mirror of
https://github.com/pgsty/minio.git
synced 2026-09-30 23:05:59 +03:00
Compare commits
24 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2a4d51406b | |||
| 0596685ae7 | |||
| 358ab38fb0 | |||
| 254b19ac07 | |||
| eb4f5e5b31 | |||
| 8d06424b12 | |||
| fced863036 | |||
| 5af0865aab | |||
| 83821f0f1f | |||
| bf053386a4 | |||
| e7963381ba | |||
| 37c0edc7ca | |||
| a2fe70424e | |||
| 027d43b4eb | |||
| f99ed829b5 | |||
| 956a1a8e33 | |||
| 82f0a9828e | |||
| 584b5a3a8f | |||
| 39f49d548f | |||
| a168576adb | |||
| e85c4c8dfb | |||
| 0e3c43778e | |||
| 15090dc4fd | |||
| e791640dac |
@@ -96,6 +96,12 @@ jobs:
|
||||
- name: Run multipart listing and cancellation tests under race detector
|
||||
run: go test -race ./cmd -run '^Test(MultipartListing|MultipartAbort|PaginateMultipartUploads|ListMultipartUploads)' -count=1 -timeout=5m
|
||||
|
||||
- name: Run CPU metrics tests under race detector
|
||||
run: go test -race ./cmd -run '^TestLoadCPUMetrics' -count=1 -timeout=5m
|
||||
|
||||
- name: Run tag replication tests under race detector
|
||||
run: go test -race ./cmd -run '^TestAPITagging' -count=1 -timeout=5m
|
||||
|
||||
crosscompile:
|
||||
name: Cross Compile
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -132,9 +132,9 @@ jobs:
|
||||
cosign-release: v3.1.2
|
||||
|
||||
- name: Build Draft release with GoReleaser
|
||||
uses: goreleaser/goreleaser-action@v7
|
||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
|
||||
with:
|
||||
version: "~> v2"
|
||||
version: v2.18.1
|
||||
args: release --clean --skip=validate --config .github/goreleaser.yml
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -4,6 +4,8 @@ on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
paths:
|
||||
- "go.mod"
|
||||
- "go.sum"
|
||||
- ".github/goreleaser.yml"
|
||||
- ".github/nfpm.yml"
|
||||
- "Dockerfile.goreleaser"
|
||||
@@ -93,9 +95,9 @@ jobs:
|
||||
echo "LDFLAGS: ${LDFLAGS}"
|
||||
|
||||
- name: GoReleaser config check
|
||||
uses: goreleaser/goreleaser-action@v7
|
||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
|
||||
with:
|
||||
version: "~> v2"
|
||||
version: v2.18.1
|
||||
args: check --config .github/goreleaser.yml
|
||||
|
||||
- name: Validate Helm chart and legacy upgrade identity
|
||||
@@ -107,9 +109,9 @@ jobs:
|
||||
syft-version: v1.50.0
|
||||
|
||||
- name: Build snapshot artifacts
|
||||
uses: goreleaser/goreleaser-action@v7
|
||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
|
||||
with:
|
||||
version: "~> v2"
|
||||
version: v2.18.1
|
||||
# A pull-request snapshot has no trusted release identity. Exercise
|
||||
# the SBOM/checksum pipeline here, and reserve keyless signing for
|
||||
# the tag-triggered release workflow with GitHub OIDC.
|
||||
|
||||
+2
-2
@@ -62,10 +62,10 @@ dist/
|
||||
|
||||
.claude/
|
||||
.codex/
|
||||
AGENTS.md
|
||||
CLAUDE.md
|
||||
_bmad/
|
||||
_bmad-output/
|
||||
# Investigation and editorial working files belong outside this repository.
|
||||
docs/investigations/
|
||||
docs/security/
|
||||
docs/rebranding.md
|
||||
.release-sign/
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
# SILO Repository Guide
|
||||
|
||||
## Project map
|
||||
|
||||
- Server: `pgsty/silo` (this checkout, default branch `main`)
|
||||
- Console: `pgsty/silo-console` (usual sibling checkout `../silo-console`)
|
||||
- Client: `pgsty/mc`, shipped as `mcli` (usual sibling checkout `../mc`)
|
||||
- Shared packages: `pgsty/silo-pkg` (usual sibling checkout `../silo-pkg`)
|
||||
- Documentation: `pgsty/silo.pgsty.com` (usual sibling checkout `../silo.pgsty.com`),
|
||||
published at <https://silo.pgsty.com/>
|
||||
|
||||
The server executable, package, systemd service, and container image use `silo`;
|
||||
the public Docker image is `docker.io/pgsty/silo`. The server module remains
|
||||
`github.com/minio/minio`. Consult the current `go.mod` and release configuration
|
||||
for selected component versions and compatibility replacements.
|
||||
|
||||
## Supported stack and compatibility policy
|
||||
|
||||
The maintained, release-gating product graph is the coordinated PGSTY stack:
|
||||
`silo` + `silo-console` + `mc` + `silo-pkg`.
|
||||
|
||||
Keep compatibility with upstream MinIO/MC on a best-effort basis. Preserve
|
||||
inexpensive wire, configuration, CLI, migration, and import compatibility when
|
||||
it helps users, and document known differences. Do not infer from source
|
||||
lineage, MinIO-compatible protocols, retained `MINIO_*`/`MC_*` names, or an
|
||||
inherited upstream test that unmodified upstream MinIO is a supported release
|
||||
target.
|
||||
|
||||
Upstream-only compatibility checks may remain as advisory evidence, but they
|
||||
must not force a downgrade of a maintained SILO component, a fork-only API
|
||||
workaround, or a release block. Make upstream compatibility a hard gate only
|
||||
when the user explicitly requests that scope.
|
||||
|
||||
## Dependency selection
|
||||
|
||||
- When PGSTY maintains a component, import and require it directly under its own
|
||||
module path. In particular, prefer `github.com/pgsty/silo-pkg/v3` over
|
||||
`github.com/minio/pkg/v3` in maintained SILO source.
|
||||
- Do not use an upstream package merely to make unmodified upstream MinIO or MC
|
||||
compile. Unavoidable transitive upstream modules should be documented and
|
||||
kept separate from the maintained product dependency.
|
||||
- `github.com/minio/minio-go/v7` is the explicit exception: use the verified
|
||||
upstream module/commit while it contains the required fixes; do not recreate
|
||||
a SILO fork without a concrete functional divergence.
|
||||
- Check the current `go.mod` before changing versions. Coordinate breaking
|
||||
import-path changes across package, client, Console, and server releases.
|
||||
|
||||
## Documentation and delivery
|
||||
|
||||
The companion site owns product, operations, migration, security-advisory,
|
||||
design, and release documentation. Maintain English and Chinese content in
|
||||
that repository and link to its canonical URLs from this one. The site's
|
||||
`content/docs/` is the documentation entry point; detailed pages live under
|
||||
`content/operations/`, `administration/`, `reference/`, `compatibility/`,
|
||||
`about/`, and `blog/`, following the existing structure.
|
||||
|
||||
Keep repository entry points and contributor instructions here. Retained
|
||||
upstream documents, examples, and test fixtures under `docs/` may be updated
|
||||
when a code or tooling change requires it; do not build a second documentation
|
||||
site in this tree.
|
||||
|
||||
Investigation plans, prompts, AI session transcripts, execution logs, temporary
|
||||
reports, and private security material belong outside the checkout, for example
|
||||
in a task-specific directory under `~/tmp/`. Do not recreate
|
||||
`docs/investigations/`, `docs/security/`, or `docs/rebranding.md`, or move their
|
||||
working material to another repository directory. Extract reusable, verified
|
||||
knowledge into the companion site; keep private evidence outside public Git.
|
||||
|
||||
Compatibility pages may continue to describe similarities with upstream, but
|
||||
must label that compatibility as best effort. The supported and tested server
|
||||
for Console and mcli administration is `pgsty/silo`.
|
||||
|
||||
Before removing or moving documentation, check repository links and scripts,
|
||||
the companion site's links and anchors, and references from this guide. Use
|
||||
fixed commit URLs for historical evidence that should survive deletion from
|
||||
the current tree. Run the site's `make check` for site changes and this
|
||||
repository's `make rebrand-guard` for documentation cleanup that changes the
|
||||
identifier inventory. When new public URLs are involved, publish the site
|
||||
before publishing source changes that depend on those URLs.
|
||||
|
||||
Keep changes in the repository that owns the affected surface. Treat every
|
||||
repository commit, tag, release, image, documentation update, and deployment as
|
||||
a separate deliverable. Follow `CONTRIBUTING.md`, including DCO sign-off
|
||||
(`git commit -s`). `CLAUDE.md` imports this guide so both agents use one policy.
|
||||
+92
-33
@@ -2,13 +2,18 @@
|
||||
|
||||
## Unreleased
|
||||
|
||||
The entries below describe source changes on main since the latest published Server.
|
||||
Preparation target: `RELEASE.2026-09-16T00-00-00Z` (package version
|
||||
`20260916000000.0.0`). The entries below describe the candidate changes since
|
||||
the latest published Server.
|
||||
**The latest published Server remains 20260903.** These changes are not in its
|
||||
binaries, packages or images. See the [component matrix](https://silo.pgsty.com/compatibility/versions/)
|
||||
and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-09-03T13-18-01Z...main).
|
||||
|
||||
### Authorization and security
|
||||
|
||||
- Synchronize CPU metrics reads with resource-metrics updates (#210), preventing
|
||||
concurrent map access from terminating the server during Prometheus scraping.
|
||||
Metric names, values and authentication requirements are unchanged.
|
||||
- Restrict embedded Console's anonymous sharing proxy to object-content GETs
|
||||
at the configured S3 origin, and reject every redirect. Internal metrics,
|
||||
system paths and non-download S3 operations cannot be reached through it.
|
||||
@@ -29,7 +34,7 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0
|
||||
recreated parents and explicitly reconcile pre-upgrade revocations whose
|
||||
history is already lost. Restoring an older backup can lose later revocations;
|
||||
keep affected sites isolated until reconciliation/rekeying is complete. See
|
||||
[the operator runbook](https://github.com/pgsty/silo.pgsty.com/blob/29c7f220b3acc556ad570694056d35e11246f1b9/content/operations/replication/iam-upgrade.md).
|
||||
[the operator runbook](https://silo.pgsty.com/operations/replication/iam-upgrade/).
|
||||
- Enforce an absolute HTTP/1 request-header deadline through the connection
|
||||
wrapper (#196). Repeated small reads no longer extend that deadline, and
|
||||
`--read-header-timeout` / `MINIO_READ_HEADER_TIMEOUT` now reaches the HTTP
|
||||
@@ -38,7 +43,7 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0
|
||||
TLS handshake reads. The wrapper's strict header mode is not applied to HTTP/2.
|
||||
- Reject unsigned `x-amz-*` request headers that could turn a signed PUT into a
|
||||
copy of another object accessible to the signer (SN-2026-011). The latest
|
||||
public Server is affected; the fix is on main. See [the advisory ledger](docs/security/advisories.md).
|
||||
public Server is affected; the fix is on main. See [the advisory ledger](https://silo.pgsty.com/about/security-advisories/).
|
||||
- Align signed request fields with policy conditions and enforce header-only
|
||||
presigned payload checksums. See [the signed-header review](https://silo.pgsty.com/blog/design/signed-header-coverage/).
|
||||
- **Breaking policy semantics:** separate self-service `admin:ChangeMyPassword`
|
||||
@@ -56,19 +61,28 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0
|
||||
quorum-written `xl.meta`; completion removes those upload-only fields. Native
|
||||
markers remain usable after their upload is completed or canceled. Strict
|
||||
listing returns a diagnostic 503 for legacy uploads or uncertain coverage;
|
||||
`api multipart_listing=legacy` is an explicit temporary migration mode.
|
||||
Upgrade every writer, drain old uploads and check the read-only admin
|
||||
`multipart-preflight` report before relying on strict listing. Per-process
|
||||
the default remains the released exact-key/cache-based `legacy` behavior.
|
||||
Opt into strict mode only through `MINIO_API_MULTIPART_LISTING=strict`, after
|
||||
upgrading every writer, draining old uploads, checking the read-only admin
|
||||
`multipart-preflight` report and validating scan capacity. The process-only
|
||||
setting is not persisted into shared API configuration. Historical
|
||||
`multipart_listing` keys are ignored and can be removed with a targeted
|
||||
`mcli admin config reset ALIAS api multipart_listing` before rollback. Per-process
|
||||
admission, directory-entry, worker and time budgets bound scan scheduling;
|
||||
each page still scans durable state. See [issue #79](https://github.com/pgsty/silo/issues/79)
|
||||
and its [design record](https://silo.pgsty.com/blog/design/list-multipart-uploads/).
|
||||
Thanks to mr javad seydi (@mrjavadseydi) for the original implementation.
|
||||
- Confirm multipart cancellation on a strict majority of each relevant set,
|
||||
and allow retries after partial deletion. Uncertain pools or insufficient
|
||||
confirmations return 503 rather than acknowledging a cancellation whose
|
||||
static remnants can later become readable. **Known boundary:** creation
|
||||
writes that finish after a storage timeout can still restore an upload after
|
||||
successful cancellation; this change does not add a durable creation fence.
|
||||
- Retain released read-quorum and best-effort multipart cancellation in default
|
||||
legacy mode. Strict mode requires majority deletion acknowledgements and
|
||||
permits retries below read quorum. When most drives were already empty,
|
||||
failed deletion of an observed remnant now returns 503 instead of being
|
||||
masked by empty-drive successes. Wrong-key or wrong-bucket cancellation
|
||||
preserves the valid upload's cache entry; successful cancellation notifies
|
||||
peers with the request context after the distributed lock is released.
|
||||
The HTTP response for an absent
|
||||
upload remains 204; this is not proof of physical cleanup. **Known boundary:**
|
||||
delayed creation writes can still restore an upload after cancellation;
|
||||
this change does not add a durable creation fence.
|
||||
- Preserve object tags during multi-pool metadata reconciliation by reading the
|
||||
resolved tag field together with its revision (#189). Previously, reconciliation
|
||||
could replace existing tags with an empty value.
|
||||
@@ -91,9 +105,38 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0
|
||||
ordinary metadata. Thanks to Mikhail Khadarenka (@chodorenko) for the fix in #187.
|
||||
**Existing data:** these repairs prevent new errors; they do not scan or rewrite
|
||||
historical object metadata, recover lost tags or prove that old purge work has
|
||||
converged. Follow the [read-only audit procedure](https://github.com/pgsty/silo.pgsty.com/blob/29c7f220b3acc556ad570694056d35e11246f1b9/content/operations/replication/replica-metadata-audit.md)
|
||||
converged. Follow the [read-only audit procedure](https://silo.pgsty.com/operations/replication/replica-metadata-audit/)
|
||||
before planning any repair of stored state.
|
||||
|
||||
- Make exact-version delete-marker purges converge in replicated buckets
|
||||
(`eb4f5e5b3`, `254b19ac0`, `358ab38fb`). A purge no longer creates the
|
||||
marker on drives that lacked it; a retried purge of a missing version is
|
||||
acknowledged only when a write-quorum majority of drives report it absent;
|
||||
purge results merge removed and reliably absent replies; healing a marker
|
||||
preserves its stored replication and purge metadata; and a queued marker
|
||||
creation is re-checked against the source under the replication lock before
|
||||
it is sent, so a purge that already reached the targets is not undone by a
|
||||
stale task from another frontend, a GET/LIST heal, the scanner or MRF.
|
||||
Purging a data version whose earlier purge is still pending reports it as a
|
||||
data version. **Known limitations:** creations already in flight or replayed
|
||||
from another site, and minority marker copies left by a crash after a
|
||||
majority-acknowledged purge, are tracked in #217. See
|
||||
[the replication reliability record](https://silo.pgsty.com/blog/design/replication-reliability/).
|
||||
- Keep a null object version that has listing quorum when a newer minority of
|
||||
drives sorts first (`8d06424b1`). The resolver recounts per header only when
|
||||
the original selection lacks quorum, every non-empty drive stream holds
|
||||
exactly one ordinary null version and all share the same erasure layout;
|
||||
mixed histories keep their previous behavior. **Known limitation:** a
|
||||
successful ListObjects can still omit readable keys during rolling restarts
|
||||
with concurrent overwrites (#218). Do not run destination-deleting sync tools
|
||||
against a listing taken during a rolling restart; list again once the
|
||||
cluster is stable.
|
||||
- Carry object tags through rebalance and decommission for ordinary and
|
||||
multipart writes (`fced86303`). Both migration entry points restore the tags
|
||||
and their revision fields when rewriting the object in the destination pool.
|
||||
Tags dropped by earlier migrations are not recovered; audit tag-dependent
|
||||
lifecycle and policy rules for pools migrated with an older build.
|
||||
|
||||
- Evaluate conditional multipart completion against the logical current object
|
||||
across all pools while holding the existing object lock. A stale `If-Match`
|
||||
can no longer replace newer data in another pool, and the current ETag is no
|
||||
@@ -104,9 +147,20 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0
|
||||
applies when the unreadable pool may not hold the object: absence cannot be
|
||||
verified. Retry after the pool recovers. Unconditional completion and the
|
||||
single-pool path retain their existing behavior.
|
||||
Ordinary conditional PUT has a separate cross-pool precondition gap tracked
|
||||
in [#199](https://github.com/pgsty/silo/issues/199); the multipart repair does
|
||||
not resolve it.
|
||||
|
||||
- Evaluate ordinary multi-pool conditional PUT against the logical current
|
||||
object across all pools, including draining pools, under the existing object
|
||||
lock (#207). A stale destination copy no longer accepts a stale ETag or rejects
|
||||
the current one; a current delete marker is treated as absence.
|
||||
**Availability change:** if any pool's object metadata cannot be verified,
|
||||
the condition fails even when GET can use another pool; read-quorum failures
|
||||
return 503. Restore readability or heal before retrying. Unconditional PUT,
|
||||
single-pool conditions and internal replication retain their existing behavior.
|
||||
A public condition with a destination `versionId` compares the current object
|
||||
while preserving the requested write version. This change does not retire
|
||||
stale copies in other pools, undo historical accepted overwrites or provide
|
||||
a new global clock-ordering guarantee. The multipart-completion repair in #190
|
||||
neither introduced nor repaired this separate PUT defect.
|
||||
|
||||
- Reconcile ordinary single-object version DELETE across all pools, including
|
||||
null versions, delete markers and unqualified directory-marker DELETE. This
|
||||
@@ -125,7 +179,7 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0
|
||||
its tracker, mover, scanner hooks, configuration, XML actions and metrics.
|
||||
Accept and ignore retired configuration/XML and preserve ordinary statistics
|
||||
when reading v9 caches. See [migration notes](docs/bucket/lifecycle/access-tiering-removal.md).
|
||||
The [decision record](docs/investigations/access-tiering-revert.md) preserves
|
||||
The [decision record](https://silo.pgsty.com/compatibility/access-tiering-removal/) preserves
|
||||
the feature's introduction, subsequent fixes, rollback scope and review history.
|
||||
- Preserve the independent multi-pool write, metadata, healing and conditional
|
||||
deletion fixes from PR #178, including shared remote-tier reference protection.
|
||||
@@ -139,7 +193,7 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0
|
||||
reflect actual work, and report bounded MRF drops.
|
||||
- Converge bucket metadata with deterministic source state, deletion tombstones,
|
||||
creation time recovery and diagnostics. The mixed-version export gate requires
|
||||
coordinated upgrades before tombstones are exported. See [the #77 record](docs/investigations/issue-77-current.md).
|
||||
coordinated upgrades before tombstones are exported. See [the #77 record](https://silo.pgsty.com/blog/design/bucket-metadata-convergence/).
|
||||
- Include per-bucket CORS in metadata export/import, close metadata publication
|
||||
and logger races, and report effective bucket quotas in metrics.
|
||||
|
||||
@@ -147,23 +201,28 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0
|
||||
|
||||
- Restore embedded Console login over loopback TLS, trusted-proxy handling and
|
||||
all four WebSocket connection limits. Preserve Go TLS defaults across transports.
|
||||
- Directly require `github.com/pgsty/silo-pkg/v3` v3.14.0; select Console
|
||||
`v0.0.0-20260916034812-56dfe455ac2f` and MC
|
||||
`v0.0.0-20260913012246-4f609a4da3bb` with explicit PGSTY replacements.
|
||||
- Pin upstream minio-go `v7.3.1-0.20260910142817-60bd07042d49`; refresh Go x/*
|
||||
modules and security fixes including bounded AMQP frame handling. Keep Go
|
||||
1.27.1 and go-systemd v22.6.0's NetBSD compatibility replacement.
|
||||
- Directly require `github.com/pgsty/silo-pkg/v3` v3.14.1; select released Console
|
||||
v2.4.1 (`v0.0.0-20260916075814-1360e26d976d`) and mcli 20260916
|
||||
(`v0.0.0-20260916070421-e952aa78f10a`) with explicit PGSTY replacements.
|
||||
The embedded frontend identifies itself as Console v2.4.1.
|
||||
- Pin upstream minio-go `v7.3.1-0.20260915093545-32e1f32cb176` to handle
|
||||
CopyObject errors embedded in HTTP 200 responses. Update JWX to v3.3.0 for
|
||||
JSON field-name escaping, strfmt to v0.27.2 for Go 1.27 hostname validation,
|
||||
and LZ4 to v4.1.30 for frame-reader, partial-read and concurrency fixes.
|
||||
Retain the earlier Go x/* and bounded AMQP frame updates, Go 1.27.1, and
|
||||
go-systemd v22.6.0's NetBSD compatibility replacement.
|
||||
- Refresh container base digests and build static curl 8.22.0 from verified
|
||||
source for both Linux architectures. Pin the actual mcli 20260913 archives and
|
||||
hashes. Helm's client image follows that release; its Server image still names
|
||||
the latest published Server 20260903.
|
||||
source for both Linux architectures. Pin the published mcli 20260916 archives
|
||||
and hashes in the container and update the client installer default.
|
||||
- Prepare Helm chart 7.0.3 with Server and client defaults for the September 16
|
||||
batch. Publish the chart only after the corresponding Server image exists.
|
||||
- Pin GoReleaser v2.18.1 and its action commit identically in snapshot and release
|
||||
workflows. Dependency-only PRs now run the Test Release Pipeline too.
|
||||
|
||||
The dependency update passed the final candidate's Go, vulnerability and Test
|
||||
Release workflows; native curl builds passed on both architectures. A local
|
||||
ARM64 image passed startup, health, S3 transfer and embedded Console checks.
|
||||
These checks do not publish a Server tag or production image and do not replace
|
||||
cluster upgrade/rollback acceptance for the next release. Dated investigations
|
||||
retain the exact source and runtime boundaries they tested.
|
||||
Validation of earlier source revisions does not establish acceptance of this
|
||||
candidate. Final source, package, image and multi-process checks are tracked
|
||||
separately in [#203](https://github.com/pgsty/silo/issues/203). No Server release
|
||||
or production rollout is implied by this preparation target.
|
||||
|
||||
## RELEASE.2026-09-03T13-18-01Z
|
||||
|
||||
|
||||
@@ -77,6 +77,16 @@ test evidence, compatibility notes, and documentation impact. Public product
|
||||
documentation is owned by the separate
|
||||
[`pgsty/silo.pgsty.com`](https://github.com/pgsty/silo.pgsty.com) repository.
|
||||
|
||||
## Contributor recognition
|
||||
|
||||
Every human issue or pull-request author is recognized in [CONTRIBUTORS.md](CONTRIBUTORS.md),
|
||||
including open issues, draft PRs, and PRs closed without merging. Merged fixes,
|
||||
adopted proposals, and reports that lead to fixes receive greater prominence;
|
||||
first participation guides the remaining order. Work incorporated through a
|
||||
later PR retains credit without changing the original PR's recorded status.
|
||||
Security disclosures are credited with the reporter's agreement. DCO sign-off
|
||||
applies to code commits, not to opening an issue.
|
||||
|
||||
## Licensing of Contributions
|
||||
|
||||
Code contributions to PGSTY SILO (`pgsty/silo`) are accepted under the
|
||||
|
||||
+116
-124
File diff suppressed because one or more lines are too long
@@ -17,9 +17,9 @@ ENV GOPATH=/go
|
||||
ENV CGO_ENABLED=0
|
||||
|
||||
ARG MC_REPO=pgsty/mc
|
||||
ARG MC_VERSION=RELEASE.2026-09-13T00-00-00Z
|
||||
ARG MC_AMD64_SHA256=9d2a92de9c7b887d9b944fe9ddce68d23f1b6df3415092e737594e56593f5e2b
|
||||
ARG MC_ARM64_SHA256=3d82e9ea6c601c4cb44fe5dd5f2ad1b7d7d64369378110f9ada9c524688a452a
|
||||
ARG MC_VERSION=RELEASE.2026-09-16T00-00-00Z
|
||||
ARG MC_AMD64_SHA256=4ba2814fd5507fbe6b4d237c359750b9119d28d7217495fa5b48002fcbd397ef
|
||||
ARG MC_ARM64_SHA256=b7008ca2a1bc5735b6585981c59a3640a0daa152dc789df3d1a0d0438787de82
|
||||
|
||||
RUN apk add -U --no-cache \
|
||||
ca-certificates \
|
||||
|
||||
@@ -38,7 +38,7 @@
|
||||
## Current release and main branch
|
||||
|
||||
The latest published Server is [20260903](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z).
|
||||
As of 2026-09-13, the main branch has newer security, storage, Console and
|
||||
As of 2026-09-16, the main branch has newer security, storage, Console and
|
||||
shared-package changes that have not shipped in a Server release. See
|
||||
[CHANGELOG.md](CHANGELOG.md) and the [component version matrix](https://silo.pgsty.com/compatibility/versions/)
|
||||
for the exact release/source boundary, including SN-2026-011 and password-policy migration.
|
||||
@@ -93,13 +93,16 @@ Every release ships checksums, SPDX SBOMs, Sigstore-signed manifests, and GitHub
|
||||
|
||||
## Compatibility
|
||||
|
||||
The S3 API, `MINIO_*` variables, `minio_*` metrics, `x-minio-*` headers, `/minio/*` routes, the `github.com/minio/*` import paths, and the on-disk format (including `.minio.sys`) are preserved and held in place by a CI compatibility check. Only Silo-owned delivery surfaces change: the `silo` executable, package, service, Helm chart, and container image — no `minio` binary alias is installed.
|
||||
Silo preserves S3 and storage-format compatibility, including existing `MINIO_*` variables, `minio_*` metrics, `x-minio-*` headers, `/minio/*` routes, and `.minio.sys` data. CI guards selected compatibility identifiers; release notes document intentional security and behavior changes. Silo-owned delivery surfaces use the `silo` executable, package, service, Helm chart, and container image; no `minio` server binary alias is installed.
|
||||
|
||||
The supported release stack is `pgsty/silo` + `pgsty/silo-console` + `pgsty/mc` + `pgsty/silo-pkg`; compatibility with unmodified upstream MinIO/MC is best effort. The Server, Console, and client retain their historical module paths where needed, while maintained code imports `github.com/pgsty/silo-pkg/v3` directly. The SDK `github.com/minio/minio-go/v7` is an explicit upstream dependency. See the current [go.mod](go.mod) for versions and replacements.
|
||||
|
||||
Every divergence from upstream is listed in the code-verified [compatibility audit](https://silo.pgsty.com/compatibility/server/). Treat each release as a downstream upgrade: pin versions, read the [release notes](https://silo.pgsty.com/tags/silo/), and keep a rollback path.
|
||||
|
||||
### TLS and Go upgrades
|
||||
|
||||
TLS key exchange follows Go's defaults across the S3 listener, node links,
|
||||
The following TLS repair is on main and is not included in Server 20260903.
|
||||
With that repair, TLS key exchange follows Go's defaults across the S3 listener, node links,
|
||||
replication, identity providers, etcd, and external HTTP services. If an endpoint
|
||||
cannot accept ML-KEM, `GODEBUG=tlsmlkem=0` disables the default hybrid exchanges
|
||||
for the process; certificate verification remains enabled. This option does not
|
||||
@@ -115,7 +118,16 @@ values to restore Keychain trust. Explicit certificates in the configured `CAs`
|
||||
directory remain additive to the selected root pool.
|
||||
Go 1.27 binaries require macOS 13 or later. See the
|
||||
[Go release notes](https://go.dev/doc/go1.27) and the
|
||||
[SILO stack investigation](docs/investigations/go127-stack.md).
|
||||
[Go 1.27 TLS and OIDC discovery guide](https://silo.pgsty.com/blog/design/go127-tls-oidc-discovery/).
|
||||
|
||||
## Documentation ownership
|
||||
|
||||
User documentation is maintained at [silo.pgsty.com](https://silo.pgsty.com/docs/),
|
||||
with source in [pgsty/silo.pgsty.com](https://github.com/pgsty/silo.pgsty.com).
|
||||
The remaining `docs/` tree contains inherited references, examples, and tooling
|
||||
fixtures. Investigation logs, AI work records, and temporary reports are kept
|
||||
outside this repository; reusable findings belong in the companion site.
|
||||
See [AGENTS.md](AGENTS.md) for repository ownership and maintenance rules.
|
||||
|
||||
## Security & Contributing
|
||||
|
||||
|
||||
+17
-2
@@ -38,7 +38,7 @@
|
||||
## 当前发行版与主分支
|
||||
|
||||
最新已发布的 Server 仍为 [20260903](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z)。
|
||||
截至 2026-09-13,主分支已合入更新的安全、存储、Console 与共享包改动,但尚未发布新 Server。
|
||||
截至 2026-09-16,主分支已合入更新的安全、存储、Console 与共享包改动,但尚未发布新 Server。
|
||||
准确的已发布/源码边界见 [CHANGELOG.md](CHANGELOG.md) 与[组件版本矩阵](https://silo.pgsty.com/zh/compatibility/versions/),
|
||||
其中包括 SN-2026-011 修复状态与密码权限迁移要求。
|
||||
|
||||
@@ -92,10 +92,25 @@ docker exec silo mcli mb local/demo && docker exec silo mcli ls local
|
||||
|
||||
## 兼容性
|
||||
|
||||
S3 API、`MINIO_*` 环境变量、`minio_*` 指标、`x-minio-*` 头、`/minio/*` 路由、`github.com/minio/*` 导入路径与磁盘格式(含 `.minio.sys`)原样保留,并由 CI 兼容性门禁冻结。只有 Silo 自有交付面改名:`silo` 可执行文件、软件包、服务、Helm Chart 与容器镜像 —— 原生交付物不会安装 `minio` 二进制别名。
|
||||
Silo 保留 S3 与存储格式兼容性,包括既有 `MINIO_*` 环境变量、`minio_*` 指标、`x-minio-*` 头、`/minio/*` 路由与 `.minio.sys` 数据。CI 守卫检查选定的兼容性标识,有意的安全与行为变化在发布说明中记录。Silo 自有交付面使用 `silo` 可执行文件、软件包、服务、Helm Chart 与容器镜像;原生交付物不会安装 `minio` 服务端二进制别名。
|
||||
|
||||
正式支持和发布验收的组合为 `pgsty/silo` + `pgsty/silo-console` + `pgsty/mc` + `pgsty/silo-pkg`,对未修改的上游 MinIO/MC 尽最大努力保持兼容。Server、Console 与客户端按需保留历史模块路径,维护源码直接导入 `github.com/pgsty/silo-pkg/v3`;SDK `github.com/minio/minio-go/v7` 是明确保留的上游依赖。具体版本与 replace 以当前 [go.mod](go.mod) 为准。
|
||||
|
||||
与上游的全部分歧,以逐项核验代码的[兼容性审计](https://silo.pgsty.com/zh/compatibility/server/)形式维护。每个版本仍应视为下游升级:锁定版本,阅读[版本说明](https://silo.pgsty.com/zh/tags/silo/),并保留回滚路径。
|
||||
|
||||
### TLS 与 Go 升级
|
||||
|
||||
Server 恢复 Go 默认密钥交换策略的修复已在 main,尚未包含在 Server 20260903。
|
||||
`GODEBUG=tlsmlkem=0`、`tlssecpmlkem=0` 的适用范围、macOS 根证书来源变化,
|
||||
以及 OIDC discovery 的诊断方法见 [Go 1.27 TLS 与 OIDC 指南](https://silo.pgsty.com/zh/blog/design/go127-tls-oidc-discovery/)。
|
||||
|
||||
## 文档归属
|
||||
|
||||
用户文档统一维护在 [silo.pgsty.com](https://silo.pgsty.com/zh/docs/),源码位于
|
||||
[pgsty/silo.pgsty.com](https://github.com/pgsty/silo.pgsty.com)。本仓库保留的 `docs/`
|
||||
主要是继承的参考材料、示例和工具测试夹具。调查日志、AI 工作记录与临时报告放在仓库外;
|
||||
可复用的结论应整理进伴生文档站。仓库职责与维护规则见 [AGENTS.md](AGENTS.md)。
|
||||
|
||||
## 安全与贡献
|
||||
|
||||
请按照 [`SECURITY.md`](SECURITY.md) 私密报告漏洞;每项修复都会发布公开[安全公告](https://silo.pgsty.com/zh/blog/security/)。本项目不要求签署 CLA:贡献按 AGPL-3.0-or-later(inbound=outbound)接收,只需 DCO 签署(`git commit -s`),详见 [`CONTRIBUTING.md`](CONTRIBUTING.md)。
|
||||
|
||||
+3
-3
@@ -7,7 +7,7 @@ Silo-specific fixes or release notes.
|
||||
## Supported Versions
|
||||
|
||||
Security fixes are tracked on the active development branch and summarized in
|
||||
[docs/security/advisories.md](docs/security/advisories.md). Only the current
|
||||
[the security advisory ledger](https://silo.pgsty.com/about/security-advisories/). Only the current
|
||||
Silo release line is supported unless an advisory says otherwise.
|
||||
|
||||
## Inherited Fix Evidence
|
||||
@@ -26,7 +26,7 @@ separately even when the fork preserves the original commit object and SHA.
|
||||
The inherited [service-account](https://github.com/pgsty/silo/blob/c1a49490c78e9c3ebcad86ba0662319138ace190/cmd/admin-handlers-users_test.go#L211-L212)
|
||||
and [STS](https://github.com/pgsty/silo/blob/c1a49490c78e9c3ebcad86ba0662319138ace190/cmd/sts-handlers_test.go#L45-L46)
|
||||
regression groups remain part of `go test ./cmd`; see the
|
||||
[canonical ledger](docs/security/advisories.md#inherited-upstream-advisory-baseline)
|
||||
[canonical ledger](https://silo.pgsty.com/about/security-advisories/#inherited)
|
||||
for the operator-facing record.
|
||||
|
||||
## Reporting a Vulnerability
|
||||
@@ -42,4 +42,4 @@ For vulnerabilities in this fork:
|
||||
|
||||
## Disclosure Process
|
||||
|
||||
Fork-specific fixes and user-visible upgrade notes are published in [docs/security/advisories.md](docs/security/advisories.md). The fork-specific triage and remediation process is described in [VULNERABILITY_REPORT.md](VULNERABILITY_REPORT.md).
|
||||
Fork-specific fixes and user-visible upgrade notes are published in [the security advisory ledger](https://silo.pgsty.com/about/security-advisories/). The fork-specific triage and remediation process is described in [VULNERABILITY_REPORT.md](VULNERABILITY_REPORT.md).
|
||||
|
||||
@@ -34,4 +34,4 @@ Based on the report, the Silo maintainers investigate:
|
||||
If the vulnerability exists in this fork itself, the maintainers will, when
|
||||
feasible, fix the issue or implement reasonable countermeasures such that the
|
||||
vulnerability can no longer be exploited. Fork-specific upgrade notes and
|
||||
security advisories are published in `docs/security/advisories.md`.
|
||||
security advisories are published in [the security advisory ledger](https://silo.pgsty.com/about/security-advisories/).
|
||||
|
||||
@@ -40,7 +40,7 @@ if [ -n "${MCLI_BIN:-}" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
release=${MCLI_RELEASE:-RELEASE.2026-09-13T00-00-00Z}
|
||||
release=${MCLI_RELEASE:-RELEASE.2026-09-16T00-00-00Z}
|
||||
version_hyphen=${release#RELEASE.}
|
||||
package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')
|
||||
if [ "${package_version}" = "${version_hyphen}" ]; then
|
||||
|
||||
@@ -132,7 +132,6 @@
|
||||
"MINIO_API_DELETE_CLEANUP_INTERVAL",
|
||||
"MINIO_API_DISABLE_ODIRECT",
|
||||
"MINIO_API_GZIP_OBJECTS",
|
||||
"MINIO_API_LEGACY_BUCKET_RESOURCE_MATCH",
|
||||
"MINIO_API_LIST_QUORUM",
|
||||
"MINIO_API_MULTIPART_LISTING",
|
||||
"MINIO_API_OBJECT_MAX_VERSIONS",
|
||||
@@ -572,6 +571,14 @@
|
||||
"minio_resource_stats",
|
||||
"minio_s3",
|
||||
"minio_stats",
|
||||
"minio_system_cpu_avg_idle",
|
||||
"minio_system_cpu_avg_iowait",
|
||||
"minio_system_cpu_load",
|
||||
"minio_system_cpu_load_perc",
|
||||
"minio_system_cpu_nice",
|
||||
"minio_system_cpu_steal",
|
||||
"minio_system_cpu_system",
|
||||
"minio_system_cpu_user",
|
||||
"minio_test"
|
||||
],
|
||||
"headers": [
|
||||
@@ -628,7 +635,6 @@
|
||||
"x-minio-origin-endpoint",
|
||||
"x-minio-prefixes-total",
|
||||
"x-minio-read-quorum",
|
||||
"x-minio-replication",
|
||||
"x-minio-replication-actual-object-size",
|
||||
"x-minio-replication-delete-status",
|
||||
"x-minio-replication-deletemarker-status",
|
||||
|
||||
@@ -115,9 +115,8 @@ func collect(repo string) (manifest, error) {
|
||||
fset := token.NewFileSet()
|
||||
|
||||
for _, rel := range files {
|
||||
// Investigation artifacts contain synthetic routes and archived configurations.
|
||||
// Migration notes and guard fixtures contain archived identifiers.
|
||||
if rel == "SILO_REBRANDING_MIGRATION.md" ||
|
||||
strings.HasPrefix(rel, "docs/investigations/") ||
|
||||
strings.HasPrefix(rel, "buildscripts/rebrand-guard/") ||
|
||||
strings.HasPrefix(rel, "buildscripts/helm-migration-guard/") {
|
||||
continue
|
||||
|
||||
@@ -113,7 +113,7 @@ helm_run template my-release "${new_chart}" \
|
||||
go run ./buildscripts/helm-migration-guard "${old_render}" "${new_render}"
|
||||
|
||||
helm_run package "${new_chart}" --destination "${output_dir}" >/dev/null
|
||||
test -s "${work_dir}/silo-7.0.2.tgz"
|
||||
test -s "${work_dir}/silo-7.0.3.tgz"
|
||||
if find "${work_dir}" -maxdepth 1 -type f -name 'minio-*.tgz' | grep -q .; then
|
||||
echo "Helm packaging emitted a legacy MinIO chart name" >&2
|
||||
exit 1
|
||||
|
||||
@@ -504,6 +504,35 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
|
||||
if !isPurge && dobj.DeleteMarkerVersionID != "" {
|
||||
// A creation task carries the marker as it looked when it was queued
|
||||
// by the DELETE handler, a GET/HEAD/LIST heal, the scanner or MRF.
|
||||
// While it waited for this lock, another frontend may have purged the
|
||||
// marker and replicated that purge. The targets no longer hold the
|
||||
// marker, so the queued creation would recreate it from the stale
|
||||
// snapshot. Confirm the source version under the lock first.
|
||||
switch deleteMarkerCreationState(ctx, objectAPI, dobj) {
|
||||
case creationStale:
|
||||
return replicatedInfos{}
|
||||
case creationUnverified:
|
||||
dobj.RetryCount++
|
||||
globalReplicationPool.Get().queueMRFSave(dobj.ToMRFEntry())
|
||||
sendEvent(eventArgs{
|
||||
BucketName: bucket,
|
||||
Object: ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: dobj.ObjectName,
|
||||
VersionID: versionID,
|
||||
DeleteMarker: dobj.DeleteMarker,
|
||||
},
|
||||
UserAgent: "Internal: [Replication]",
|
||||
Host: globalLocalNodeName,
|
||||
EventName: event.ObjectReplicationNotTracked,
|
||||
})
|
||||
return replicatedInfos{}
|
||||
}
|
||||
}
|
||||
|
||||
rinfos := replicatedInfos{Targets: make([]replicatedTargetInfo, 0, len(dsc.targetsMap))}
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
@@ -1955,6 +1984,41 @@ func (di DeletedObjectReplicationInfo) isVersionPurge() bool {
|
||||
return di.VersionID != "" || di.DeleteMarkerVersionID != "" && !di.VersionPurgeStatus().Empty()
|
||||
}
|
||||
|
||||
// creationState is the outcome of re-reading a queued delete-marker creation
|
||||
// against the source.
|
||||
type creationState int
|
||||
|
||||
const (
|
||||
creationCurrent creationState = iota
|
||||
creationStale
|
||||
creationUnverified
|
||||
)
|
||||
|
||||
// deleteMarkerCreationState re-reads the marker a queued creation task refers
|
||||
// to. The version being absent, no longer a marker, or under a version purge
|
||||
// makes the creation stale. A failed read is not absence: the caller retries
|
||||
// later instead of guessing.
|
||||
func deleteMarkerCreationState(ctx context.Context, objectAPI ObjectLayer, dobj DeletedObjectReplicationInfo) creationState {
|
||||
oi, err := objectAPI.GetObjectInfo(ctx, dobj.Bucket, dobj.ObjectName, ObjectOptions{
|
||||
VersionID: dobj.DeleteMarkerVersionID,
|
||||
Versioned: globalBucketVersioningSys.PrefixEnabled(dobj.Bucket, dobj.ObjectName),
|
||||
VersionSuspended: globalBucketVersioningSys.Suspended(dobj.Bucket),
|
||||
})
|
||||
switch {
|
||||
case isErrObjectNotFound(err), isErrVersionNotFound(err):
|
||||
return creationStale
|
||||
case err != nil && !isErrMethodNotAllowed(err):
|
||||
return creationUnverified
|
||||
}
|
||||
if !oi.DeleteMarker || oi.VersionID != dobj.DeleteMarkerVersionID {
|
||||
return creationStale
|
||||
}
|
||||
if !oi.VersionPurgeStatus.Empty() || oi.VersionPurgeStatusInternal != "" {
|
||||
return creationStale
|
||||
}
|
||||
return creationCurrent
|
||||
}
|
||||
|
||||
// Purge metadata uses COMPLETE; operation statistics and audit use COMPLETED.
|
||||
func purgeReplicationStatus(status VersionPurgeStatusType) replication.StatusType {
|
||||
if replication.StatusType(status) == replication.CompletedLegacy {
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
// An exact-version DELETE without a replication decision physically purges
|
||||
// the version. Its response must describe the stored version: a data version
|
||||
// is not a delete marker even while pending-purge metadata makes the lookup
|
||||
// expose it as deleted, and a stored marker stays a marker.
|
||||
func TestDeleteMarkerPurgeResponseIdentity(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, endpoints: []string{"DeleteObject"}, objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, creds auth.Credentials, t *testing.T) {
|
||||
defer replicationTestCapacity(obj)()
|
||||
ctx := t.Context()
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Seed the purge state the DELETE handler would record for a target.
|
||||
arn := "arn:minio:replication::" + mustGetUUID() + ":bucket"
|
||||
pendingPurge := ReplicationState{
|
||||
VersionPurgeStatusInternal: arn + "=PENDING;",
|
||||
PurgeTargets: map[string]VersionPurgeStatusType{arn: replication.VersionPurgePending},
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
marker, purged bool
|
||||
}{
|
||||
{name: "data"},
|
||||
{name: "data-pending-purge", purged: true},
|
||||
{name: "marker", marker: true},
|
||||
{name: "marker-pending-purge", marker: true, purged: true},
|
||||
} {
|
||||
name := "purge-response-" + mustGetUUID()
|
||||
oi, err := obj.PutObject(ctx, bucket, name, mustGetPutObjReader(t, bytes.NewReader([]byte("data")), 4, "", ""), ObjectOptions{Versioned: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
version := oi.VersionID
|
||||
if tc.marker {
|
||||
opts := ObjectOptions{Versioned: true, VersionID: mustGetUUID(), DeleteMarker: true, MTime: UTCNow(), ReplicationRequest: true}
|
||||
opts.SetReplicaStatus(replication.Replica)
|
||||
if _, err := obj.DeleteObject(ctx, bucket, name, opts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
version = opts.VersionID
|
||||
}
|
||||
if tc.purged {
|
||||
if _, err := obj.DeleteObject(ctx, bucket, name, ObjectOptions{Versioned: true, VersionID: version, DeleteReplication: pendingPurge}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
before, err := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{Versioned: true, VersionID: version})
|
||||
if tc.marker || tc.purged {
|
||||
if !isErrMethodNotAllowed(err) || !before.DeleteMarker {
|
||||
t.Fatalf("%s: seeded version not exposed as deleted: %+v %v", tc.name, before, err)
|
||||
}
|
||||
} else if err != nil || before.DeleteMarker {
|
||||
t.Fatalf("%s: seeded data version: %+v %v", tc.name, before, err)
|
||||
}
|
||||
if tc.purged && before.VersionPurgeStatus != replication.VersionPurgePending {
|
||||
t.Fatalf("%s: purge state not seeded: %+v", tc.name, before)
|
||||
}
|
||||
req, err := newTestSignedRequestV4(http.MethodDelete, "/"+bucket+"/"+name+"?versionId="+version, 0, nil, creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("%s: status=%d: %s", tc.name, rec.Code, rec.Body.String())
|
||||
}
|
||||
// The handler sets these headers by map key, not canonical name.
|
||||
if got := rec.Header()[xhttp.AmzVersionID]; len(got) != 1 || got[0] != version {
|
||||
t.Fatalf("%s: response version %q, want %q", tc.name, got, version)
|
||||
}
|
||||
if got := len(rec.Header()[xhttp.AmzDeleteMarker]) == 1 && rec.Header()[xhttp.AmzDeleteMarker][0] == "true"; got != tc.marker {
|
||||
t.Fatalf("%s: x-amz-delete-marker=%v for a stored marker=%v", tc.name, got, tc.marker)
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{Versioned: true, VersionID: version}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Fatalf("%s: version not purged: %v", tc.name, err)
|
||||
}
|
||||
t.Logf("%s %s: purged with x-amz-delete-marker=%q", backend, tc.name, rec.Header()[xhttp.AmzDeleteMarker])
|
||||
}
|
||||
}})
|
||||
}
|
||||
@@ -0,0 +1,524 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/once"
|
||||
)
|
||||
|
||||
func TestDeleteMarkerPurgeIntent(t *testing.T) {
|
||||
version := mustGetUUID()
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
edit func(*ObjectOptions)
|
||||
want bool
|
||||
}{
|
||||
{"ordinary", func(*ObjectOptions) {}, true},
|
||||
{"receiver", func(o *ObjectOptions) { o.ReplicationRequest = true; o.SetReplicaStatus(replication.Replica) }, true},
|
||||
{"untrusted-replica", func(o *ObjectOptions) { o.SetReplicaStatus(replication.Replica) }, false},
|
||||
{"complete-purge", func(o *ObjectOptions) {
|
||||
o.DeleteReplication.VersionPurgeStatusInternal = string(replication.VersionPurgeComplete)
|
||||
}, true},
|
||||
{"pending-purge", func(o *ObjectOptions) { o.DeleteReplication.VersionPurgeStatusInternal = "PENDING" }, false},
|
||||
{"failed-purge", func(o *ObjectOptions) { o.DeleteReplication.VersionPurgeStatusInternal = "FAILED" }, false},
|
||||
{"unknown-purge", func(o *ObjectOptions) { o.DeleteReplication.VersionPurgeStatusInternal = "future-state" }, false},
|
||||
{"pending-creation", func(o *ObjectOptions) { o.DeleteReplication.ReplicationStatusInternal = "PENDING" }, false},
|
||||
{"failed-creation", func(o *ObjectOptions) { o.DeleteReplication.ReplicationStatusInternal = "FAILED" }, false},
|
||||
{"completed-creation", func(o *ObjectOptions) { o.DeleteReplication.ReplicationStatusInternal = "COMPLETED" }, false},
|
||||
{"create-marker", func(o *ObjectOptions) { o.DeleteMarker = true }, false},
|
||||
{"empty", func(o *ObjectOptions) { o.VersionID = "" }, false},
|
||||
{"null", func(o *ObjectOptions) { o.VersionID = nullVersionID }, false},
|
||||
{"invalid", func(o *ObjectOptions) { o.VersionID = "invalid" }, false},
|
||||
{"zero-uuid", func(o *ObjectOptions) { o.VersionID = emptyUUID }, false},
|
||||
{"movement", func(o *ObjectOptions) { o.DataMovement = true }, false},
|
||||
{"free-version", func(o *ObjectOptions) { o.InclFreeVersions = true }, false},
|
||||
{"expiration", func(o *ObjectOptions) { o.Expiration.Expire = true }, false},
|
||||
{"transition", func(o *ObjectOptions) { o.Transition.Status = "complete" }, false},
|
||||
{"restored-expiration", func(o *ObjectOptions) { o.Transition.ExpireRestored = true }, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
o := ObjectOptions{VersionID: version, Versioned: true}
|
||||
tc.edit(&o)
|
||||
if got := o.isVersionPurge(); got != tc.want {
|
||||
t.Fatalf("purge=%v want=%v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Every other StorageAPI method panics through the nil embedding: a proof may
|
||||
// read metadata, but must never delete, heal, or ask a different storage API.
|
||||
type absenceProofDisk struct {
|
||||
StorageAPI
|
||||
t *testing.T
|
||||
err error
|
||||
reads *atomic.Int32
|
||||
deletes *atomic.Int32
|
||||
}
|
||||
|
||||
func (d absenceProofDisk) ReadVersion(_ context.Context, _, _, _, _ string, opts ReadOptions) (FileInfo, error) {
|
||||
if opts.ReadData || opts.Healing {
|
||||
d.t.Error("absence proof requested data/healing")
|
||||
}
|
||||
d.reads.Add(1)
|
||||
return FileInfo{}, d.err
|
||||
}
|
||||
|
||||
func (d absenceProofDisk) DeleteVersion(_ context.Context, _, _ string, _ FileInfo, _ bool, _ DeleteOptions) error {
|
||||
if d.deletes == nil {
|
||||
d.t.Error("read-only absence proof attempted a deletion")
|
||||
} else {
|
||||
d.deletes.Add(1)
|
||||
}
|
||||
return d.err
|
||||
}
|
||||
|
||||
func TestVersionPurgeAbsenceProof(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
errs []error
|
||||
quorum, all bool
|
||||
}{
|
||||
{"all-absent", []error{errFileNotFound, errFileVersionNotFound, errFileNotFound, errFileVersionNotFound}, true, true},
|
||||
{"majority-absent", []error{errFileNotFound, errFileVersionNotFound, errFileNotFound, nil}, true, false},
|
||||
{"half-absent", []error{errFileNotFound, errFileVersionNotFound, errDiskNotFound, errDiskNotFound}, false, false},
|
||||
{"corrupt", []error{errFileNotFound, errFileVersionNotFound, errFileCorrupt, errFileCorrupt}, false, false},
|
||||
{"permission", []error{errFileNotFound, errFileVersionNotFound, errDiskAccessDenied, errVolumeAccessDenied}, false, false},
|
||||
{"volume-missing", []error{errFileNotFound, errFileVersionNotFound, errVolumeNotFound, errVolumeNotFound}, false, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var reads atomic.Int32
|
||||
disks := make([]StorageAPI, len(tc.errs))
|
||||
for i, err := range tc.errs {
|
||||
disks[i] = absenceProofDisk{t: t, err: err, reads: &reads}
|
||||
}
|
||||
er := erasureObjects{getDisks: func() []StorageAPI { return disks }}
|
||||
oldQueue := globalMRFState.opCh
|
||||
globalMRFState.opCh = make(chan PartialOperation, 1)
|
||||
defer func() { globalMRFState.opCh = oldQueue }()
|
||||
all, err := er.confirmVersionAbsent(t.Context(), "bucket", "object", mustGetUUID())
|
||||
if (err == nil) != tc.quorum || all != tc.all || reads.Load() != int32(len(disks)) || len(globalMRFState.opCh) != 0 {
|
||||
t.Fatalf("proof all=%v error=%v reads=%d queued=%d", all, err, reads.Load(), len(globalMRFState.opCh))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionPurgeAggregation(t *testing.T) {
|
||||
for _, pure := range []bool{false, true} {
|
||||
for _, fault := range []error{nil, errFileCorrupt, errDiskAccessDenied} {
|
||||
t.Run(fmt.Sprintf("purge_%v_fault_%v", pure, fault), func(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
disks := make([]StorageAPI, 4)
|
||||
for i, err := range []error{nil, errFileNotFound, errFileVersionNotFound, fault} {
|
||||
disks[i] = absenceProofDisk{t: t, err: err, deletes: &calls}
|
||||
}
|
||||
er := erasureObjects{getDisks: func() []StorageAPI { return disks }}
|
||||
err := er.deleteObjectVersion(t.Context(), "bucket", "object", FileInfo{VersionID: mustGetUUID()}, false, pure)
|
||||
if (err == nil) != pure || calls.Load() != 4 {
|
||||
t.Fatalf("aggregation error=%v calls=%d", err, calls.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMarkerPurgeOmittedPool(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
defer replicationTestCapacity(z)()
|
||||
const name = "partially-absent"
|
||||
er := z.serverPools[1].getHashedSet(name)
|
||||
_, opts := seedPurgeMarker(t, er, bucket, name, false)
|
||||
opts.ReplicationRequest = false
|
||||
opts.DeleteReplication = ReplicationState{}
|
||||
// Pool 0 appears absent at read quorum, but half of it cannot be read.
|
||||
missing := z.serverPools[0].getHashedSet(name)
|
||||
original := missing.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
clear(disks[len(disks)/2:])
|
||||
missing.getDisks = func() []StorageAPI { return disks }
|
||||
defer func() { missing.getDisks = original }()
|
||||
_, err := z.DeleteObject(t.Context(), bucket, name, opts)
|
||||
if !isErrWriteQuorum(err) {
|
||||
t.Errorf("omitted pool acknowledged deletion: %T %v", err, err)
|
||||
}
|
||||
if countPurgeMarkers(t, er.getDisks(), bucket, name, opts.VersionID) != 16 {
|
||||
t.Fatal("mutated known copies before checking omitted pool")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMarkerPurgeReceivingPool(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
defer replicationTestCapacity(z)()
|
||||
for _, duplicate := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("duplicate_%v", duplicate), func(t *testing.T) {
|
||||
name := "receiver-" + mustGetUUID()
|
||||
_, opts := seedPurgeMarker(t, z.serverPools[0].getHashedSet(name), bucket, name, false)
|
||||
if duplicate {
|
||||
create := opts
|
||||
create.DeleteMarker = true
|
||||
if _, err := z.serverPools[1].DeleteObject(t.Context(), bucket, name, create); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
// Latest-key routing selects pool 1, but the addressed marker is
|
||||
// in pool 0. A replica purge addresses a version, not the latest.
|
||||
putConsistencyObject(t, z, bucket, name, 1, "newer-data", ObjectOptions{Versioned: true})
|
||||
}
|
||||
if _, err := z.DeleteObject(t.Context(), bucket, name, opts); err != nil {
|
||||
t.Errorf("replica purge did not find its addressed version: %v", err)
|
||||
}
|
||||
for i, pool := range z.serverPools {
|
||||
if got := countPurgeMarkers(t, pool.getHashedSet(name).getDisks(), bucket, name, opts.VersionID); got != 0 {
|
||||
t.Errorf("replica purge left %d marker copies in pool %d", got, i)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMarkerPurgeCallbackIntent(t *testing.T) {
|
||||
for _, loseCopies := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("missing_after_callback_%v", loseCopies), func(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
defer replicationTestCapacity(z)()
|
||||
const name = "callback-pending"
|
||||
er := z.serverPools[0].getHashedSet(name)
|
||||
_, opts := seedPurgeMarker(t, er, bucket, name, false)
|
||||
original := er.getDisks
|
||||
all := append([]StorageAPI(nil), original()...)
|
||||
defer func() { er.getDisks = original }()
|
||||
opts.ReplicationRequest = false
|
||||
opts.DeleteReplication = ReplicationState{}
|
||||
metadata, retention := 0, 0
|
||||
opts.EvalRetentionBypassFn = func(oi ObjectInfo, err error) error {
|
||||
retention++
|
||||
if !oi.DeleteMarker || !isErrMethodNotAllowed(err) {
|
||||
t.Fatalf("retention callback lost marker: %+v %v", oi, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
opts.EvalMetadataFn = func(*ObjectInfo, error) (ReplicateDecision, error) {
|
||||
metadata++
|
||||
if loseCopies {
|
||||
// Inject a disk-state change after preflight, before the set's
|
||||
// metadata-update read. It returns NotFound at read quorum.
|
||||
for _, disk := range all[:8] {
|
||||
if err := disk.DeleteVersion(t.Context(), bucket, name, FileInfo{VersionID: opts.VersionID}, false, DeleteOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
partial := make([]StorageAPI, len(all))
|
||||
copy(partial, all[:8])
|
||||
er.getDisks = func() []StorageAPI { return partial }
|
||||
}
|
||||
decision := ReplicateDecision{}
|
||||
decision.Set(newReplicateTargetDecision("arn1", true, false))
|
||||
return decision, nil
|
||||
}
|
||||
_, err := z.DeleteObject(t.Context(), bucket, name, opts)
|
||||
if loseCopies {
|
||||
if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Errorf("pending update misclassified as physical purge: %T %v", err, err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Errorf("pending update failed: %v", err)
|
||||
}
|
||||
want := 16
|
||||
if loseCopies {
|
||||
want = 8
|
||||
}
|
||||
if got := countPurgeMarkers(t, all, bucket, name, opts.VersionID); got != want || metadata != 1 || retention != 1 {
|
||||
t.Errorf("pending update: copies=%d want=%d metadata=%d retention=%d", got, want, metadata, retention)
|
||||
}
|
||||
if !loseCopies {
|
||||
fi, err := all[0].ReadVersion(t.Context(), "", bucket, name, opts.VersionID, ReadOptions{})
|
||||
if err != nil || fi.VersionPurgeStatus() != replication.VersionPurgePending {
|
||||
t.Errorf("pending state not persisted: %+v %v", fi.ReplicationState, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMarkerReceivingMetadataUpdate(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
defer replicationTestCapacity(z)()
|
||||
for _, state := range []VersionPurgeStatusType{replication.VersionPurgePending, replication.VersionPurgeFailed} {
|
||||
t.Run(string(state), func(t *testing.T) {
|
||||
name := "update-" + mustGetUUID()
|
||||
_, opts := seedPurgeMarker(t, z.serverPools[0].getHashedSet(name), bucket, name, false)
|
||||
create := opts
|
||||
create.DeleteMarker = true
|
||||
if _, err := z.serverPools[1].DeleteObject(t.Context(), bucket, name, create); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.DeleteReplication = ReplicationState{VersionPurgeStatusInternal: string(state)}
|
||||
if _, err := z.DeleteObject(t.Context(), bucket, name, opts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, pool := range z.serverPools {
|
||||
if got := countPurgeMarkers(t, pool.getHashedSet(name).getDisks(), bucket, name, opts.VersionID); got != 16 {
|
||||
t.Errorf("metadata update removed marker copies in pool %d: %d", i, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionPurgeRetentionGate(t *testing.T) {
|
||||
z, er, all, bucket := markerPurgeFixture(t, 4)
|
||||
data, _ := seedPurgeMarker(t, er, bucket, "protected-data", true)
|
||||
denied := errors.New("retention denied")
|
||||
opts := ObjectOptions{Versioned: true, VersionID: data, EvalRetentionBypassFn: func(oi ObjectInfo, err error) error {
|
||||
if err != nil || oi.VersionID != data || oi.DeleteMarker {
|
||||
t.Errorf("wrong retained version: %+v error=%v", oi, err)
|
||||
}
|
||||
return denied
|
||||
}}
|
||||
_, err := z.DeleteObject(t.Context(), bucket, "protected-data", opts)
|
||||
if !errors.Is(err, denied) {
|
||||
t.Fatalf("retention gate: %v", err)
|
||||
}
|
||||
for _, disk := range all {
|
||||
if _, err := disk.ReadVersion(t.Context(), "", bucket, "protected-data", data, ReadOptions{}); err != nil {
|
||||
t.Errorf("protected version changed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func markerPurgeFixture(t *testing.T, n int) (*erasureServerPools, *erasureObjects, []StorageAPI, string) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
obj, dirs, err := prepareErasure(ctx, n)
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatal(err)
|
||||
}
|
||||
restore := replicationTestCapacity(obj)
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
restore()
|
||||
obj.Shutdown(context.Background())
|
||||
removeRoots(dirs)
|
||||
})
|
||||
bucket := getRandomBucketName()
|
||||
if err := obj.MakeBucket(ctx, bucket, MakeBucketOptions{VersioningEnabled: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
er := z.serverPools[0].sets[0]
|
||||
original := er.getDisks
|
||||
t.Cleanup(func() { er.getDisks = original })
|
||||
return z, er, append([]StorageAPI(nil), original()...), bucket
|
||||
}
|
||||
|
||||
func seedPurgeMarker(t *testing.T, er *erasureObjects, bucket, name string, withData bool) (string, ObjectOptions) {
|
||||
t.Helper()
|
||||
dataVersion := ""
|
||||
if withData {
|
||||
oi, err := er.PutObject(t.Context(), bucket, name, mustGetPutObjReader(t, bytes.NewReader([]byte("data")), 4, "", ""), ObjectOptions{Versioned: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dataVersion = oi.VersionID
|
||||
}
|
||||
opts := ObjectOptions{Versioned: true, VersionID: mustGetUUID(), DeleteMarker: true, ReplicationRequest: true, MTime: UTCNow().Add(-time.Hour), NoAuditLog: true}
|
||||
opts.SetReplicaStatus(replication.Replica)
|
||||
if _, err := er.DeleteObject(t.Context(), bucket, name, opts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.DeleteMarker = false
|
||||
return dataVersion, opts
|
||||
}
|
||||
|
||||
func countPurgeMarkers(t *testing.T, disks []StorageAPI, bucket, name, version string) int {
|
||||
t.Helper()
|
||||
n := 0
|
||||
for _, disk := range disks {
|
||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, name, version, ReadOptions{})
|
||||
if err == nil && fi.Deleted {
|
||||
n++
|
||||
} else if err != errFileNotFound && err != errFileVersionNotFound {
|
||||
t.Fatalf("unexpected disk version: deleted=%v error=%v", fi.Deleted, err)
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func TestDeleteMarkerPurgeQuorum(t *testing.T) {
|
||||
for _, tc := range []struct{ disks, online int }{{4, 2}, {16, 7}, {16, 8}, {16, 9}} {
|
||||
t.Run(fmt.Sprintf("%d_disks_%d_online", tc.disks, tc.online), func(t *testing.T) {
|
||||
z, er, all, bucket := markerPurgeFixture(t, tc.disks)
|
||||
const name = "marker"
|
||||
dataVersion, opts := seedPurgeMarker(t, er, bucket, name, true)
|
||||
partial := make([]StorageAPI, len(all))
|
||||
copy(partial, all[:tc.online])
|
||||
er.getDisks = func() []StorageAPI { return partial }
|
||||
_, first := z.DeleteObject(t.Context(), bucket, name, opts)
|
||||
_, retry := z.DeleteObject(t.Context(), bucket, name, opts)
|
||||
if tc.online <= tc.disks/2 {
|
||||
if !isErrWriteQuorum(first) || !isErrWriteQuorum(retry) {
|
||||
t.Errorf("below write quorum: first=%T %v, retry=%T %v", first, first, retry, retry)
|
||||
}
|
||||
} else if first != nil || (retry != nil && !isErrVersionNotFound(retry) && !isErrObjectNotFound(retry)) {
|
||||
t.Errorf("write quorum available: first=%v retry=%v", first, retry)
|
||||
}
|
||||
want := tc.disks - tc.online
|
||||
if tc.online < tc.disks/2 {
|
||||
want = tc.disks
|
||||
}
|
||||
if got := countPurgeMarkers(t, all, bucket, name, opts.VersionID); got != want {
|
||||
t.Errorf("partial purge markers=%d want=%d", got, want)
|
||||
}
|
||||
er.getDisks = func() []StorageAPI { return all }
|
||||
_, err := z.DeleteObject(t.Context(), bucket, name, opts)
|
||||
if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Errorf("full-online retry: %v", err)
|
||||
}
|
||||
if tc.online <= tc.disks/2 {
|
||||
if got := countPurgeMarkers(t, all, bucket, name, opts.VersionID); got != 0 {
|
||||
t.Errorf("full-online retry recreated/retained %d marker copies", got)
|
||||
}
|
||||
}
|
||||
// A quorum-confirmed missing retry can leave minority residue. Exercise
|
||||
// the real dangling-version healer after every disk has returned.
|
||||
_, _ = er.HealObject(t.Context(), bucket, name, opts.VersionID, madmin.HealOpts{Remove: true})
|
||||
if got := countPurgeMarkers(t, all, bucket, name, opts.VersionID); got != 0 {
|
||||
t.Errorf("marker copies after retry and heal=%d", got)
|
||||
}
|
||||
oi, err := z.GetObjectInfo(t.Context(), bucket, name, ObjectOptions{Versioned: true})
|
||||
if err != nil || oi.DeleteMarker || oi.VersionID != dataVersion {
|
||||
t.Errorf("underlying version not visible: %+v error=%v", oi, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMarkerPurgeMissingKey(t *testing.T) {
|
||||
for _, pooled := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("pooled_%v", pooled), func(t *testing.T) {
|
||||
z, er, all, bucket := markerPurgeFixture(t, 4)
|
||||
_, opts := seedPurgeMarker(t, er, bucket, "marker-only", false)
|
||||
er.getDisks = func() []StorageAPI { return []StorageAPI{all[0], all[1], nil, nil} }
|
||||
deleteFn := er.DeleteObject
|
||||
if pooled {
|
||||
deleteFn = z.DeleteObject
|
||||
}
|
||||
_, _ = deleteFn(t.Context(), bucket, "marker-only", opts)
|
||||
_, err := deleteFn(t.Context(), bucket, "marker-only", opts)
|
||||
if !isErrWriteQuorum(err) {
|
||||
t.Errorf("missing retry with only 2/4 absence votes returned %T %v", err, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMarkerHealReplicationIdentity(t *testing.T) {
|
||||
obj, er, all, bucket := markerPurgeFixture(t, 4)
|
||||
const name = "healed-marker"
|
||||
_, opts := seedPurgeMarker(t, er, bucket, name, true)
|
||||
original, err := all[3].ReadVersion(t.Context(), "", bucket, name, opts.VersionID, ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, disk := range all[:2] {
|
||||
if err := disk.DeleteVersion(t.Context(), bucket, name, FileInfo{VersionID: opts.VersionID}, false, DeleteOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := er.HealObject(t.Context(), bucket, name, opts.VersionID, madmin.HealOpts{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, disk := range all {
|
||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, name, opts.VersionID, ReadOptions{})
|
||||
if err != nil || !maps.Equal(fi.Metadata, original.Metadata) {
|
||||
t.Errorf("disk %d lost marker metadata: before=%v after=%v error=%v", i, original.Metadata, fi.Metadata, err)
|
||||
}
|
||||
}
|
||||
// Only the healed half remains readable. Read actual disk state before
|
||||
// invoking scanner replication, rather than constructing an ObjectInfo.
|
||||
er.getDisks = func() []StorageAPI { return []StorageAPI{all[0], all[1], nil, nil} }
|
||||
oi, err := er.GetObjectInfo(t.Context(), bucket, name, ObjectOptions{VersionID: opts.VersionID, Versioned: true})
|
||||
if !oi.DeleteMarker || !isErrMethodNotAllowed(err) {
|
||||
t.Fatalf("healed marker unreadable: %+v %v", oi, err)
|
||||
}
|
||||
er.getDisks = func() []StorageAPI { return all }
|
||||
var outbound atomic.Int32
|
||||
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodDelete {
|
||||
outbound.Add(1)
|
||||
if r.Header.Get(xhttp.MinIOSourceDeleteMarker) != "true" || r.URL.Query().Get("versionId") != opts.VersionID {
|
||||
t.Errorf("unexpected outbound request: %s %s", r.Method, r.URL)
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer remote.Close()
|
||||
client, err := minio.New(strings.TrimPrefix(remote.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
arn := "arn:minio:replication::" + mustGetUUID() + ":bucket"
|
||||
target := &TargetClient{Client: client, ARN: arn, Bucket: "target"}
|
||||
oldTargets := globalBucketTargetSys
|
||||
globalBucketTargetSys = &BucketTargetSys{arnRemotesMap: map[string]arnTarget{arn: {Client: target, lastRefresh: UTCNow()}}, targetsMap: map[string][]madmin.BucketTarget{bucket: {{Arn: arn, TargetBucket: "target"}}}, hc: map[string]epHealth{client.EndpointURL().Host: {Online: true}}}
|
||||
defer func() { globalBucketTargetSys = oldTargets }()
|
||||
cfg := configs[0]
|
||||
cfg.RoleArn = arn
|
||||
meta, err := globalBucketMetadataSys.Get(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta.replicationConfig = &cfg
|
||||
globalBucketMetadataSys.Set(bucket, meta)
|
||||
worker := make(chan ReplicationWorkerOperation, 1)
|
||||
oldPool := globalReplicationPool
|
||||
globalReplicationPool = once.NewSingleton[ReplicationPool]()
|
||||
globalReplicationPool.Set(&ReplicationPool{ctx: t.Context(), objLayer: obj, workers: []chan ReplicationWorkerOperation{worker}, stats: globalReplicationStats.Load(), mrfSaveCh: make(chan MRFReplicateEntry, 1)})
|
||||
defer func() { globalReplicationPool = oldPool }()
|
||||
roi := queueReplicationHeal(t.Context(), bucket, oi, replicationConfig{Config: &cfg, remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{Arn: arn, TargetBucket: "target"}}}}, 0)
|
||||
select {
|
||||
case op := <-worker:
|
||||
d := op.(DeletedObjectReplicationInfo)
|
||||
result := replicateDeleteToTarget(t.Context(), d, target)
|
||||
t.Errorf("healed replica scheduled for creation: version=%s existing=%v result=%+v", d.DeleteMarkerVersionID, roi.ExistingObjResync.mustResync(), result)
|
||||
default:
|
||||
}
|
||||
if outbound.Load() != 0 {
|
||||
t.Errorf("healed replica sent %d new marker creations", outbound.Load())
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,7 @@ func TestMultipartListingAbortBetweenHTTPPages(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
setMultipartListingTestMode(t, false)
|
||||
var firstID, secondID string
|
||||
for attempt := 0; attempt < 32; attempt++ {
|
||||
one, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
@@ -147,6 +148,22 @@ func TestMultipartListingLegacyPreflight(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z.mpCache.Clear()
|
||||
t.Run("legacy-upgrade", func(t *testing.T) {
|
||||
setMultipartListingTestMode(t, true)
|
||||
exact, err := z.ListMultipartUploads(t.Context(), other, "old", "", "", "", 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, exact, "old")
|
||||
const empty = "multipart-legacy-empty"
|
||||
if err := z.MakeBucket(t.Context(), empty, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := z.ListMultipartUploads(t.Context(), empty, "", "", "", "", 10)
|
||||
if err != nil || len(got.Uploads) != 0 {
|
||||
t.Fatalf("old upload in another bucket broke legacy listing: %+v %v", got, err)
|
||||
}
|
||||
})
|
||||
_, err = z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
|
||||
if !errors.Is(err, errMultipartListingLegacy) {
|
||||
t.Fatalf("old upload in another bucket: %v", err)
|
||||
@@ -214,6 +231,7 @@ func TestMultipartListingIdentityFallback(t *testing.T) {
|
||||
|
||||
func TestMultipartAbortPoolsAndRetry(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
setMultipartListingTestMode(t, false)
|
||||
mp, err := z.serverPools[1].NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -286,6 +304,7 @@ func TestMultipartListingMarkerHTTP(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
setMultipartListingTestMode(t, false)
|
||||
for _, tc := range []struct {
|
||||
key, marker string
|
||||
status int
|
||||
@@ -369,6 +388,7 @@ func TestMultipartAbortLateCreateBoundary(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
setMultipartListingTestMode(t, false)
|
||||
t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
|
||||
saved := globalStorageClass
|
||||
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 8}})
|
||||
@@ -454,6 +474,7 @@ func TestMultipartListingScanCosts(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
setMultipartListingTestMode(t, false)
|
||||
t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
|
||||
const bucket, otherBucket = "r9-scan-target", "r9-scan-unrelated"
|
||||
for _, name := range []string{bucket, otherBucket} {
|
||||
@@ -523,6 +544,7 @@ func multipartListingFixture(t *testing.T) (*erasureServerPools, *erasureObjects
|
||||
if err := z.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
setMultipartListingTestMode(t, false)
|
||||
return z, z.serverPools[0].getHashedSet("a"), bucket
|
||||
}
|
||||
|
||||
@@ -673,6 +695,7 @@ func TestMultipartListingAdmissionHTTP(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
setMultipartListingTestMode(t, false)
|
||||
for range cap(multipartScanSlots) {
|
||||
scan, err := startMultipartScan(t.Context(), false)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
// Copyright (c) 2026 Ruohang Feng
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/config/api"
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
"github.com/minio/minio/internal/grid"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
)
|
||||
|
||||
func setMultipartListingTestMode(t *testing.T, legacy bool) {
|
||||
t.Helper()
|
||||
globalAPIConfig.mu.Lock()
|
||||
previous := globalAPIConfig.multipartListingStrict
|
||||
globalAPIConfig.multipartListingStrict = !legacy
|
||||
globalAPIConfig.mu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
globalAPIConfig.mu.Lock()
|
||||
globalAPIConfig.multipartListingStrict = previous
|
||||
globalAPIConfig.mu.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
func TestMultipartListingDefaultMode(t *testing.T) {
|
||||
var uninitialized apiConfig
|
||||
if !uninitialized.getMultipartListingLegacy() {
|
||||
t.Fatal("runtime zero value enabled strict mode before config initialization")
|
||||
}
|
||||
for _, mode := range []string{"", "legacy", "strict"} {
|
||||
var local apiConfig
|
||||
local.init(api.Config{RequestsMax: 1, MultipartListing: mode}, []int{4}, false)
|
||||
if local.getMultipartListingLegacy() != (mode != "strict") {
|
||||
t.Fatalf("unexpected mode for %q", mode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAbortLegacyAvailability(t *testing.T) {
|
||||
for _, drives := range []int{4, 16} {
|
||||
for _, offline := range []int{0, 1, drives / 2} {
|
||||
t.Run(fmt.Sprintf("drives=%d/offline=%d", drives, offline), func(t *testing.T) {
|
||||
obj, dirs, err := prepareErasure(t.Context(), drives)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
|
||||
setMultipartListingTestMode(t, true)
|
||||
previous := globalStorageClass
|
||||
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: drives / 2}})
|
||||
t.Cleanup(func() { globalStorageClass.Update(previous) })
|
||||
const bucket, key = "multipart-legacy-availability", "keep-available"
|
||||
if err := z.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, key, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
set := z.serverPools[0].getHashedSet(key)
|
||||
original := set.getDisks
|
||||
disks := original()
|
||||
set.getDisks = func() []StorageAPI {
|
||||
visible := append([]StorageAPI(nil), disks...)
|
||||
clear(visible[:offline])
|
||||
return visible
|
||||
}
|
||||
t.Cleanup(func() { set.getDisks = original })
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, key, mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("released read-quorum availability regressed: %v", err)
|
||||
}
|
||||
for _, disk := range disks[offline:] {
|
||||
_, err := disk.ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, set.getUploadIDDir(bucket, key, mp.UploadID), "", ReadOptions{})
|
||||
if !errors.Is(err, errFileNotFound) {
|
||||
t.Fatalf("online replica not cleaned: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAbortLegacyPoolOrder(t *testing.T) {
|
||||
for _, owner := range []int{0, 1} {
|
||||
t.Run(fmt.Sprintf("owner=%d", owner), func(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
setMultipartListingTestMode(t, true)
|
||||
mp, err := z.serverPools[owner].NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
emptySet := z.serverPools[1-owner].getHashedSet("a")
|
||||
original := emptySet.getDisks
|
||||
t.Cleanup(func() { emptySet.getDisks = original })
|
||||
emptySet.getDisks = func() []StorageAPI { return make([]StorageAPI, emptySet.setDriveCount) }
|
||||
err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{})
|
||||
if owner == 0 && err != nil {
|
||||
t.Fatalf("unrelated later pool blocked legacy cancellation: %v", err)
|
||||
}
|
||||
if owner == 1 {
|
||||
if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 {
|
||||
t.Fatalf("unknown earlier pool must retain released error behavior: %v", err)
|
||||
}
|
||||
if _, err := z.serverPools[owner].GetMultipartInfo(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("later pool visited despite earlier error: %v", err)
|
||||
}
|
||||
emptySet.getDisks = original
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAbortMinorityLegacyCleanup(t *testing.T) {
|
||||
for _, failDelete := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("delete-fails=%v", failDelete), func(t *testing.T) {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "old", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := set.getUploadIDDir(bucket, "old", mp.UploadID)
|
||||
original := set.getDisks
|
||||
disks := original()
|
||||
fi, err := disks[0].ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, path, "", ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
delete(fi.Metadata, multipartMetaBucket)
|
||||
delete(fi.Metadata, multipartMetaObject)
|
||||
if err := disks[0].WriteMetadata(t.Context(), bucket, minioMetaMultipartBucket, path, fi); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, d := range disks[1:] {
|
||||
if err := d.Delete(t.Context(), minioMetaMultipartBucket, path, DeleteOptions{Recursive: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
report, err := z.multipartPreflight(t.Context())
|
||||
if err != nil || report.Ready || report.LegacyUploads != 1 {
|
||||
t.Fatalf("invalid legacy fixture: %+v %v", report, err)
|
||||
}
|
||||
if failDelete {
|
||||
set.getDisks = func() []StorageAPI {
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
wrapped[0] = multipartListingFaultDisk{StorageAPI: disks[0], delete: func(context.Context, string, string, DeleteOptions) error { return errFaultyDisk }}
|
||||
return wrapped
|
||||
}
|
||||
t.Cleanup(func() { set.getDisks = original })
|
||||
err := z.AbortMultipartUpload(t.Context(), bucket, "old", mp.UploadID, ObjectOptions{})
|
||||
if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 {
|
||||
t.Fatalf("empty drives masked failed deletion of the observed replica: %v", err)
|
||||
}
|
||||
if _, present := z.mpCache.Load(mp.UploadID); !present {
|
||||
t.Fatal("failed cancellation evicted upload from cache")
|
||||
}
|
||||
set.getDisks = original
|
||||
}
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "old", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
report, err = z.multipartPreflight(t.Context())
|
||||
if err != nil || !report.Ready || report.LegacyUploads != 0 {
|
||||
t.Fatalf("acknowledged cleanup left online legacy remnants: %+v %v", report, err)
|
||||
}
|
||||
if _, present := z.mpCache.Load(mp.UploadID); present {
|
||||
t.Fatal("successful cancellation retained cache entry")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAbortWrongTargetKeepsCache(t *testing.T) {
|
||||
for _, legacy := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("legacy=%v", legacy), func(t *testing.T) {
|
||||
z, _, bucket := multipartListingFixture(t)
|
||||
setMultipartListingTestMode(t, legacy)
|
||||
const other = "multipart-other-bucket"
|
||||
if err := z.MakeBucket(t.Context(), other, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "valid-key", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, target := range [][2]string{{bucket, "wrong-key"}, {other, "valid-key"}} {
|
||||
err := z.AbortMultipartUpload(t.Context(), target[0], target[1], mp.UploadID, ObjectOptions{})
|
||||
var invalid InvalidUploadID
|
||||
if !errors.As(err, &invalid) {
|
||||
t.Fatalf("wrong target result: %v", err)
|
||||
}
|
||||
if _, present := z.mpCache.Load(mp.UploadID); !present {
|
||||
t.Fatal("wrong-target abort evicted another upload")
|
||||
}
|
||||
if _, err := z.GetMultipartInfo(t.Context(), bucket, "valid-key", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("wrong-target abort harmed valid upload: %v", err)
|
||||
}
|
||||
}
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "valid-key", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var invalid InvalidUploadID
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "valid-key", mp.UploadID, ObjectOptions{}); !errors.As(err, &invalid) {
|
||||
t.Fatalf("repeat abort: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAbortRejectsUnsafeID(t *testing.T) {
|
||||
for _, legacy := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("legacy=%v", legacy), func(t *testing.T) {
|
||||
z, _, bucket := multipartListingFixture(t)
|
||||
setMultipartListingTestMode(t, legacy)
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "keep", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, suffix := range []string{".", "..", "../other", "a/b", "a\\b", ""} {
|
||||
id := base64.RawURLEncoding.EncodeToString([]byte("deployment." + suffix))
|
||||
err := z.AbortMultipartUpload(t.Context(), bucket, "keep", id, ObjectOptions{})
|
||||
var invalid InvalidUploadID
|
||||
if !errors.As(err, &invalid) {
|
||||
t.Fatalf("unsafe ID accepted: suffix=%q err=%v", suffix, err)
|
||||
}
|
||||
if _, err := z.GetMultipartInfo(t.Context(), bucket, "keep", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("valid upload harmed by rejected ID: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAbortPeerNotification(t *testing.T) {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
tg, err := grid.SetupTestGrid(2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(tg.Cleanup)
|
||||
var notifications atomic.Int32
|
||||
if err := cleanupUploadIDCacheMetaRPC.Register(tg.Managers[1], func(*grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
|
||||
notifications.Add(1)
|
||||
return grid.NoPayload{}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
host, err := xnet.ParseHost(strings.TrimPrefix(tg.Hosts[1], "http://"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previous := globalNotificationSys
|
||||
globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{{
|
||||
host: host,
|
||||
gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[1]) },
|
||||
}}}
|
||||
t.Cleanup(func() { globalNotificationSys = previous })
|
||||
// Use the real distributed lock implementation: Unlock cancels its derived
|
||||
// context. Local locks do not, and would hide a broken notification context.
|
||||
oldMutex, oldLockers := set.nsMutex, set.getLockers
|
||||
lockers := []dsync.NetLocker{newLocker(), newLocker()}
|
||||
set.nsMutex = newNSLock(true)
|
||||
set.getLockers = func() ([]dsync.NetLocker, string) { return lockers, "multipart-test" }
|
||||
t.Cleanup(func() { set.nsMutex, set.getLockers = oldMutex, oldLockers })
|
||||
for _, legacy := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("legacy=%v", legacy), func(t *testing.T) {
|
||||
setMultipartListingTestMode(t, legacy)
|
||||
for range 4 {
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "valid", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := notifications.Load()
|
||||
var invalid InvalidUploadID
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "wrong", mp.UploadID, ObjectOptions{}); !errors.As(err, &invalid) {
|
||||
t.Fatalf("wrong target: %v", err)
|
||||
}
|
||||
if notifications.Load() != before {
|
||||
t.Fatal("wrong-target abort sent a destructive peer notification")
|
||||
}
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "valid", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if notifications.Load() != before+1 {
|
||||
t.Fatal("successful abort lost peer notification after unlocking")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAbortStrictMajority(t *testing.T) {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
original := set.getDisks
|
||||
disks := original()
|
||||
set.getDisks = func() []StorageAPI {
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
wrapped[0] = multipartListingFaultDisk{StorageAPI: disks[0], delete: func(context.Context, string, string, DeleteOptions) error { return errFaultyDisk }}
|
||||
return wrapped
|
||||
}
|
||||
t.Cleanup(func() { set.getDisks = original })
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("ordinary majority cancellation was tightened: %v", err)
|
||||
}
|
||||
// One failed deletion is tolerated in the ordinary majority path. Retrying
|
||||
// after recovery must also clean the now-minority remnant.
|
||||
if _, err := disks[0].ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, set.getUploadIDDir(bucket, "a", mp.UploadID), "", ReadOptions{}); err != nil {
|
||||
t.Fatalf("fault fixture did not leave a remnant: %v", err)
|
||||
}
|
||||
set.getDisks = original
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
+37
-12
@@ -1753,11 +1753,11 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
|
||||
}
|
||||
|
||||
// abortMultipartUpload confirms absence on a strict majority of this set.
|
||||
// Unlike an existence read followed by best-effort deletion, it also permits
|
||||
// retrying a partial deletion which no longer has a readable metadata quorum.
|
||||
// This does not fence creation writes still executing after a storage timeout.
|
||||
func (er erasureObjects) abortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (bool, error) {
|
||||
// abortMultipartUpload retains read-quorum validation and best-effort cleanup
|
||||
// in legacy mode. Strict mode requires majority deletion acknowledgements and
|
||||
// permits retrying remnants below read quorum. Neither mode fences creation
|
||||
// writes still executing after a storage timeout.
|
||||
func (er erasureObjects) abortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions, legacy bool) (bool, error) {
|
||||
if !opts.NoAuditLog {
|
||||
auditObjectErasureSet(ctx, "AbortMultipartUpload", object, &er)
|
||||
}
|
||||
@@ -1769,21 +1769,34 @@ func (er erasureObjects) abortMultipartUpload(ctx context.Context, bucket, objec
|
||||
if !ok || internalID == "" || internalID == "." || internalID == ".." || strings.ContainsAny(internalID, "/\\") {
|
||||
return false, InvalidUploadID{Bucket: bucket, Object: object, UploadID: uploadID}
|
||||
}
|
||||
if legacy {
|
||||
// Keep the released read-quorum and best-effort cleanup behavior.
|
||||
// The upload ID safety check above applies to both modes.
|
||||
defer er.deleteAll(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID))
|
||||
_, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
|
||||
err = toObjectErr(err, bucket, object, uploadID)
|
||||
if _, absent := err.(InvalidUploadID); absent {
|
||||
return false, nil
|
||||
}
|
||||
return err == nil, err
|
||||
}
|
||||
disks := er.getDisks()
|
||||
uploadPath := er.getUploadIDDir(bucket, object, uploadID)
|
||||
_, errs := readAllFileInfo(ctx, disks, bucket, minioMetaMultipartBucket, uploadPath, "", false, false)
|
||||
quorum := er.setDriveCount/2 + 1
|
||||
found, absent := false, 0
|
||||
for _, err := range errs {
|
||||
observed := make([]bool, len(disks))
|
||||
for i, err := range errs {
|
||||
switch {
|
||||
case err == nil, errors.Is(err, errFileCorrupt):
|
||||
found = true
|
||||
observed[i] = true
|
||||
case errors.Is(err, errFileNotFound), errors.Is(err, errFileVersionNotFound):
|
||||
absent++
|
||||
}
|
||||
}
|
||||
if absent >= quorum {
|
||||
return found, nil
|
||||
if absent >= quorum && !found {
|
||||
return false, nil
|
||||
}
|
||||
if !found {
|
||||
return false, toObjectErr(errErasureReadQuorum, bucket, object, uploadID)
|
||||
@@ -1801,13 +1814,25 @@ func (er erasureObjects) abortMultipartUpload(ctx context.Context, bucket, objec
|
||||
return err
|
||||
}, i)
|
||||
}
|
||||
return true, toObjectErr(reduceWriteQuorumErrs(ctx, g.Wait(), nil, quorum), bucket, object, uploadID)
|
||||
deleteErrs := g.Wait()
|
||||
if err := reduceWriteQuorumErrs(ctx, deleteErrs, nil, quorum); err != nil {
|
||||
return true, toObjectErr(err, bucket, object, uploadID)
|
||||
}
|
||||
if absent >= quorum {
|
||||
// Missing disks must not mask a failed cleanup of known remnants.
|
||||
for i, found := range observed {
|
||||
if found && deleteErrs[i] != nil {
|
||||
return true, toObjectErr(errErasureWriteQuorum, bucket, object, uploadID)
|
||||
}
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// AbortMultipartUpload confirms logical cancellation. Offline part data may
|
||||
// still need stale-upload cleanup after its drives return.
|
||||
// AbortMultipartUpload cancels an upload using the configured mode. Offline
|
||||
// part data may still need stale-upload cleanup after its drives return.
|
||||
func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (err error) {
|
||||
found, err := er.abortMultipartUpload(ctx, bucket, object, uploadID, opts)
|
||||
found, err := er.abortMultipartUpload(ctx, bucket, object, uploadID, opts, globalAPIConfig.getMultipartListingLegacy())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+32
-6
@@ -1657,7 +1657,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
|
||||
}
|
||||
|
||||
func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object string, fi FileInfo, forceDelMarker bool) error {
|
||||
func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object string, fi FileInfo, forceDelMarker, purge bool) error {
|
||||
disks := er.getDisks()
|
||||
// Assume (N/2 + 1) quorum for Delete()
|
||||
// this is a theoretical assumption such that
|
||||
@@ -1673,7 +1673,13 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
|
||||
if disks[index] == nil {
|
||||
return errDiskNotFound
|
||||
}
|
||||
return disks[index].DeleteVersion(ctx, bucket, object, fi, forceDelMarker, DeleteOptions{})
|
||||
err := disks[index].DeleteVersion(ctx, bucket, object, fi, forceDelMarker, DeleteOptions{})
|
||||
// Physical removal and reliable already-absent replies are the same
|
||||
// outcome. Creation and metadata updates must not use this quorum.
|
||||
if purge && (err == errFileNotFound || err == errFileVersionNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}, index)
|
||||
}
|
||||
// return errors if any during deletion
|
||||
@@ -2023,6 +2029,11 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
|
||||
if opts.DeleteMarker {
|
||||
versionFound = false
|
||||
} else if !tryDel {
|
||||
if opts.isVersionPurge() && (isErrObjectNotFound(gerr) || isErrVersionNotFound(gerr)) {
|
||||
if err := er.checkPurgeAbsent(ctx, bucket, object, opts.VersionID); err != nil {
|
||||
return objInfo, err
|
||||
}
|
||||
}
|
||||
return objInfo, gerr
|
||||
}
|
||||
}
|
||||
@@ -2105,6 +2116,13 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
|
||||
}
|
||||
}
|
||||
|
||||
purge := opts.isVersionPurge()
|
||||
if purge {
|
||||
// A delete marker describes the version being removed; it is not an
|
||||
// instruction to create that marker on disks which already lack it.
|
||||
markDelete, deleteMarker = false, false
|
||||
}
|
||||
|
||||
modTime := opts.MTime
|
||||
if opts.MTime.IsZero() {
|
||||
modTime = UTCNow()
|
||||
@@ -2151,7 +2169,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
|
||||
// delete marker. Add delete marker, since we don't have
|
||||
// any version specified explicitly. Or if a particular
|
||||
// version id needs to be replicated.
|
||||
if err = er.deleteObjectVersion(ctx, bucket, object, fi, opts.DeleteMarker); err != nil {
|
||||
if err = er.deleteObjectVersion(ctx, bucket, object, fi, opts.DeleteMarker, false); err != nil {
|
||||
return objInfo, toObjectErr(err, bucket, object)
|
||||
}
|
||||
oi := fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
|
||||
@@ -2174,11 +2192,19 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
|
||||
if opts.SkipFreeVersion {
|
||||
dfi.SetSkipTierFreeVersion()
|
||||
}
|
||||
if err = er.deleteObjectVersion(ctx, bucket, object, dfi, opts.DeleteMarker); err != nil {
|
||||
if err = er.deleteObjectVersion(ctx, bucket, object, dfi, opts.DeleteMarker, purge); err != nil {
|
||||
return objInfo, toObjectErr(err, bucket, object)
|
||||
}
|
||||
|
||||
return dfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
|
||||
oi := dfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
|
||||
if purge {
|
||||
// Preserve the DELETE response's identity without reusing it as a disk
|
||||
// instruction to create a marker. The lookup also exposes a data
|
||||
// version pending purge as deleted for visibility; only a stored
|
||||
// marker, which carries no erasure layout, is reported as one.
|
||||
oi.DeleteMarker = goi.DeleteMarker && goi.DataBlocks == 0
|
||||
}
|
||||
return oi, nil
|
||||
}
|
||||
|
||||
// Send the successful but partial upload/delete, however ignore
|
||||
@@ -2467,7 +2493,7 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
|
||||
|
||||
storageDisks := er.getDisks()
|
||||
|
||||
if err = er.deleteObjectVersion(ctx, bucket, object, fi, false); err != nil {
|
||||
if err = er.deleteObjectVersion(ctx, bucket, object, fi, false, false); err != nil {
|
||||
eventName = event.ObjectTransitionFailed
|
||||
}
|
||||
|
||||
|
||||
@@ -316,6 +316,11 @@ func (z *erasureServerPools) retireReplicaCopies(ctx context.Context, bucket, ob
|
||||
func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
|
||||
copies, err := z.objectPoolInfos(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
if opts.isVersionPurge() && (isErrObjectNotFound(err) || isErrVersionNotFound(err)) {
|
||||
if quorumErr := z.checkPurgeAbsent(ctx, bucket, object, opts.VersionID); quorumErr != nil {
|
||||
return ObjectInfo{}, quorumErr
|
||||
}
|
||||
}
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
primary := copies[0]
|
||||
@@ -365,6 +370,21 @@ func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket,
|
||||
opts.EvalMetadataFn = nil
|
||||
}
|
||||
}
|
||||
if opts.isVersionPurge() {
|
||||
// A missing pool may have only read-quorum absence. Verify every pool
|
||||
// omitted by lookup before mutating the known copies.
|
||||
for i, pool := range z.serverPools {
|
||||
found := false
|
||||
for _, copy := range copies {
|
||||
found = found || copy.Index == i
|
||||
}
|
||||
if !found {
|
||||
if err := pool.getHashedSet(object).checkPurgeAbsent(ctx, bucket, object, opts.VersionID); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if opts.VersionID == "" && (opts.Versioned || opts.VersionSuspended) {
|
||||
// A single new delete marker hides the current version. Older versions
|
||||
// remain history and must not be removed by an unqualified DELETE.
|
||||
|
||||
@@ -618,7 +618,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, idx int, bu
|
||||
if objInfo.isMultipart() {
|
||||
res, err := z.NewMultipartUpload(ctx, bucket, objInfo.Name, ObjectOptions{
|
||||
VersionID: objInfo.VersionID,
|
||||
UserDefined: objInfo.UserDefined,
|
||||
UserDefined: migrationObjectMetadata(objInfo),
|
||||
NoAuditLog: true,
|
||||
SrcPoolIdx: idx,
|
||||
DataMovement: true,
|
||||
@@ -681,7 +681,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, idx int, bu
|
||||
SrcPoolIdx: idx,
|
||||
VersionID: objInfo.VersionID,
|
||||
MTime: objInfo.ModTime,
|
||||
UserDefined: objInfo.UserDefined,
|
||||
UserDefined: migrationObjectMetadata(objInfo),
|
||||
PreserveETag: objInfo.ETag, // Preserve original ETag to ensure same metadata.
|
||||
IndexCB: func() []byte {
|
||||
return objInfo.Parts[0].Index // Preserve part Index to ensure decompression works.
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import xhttp "github.com/minio/minio/internal/http"
|
||||
|
||||
// migrationObjectMetadata restores tags removed from the read representation.
|
||||
// Keep the original revision, including ordered empty states, for the existing
|
||||
// locked reconciliation at PUT or multipart completion. Moving is not a new
|
||||
// tagging mutation, and the write must not modify the reader's metadata map.
|
||||
func migrationObjectMetadata(oi ObjectInfo) map[string]string {
|
||||
metadata := cloneMSS(oi.UserDefined)
|
||||
delete(metadata, xhttp.AmzObjectTagging)
|
||||
if oi.UserTags != "" {
|
||||
metadata[xhttp.AmzObjectTagging] = oi.UserTags
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
@@ -0,0 +1,592 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
const migrationTagRevision = ReservedMetadataPrefixLower + TaggingTimestamp
|
||||
|
||||
var migrationTagStates = []struct {
|
||||
name, tags, revision string
|
||||
hasRevision bool
|
||||
}{
|
||||
{name: "initial", tags: "team=storage&path=a%2Fb"},
|
||||
{name: "empty-revision", tags: "team=storage", hasRevision: true},
|
||||
{name: "invalid-revision", tags: "team=storage", revision: "invalid", hasRevision: true},
|
||||
{name: "ordered", tags: "team=storage", revision: "2026-09-16T09:00:00Z", hasRevision: true},
|
||||
{name: "cleared", revision: "2026-09-16T09:00:00Z", hasRevision: true},
|
||||
{name: "never-tagged"},
|
||||
{name: "empty-empty-revision", hasRevision: true},
|
||||
{name: "empty-invalid-revision", revision: "invalid", hasRevision: true},
|
||||
}
|
||||
|
||||
func TestMigrationObjectMetadata(t *testing.T) {
|
||||
for _, state := range migrationTagStates {
|
||||
for _, residual := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/residual=%t", state.name, residual), func(t *testing.T) {
|
||||
raw := map[string]string{
|
||||
xhttp.AmzObjectTagging: state.tags,
|
||||
"x-amz-meta-team": "storage",
|
||||
"x-amz-meta-empty": "",
|
||||
ReservedMetadataPrefix + "compression": "opaque-compression-state",
|
||||
ReservedMetadataPrefix + "sealed-key": "opaque-key-state",
|
||||
"x-amz-object-lock-retain-until-date": "2030-01-01T00:00:00Z",
|
||||
"x-amz-object-lock-legal-hold": "ON",
|
||||
"x-amz-tagging": "keep-noncanonical-metadata",
|
||||
ReservedMetadataPrefix + "actual-size": "42",
|
||||
ReservedMetadataPrefix + "replica-status": "opaque-replication-state",
|
||||
}
|
||||
if state.hasRevision {
|
||||
raw[migrationTagRevision] = state.revision
|
||||
}
|
||||
oi := (FileInfo{Metadata: raw}).ToObjectInfo("bucket", "object", false)
|
||||
if residual {
|
||||
oi.UserDefined[xhttp.AmzObjectTagging] = "stale=raw"
|
||||
}
|
||||
before := maps.Clone(oi.UserDefined)
|
||||
got := migrationObjectMetadata(oi)
|
||||
if got == nil || !maps.Equal(before, oi.UserDefined) {
|
||||
t.Fatal("helper must return a new non-nil map without changing the read snapshot")
|
||||
}
|
||||
value, exists := got[xhttp.AmzObjectTagging]
|
||||
if value != state.tags || exists != (state.tags != "") {
|
||||
t.Fatalf("raw tags=(%q,%t), want (%q,%t)", value, exists, state.tags, state.tags != "")
|
||||
}
|
||||
stamp, present := got[migrationTagRevision]
|
||||
if stamp != state.revision || present != state.hasRevision {
|
||||
t.Fatalf("revision=(%q,%t), want (%q,%t)", stamp, present, state.revision, state.hasRevision)
|
||||
}
|
||||
for key, value := range before {
|
||||
if stored, exists := got[key]; key != xhttp.AmzObjectTagging && (!exists || stored != value) {
|
||||
t.Errorf("lost metadata %q", key)
|
||||
}
|
||||
}
|
||||
got["x-amz-meta-team"] = "changed"
|
||||
if !maps.Equal(before, oi.UserDefined) {
|
||||
t.Fatal("output aliases the read snapshot")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, metadata := range []map[string]string{nil, {}} {
|
||||
got := migrationObjectMetadata(ObjectInfo{UserDefined: metadata})
|
||||
if got == nil || len(got) != 0 {
|
||||
t.Fatalf("empty input must yield a writable empty map: %#v", got)
|
||||
}
|
||||
got["new"] = "value"
|
||||
if len(metadata) != 0 {
|
||||
t.Fatal("empty map input was aliased")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Allocation ignores the host's used-space percentage, as in the existing tag
|
||||
// fixtures. All object data and metadata still use real erasure-storage disks.
|
||||
func migrationTestPools(t *testing.T) (*erasureServerPools, string) {
|
||||
t.Helper()
|
||||
z, bucket := consistencyPools(t)
|
||||
for _, pool := range z.serverPools {
|
||||
for _, set := range pool.sets {
|
||||
original := set.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i := range disks {
|
||||
disks[i] = tagTestCapacityDisk{StorageAPI: disks[i]}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
t.Cleanup(func() { set.getDisks = original })
|
||||
}
|
||||
}
|
||||
return z, bucket
|
||||
}
|
||||
|
||||
func migrationTestMover(t *testing.T, z *erasureServerPools, source int, kind string) func(context.Context, int, string, *GetObjectReader) error {
|
||||
t.Helper()
|
||||
if kind == "rebalance" {
|
||||
z.rebalMu.Lock()
|
||||
z.rebalMeta = &rebalanceMeta{PoolStats: []*rebalanceStats{{}, {}}}
|
||||
z.rebalMeta.PoolStats[source] = &rebalanceStats{Participating: true, Info: rebalanceInfo{Status: rebalStarted}}
|
||||
z.rebalMu.Unlock()
|
||||
t.Cleanup(func() { z.rebalMu.Lock(); z.rebalMeta = nil; z.rebalMu.Unlock() })
|
||||
return z.rebalanceObject
|
||||
}
|
||||
z.poolMetaMutex.Lock()
|
||||
z.poolMeta.Pools[source].Decommission = &PoolDecommissionInfo{}
|
||||
z.poolMetaMutex.Unlock()
|
||||
t.Cleanup(func() { z.poolMetaMutex.Lock(); z.poolMeta.Pools[source].Decommission = nil; z.poolMetaMutex.Unlock() })
|
||||
return z.decommissionObject
|
||||
}
|
||||
|
||||
func migrationTestObject(t *testing.T, z *erasureServerPools, bucket, object string, source int, multipart bool, parts []string, opts ObjectOptions) ObjectInfo {
|
||||
t.Helper()
|
||||
if !multipart {
|
||||
oi := putConsistencyObject(t, z, bucket, object, source, strings.Join(parts, ""), opts)
|
||||
return migrationTestPersistedInfo(t, z, source, oi)
|
||||
}
|
||||
mp, err := z.serverPools[source].NewMultipartUpload(t.Context(), bucket, object, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
completed := make([]CompletePart, len(parts))
|
||||
for i, body := range parts {
|
||||
part, err := z.serverPools[source].PutObjectPart(t.Context(), bucket, object, mp.UploadID, i+1,
|
||||
mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
completed[i] = CompletePart{PartNumber: i + 1, ETag: part.ETag}
|
||||
}
|
||||
oi, err := z.serverPools[source].CompleteMultipartUpload(t.Context(), bucket, object, mp.UploadID, completed, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !oi.isMultipart() {
|
||||
t.Fatal("fixture did not create a multipart object")
|
||||
}
|
||||
return migrationTestPersistedInfo(t, z, source, oi)
|
||||
}
|
||||
|
||||
func migrationTestPersistedInfo(t *testing.T, z *erasureServerPools, pool int, oi ObjectInfo) ObjectInfo {
|
||||
t.Helper()
|
||||
// PUT can return transient fields such as tier-free-versionID that are not
|
||||
// stored. Take the same persisted read representation used by the movers.
|
||||
got, err := z.serverPools[pool].GetObjectInfo(t.Context(), oi.Bucket, oi.Name, ObjectOptions{VersionID: migrationTestVersion(oi)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func migrationTestVersion(oi ObjectInfo) string {
|
||||
if oi.VersionID == "" {
|
||||
return nullVersionID
|
||||
}
|
||||
return oi.VersionID
|
||||
}
|
||||
|
||||
func migrationTestReader(t *testing.T, z *erasureServerPools, source int, oi ObjectInfo) *GetObjectReader {
|
||||
t.Helper()
|
||||
gr, err := z.serverPools[source].GetObjectNInfo(t.Context(), oi.Bucket, oi.Name, nil, nil,
|
||||
ObjectOptions{VersionID: migrationTestVersion(oi), NoLock: true, NoDecryption: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { gr.Close() })
|
||||
return gr
|
||||
}
|
||||
|
||||
func migrationTestStored(t *testing.T, z *erasureServerPools, pool int, original ObjectInfo, tags, revision string, hasRevision bool, body string) {
|
||||
t.Helper()
|
||||
version := migrationTestVersion(original)
|
||||
got, err := z.serverPools[pool].GetObjectInfo(t.Context(), original.Bucket, original.Name, ObjectOptions{VersionID: version})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.UserTags != tags || got.ETag != original.ETag || !got.ModTime.Equal(original.ModTime) || got.VersionID != original.VersionID || got.Size != original.Size {
|
||||
t.Errorf("pool %d changed tags/object identity: tags=%q want=%q version=%q/%q etag=%q/%q size=%d/%d mtime=%v/%v", pool, got.UserTags, tags, got.VersionID, original.VersionID, got.ETag, original.ETag, got.Size, original.Size, got.ModTime, original.ModTime)
|
||||
}
|
||||
infos, errs := readAllFileInfo(t.Context(), z.serverPools[pool].getHashedSet(original.Name).getDisks(), "", original.Bucket, original.Name, version, false, false)
|
||||
for disk, fi := range infos {
|
||||
if errs[disk] != nil {
|
||||
t.Fatalf("pool %d disk %d: %v", pool, disk, errs[disk])
|
||||
}
|
||||
stamp, exists := fi.Metadata[migrationTagRevision]
|
||||
if fi.Metadata[xhttp.AmzObjectTagging] != tags || stamp != revision || exists != hasRevision {
|
||||
t.Errorf("pool %d disk %d: tags=%q revision=(%q,%t), want %q (%q,%t)", pool, disk, fi.Metadata[xhttp.AmzObjectTagging], stamp, exists, tags, revision, hasRevision)
|
||||
}
|
||||
for key, value := range original.UserDefined {
|
||||
if key != migrationTagRevision && fi.Metadata[key] != value {
|
||||
t.Errorf("pool %d disk %d lost metadata %q", pool, disk, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
gr, err := z.serverPools[pool].GetObjectNInfo(t.Context(), original.Bucket, original.Name, nil, nil, ObjectOptions{VersionID: version})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := io.ReadAll(gr)
|
||||
gr.Close()
|
||||
if err != nil || !bytes.Equal(data, []byte(body)) {
|
||||
t.Fatalf("pool %d content changed: bytes=%d err=%v", pool, len(data), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMigrationPreservesTagState(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for _, method := range []string{"put", "multipart"} {
|
||||
for source := range 2 {
|
||||
for _, state := range migrationTagStates {
|
||||
t.Run(fmt.Sprintf("%s/%s/source=%d/%s", kind, method, source, state.name), func(t *testing.T) {
|
||||
move := migrationTestMover(t, z, source, kind)
|
||||
meta := map[string]string{xhttp.AmzObjectTagging: state.tags, "x-amz-meta-owner": "retained"}
|
||||
if state.hasRevision {
|
||||
meta[migrationTagRevision] = state.revision
|
||||
}
|
||||
oi := migrationTestObject(t, z, bucket, t.Name(), source, method == "multipart", []string{"payload"}, ObjectOptions{Versioned: true, UserDefined: meta})
|
||||
migrationTestStored(t, z, source, oi, state.tags, state.revision, state.hasRevision, "payload")
|
||||
gr := migrationTestReader(t, z, source, oi)
|
||||
before := maps.Clone(gr.ObjInfo.UserDefined)
|
||||
if err := move(t.Context(), source, bucket, gr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !maps.Equal(before, gr.ObjInfo.UserDefined) {
|
||||
t.Error("migration modified its source snapshot")
|
||||
}
|
||||
migrationTestStored(t, z, 1-source, oi, state.tags, state.revision, state.hasRevision, "payload")
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type migrationTestGate struct {
|
||||
entered, resume chan struct{}
|
||||
once, release sync.Once
|
||||
}
|
||||
|
||||
func newMigrationTestGate() *migrationTestGate {
|
||||
return &migrationTestGate{entered: make(chan struct{}), resume: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (g *migrationTestGate) wait(ctx context.Context) error {
|
||||
g.once.Do(func() { close(g.entered) })
|
||||
select {
|
||||
case <-g.resume:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (g *migrationTestGate) unblock() { g.release.Do(func() { close(g.resume) }) }
|
||||
|
||||
type migrationTestGateReader struct {
|
||||
io.Reader
|
||||
ctx context.Context
|
||||
gate *migrationTestGate
|
||||
}
|
||||
|
||||
func (r migrationTestGateReader) Read(p []byte) (int, error) {
|
||||
if err := r.gate.wait(r.ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return r.Reader.Read(p)
|
||||
}
|
||||
|
||||
func TestPoolsMigrationRechecksTags(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
const recent = "2026-09-16T10:00:00Z"
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for _, method := range []string{"put", "multipart"} {
|
||||
for source := range 2 {
|
||||
for _, tags := range []string{"state=after", ""} {
|
||||
t.Run(fmt.Sprintf("%s/%s/source=%d/clear=%t", kind, method, source, tags == ""), func(t *testing.T) {
|
||||
move := migrationTestMover(t, z, source, kind)
|
||||
oi := migrationTestObject(t, z, bucket, t.Name(), source, method == "multipart", []string{"payload"}, ObjectOptions{
|
||||
Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "state=before"},
|
||||
})
|
||||
gr := migrationTestReader(t, z, source, oi)
|
||||
before := maps.Clone(gr.ObjInfo.UserDefined)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
gate := newMigrationTestGate()
|
||||
done, finished := make(chan error, 1), make(chan struct{})
|
||||
if method == "multipart" {
|
||||
// The first part read is after persisted upload initialization,
|
||||
// but before completion takes the object lock and reconciles.
|
||||
gr.Reader = migrationTestGateReader{Reader: gr.Reader, ctx: ctx, gate: gate}
|
||||
go func() { defer close(finished); done <- move(ctx, source, bucket, gr) }()
|
||||
defer func() { gate.unblock(); cancel(); <-finished }()
|
||||
select {
|
||||
case <-gate.entered:
|
||||
case err := <-done:
|
||||
t.Fatalf("migration finished before the part barrier: %v", err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
uploads, err := z.serverPools[1-source].listMultipartUploadsExact(ctx, bucket, oi.Name)
|
||||
if err != nil || len(uploads.Uploads) != 1 {
|
||||
t.Fatalf("upload not persisted at barrier: %+v, %v", uploads, err)
|
||||
}
|
||||
info, err := z.serverPools[1-source].GetMultipartInfo(ctx, bucket, oi.Name, uploads.Uploads[0].UploadID, ObjectOptions{})
|
||||
if err != nil || info.UserDefined[xhttp.AmzObjectTagging] != "state=before" {
|
||||
t.Fatalf("upload lost snapshot tags: %v, %v", info.UserDefined, err)
|
||||
}
|
||||
}
|
||||
// For ordinary PUT this must finish before calling the mover:
|
||||
// blocking its data Read would already hold the object lock.
|
||||
opts := ObjectOptions{VersionID: migrationTestVersion(oi), UserDefined: map[string]string{migrationTagRevision: recent}}
|
||||
var err error
|
||||
if tags == "" {
|
||||
_, err = z.DeleteObjectTags(ctx, bucket, oi.Name, opts)
|
||||
} else {
|
||||
_, err = z.PutObjectTags(ctx, bucket, oi.Name, tags, opts)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if gr.ObjInfo.UserTags != "state=before" || !maps.Equal(before, gr.ObjInfo.UserDefined) {
|
||||
t.Fatal("test must retain the old source snapshot")
|
||||
}
|
||||
migrationTestStored(t, z, source, oi, tags, recent, true, "payload")
|
||||
if method == "multipart" {
|
||||
gate.unblock()
|
||||
err = <-done
|
||||
} else {
|
||||
err = move(ctx, source, bucket, gr)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !maps.Equal(before, gr.ObjInfo.UserDefined) {
|
||||
t.Error("reconciliation changed the reader's map")
|
||||
}
|
||||
migrationTestStored(t, z, 1-source, oi, tags, recent, true, "payload")
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type migrationTestMetadataGateDisk struct {
|
||||
StorageAPI
|
||||
bucket, object string
|
||||
gate *migrationTestGate
|
||||
}
|
||||
|
||||
func (d migrationTestMetadataGateDisk) UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo, opts UpdateMetadataOpts) error {
|
||||
if volume == d.bucket && path == d.object {
|
||||
if err := d.gate.wait(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return d.StorageAPI.UpdateMetadata(ctx, volume, path, fi, opts)
|
||||
}
|
||||
|
||||
func TestPoolsMigrationTagsDuringCleanup(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
const recent = "2026-09-16T10:00:00Z"
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for source := range 2 {
|
||||
for _, tags := range []string{"state=after", ""} {
|
||||
for _, interrupt := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/source=%d/clear=%t/interrupt=%t", kind, source, tags == "", interrupt), func(t *testing.T) {
|
||||
move := migrationTestMover(t, z, source, kind)
|
||||
oi := migrationTestObject(t, z, bucket, t.Name(), source, false, []string{"payload"}, ObjectOptions{
|
||||
Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "state=before"},
|
||||
})
|
||||
if err := move(t.Context(), source, bucket, migrationTestReader(t, z, source, oi)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
migrationTestStored(t, z, 1-source, oi, "state=before", "", false, "payload")
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
mutate := func() (ObjectInfo, error) {
|
||||
opts := ObjectOptions{VersionID: migrationTestVersion(oi), UserDefined: map[string]string{migrationTagRevision: recent}}
|
||||
if tags == "" {
|
||||
return z.DeleteObjectTags(ctx, bucket, oi.Name, opts)
|
||||
}
|
||||
return z.PutObjectTags(ctx, bucket, oi.Name, tags, opts)
|
||||
}
|
||||
set := z.serverPools[source].getHashedSet(oi.Name)
|
||||
cleanup := func() {
|
||||
// Use the same source prefix cleanup as both outer movers.
|
||||
if _, err := set.DeleteObject(ctx, bucket, oi.Name, ObjectOptions{DeletePrefix: true, DeletePrefixObject: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var updated ObjectInfo
|
||||
if !interrupt {
|
||||
var err error
|
||||
updated, err = mutate()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cleanup()
|
||||
} else {
|
||||
gate := newMigrationTestGate()
|
||||
original := set.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i := range disks {
|
||||
disks[i] = migrationTestMetadataGateDisk{StorageAPI: disks[i], bucket: bucket, object: oi.Name, gate: gate}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
defer func() { set.getDisks = original }()
|
||||
done, finished := make(chan error, 1), make(chan struct{})
|
||||
go func() { defer close(finished); _, err := mutate(); done <- err }()
|
||||
defer func() { gate.unblock(); cancel(); <-finished }()
|
||||
select {
|
||||
case <-gate.entered:
|
||||
case err := <-done:
|
||||
t.Fatalf("metadata update missed the barrier: %v", err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
cleanup()
|
||||
gate.unblock()
|
||||
if err := <-done; err == nil {
|
||||
t.Fatal("update reported success after its source copy was removed")
|
||||
}
|
||||
<-finished
|
||||
set.getDisks = original
|
||||
var err error
|
||||
updated, err = mutate()
|
||||
if err != nil {
|
||||
t.Fatalf("metadata retry did not converge: %v", err)
|
||||
}
|
||||
}
|
||||
if _, err := z.serverPools[source].GetObjectInfo(ctx, bucket, oi.Name, ObjectOptions{VersionID: oi.VersionID}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Fatalf("source cleanup left a copy: %v", err)
|
||||
}
|
||||
migrationTestStored(t, z, 1-source, oi, tags, updated.UserDefined[migrationTagRevision], true, "payload")
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMigrationVersionHistory(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for _, method := range []string{"put", "multipart"} {
|
||||
for _, versioning := range []string{"unversioned", "null", "history"} {
|
||||
t.Run(kind+"/"+method+"/"+versioning, func(t *testing.T) {
|
||||
move := migrationTestMover(t, z, 0, kind)
|
||||
opts := ObjectOptions{
|
||||
Versioned: versioning == "history", VersionSuspended: versioning == "null",
|
||||
MTime: time.Date(2026, 9, 16, 8, 0, 0, 0, time.UTC),
|
||||
UserDefined: map[string]string{xhttp.AmzObjectTagging: "version=old"},
|
||||
}
|
||||
versions := []ObjectInfo{migrationTestObject(t, z, bucket, t.Name(), 0, method == "multipart", []string{"old"}, opts)}
|
||||
if versioning == "history" {
|
||||
opts.MTime = opts.MTime.Add(time.Minute)
|
||||
versions = append(versions, migrationTestObject(t, z, bucket, t.Name(), 0, method == "multipart", []string{"old"}, opts))
|
||||
}
|
||||
var newest ObjectInfo
|
||||
if versioning != "unversioned" {
|
||||
newest = migrationTestObject(t, z, bucket, t.Name(), 0, false, []string{"latest"}, ObjectOptions{
|
||||
Versioned: true, MTime: opts.MTime.Add(time.Minute),
|
||||
UserDefined: map[string]string{xhttp.AmzObjectTagging: "version=new"},
|
||||
})
|
||||
}
|
||||
for _, oi := range versions {
|
||||
if err := move(t.Context(), 0, bucket, migrationTestReader(t, z, 0, oi)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
migrationTestStored(t, z, 1, oi, "version=old", "", false, "old")
|
||||
}
|
||||
if versioning != "unversioned" {
|
||||
migrationTestStored(t, z, 0, newest, "version=new", "", false, "latest")
|
||||
if _, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, t.Name(), ObjectOptions{VersionID: newest.VersionID}); !isErrVersionNotFound(err) {
|
||||
t.Fatalf("moving an addressed version affected the newer version: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMigrationMultipartParts(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
parts := []string{strings.Repeat("a", 5<<20), "tail-with-tags"}
|
||||
body := strings.Join(parts, "")
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for source := range 2 {
|
||||
t.Run(fmt.Sprintf("%s/source=%d", kind, source), func(t *testing.T) {
|
||||
move := migrationTestMover(t, z, source, kind)
|
||||
oi := migrationTestObject(t, z, bucket, t.Name(), source, true, parts, ObjectOptions{
|
||||
Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "multipart=kept"},
|
||||
})
|
||||
if err := move(t.Context(), source, bucket, migrationTestReader(t, z, source, oi)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
migrationTestStored(t, z, 1-source, oi, "multipart=kept", "", false, body)
|
||||
got, err := z.serverPools[1-source].GetObjectInfo(t.Context(), bucket, oi.Name, ObjectOptions{VersionID: oi.VersionID})
|
||||
if err != nil || len(got.Parts) != len(oi.Parts) {
|
||||
t.Fatalf("parts changed: %+v, %v", got.Parts, err)
|
||||
}
|
||||
for i, part := range got.Parts {
|
||||
old := oi.Parts[i]
|
||||
if part.Number != old.Number || part.Size != old.Size || part.ActualSize != old.ActualSize || part.ETag != old.ETag || !bytes.Equal(part.Index, old.Index) {
|
||||
t.Errorf("part %d changed: %+v / %+v", i, part, old)
|
||||
}
|
||||
}
|
||||
start := int64(len(parts[0]) - 3)
|
||||
gr, err := z.serverPools[1-source].GetObjectNInfo(t.Context(), bucket, oi.Name, &HTTPRangeSpec{Start: start, End: start + 7}, nil, ObjectOptions{VersionID: oi.VersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := io.ReadAll(gr)
|
||||
gr.Close()
|
||||
if err != nil || string(data) != body[start:start+8] {
|
||||
t.Fatalf("cross-part range changed: %q, %v", data, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMigrationUnreadablePool(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for _, method := range []string{"put", "multipart"} {
|
||||
t.Run(kind+"/"+method, func(t *testing.T) {
|
||||
move := migrationTestMover(t, z, 0, kind)
|
||||
oi := migrationTestObject(t, z, bucket, t.Name(), 0, method == "multipart", []string{"payload"}, ObjectOptions{
|
||||
Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "keep=source"},
|
||||
})
|
||||
gr := migrationTestReader(t, z, 0, oi)
|
||||
set := z.serverPools[0].getHashedSet(oi.Name)
|
||||
original := set.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i := range disks {
|
||||
disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: oi.Name}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
defer func() { set.getDisks = original }()
|
||||
err := move(t.Context(), 0, bucket, gr)
|
||||
var quorum InsufficientReadQuorum
|
||||
if !errors.As(err, &quorum) {
|
||||
t.Fatalf("unreadable pool must fail migration with read quorum error: %v", err)
|
||||
}
|
||||
set.getDisks = original
|
||||
migrationTestStored(t, z, 0, oi, "keep=source", "", false, "payload")
|
||||
if _, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, oi.Name, ObjectOptions{VersionID: oi.VersionID}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Fatalf("failed migration committed a target version: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -865,7 +865,7 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, poolIdx int, b
|
||||
if oi.isMultipart() {
|
||||
res, err := z.NewMultipartUpload(ctx, bucket, oi.Name, ObjectOptions{
|
||||
VersionID: oi.VersionID,
|
||||
UserDefined: oi.UserDefined,
|
||||
UserDefined: migrationObjectMetadata(oi),
|
||||
NoAuditLog: true,
|
||||
DataMovement: true,
|
||||
SrcPoolIdx: poolIdx,
|
||||
@@ -924,7 +924,7 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, poolIdx int, b
|
||||
DataMovement: true,
|
||||
VersionID: oi.VersionID,
|
||||
MTime: oi.ModTime,
|
||||
UserDefined: oi.UserDefined,
|
||||
UserDefined: migrationObjectMetadata(oi),
|
||||
PreserveETag: oi.ETag, // Preserve original ETag to ensure same metadata.
|
||||
IndexCB: func() []byte {
|
||||
return oi.Parts[0].Index // Preserve part Index to ensure decompression works.
|
||||
|
||||
@@ -1239,9 +1239,12 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
||||
return ObjectInfo{}, z.deletePrefix(ctx, bucket, object)
|
||||
}
|
||||
|
||||
// Reconcile ordinary addressed-version deletes independently of pool movement.
|
||||
// Resolve a physical purge by its addressed version even on a replica
|
||||
// receiver: latest-key routing can select a different pool or miss copies.
|
||||
// Marker creation and specialized movement/scanner operations retain their
|
||||
// existing routing.
|
||||
reconcileVersion := opts.VersionID != "" && !opts.DataMovement &&
|
||||
!opts.ReplicationRequest && !opts.Expiration.Expire && !opts.InclFreeVersions
|
||||
(!opts.ReplicationRequest || opts.isVersionPurge()) && !opts.Expiration.Expire && !opts.InclFreeVersions
|
||||
if !z.SinglePool() && (opts.CheckPrecondFn != nil || reconcileVersion) {
|
||||
return z.deleteObjectReconciled(ctx, bucket, object, opts)
|
||||
}
|
||||
@@ -1254,6 +1257,13 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
||||
if _, ok := err.(InsufficientReadQuorum); ok {
|
||||
return objInfo, InsufficientWriteQuorum{}
|
||||
}
|
||||
// Lookup can return before the set's purge confirmation. Check here,
|
||||
// before any callback can change the request into a metadata update.
|
||||
if opts.isVersionPurge() && (isErrObjectNotFound(err) || isErrVersionNotFound(err)) {
|
||||
if quorumErr := z.checkPurgeAbsent(ctx, bucket, object, opts.VersionID); quorumErr != nil {
|
||||
return objInfo, quorumErr
|
||||
}
|
||||
}
|
||||
// A conditional (If-Match) delete addressing a specific version treats an
|
||||
// absent key as an absent version. getPoolInfoExistingWithOpts strips
|
||||
// VersionID, so a missing key surfaces ObjectNotFound here even for a
|
||||
@@ -1284,6 +1294,11 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
||||
if verr != nil && (!isErrMethodNotAllowed(verr) || !vi.DeleteMarker) {
|
||||
// Genuine read failure for the addressed version: a missing
|
||||
// version -> VersionNotFound (NoSuchVersion), read-quorum loss, etc.
|
||||
if opts.isVersionPurge() && (isErrObjectNotFound(verr) || isErrVersionNotFound(verr)) {
|
||||
if quorumErr := z.checkPurgeAbsent(ctx, bucket, object, opts.VersionID); quorumErr != nil {
|
||||
return objInfo, quorumErr
|
||||
}
|
||||
}
|
||||
return objInfo, verr
|
||||
}
|
||||
// verr is nil for a live version, or MethodNotAllowed with a populated
|
||||
@@ -2160,13 +2175,14 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
|
||||
return toObjectErr(err, bucket)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_, absent := err.(InvalidUploadID)
|
||||
if err == nil || absent {
|
||||
// Unlock cancels the derived lock context before this notification runs.
|
||||
// Keep the request context so successful cancellation reaches peer caches.
|
||||
defer func(ctx context.Context) {
|
||||
if err == nil {
|
||||
z.mpCache.Delete(uploadID)
|
||||
globalNotificationSys.DeleteUploadID(ctx, uploadID)
|
||||
}
|
||||
}()
|
||||
}(ctx)
|
||||
|
||||
lk := z.NewNSLock(bucket, pathJoin(object, uploadID))
|
||||
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||
@@ -2176,13 +2192,18 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
|
||||
legacy := globalAPIConfig.getMultipartListingLegacy()
|
||||
found := false
|
||||
var firstErr error
|
||||
for idx, pool := range z.serverPools {
|
||||
if z.IsSuspended(idx) {
|
||||
continue
|
||||
}
|
||||
poolFound, err := pool.getHashedSet(object).abortMultipartUpload(ctx, bucket, object, uploadID, opts)
|
||||
poolFound, err := pool.getHashedSet(object).abortMultipartUpload(ctx, bucket, object, uploadID, opts, legacy)
|
||||
if legacy && (poolFound || err != nil) {
|
||||
// Match the released first-matching-pool behavior.
|
||||
return err
|
||||
}
|
||||
found = found || poolFound
|
||||
if err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
)
|
||||
|
||||
// isVersionPurge distinguishes physical removal from marker creation and
|
||||
// replication-state updates. Call it again after metadata callbacks, which
|
||||
// can change an ordinary deletion into a pending purge update.
|
||||
func (o ObjectOptions) isVersionPurge() bool {
|
||||
if o.VersionID == "" || o.VersionID == nullVersionID || o.DeleteMarker ||
|
||||
o.DeletePrefix || o.DataMovement || o.InclFreeVersions || o.Expiration.Expire ||
|
||||
o.Transition != (TransitionOptions{}) {
|
||||
return false
|
||||
}
|
||||
id, err := uuid.Parse(o.VersionID)
|
||||
if err != nil || id == uuid.Nil {
|
||||
return false
|
||||
}
|
||||
purge := o.VersionPurgeStatus()
|
||||
if purge == replication.VersionPurgeComplete {
|
||||
return true
|
||||
}
|
||||
if !purge.Empty() || o.DeleteReplication.VersionPurgeStatusInternal != "" {
|
||||
return false
|
||||
}
|
||||
status := o.DeleteMarkerReplicationStatus()
|
||||
return (status.Empty() && o.DeleteReplication.ReplicationStatusInternal == "") ||
|
||||
(status == replication.Replica && o.ReplicationRequest)
|
||||
}
|
||||
|
||||
// confirmVersionAbsent is a read-only proof for an already-missing purge.
|
||||
// Read quorum (or a synthesized NotFound) cannot acknowledge a write. Do not
|
||||
// delete here: unreadable minority copies may carry retention we cannot check.
|
||||
// The caller holds the object lock. This function never heals or enqueues work.
|
||||
func (er erasureObjects) confirmVersionAbsent(ctx context.Context, bucket, object, versionID string) (allAbsent bool, err error) {
|
||||
disks := er.getDisks()
|
||||
_, errs := readAllFileInfo(ctx, disks, "", bucket, object, versionID, false, false)
|
||||
absent := 0
|
||||
for _, err := range errs {
|
||||
if err == errFileNotFound || err == errFileVersionNotFound {
|
||||
absent++
|
||||
}
|
||||
}
|
||||
if absent < len(disks)/2+1 {
|
||||
return false, InsufficientWriteQuorum{}
|
||||
}
|
||||
return absent == len(disks), nil
|
||||
}
|
||||
|
||||
// checkPurgeAbsent schedules recovery explicitly, outside the read-only proof.
|
||||
func (er erasureObjects) checkPurgeAbsent(ctx context.Context, bucket, object, versionID string) error {
|
||||
allAbsent, err := er.confirmVersionAbsent(ctx, bucket, object, versionID)
|
||||
if !allAbsent {
|
||||
er.addPartial(bucket, object, versionID)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (z *erasureServerPools) checkPurgeAbsent(ctx context.Context, bucket, object, versionID string) error {
|
||||
for _, pool := range z.serverPools {
|
||||
if err := pool.getHashedSet(object).checkPurgeAbsent(ctx, bucket, object, versionID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+3
-3
@@ -50,7 +50,7 @@ type apiConfig struct {
|
||||
transitionWorkers int
|
||||
|
||||
staleUploadsExpiry time.Duration
|
||||
multipartListingLegacy bool
|
||||
multipartListingStrict bool
|
||||
staleUploadsCleanupInterval time.Duration
|
||||
deleteCleanupInterval time.Duration
|
||||
enableODirect bool
|
||||
@@ -182,7 +182,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int, legacy bool) {
|
||||
t.transitionWorkers = cfg.TransitionWorkers
|
||||
|
||||
t.staleUploadsExpiry = cfg.StaleUploadsExpiry
|
||||
t.multipartListingLegacy = cfg.MultipartListing == "legacy"
|
||||
t.multipartListingStrict = cfg.MultipartListing == "strict"
|
||||
t.deleteCleanupInterval = cfg.DeleteCleanupInterval
|
||||
t.enableODirect = cfg.EnableODirect
|
||||
t.gzipObjects = cfg.GzipObjects
|
||||
@@ -211,7 +211,7 @@ func (t *apiConfig) odirectEnabled() bool {
|
||||
func (t *apiConfig) getMultipartListingLegacy() bool {
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
return t.multipartListingLegacy
|
||||
return !t.multipartListingStrict
|
||||
}
|
||||
|
||||
func (t *apiConfig) shouldGzipObjects() bool {
|
||||
|
||||
@@ -47,6 +47,7 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
setMultipartListingTestMode(t, false)
|
||||
t.Cleanup(func() {
|
||||
z.Shutdown(t.Context())
|
||||
removeRoots(dirs)
|
||||
@@ -201,15 +202,7 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
|
||||
if !errors.Is(err, errMultipartListingLegacy) {
|
||||
t.Fatalf("legacy strict listing: %v", err)
|
||||
}
|
||||
globalAPIConfig.mu.Lock()
|
||||
oldLegacy := globalAPIConfig.multipartListingLegacy
|
||||
globalAPIConfig.multipartListingLegacy = true
|
||||
globalAPIConfig.mu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
globalAPIConfig.mu.Lock()
|
||||
globalAPIConfig.multipartListingLegacy = oldLegacy
|
||||
globalAPIConfig.mu.Unlock()
|
||||
})
|
||||
setMultipartListingTestMode(t, true)
|
||||
legacy, err := z.ListMultipartUploads(t.Context(), bucket, legacyObject, "", "", "", 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -267,6 +260,7 @@ func TestPaginateMultipartUploads(t *testing.T) {
|
||||
|
||||
func TestListMultipartUploadsGlobalPageAcrossPools(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
setMultipartListingTestMode(t, false)
|
||||
objects := []string{"a/one", "b/two", "c/three", "d/four"}
|
||||
for i, object := range objects {
|
||||
if _, err := z.serverPools[i%len(z.serverPools)].NewMultipartUpload(t.Context(), bucket, object, ObjectOptions{}); err != nil {
|
||||
|
||||
@@ -0,0 +1,433 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
madmin "github.com/minio/madmin-go/v3"
|
||||
)
|
||||
|
||||
// Hold nine completed disk walks while a PUT overwrites one name. The other
|
||||
// seven walks start after that PUT completes. Both generations come from real
|
||||
// disk walkers and valid PUT metadata; only the reader schedule is controlled.
|
||||
type nullQuorumWalkSchedule struct {
|
||||
bucket string
|
||||
oldReady chan struct{}
|
||||
resume chan struct{}
|
||||
mu sync.Mutex
|
||||
snapshots [16][]byte
|
||||
}
|
||||
|
||||
type nullQuorumWalkDisk struct {
|
||||
StorageAPI
|
||||
schedule *nullQuorumWalkSchedule
|
||||
index int
|
||||
}
|
||||
|
||||
func (d *nullQuorumWalkDisk) WalkDir(ctx context.Context, opts WalkDirOptions, out io.Writer) error {
|
||||
if opts.Bucket != d.schedule.bucket {
|
||||
return d.StorageAPI.WalkDir(ctx, opts, out)
|
||||
}
|
||||
var stream bytes.Buffer
|
||||
if d.index < 9 {
|
||||
if err := d.StorageAPI.WalkDir(ctx, opts, &stream); err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case d.schedule.oldReady <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-d.schedule.resume:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
if d.index >= 9 {
|
||||
if err := d.StorageAPI.WalkDir(ctx, opts, &stream); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
d.schedule.mu.Lock()
|
||||
d.schedule.snapshots[d.index] = bytes.Clone(stream.Bytes())
|
||||
d.schedule.mu.Unlock()
|
||||
_, err := out.Write(stream.Bytes())
|
||||
return err
|
||||
}
|
||||
|
||||
func nullQuorumBackend(t *testing.T) (*erasureServerPools, string, http.Handler) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
obj, dirs, err := prepareErasure16(ctx)
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
previous := newObjectLayerFn()
|
||||
setObjectLayer(z)
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
z.Shutdown(context.Background())
|
||||
removeRoots(dirs)
|
||||
setObjectLayer(previous)
|
||||
})
|
||||
bucket, router, err := initAPIHandlerTest(ctx, z, nil, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return z, bucket, router
|
||||
}
|
||||
|
||||
func nullQuorumRequest(t *testing.T, router http.Handler, method, target string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
req, err := newTestSignedRequestV4(method, target, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req.WithContext(t.Context()))
|
||||
return rec
|
||||
}
|
||||
|
||||
func TestListObjectsSingleNullQuorumHTTP(t *testing.T) {
|
||||
z, bucket, router := nullQuorumBackend(t)
|
||||
oldBody := bytes.Repeat([]byte{'a'}, 8192)
|
||||
newBody := bytes.Repeat([]byte{'b'}, 8192)
|
||||
oldTime := time.Now().UTC().Add(-time.Hour)
|
||||
newTime := oldTime.Add(time.Minute)
|
||||
var want []string
|
||||
for i := range 16 {
|
||||
name := fmt.Sprintf("fixed/%02d", i)
|
||||
want = append(want, name)
|
||||
_, err := z.PutObject(t.Context(), bucket, name, mustGetPutObjReader(t, bytes.NewReader(oldBody), int64(len(oldBody)), "", ""), ObjectOptions{MTime: oldTime})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
const overwritten = "fixed/10"
|
||||
schedule := &nullQuorumWalkSchedule{bucket: bucket, oldReady: make(chan struct{}, 16), resume: make(chan struct{})}
|
||||
set := z.serverPools[0].sets[0]
|
||||
disks := set.getDisks()
|
||||
wrapped := make([]StorageAPI, len(disks))
|
||||
for i, disk := range disks {
|
||||
wrapped[i] = &nullQuorumWalkDisk{StorageAPI: disk, schedule: schedule, index: i}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return wrapped }
|
||||
if globalAPIConfig.getListQuorum() != "strict" {
|
||||
t.Fatal("test requires strict listing across all sixteen disks")
|
||||
}
|
||||
writeDone := make(chan error, 1)
|
||||
putReader := mustGetPutObjReader(t, bytes.NewReader(newBody), int64(len(newBody)), "", "")
|
||||
go func() {
|
||||
defer close(schedule.resume)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
for range 9 {
|
||||
select {
|
||||
case <-schedule.oldReady:
|
||||
case <-ctx.Done():
|
||||
writeDone <- ctx.Err()
|
||||
return
|
||||
}
|
||||
}
|
||||
_, err := z.PutObject(ctx, bucket, overwritten, putReader, ObjectOptions{MTime: newTime})
|
||||
writeDone <- err
|
||||
}()
|
||||
rec := nullQuorumRequest(t, router, http.MethodGet, getListObjectsV2URL("", bucket, "fixed/", "1000", "", "", ""))
|
||||
if err := <-writeDone; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var list ListObjectsV2Response
|
||||
if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &list) != nil {
|
||||
t.Fatalf("LIST: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var got []string
|
||||
for _, object := range list.Contents {
|
||||
got = append(got, object.Key)
|
||||
}
|
||||
t.Logf("LIST HTTP=%d KeyCount=%d IsTruncated=%v keys=%v", rec.Code, list.KeyCount, list.IsTruncated, got)
|
||||
if list.KeyCount != 16 || list.IsTruncated || !slices.Equal(got, want) {
|
||||
t.Errorf("LIST omitted a name during overwrite: got %v, want %v", got, want)
|
||||
}
|
||||
|
||||
// Verify the actual reader inputs, including shape and EC, instead of
|
||||
// assuming the scheduling barrier produced the intended resolver case.
|
||||
schedule.mu.Lock()
|
||||
snapshots := schedule.snapshots
|
||||
schedule.mu.Unlock()
|
||||
for i, snapshot := range snapshots {
|
||||
reader := newMetacacheReader(bytes.NewReader(snapshot))
|
||||
var found bool
|
||||
for {
|
||||
entry, err := reader.next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if entry.name != overwritten {
|
||||
continue
|
||||
}
|
||||
xl, err := entry.xlmeta()
|
||||
if err != nil || len(xl.versions) != 1 {
|
||||
t.Fatalf("disk %d: invalid input shape: %v", i, err)
|
||||
}
|
||||
header := xl.versions[0].header
|
||||
wantTime := oldTime
|
||||
if i >= 9 {
|
||||
wantTime = newTime
|
||||
}
|
||||
if header.ModTime != wantTime.UnixNano() || header.VersionID != [16]byte{} || header.Type != ObjectType || header.FreeVersion() {
|
||||
t.Fatalf("disk %d: unexpected header %v", i, header)
|
||||
}
|
||||
t.Logf("disk=%d versions=1 header=%v", i, header)
|
||||
found = true
|
||||
}
|
||||
reader.Close()
|
||||
if !found {
|
||||
t.Fatalf("disk %d did not emit the overwritten name", i)
|
||||
}
|
||||
}
|
||||
get := nullQuorumRequest(t, router, http.MethodGet, getGetObjectURL("", bucket, overwritten))
|
||||
head := nullQuorumRequest(t, router, http.MethodHead, getGetObjectURL("", bucket, overwritten))
|
||||
t.Logf("completed PUT readback: GET=%d bytes=%d HEAD=%d length=%s", get.Code, get.Body.Len(), head.Code, head.Header().Get("Content-Length"))
|
||||
if get.Code != http.StatusOK || !bytes.Equal(get.Body.Bytes(), newBody) || head.Code != http.StatusOK || head.Header().Get("Content-Length") != "8192" {
|
||||
t.Fatalf("completed PUT was not readable: GET=%d HEAD=%d", get.Code, head.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingleNullQuorumReadAndConsumers(t *testing.T) {
|
||||
z, bucket, router := nullQuorumBackend(t)
|
||||
set := z.serverPools[0].sets[0]
|
||||
disks := set.getDisks()
|
||||
const object = "object"
|
||||
oldBody, newBody := bytes.Repeat([]byte{'a'}, 8192), bytes.Repeat([]byte{'b'}, 8192)
|
||||
oldTime := time.Now().UTC().Add(-time.Hour)
|
||||
put := func(body []byte, modTime time.Time) {
|
||||
t.Helper()
|
||||
_, err := z.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{MTime: modTime})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
put(oldBody, oldTime)
|
||||
oldMeta := make([][]byte, len(disks))
|
||||
for i, disk := range disks {
|
||||
var err error
|
||||
oldMeta[i], err = disk.ReadAll(t.Context(), bucket, object+"/"+xlStorageFormatFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
put(newBody, oldTime.Add(time.Minute))
|
||||
newMinorityMeta := mustReadNullQuorumMeta(t, disks[14], bucket, object)
|
||||
// Model an incomplete overwrite using real inline shards: fifteen old
|
||||
// copies retain a read quorum; the final disk contains the newer minority.
|
||||
for i := range 15 {
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
checkRead := func(body []byte, modTime time.Time) {
|
||||
t.Helper()
|
||||
get := nullQuorumRequest(t, router, http.MethodGet, getGetObjectURL("", bucket, object))
|
||||
head := nullQuorumRequest(t, router, http.MethodHead, getGetObjectURL("", bucket, object))
|
||||
if get.Code != 200 || !bytes.Equal(get.Body.Bytes(), body) || head.Code != 200 || head.Header().Get("Content-Length") != "8192" {
|
||||
t.Fatalf("GET/HEAD lost readable quorum: GET=%d HEAD=%d body=%q", get.Code, head.Code, get.Body.String())
|
||||
}
|
||||
oi, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||
if err != nil || !oi.ModTime.Equal(modTime) {
|
||||
t.Fatalf("wrong readable generation: %v, %v", oi.ModTime, err)
|
||||
}
|
||||
}
|
||||
checkRead(oldBody, oldTime)
|
||||
// Reading must not reinterpret the minority as absent or delete it.
|
||||
minority, err := disks[15].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||
if err != nil || !minority.ModTime.Equal(oldTime.Add(time.Minute)) {
|
||||
t.Fatalf("read mutated the minority: %+v, %v", minority, err)
|
||||
}
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
var names []string
|
||||
consume := func(entry metaCacheEntry) {
|
||||
versions, err := entry.fileInfoVersions(bucket)
|
||||
if err != nil || len(versions.Versions) != 1 || !versions.Versions[0].ModTime.Equal(oldTime) {
|
||||
t.Errorf("invalid migration metadata: %+v, %v", versions, err)
|
||||
return
|
||||
}
|
||||
// Migration reopens the selected name/version on all drives;
|
||||
// it must not treat the selected listing metadata as disk agreement.
|
||||
reader, err := set.GetObjectNInfo(t.Context(), bucket, entry.name, nil, nil, ObjectOptions{VersionID: nullVersionID, NoLock: true, NoDecryption: true})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
body, err := io.ReadAll(reader)
|
||||
reader.Close()
|
||||
if err != nil || !bytes.Equal(body, oldBody) {
|
||||
t.Errorf("migration could not read selected object data: %v", err)
|
||||
}
|
||||
names = append(names, entry.name)
|
||||
}
|
||||
var err error
|
||||
if kind == "rebalance" {
|
||||
err = set.listObjectsToRebalance(t.Context(), bucket, consume)
|
||||
} else {
|
||||
err = set.listObjectsToDecommission(t.Context(), decomBucketInfo{Name: bucket}, consume)
|
||||
}
|
||||
if err != nil || !slices.Equal(names, []string{object}) {
|
||||
t.Fatalf("migration listing dropped the quorum: %v, %v", names, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, latestOnly := range []bool{false, true} {
|
||||
results := make(chan itemOrErr[ObjectInfo], 16)
|
||||
if err := z.Walk(t.Context(), bucket, "", results, WalkOptions{AskDisks: "strict", LatestOnly: latestOnly}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var names []string
|
||||
for result := range results {
|
||||
if result.Err != nil || !result.Item.ModTime.Equal(oldTime) {
|
||||
t.Fatalf("Walk returned invalid metadata: %+v", result)
|
||||
}
|
||||
names = append(names, result.Item.Name)
|
||||
}
|
||||
if !slices.Equal(names, []string{object}) {
|
||||
t.Fatalf("Walk dropped the quorum: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
// Exercise the scanner's actual abandoned-child path. Make the scan drive
|
||||
// miss the object, while keeping an older quorum and a newer minority on
|
||||
// the remaining disks. Its queued heal must read all disks and repair both.
|
||||
if err := disks[14].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, newMinorityMeta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(disks[15].Endpoint().Path, bucket, object)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
scanNullQuorumAbandoned(t, z, bucket, object, oldTime)
|
||||
for i, disk := range disks {
|
||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{Healing: true})
|
||||
if err != nil || !fi.ModTime.Equal(oldTime) {
|
||||
t.Fatalf("scanner heal did not reconcile disk %d: %v, %v", i, fi.ModTime, err)
|
||||
}
|
||||
}
|
||||
checkRead(oldBody, oldTime)
|
||||
put(newBody, oldTime.Add(2*time.Minute))
|
||||
checkRead(newBody, oldTime.Add(2*time.Minute))
|
||||
}
|
||||
|
||||
func mustReadNullQuorumMeta(t *testing.T, disk StorageAPI, bucket, object string) []byte {
|
||||
t.Helper()
|
||||
data, err := disk.ReadAll(t.Context(), bucket, object+"/"+xlStorageFormatFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func scanNullQuorumAbandoned(t *testing.T, z *erasureServerPools, bucket, object string, oldTime time.Time) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
disks := z.serverPools[0].sets[0].getDisks()
|
||||
seq := newBgHealSequence()
|
||||
defer seq.cancelCtx()
|
||||
previousState, previousRoutine := globalBackgroundHealState, globalBackgroundHealRoutine
|
||||
globalBackgroundHealState = &allHealState{healSeqMap: map[string]*healSequence{"test": seq}}
|
||||
routine := &healRoutine{tasks: make(chan healTask)}
|
||||
globalBackgroundHealRoutine = routine
|
||||
defer func() { globalBackgroundHealState, globalBackgroundHealRoutine = previousState, previousRoutine }()
|
||||
workerDone := make(chan struct{})
|
||||
var healed []string
|
||||
go func() {
|
||||
defer close(workerDone)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case task := <-routine.tasks:
|
||||
var result madmin.HealResultItem
|
||||
var err error
|
||||
if task.object == "" {
|
||||
result, err = z.HealBucket(ctx, task.bucket, task.opts)
|
||||
} else {
|
||||
healed = append(healed, task.object+"/"+task.versionID)
|
||||
result, err = z.HealObject(ctx, task.bucket, task.object, task.versionID, task.opts)
|
||||
}
|
||||
task.respCh <- healResult{result: result, err: err}
|
||||
}
|
||||
}
|
||||
}()
|
||||
cache := dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}
|
||||
cache.replace(bucket, "", dataUsageEntry{})
|
||||
cache.replace(bucket+"/"+object, bucket, dataUsageEntry{Size: 8192, Objects: 1, Versions: 1})
|
||||
scanner := folderScanner{
|
||||
root: disks[15].Endpoint().Path, oldCache: cache,
|
||||
newCache: dataUsageCache{Info: cache.Info}, updateCache: dataUsageCache{Info: cache.Info},
|
||||
disks: disks, disksQuorum: 8, healObjectSelect: 1,
|
||||
weSleep: func() bool { return false }, shouldHeal: func() bool { return true }, updateCurrentPath: func(string) {},
|
||||
getSize: func(item scannerItem) (sizeSummary, error) {
|
||||
if filepath.Base(item.Path) != xlStorageFormatFile {
|
||||
return sizeSummary{}, errSkipFile
|
||||
}
|
||||
data, err := os.ReadFile(item.Path)
|
||||
if err != nil {
|
||||
return sizeSummary{}, err
|
||||
}
|
||||
var xl xlMetaV2
|
||||
if err := xl.Load(data); err != nil {
|
||||
return sizeSummary{}, err
|
||||
}
|
||||
fi, err := xl.ToFileInfo(bucket, object, "", false, false)
|
||||
if err != nil || !fi.ModTime.Equal(oldTime) {
|
||||
return sizeSummary{}, fmt.Errorf("scanner re-read wrong generation: %v, %v", fi.ModTime, err)
|
||||
}
|
||||
return sizeSummary{totalSize: fi.Size, versions: 1}, nil
|
||||
},
|
||||
}
|
||||
var usage dataUsageEntry
|
||||
err := scanner.scanFolder(ctx, cachedFolder{name: bucket, objectHealProbDiv: 1}, &usage)
|
||||
cancel()
|
||||
<-workerDone
|
||||
if err != nil || len(healed) != 1 || healed[0] != object+"/"+nullVersionID && healed[0] != object+"/" {
|
||||
t.Fatalf("scanner did not queue a name-based heal: %v, %v", healed, err)
|
||||
}
|
||||
flat := scanner.newCache.sizeRecursive(bucket)
|
||||
if flat == nil || flat.Objects != 1 || flat.Size != 8192 {
|
||||
t.Fatalf("scanner lost the healed object from usage: %+v", flat)
|
||||
}
|
||||
t.Logf("scanner queued %v; healed old quorum, repaired minority/missing copies, and retained one 8192-byte object", healed)
|
||||
}
|
||||
@@ -69,6 +69,8 @@ func loadCPUMetrics(ctx context.Context, m MetricValues, c *metricsCache) error
|
||||
|
||||
// metrics-resource.go runs a job to collect resource metrics including their Avg values and
|
||||
// stores them in resourceMetricsMap. We can use it to get the Avg values of CPU idle and IOWait.
|
||||
resourceMetricsMapMu.RLock()
|
||||
defer resourceMetricsMapMu.RUnlock()
|
||||
cpuResourceMetrics, found := resourceMetricsMap[cpuSubsystem]
|
||||
if found {
|
||||
if cpuIdleMetric, ok := cpuResourceMetrics[getResourceKey(cpuIdle, nil)]; ok {
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
// Copyright (c) 2026 Ruohang Feng
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/cachevalue"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
cpustats "github.com/shirou/gopsutil/v3/cpu"
|
||||
"github.com/shirou/gopsutil/v3/load"
|
||||
)
|
||||
|
||||
// These tests replace global resource metrics and must not run in parallel.
|
||||
// Concurrent writers must finish before the fixture is restored.
|
||||
func setCPUResourceMetricsForTest(t *testing.T, value map[MetricSubsystem]ResourceMetrics) {
|
||||
t.Helper()
|
||||
resourceMetricsMapMu.Lock()
|
||||
saved := resourceMetricsMap
|
||||
resourceMetricsMap = value
|
||||
resourceMetricsMapMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
resourceMetricsMapMu.Lock()
|
||||
resourceMetricsMap = saved
|
||||
resourceMetricsMapMu.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
func newCPUMetricsTestRegistry() *prometheus.Registry {
|
||||
c := &metricsCache{cpuMetrics: cachevalue.NewFromFunc(time.Hour, cachevalue.Opts{},
|
||||
func(context.Context) (madmin.CPUMetrics, error) {
|
||||
return madmin.CPUMetrics{
|
||||
CPUCount: 4,
|
||||
LoadStat: &load.AvgStat{Load1: 2},
|
||||
TimesStat: &cpustats.TimesStat{
|
||||
User: 10, System: 20, Idle: 60, Iowait: 5, Nice: 3, Steal: 2,
|
||||
},
|
||||
}, nil
|
||||
})}
|
||||
_, _ = c.cpuMetrics.Get() // Warm the cache: the cache does not protect the resource map.
|
||||
g := NewMetricsGroup(systemCPUCollectorPath, []MetricDescriptor{
|
||||
sysCPUAvgIdleMD, sysCPUAvgIOWaitMD, sysCPULoadMD, sysCPULoadPercMD,
|
||||
sysCPUNiceMD, sysCPUStealMD, sysCPUSystemMD, sysCPUUserMD,
|
||||
}, loadCPUMetrics)
|
||||
g.SetCache(c)
|
||||
r := prometheus.NewPedanticRegistry()
|
||||
r.MustRegister(g)
|
||||
return r
|
||||
}
|
||||
|
||||
func TestLoadCPUMetricsValues(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
data map[MetricSubsystem]ResourceMetrics
|
||||
want map[string]float64
|
||||
}{
|
||||
{name: "nil-map"},
|
||||
{name: "empty-map", data: map[MetricSubsystem]ResourceMetrics{}},
|
||||
{name: "no-cpu", data: map[MetricSubsystem]ResourceMetrics{memSubsystem: {}}},
|
||||
{name: "nil-cpu", data: map[MetricSubsystem]ResourceMetrics{cpuSubsystem: nil}},
|
||||
{name: "empty-cpu", data: map[MetricSubsystem]ResourceMetrics{cpuSubsystem: {}}},
|
||||
{name: "idle-only", data: map[MetricSubsystem]ResourceMetrics{cpuSubsystem: {
|
||||
getResourceKey(cpuIdle, nil): {Avg: 87.654},
|
||||
}}, want: map[string]float64{"minio_system_cpu_avg_idle": 87.65}},
|
||||
{name: "iowait-only", data: map[MetricSubsystem]ResourceMetrics{cpuSubsystem: {
|
||||
getResourceKey(cpuIOWait, nil): {Avg: 1.236},
|
||||
}}, want: map[string]float64{"minio_system_cpu_avg_iowait": 1.24}},
|
||||
{name: "both-rounded", data: map[MetricSubsystem]ResourceMetrics{cpuSubsystem: {
|
||||
getResourceKey(cpuIdle, nil): {Avg: 87.654},
|
||||
getResourceKey(cpuIOWait, nil): {Avg: 1.236},
|
||||
}}, want: map[string]float64{"minio_system_cpu_avg_idle": 87.65, "minio_system_cpu_avg_iowait": 1.24}},
|
||||
{name: "zero-keeps-existing-omission", data: map[MetricSubsystem]ResourceMetrics{cpuSubsystem: {
|
||||
getResourceKey(cpuIdle, nil): {Avg: 0},
|
||||
getResourceKey(cpuIOWait, nil): {Avg: 0},
|
||||
}}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
setCPUResourceMetricsForTest(t, tc.data)
|
||||
families, err := newCPUMetricsTestRegistry().Gather()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]float64{
|
||||
"minio_system_cpu_load": 2, "minio_system_cpu_load_perc": 50,
|
||||
"minio_system_cpu_nice": 3, "minio_system_cpu_steal": 2,
|
||||
"minio_system_cpu_system": 20, "minio_system_cpu_user": 10,
|
||||
}
|
||||
for k, v := range tc.want {
|
||||
want[k] = v
|
||||
}
|
||||
if len(families) != len(want) {
|
||||
t.Fatalf("got %d families, want %d", len(families), len(want))
|
||||
}
|
||||
for _, family := range families {
|
||||
v, ok := want[family.GetName()]
|
||||
if !ok || family.GetType() != dto.MetricType_GAUGE || len(family.Metric) != 1 || family.Metric[0].GetGauge().GetValue() != v {
|
||||
t.Errorf("unexpected family: %v (want %v)", family, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadCPUMetricsConcurrentUpdate(t *testing.T) {
|
||||
for _, subsystem := range []MetricSubsystem{cpuSubsystem, memSubsystem} {
|
||||
for _, readers := range []int{1, 4} {
|
||||
t.Run(fmt.Sprintf("writer-%s/readers-%d", subsystem, readers), func(t *testing.T) {
|
||||
setCPUResourceMetricsForTest(t, map[MetricSubsystem]ResourceMetrics{})
|
||||
updateResourceMetrics(cpuSubsystem, cpuIdle, 80, nil, false)
|
||||
updateResourceMetrics(cpuSubsystem, cpuIOWait, 5, nil, false)
|
||||
r := newCPUMetricsTestRegistry()
|
||||
stop := make(chan struct{})
|
||||
done := make(chan struct{})
|
||||
ready := make(chan struct{})
|
||||
var updates atomic.Uint64
|
||||
go func() {
|
||||
defer close(done)
|
||||
if subsystem == cpuSubsystem {
|
||||
updateResourceMetrics(subsystem, cpuIdle, 80, nil, false)
|
||||
} else {
|
||||
updateResourceMetrics(subsystem, memUsed, 1024, nil, false)
|
||||
}
|
||||
close(ready)
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
default:
|
||||
}
|
||||
if subsystem == cpuSubsystem {
|
||||
updateResourceMetrics(subsystem, cpuIdle, 80, nil, false)
|
||||
updateResourceMetrics(subsystem, cpuIOWait, 5, nil, false)
|
||||
} else {
|
||||
updateResourceMetrics(subsystem, memUsed, 1024, nil, false)
|
||||
}
|
||||
updates.Add(1)
|
||||
runtime.Gosched()
|
||||
}
|
||||
}()
|
||||
<-ready
|
||||
defer func() { close(stop); <-done }()
|
||||
var wg sync.WaitGroup
|
||||
for range readers {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for range 1000 {
|
||||
families, err := r.Gather()
|
||||
if err != nil || len(families) != 8 {
|
||||
t.Errorf("Gather: families=%d, error=%v", len(families), err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if updates.Load() == 0 {
|
||||
t.Fatal("writer made no progress")
|
||||
}
|
||||
t.Logf("completed %d gathers with %d concurrent update iterations", readers*1000, updates.Load())
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -82,17 +82,26 @@ type markerRecoveryTarget struct {
|
||||
func replicationTestCapacity(obj ObjectLayer) func() {
|
||||
var restore []func()
|
||||
for _, pool := range obj.(*erasureServerPools).serverPools {
|
||||
pool.erasureDisksMu.Lock()
|
||||
for _, set := range pool.sets {
|
||||
original := set.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
original := pool.erasureDisks[set.setIndex]
|
||||
disks := append([]StorageAPI(nil), original...)
|
||||
for i, disk := range disks {
|
||||
if disk != nil {
|
||||
disks[i] = tagTestCapacityDisk{StorageAPI: disk}
|
||||
}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
restore = append(restore, func() { set.getDisks = original })
|
||||
// GetDisks copies this list under the same mutex. Keep its function
|
||||
// stable while background IAM scans use the fixture, both when
|
||||
// installing the adapter and when restoring the original disks.
|
||||
pool.erasureDisks[set.setIndex] = disks
|
||||
restore = append(restore, func() {
|
||||
pool.erasureDisksMu.Lock()
|
||||
pool.erasureDisks[set.setIndex] = original
|
||||
pool.erasureDisksMu.Unlock()
|
||||
})
|
||||
}
|
||||
pool.erasureDisksMu.Unlock()
|
||||
}
|
||||
return func() {
|
||||
for _, fn := range restore {
|
||||
@@ -101,6 +110,34 @@ func replicationTestCapacity(obj ObjectLayer) func() {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplicationCapacityConcurrentIAM(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
if _, _, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
iam := globalIAMSys
|
||||
started, finished := make(chan struct{}), make(chan error, 1)
|
||||
go func() {
|
||||
close(started)
|
||||
for range 20 {
|
||||
if err := iam.Load(ctx, false); err != nil {
|
||||
finished <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
finished <- nil
|
||||
}()
|
||||
<-started
|
||||
for range 5000 {
|
||||
replicationTestCapacity(z)()
|
||||
}
|
||||
if err := <-finished; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func testReplicationMRFMarkerRecovery(t *testing.T, obj ObjectLayer, backend, bucket string, router http.Handler, creds auth.Credentials, tc markerRecoveryCase) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/once"
|
||||
)
|
||||
|
||||
type staleCreationCase struct {
|
||||
name string
|
||||
purged, pendingPurge, unreadable bool
|
||||
}
|
||||
|
||||
// A queued delete-marker creation must be checked against the source marker
|
||||
// under the replication lock before it is sent. Between queueing (DELETE
|
||||
// handler, GET/HEAD/LIST heal, scanner, MRF) and sending, a user purge of the
|
||||
// same marker can complete and reach the targets; the stale creation would
|
||||
// then recreate the marker there.
|
||||
func TestReplicationDeleteMarkerCreationRevalidated(t *testing.T) {
|
||||
for _, tc := range []staleCreationCase{
|
||||
{name: "current"},
|
||||
{name: "purged", purged: true},
|
||||
{name: "purge-pending", pendingPurge: true},
|
||||
{name: "unreadable", unreadable: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, endpoints: []string{"DeleteObject"}, objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, creds auth.Credentials, t *testing.T) {
|
||||
testReplicationDeleteMarkerCreationRevalidated(t, obj, backend, bucket, router, creds, tc)
|
||||
}})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type staleCreationTarget struct {
|
||||
arn, bucket string
|
||||
creations atomic.Int32
|
||||
purges atomic.Int32
|
||||
}
|
||||
|
||||
func testReplicationDeleteMarkerCreationRevalidated(t *testing.T, obj ObjectLayer, backend, bucket string, router http.Handler, creds auth.Credentials, tc staleCreationCase) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
defer replicationTestCapacity(obj)()
|
||||
stats := NewReplicationStats(ctx, nil)
|
||||
oldStats := globalReplicationStats.Swap(stats)
|
||||
defer globalReplicationStats.Store(oldStats)
|
||||
oldPool := globalReplicationPool
|
||||
defer func() { globalReplicationPool = oldPool }()
|
||||
const name = "marker"
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := obj.PutObject(ctx, bucket, name, mustGetPutObjReader(t, bytes.NewReader([]byte("data")), 4, "", ""), ObjectOptions{Versioned: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target := &staleCreationTarget{arn: "arn:minio:replication::" + mustGetUUID() + ":bucket", bucket: getRandomBucketName()}
|
||||
if err := obj.MakeBucket(ctx, target.bucket, MakeBucketOptions{VersioningEnabled: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := obj.PutObject(ctx, target.bucket, name, mustGetPutObjReader(t, bytes.NewReader([]byte("data")), 4, "", ""), ObjectOptions{Versioned: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The remote behaves like a real receiver: a marker lookup and a
|
||||
// replicated DELETE applied to the target bucket on the same fixture.
|
||||
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
opts := ObjectOptions{VersionID: r.URL.Query().Get("versionId"), Versioned: true}
|
||||
switch r.Method {
|
||||
case http.MethodHead:
|
||||
oi, err := obj.GetObjectInfo(r.Context(), target.bucket, name, opts)
|
||||
if oi.DeleteMarker {
|
||||
w.Header().Set(xhttp.AmzDeleteMarker, "true")
|
||||
w.Header().Set(xhttp.AmzVersionID, oi.VersionID)
|
||||
}
|
||||
if err != nil {
|
||||
writeErrorResponseHeadersOnly(w, toAPIError(r.Context(), err))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case http.MethodDelete:
|
||||
opts.DeleteMarker = r.Header.Get(xhttp.MinIOSourceDeleteMarker) == "true"
|
||||
if opts.DeleteMarker {
|
||||
target.creations.Add(1)
|
||||
} else {
|
||||
target.purges.Add(1)
|
||||
}
|
||||
opts.ReplicationRequest = true
|
||||
opts.SetReplicaStatus(replication.Replica)
|
||||
if _, err := obj.DeleteObject(r.Context(), target.bucket, name, opts); err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
default:
|
||||
t.Errorf("unexpected remote method %s", r.Method)
|
||||
}
|
||||
}))
|
||||
defer remote.Close()
|
||||
client, err := minio.New(strings.TrimPrefix(remote.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
globalBucketTargetSys.Lock()
|
||||
globalBucketTargetSys.arnRemotesMap[target.arn] = arnTarget{Client: &TargetClient{Client: client, ARN: target.arn, Bucket: target.bucket}, lastRefresh: UTCNow()}
|
||||
globalBucketTargetSys.targetsMap[bucket] = append(globalBucketTargetSys.targetsMap[bucket], madmin.BucketTarget{Arn: target.arn, TargetBucket: target.bucket})
|
||||
globalBucketTargetSys.Unlock()
|
||||
globalBucketTargetSys.hMutex.Lock()
|
||||
globalBucketTargetSys.hc[client.EndpointURL().Host] = epHealth{Online: true}
|
||||
globalBucketTargetSys.hMutex.Unlock()
|
||||
rule := configs[0].Rules[0]
|
||||
rule.Destination = replication.Destination{ARN: target.arn, Bucket: target.bucket}
|
||||
cfg := replication.Config{RoleArn: target.arn, Rules: []replication.Rule{rule}}
|
||||
meta, err := globalBucketMetadataSys.Get(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta.replicationConfig = &cfg
|
||||
globalBucketMetadataSys.Set(bucket, meta)
|
||||
p := &ReplicationPool{ctx: ctx, objLayer: obj, workers: []chan ReplicationWorkerOperation{make(chan ReplicationWorkerOperation, 8)}, stats: stats, mrfSaveCh: make(chan MRFReplicateEntry, 8)}
|
||||
globalReplicationPool = once.NewSingleton[ReplicationPool]()
|
||||
globalReplicationPool.Set(p)
|
||||
|
||||
// The real DELETE handler creates the pending marker and queues the
|
||||
// creation task that a worker would later run.
|
||||
req, err := newTestSignedRequestV4(http.MethodDelete, "/"+bucket+"/"+name, 0, nil, creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusNoContent {
|
||||
t.Fatalf("source DELETE status %d: %s", w.Code, w.Body)
|
||||
}
|
||||
var creation DeletedObjectReplicationInfo
|
||||
select {
|
||||
case op := <-p.workers[0]:
|
||||
creation = op.(DeletedObjectReplicationInfo)
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("handler queued no creation task")
|
||||
}
|
||||
version := creation.DeleteMarkerVersionID
|
||||
if version == "" || creation.VersionID != "" || !creation.DeleteMarker {
|
||||
t.Fatalf("handler queued a non-creation task: %+v", creation)
|
||||
}
|
||||
before, err := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: version, Versioned: true})
|
||||
if !isErrMethodNotAllowed(err) || !before.DeleteMarker || before.ReplicationStatus != replication.Pending {
|
||||
t.Fatalf("source marker not pending: %+v %v", before, err)
|
||||
}
|
||||
|
||||
// Meanwhile the user purges the marker through another path. The purge is
|
||||
// either complete (version gone) or still pending on the source.
|
||||
switch {
|
||||
case tc.purged:
|
||||
if _, err := obj.DeleteObject(ctx, bucket, name, ObjectOptions{VersionID: version, Versioned: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: version, Versioned: true}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Fatalf("source purge left the marker: %v", err)
|
||||
}
|
||||
case tc.pendingPurge:
|
||||
opts := ObjectOptions{VersionID: version, Versioned: true, DeleteReplication: ReplicationState{
|
||||
ReplicateDecisionStr: creation.ReplicationState.ReplicateDecisionStr,
|
||||
VersionPurgeStatusInternal: target.arn + "=PENDING;",
|
||||
PurgeTargets: map[string]VersionPurgeStatusType{target.arn: replication.VersionPurgePending},
|
||||
}}
|
||||
if _, err := obj.DeleteObject(ctx, bucket, name, opts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oi, err := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: version, Versioned: true})
|
||||
if !isErrMethodNotAllowed(err) || !oi.DeleteMarker || oi.VersionPurgeStatus != replication.VersionPurgePending {
|
||||
t.Fatalf("source marker not pending purge: %+v %v", oi, err)
|
||||
}
|
||||
}
|
||||
source := obj
|
||||
if tc.unreadable {
|
||||
source = staleLookupLayer{ObjectLayer: obj}
|
||||
}
|
||||
pending, _ := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: version, Versioned: true})
|
||||
|
||||
result := replicateDelete(ctx, creation, source)
|
||||
|
||||
after, aerr := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: version, Versioned: true})
|
||||
_, terr := obj.GetObjectInfo(ctx, target.bucket, name, ObjectOptions{VersionID: version, Versioned: true})
|
||||
targetHasMarker := isErrMethodNotAllowed(terr)
|
||||
switch {
|
||||
case tc.purged, tc.pendingPurge, tc.unreadable:
|
||||
if len(result.Targets) != 0 || target.creations.Load() != 0 || target.purges.Load() != 0 {
|
||||
t.Fatalf("%s: stale creation was sent: result=%+v creations=%d purges=%d", tc.name, result, target.creations.Load(), target.purges.Load())
|
||||
}
|
||||
if targetHasMarker {
|
||||
t.Fatalf("%s: target marker recreated from the stale creation", tc.name)
|
||||
}
|
||||
if tc.purged {
|
||||
if !isErrVersionNotFound(aerr) && !isErrObjectNotFound(aerr) {
|
||||
t.Fatalf("stale creation resurrected the source marker: %+v %v", after, aerr)
|
||||
}
|
||||
} else if !isErrMethodNotAllowed(aerr) || !after.DeleteMarker ||
|
||||
after.ReplicationStatusInternal != pending.ReplicationStatusInternal ||
|
||||
after.VersionPurgeStatusInternal != pending.VersionPurgeStatusInternal ||
|
||||
after.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] != pending.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] {
|
||||
t.Fatalf("skipped creation rewrote the source marker: before=%+v after=%+v %v", pending, after, aerr)
|
||||
}
|
||||
if tc.unreadable {
|
||||
select {
|
||||
case entry := <-p.mrfSaveCh:
|
||||
if entry.versionID != version || entry.RetryCount != 1 || entry.Bucket != bucket || entry.Object != name {
|
||||
t.Fatalf("unverified creation queued wrong MRF entry: %+v", entry)
|
||||
}
|
||||
default:
|
||||
t.Fatal("unverified source read did not queue a retry")
|
||||
}
|
||||
}
|
||||
if len(p.mrfSaveCh) != 0 {
|
||||
t.Fatalf("%s: unexpected MRF entries queued: %d", tc.name, len(p.mrfSaveCh))
|
||||
}
|
||||
default:
|
||||
if result.ReplicationStatus() != replication.Completed || target.creations.Load() != 1 || !targetHasMarker {
|
||||
t.Fatalf("current creation not replicated: result=%+v creations=%d targetMarker=%v", result, target.creations.Load(), targetHasMarker)
|
||||
}
|
||||
if !isErrMethodNotAllowed(aerr) || !after.DeleteMarker || replicationStatusesMap(after.ReplicationStatusInternal)[target.arn] != replication.Completed {
|
||||
t.Fatalf("source creation state not completed: %+v %v", after, aerr)
|
||||
}
|
||||
if len(p.mrfSaveCh) != 0 {
|
||||
t.Fatal("completed creation queued MRF work")
|
||||
}
|
||||
}
|
||||
t.Logf("%s: %s checked; creations=%d purges=%d", backend, tc.name, target.creations.Load(), target.purges.Load())
|
||||
}
|
||||
|
||||
// staleLookupLayer cannot confirm the source marker: the read fails without
|
||||
// saying whether the version is present.
|
||||
type staleLookupLayer struct{ ObjectLayer }
|
||||
|
||||
func (staleLookupLayer) GetObjectInfo(context.Context, string, string, ObjectOptions) (ObjectInfo, error) {
|
||||
return ObjectInfo{}, InsufficientReadQuorum{}
|
||||
}
|
||||
@@ -41,15 +41,23 @@ const r5TagStamp = ReservedMetadataPrefixLower + TaggingTimestamp
|
||||
func r5Capacity(z *erasureServerPools) func() {
|
||||
var restores []func()
|
||||
for _, pool := range z.serverPools {
|
||||
pool.erasureDisksMu.Lock()
|
||||
for _, set := range pool.sets {
|
||||
old := set.getDisks
|
||||
disks := append([]StorageAPI(nil), old()...)
|
||||
old := pool.erasureDisks[set.setIndex]
|
||||
disks := append([]StorageAPI(nil), old...)
|
||||
for i := range disks {
|
||||
disks[i] = tagTestCapacityDisk{StorageAPI: disks[i]}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
restores = append(restores, func() { set.getDisks = old })
|
||||
// GetDisks copies this list under the same mutex. Keep its function
|
||||
// stable while background IAM scans are using the fixture.
|
||||
pool.erasureDisks[set.setIndex] = disks
|
||||
restores = append(restores, func() {
|
||||
pool.erasureDisksMu.Lock()
|
||||
pool.erasureDisks[set.setIndex] = old
|
||||
pool.erasureDisksMu.Unlock()
|
||||
})
|
||||
}
|
||||
pool.erasureDisksMu.Unlock()
|
||||
}
|
||||
return func() {
|
||||
for _, restore := range restores {
|
||||
@@ -58,6 +66,34 @@ func r5Capacity(z *erasureServerPools) func() {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPITaggingCapacityConcurrentIAM(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
if _, _, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
iam := globalIAMSys
|
||||
started, finished := make(chan struct{}), make(chan error, 1)
|
||||
go func() {
|
||||
close(started)
|
||||
for range 20 {
|
||||
if err := iam.Load(ctx, false); err != nil {
|
||||
finished <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
finished <- nil
|
||||
}()
|
||||
<-started
|
||||
for range 5000 {
|
||||
r5Capacity(z)()
|
||||
}
|
||||
if err := <-finished; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func r5Request(t *testing.T, router http.Handler, cred auth.Credentials, method, path, body string, headers map[string]string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
r, err := newTestSignedRequestV4(method, path, int64(len(body)), strings.NewReader(body), cred.AccessKey, cred.SecretKey, headers)
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/iotest"
|
||||
|
||||
"github.com/pierrec/lz4/v4"
|
||||
)
|
||||
|
||||
func TestUntarLZ4(t *testing.T) {
|
||||
for _, empty := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("empty=%t", empty), func(t *testing.T) {
|
||||
want := map[string][]byte{}
|
||||
if !empty {
|
||||
want["small.txt"] = bytes.Repeat([]byte("small object\n"), 10)
|
||||
want["large.txt"] = bytes.Repeat([]byte("large object\n"), 32768)
|
||||
}
|
||||
var compressed bytes.Buffer
|
||||
compressor := lz4.NewWriter(&compressed)
|
||||
archive := tar.NewWriter(compressor)
|
||||
for name, data := range want {
|
||||
if err := archive.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(data))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := archive.Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := archive.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := compressor.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var mu sync.Mutex
|
||||
got := map[string][]byte{}
|
||||
err := untar(t.Context(), iotest.HalfReader(bytes.NewReader(compressed.Bytes())), func(r io.Reader, info os.FileInfo, name string) error {
|
||||
// Exercise a short first read followed by streaming the remaining content.
|
||||
var output bytes.Buffer
|
||||
prefix := make([]byte, 7)
|
||||
if _, err := io.ReadFull(r, prefix); err != nil {
|
||||
return err
|
||||
}
|
||||
output.Write(prefix)
|
||||
if _, err := io.Copy(&output, r); err != nil {
|
||||
return err
|
||||
}
|
||||
if int64(output.Len()) != info.Size() {
|
||||
return fmt.Errorf("size mismatch for %s", name)
|
||||
}
|
||||
mu.Lock()
|
||||
got[name] = output.Bytes()
|
||||
mu.Unlock()
|
||||
return nil
|
||||
}, untarOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("objects=%d, want %d", len(got), len(want))
|
||||
}
|
||||
for name, data := range want {
|
||||
if !bytes.Equal(got[name], data) {
|
||||
t.Errorf("content mismatch for %s", name)
|
||||
}
|
||||
}
|
||||
t.Run("corrupt-header", func(t *testing.T) {
|
||||
damaged := bytes.Clone(compressed.Bytes())
|
||||
damaged[6] ^= 0xff
|
||||
err := untar(t.Context(), bytes.NewReader(damaged), func(io.Reader, os.FileInfo, string) error {
|
||||
t.Error("corrupt header must be rejected before uploading objects")
|
||||
return nil
|
||||
}, untarOptions{})
|
||||
if err == nil {
|
||||
t.Fatal("corrupt LZ4 header accepted")
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
)
|
||||
|
||||
func TestDeleteMarkerReadDataControls(t *testing.T) {
|
||||
obj, er, disks, bucket := markerPurgeFixture(t, 4)
|
||||
for _, body := range []string{"", "inline-data-control"} {
|
||||
name := "data-" + mustGetUUID()
|
||||
oi, err := er.PutObject(t.Context(), bucket, name, mustGetPutObjReader(t, bytes.NewBufferString(body), int64(len(body)), "", ""), ObjectOptions{Versioned: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, disk := range disks {
|
||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, name, oi.VersionID, ReadOptions{ReadData: true})
|
||||
if err != nil || fi.Deleted || fi.Size != int64(len(body)) || !fi.InlineData() || (body != "" && len(fi.Data) == 0) {
|
||||
t.Fatalf("data read: size=%d inline=%v bytes=%d error=%v", fi.Size, fi.InlineData(), len(fi.Data), err)
|
||||
}
|
||||
// Legacy inline data without its annotation still gets the original
|
||||
// ReadData behavior; the new marker guard must not affect it.
|
||||
delete(fi.Metadata, ReservedMetadataPrefixLower+"inline-data")
|
||||
if err := disk.WriteMetadata(t.Context(), "", bucket, name, fi); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fi, err = disk.ReadVersion(t.Context(), "", bucket, name, oi.VersionID, ReadOptions{ReadData: true})
|
||||
if err != nil || !fi.InlineData() {
|
||||
t.Fatalf("legacy inline read: %+v error=%v", fi, err)
|
||||
}
|
||||
}
|
||||
reader, err := obj.GetObjectNInfo(t.Context(), bucket, name, nil, http.Header{}, ObjectOptions{VersionID: oi.VersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := io.ReadAll(reader)
|
||||
reader.Close()
|
||||
if err != nil || string(got) != body {
|
||||
t.Fatalf("payload changed: %q error=%v", got, err)
|
||||
}
|
||||
}
|
||||
_, opts := seedPurgeMarker(t, er, bucket, "stored-marker-key", false)
|
||||
for _, disk := range disks {
|
||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, "stored-marker-key", opts.VersionID, ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Preserve even an existing unusual marker key; suppress only the
|
||||
// manufacture of a new inline annotation by the data-read branch.
|
||||
fi.Metadata[ReservedMetadataPrefixLower+"inline-data"] = "original"
|
||||
if err := disk.WriteMetadata(t.Context(), "", bucket, "stored-marker-key", fi); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fi, err = disk.ReadVersion(t.Context(), "", bucket, "stored-marker-key", opts.VersionID, ReadOptions{ReadData: true})
|
||||
if err != nil || fi.Metadata[ReservedMetadataPrefixLower+"inline-data"] != "original" {
|
||||
t.Errorf("existing marker key changed: metadata=%v error=%v", fi.Metadata, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMarkerMetadataRoundTrip(t *testing.T) {
|
||||
stamp := time.Date(2026, 9, 1, 12, 1, 2, 345, time.UTC)
|
||||
for _, free := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "replication", true: "free-version"}[free], func(t *testing.T) {
|
||||
metadata := map[string]string{
|
||||
ReservedMetadataPrefixLower + ReplicaStatus: "REPLICA",
|
||||
ReservedMetadataPrefixLower + ReplicaTimestamp: stamp.Format(time.RFC3339Nano),
|
||||
ReservedMetadataPrefixLower + ReplicationStatus: "arn1=COMPLETED;arn2=FAILED;",
|
||||
ReservedMetadataPrefixLower + ReplicationTimestamp: stamp.Add(-time.Hour).Format(time.RFC3339Nano),
|
||||
VersionPurgeStatusKey: "arn1=PENDING;arn2=FAILED;",
|
||||
targetResetHeader("arn1"): "original;reset1",
|
||||
targetResetHeader("arn2"): "original;reset2",
|
||||
ReservedMetadataPrefixLower + "unknown": "",
|
||||
}
|
||||
if free {
|
||||
metadata[ReservedMetadataPrefixLower+freeVersion] = ""
|
||||
metadata[metaTierName] = "tier"
|
||||
metadata[metaTierObjName] = "remote-object"
|
||||
metadata[metaTierVersionID] = "remote-version"
|
||||
}
|
||||
fi := FileInfo{VersionID: mustGetUUID(), Deleted: true, ModTime: stamp, Metadata: maps.Clone(metadata)}
|
||||
// Deliberately conflicting parsed state must never replace stored
|
||||
// values or add a key missing from the raw metadata.
|
||||
fi.ReplicationState = ReplicationState{ReplicaStatus: replication.Failed, ReplicaTimeStamp: stamp.Add(time.Hour), ResetStatusesMap: map[string]string{"new-arn": "invented"}}
|
||||
fi.SetHealing()
|
||||
fi.SetDataMov()
|
||||
fi.SetTierFreeVersionID(mustGetUUID())
|
||||
fi.SetTierFreeVersion()
|
||||
fi.SetSkipTierFreeVersion()
|
||||
xl := xlMetaV2{}
|
||||
if err := xl.AddVersion(fi); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stored, err := xl.getIdx(0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := make(map[string]string)
|
||||
for k, v := range stored.DeleteMarker.MetaSys {
|
||||
got[k] = string(v)
|
||||
}
|
||||
if !maps.Equal(got, metadata) {
|
||||
t.Errorf("stored metadata=%v want=%v", got, metadata)
|
||||
}
|
||||
decoded, err := xl.ToFileInfo("bucket", "marker", fi.VersionID, true, true)
|
||||
if err != nil || !decoded.ModTime.Equal(fi.ModTime) || decoded.TierFreeVersion() != free {
|
||||
t.Errorf("round trip identity: %+v error=%v", decoded, err)
|
||||
}
|
||||
if decoded.ReplicationState.ReplicaStatus != replication.Replica || decoded.ReplicationState.PurgeTargets["arn2"] != replication.VersionPurgeFailed || decoded.ReplicationState.Targets["arn1"] != replication.Completed {
|
||||
t.Errorf("lost parsed replication state: %+v", decoded.ReplicationState)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMarkerCreationMetadata(t *testing.T) {
|
||||
_, er, disks, bucket := markerPurgeFixture(t, 4)
|
||||
_, opts := seedPurgeMarker(t, er, bucket, "empty-key", false)
|
||||
for i, disk := range disks {
|
||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, "empty-key", opts.VersionID, ReadOptions{})
|
||||
if err != nil || fi.Metadata[ReservedMetadataPrefixLower+ReplicaStatus] != "REPLICA" || fi.Metadata[ReservedMetadataPrefixLower+ReplicaTimestamp] != opts.DeleteReplication.ReplicaTimeStamp.Format(time.RFC3339Nano) {
|
||||
t.Errorf("disk %d lost new marker replica identity: metadata=%v error=%v", i, fi.Metadata, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// deleteMarkerMetadata preserves the complete stored marker state during
|
||||
// healing. The parsed ReplicationState is lossy; even absent raw keys must not
|
||||
// be reconstructed from it. Only creation writers without stored metadata
|
||||
// use typed state, and never invent timestamps.
|
||||
func deleteMarkerMetadata(fi FileInfo) map[string][]byte {
|
||||
meta := make(map[string][]byte, len(fi.Metadata))
|
||||
for k, v := range fi.Metadata {
|
||||
switch k {
|
||||
case xMinIOHealing, xMinIODataMov,
|
||||
ReservedMetadataPrefixLower + tierFVID,
|
||||
ReservedMetadataPrefixLower + tierFVMarker,
|
||||
ReservedMetadataPrefixLower + tierSkipFVID:
|
||||
continue
|
||||
}
|
||||
meta[k] = []byte(v)
|
||||
}
|
||||
if len(meta) != 0 || fi.Healing() || fi.DataMov() {
|
||||
return meta
|
||||
}
|
||||
rs := fi.ReplicationState
|
||||
if !rs.ReplicaStatus.Empty() {
|
||||
meta[ReservedMetadataPrefixLower+ReplicaStatus] = []byte(rs.ReplicaStatus)
|
||||
if !rs.ReplicaTimeStamp.IsZero() {
|
||||
meta[ReservedMetadataPrefixLower+ReplicaTimestamp] = []byte(rs.ReplicaTimeStamp.UTC().Format(time.RFC3339Nano))
|
||||
}
|
||||
}
|
||||
if rs.ReplicationStatusInternal != "" {
|
||||
meta[ReservedMetadataPrefixLower+ReplicationStatus] = []byte(rs.ReplicationStatusInternal)
|
||||
if !rs.ReplicationTimeStamp.IsZero() {
|
||||
meta[ReservedMetadataPrefixLower+ReplicationTimestamp] = []byte(rs.ReplicationTimeStamp.UTC().Format(time.RFC3339Nano))
|
||||
}
|
||||
}
|
||||
if rs.VersionPurgeStatusInternal != "" {
|
||||
meta[VersionPurgeStatusKey] = []byte(rs.VersionPurgeStatusInternal)
|
||||
}
|
||||
for k, v := range rs.ResetStatusesMap {
|
||||
if !strings.HasPrefix(k, ReservedMetadataPrefixLower+ReplicationReset) {
|
||||
k = targetResetHeader(k)
|
||||
}
|
||||
meta[k] = []byte(v)
|
||||
}
|
||||
return meta
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"reflect"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Use complete metadata so the selected version can also be decoded by LIST
|
||||
// and the object read path, rather than testing only synthetic shallow headers.
|
||||
func quorumNullVersion(t testing.TB, generation int64) xlMetaV2ShallowVersion {
|
||||
t.Helper()
|
||||
fi := newFileInfo("fixed/object", 12, 4)
|
||||
fi.Erasure.Index = 1
|
||||
fi.DataDir = "11111111-1111-1111-1111-111111111111"
|
||||
fi.ModTime = time.Unix(generation, 0).UTC()
|
||||
fi.Size = 8192
|
||||
fi.Parts = []ObjectPartInfo{{Number: 1, Size: fi.Size, ActualSize: fi.Size}}
|
||||
fi.Metadata = map[string]string{"etag": fmt.Sprintf("%032d", generation)}
|
||||
var xl xlMetaV2
|
||||
if err := xl.AddVersion(fi); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return xl.versions[0]
|
||||
}
|
||||
|
||||
// Start with forward, reverse and interleaved orders, then fixed-seed shuffles.
|
||||
func quorumVersionOrders(input [][]xlMetaV2ShallowVersion) [][][]xlMetaV2ShallowVersion {
|
||||
orders := [][][]xlMetaV2ShallowVersion{slices.Clone(input), slices.Clone(input)}
|
||||
slices.Reverse(orders[1])
|
||||
interleaved := make([][]xlMetaV2ShallowVersion, 0, len(input))
|
||||
for i, j := 0, len(input)-1; i <= j; i, j = i+1, j-1 {
|
||||
interleaved = append(interleaved, input[i])
|
||||
if i != j {
|
||||
interleaved = append(interleaved, input[j])
|
||||
}
|
||||
}
|
||||
orders = append(orders, interleaved)
|
||||
for seed := range int64(32) {
|
||||
order := slices.Clone(input)
|
||||
rand.New(rand.NewSource(seed)).Shuffle(len(order), func(i, j int) {
|
||||
order[i], order[j] = order[j], order[i]
|
||||
})
|
||||
orders = append(orders, order)
|
||||
}
|
||||
return orders
|
||||
}
|
||||
|
||||
func TestMergeXLV2SingleNullQuorum(t *testing.T) {
|
||||
v := []xlMetaV2ShallowVersion{quorumNullVersion(t, 100), quorumNullVersion(t, 200), quorumNullVersion(t, 300)}
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
counts [3]int
|
||||
empty int
|
||||
quorum int
|
||||
want int
|
||||
}{
|
||||
{"consistent", [3]int{16}, 0, 8, 0},
|
||||
{"old9-new7", [3]int{9, 7}, 0, 8, 0},
|
||||
{"old15-new1", [3]int{15, 1}, 0, 8, 0},
|
||||
{"old3-new1", [3]int{3, 1}, 0, 3, 0},
|
||||
{"two-quorums", [3]int{8, 8}, 0, 8, 1},
|
||||
{"empty-streams", [3]int{9, 1}, 6, 8, 0},
|
||||
{"two-subquorums", [3]int{7, 7}, 2, 8, -1},
|
||||
{"three-subquorums", [3]int{6, 5, 5}, 0, 8, -1},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var input [][]xlMetaV2ShallowVersion
|
||||
for g, n := range tt.counts {
|
||||
for range n {
|
||||
input = append(input, []xlMetaV2ShallowVersion{v[g]})
|
||||
}
|
||||
}
|
||||
input = append(input, make([][]xlMetaV2ShallowVersion, tt.empty)...)
|
||||
want := []xlMetaV2ShallowVersion{}
|
||||
if tt.want >= 0 {
|
||||
want = append(want, v[tt.want])
|
||||
}
|
||||
for order, versions := range quorumVersionOrders(input) {
|
||||
for _, strict := range []bool{false, true} {
|
||||
for _, requested := range []int{0, 1} {
|
||||
got := mergeXLV2Versions(tt.quorum, strict, requested, versions...)
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("order=%d strict=%v requested=%d: got %#v, want %#v", order, strict, requested, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeXLV2SingleNullHeaderGroups(t *testing.T) {
|
||||
old := quorumNullVersion(t, 100)
|
||||
newer := quorumNullVersion(t, 200)
|
||||
differentSig := old
|
||||
differentSig.header.Signature[0]++
|
||||
differentFlags := old
|
||||
differentFlags.header.Flags ^= xlFlagInlineData
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
input [][]xlMetaV2ShallowVersion
|
||||
strict bool
|
||||
want []xlMetaV2ShallowVersion
|
||||
}{
|
||||
{"signature-nonstrict", [][]xlMetaV2ShallowVersion{{old}, {old}, {differentSig}, {newer}}, false, []xlMetaV2ShallowVersion{differentSig}},
|
||||
{"signature-strict", [][]xlMetaV2ShallowVersion{{old}, {old}, {differentSig}, {newer}}, true, []xlMetaV2ShallowVersion{}},
|
||||
{"flags-nonstrict", [][]xlMetaV2ShallowVersion{{old}, {old}, {differentFlags}, {newer}}, false, []xlMetaV2ShallowVersion{}},
|
||||
{"flags-strict", [][]xlMetaV2ShallowVersion{{old}, {old}, {differentFlags}, {newer}}, true, []xlMetaV2ShallowVersion{}},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := mergeXLV2Versions(3, tt.strict, 0, tt.input...)
|
||||
if !reflect.DeepEqual(got, tt.want) {
|
||||
t.Fatalf("got %#v, want %#v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveSingleNullQuorum(t *testing.T) {
|
||||
old, newer := quorumNullVersion(t, 100), quorumNullVersion(t, 200)
|
||||
input := make([][]xlMetaV2ShallowVersion, 16)
|
||||
for i := range input {
|
||||
input[i] = []xlMetaV2ShallowVersion{old}
|
||||
if i >= 9 {
|
||||
input[i] = []xlMetaV2ShallowVersion{newer}
|
||||
}
|
||||
}
|
||||
for order, versions := range quorumVersionOrders(input) {
|
||||
entries := make(metaCacheEntries, len(versions))
|
||||
for i := range versions {
|
||||
xl := &xlMetaV2{versions: versions[i]}
|
||||
metadata, err := xl.AppendTo(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entries[i] = metaCacheEntry{name: "fixed/object", metadata: metadata}
|
||||
}
|
||||
selected, ok := entries.resolve(&metadataResolutionParams{objQuorum: 8, dirQuorum: 8, requestedVersions: 1})
|
||||
if !ok {
|
||||
t.Fatalf("order %d: a complete null object version has quorum but was omitted", order)
|
||||
}
|
||||
xl, err := selected.xlmeta()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(xl.versions, []xlMetaV2ShallowVersion{old}) {
|
||||
t.Fatalf("order %d: incorrect complete selected version: %#v", order, xl.versions)
|
||||
}
|
||||
fi, err := xl.ToFileInfo("bucket", "fixed/object", "", false, false)
|
||||
if err != nil || !fi.IsValid() || fi.Size != 8192 || !fi.ModTime.Equal(time.Unix(100, 0)) {
|
||||
t.Fatalf("order %d: selected metadata cannot be read: %+v, %v", order, fi, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// These are compatibility assertions for inputs outside the narrow repair.
|
||||
// In particular, the mixed-version outputs below are not a new correctness
|
||||
// contract for version enumeration; that behavior requires a separate change.
|
||||
func TestMergeXLV2NullHistoriesUnchanged(t *testing.T) {
|
||||
old, newer := quorumNullVersion(t, 100), quorumNullVersion(t, 300)
|
||||
middle := quorumNullVersion(t, 200)
|
||||
middle.header.VersionID = [16]byte{1}
|
||||
highest := quorumNullVersion(t, 400)
|
||||
highest.header.VersionID = [16]byte{2}
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
input [][]xlMetaV2ShallowVersion
|
||||
want []xlMetaV2ShallowVersion
|
||||
}{
|
||||
{"mixed-forward", [][]xlMetaV2ShallowVersion{{old}, {old}, {old}, {newer, middle}, {middle}, {middle}}, []xlMetaV2ShallowVersion{middle}},
|
||||
{"mixed-reverse", [][]xlMetaV2ShallowVersion{{middle}, {middle}, {newer, middle}, {old}, {old}, {old}}, []xlMetaV2ShallowVersion{old}},
|
||||
{"history-pruned-first", [][]xlMetaV2ShallowVersion{{highest, old}, {highest, old}, {highest, old}, {highest, newer}}, []xlMetaV2ShallowVersion{highest}},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := mergeXLV2Versions(3, false, 0, tt.input...)
|
||||
if !reflect.DeepEqual(got, tt.want) {
|
||||
t.Fatalf("excluded history changed: got %#v, want %#v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, kind := range []string{"delete", "free", "legacy", "mixed-ec", "legacy-modern-ec", "nonzero-strict"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
a, b := old, newer
|
||||
switch kind {
|
||||
case "delete":
|
||||
b.header.Type = DeleteType
|
||||
case "free":
|
||||
b.header.Flags |= xlFlagFreeVersion
|
||||
case "legacy":
|
||||
b.header.Type = LegacyType
|
||||
case "mixed-ec":
|
||||
b.header.EcN, b.header.EcM = 8, 8
|
||||
case "legacy-modern-ec":
|
||||
b.header.EcN, b.header.EcM = 0, 0
|
||||
case "nonzero-strict":
|
||||
a.header.VersionID, b.header.VersionID = [16]byte{1}, [16]byte{1}
|
||||
}
|
||||
for _, requested := range []int{0, 1} {
|
||||
got := mergeXLV2Versions(3, true, requested, []xlMetaV2ShallowVersion{a}, []xlMetaV2ShallowVersion{a}, []xlMetaV2ShallowVersion{a}, []xlMetaV2ShallowVersion{b})
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("excluded input changed: requested=%d got %#v", requested, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
// All-legacy EC headers are eligible, unlike mixing legacy and modern EC.
|
||||
old.header.EcN, old.header.EcM = 0, 0
|
||||
newer.header.EcN, newer.header.EcM = 0, 0
|
||||
got := mergeXLV2Versions(3, false, 0, []xlMetaV2ShallowVersion{old}, []xlMetaV2ShallowVersion{old}, []xlMetaV2ShallowVersion{old}, []xlMetaV2ShallowVersion{newer})
|
||||
if !reflect.DeepEqual(got, []xlMetaV2ShallowVersion{old}) {
|
||||
t.Fatalf("all-legacy EC lost the old quorum: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkResolveSingleNullQuorumPage(b *testing.B) {
|
||||
old, newer := quorumNullVersion(b, 100), quorumNullVersion(b, 200)
|
||||
for _, kind := range []string{"consistent", "old-first", "new-first"} {
|
||||
b.Run(kind, func(b *testing.B) {
|
||||
var templates [16]metaCacheEntry
|
||||
for i := range templates {
|
||||
v := old
|
||||
if kind == "old-first" && i >= 9 || kind == "new-first" && i < 7 {
|
||||
v = newer
|
||||
}
|
||||
xl := &xlMetaV2{versions: []xlMetaV2ShallowVersion{v}}
|
||||
data, err := xl.AppendTo(nil)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
templates[i] = metaCacheEntry{metadata: data}
|
||||
}
|
||||
var names [1000]string
|
||||
for i := range names {
|
||||
names[i] = fmt.Sprintf("fixed/%04d", i)
|
||||
}
|
||||
resolver := metadataResolutionParams{objQuorum: 8, dirQuorum: 8, requestedVersions: 1}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
for _, name := range names {
|
||||
entries := templates
|
||||
for i := range entries {
|
||||
entries[i].name = name
|
||||
}
|
||||
selected, ok := metaCacheEntries(entries[:]).resolve(&resolver)
|
||||
if ok && selected.reusable {
|
||||
metaDataPoolPut(selected.metadata)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+83
-36
@@ -1630,7 +1630,7 @@ func (x *xlMetaV2) AddVersion(fi FileInfo) error {
|
||||
ventry.DeleteMarker = &xlMetaV2DeleteMarker{
|
||||
VersionID: uv,
|
||||
ModTime: fi.ModTime.UnixNano(),
|
||||
MetaSys: make(map[string][]byte),
|
||||
MetaSys: deleteMarkerMetadata(fi),
|
||||
}
|
||||
} else {
|
||||
ventry.Type = ObjectType
|
||||
@@ -1941,6 +1941,10 @@ func mergeXLV2Versions(quorum int, strict bool, requestedVersions int, versions
|
||||
// No need for non-strict checks if quorum is 1.
|
||||
strict = true
|
||||
}
|
||||
// Keep the original stream shapes: pruning must not make a versioned object
|
||||
// eligible for the single null-version recount below.
|
||||
originalVersions := versions
|
||||
var checkedSingleNull, singleNull bool
|
||||
// Shallow copy input
|
||||
versions = append(make([][]xlMetaV2ShallowVersion, 0, len(versions)), versions...)
|
||||
|
||||
@@ -2015,44 +2019,23 @@ func mergeXLV2Versions(quorum int, strict bool, requestedVersions int, versions
|
||||
// Version IDs match, but otherwise unable to resolve.
|
||||
// We are either strict, or don't have enough information to match.
|
||||
// Switch to a pure counting algo.
|
||||
x := make(map[xlMetaV2VersionHeader]int, len(tops))
|
||||
for _, a := range tops {
|
||||
if a.header.VersionID != ver.header.VersionID {
|
||||
continue
|
||||
}
|
||||
if !strict {
|
||||
// we must match EC, when we are not strict.
|
||||
if !a.header.matchesEC(ver.header) {
|
||||
continue
|
||||
}
|
||||
|
||||
a.header.Signature = [4]byte{}
|
||||
}
|
||||
x[a.header]++
|
||||
}
|
||||
latestCount = 0
|
||||
for k, v := range x {
|
||||
if v < latestCount {
|
||||
continue
|
||||
}
|
||||
if v == latestCount && latest.header.sortsBefore(k) {
|
||||
// Tiebreak, use sort.
|
||||
continue
|
||||
}
|
||||
for _, a := range tops {
|
||||
hdr := a.header
|
||||
if !strict {
|
||||
hdr.Signature = [4]byte{}
|
||||
}
|
||||
if hdr == k {
|
||||
latest = a
|
||||
}
|
||||
}
|
||||
latestCount = v
|
||||
}
|
||||
latest, latestCount = countXLV2Versions(tops, ver.header, latest, strict)
|
||||
break
|
||||
}
|
||||
}
|
||||
if latestCount < quorum {
|
||||
if !checkedSingleNull {
|
||||
singleNull = singleNullVersionStreams(originalVersions)
|
||||
checkedSingleNull = true
|
||||
}
|
||||
if singleNull {
|
||||
// A newer minority at the end can hide an older quorum from
|
||||
// the selection loop. Recount before discarding the null ID.
|
||||
if candidate, count := countXLV2Versions(tops, latest.header, latest, strict); count >= quorum {
|
||||
latest, latestCount = candidate, count
|
||||
}
|
||||
}
|
||||
}
|
||||
if latestCount >= quorum {
|
||||
merged = append(merged, latest)
|
||||
|
||||
@@ -2113,6 +2096,70 @@ func mergeXLV2Versions(quorum int, strict bool, requestedVersions int, versions
|
||||
return merged
|
||||
}
|
||||
|
||||
// singleNullVersionStreams excludes histories and other version types from the
|
||||
// additional recount. Check the original inputs, before any stream is pruned.
|
||||
func singleNullVersionStreams(versions [][]xlMetaV2ShallowVersion) bool {
|
||||
var ec xlMetaV2VersionHeader
|
||||
var haveEC bool
|
||||
for _, stream := range versions {
|
||||
if len(stream) == 0 {
|
||||
continue
|
||||
}
|
||||
if len(stream) != 1 {
|
||||
return false
|
||||
}
|
||||
h := stream[0].header
|
||||
if h.VersionID != [16]byte{} || h.Type != ObjectType || h.FreeVersion() {
|
||||
return false
|
||||
}
|
||||
if haveEC && (h.EcN != ec.EcN || h.EcM != ec.EcM) {
|
||||
return false
|
||||
}
|
||||
ec, haveEC = h, true
|
||||
}
|
||||
return haveEC
|
||||
}
|
||||
|
||||
// countXLV2Versions selects the most frequent compatible header for reference's
|
||||
// VersionID, retaining the existing sort tiebreak and last matching entry.
|
||||
func countXLV2Versions(tops []xlMetaV2ShallowVersion, reference xlMetaV2VersionHeader, latest xlMetaV2ShallowVersion, strict bool) (xlMetaV2ShallowVersion, int) {
|
||||
x := make(map[xlMetaV2VersionHeader]int, len(tops))
|
||||
for _, a := range tops {
|
||||
if a.header.VersionID != reference.VersionID {
|
||||
continue
|
||||
}
|
||||
if !strict {
|
||||
// we must match EC, when we are not strict.
|
||||
if !a.header.matchesEC(reference) {
|
||||
continue
|
||||
}
|
||||
a.header.Signature = [4]byte{}
|
||||
}
|
||||
x[a.header]++
|
||||
}
|
||||
var latestCount int
|
||||
for k, v := range x {
|
||||
if v < latestCount {
|
||||
continue
|
||||
}
|
||||
if v == latestCount && latest.header.sortsBefore(k) {
|
||||
// Tiebreak, use sort.
|
||||
continue
|
||||
}
|
||||
for _, a := range tops {
|
||||
hdr := a.header
|
||||
if !strict {
|
||||
hdr.Signature = [4]byte{}
|
||||
}
|
||||
if hdr == k {
|
||||
latest = a
|
||||
}
|
||||
}
|
||||
latestCount = v
|
||||
}
|
||||
return latest, latestCount
|
||||
}
|
||||
|
||||
type xlMetaBuf []byte
|
||||
|
||||
// ToFileInfo converts xlMetaV2 into a common FileInfo datastructure
|
||||
|
||||
+3
-1
@@ -1719,7 +1719,9 @@ func (s *xlStorage) ReadVersion(ctx context.Context, origvolume, volume, path, v
|
||||
defer metaDataPoolPut(buf)
|
||||
}
|
||||
|
||||
if readData {
|
||||
// Delete markers have no payload. In particular, do not manufacture an
|
||||
// inline-data metadata key when healing their zero-size FileInfo.
|
||||
if readData && !fi.Deleted {
|
||||
if len(fi.Data) > 0 || fi.Size == 0 {
|
||||
if fi.InlineData() {
|
||||
// If written with header we are fine.
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
PR #60 introduced an opt-in scheduler that moved objects between local server pools according to GET frequency. It and its feature-specific fixes have been removed. This does not remove ordinary lifecycle expiration, transitions to remote tiers, rebalance, decommission, or the general multi-pool correctness fixes from PR #178.
|
||||
|
||||
The [introduction and rollback record](../../investigations/access-tiering-revert.md) documents the commit history, scope decision, review corrections and unresolved validation findings.
|
||||
The [introduction and rollback record](https://silo.pgsty.com/compatibility/access-tiering-removal/) documents the commit history, scope decision, review corrections and unresolved validation findings.
|
||||
|
||||
The published Server 20260903 predates this feature. These instructions concern main/snapshot deployments that included #60; upgrading from the published version does not require access-tier configuration cleanup.
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# Compression Guide
|
||||
|
||||
For SILO's current SSE-C compression and historical-object boundary, see the [maintained encryption guide](https://silo.pgsty.com/administration/server-side-encryption/server-side-encryption-sse-c/) and [SSE-C replica design](https://silo.pgsty.com/blog/design/ssec-replica-integrity/). Check the documented release boundary before applying main-branch behavior to an older binary.
|
||||
|
||||
Silo server allows streaming compression to ensure efficient disk space usage.
|
||||
Compression happens inflight, i.e objects are compressed before being written to disk(s).
|
||||
Silo uses [`klauspost/compress/s2`](https://github.com/klauspost/compress/tree/master/s2)
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# Federation Quickstart Guide *Federation feature is deprecated and should be avoided for future deployments*
|
||||
|
||||
The maintained [federated CopyObject design](https://silo.pgsty.com/blog/design/federated-copy-object/) records the destination encryption, checksum, Object Lock and committed-response contract, including its Server release boundary.
|
||||
|
||||
This document explains how to configure Silo with `Bucket lookup from DNS` style federation.
|
||||
|
||||
## Cross-deployment copy behavior
|
||||
|
||||
@@ -1,193 +0,0 @@
|
||||
# 访问频率分层:引入、修复与回退记录
|
||||
|
||||
记录日期:2026-09-15。本记录说明 PR #60 的设计、合入后的取舍,以及此次回退为什么同时保留并补齐通用多池正确性修复。操作步骤见[退役迁移说明](../bucket/lifecycle/access-tiering-removal.md)。合并、正式发布和生产部署是不同状态;本记录随回退变更交付,不代表已经发布。
|
||||
|
||||
初稿审查时(2026-09-15 03:20 UTC),最终候选尚未移植到远端基线、尚未冻结 PR head,合并前全量检查与三次有效 Linux 运行尚未执行,本变更尚未合并。后续执行状态以承载本记录的 PR 及其绑定提交的验收记录为准;下文的历史实验不替代这些检查。
|
||||
|
||||
**当前验收状态:** 回退、普通版本 DELETE 调和、扫描复用及文档由 [PR #188](https://github.com/pgsty/silo/pull/188) 交付。本地与 CI 检查通过;首次 Linux 验收因 DELETE204 后的 HEAD/GET503 停止。随后完成可控机制实验、匹配写入负载对照,并修正准备检查;独立的新轮次 R-Upgrade-2 三次完整升级验收均通过。旧失败没有改判,原单次请求的逐盘状态仍不可追溯。最终合并状态以 PR 为准,正式发布与部署另行验收。详情见第 9 至 11 节。
|
||||
|
||||
## 1. 引入的目标和实际范围
|
||||
|
||||
[@mrjavadseydi](https://github.com/mrjavadseydi) 在 [PR #60](https://github.com/pgsty/silo/pull/60) 提出了基于 GET 频率的本地池间分层。成功 GET 更新有界滚动计数,后台调度器把热对象提升到配置中的首个池,把已经迁移且变冷的对象降到末个池。功能默认关闭,至少需要两个池;它与普通生命周期过期、远端对象存储 transition、rebalance 和 decommission 是不同机制。
|
||||
|
||||
实现不只是一个后台任务:它增加了 GET 计数入口、leader 调度、跨池版本栈复制、来源清理及失败恢复、热池配额、生命周期 XML 扩展、配置项、指标和扫描统计。访问热度统计使 data-usage cache 从 v8 升到 v9。对象本体的存储格式没有因此改变。
|
||||
|
||||
搬移要保持完整版本历史、删除标记、null version、时间戳、ETag、校验和和加密元数据;同时需要处理并发写入、目的端已有版本、来源部分删除和远端 tier 引用。合入前的修补和测试针对并覆盖了这些边界,不应把回退解释成贡献无效。贡献者署名继续保留,独立的其他贡献也不回退。
|
||||
|
||||
## 2. 可追溯时间线
|
||||
|
||||
下表的 PR/Issue 时间使用 UTC;提交链接对应具体代码,不把“报告时间”当作“缺陷首次出现时间”。
|
||||
|
||||
| 时间 | 事件 | 本次处理 |
|
||||
| --- | --- | --- |
|
||||
| 2026-08-15 10:15 | #60 创建;原始实现 [`7a060cab1`](https://github.com/pgsty/silo/commit/7a060cab1edd5bbc17da7f703bbd1ab7415b6f7c) | 随特性撤销 |
|
||||
| 2026-09-06 00:09 | [#133](https://github.com/pgsty/silo/issues/133) 报告多池副本写不能权威调和 Object Lock 状态 | 保留解决它的通用修复 |
|
||||
| 2026-09-06 15:12 | [#144](https://github.com/pgsty/silo/issues/144) 报告条件 DELETE 原子性仅限单个纠删码集合 | 保留解决它的通用修复 |
|
||||
| 2026-09-08 05:18 | [`9a6e1477f`](https://github.com/pgsty/silo/commit/9a6e1477f45067559def8423d431ee177795134f) 补访问分层兼容标识清单 | 删除功能专属标识,保留有依据的退役兼容 |
|
||||
| 2026-09-08 07:08 | [`374de0fa3`](https://github.com/pgsty/silo/commit/374de0fa32aa1d6eda57dbba6ac522ebf793b6be) 修复搬移的版本保全、写隔离和删除范围 | 随专属搬移器撤销 |
|
||||
| 2026-09-08 07:28 | [`5ac33e158`](https://github.com/pgsty/silo/commit/5ac33e1583e838ade3f56c7d60e80e7a854f9a88) 确定性覆盖搬移失败恢复 | 随已删除搬移器的专属测试撤销 |
|
||||
| 2026-09-08 07:42 | #60 以 [`a3df317ae`](https://github.com/pgsty/silo/commit/a3df317ae0725eb650e4d3e21551154f69be6229) 合入 | 以该 merge 的第一父差异确定功能边界 |
|
||||
| 2026-09-11 12:34 | [#178](https://github.com/pgsty/silo/pull/178) 合入通用多池写入、元数据与条件删除调和 | 保留,解除测试对访问搬移器的依赖 |
|
||||
| 2026-09-13 | [`2dd1e00da`](https://github.com/pgsty/silo/commit/2dd1e00da49faf995f2db29807fc6211b8376d7d) 的 CHANGELOG 同时汇总访问分层和通用多池修复 | 拆开表述,不整条删除独立修复历史 |
|
||||
| 2026-09-15 | 维护者决定收缩访问分层;完成来源分析、三种候选反证、退役兼容、普通版本 DELETE 修补与外部评审 | 形成此次选择性回退 |
|
||||
|
||||
#178 中的 [`e59a3d938`](https://github.com/pgsty/silo/commit/e59a3d938ed25c1bcd51efbb4ad6955073d195f7) 提供池级串行化、字段调和和条件删除;[`ccb676e60`](https://github.com/pgsty/silo/commit/ccb676e60cb7441ee65ff7c35f3b7828979101fd) 保护仍被其他副本使用的远端 tier 引用;[`51d41345f`](https://github.com/pgsty/silo/commit/51d41345f7ac532f9c6fea2b7dba5f29da930b8f) 保存 Linux 重启及 OIDC 验收记录。这三项不属于仅为访问频率调度而存在的代码。
|
||||
|
||||
截至回退评估时,公开 Server `RELEASE.2026-09-03T13-18-01Z` 早于 #60 合入。退役迁移主要针对运行过后续 main、自行构建或快照版本的实例;不能据此声称正式 release 用户普遍启用过此特性。
|
||||
|
||||
## 3. 为什么回退,为什么不能整批撤销后续修复
|
||||
|
||||
维护者的取舍是:默认关闭的可选调度能力,对配置、生命周期、缓存、统计和核心多池写入路径带来了过大的维护面。此次移除的是该能力及专属实现,没有测量并宣称吞吐量提升、延迟降低或固定减少一次分布式锁往返。
|
||||
|
||||
“后来改过同一文件”不等于“由 #60 引发”。#133/#144 的报告早于 #60 合入;更关键的是,原有 rebalance、decommission 和复制写入也会使同一版本暂时存在于多个池。访问分层消失后,这些状态仍然合法存在。移除调和与锁纪律会重新允许旧副本遮蔽新元数据、条件删除选错版本、清理错误被吞掉等问题。
|
||||
|
||||
评估在隔离工作树中实际比较了三条路线:
|
||||
|
||||
| 候选 | 通用多池回归 | 普通指定版本 DELETE |
|
||||
| --- | --- | --- |
|
||||
| A:撤销 #60,保留 #178 | 原有 13 组通过 | 仍能成功返回后留下可读副本 |
|
||||
| B:同时撤销 #60 和 #178 的存储修补 | 相同 13 组中 10 组失败 | 问题仍在 |
|
||||
| C:A 加普通版本 DELETE 调和 | 13 组原有及当时新增的 8 组通过 | 同一复现通过 |
|
||||
|
||||
这些是 2026-09-15 的历史对照结果,不是最终 PR head 的发布验收。后续补上目录标记、真实 rebalance 中断等覆盖后,通用多池测试达到 23 组。测试通过不能替代来源分析,来源分析也不能替代最终候选的运行验证。
|
||||
|
||||
## 4. 最终保留与删除的边界
|
||||
|
||||
- 删除访问 tracker、调度/搬移器、GET 和 scanner 钩子、热池配额、专属配置帮助、生命周期动作、指标及专属测试。
|
||||
- 保留普通过期、远端 transition、rebalance/decommission、复制写入,以及 #178 的 Object Lock、标签、条件删除、元数据调和与远端引用保护。
|
||||
- 原有 data-usage 生成解码器恢复到 #60 前的实现;允许读取 v8/v9,利用字段编码跳过已退役热度字段,继续写 v8。普通字段保真由历史真实 v9 样本测试覆盖。
|
||||
- 仅容忍准确的十个退役 ILM 键;读取生命周期时丢弃退役扩展。纯访问动作的规则需先清理才能再次编辑;混合规则保留普通动作。
|
||||
- 已经搬移的对象留在当前池;没有全量搬回、自动删除所有重复版本、后台清理服务或对象元数据重写。
|
||||
|
||||
曾对本地审查提交 `d06f1c614` 做过声明来源复核:消失的 357 个声明均不在 #60 之前,删除的十个文件均由 #60 引入;#60 原先删换的 41 行旧文本按忽略空白比较有 40 行恢复,剩下一行保留 #178 在已持有池锁时调用 `getWritePoolIdx(..., true)` 的修正,避免对同一对象再次取锁。生成缓存解码器与功能前逐字节一致,原有 13 组通用测试没有删除。这是该审查版本的保全证据,不能把数字脱离 SHA 当作未来所有版本的保证。
|
||||
|
||||
## 5. 普通版本 DELETE 是独立补洞
|
||||
|
||||
功能移除不会自动消除历史重复副本。普通单对象指定版本 DELETE 因而复用既有调和路径:在池级对象锁内读取每个池的目标版本,计算一次条件及回调,先删除非权威副本,再处理权威副本;任何不可读池或清理错误都不能当作成功。
|
||||
|
||||
范围包括 UUID、null version、delete marker,以及原先就被解析为 null version 的未指定版本目录标记 DELETE。入站复制、搬移内部调用、生命周期过期和 free-version 清理保留各自语义;批量 `DeleteObjects` 原本就会向池并发扇出,不是此次遗漏。
|
||||
|
||||
删除标记需要向 retention/metadata 回调传入与 set 层相同的 `MethodNotAllowed` 或 `ObjectNotFound` 语义。直接复用拒绝 marker 的元数据更新入口会错误地拒绝合法版本删除。回调从所有副本合并独立更新的 Object Lock 和标签,不能随意只采用一个池的状态。
|
||||
|
||||
存在两项明确的成功/失败边界:
|
||||
|
||||
1. 读法定多数不足时返回 `503 SlowDownRead`,即使另一个池有可读副本。旧路径的结果会受池遍历顺序影响;新路径把失败语义统一。它是正确性与可用性的取舍,需要恢复后重试。
|
||||
2. 出站删除复制尚未完成时,成功响应可以表示各副本进入 `VersionPurgePending`,由既有 worker 完成清理。原有每池 quorum 规则也继续适用;不能把成功响应等同于每一块盘立即物理删除。
|
||||
|
||||
## 6. 审查如何改变了方案
|
||||
|
||||
本地先形成三个线性审查提交:`8fdfdabd9` 移除特性,`d06f1c614` 补普通 DELETE,`6e3fdca97` 去除重复扫描。它们记录审查演进,最终 PR 在独立远端基线上重放,提交 ID 会改变;不应把线性演进误认为三份同时维护的实现。
|
||||
|
||||
Claude Code Opus 5 / max 的五轮实现评审要求补齐退役兼容、说明协调停机及环境一致性、验证真实池故障,并纠正运行证据措辞。随后 Claude 与 ZCode 的独立复核再次确认了回退边界和普通 DELETE 语义;最终两轮计划商榷收束了交付流程。
|
||||
|
||||
| 意见 | 裁定与处理 |
|
||||
| --- | --- |
|
||||
| 指定版本 DELETE 重复扫描所有池 | 接受。第一次扫描已在同一锁内得到目标副本,直接合并其结果。16 盘池在回调前的读取计数从 32 降为 16;这不等于总 I/O 或延迟减半。 |
|
||||
| N 个副本产生 N 条 DELETE 审计 | 反驳。底层调用只追加上下文标签,HTTP 层向每个配置审计目标发送一次请求事件。成功完成调和时池标签最终指向 primary;不增加额外 NoAuditLog 修改。 |
|
||||
| retention/metadata 顺序与 set 层不同 | 差异存在,但已有明确注释。保留 retention 优先,拒绝后不删除、不调度删除复制、不 Sweep;不宣称任意自定义回调都能交换。 |
|
||||
| 把优化 amend 进旧提交并直接丢弃 main 脏修改 | 不 amend 已审历史。先保存完整文件、补丁、哈希及可达恢复引用,核对覆盖后受控恢复。八组新增通用测试承接,访问搬移专用测试由真实 rebalance 中断覆盖替代。 |
|
||||
| 从当前本地分支直接开 PR | 调整。其祖先含另一个任务的 IAM/超时提交 `ebc9937d9`;从远端 `89637554d` 仅移植本次变更,不夹带或删除独立工作。 |
|
||||
| 合并之后再跑全量与 Linux 验收 | 不接受。先完成文档和移植,冻结 PR head,再验收;不能把旧 SHA 的测试直接提升为新基线的通过记录。 |
|
||||
| 不同基线 diff 必须逐字节一致 | 改为每提交 stable patch-id、路径与完整树等价性核验。blob hash 和行号随基线变化,不应成为错误的拒绝依据。 |
|
||||
|
||||
`objectPoolInfos` 的并行查询作为独立性能跟进,本次不增加并发实现。Contributor 署名、#132 配额指标、#77 桶元数据、federated COPY、IAM、超时及其他独立修复不因本次取舍被整体撤销。
|
||||
|
||||
## 7. 历史运行证据与未定案事项
|
||||
|
||||
以下是移植前 `d06f1c614` 的历史实验,不是最终 PR head 的验收替身。
|
||||
|
||||
| 运行 | 实际观察 | 不应推导的结论 |
|
||||
| --- | --- | --- |
|
||||
| `f590538f`,四节点双池 | 旧版真实 rebalance 中断留下 9 个重复 UUID;协调停机换新后对象四节点可读,旧配置与普通规则可编辑,真实缓存头 v9→v8;删除一个 addressed UUID 后四节点 HEAD/GET 404,另一个版本仍可读 | 没有验证全部 9 个不同 UUID 收敛;物理元数据仅每池抽查一盘;没有整份运行时统计守恒测量 |
|
||||
| `fd93cd37`,三节点双池 | 停掉一个池所在节点,保留 namespace 锁的 2/3 法定多数;DELETE 返回 503 SlowDownRead,源全部四盘保留目标版本;恢复后 DELETE204,各节点 HEAD/GET404 | 不能推广为跨主机网络、持久盘及压力验收 |
|
||||
| 被弃用的四节点停池拓扑 | 同时丢失 namespace 锁法定多数,发生客户端超时,记录脚本还遇到 NoneType 错误 | 不是“池读取返回503”的证明 |
|
||||
| `83676ca2` | 升级后配置/生命周期编辑之后一次 HeadObject 返回503;artifact 没有记录 DELETE 自身响应 | “DELETE之后”仅来自脚本顺序,不能写成已证实 DELETE204 后异常,也不能归类为已修复、既有问题或暂态 |
|
||||
|
||||
**开放项:`83676ca2` 的 HEAD503 仍未定案。** 可直接比较的目标阶段历史运行是一失败、一成功;一次未复现不足以关闭问题。仅凭 `SlowDownWrite` 等错误名字也不能给其他失败确定容量或环境根因。
|
||||
|
||||
最终候选的合并前复核采用有界规则:要求三次有效升级后 DELETE/HEAD 运行,最多五次总尝试;每次记录 DELETE 码/耗时、失败 HEAD 的节点与版本、GET 错误码、两池全盘元数据、固定间隔重试时序和旧版同拓扑对照。旧版可能保留副本返回200,不要求它满足新增跨池删除契约。
|
||||
|
||||
有证据证明在目标操作前失败的 harness 尝试才能不计入有效运行,但仍计入总尝试。任何目标阶段的新503或数据不变量失败都不能通过补跑抹掉,必须暂停合并并定位。三次通过也只满足这项工程检查,不证明历史异常已消失;开放项在合并和正式发布评估时仍须可见。
|
||||
|
||||
## 8. 交付与恢复纪律
|
||||
|
||||
最终候选使用独立分支;main 的五个旧修改先保存完整内容、二进制补丁、SHA-256 和具名 Git 恢复引用,再核对原 HEAD/哈希及测试覆盖,只恢复这五个文件。禁止用整树 reset 或 clean 代替受控归一;若用户已有新增编辑,应保留并重新核对。
|
||||
|
||||
全量 cmd/internal、相关 race、构建、vet、lint、生成文件和兼容检查,以及上述 Linux 验收,绑定最终候选的实际 SHA。若纳入新的远端提交,重新记录基线与 head,复核变更并重跑受影响验收。文档与原始测试记录各自保留其对应版本,不篡改旧失败、不用新通过覆盖旧记录。
|
||||
|
||||
逐节点滚动升级未通过既有二进制校验检查,采用[协调停机方案](../bucket/lifecycle/access-tiering-removal.md#before-upgrading-a-build-with-access-tiering)。实验使用单个 Docker Linux VM 和 tmpfs;正式 tag、包、镜像、跨主机及生产部署仍是独立交付。未验证全部重复 UUID 或运行时统计守恒应如实披露,不能反向引入自动搬回/清理需求或无关重构。
|
||||
|
||||
## 9. 执行后记:最终基线、恢复窗口与验收
|
||||
|
||||
本次实际交付由 [PR #188](https://github.com/pgsty/silo/pull/188) 承载。选择的远端基线为 `89637554d60c27cfc51d2281d0a4fe15e415f06d`,移植没有包含本地独立 IAM/超时提交 `ebc9937d9`。前三项实现和历史文档逐提交通过 stable patch-id 对照;虚拟补回独立 IAM 差异后,完整树与原审查分支一致。
|
||||
|
||||
首次本地 lint 发现新增回调选择分支触发 `gocritic/ifElseChain`,因此追加等价的无表达式 `switch` 改写,保持 marker、指定版本和普通元数据查找的条件顺序及分支体。重新固定的代码候选为 [`41aa84609`](https://github.com/pgsty/silo/commit/41aa84609754769cfb1861d7fd060c2e84182b98)。这一提交上,全量 cmd/internal 得到 6,428 个测试及子测试通过、166 个跳过,50 个有测试的包通过;相关 race 得到 283 个测试及子测试通过。`make build`、全包构建、vet、lint、生成文件及 rebrand/compat 检查均通过。[Go CI](https://github.com/pgsty/silo/actions/runs/34925534139)、[DCO](https://github.com/pgsty/silo/actions/runs/34925534110)、[VulnCheck](https://github.com/pgsty/silo/actions/runs/34925534142) 和[发布流水线的测试运行](https://github.com/pgsty/silo/actions/runs/34925534198)共 11 项检查通过;后者没有发布正式制品。
|
||||
|
||||
第一次最终候选停池实验 `dcd5c2e5` 在源版本保全断言后失败:停掉另一池返回 `503 SlowDownRead`,源四盘版本保留;恢复后 DELETE 成功,节点 0/2 的 HEAD/GET 返回 404,节点 1 返回 503。DELETE 成功由脚本已通过的 204 断言确定,原输出没有单独保存该次 DELETE 响应。此次失败如实保留,不能把后来的成功写回原记录。
|
||||
|
||||
复核发现,`ListBuckets` 以及位于源池的现存版本 GET,不能证明每个协调节点对另一池的读取连接已经恢复。随后进行了旧基线与候选的同拓扑对照,探测的是从未写入过的随机 UUID,并且在这些探测之前没有执行任何 DELETE:
|
||||
|
||||
| 高时间分辨率对照 | 桶列表和现存版本 | 从未写入版本的 HEAD/GET | 观察到的恢复窗口 |
|
||||
| --- | --- | --- | --- |
|
||||
| `20502a2f`,旧基线 `89637554d` | 三节点均为 200 | 节点 0/2 为 404,节点 1 为 503 SlowDownRead,连续 16 组 | 从重启后的观察循环起算约 1.60–2.50 秒 |
|
||||
| `246aaf77`,候选 `41aa84609` | 三节点均为 200 | 同样是节点 1 的 503,连续 11 组 | 约 1.67–2.30 秒 |
|
||||
|
||||
两边在缺失版本全节点连续三轮返回 404 后执行 DELETE,均得到 204、全节点 HEAD/GET 404、目标 UUID 在八盘均不存在且其他版本可读。另有两次较低时间分辨率诊断未捕捉到窗口,同样保留;这四次诊断不计入三次升级验收。
|
||||
|
||||
这给出了基线在零 DELETE 下的正向复现,证明原恢复条件不足。`getLatestObjectInfoWithIdx` 的读选择函数与基线文本相同:现存副本可以遮蔽另一池的读错误;缺失版本则必须确认所有池,不可读时返回 503。Claude 复核后同意修正实验准备条件并继续验收,明确反对把这一路径的 503 改成 404。最初失败没有瞬时 RPC 全貌,不能逐请求追溯每条连接;这些对照也不能给 `83676ca2` 归因。
|
||||
|
||||
修订后的验收在升级/恢复后逐节点探测从未写入的 UUID,记录首次全 404 时刻,要求连续三轮全 404;并列保存各节点 `admin info` 的全盘状态。最多等待 30 秒,超时仍失败,DELETE 之后仍严格要求 204/404,不把 503 纳入通过条件。后续失败保留实验资源供即时取证,再受控清理。
|
||||
|
||||
修订准备条件后的独立停池验收 `40a59b3b` 通过:离线 DELETE `503 SlowDownRead`,源四盘版本保留;恢复门约 1.48 秒完成,DELETE `204`,三节点 HEAD/GET `404`,目标在八盘均不存在,另一版本仍可读。恢复早期 `admin info` 中也记录到了各节点不同的离线盘视图,最后恢复为全盘正常。
|
||||
|
||||
完整升级验收随后实际进行了三次尝试:
|
||||
|
||||
| 尝试 | 运行 | 结果与证据边界 |
|
||||
| --- | --- | --- |
|
||||
| 1 | `83f88c59` | 准备失败,未启动候选。旧版 rebalance 报 Completed、搬移版本数为 0;源池占用约 6.5%,到平均空闲目标的差值约 3.13%,落入代码既有 5% 容差。增加造数从 64 到 192 个 2 MiB 版本后再试;本次仍计入五次总尝试上限。 |
|
||||
| 2,第一轮有效运行 | `ea58d0c5` | 通过。实际 rebalance 中断产生跨八盘的重复版本;停机复制同一数据供旧版对照。旧版 DELETE204 后仍可读;候选 DELETE204 后四节点立即及后续固定间隔 HEAD/GET 均404,八盘目标清除、另一版本四节点可读;旧 ILM/生命周期编辑和真实缓存 v9→v8 通过。 |
|
||||
| 3,第二轮有效运行 | `2059bc6b` | **候选失败,阻断合并。** 两阶段逐节点缺失版本连续三轮404、admin info全盘ok之后,DELETE明确204(约12.98ms);节点2随后的HEAD503/GET503 SlowDownRead,另外三节点404;八盘快照均无目标,首次重试及后续采样全404,其他版本四节点可读。首次失败保留,不因重试恢复改判。 |
|
||||
|
||||
第三次尝试发生后停止剩余验收,保留原容器、卷、元数据与响应时序进行诊断。不能把四节点顺序探测中的“节点2异常”直接解释为永久节点故障:它也可能与采样时间有关。随后在保留环境中,对三个额外重复版本做并发、不同顺序的“刚删 UUID / 从未写入 UUID”对照,候选稳定期均为204/404,未复现;旧版克隆数据的双次 DELETE 对照则遇到 `SlowDownWrite`,没有完成其全部断言。这些是诊断结果,不补入有效升级通过计数,也不证明第三次尝试已解释。
|
||||
|
||||
为观察逐盘返回,另在独立临时工作树编译仅增加日志的诊断二进制,**没有进入 PR**。它证明了第二层准备检查盲点:`getObjectFileInfo` 的四个响应信号中,可以只有两个实际 `file version not found`,其余两个是被跳过的盘所保留的 `errDiskOngoingReq`;`objectQuorumFromMeta` 的预期读 quorum 为2,因而仍可返回404。同期 `admin info` 汇集各服务器本地盘态为ok,不能证明请求节点到各盘的路径都可用。一次诊断 DELETE 的逐盘返回为 `[nil, nil, drive not found, drive not found]`,达不到写 quorum 3,故返回 `SlowDownWrite`。
|
||||
|
||||
Claude 撤回了此前“逐节点三轮404已是最强全池准备条件”的表述,同意这只能证明读 quorum,不能证明全盘可达或写 quorum。诊断给出了候选机制,但**第三次尝试失败瞬间没有逐请求逐盘日志,仍不足以确定其具体原因**;不能把后来稳定期的成功、诊断中的配额不足,或对错误名的解释当成该次故障的直接证据。
|
||||
|
||||
## 10. 首次执行的停止位置与恢复资料
|
||||
|
||||
首次执行停止时,代码为 `41aa84609`;后续提交只记录执行,不改变已测试的生产代码。当时有效升级运行是一通过、一失败,未满足三次有效运行全部通过的约定;总尝试3次,没有通过继续补跑消耗剩余次数来冲淡失败。`2059bc6b` 和先前的 `83676ca2` 均保持 **OPEN**。Claude 与 Codex 当时的裁定是 **NO-GO for merge**,没有把证据不足升级为“已修复”“既有问题”或“暂态”。后续收尾见第 11 节;实际合并状态以 [#188](https://github.com/pgsty/silo/pull/188) 为准。
|
||||
|
||||
主工作区原五文件的完整内容、二进制补丁和 SHA-256 已归档;另有可达 Git 引用 `refs/archive/access-tiering-main-five-files-20260915`,指向快照 `c7fbfc6ada0f0f2abcbe8c0a9681f07e12dafe52`。逐文件核验快照与原已审内容一致。首次停止时因验收未通过,没有执行五文件恢复,也没有移动或改写独立 IAM 提交 `ebc9937d9`。当时唯一待交付候选在 PR 分支,旧变体冻结等待验收裁定。
|
||||
|
||||
本机执行资料归档在 `~/.codex/outputs/silo-access-revert-assessment-20260915/final-execution/`:保存了每次尝试、旧/新基线对照、二进制 SHA-256、准备条件、源/目的全盘元数据、诊断补丁、独立评审意见,以及受控清理记录。原始失败记录不覆写;保存的诊断卷内容用于继续调查,不是生产数据或发布制品。
|
||||
|
||||
继续推进需要一次能区分机制的取证:在条件可控的升级实验中,于失败请求当时记录每池每盘的真实应答和错误类型,并同时读刚删版本与从未写入版本;明确区分盘面不同步、请求节点的不可用路径与其他原因。若四盘均真实应答仍返回503,应沿错误归约/元数据路径定位;若盘不可用,应查明连接或初始化状态,并验证准备条件。后续成功本身不能关闭本次失败,更不能通过把不可判定的503改成404来满足验收。正式发布、制品和生产部署继续作为独立交付。
|
||||
|
||||
## 11. 收尾复核:准备检查、可控机制与独立新轮次
|
||||
|
||||
后续收尾没有继续修改生产代码。旧基线 `89637554d` 与候选 `41aa84609` 使用各自的独立诊断构建,仅对测试桶记录逐盘应答;这些日志补丁没有进入 PR。第一轮完整诊断 `53ebe161` 通过但未复现503,仅作为一个样本保留。第二轮 `c951f762` 得到了不同的、可直接解释的失败:两个池的删除调用均记录 `quorum=3 errs=[nil,nil,nil,drive not found]`,DELETE204、所有读样本404,但八盘快照的盘3、7仍保留目标版本。它符合既有写 quorum 契约,并正向证明旧准备门会放行尚未完成挂盘的协调节点;它没有复现或解释 `2059bc6b` 那一次请求。
|
||||
|
||||
审查还纠正了两项推断:后续诊断进程在04:19的重连日志不能用于解释03:56的原失败;错误归约按具体错误值计数,不能把“最高同值计数不足”简单等同于“实际应答盘数不足”。原失败之后的全盘快照也不是失败瞬间的原子快照。这些界限继续保留。
|
||||
|
||||
准备检查改用服务器已有的 storage trace:从每个 S3 节点,对各自唯一、从未写入的对象执行 `GetObjectTagging`,该路径等待所有盘;按唯一对象名和后端节点、盘路径匹配真实 `storage.ReadVersion` 应答。四节点双池需要每轮32条实际缺失应答,连续三轮完成才满足准备条件;单纯404或 `admin info` 的本地盘状态不够。探针不创建对象,不改变服务器读写语义。trace 输出是多行 JSON 对象流,按流解析;缺失 trace 证据会使检查失败,不能据此假定对应盘健康。
|
||||
|
||||
为了比较相同的跨池写入负载,停止真实 rebalance 夹具后复制两份相同数据。旧基线使用 #178 已有的 `If-Match` 条件 DELETE,候选使用普通指定版本 DELETE,二者都处理同一目标的两个池副本。两臂分别完成96次“已删版本/从未写入版本”的 HEAD/GET 对照,均404,目标八盘均清除。准备检查分别耗时约0.60秒和17.08秒。这是一对匹配样本,没有观察到候选专属差异,不是吞吐比较,也不能排除所有可能的故障机制。
|
||||
|
||||
`f55967b7` 另用明确制造的重复副本夹具完成因果实验,而非冒充真实 rebalance:pool0 的四盘由一个节点承载并保留 namespace 锁;pool1 的四盘分布到另外四个节点。先停止 pool1 一盘,DELETE204 后直接确认只有该盘残留目标版本。重新接入这份副本,再停止 pool1 两块已删除该版本的盘,保留“一盘返回版本、一盘返回缺失”的读视图。三个被测协调节点的已删版本均返回 `503 SlowDownRead`,从未写入版本均返回404;同一失败请求的逐盘记录明确为 `[drive not found, drive not found, file version not found, nil]`,没有足够的同值应答达到读 quorum。整个实验没有丢失 pool0 的 namespace 锁,也没有将不确定状态改为404。
|
||||
|
||||
该因果实验的部分节点重启未在35秒内恢复全部40条访问路径,原实验因此仍记 FAIL;正向机制观察与这个恢复失败分开记录。随后协调重启全部五节点,独立恢复检查通过,五节点 HEAD/GET 均404。这不是滚动恢复已通过的声明,也不追溯原 `2059bc6b` 的逐盘状态。可控复现确定的是故障机制类别,原单次实例继续 **OPEN**。
|
||||
|
||||
Claude 复核上述证据后同意显式开启 **R-Upgrade-2**:原轮次的一通过、一失败和总尝试3次保持原样,不合并计数,不静默重置。新轮使用未经诊断修改的 `41aa84609` 二进制,要求三次有效运行全部通过、最多五次总尝试;任何新503或数据不变量失败仍须停止。每次 DELETE 前后均检查32条路径,响应后的即时 HEAD/GET 先于后置准备检查执行,避免等待掩盖短暂错误。
|
||||
|
||||
| 新轮次运行 | 完整运行耗时 | 升级后32路径准备耗时 | 验收结果 |
|
||||
| --- | --- | --- | --- |
|
||||
| `8eb016db` | 90.20秒 | 17.04秒 | PASS |
|
||||
| `2a2b7b64` | 80.83秒 | 0.62秒 | PASS |
|
||||
| `371e7b9f` | 96.28秒 | 0.60秒 | PASS |
|
||||
|
||||
三次均实际走到候选阶段:DELETE204,所有即时及后续 HEAD/GET 样本404,目标在八盘均不存在,其他版本从四节点读回;旧配置、普通生命周期规则编辑及真实缓存 v9→v8 均通过。删除前后各三轮32路径检查也全部通过。准备时间有明显波动,应验证访问路径,不能用固定等待秒数代替检查。这些结果满足修正准备条件后的有界验收;不把有限样本写成“历史503已消失”,不把不同阶段的诊断通过计入新轮次。
|
||||
|
||||
新轮仍绑定生产代码 `41aa84609`(Linux 二进制 SHA-256 `28e1339d630a22fa5a0e4659b6182224e81f6cd7cd4079856534390e40a697b1`),后续仅更新文档。合并前须核对源码等价性和最终 CI,按第8节的恢复纪律处理旧五文件,保留独立 IAM 提交。原 `2059bc6b`/`83676ca2`、部分重启恢复边界、单 VM/tmpfs、未验证全部不同重复 UUID 和整份运行时统计守恒继续可见;不为此新增自动搬回、清理服务或读错误降级。
|
||||
|
||||
本轮详细资料位于原归档的 `final-execution/closure-20260915/`,包括匹配对照、同请求逐盘日志、`qualification-summary.json`、独立 R-Upgrade-2 账本、诊断补丁和完整卷归档。原失败现场、后续对照及恢复后的卷分别标注时点。临时实验资源在归档验证后清理;这些资料与正式发布制品区分管理。
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,190 +0,0 @@
|
||||
# SILO stack: Go 1.27 compatibility audit
|
||||
|
||||
> This is a dated investigation, with the source and runtime boundaries recorded
|
||||
> below. It does not establish the current dependency pins or a later release.
|
||||
> See [the current changelog](../../CHANGELOG.md) and
|
||||
> [component matrix](https://silo.pgsty.com/compatibility/versions/).
|
||||
|
||||
|
||||
2026-09-09. Scope: the maintained Server, silo-pkg, mcli, and Console. This extends
|
||||
the [OIDC #154 investigation](issue-154.md) to other paths using the same TLS
|
||||
configuration and to adjacent standard-library changes. It records local
|
||||
validation performed before commit. No pushes, issue comments, releases, or
|
||||
production changes were made during the audit.
|
||||
|
||||
## Findings and changes
|
||||
|
||||
| Component | Finding | Change |
|
||||
| --- | --- | --- |
|
||||
| Server, baseline `d1105bbb3d4a0afa33b3a4ac11b821235038ed0e` | Eight TLS configuration sites explicitly use the same ML-KEM-containing curve list, overriding `tlsmlkem=0` in Go 1.27. | Remove the eight assignments and obsolete `TLSCurveIDs` helper. Retain Go defaults in HTTP clients, replication, cloud client-certificate transport, both grid links, etcd, and the S3 listener. Add wire-level regression tests. |
|
||||
| silo-pkg, baseline `a92c54d` | No built-in explicit PQ curve list. Web environment transport uses defaults; LDAP/OIDC accept caller configuration. | Add a real web-environment TLS regression test and document runtime-default selection. Keep the Go 1.26 library floor. |
|
||||
| mcli, baseline `fcd5cad8` | S3/Admin transport and alias/TOFU dialer use default curves. No instance of the Server's override bug found. | Add real S3/Admin transport and alias-dialer handshake tests; update Go/TLS upgrade notes. |
|
||||
| Console, baseline `c103d08ec` | IdP, SILO/STS, Prometheus, and webhook clients share a transport using defaults. The HTTPS listener explicitly uses only P-256. | Add real IdP/SILO-client handshake tests; update Go/TLS upgrade notes. Retain the existing listener policy. |
|
||||
|
||||
The Server's general external HTTP transport is also used by OpenID discovery
|
||||
and JWKS, identity plugins, notification/lambda checks, audit/log webhooks, and
|
||||
S3 cloud backends. Fixing only the two OIDC callers would leave those other paths
|
||||
affected. The broader correction supersedes the earlier OIDC-only candidate.
|
||||
|
||||
The retained defaults follow Go's implementation rather than duplicating its
|
||||
`GODEBUG` parser. Go 1.27 explicitly changed the interaction between manually
|
||||
selected curves and the `tlsmlkem`/`tlssecpmlkem` default controls. See the
|
||||
[Go TLS release notes](https://go.dev/doc/go1.27#crypto/tls).
|
||||
|
||||
With no opt-out, default curves additionally include SecP256r1MLKEM768 and
|
||||
SecP384r1MLKEM1024. With `GODEBUG=tlsmlkem=0`, hybrid exchanges are disabled for
|
||||
default-configured TLS throughout the process. This is an intentional change
|
||||
from the old fixed subset. `GODEBUG=tlssecpmlkem=0` disables just the SecP
|
||||
hybrids while retaining X25519MLKEM768. TLS versions, cipher-suite policy,
|
||||
certificate and hostname checks, client certificates, proxies, and HTTP/2 choices are not
|
||||
relaxed by this patch. No automatic fallback after a TLS error is introduced.
|
||||
|
||||
## Reproduction and regression evidence
|
||||
|
||||
Before changing product code, the new Server test failed for all five tested
|
||||
outbound constructors with `tlsmlkem=0`: general external HTTP, internode HTTP,
|
||||
replication, cloud client certificates, and etcd. The server observed
|
||||
`[X25519MLKEM768 X25519 P256 P384 P521]` in every case. Both TLS 1.2 and TLS 1.3
|
||||
peers reproduced the problem. The inbound Server also accepted a PQ-only client
|
||||
despite the same opt-out.
|
||||
|
||||
After the change, those tests pass on darwin/arm64 and linux/arm64. They exercise
|
||||
20 outbound combinations (five constructors, two peer TLS versions, two debug
|
||||
settings), plus inbound classical/PQ-only peers with the opt-out enabled and
|
||||
disabled. The inbound opt-out case rejects a PQ-only peer while continuing to
|
||||
accept P-256. The etcd test exercises its actual TLS configuration, not an etcd
|
||||
cluster; the client-certificate test exercises construction/loading, not a full
|
||||
mutual-authentication service.
|
||||
|
||||
The mc, Console, and silo-pkg handshake tests pass without product code changes.
|
||||
All use a trusted synthetic certificate and inspect a real ClientHello. They
|
||||
check both disabling and retaining ML-KEM. Console also retains its existing
|
||||
unknown-CA, hostname, and endpoint-scoping regression checks.
|
||||
|
||||
A freshly source-built complete Linux Server then passed three isolated
|
||||
integration scenarios using the existing synthetic IdP fixture:
|
||||
|
||||
| Scenario | Result |
|
||||
| --- | --- |
|
||||
| ML-KEM-intolerant IdP + `tlsmlkem=0` | Discovery/JWKS, IAM, Console, and authenticated Admin calls succeed; login 204 and bucket list 200. |
|
||||
| Normal TLS 1.3 IdP, no opt-out | Same successful login, token exchange, STS, session, and bucket-list chain. |
|
||||
| Add OIDC to a running Server | Actual Admin API accepts the provider under the compatibility setting. |
|
||||
|
||||
Both login scenarios reject modified JWT signatures and a wrong audience: no
|
||||
session cookie is issued and authenticated bucket access returns 403. Login
|
||||
currently reports these authentication failures as 500; that existing error
|
||||
mapping is outside this TLS change. Curl in the same namespace returns HTTP/2
|
||||
200. The containers use a pre-existing generic Debian image, `--network none`,
|
||||
and no published ports; no Server or Console image was downloaded or used.
|
||||
|
||||
The fixture drives real Console HTTP APIs, not rendered browser interaction.
|
||||
This is a conditional interoperability reproduction, not proof of the actual
|
||||
customer ingress behavior. Grid's existing tests pass, but a production-style
|
||||
distributed TLS cluster, external cloud providers, and real etcd/LDAP/Keycloak
|
||||
deployments were not exercised. Disabling ML-KEM does not disable new ML-DSA
|
||||
signature offers and does not repair an ingress that rejects those offers.
|
||||
|
||||
## Other Go changes checked
|
||||
|
||||
### macOS root certificates: a confirmed upgrade-visible change
|
||||
|
||||
Using the same public synthetic CA and `certs.GetRootCAs`, fresh-process probes
|
||||
produce the following results when `SSL_CERT_FILE` points at that CA and
|
||||
`SSL_CERT_DIR` points at an empty directory:
|
||||
|
||||
| Compiler / main module | CA from environment trusted? | Explicit CA argument trusted? |
|
||||
| --- | --- | --- |
|
||||
| Go 1.26.5 / `go 1.26.0` | No | Yes |
|
||||
| Go 1.27.1 / `go 1.26.0` | No | Yes |
|
||||
| Go 1.27.1 / `go 1.27.1` | Yes | Yes |
|
||||
|
||||
The Go 1.26 module built by Go 1.27 carries
|
||||
`DefaultGODEBUG=...x509sslcertoverrideplatform=0`; explicitly setting that option
|
||||
to `1` enables the new behavior. A Go 1.27 application can explicitly set it to
|
||||
`0` to recover the previous platform behavior. The diagnostic source is
|
||||
[cert-roots.go](issue-154/cert-roots.go). The consuming application's defaults
|
||||
apply to library calls as well, so silo-pkg's older `go` directive does not
|
||||
prevent the behavior in Server, mc, or Console.
|
||||
|
||||
This is expected standard-library behavior, not a reason to silently discard
|
||||
configured CA variables or skip verification. Setting either variable replaces
|
||||
Keychain trust with on-disk roots and Go's verifier; stale or incomplete paths
|
||||
can break previously trusted connections. Unset inherited variables to restore
|
||||
Keychain trust; explicit additional CAs still work. The three application
|
||||
READMEs and the package README now document this. The package's Windows loader enumerates
|
||||
the native ROOT store directly and does not call `SystemCertPool`; it does not
|
||||
inherit this particular new setting. Windows behavior was reviewed in source,
|
||||
not runtime-tested.
|
||||
|
||||
### JSON, HTTP, timers, and other compatibility controls
|
||||
|
||||
- **JSON:** checked owned JSON error handling and exercised policy/condition,
|
||||
config, and authentication tests. `quick` uses typed `SyntaxError` and
|
||||
`UnmarshalTypeError`; no owned decision depending on changed standard JSON
|
||||
error text was found. silo-pkg's full suite passes with both Go compilers;
|
||||
mc's command and Console's API/auth suites pass under Go 1.27. No broad
|
||||
`nojsonv2` opt-out or serialization rewrite is justified by these results.
|
||||
- **HTTP response closing:** Go 1.27's standard-library drain is bounded at
|
||||
256 KiB and 50 ms. mc's actual early-close/cancel regression passes, including
|
||||
the compressed S3 Select stream. Existing owned drain helpers can still have
|
||||
independent timeout concerns; those are not newly caused by this Go change.
|
||||
- **ALPN and custom connections:** mc's TLS dialer returns a real `*tls.Conn`;
|
||||
its deadline wrapper is on the TCP dial path. No new accidental HTTP/2 opt-in
|
||||
from the expanded `ConnectionState` interface support was identified in
|
||||
these transports. Console keeps its existing transport policy.
|
||||
- **Removed switches:** no maintained runtime/config reliance on `tlsrsakex`,
|
||||
`tls3des`, `tls10server`, `x509keypairleaf`, or `asynctimerchan` was found.
|
||||
Existing explicit cipher policies continue to be explicit. Timer uses in
|
||||
package certificate reload and license refresh do not depend on buffered
|
||||
timer-channel length/capacity. Unix EOF error changes do not expose a matching
|
||||
owned error-type assumption on the reviewed Unix-socket paths.
|
||||
- **Platform floor:** application READMEs now identify macOS 13 as the minimum
|
||||
for Go 1.27 binaries. No Windows, old macOS, or PowerPC runtime claim is made.
|
||||
|
||||
## Validation and delivery
|
||||
|
||||
- Server: focused TLS tests on macOS and Linux; macOS race run; crypto, HTTP,
|
||||
grid, and all `internal/config/...` tests; full Linux Server build and the
|
||||
three integration scenarios above.
|
||||
- silo-pkg: `make test` (lint and full race suite) on Go 1.27.1; full suite on
|
||||
Go 1.26.5, preserving the library floor.
|
||||
- mc: complete `./cmd` suite, including the new TLS test and existing S3 Select
|
||||
early-cancel test.
|
||||
- Console: complete `./api/... ./pkg/...` suites, including identity and STS
|
||||
validation and the new handshake tests.
|
||||
|
||||
Go lint reports zero issues in all four repositories. Server's optional spelling
|
||||
check is skipped because `typos` is not installed. Its lint target used the
|
||||
already-installed matching golangci-lint v2.13.1 after a redundant download was
|
||||
stopped; mc's lint was rerun serially after the global linter lock prevented the
|
||||
first concurrent attempt. These tooling retries did not require source changes.
|
||||
|
||||
An independent Fable 5.1 Max review reproduced the old-code failures and ran the
|
||||
new TLS tests with the race detector. Its shuffled mc and Console suites passed.
|
||||
One shuffled silo-pkg run failed the untouched `certs.TestValidPairAfterWrite`;
|
||||
that test passed three plain reruns, the same shuffle seed, and the full certs
|
||||
package rerun. The certs package runs in a separate test binary from the changed
|
||||
env tests; this was classified as an existing timing flake.
|
||||
|
||||
The review also found that committing the diagnostic fixture made rebrand CI
|
||||
count synthetic IdP paths as product routes. The guard now excludes
|
||||
`docs/investigations/`; the product compatibility baseline is unchanged.
|
||||
The full proposed file set passes the guard, and a negative control adding a
|
||||
product route still fails. After the review corrections, a fresh Linux Server
|
||||
build passed all three integration scenarios again; the new binary identity and
|
||||
rerun results are recorded in the evidence file's `post_review_validation`.
|
||||
|
||||
All source and dependency choices remain those of the maintained PGSTY stack.
|
||||
`go.mod` and `go.sum` are unchanged in every repository. The Server uses its
|
||||
existing pinned Console/pkg/mc modules; the companion changes are tests and
|
||||
documentation, so no replacement graph or unpublished dependency version is
|
||||
needed to build the runtime fix.
|
||||
|
||||
Validation used separate worktrees for Server, silo-pkg, mc, and Console, leaving
|
||||
the original repositories and their `main` branches untouched. Build identities,
|
||||
fixture results, and command outcomes are recorded in
|
||||
[go127-stack-evidence.json](go127-stack-evidence.json). Absolute paths and branch
|
||||
names in the captured evidence identify the environment at recording time.
|
||||
The recorded module graph identifies the dependencies selected for those
|
||||
builds; `go.sum` also retains checksums for unselected versions. Local image IDs
|
||||
and temporary paths are historical evidence, not portable setup instructions.
|
||||
@@ -1,534 +0,0 @@
|
||||
{
|
||||
"date": "2026-09-11",
|
||||
"runs": [
|
||||
{
|
||||
"runid": "silo-v1-0806-9c53f14c",
|
||||
"version": "0806",
|
||||
"image": "sha256:29a498b24669cae1fed11c1a2fb2b3d73c68829a0a9c0b14e71b386671d38fac",
|
||||
"nodes": 4,
|
||||
"drives": 4,
|
||||
"filesystem": "Linux tmpfs named volumes, held mounted across server restarts",
|
||||
"drive_bytes": 268435456,
|
||||
"status": "PASS",
|
||||
"phases": [
|
||||
{
|
||||
"phase": "startup-admin",
|
||||
"seconds_from_start": 4.157,
|
||||
"first_online_by_coordinator": [
|
||||
4.037,
|
||||
4.078,
|
||||
4.118,
|
||||
3.737
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "startup-canary",
|
||||
"attempts": 1,
|
||||
"puts": 4,
|
||||
"gets": 16,
|
||||
"hard_deadline_seconds": 60,
|
||||
"gate_seconds": 0.239,
|
||||
"seconds_from_start": 4.396,
|
||||
"first_put_seconds_after_admin_by_coordinator": [
|
||||
0.096,
|
||||
0.116,
|
||||
0.127,
|
||||
0.138
|
||||
],
|
||||
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||
0.162,
|
||||
0.194,
|
||||
0.217,
|
||||
0.239
|
||||
],
|
||||
"transient_error_count": 0,
|
||||
"first_transient_errors": []
|
||||
},
|
||||
{
|
||||
"phase": "full-restart-admin",
|
||||
"seconds_from_start": 2.084,
|
||||
"first_online_by_coordinator": [
|
||||
1.937,
|
||||
1.976,
|
||||
2.016,
|
||||
1.643
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "full-restart-canary",
|
||||
"attempts": 45,
|
||||
"puts": 4,
|
||||
"gets": 16,
|
||||
"hard_deadline_seconds": 60,
|
||||
"gate_seconds": 14.449,
|
||||
"seconds_from_start": 16.534,
|
||||
"first_put_seconds_after_admin_by_coordinator": [
|
||||
14.315,
|
||||
13.573,
|
||||
13.953,
|
||||
0.05
|
||||
],
|
||||
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||
14.373,
|
||||
14.396,
|
||||
14.426,
|
||||
14.449
|
||||
],
|
||||
"transient_error_count": 129,
|
||||
"first_transient_errors": [
|
||||
{
|
||||
"attempt": 1,
|
||||
"operation": "put",
|
||||
"node": 0,
|
||||
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||
},
|
||||
{
|
||||
"attempt": 1,
|
||||
"operation": "put",
|
||||
"node": 1,
|
||||
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||
},
|
||||
{
|
||||
"attempt": 1,
|
||||
"operation": "put",
|
||||
"node": 2,
|
||||
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||
},
|
||||
{
|
||||
"attempt": 2,
|
||||
"operation": "put",
|
||||
"node": 0,
|
||||
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "readback-15s",
|
||||
"seconds_after_canary": 17.596,
|
||||
"objects": 55,
|
||||
"reads": 220,
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"phase": "readback-30s",
|
||||
"seconds_after_canary": 31.357,
|
||||
"objects": 55,
|
||||
"reads": 220,
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"phase": "readback-60s",
|
||||
"seconds_after_canary": 61.566,
|
||||
"objects": 55,
|
||||
"reads": 220,
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"phase": "one-node-outage",
|
||||
"existing_read": true,
|
||||
"put": true,
|
||||
"readers": 3
|
||||
},
|
||||
{
|
||||
"phase": "rejoin-admin",
|
||||
"seconds_from_start": 1.382,
|
||||
"first_online_by_coordinator": [
|
||||
1.27,
|
||||
1.307,
|
||||
1.343,
|
||||
1.382
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "rejoin-canary",
|
||||
"attempts": 1,
|
||||
"puts": 4,
|
||||
"gets": 16,
|
||||
"hard_deadline_seconds": 60,
|
||||
"gate_seconds": 0.158,
|
||||
"seconds_from_start": 1.541,
|
||||
"first_put_seconds_after_admin_by_coordinator": [
|
||||
0.013,
|
||||
0.029,
|
||||
0.039,
|
||||
0.05
|
||||
],
|
||||
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||
0.077,
|
||||
0.102,
|
||||
0.132,
|
||||
0.158
|
||||
],
|
||||
"transient_error_count": 0,
|
||||
"first_transient_errors": []
|
||||
},
|
||||
{
|
||||
"phase": "final-readback",
|
||||
"seconds_after_canary": 1.71,
|
||||
"objects": 60,
|
||||
"reads": 240,
|
||||
"errors": []
|
||||
}
|
||||
],
|
||||
"acknowledged_objects": 60,
|
||||
"version_ids_recorded": true,
|
||||
"sha256_recorded": true
|
||||
},
|
||||
{
|
||||
"runid": "silo-v1-0903-a457573c",
|
||||
"version": "0903",
|
||||
"image": "sha256:b616a0cf8cb281e7e6bb3c9b1fb53875b4016a2878223925541c18f82d6c5ca3",
|
||||
"nodes": 4,
|
||||
"drives": 4,
|
||||
"filesystem": "Linux tmpfs named volumes, held mounted across server restarts",
|
||||
"drive_bytes": 268435456,
|
||||
"status": "PASS",
|
||||
"phases": [
|
||||
{
|
||||
"phase": "startup-admin",
|
||||
"seconds_from_start": 4.271,
|
||||
"first_online_by_coordinator": [
|
||||
4.158,
|
||||
4.194,
|
||||
3.827,
|
||||
3.867
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "startup-canary",
|
||||
"attempts": 1,
|
||||
"puts": 4,
|
||||
"gets": 16,
|
||||
"hard_deadline_seconds": 60,
|
||||
"gate_seconds": 0.2,
|
||||
"seconds_from_start": 4.471,
|
||||
"first_put_seconds_after_admin_by_coordinator": [
|
||||
0.054,
|
||||
0.068,
|
||||
0.081,
|
||||
0.092
|
||||
],
|
||||
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||
0.118,
|
||||
0.146,
|
||||
0.167,
|
||||
0.2
|
||||
],
|
||||
"transient_error_count": 0,
|
||||
"first_transient_errors": []
|
||||
},
|
||||
{
|
||||
"phase": "full-restart-admin",
|
||||
"seconds_from_start": 2.284,
|
||||
"first_online_by_coordinator": [
|
||||
2.143,
|
||||
2.193,
|
||||
0.849,
|
||||
0.902
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "full-restart-canary",
|
||||
"attempts": 1,
|
||||
"puts": 4,
|
||||
"gets": 16,
|
||||
"hard_deadline_seconds": 60,
|
||||
"gate_seconds": 0.191,
|
||||
"seconds_from_start": 2.476,
|
||||
"first_put_seconds_after_admin_by_coordinator": [
|
||||
0.016,
|
||||
0.029,
|
||||
0.044,
|
||||
0.057
|
||||
],
|
||||
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||
0.093,
|
||||
0.124,
|
||||
0.158,
|
||||
0.192
|
||||
],
|
||||
"transient_error_count": 0,
|
||||
"first_transient_errors": []
|
||||
},
|
||||
{
|
||||
"phase": "readback-15s",
|
||||
"seconds_after_canary": 15.235,
|
||||
"objects": 8,
|
||||
"reads": 32,
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"phase": "readback-30s",
|
||||
"seconds_after_canary": 30.356,
|
||||
"objects": 8,
|
||||
"reads": 32,
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"phase": "readback-60s",
|
||||
"seconds_after_canary": 60.192,
|
||||
"objects": 8,
|
||||
"reads": 32,
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"phase": "one-node-outage",
|
||||
"existing_read": true,
|
||||
"put": true,
|
||||
"readers": 3
|
||||
},
|
||||
{
|
||||
"phase": "rejoin-admin",
|
||||
"seconds_from_start": 0.903,
|
||||
"first_online_by_coordinator": [
|
||||
0.791,
|
||||
0.829,
|
||||
0.865,
|
||||
0.903
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "rejoin-canary",
|
||||
"attempts": 1,
|
||||
"puts": 4,
|
||||
"gets": 16,
|
||||
"hard_deadline_seconds": 60,
|
||||
"gate_seconds": 0.141,
|
||||
"seconds_from_start": 1.044,
|
||||
"first_put_seconds_after_admin_by_coordinator": [
|
||||
0.012,
|
||||
0.022,
|
||||
0.035,
|
||||
0.045
|
||||
],
|
||||
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||
0.069,
|
||||
0.096,
|
||||
0.12,
|
||||
0.141
|
||||
],
|
||||
"transient_error_count": 0,
|
||||
"first_transient_errors": []
|
||||
},
|
||||
{
|
||||
"phase": "final-readback",
|
||||
"seconds_after_canary": 0.301,
|
||||
"objects": 13,
|
||||
"reads": 52,
|
||||
"errors": []
|
||||
}
|
||||
],
|
||||
"acknowledged_objects": 13,
|
||||
"version_ids_recorded": true,
|
||||
"sha256_recorded": true
|
||||
},
|
||||
{
|
||||
"runid": "silo-v1-current-6a14eb18",
|
||||
"version": "current",
|
||||
"image": "pgsty/d12a:build",
|
||||
"image_id": "sha256:307af7711e2e04ab75759cb42a1eef45c43c4404894c0e30dd19f742b107b922",
|
||||
"platform": "linux/arm64",
|
||||
"nodes": 4,
|
||||
"drives": 4,
|
||||
"filesystem": "Linux tmpfs named volumes, held mounted across server restarts",
|
||||
"drive_bytes": 268435456,
|
||||
"status": "PASS",
|
||||
"binary_sha256": "1e4cd7b78ecfa1b0cf28f1961d48a220a712c6be1fd3a01b60ee99aec62f04a4",
|
||||
"source_revision": "b32f2d9dd01a383a9991d34ffe248063463a031d+pool-consistency",
|
||||
"phases": [
|
||||
{
|
||||
"phase": "startup-admin",
|
||||
"seconds_from_start": 7.374,
|
||||
"first_online_by_coordinator": [
|
||||
7.207,
|
||||
6.677,
|
||||
6.74,
|
||||
6.8
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "startup-canary",
|
||||
"attempts": 1,
|
||||
"puts": 4,
|
||||
"gets": 16,
|
||||
"hard_deadline_seconds": 60,
|
||||
"gate_seconds": 0.479,
|
||||
"seconds_from_start": 7.854,
|
||||
"first_put_seconds_after_admin_by_coordinator": [
|
||||
0.266,
|
||||
0.279,
|
||||
0.292,
|
||||
0.305
|
||||
],
|
||||
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||
0.336,
|
||||
0.378,
|
||||
0.431,
|
||||
0.479
|
||||
],
|
||||
"transient_error_count": 0,
|
||||
"first_transient_errors": []
|
||||
},
|
||||
{
|
||||
"phase": "full-restart-admin",
|
||||
"seconds_from_start": 2.461,
|
||||
"first_online_by_coordinator": [
|
||||
2.264,
|
||||
1.414,
|
||||
2.396,
|
||||
2.461
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "full-restart-canary",
|
||||
"attempts": 36,
|
||||
"puts": 4,
|
||||
"gets": 16,
|
||||
"hard_deadline_seconds": 60,
|
||||
"gate_seconds": 14.489,
|
||||
"seconds_from_start": 16.951,
|
||||
"first_put_seconds_after_admin_by_coordinator": [
|
||||
0.013,
|
||||
0.025,
|
||||
14.301,
|
||||
0.046
|
||||
],
|
||||
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||
14.358,
|
||||
14.394,
|
||||
14.435,
|
||||
14.489
|
||||
],
|
||||
"transient_error_count": 104,
|
||||
"first_transient_errors": [
|
||||
{
|
||||
"attempt": 1,
|
||||
"operation": "put",
|
||||
"node": 2,
|
||||
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||
},
|
||||
{
|
||||
"attempt": 1,
|
||||
"operation": "get",
|
||||
"node": 2,
|
||||
"key": "full-restart-attempt-1-node-0",
|
||||
"error": "An error occurred (SlowDownRead) when calling the GetObject operation (reached max retries: 0): Resource requested is unreadable, please reduce your request rate"
|
||||
},
|
||||
{
|
||||
"attempt": 1,
|
||||
"operation": "get",
|
||||
"node": 2,
|
||||
"key": "full-restart-attempt-1-node-3",
|
||||
"error": "An error occurred (SlowDownRead) when calling the GetObject operation (reached max retries: 0): Resource requested is unreadable, please reduce your request rate"
|
||||
},
|
||||
{
|
||||
"attempt": 2,
|
||||
"operation": "put",
|
||||
"node": 2,
|
||||
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "readback-15s",
|
||||
"started_seconds_after_canary": 15.0,
|
||||
"seconds_after_canary": 22.006,
|
||||
"objects": 113,
|
||||
"reads": 452,
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"phase": "readback-30s",
|
||||
"started_seconds_after_canary": 30.003,
|
||||
"seconds_after_canary": 36.571,
|
||||
"objects": 113,
|
||||
"reads": 452,
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"phase": "readback-60s",
|
||||
"started_seconds_after_canary": 60.0,
|
||||
"seconds_after_canary": 65.743,
|
||||
"objects": 113,
|
||||
"reads": 452,
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"phase": "one-node-outage",
|
||||
"existing_read": true,
|
||||
"put": true,
|
||||
"readers": 3
|
||||
},
|
||||
{
|
||||
"phase": "rejoin-admin",
|
||||
"seconds_from_start": 2.49,
|
||||
"first_online_by_coordinator": [
|
||||
1.165,
|
||||
2.417,
|
||||
1.646,
|
||||
2.087
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "rejoin-canary",
|
||||
"attempts": 37,
|
||||
"puts": 4,
|
||||
"gets": 16,
|
||||
"hard_deadline_seconds": 60,
|
||||
"gate_seconds": 13.977,
|
||||
"seconds_from_start": 16.468,
|
||||
"first_put_seconds_after_admin_by_coordinator": [
|
||||
0.013,
|
||||
0.024,
|
||||
0.035,
|
||||
13.889
|
||||
],
|
||||
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||
13.909,
|
||||
13.927,
|
||||
13.953,
|
||||
13.977
|
||||
],
|
||||
"transient_error_count": 36,
|
||||
"first_transient_errors": [
|
||||
{
|
||||
"attempt": 1,
|
||||
"operation": "put",
|
||||
"node": 3,
|
||||
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||
},
|
||||
{
|
||||
"attempt": 2,
|
||||
"operation": "put",
|
||||
"node": 3,
|
||||
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||
},
|
||||
{
|
||||
"attempt": 3,
|
||||
"operation": "put",
|
||||
"node": 3,
|
||||
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||
},
|
||||
{
|
||||
"attempt": 4,
|
||||
"operation": "put",
|
||||
"node": 3,
|
||||
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "final-readback",
|
||||
"started_seconds_after_canary": 0.0,
|
||||
"seconds_after_canary": 6.507,
|
||||
"objects": 226,
|
||||
"reads": 904,
|
||||
"errors": []
|
||||
}
|
||||
],
|
||||
"acknowledged_objects": 226,
|
||||
"version_ids_recorded": true,
|
||||
"sha256_recorded": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,327 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded four-container restart/readback acceptance for pgsty/silo#116."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
|
||||
ROOT = None
|
||||
MCLI = None
|
||||
CURRENT_BINARY = None
|
||||
SOURCE_REVISION = None
|
||||
IMAGES = {
|
||||
'0806': 'pgsty/silo:RELEASE.2026-08-06T00-00-00Z',
|
||||
'0903': 'pgsty/silo:RELEASE.2026-09-03T13-18-01Z',
|
||||
'current': 'pgsty/d12a:build',
|
||||
}
|
||||
|
||||
|
||||
def docker(*args, timeout=40, check=True):
|
||||
return subprocess.run(['docker', *args], check=check, capture_output=True, text=True, timeout=timeout)
|
||||
|
||||
|
||||
class Deadline(BaseException):
|
||||
pass
|
||||
|
||||
|
||||
def bounded(seconds, operation):
|
||||
def expired(*_):
|
||||
raise Deadline(f'hard deadline of {seconds}s exceeded')
|
||||
old = signal.signal(signal.SIGALRM, expired)
|
||||
signal.setitimer(signal.ITIMER_REAL, seconds)
|
||||
try:
|
||||
return operation(time.monotonic() + seconds)
|
||||
finally:
|
||||
signal.setitimer(signal.ITIMER_REAL, 0)
|
||||
signal.signal(signal.SIGALRM, old)
|
||||
|
||||
|
||||
def remaining(deadline):
|
||||
value = deadline - time.monotonic()
|
||||
if value <= 0:
|
||||
raise Deadline('absolute deadline exceeded')
|
||||
return value
|
||||
|
||||
|
||||
def run(version):
|
||||
image_info = json.loads(docker('image', 'inspect', IMAGES[version]).stdout)[0]
|
||||
runid = f'silo-v1-{version}-{uuid.uuid4().hex[:8]}'
|
||||
out = ROOT / runid
|
||||
out.mkdir(mode=0o700)
|
||||
user, password = 'local116', secrets.token_urlsafe(24)
|
||||
envfile = out / 'credentials.env'
|
||||
envfile.write_text(f'MINIO_ROOT_USER={user}\nMINIO_ROOT_PASSWORD={password}\nMINIO_CI_CD=1\nMINIO_BROWSER=off\nGOMAXPROCS=2\n')
|
||||
envfile.chmod(0o600)
|
||||
env = {k: v for k, v in os.environ.items() if not k.startswith(('MINIO_', 'SILO_', 'MC_'))
|
||||
and k.lower() not in {'http_proxy', 'https_proxy', 'all_proxy', 'no_proxy'}}
|
||||
nodes = [f'{runid}-n{i}' for i in range(4)]
|
||||
sockets = [socket.socket() for _ in nodes]
|
||||
for sock in sockets:
|
||||
sock.bind(('127.0.0.1', 0))
|
||||
ports = [sock.getsockname()[1] for sock in sockets]
|
||||
for sock in sockets:
|
||||
sock.close()
|
||||
volumes = [n + '-data' for n in nodes]
|
||||
endpoints, ledger = [], []
|
||||
result = {'runid': runid, 'version': version, 'image': IMAGES[version],
|
||||
'image_id': image_info['Id'], 'platform': image_info['Os']+'/'+image_info['Architecture'],
|
||||
'nodes': 4,
|
||||
'drives': 4, 'filesystem': 'Linux tmpfs named volumes, held mounted across server restarts',
|
||||
'drive_bytes': 268435456, 'phases': [], 'status': 'RUNNING'}
|
||||
if version == 'current':
|
||||
result['binary_sha256'] = hashlib.sha256(CURRENT_BINARY.read_bytes()).hexdigest()
|
||||
result['source_revision'] = SOURCE_REVISION
|
||||
bucket = 'canary-' + uuid.uuid4().hex[:10]
|
||||
|
||||
def save(event=None):
|
||||
if event is not None:
|
||||
result['phases'].append(event)
|
||||
compact = {k: (len(v) if k in ('transient_errors', 'errors') else v) for k, v in event.items()}
|
||||
print(json.dumps({'version': version, **compact}), flush=True)
|
||||
(out / 'result.json').write_text(json.dumps(result, indent=2) + '\n')
|
||||
(out / 'acknowledged.json').write_text(json.dumps(ledger, indent=2) + '\n')
|
||||
|
||||
def parallel(fn, values):
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
return list(pool.map(fn, values))
|
||||
|
||||
def client(i, deadline):
|
||||
timeout = remaining(deadline)
|
||||
return boto3.client('s3', endpoint_url=endpoints[i], aws_access_key_id=user,
|
||||
aws_secret_access_key=password, region_name='us-east-1',
|
||||
config=Config(proxies={}, signature_version='s3v4', s3={'addressing_style': 'path'},
|
||||
retries={'total_max_attempts': 1}, connect_timeout=timeout,
|
||||
read_timeout=timeout, request_checksum_calculation='when_required',
|
||||
response_checksum_validation='when_required'))
|
||||
|
||||
def admin_gate(origin, phase):
|
||||
def check(deadline):
|
||||
first = [None] * 4
|
||||
while True:
|
||||
states = []
|
||||
for i in range(4):
|
||||
try:
|
||||
p = subprocess.run([str(MCLI), '--config-dir', str(out / 'mcli'), '--json',
|
||||
'admin', 'info', f'n{i}'], env=env, capture_output=True,
|
||||
text=True, timeout=min(5, remaining(deadline)))
|
||||
info = json.loads(p.stdout)['info']
|
||||
servers = info['servers']
|
||||
ok = len(servers) == 4 and all(s.get('state') == 'online' and s.get('drives')
|
||||
and all(d.get('state') == 'ok' for d in s['drives'])
|
||||
and all(v == 'online' for v in s.get('network', {}).values()) for s in servers)
|
||||
if ok:
|
||||
(out / f'{phase}-admin-{i}.json').write_text(json.dumps(info, indent=2) + '\n')
|
||||
if first[i] is None:
|
||||
first[i] = round(time.monotonic() - origin, 3)
|
||||
states.append(ok)
|
||||
except (Exception,):
|
||||
states.append(False)
|
||||
if all(states):
|
||||
event = {'phase': phase + '-admin', 'seconds_from_start': round(time.monotonic()-origin, 3),
|
||||
'first_online_by_coordinator': first}
|
||||
save(event)
|
||||
return time.monotonic()
|
||||
time.sleep(min(.25, remaining(deadline)))
|
||||
return bounded(90, check)
|
||||
|
||||
def canary(phase, origin, admin_time, setup=False):
|
||||
def check(deadline):
|
||||
started = time.monotonic()
|
||||
first_put, first_reads = [None] * 4, [None] * 4
|
||||
errors, attempt, setup_done = [], 0, not setup
|
||||
result['active_canary'] = {'phase': phase, 'errors': errors}
|
||||
while True:
|
||||
attempt += 1
|
||||
remaining(deadline)
|
||||
if not setup_done:
|
||||
try:
|
||||
try:
|
||||
client(0, deadline).create_bucket(Bucket=bucket)
|
||||
except Exception as e:
|
||||
if 'BucketAlreadyOwnedByYou' not in str(e):
|
||||
raise
|
||||
client(0, deadline).put_bucket_versioning(Bucket=bucket, VersioningConfiguration={'Status': 'Enabled'})
|
||||
setup_done = True
|
||||
except Exception as e:
|
||||
errors.append({'attempt': attempt, 'operation': 'setup', 'error': str(e)[:250]})
|
||||
time.sleep(min(.25, remaining(deadline)))
|
||||
continue
|
||||
acked = []
|
||||
for i in range(4):
|
||||
key = f'{phase}-attempt-{attempt}-node-{i}'
|
||||
payload = (key + '\n').encode() * 16384
|
||||
try:
|
||||
vid = client(i, deadline).put_object(Bucket=bucket, Key=key, Body=payload)['VersionId']
|
||||
entry = {'phase': phase, 'key': key, 'version': vid, 'bytes': len(payload),
|
||||
'sha256': hashlib.sha256(payload).hexdigest(), 'writer': i,
|
||||
'ack_seconds_from_start': round(time.monotonic()-origin, 3)}
|
||||
ledger.append(entry)
|
||||
save() # Persist every acknowledged write, including failed rounds.
|
||||
acked.append(entry)
|
||||
if first_put[i] is None:
|
||||
first_put[i] = round(time.monotonic()-admin_time, 3)
|
||||
except Exception as e:
|
||||
errors.append({'attempt': attempt, 'operation': 'put', 'node': i, 'error': str(e)[:250]})
|
||||
reads = 0
|
||||
for i in range(4):
|
||||
own_reads = 0
|
||||
for entry in acked:
|
||||
try:
|
||||
verify(i, entry, deadline)
|
||||
reads += 1
|
||||
own_reads += 1
|
||||
except Exception as e:
|
||||
errors.append({'attempt': attempt, 'operation': 'get', 'node': i,
|
||||
'key': entry['key'], 'error': str(e)[:250]})
|
||||
if own_reads == 4 and first_reads[i] is None:
|
||||
first_reads[i] = round(time.monotonic()-admin_time, 3)
|
||||
remaining(deadline)
|
||||
if len(acked) == 4 and reads == 16:
|
||||
result.pop('active_canary', None)
|
||||
save({'phase': phase + '-canary', 'attempts': attempt, 'puts': 4, 'gets': 16,
|
||||
'hard_deadline_seconds': 60, 'gate_seconds': round(time.monotonic()-started, 3),
|
||||
'seconds_from_start': round(time.monotonic()-origin, 3),
|
||||
'first_put_seconds_after_admin_by_coordinator': first_put,
|
||||
'first_four_reads_seconds_after_admin_by_coordinator': first_reads, 'transient_errors': errors})
|
||||
return time.monotonic()
|
||||
(out / f'{phase}-canary-errors.json').write_text(json.dumps(errors, indent=2) + '\n')
|
||||
if attempt == 1:
|
||||
print(json.dumps({'version': version, 'phase': phase, 'first_attempt_errors': errors}), flush=True)
|
||||
time.sleep(min(.25, remaining(deadline)))
|
||||
return bounded(60, check)
|
||||
|
||||
def verify(i, entry, deadline):
|
||||
got = client(i, deadline).get_object(Bucket=bucket, Key=entry['key'], VersionId=entry['version'])
|
||||
try:
|
||||
data = got['Body'].read()
|
||||
finally:
|
||||
got['Body'].close()
|
||||
assert len(data) == entry['bytes'] and hashlib.sha256(data).hexdigest() == entry['sha256'], entry['key']
|
||||
assert got.get('VersionId') == entry['version'], entry['key']
|
||||
remaining(deadline)
|
||||
|
||||
def readback(label, origin):
|
||||
def check(deadline):
|
||||
started = time.monotonic()
|
||||
errors = []
|
||||
for entry in ledger:
|
||||
for i in range(4):
|
||||
try:
|
||||
verify(i, entry, deadline)
|
||||
except Exception as e:
|
||||
errors.append({'key': entry['key'], 'node': i, 'error': str(e)[:250]})
|
||||
save({'phase': label, 'started_seconds_after_canary': round(started-origin, 3),
|
||||
'seconds_after_canary': round(time.monotonic()-origin, 3),
|
||||
'objects': len(ledger), 'reads': len(ledger)*4, 'errors': errors})
|
||||
return errors
|
||||
return bounded(60, check)
|
||||
|
||||
try:
|
||||
docker('network', 'create', runid)
|
||||
for volume in volumes:
|
||||
docker('volume', 'create', '--driver', 'local', '--opt', 'type=tmpfs',
|
||||
'--opt', 'device=tmpfs', '--opt', 'o=size=256m', volume)
|
||||
mounts = [arg for i, v in enumerate(volumes) for arg in ('--mount', f'type=volume,source={v},target=/keep/{i}')]
|
||||
docker('run', '-d', '--pull=never', '--network', 'none', '--name', runid + '-keeper',
|
||||
*mounts, '--entrypoint', 'sleep', 'alpine:3.23', '1800')
|
||||
urls = [f'http://{n}:9000/data' for n in nodes]
|
||||
def create(i):
|
||||
extra = []
|
||||
if version == 'current':
|
||||
extra = ['--mount', f'type=bind,source={CURRENT_BINARY},target=/lab/silo,readonly',
|
||||
'--entrypoint', '/lab/silo']
|
||||
docker('create', '--pull=never', '--name', nodes[i], '--network', runid,
|
||||
'--hostname', nodes[i], '--cpus', '2', '--memory', '3g', '--env-file', str(envfile),
|
||||
'--mount', f'type=volume,source={volumes[i]},target=/data',
|
||||
'-p', f'127.0.0.1:{ports[i]}:9000', *extra, image_info['Id'], 'server', '--address', ':9000',
|
||||
'--console-address', ':9001', *urls)
|
||||
parallel(create, range(4))
|
||||
start = time.monotonic()
|
||||
parallel(lambda n: docker('start', n), nodes)
|
||||
for i, n in enumerate(nodes):
|
||||
endpoint = 'http://' + docker('port', n, '9000/tcp').stdout.strip()
|
||||
endpoints.append(endpoint)
|
||||
env[f'MC_HOST_n{i}'] = endpoint.replace('http://', f'http://{user}:{password}@')
|
||||
admin = admin_gate(start, 'startup')
|
||||
canary('startup', start, admin, setup=True)
|
||||
parallel(lambda n: docker('stop', '-t', '10', n), nodes)
|
||||
start = time.monotonic()
|
||||
parallel(lambda n: docker('start', n), nodes)
|
||||
admin = admin_gate(start, 'full-restart')
|
||||
gate = canary('full-restart', start, admin)
|
||||
read_errors = []
|
||||
for after in (15, 30, 60):
|
||||
time.sleep(max(0, gate + after - time.monotonic()))
|
||||
read_errors.extend(readback(f'readback-{after}s', gate))
|
||||
assert not read_errors, 'acknowledged object readback failure; see result.json'
|
||||
docker('stop', '-t', '10', nodes[3])
|
||||
def outage(deadline):
|
||||
verify(0, ledger[0], deadline)
|
||||
payload = b'acknowledged with one Linux node offline' * 16384
|
||||
key = 'one-node-outage'
|
||||
vid = client(0, deadline).put_object(Bucket=bucket, Key=key, Body=payload)['VersionId']
|
||||
entry = {'phase': 'outage', 'key': key, 'version': vid, 'bytes': len(payload),
|
||||
'sha256': hashlib.sha256(payload).hexdigest(), 'writer': 0}
|
||||
ledger.append(entry)
|
||||
save()
|
||||
for i in range(3):
|
||||
verify(i, entry, deadline)
|
||||
save({'phase': 'one-node-outage', 'existing_read': True, 'put': True, 'readers': 3})
|
||||
bounded(60, outage)
|
||||
start = time.monotonic()
|
||||
docker('start', nodes[3])
|
||||
admin = admin_gate(start, 'rejoin')
|
||||
gate = canary('rejoin', start, admin)
|
||||
assert not readback('final-readback', gate)
|
||||
result['status'] = 'PASS'
|
||||
except BaseException as e:
|
||||
result['status'] = 'FAIL'
|
||||
result['error'] = f'{type(e).__name__}: {e}'
|
||||
raise
|
||||
finally:
|
||||
save()
|
||||
for n in nodes:
|
||||
log = docker('logs', n, check=False)
|
||||
(out / (n + '.log')).write_text(log.stdout + log.stderr)
|
||||
docker('rm', '-f', n, check=False)
|
||||
docker('rm', '-f', runid + '-keeper', check=False)
|
||||
for v in volumes:
|
||||
docker('volume', 'rm', v, check=False)
|
||||
docker('network', 'rm', runid, check=False)
|
||||
envfile.unlink(missing_ok=True)
|
||||
print(json.dumps({'version': version, 'status': result['status'], 'evidence': str(out)}), flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('versions', nargs='+', choices=list(IMAGES))
|
||||
parser.add_argument('--output', type=Path, required=True, help='directory for retained evidence')
|
||||
parser.add_argument('--mcli', default=shutil.which('mcli'), help='native mcli executable')
|
||||
parser.add_argument('--current-binary', type=Path, help='Linux binary matching the Docker architecture')
|
||||
parser.add_argument('--source-revision', help='Git revision of --current-binary')
|
||||
parser.add_argument('--current-image', default=IMAGES['current'], help='cached Linux base image for current binary')
|
||||
args = parser.parse_args()
|
||||
if not args.mcli or not Path(args.mcli).is_file():
|
||||
parser.error('--mcli must point to an executable file')
|
||||
if 'current' in args.versions and (not args.current_binary or not args.current_binary.is_file()):
|
||||
parser.error('current requires --current-binary')
|
||||
ROOT = args.output.resolve()
|
||||
ROOT.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
MCLI = Path(args.mcli).resolve()
|
||||
CURRENT_BINARY = args.current_binary.resolve() if args.current_binary else None
|
||||
SOURCE_REVISION = args.source_revision
|
||||
IMAGES['current'] = args.current_image
|
||||
for version in args.versions:
|
||||
run(version)
|
||||
@@ -1,409 +0,0 @@
|
||||
# SILO #154:OIDC discovery 连接重置调查
|
||||
|
||||
> This is a dated investigation, with the source and runtime boundaries recorded
|
||||
> below. It does not establish the current dependency pins or a later release.
|
||||
> See [the current changelog](../../CHANGELOG.md) and
|
||||
> [component matrix](https://silo.pgsty.com/compatibility/versions/).
|
||||
|
||||
|
||||
前两轮调查时间:2026-09-09;公开 issue 最后核对于 07:53 UTC。第二轮补充同源码、同依赖、不同 Go 工具链的 Linux 完整 Server 对照和候选补丁认证链路验证。
|
||||
|
||||
**后续更新:用户已授权扩展至整个 SILO 技术栈并修复。现已确认 Server 其他 TLS 路径也存在同类覆盖问题,并在产品工作区完成统一使用 Go 默认曲线的修复。当前实现、验证和交付状态见 [全栈调查](go127-stack.md)。下文保留前两轮的诊断与当时的 OIDC 局部候选;“未修改产品”和“不要扩大范围”等表述仅适用于当时的调查阶段,局部候选已被后续全路径修复取代。**
|
||||
|
||||
## 判断与处理顺序
|
||||
|
||||
**目前可以确认是 Server 发出的 discovery GET 失败,随后 IAM 初始化等待,Console 初始化也被延后;还不能确认真实连接由谁、在哪个协议阶段重置。优先调查 SILO/Go 客户端与 IdP 前置 TLS 终止器、代理或 WAF 的互操作。** 现有证据不支持将其定性为证书错误、Keycloak 配置错误、JWT 校验错误或 `coreos/go-oidc` 回归。
|
||||
|
||||
有两项与版本相关、可在本地验证的 TLS 差异:
|
||||
|
||||
1. **Go 1.27 改变了 `GODEBUG=tlsmlkem=0` 与显式 `CurvePreferences` 的关系。** SILO 两个版本都显式包含 X25519MLKEM768。旧版 Go 1.26.5 会根据该环境选项移除它;Go 1.27.1 保留显式配置。在模拟拒绝 ML-KEM 的入口上,能重现“相同选项下旧 Server 启动成功,新 Server 持续 reset”。**客户是否设置过这个选项尚未知,不能把条件性复现当成客户根因。**
|
||||
2. **Go 1.27 的 ClientHello 新增 ML-DSA 签名算法。** 没有上述环境选项时,新旧版本也会发送不同的握手。人为拒绝新算法编号的入口同样能产生旧成功、新失败。真实入口是否存在这种行为尚未知。
|
||||
|
||||
另外,HTTP User-Agent 从 `MinIO` 变成了 `Silo`;若连接在 TLS 完成、GET 发出之后被重置,应优先查 WAF、User-Agent 规则和 HTTP 路由,而不是继续调整 TLS。
|
||||
|
||||
最短路径:**先在故障进程所在环境拿到实际 Go 版本、是否设置 `tlsmlkem=0`、目标 IP 和 TLS 完成与否;再针对已证实的分支处理。** 优先修正入口兼容性或错误路由。若确认是上述环境选项失效,只为 OpenID 出站请求恢复 Go 默认曲线选择,是目前最小的代码候选。现阶段不宜做全局 TLS 改动或整体依赖回退。
|
||||
|
||||
## 第二轮结论:已隔离 Go 因素,局部候选修复通过 Linux 验证
|
||||
|
||||
**可以复现一种由 Go 1.26.5 → 1.27.1 单独触发的兼容性回归。建议保留 Go、针对已确认分支修复;整体回退只用于临时恢复服务。** 这里的“已确认”指本地实验机制,仍不等于已经确认客户入口的根因。
|
||||
|
||||
四个完整 Server 都从本地源码编译为 `linux/arm64`,在现成通用 Debian 12 基础镜像的 `--network none` 容器中运行;Server、合成 IdP、Console 和 curl 共用同一 loopback 网络,没有暴露端口。没有下载或运行 Server/Console 镜像。旧源码为 `d88f46ccee345a9c2fabe2d221d9a9e56bc11aec`,当前源码为 `d1105bbb3d4a0afa33b3a4ac11b821235038ed0e`。
|
||||
|
||||
旧源码两次构建使用完全相同的 `go.mod`、`go.sum` 和实际链接模块版本,仅替换编译器;当前源码与候选补丁构建的依赖图也完全相同。实际二进制 SHA-256、`go version -m` 依赖和构建选项保存在 [Linux 证据](issue-154/linux-evidence.json)。
|
||||
|
||||
下表的入口**人为设置为见到 X25519MLKEM768 就发 TCP RST**,Server 都设置 `GODEBUG=tlsmlkem=0`:
|
||||
|
||||
| 源码与编译器 | Server 初始 ClientHello | 完整 Server 结果 | 同容器 curl |
|
||||
| --- | --- | --- | --- |
|
||||
| 同一份旧源码 + Go 1.26.5 | 275 字节,无 ML-KEM | discovery、JWKS、IAM、Console 正常;cluster 200 | HTTP/2 200 |
|
||||
| 同一份旧源码 + Go 1.27.1 | 1509 字节,仍包含 ML-KEM | `connection reset by peer`;IAM 等待;cluster 503;Console 未启动 | HTTP/2 200 |
|
||||
| 当前源码 + Go 1.27.1 | 1509 字节,仍包含 ML-KEM | 同样失败 | HTTP/2 200 |
|
||||
| 当前源码 + 局部候选补丁 + Go 1.27.1 | 287 字节,无 ML-KEM | 完整启动及合成 OIDC 登录成功 | HTTP/2 200 |
|
||||
|
||||
这将该条件下的回归定位到工具链行为,而非 Console、pkg、mc 或 `coreos/go-oidc` 升级。Go 1.27 的发布说明明确将此作为有意改变:`tlsmlkem` / `tlssecpmlkem` 只控制默认曲线集合,显式指定的集合可以继续启用这些算法。SILO 现有曲线列表显式包含该算法,因而原来的兼容开关在这条路径失效。[Go 1.27 crypto/tls 说明](https://go.dev/doc/go1.27#crypto/tls)。
|
||||
|
||||
共完成 **12 个 Linux 场景**,其中失败用例按预期失败:
|
||||
|
||||
- 正常 TLS 1.2 / P-256 / RSA / AES-256-GCM 的 IdP,旧源码用两个 Go 版本编译均正常,证明不是 Go 1.27 普遍无法连接这套 TLS。
|
||||
- 上表四个对照均符合预期;候选补丁如果不设置 `tlsmlkem=0`,仍被 ML-KEM 拒绝规则拦截。补丁恢复显式兼容选项的作用,不会自行关闭后量子算法。
|
||||
- 候选补丁在上述 TLS 1.2 兼容场景、以及无 GODEBUG 的正常 TLS 1.3 场景,都完成 Console 登录信息获取 → IdP authorization redirect → callback → token exchange → STS 凭据 → Console 会话 → 桶列表读取。登录 API 为 204,桶列表为 200。
|
||||
- 两个登录场景分别提交错误签名与错误 audience 的 JWT,登录返回 500、桶列表返回 403,没有生成会话 cookie。这里只确认认证未放行,没有将现有 500 状态码行为改为另一项修复。
|
||||
- 不信任 CA 时,候选 Server 仍因 `x509: certificate signed by unknown authority` 停在 IAM 初始化。没有关闭证书校验。
|
||||
- 不配置 OIDC 启动后,经实际 Admin API 添加同一合成 provider:当前源码失败且返回 reset;候选补丁成功。
|
||||
- 入口改为拒绝 ML-DSA 签名编号,候选补丁加 `tlsmlkem=0` 仍失败。这是另一条机制,当前补丁没有解决它。
|
||||
|
||||
认证流程由 Python 驱动真实 Console HTTP API;IdP 使用一次性合成授权码和自行签发的实验 JWT,没有客户账户。没有执行浏览器页面交互、登出、真实 Keycloak 或生产数据升级测试。TLS 1.3 对照实际协商 TLS 1.3/P-256,不是所有新增后量子曲线的互操作覆盖。
|
||||
|
||||
### 修复与回退的取舍
|
||||
|
||||
[候选补丁](issue-154/openid-default-curves.patch) 只有三个文件:增加 OpenID 专用 transport helper,将其 `TLSClientConfig.CurvePreferences` 设为 `nil`,再替换 IAM 初始化和 OpenID 配置校验两个调用点。测试时仅应用于隔离源码副本,当前产品工作区未应用;Go 版本与所有依赖保持不变。它让 Go 默认策略和现有兼容开关接管外部 IdP 的密钥交换,其他 TLS 参数继续来自现有构造函数。
|
||||
|
||||
如果客户证据确认此分支,建议采用该局部修复,并在客户实际入口复验。如果客户没有设置 `tlsmlkem=0`,它不能单独解释旧版成功:旧版默认也发送 ML-KEM。此时应继续区分 ML-DSA、其他握手变化、HTTP/WAF 规则与网络路径;不把此补丁直接宣称为 #154 的完整修复。
|
||||
|
||||
**不建议将当前产品直接改回 Go 1.26。** 实测以 Go 1.26.7 和 `GOTOOLCHAIN=local` 读取当前源码即被 `go.mod requires go >= 1.27.1` 拒绝;所固定的 Server、Console、mc 均声明 Go 1.27.1。回退需要进一步调整这些模块及可能的传递依赖,不是只换一个编译器版本。这个报错证明当前依赖图不能原样回编,并不证明经过额外适配后绝对无法回编。报告中已恢复服务的旧版可作为临时运行状态,不能把这种处置等同于完成兼容修复。
|
||||
|
||||
第二轮的运行脚本是 [run-linux.py](issue-154/run-linux.py),结果是 [linux-evidence.json](issue-154/linux-evidence.json),具体构建和运行步骤见文末。没有 issue 评论、提交、推送、合并或发布。
|
||||
|
||||
候选副本另通过 Go 1.27.1、darwin/arm64、CGO 关闭的 `go test -mod=readonly -count=1 ./internal/http ./internal/config/identity/openid`;补丁可以干净应用到调查基线。这些包级测试与上述 Linux 集成验证分别记录,不将其当成 Linux 单元测试结果。
|
||||
|
||||
## 范围、版本与公开证据
|
||||
|
||||
- 初始工作区干净,处于 detached HEAD;调查分支为 `codex/investigate-oidc-154`,基线与远端 main 均为 `d1105bbb3d4a0afa33b3a4ac11b821235038ed0e`。
|
||||
- 已读取 `/Users/vonng/pgsty/silo/AGENTS.md` 及工作区适用说明。维护范围为 PGSTY 的 Server、Console、mc、silo-pkg;上游 MinIO 仅作参考。
|
||||
- 第一轮 Server 与 transport 探针为本地源码构建的 darwin/arm64 程序,第二轮完整 Server 交叉构建为 linux/arm64;均使用 `CGO_ENABLED=0 GOWORK=off`。fixture 与 Admin 辅助程序也由本机 Go 构建。历史源码使用 `git archive` 导入隔离临时目录。没有下载或运行 Server/Console Docker 镜像,没有访问客户端点,没有改动真实服务或数据,没有评论 issue、推送、合并或发布。
|
||||
- 产品源码、`go.mod`、`go.sum` 未修改。本目录中的 Go 文件是显式运行的调查工具,带 `//go:build ignore`,不进入正常构建。
|
||||
|
||||
| 项目 | 旧版:2026-08-04 | 报告故障版:2026-09-03 | 调查时 main |
|
||||
| --- | --- | --- | --- |
|
||||
| 完整 tag | `RELEASE.2026-08-04T00-00-00Z` | `RELEASE.2026-09-03T13-18-01Z` | 无新 release 声明 |
|
||||
| 源码 commit | `d88f46ccee345a9c2fabe2d221d9a9e56bc11aec` | `9b11dc9469e650815b775cb47b039610644f5da4` | `d1105bbb3d4a0afa33b3a4ac11b821235038ed0e` |
|
||||
| go.mod / Docker 构建定义 | Go 1.26.5 | Go 1.27.1 | Go 1.27.1 |
|
||||
| 本地 Server / 探针实际编译器 | Go 1.26.5 | Go 1.27.1 | Go 1.27.1 |
|
||||
| Console replacement | `v0.0.0-20260804042150-b952a1202869` | `v0.0.0-20260903111932-464a59d73ada` | `v0.0.0-20260908142700-c103d08ec36a` |
|
||||
| mc replacement | `v0.0.0-20260801042411-ad10a2a10b76` | `v0.0.0-20260903063637-a2ef95c035d9` | `v0.0.0-20260909015522-fcd5cad8247f` |
|
||||
| PGSTY silo-pkg | v3.11.0,替换历史 minio/pkg 路径 | v3.13.2,直接依赖 | v3.13.3,直接依赖 |
|
||||
| coreos/go-oidc/v3 | v3.17.0 | v3.21.0 | v3.21.0 |
|
||||
| x/crypto | v0.54.0 | v0.56.0 | v0.56.0 |
|
||||
| x/net | v0.57.0 | v0.58.0 | v0.58.0 |
|
||||
| x/oauth2 | v0.36.0 | v0.36.0 | v0.36.0 |
|
||||
|
||||
版本依据:[旧版 go.mod](https://github.com/pgsty/silo/blob/d88f46ccee345a9c2fabe2d221d9a9e56bc11aec/go.mod)、[故障版 go.mod](https://github.com/pgsty/silo/blob/9b11dc9469e650815b775cb47b039610644f5da4/go.mod)、[本次 main go.mod](https://github.com/pgsty/silo/blob/d1105bbb3d4a0afa33b3a4ac11b821235038ed0e/go.mod)。还核对了各 tag 的 `Dockerfile.goreleaser` 和 release workflow。它们声明相应 Go 构建版本、禁用 CGO;历史归档、本次探针和 Server 均用 `go version -m` 核实实际编译器与 replacement。**这些不是客户实际镜像 digest 或其内二进制的取证,后者仍需 `--version` / build info 确认。**
|
||||
|
||||
[Issue #154](https://github.com/pgsty/silo/issues/154) 当前 OPEN,最后更新时间 `2026-09-08T05:30:42Z`,评论数为 0。报告包含升级后 Server 初始化失败、旧版回退恢复、测试实例添加 OIDC 失败,以及 curl 成功的输出。
|
||||
|
||||
curl 那次连接验证了所收到的证书链,并选择 TLS 1.2、`ECDHE-RSA-AES256-GCM-SHA384`、P-256 和 HTTP/2;解析得到两个 IPv4,记录中实际访问了其中一个。**公开命令使用 `docker run --rm` 新建容器,并非 `docker exec` 进入原故障容器;同镜像不能证明同网络命名空间、环境变量、CA 挂载、DNS 结果或出口。** 域名、realm、IP 已脱敏,本次不推测其真实值。
|
||||
|
||||
## 实际请求链
|
||||
|
||||
### IAM 与配置校验
|
||||
|
||||
```text
|
||||
Server startup
|
||||
IAMSys.Init
|
||||
openid.LookupConfig
|
||||
parseDiscoveryDoc: GET .well-known/openid-configuration
|
||||
PopulatePublicKey: GET discovery 中的 jwks_uri
|
||||
IAM store 初始化
|
||||
Console 初始化
|
||||
|
||||
Console 添加 OIDC
|
||||
AdminClient.AddOrUpdateIDPConfig
|
||||
Server addOrUpdateIDPHandler
|
||||
validateConfig(identity_openid)
|
||||
同一个 openid.LookupConfig / NewHTTPTransport
|
||||
```
|
||||
|
||||
`parseDiscoveryDoc` 是 SILO 自己的实现,用标准库 `http.Client` 发 GET,收到成功响应后才解码 JSON;这次日志中的 `Get ... read tcp ... reset` 发生在该调用返回响应之前,不能进一步区分 TLS 与 HTTP。请求本身不需要客户 client secret、token 或私钥。[IAM 调用点](https://github.com/pgsty/silo/blob/d1105bbb3d4a0afa33b3a4ac11b821235038ed0e/cmd/iam.go#L278-L288)、[discovery 实现](https://github.com/pgsty/silo/blob/d1105bbb3d4a0afa33b3a4ac11b821235038ed0e/internal/config/identity/openid/jwt.go#L261-L285)、[JWKS 实现](https://github.com/pgsty/silo/blob/d1105bbb3d4a0afa33b3a4ac11b821235038ed0e/internal/config/identity/openid/jwt.go#L89-L110)。
|
||||
|
||||
Console 的添加表单通过 Admin API 触发 Server 配置校验。校验成功才写入配置;本地已重现该 API 返回同样的 reset,恢复 IdP 后再次创建成功并要求重启。[Console 调用](https://github.com/pgsty/silo-console/blob/c103d08ec36a/api/admin_idp.go#L79-L114)、[Admin 客户端](https://github.com/pgsty/silo-console/blob/c103d08ec36a/api/client-admin.go#L593-L595)、[Server 校验和保存边界](https://github.com/pgsty/silo/blob/d1105bbb3d4a0afa33b3a4ac11b821235038ed0e/cmd/admin-handlers-idp-config.go#L127-L150)、[OpenID 配置校验](https://github.com/pgsty/silo/blob/d1105bbb3d4a0afa33b3a4ac11b821235038ed0e/cmd/config-current.go#L353-L359)。
|
||||
|
||||
`coreos/go-oidc.NewProvider` 在 Server 中的使用是 `MockOpenIDTestUserInteraction` 测试辅助函数,不是这次 IAM discovery 调用链。升级此依赖不能单独解释或修复该 GET。`crypto/tls` 和这里的 `net/http` 属于 Go 标准库,不能用 go.mod 中 `x/crypto`、`x/net` 的版本代替它们的实际行为。[辅助函数](https://github.com/pgsty/silo/blob/d1105bbb3d4a0afa33b3a4ac11b821235038ed0e/cmd/utils.go#L994-L1012)。
|
||||
|
||||
### transport 参数与环境
|
||||
|
||||
| 项目 | 实际行为及意义 |
|
||||
| --- | --- |
|
||||
| 构造 | `NewHTTPTransport()` → `NewHTTPTransportWithTimeout(time.Minute)` → `xhttp.ConnSettings`。每次 IAM 初始化重试都会重新构造。 |
|
||||
| TLS 版本 | 未显式设置 Min/Max;所比较 Go 工具链的正常默认范围是 TLS 1.2–1.3。证书、主机名验证开启。 |
|
||||
| 密码套件 | 显式 `TLSCiphersBackwardCompatible()`,包含 curl 成功使用的 ECDHE-RSA/AES-256-GCM。没有理由为本 issue 添加旧 RSA、3DES 或 SHA-1 例外。 |
|
||||
| 曲线 | 显式 `{X25519MLKEM768, P256, X25519, P384, P521}`,两个 release 和当前 main 相同。实际上线顺序还受 Go 实现控制,见实验。 |
|
||||
| ALPN / HTTP | `EnableHTTP2=false`,同时设置自定义 TLS config 和 DialContext;实测 ClientHello 没有 ALPN,GET 使用 HTTP/1.1。三个版本一致。`GODEBUG=http2client=0` 对此基线路径无修复价值。 |
|
||||
| 代理 | `http.ProxyFromEnvironment`;HTTPS URL 使用 `HTTPS_PROXY` / `https_proxy` 与 `NO_PROXY` / `no_proxy`。这两版标准库均优先非空大写值,不使用 `ALL_PROXY`;设置在进程内缓存。不能根据 curl 的路由推断它。注意 go.mod 的 x/net v0.58.0 代码不是这里所用的标准库 vendored 实现。 |
|
||||
| DNS | `globalDNSCache.LookupHost`,dnscache v0.1.1;默认刷新窗口在容器/Kubernetes 为 30 秒,其他环境为 10 分钟,可配置。按返回地址逐个尝试 TCP,首次 TCP 成功就返回;随后 TLS/HTTP 失败不会回到这个循环尝试另一 IP。代码注释称随机选择,但该循环没有 shuffle。 |
|
||||
| Linux TCP 参数 | 使用 SILO 的自定义 dialer,包含 TCP fast open/keepalive 等设置,部分参数来自 Server CLI,包括 interface、buffer、user timeout。第二轮执行了完整 Linux Server 的正常 CLI 初始化,但只验证 loopback;不能排除实际出口设备、路由或非默认 CLI 参数的交互。 |
|
||||
| CA | `silo-pkg/certs.GetRootCAs` 加载系统根、Kubernetes CA 目录和 `certs/CAs`;Server 还加入自己的公开服务证书。相关 pkg CA 加载实现未在这次版本比较中变化;实际文件和路径仍可能因部署变化不同。 |
|
||||
| 超时 | TCP 拨号 5 秒,TLS 握手 10 秒,响应头 1 分钟;discovery/JWKS 的 Client 没有总超时,discovery 使用的 Request 也没有调用方 context。响应体停滞不受响应头超时保护。 |
|
||||
| 复用 | keep-alive 开启,idle 15 秒,TLS session cache 100。一次初始化内 discovery 与 JWKS 可复用连接;初始化重试的新 transport 没有旧连接或 session。持续首次握手失败不能用清理空闲连接解释。 |
|
||||
| 请求标识 | UA 包含产品、OS、架构、模式和构建信息,产品名从 `MinIO` 变为 `Silo`;传输层禁用自动压缩。UA 规则只能在 HTTPS 被终止、HTTP 请求可见之后起作用。 |
|
||||
|
||||
源码:[构造与超时](https://github.com/pgsty/silo/blob/d1105bbb3d4a0afa33b3a4ac11b821235038ed0e/cmd/utils.go#L651-L670)、[HTTP/TLS 参数](https://github.com/pgsty/silo/blob/d1105bbb3d4a0afa33b3a4ac11b821235038ed0e/internal/http/transports.go#L44-L94)、[密码套件与曲线](https://github.com/pgsty/silo/blob/d1105bbb3d4a0afa33b3a4ac11b821235038ed0e/internal/crypto/crypto.go#L52-L78)、[DNS 拨号循环](https://github.com/pgsty/silo/blob/d1105bbb3d4a0afa33b3a4ac11b821235038ed0e/internal/http/dial_dnscache.go#L42-L84)、[缓存刷新](https://github.com/pgsty/silo/blob/d1105bbb3d4a0afa33b3a4ac11b821235038ed0e/cmd/common-main.go#L551-L578)、[CA 加载](https://github.com/pgsty/silo/blob/d1105bbb3d4a0afa33b3a4ac11b821235038ed0e/cmd/server-main.go#L383-L395)、[UA](https://github.com/pgsty/silo/blob/d1105bbb3d4a0afa33b3a4ac11b821235038ed0e/cmd/update.go#L228-L266)。
|
||||
|
||||
比较旧 tag 与故障 tag,OpenID discovery、transport、曲线实现的变动仅为 pkg 导入路径调整;IAM 另有品牌日志变更。比较故障 tag 与本次 main,这些关键实现没有变动。因此不能把当前依赖推进当成 #154 已解决的证据。
|
||||
|
||||
**Console 登录阶段是另一条出站路径。** 当前及故障版 Console 通过 `GetConsoleHTTPClient` / `GlobalTransport` 再获取 discovery、交换令牌,TLS config 未指定 CurvePreferences,仍做完整证书验证;不能把“添加配置走 Server”推广为所有 Console OIDC 请求都走 Server transport。任何修复最终都必须验证完整登录。`CONSOLE_MINIO_SERVER_TLS_SKIP_VERIFY` 只针对 SILO 端点,不能作为 IdP 修复。[Console transport](https://github.com/pgsty/silo-console/blob/c103d08ec36a/api/config.go#L68-L121)、[IdP 客户端](https://github.com/pgsty/silo-console/blob/c103d08ec36a/api/tls.go#L59-L70)、[Console discovery](https://github.com/pgsty/silo-console/blob/c103d08ec36a/pkg/auth/idp/oauth2/provider.go#L428-L449)。
|
||||
|
||||
## 本地实验与能排除的假设
|
||||
|
||||
工具与原始结果放在 [issue-154/](issue-154/):
|
||||
|
||||
- `probe.go` 直接调用 `cmd.NewHTTPTransport()`,用同版本 `certs.GetRootCAs` 装入实验 CA;记录 TCP、TLS、HTTP 阶段。诊断参数仅修改被比较的一项。
|
||||
- `fixture.go` 仅监听 IPv4 loopback;使用即时生成的 RSA 测试证书与专用 CA,提供 discovery 和有效 JWKS。正常基线限制 TLS 1.2、P-256、`TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384`,支持 HTTP/1.1 和 HTTP/2。
|
||||
- `admin-check.go` 仅允许 loopback Server,验证 Console 使用的 Admin API;使用本地虚构配置和临时凭据。
|
||||
- `evidence.json` 保存版本、握手参数、分阶段 trace 和完整 Server 实验结果;无客户信息、私钥或 token。
|
||||
- 第二轮扩展 fixture 的合成授权码、token 和 JWKS 流程;`run-linux.py` 驱动 Linux 完整 Server 及 Console API,`linux-evidence.json` 保存十二组场景和构建身份。敏感运行值不写入结果。
|
||||
|
||||
### 正常服务及握手差异
|
||||
|
||||
三个源码版本的实际 transport 都能通过完整证书验证,以 TLS 1.2 / AES-256-GCM / HTTP/1.1 获取测试文档。**Go 1.27 本身并非不能连接 TLS 1.2、RSA 证书或这套密码套件。** 正常服务也接受诊断性的 HTTP/2。
|
||||
|
||||
| 构建与选项 | fixture 读到的初始握手字节数 | 支持的 group ID | 新增 ML-DSA 签名编号 |
|
||||
| --- | ---: | --- | --- |
|
||||
| 旧源码 + Go 1.26.5,默认 | 1497 | 4588, 29, 23, 24, 25 | 无 |
|
||||
| 故障源码 + Go 1.27.1,默认 | 1509 | 同上 | 0x0904, 0x0905, 0x0906 |
|
||||
| 当前源码 + Go 1.27.1,默认 | 1509 | 同上 | 同上 |
|
||||
| 旧源码 + Go 1.26.5,`tlsmlkem=0` | 275 | 29, 23, 24, 25 | 无 |
|
||||
| 故障/当前源码 + Go 1.27.1,`tlsmlkem=0` | 1509 | 4588, 29, 23, 24, 25 | 有 |
|
||||
| 旧源码、旧依赖,只改为 Go 1.27.1 | 1509 | 4588, 29, 23, 24, 25 | 有;`tlsmlkem=0` 也不再移除 4588 |
|
||||
|
||||
4588 是 X25519MLKEM768,29 是 X25519,23/24/25 是 P-256/384/521。这些字节数包含本地 fixture 收到的 TLS record,测试 URL 是 IP,没有 DNS SNI;不能直接当成客户网络中的包长或 MTU 证据。默认新旧差异约 12 字节,没有证据支持“本次才突然出现巨大 ML-KEM 握手”的说法。设置上述环境选项时的差异则显著不同。
|
||||
|
||||
旧源码保留旧依赖、仅更换 Go 编译器后,行为随编译器变化,隔离了本次发现与 silo-pkg/Console 版本更新之间的关系。Go 1.27 官方说明也明确记录显式曲线配置不再受这些默认值开关限制,并新增 ML-DSA 支持。[Go 1.27 发布说明](https://go.dev/doc/go1.27)。本地进一步核对了两版 `crypto/tls/defaults.go`、`common.go` 的 `curvePreferences`/`supportsCurve` 和 `handshake_client.go`。
|
||||
|
||||
### 主动拒绝与对照结果
|
||||
|
||||
下列拒绝规则是人为设置的模型,**只证明机制可以产生相同症状,不证明真实 Keycloak 或其入口使用这些规则**。
|
||||
|
||||
| fixture 规则 | 对照结果 | 能支持的结论 |
|
||||
| --- | --- | --- |
|
||||
| ClientHello 带 ML-KEM 就发送 TCP RST | 旧版默认也失败;旧版 + `tlsmlkem=0` 成功;故障版 + 同选项失败;故障版显式 classical 曲线成功 | 需要旧环境选项或其他变化,才能用该机制解释升级回归。仅说 ML-KEM 不兼容不充分。 |
|
||||
| ClientHello 带 ML-DSA 编号就 RST | 旧版默认成功;故障版默认/仅 classical 都失败;故障版 TLS 1.2-only 成功 | 新的签名算法列表是另一种可区分机制。ML-DSA 与 ML-KEM 不是同一项。TLS 1.2-only 会同时改变多项 ClientHello,成功不等于唯一定位 ML-DSA。 |
|
||||
| 必须提供 h2 ALPN | 旧、新 Server transport 默认都失败;`-h2` 成功 | 可以解释 curl 与 Server 的不同,单独不能解释新旧版本差异。 |
|
||||
| TLS 完成后,对 `Silo` UA 的 GET 发送 RST | 同一个新 transport,`MinIO` UA 成功、`Silo` UA 失败 | 报告的外层 `Get ... reset` 错误也可能来自 HTTP 层,必须先判断 TLS 是否完成。 |
|
||||
| 不信任测试 CA | 返回 `*tls.CertificateVerificationError` / x509 类错误 | 与人为 RST 的错误不同;没有证据要求跳过证书验证。仍须比较客户实际连接收到的链。 |
|
||||
| 正常连接复用 | 第二个请求 `reused=true`;关闭 idle 连接后重新建 TCP 可恢复 TLS session | keep-alive 与 TLS session 复用是两件事。首次新 transport 就失败时,此分支优先级低。 |
|
||||
|
||||
### 完整 Server、IAM 和恢复
|
||||
|
||||
使用三个本地编译的完整 Server,独立空数据/配置目录、独立 CA 和临时凭据;未登录真实 IdP。
|
||||
|
||||
| 场景 | 实际结果 |
|
||||
| --- | --- |
|
||||
| 旧版、故障版连接正常 IdP | discovery + JWKS 成功;一条 TLS 连接;cluster health 200;Console HTTP 200;Admin ListUsers 成功。 |
|
||||
| 当前版连接持续 reset 的 IdP | IAM 持续等待;cluster health 503,`X-Minio-Server-Status: iam-offline`;Console 端口尚未提供服务;5 秒限时的 Admin ListUsers 未完成。 |
|
||||
| 上述场景恢复 IdP,不重启 Server | 约 0.43 秒后 cluster/Console 200,ListUsers 成功。该时间是单次本地样本,不是恢复 SLA。 |
|
||||
| 旧版 + `tlsmlkem=0`,IdP 拒绝 ML-KEM | 完整启动成功。 |
|
||||
| 故障版 + `tlsmlkem=0`,相同拒绝规则 | IAM 等待、cluster 503;撤掉规则后约 1.38 秒内完整恢复,无需重启。 |
|
||||
| 当前版 discovery 成功、JWKS 返回 503 | 同样阻塞 IAM;JWKS 恢复后约 0.33 秒内恢复。只验证 discovery 200 不够。 |
|
||||
| 当前版先不配置 OIDC,再经 Admin API 添加 | IdP reset 时返回同型 `Get ... read tcp ... connection reset by peer`;恢复后同名创建成功,`restart=true`。失败校验没有保存该 provider。 |
|
||||
|
||||
在本地 IAM 阻塞的场景中,`/minio/health/live` 和 `/minio/health/ready` **仍为 200**。判断这次恢复应使用 `/minio/health/cluster` 并验证受认证操作和 Console,不能只看 ready。源码上 cluster 的 `checkHealth` 检查 IAM,而 ready 没有此检查;这是已有行为,本文不扩展为一次健康检查重构。[health 检查](https://github.com/pgsty/silo/blob/d1105bbb3d4a0afa33b3a4ac11b821235038ed0e/cmd/healthcheck-handler.go#L32-L65)、[ready 检查](https://github.com/pgsty/silo/blob/d1105bbb3d4a0afa33b3a4ac11b821235038ed0e/cmd/healthcheck-handler.go#L129-L186)。
|
||||
|
||||
现有 IAM 重试间隔随机为 0–3 秒,GET 本身还会占用网络等待时间;初始化成功前不会继续创建 IAM store,Console 又在 IAM 调用返回后启动。外部连接恢复可由现有重试自行恢复;这不意味着靠增加重试能修复持续不兼容。缺少整体请求期限、请求取消的部分是另外一个可单独修复的启动健壮性问题。[重试逻辑](https://github.com/pgsty/silo/blob/d1105bbb3d4a0afa33b3a4ac11b821235038ed0e/cmd/iam.go#L342-L365)、[Console 启动顺序](https://github.com/pgsty/silo/blob/d1105bbb3d4a0afa33b3a4ac11b821235038ed0e/cmd/server-main.go#L1007-L1027)。
|
||||
|
||||
已通过:`CGO_ENABLED=0 GOWORK=off go test -mod=readonly -count=1 ./internal/http ./internal/config/identity/openid`。这些测试没有替代真实 Keycloak 登录或 Linux 故障网络的验证。
|
||||
|
||||
## 最少补充证据与诊断命令
|
||||
|
||||
先收集第一轮,按结果才展开后续。所有请求只取公开 discovery,不需要客户端密钥、密码、token 或私钥。不要求环境变量全量导出、配置导出或证书私钥。
|
||||
|
||||
### 第一轮:运行身份、网络一致性、协议阶段
|
||||
|
||||
在**已有故障容器/Pod 的实际网络环境**中运行;旧版也做同样检查。不要以新建默认网络容器替代。如果实际进程经启动脚本修改过环境,探针也应使用修改后的相关环境和同一 CA 挂载。
|
||||
|
||||
```sh
|
||||
# 选择实际 Server 可执行文件;记录输出中的版本、Go、OS/架构。
|
||||
silo --version
|
||||
# 旧镜像的程序名可能是 minio。
|
||||
|
||||
# Linux:PID 设为实际 Server 进程 PID,不预设一定是 1。
|
||||
# 只输出 Go 调试选项以及相关环境项是否存在,不输出代理凭据/地址。
|
||||
tr '\0' '\n' < "/proc/$PID/environ" | awk '
|
||||
/^GODEBUG=/ { print; next }
|
||||
/^(HTTP_PROXY|HTTPS_PROXY|NO_PROXY|ALL_PROXY|http_proxy|https_proxy|no_proxy|all_proxy|SSL_CERT_FILE|SSL_CERT_DIR)=/ {
|
||||
split($0, a, "="); print a[1] "=<set>"
|
||||
}'
|
||||
|
||||
# OIDC_URL 仅在本地设为原 config_url;URL 不应带凭据或令牌。
|
||||
export OIDC_URL
|
||||
curl --http1.1 --connect-timeout 5 --max-time 20 -sS -o /dev/null \
|
||||
-w 'ip=%{remote_ip} http=%{http_version} status=%{http_code} verify=%{ssl_verify_result}\n' "$OIDC_URL"
|
||||
curl --http2 --connect-timeout 5 --max-time 20 -sS -o /dev/null \
|
||||
-w 'ip=%{remote_ip} http=%{http_version} status=%{http_code} verify=%{ssl_verify_result}\n' "$OIDC_URL"
|
||||
|
||||
# 由维护者从对应源码构建的探针;OIDC_CA 是与 Server 相同的 CAs 目录。
|
||||
./oidc-probe -ca "$OIDC_CA"
|
||||
```
|
||||
|
||||
探针从 `OIDC_URL` 读取 URL。URL、响应体、请求头不会打印;输出的 IP 可按一致映射替换为 IP-A/IP-B。它使用真实 Server transport 构造函数,但添加 20 秒总期限、不跟随重定向、限制响应体读取为 1 MiB,并使用 `issue-154-probe` UA;因此是定位连接阶段的工具,不是完整 OIDC 流程。它没有执行 Server 的 CLI 初始化,也不继承该进程的 `--interface`、socket buffer、TCP user timeout 或存量 DNS 缓存;若使用这些非默认参数,必须对齐后才能归因。若 Server 将自己的公开服务证书也作为根信任,需要把相同公开证书加入探针的临时 CA 目录。
|
||||
|
||||
读结果的方法:
|
||||
|
||||
- `tls_start` 后 `tls_done ... err=reset`:先查 TLS ClientHello、代理 CONNECT 或 TLS 终止设备;尚不能从客户端确定 RST 由终端还是中间设备发出。
|
||||
- `tls_done ... err=none`、`wrote_request` 后 reset:检查 HTTP/WAF/UA/入口路由。此时更换证书信任或密钥交换没有针对性。
|
||||
- 出现 x509 类错误:比较该进程实际收到的证书链、SNI、系统 CA 和自定义 CA;保持验证开启。
|
||||
- 两次 curl 的 IP 不同,或探针目标与 curl 不同:先做同 IP 对照。curl HTTP/1.1 成功也不代表 Go ClientHello 相同。
|
||||
- 仅 curl h2 成功:确认实际协商的 `http_version` 是 2,再检查入口 HTTP/1.1 支持;不要先全局启用 Server HTTP/2。
|
||||
|
||||
### 只对命中的分支做 A/B
|
||||
|
||||
```sh
|
||||
# TLS 阶段失败:仅移除 hybrid key exchange,仍支持 TLS 1.3 和验证证书。
|
||||
./oidc-probe -ca "$OIDC_CA" -classical
|
||||
|
||||
# 如果旧进程确实使用 tlsmlkem=0,验证最小候选是否恢复其效果。
|
||||
# 应保留原 GODEBUG 的其他相关选项;以下假定没有需要保留的其他值。
|
||||
GODEBUG=tlsmlkem=0 ./oidc-probe -ca "$OIDC_CA" -default-curves
|
||||
|
||||
# 只有 classical 仍失败时,作为鉴别实验测试 TLS 1.2-only。
|
||||
./oidc-probe -ca "$OIDC_CA" -tls12
|
||||
|
||||
# 只有 HTTP/ALPN 对照指向此分支时才测试。
|
||||
./oidc-probe -ca "$OIDC_CA" -h2
|
||||
|
||||
# HTTP 阶段才失败:UA_OLD / UA_NEW 是两版实际请求的完整 UA,非密钥。
|
||||
./oidc-probe -ca "$OIDC_CA" -ua "$UA_OLD"
|
||||
./oidc-probe -ca "$OIDC_CA" -ua "$UA_NEW"
|
||||
```
|
||||
|
||||
若 first-hop 使用代理,先比较两进程的代理选择和 `NO_PROXY`,不公开含密码的代理 URL。只在明确允许直连的部署中使用 `-direct`。确认直连后,可针对每个 DNS 地址运行以下两项,保留原 URL 主机名与 SNI,不能直接把 HTTPS URL 改成 IP:
|
||||
|
||||
```sh
|
||||
curl --noproxy '*' --resolve "$OIDC_HOST:443:$IP_A" --http1.1 \
|
||||
--connect-timeout 5 --max-time 20 -sS -o /dev/null \
|
||||
-w 'ip=%{remote_ip} status=%{http_code} verify=%{ssl_verify_result}\n' "$OIDC_URL"
|
||||
./oidc-probe -ca "$OIDC_CA" -direct -ip "$IP_A"
|
||||
# 对 IP-B 重复;端口不是 443 时据实修改 --resolve。
|
||||
```
|
||||
|
||||
仅第二次请求失败时才补 `-n 2` 与 `-n 2 -fresh`;后者重建 TCP,但同一 transport 的 TLS session cache 仍保留。若仍无法区分,下一步要的是入口侧同一时间窗口的握手失败原因/命中规则,或由客户自行脱敏后的 ClientHello 参数和 RST 阶段,不是完整认证流量包。
|
||||
|
||||
## 条件性最小修复方案
|
||||
|
||||
### 1. 路由、代理或入口策略差异已证实
|
||||
|
||||
优先统一有问题的 IdP 入口、修正具体域名的代理/NO_PROXY 或后端节点配置;更新错误拒绝合法 ClientHello 的 TLS 终止器/WAF。证书仍按原 hostname 和有效 CA 验证,OIDC issuer 不随意改名。若是 `Silo` UA 被规则拒绝,调整该规则;不把全产品 UA 改回 MinIO。
|
||||
|
||||
这是配置层处理,可以不改 SILO。工作量取决于入口归属;成功标准是原版故障二进制在原网络环境中完成 discovery、JWKS 和完整登录,而非仅 curl 200。
|
||||
|
||||
### 2. 证实旧环境依赖 `tlsmlkem=0`,且 classical / default-curves 对照成功
|
||||
|
||||
最小代码候选是为外部 OpenID 请求使用 Go 的默认曲线集合,恢复已有 GODEBUG 选项的效果;保留当前 trust pool、SNI、密码套件、超时、代理及 HTTP/1.1 行为。候选代码如下,**已在隔离副本完成上述验证,尚未应用到产品工作区**:
|
||||
|
||||
```go
|
||||
func NewOpenIDHTTPTransport() *http.Transport {
|
||||
tr := NewHTTPTransport()
|
||||
tr.TLSClientConfig.CurvePreferences = nil
|
||||
return tr
|
||||
}
|
||||
```
|
||||
|
||||
当时的候选只将 `cmd/iam.go` 和 `cmd/config-current.go` 两处 OpenID 调用改用该 helper,通过现有 UA wrapper 传入 `openid.LookupConfig`。后续排查确认节点互联、复制、远端存储等连接也存在同样的问题,此局部方案已被 [SILO 跨组件修复](go127-stack.md) 取代;请勿再应用下面归档的 OIDC-only 补丁。
|
||||
|
||||
候选已在真实 transport 探针及第二轮完整 Linux Server 中验证:`CurvePreferences=nil` + Go 1.27.1 + `tlsmlkem=0` 会移除 ML-KEM,并通过 `reject-mlkem` fixture、配置添加和合成登录。它仍不能通过 `reject-mldsa` fixture,说明此方案针对的是一个明确分支。
|
||||
|
||||
影响需要明确:
|
||||
|
||||
- 不设置 GODEBUG 时会采用完整 Go 默认集合,实测额外提供 SecP256r1MLKEM768 和 SecP384r1MLKEM1024;这也是一项兼容性变化。第二轮默认 TLS 1.3 登录通过,仍不能称为完全无行为变化或所有曲线均已覆盖。
|
||||
- `GODEBUG` 是进程级设置,其他使用 Go 默认曲线的客户端也可能受到影响;本次 helper 的改动范围是 OpenID,但该环境选项本身不是逐 provider 开关。
|
||||
- 禁用 hybrid 后仍有标准 ECDHE/TLS 和证书验证,失去的是相应后量子密钥交换保护。优先修正入口,临时兼容设置应有撤销条件;不能自动遇到 reset 就降级重试。
|
||||
- 当前 Console IdP transport 本来使用默认曲线,同一进程中的该 GODEBUG 策略可以与之保持一致;第二轮合成登录链路已经验证,真实 IdP 和浏览器验收仍待进行。
|
||||
- 隔离源码副本中的候选实现及上述本地验证已完成;真实端点 A/B 证据、生产适用性和发布是尚未完成的步骤。
|
||||
|
||||
### 3. 未设置上述选项,或 classical 仍失败
|
||||
|
||||
不要套用方案 2。若同 IP、同代理、同 UA 下仅 Go 1.27 失败,优先收集入口对新签名算法/扩展的处理证据。`-tls12` 成功只能缩小到 ClientHello/TLS 特征集合;它同时移除 hybrid key share 和 TLS 1.3 特有签名编号,不能唯一证明 ML-DSA。
|
||||
|
||||
优先更新入口或获得 Go 上游可复现用例。只有真实证据证明 TLS 1.2 兼容模式是必要且有效、入口又短期无法处理时,才评估**明确限定于该 IdP**的临时 TLS 1.2 选项,并保留现代 ECDHE/AEAD 和证书验证。不要把 `MaxVersion=TLS12` 全局写死,不引入自定义 ClientHello/TLS 栈或未受支持的“关闭 ML-DSA”环境选项。该分支尚不足以确定补丁或工期。
|
||||
|
||||
### 4. 启动健壮性可单独改进
|
||||
|
||||
如需小幅改善可诊断性,应单独给 discovery/JWKS 加明确阶段标识和有限总请求期限,让 IAM 重试可感知取消;在请求完成前发生 stall 时及时释放资源。日志避免输出 client secret/token,错误保留原始 cause。保留已配置 OIDC 的失败关闭行为及 IdP 恢复后的自动初始化,不自动关闭 OIDC,也不把无限重试包装成根因修复。
|
||||
|
||||
这类改动约 0.5–1 个工程日,可分别回归 stalled body、取消、重试后恢复;它不会使持续 RST 的 TLS 连接成功,不应替代前面鉴别。
|
||||
|
||||
## 临时处置与验收边界
|
||||
|
||||
当前可沿用报告中已经恢复服务的旧版回退状态,尽快完成上述定位。不要将旧版本长期保留视为解决方案,也不要为了它整体降级新版本依赖。本次空数据实验不证明任意生产数据/配置的降级兼容;再次切换版本前应沿既有备份和升级边界操作。
|
||||
|
||||
修复应按以下范围验证,均从 PGSTY 源码本地构建:
|
||||
|
||||
1. 对确认的分支增加最小回归:实际 OpenID transport、匹配的 ClientHello/入口拒绝条件、默认设置与明确 opt-out;确保通用和 internode transport 不变。
|
||||
2. TLS 1.2(报告密码套件)与 TLS 1.3、有效自定义 CA、错误 CA 和 hostname 拒绝;适用时覆盖代理及多地址入口。不放宽证书、JWT 签名或 audience 等认证校验。
|
||||
3. 完整 Server 的 discovery 与 JWKS、IdP 中断后恢复、cluster health、受认证 Admin 操作;Console 添加配置、浏览器重定向、回调、令牌交换、STS 授权和登出。mc 添加/读取配置路径应一致。
|
||||
4. 在实际 Linux 容器/Pod 网络和每个 IdP 入口地址复验。证据应绑定最终 Server/Console/pkg/mc commit 和编译器;发布、镜像及生产可用性属于后续独立验收。
|
||||
|
||||
第二轮完成 Linux loopback 上的合成 OAuth token/STS 登录;本次没有真实 Keycloak、真实 Linux 故障路径、浏览器页面或生产数据升级测试。当前 main 对正常 fixture 成功,并不能据此宣告 #154 修复。下一项最有价值的新增证据是**实际进程的 `tlsmlkem` 设置和一个带 TLS 完成事件的同环境 GET trace**。
|
||||
|
||||
## 复现实验
|
||||
|
||||
从本任务 worktree 根目录执行;工具均使用本地源码,输出目录必须是新建临时目录。
|
||||
|
||||
```sh
|
||||
LAB_DIR=$(mktemp -d)
|
||||
CGO_ENABLED=0 GOWORK=off go build -mod=readonly -tags kqueue \
|
||||
-o "$LAB_DIR/silo" .
|
||||
CGO_ENABLED=0 GOWORK=off go build -mod=readonly -tags kqueue \
|
||||
-o "$LAB_DIR/probe" docs/investigations/issue-154/probe.go
|
||||
go build -o "$LAB_DIR/fixture" docs/investigations/issue-154/fixture.go
|
||||
go build -mod=readonly -o "$LAB_DIR/admin-check" docs/investigations/issue-154/admin-check.go
|
||||
"$LAB_DIR/fixture" -dir "$LAB_DIR/idp" > "$LAB_DIR/hello.jsonl" 2> "$LAB_DIR/fixture.log" &
|
||||
FIXTURE_PID=$!
|
||||
trap 'kill "$FIXTURE_PID" 2>/dev/null || true' EXIT
|
||||
attempt=0
|
||||
while [ ! -s "$LAB_DIR/idp/url" ] && [ "$attempt" -lt 50 ]; do
|
||||
sleep 0.1
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
test -s "$LAB_DIR/idp/url" || exit 1
|
||||
# 私钥仅保存在 fixture 内存。
|
||||
OIDC_URL="$(cat "$LAB_DIR/idp/url")/.well-known/openid-configuration"
|
||||
export OIDC_URL
|
||||
"$LAB_DIR/probe" -ca "$LAB_DIR/idp/ca.pem"
|
||||
printf '%s' reject-mlkem > "$LAB_DIR/idp/mode"
|
||||
GODEBUG=tlsmlkem=0 "$LAB_DIR/probe" -ca "$LAB_DIR/idp/ca.pem"
|
||||
GODEBUG=tlsmlkem=0 "$LAB_DIR/probe" -ca "$LAB_DIR/idp/ca.pem" -default-curves
|
||||
printf '%s' reject-mldsa > "$LAB_DIR/idp/mode"
|
||||
"$LAB_DIR/probe" -ca "$LAB_DIR/idp/ca.pem" -classical
|
||||
"$LAB_DIR/probe" -ca "$LAB_DIR/idp/ca.pem" -tls12
|
||||
kill "$FIXTURE_PID"
|
||||
```
|
||||
|
||||
完整 Server 实验使用上述临时目录下的 `certs/CAs` 和数据目录,设置虚构的 `MINIO_IDENTITY_OPENID_CLIENT_ID`、fixture URL、临时 root 凭据,并显式指定 `--config-dir`、`--certs-dir`、loopback API/Console 地址。用 mode 文件切换 `reset`/`bad-jwks` 到 `normal`,同时检查 cluster health 和 Admin ListUsers;结果已保存在 `evidence.json`。`admin-check add` 只操作 `LAB_SERVER=127.0.0.1:port` 的实验实例,读取临时 `LAB_USER`、`LAB_PASSWORD`、`LAB_OIDC_URL`,使用虚构 OIDC secret。
|
||||
|
||||
跨版本构建时从各 tag `git archive` 获取源码,把 `probe.go` 放入该 module 根目录;旧版探针仅将 certs 导入替换为 `github.com/minio/pkg/v3/certs`,由旧版 go.mod 选择 PGSTY v3.11.0。分别强制 `GOTOOLCHAIN=go1.26.5` / `go1.27.1`,使用 `-mod=readonly`,不修改历史 go.mod。另将旧源码原依赖用 Go 1.27.1 构建,隔离工具链因素。若为 Linux 客户端构建探针,按已确认的架构设置 `GOOS=linux GOARCH=amd64` 或 `arm64`,不使用 Server 镜像代替。
|
||||
|
||||
### 第二轮 Linux 完整 Server 对照
|
||||
|
||||
从本任务根目录运行以下 Bash 命令。需要预先准备 Go 1.26.5、1.27.1 及源码依赖缓存,以及带 Python 3、curl/OpenSSL/HTTP2 的通用 Linux 基础镜像。`GOTOOLCHAIN` 明确选定编译器,`-mod=readonly` 保留依赖版本;实际测试构建用已安装工具链的绝对路径配合 `GOTOOLCHAIN=local`,等价地避免自动切换编译器。以下固定 `arm64` 与本次实验一致。
|
||||
|
||||
```bash
|
||||
LAB_DIR=$(mktemp -d)
|
||||
mkdir -p "$LAB_DIR/old" "$LAB_DIR/head" "$LAB_DIR/candidate" "$LAB_DIR/bin" "$LAB_DIR/out"
|
||||
git archive d88f46ccee345a9c2fabe2d221d9a9e56bc11aec | tar -x -C "$LAB_DIR/old"
|
||||
git archive d1105bbb3d4a0afa33b3a4ac11b821235038ed0e | tar -x -C "$LAB_DIR/head"
|
||||
git archive d1105bbb3d4a0afa33b3a4ac11b821235038ed0e | tar -x -C "$LAB_DIR/candidate"
|
||||
git -C "$LAB_DIR/candidate" apply "$PWD/docs/investigations/issue-154/openid-default-curves.patch"
|
||||
|
||||
(cd "$LAB_DIR/old" && CGO_ENABLED=0 GOWORK=off GOOS=linux GOARCH=arm64 \
|
||||
GOTOOLCHAIN=go1.26.5 go build -mod=readonly -tags kqueue -o "$LAB_DIR/bin/old-go126" .)
|
||||
(cd "$LAB_DIR/old" && CGO_ENABLED=0 GOWORK=off GOOS=linux GOARCH=arm64 \
|
||||
GOTOOLCHAIN=go1.27.1 go build -mod=readonly -tags kqueue -o "$LAB_DIR/bin/old-go127" .)
|
||||
(cd "$LAB_DIR/head" && CGO_ENABLED=0 GOWORK=off GOOS=linux GOARCH=arm64 \
|
||||
GOTOOLCHAIN=go1.27.1 go build -mod=readonly -tags kqueue -o "$LAB_DIR/bin/head-go127" .)
|
||||
(cd "$LAB_DIR/candidate" && CGO_ENABLED=0 GOWORK=off GOOS=linux GOARCH=arm64 \
|
||||
GOTOOLCHAIN=go1.27.1 go build -mod=readonly -tags kqueue -o "$LAB_DIR/bin/candidate-go127" .)
|
||||
CGO_ENABLED=0 GOWORK=off GOOS=linux GOARCH=arm64 GOTOOLCHAIN=go1.27.1 \
|
||||
go build -mod=readonly -o "$LAB_DIR/bin/fixture" docs/investigations/issue-154/fixture.go
|
||||
CGO_ENABLED=0 GOWORK=off GOOS=linux GOARCH=arm64 GOTOOLCHAIN=go1.27.1 \
|
||||
go build -mod=readonly -o "$LAB_DIR/bin/admin-check" docs/investigations/issue-154/admin-check.go
|
||||
|
||||
# 本机已存在的通用 Debian 12 arm64 基础镜像;不拉取镜像,不映射端口。
|
||||
LINUX_BASE_IMAGE=sha256:307af7711e2e04ab75759cb42a1eef45c43c4404894c0e30dd19f742b107b922
|
||||
docker run --rm --pull=never --network none \
|
||||
--mount "type=bind,source=$LAB_DIR/bin,target=/lab/bin,readonly" \
|
||||
--mount "type=bind,source=$LAB_DIR/out,target=/lab/out" \
|
||||
--mount "type=bind,source=$PWD/docs/investigations/issue-154/run-linux.py,target=/lab/run-linux.py,readonly" \
|
||||
--entrypoint python3 "$LINUX_BASE_IMAGE" /lab/run-linux.py
|
||||
```
|
||||
|
||||
每组场景输出一行摘要,同时在独立输出目录保存 `result.json`。脚本用 `finally` 终止其 Server/fixture,容器结束自动删除。它没有导出会话 cookie、授权码、JWT、state 或临时密码;原始运行目录只用于该次隔离实验,交付证据只保留握手、结果和构建身份。
|
||||
@@ -1,41 +0,0 @@
|
||||
//go:build ignore
|
||||
|
||||
// Loopback-only lab client for the Admin API used by Console's OIDC form.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
)
|
||||
|
||||
func main() {
|
||||
endpoint := os.Getenv("LAB_SERVER")
|
||||
host, _, err := net.SplitHostPort(endpoint)
|
||||
if err != nil || host != "127.0.0.1" {
|
||||
panic("LAB_SERVER must use IPv4 loopback")
|
||||
}
|
||||
a, err := madmin.New(endpoint, os.Getenv("LAB_USER"), os.Getenv("LAB_PASSWORD"), false)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if len(os.Args) > 1 && os.Args[1] == "add" {
|
||||
restart, err := a.AddOrUpdateIDPConfig(ctx, "openid", "local154", "enable=on client_id=local154 client_secret=local154-placeholder config_url="+os.Getenv("LAB_OIDC_URL"), false)
|
||||
fmt.Printf("add restart=%v err=%v\n", restart, err)
|
||||
if err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
users, err := a.ListUsers(ctx)
|
||||
fmt.Printf("list_users count=%d err=%v\n", len(users), err)
|
||||
if err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
//go:build ignore
|
||||
|
||||
// Run only with the public, synthetic CA made by fixture.go.
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"os"
|
||||
"runtime"
|
||||
|
||||
"github.com/pgsty/silo-pkg/v3/certs"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 3 {
|
||||
panic("usage: cert-roots synthetic-ca.pem explicit-ca-path-or-empty")
|
||||
}
|
||||
data, err := os.ReadFile(os.Args[1])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
block, _ := pem.Decode(data)
|
||||
if block == nil || block.Type != "CERTIFICATE" {
|
||||
panic("expected public certificate")
|
||||
}
|
||||
certificate, err := x509.ParseCertificate(block.Bytes)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
roots, err := certs.GetRootCAs(os.Args[2])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
_, err = certificate.Verify(x509.VerifyOptions{Roots: roots})
|
||||
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{
|
||||
"go": runtime.Version(), "os": runtime.GOOS,
|
||||
"explicit_ca": os.Args[2] != "", "trusted": err == nil,
|
||||
})
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,179 +0,0 @@
|
||||
//go:build ignore
|
||||
|
||||
// Loopback-only synthetic OIDC/TLS fixture. Only disposable lab identities are used.
|
||||
// Rejection modes model hypotheses; they are not evidence about the user's IdP.
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type observedConn struct {
|
||||
net.Conn
|
||||
readBytes int
|
||||
}
|
||||
|
||||
func (c *observedConn) Read(p []byte) (int, error) {
|
||||
n, e := c.Conn.Read(p)
|
||||
c.readBytes += n
|
||||
return n, e
|
||||
}
|
||||
func (c *observedConn) reset() { _ = c.Conn.(*net.TCPConn).SetLinger(0); _ = c.Conn.Close() }
|
||||
|
||||
type observedListener struct{ net.Listener }
|
||||
|
||||
func (l observedListener) Accept() (net.Conn, error) {
|
||||
c, e := l.Listener.Accept()
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
return &observedConn{Conn: c}, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
dir := flag.String("dir", "", "isolated output directory for public CA, URL, and mode file")
|
||||
tls13 := flag.Bool("tls13", false, "allow TLS 1.3 in addition to the TLS 1.2 baseline")
|
||||
flag.Parse()
|
||||
if *dir == "" {
|
||||
panic("-dir required")
|
||||
}
|
||||
must(os.MkdirAll(*dir, 0700))
|
||||
key, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
must(err)
|
||||
root := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "issue-154 local CA"}, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(24 * time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature}
|
||||
rootDER, err := x509.CreateCertificate(rand.Reader, root, root, &key.PublicKey, key)
|
||||
must(err)
|
||||
leaf := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "localhost"}, DNSNames: []string{"localhost"}, IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, NotBefore: root.NotBefore, NotAfter: root.NotAfter, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, KeyUsage: x509.KeyUsageDigitalSignature}
|
||||
leafDER, err := x509.CreateCertificate(rand.Reader, leaf, root, &key.PublicKey, key)
|
||||
must(err)
|
||||
must(os.WriteFile(filepath.Join(*dir, "ca.pem"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: rootDER}), 0600))
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
must(err)
|
||||
base := "https://" + ln.Addr().String()
|
||||
must(os.WriteFile(filepath.Join(*dir, "url"), []byte(base), 0600))
|
||||
mode := func() string { b, _ := os.ReadFile(filepath.Join(*dir, "mode")); return strings.TrimSpace(string(b)) }
|
||||
var mu sync.Mutex
|
||||
log := func(v any) { mu.Lock(); defer mu.Unlock(); _ = json.NewEncoder(os.Stdout).Encode(v) }
|
||||
tc := &tls.Config{Certificates: []tls.Certificate{{Certificate: [][]byte{leafDER, rootDER}, PrivateKey: key}}, MinVersion: tls.VersionTLS12, MaxVersion: tls.VersionTLS12, CurvePreferences: []tls.CurveID{tls.CurveP256}, CipherSuites: []uint16{tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384}}
|
||||
if *tls13 {
|
||||
tc.MaxVersion = tls.VersionTLS13
|
||||
}
|
||||
peerConfig := tc.Clone()
|
||||
peerConfig.NextProtos = []string{"h2", "http/1.1"}
|
||||
// net/http validates HTTP/2 support before GetConfigForClient; the fixture
|
||||
// then deliberately selects only the reported AES-256 suite.
|
||||
tc.CipherSuites = append(tc.CipherSuites, tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256)
|
||||
tc.GetConfigForClient = func(chi *tls.ClientHelloInfo) (*tls.Config, error) {
|
||||
m := mode()
|
||||
c := chi.Conn.(*observedConn)
|
||||
log(map[string]any{"event": "hello", "mode": m, "bytes_read": c.readBytes, "curves": chi.SupportedCurves, "signatures": chi.SignatureSchemes, "alpn": chi.SupportedProtos, "versions": chi.SupportedVersions})
|
||||
reject := m == "reset" || m == "reject-mlkem" && slices.Contains(chi.SupportedCurves, tls.CurveID(4588)) || m == "reject-mldsa" && slices.Contains(chi.SignatureSchemes, tls.SignatureScheme(0x0904)) || m == "require-h2" && !slices.Contains(chi.SupportedProtos, "h2")
|
||||
if reject {
|
||||
c.reset()
|
||||
return nil, errors.New("synthetic ClientHello rejection")
|
||||
}
|
||||
return peerConfig, nil
|
||||
}
|
||||
server := &http.Server{TLSConfig: tc, ReadHeaderTimeout: 5 * time.Second}
|
||||
var codes sync.Map
|
||||
server.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
log(map[string]any{"event": "request", "path": r.URL.Path, "protocol": r.Proto, "tls": r.TLS.Version, "cipher": r.TLS.CipherSuite, "resumed": r.TLS.DidResume, "ua": r.UserAgent()})
|
||||
if mode() == "reject-silo-ua" && strings.HasPrefix(r.UserAgent(), "Silo") {
|
||||
c, _, e := w.(http.Hijacker).Hijack()
|
||||
if e == nil {
|
||||
c.(*tls.Conn).NetConn().(*observedConn).reset()
|
||||
}
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/authorize":
|
||||
q := r.URL.Query()
|
||||
redirect, err := url.Parse(q.Get("redirect_uri"))
|
||||
if err != nil || redirect.Scheme != "http" || redirect.Hostname() != "127.0.0.1" || redirect.Path != "/oauth_callback" || q.Get("client_id") != "local154" {
|
||||
http.Error(w, "loopback lab authorization only", 400)
|
||||
return
|
||||
}
|
||||
codeBytes := make([]byte, 18)
|
||||
_, err = rand.Read(codeBytes)
|
||||
must(err)
|
||||
code := base64.RawURLEncoding.EncodeToString(codeBytes)
|
||||
codes.Store(code, q.Get("nonce"))
|
||||
values := redirect.Query()
|
||||
values.Set("code", code)
|
||||
values.Set("state", q.Get("state"))
|
||||
redirect.RawQuery = values.Encode()
|
||||
http.Redirect(w, r, redirect.String(), http.StatusFound)
|
||||
case "/token":
|
||||
if r.Method != http.MethodPost || r.ParseForm() != nil {
|
||||
http.Error(w, "bad token request", 400)
|
||||
return
|
||||
}
|
||||
id, secret, ok := r.BasicAuth()
|
||||
if !ok {
|
||||
id, secret = r.Form.Get("client_id"), r.Form.Get("client_secret")
|
||||
}
|
||||
nonce, found := codes.LoadAndDelete(r.Form.Get("code"))
|
||||
if id != "local154" || secret != "local154-placeholder" || !found || r.Form.Get("grant_type") != "authorization_code" {
|
||||
w.WriteHeader(400)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"error": "invalid_grant"})
|
||||
return
|
||||
}
|
||||
audience := id
|
||||
if mode() == "bad-audience" {
|
||||
audience = "different-lab-client"
|
||||
}
|
||||
claims, _ := json.Marshal(map[string]any{"iss": base, "sub": "local154-user", "aud": audience, "iat": time.Now().Unix(), "exp": time.Now().Add(time.Hour).Unix(), "policy": "readwrite", "nonce": nonce})
|
||||
header := base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"RS256","kid":"local-154","typ":"JWT"}`))
|
||||
payload := header + "." + base64.RawURLEncoding.EncodeToString(claims)
|
||||
hash := sha256.Sum256([]byte(payload))
|
||||
signature, err := rsa.SignPKCS1v15(rand.Reader, key, crypto.SHA256, hash[:])
|
||||
must(err)
|
||||
if mode() == "bad-signature" {
|
||||
signature[0] ^= 1
|
||||
}
|
||||
token := payload + "." + base64.RawURLEncoding.EncodeToString(signature)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"access_token": token, "id_token": token, "token_type": "Bearer", "expires_in": 3600})
|
||||
case "/.well-known/openid-configuration":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"issuer": base, "jwks_uri": base + "/jwks", "authorization_endpoint": base + "/authorize", "token_endpoint": base + "/token", "response_types_supported": []string{"code"}, "subject_types_supported": []string{"public"}, "id_token_signing_alg_values_supported": []string{"RS256"}, "scopes_supported": []string{"openid"}})
|
||||
case "/jwks":
|
||||
if mode() == "bad-jwks" {
|
||||
http.Error(w, "synthetic JWKS outage", 503)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"keys": []any{map[string]any{"kty": "RSA", "kid": "local-154", "use": "sig", "alg": "RS256", "n": base64.RawURLEncoding.EncodeToString(key.N.Bytes()), "e": "AQAB"}}})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
})
|
||||
fmt.Fprintln(os.Stderr, base)
|
||||
must(server.ServeTLS(observedListener{ln}, "", ""))
|
||||
}
|
||||
|
||||
func must(err error) {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,40 +0,0 @@
|
||||
# Historical OIDC-only candidate; superseded by ../go127-stack.md. Do not apply on top of the stack fix.
|
||||
--- a/cmd/utils.go
|
||||
+++ b/cmd/utils.go
|
||||
@@ -654,6 +654,14 @@
|
||||
return NewHTTPTransportWithTimeout(1 * time.Minute)
|
||||
}
|
||||
|
||||
+// NewOpenIDHTTPTransport uses Go defaults for external identity-provider key exchange.
|
||||
+// This lets tlsmlkem/tlssecpmlkem configure their documented default sets.
|
||||
+func NewOpenIDHTTPTransport() *http.Transport {
|
||||
+ tr := NewHTTPTransport()
|
||||
+ tr.TLSClientConfig.CurvePreferences = nil
|
||||
+ return tr
|
||||
+}
|
||||
+
|
||||
// Default values for dial timeout
|
||||
const defaultDialTimeout = 5 * time.Second
|
||||
|
||||
--- a/cmd/iam.go
|
||||
+++ b/cmd/iam.go
|
||||
@@ -277,7 +277,7 @@
|
||||
for {
|
||||
if !openidInit {
|
||||
openidConfig, err := openid.LookupConfig(s,
|
||||
- xhttp.WithUserAgent(NewHTTPTransport(), func() string {
|
||||
+ xhttp.WithUserAgent(NewOpenIDHTTPTransport(), func() string {
|
||||
return getUserAgent(getMinioMode())
|
||||
}), xhttp.DrainBody, globalSite.Region())
|
||||
if err != nil {
|
||||
--- a/cmd/config-current.go
|
||||
+++ b/cmd/config-current.go
|
||||
@@ -352,7 +352,7 @@
|
||||
}
|
||||
case config.IdentityOpenIDSubSys:
|
||||
if _, err := openid.LookupConfig(s,
|
||||
- xhttp.WithUserAgent(NewHTTPTransport(), func() string {
|
||||
+ xhttp.WithUserAgent(NewOpenIDHTTPTransport(), func() string {
|
||||
return getUserAgent(getMinioMode())
|
||||
}), xhttp.DrainBody, globalSite.Region()); err != nil {
|
||||
return err
|
||||
@@ -1,165 +0,0 @@
|
||||
//go:build ignore
|
||||
|
||||
// Diagnostic GET using the Server's actual transport constructor.
|
||||
// Build explicitly from the SILO module root; see ../issue-154.md.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptrace"
|
||||
"net/url"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/cmd"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/pgsty/silo-pkg/v3/certs"
|
||||
)
|
||||
|
||||
func main() {
|
||||
endpoint := flag.String("url", os.Getenv("OIDC_URL"), "discovery URL; no credentials or query string")
|
||||
ca := flag.String("ca", "", "same CA file or certs/CAs directory as Server")
|
||||
h2 := flag.Bool("h2", false, "diagnostic: opt in to HTTP/2")
|
||||
classical := flag.Bool("classical", false, "diagnostic: omit hybrid key exchange only")
|
||||
defaultCurves := flag.Bool("default-curves", false, "diagnostic: let Go choose curves and honor its GODEBUG defaults")
|
||||
tls12 := flag.Bool("tls12", false, "diagnostic: TLS 1.2 only; keeps certificate verification")
|
||||
direct := flag.Bool("direct", false, "diagnostic: bypass environment proxy")
|
||||
ip := flag.String("ip", "", "diagnostic: pin destination IP, preserving Host/SNI; requires -direct")
|
||||
fresh := flag.Bool("fresh", false, "diagnostic: close idle connections between requests")
|
||||
ua := flag.String("ua", "issue-154-probe", "HTTP User-Agent; supply actual Server UA to investigate a WAF")
|
||||
n := flag.Int("n", 1, "number of GETs (1 to 3)")
|
||||
flag.Parse()
|
||||
u, err := url.Parse(*endpoint)
|
||||
if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || *n < 1 || *n > 3 {
|
||||
fmt.Fprintln(os.Stderr, "require an HTTPS URL without credentials/query/fragment and -n between 1 and 3")
|
||||
os.Exit(2)
|
||||
}
|
||||
if *ip != "" && (!*direct || net.ParseIP(*ip) == nil) {
|
||||
fmt.Fprintln(os.Stderr, "-ip requires a literal IP and -direct")
|
||||
os.Exit(2)
|
||||
}
|
||||
if *classical && *defaultCurves {
|
||||
fmt.Fprintln(os.Stderr, "choose at most one of -classical and -default-curves")
|
||||
os.Exit(2)
|
||||
}
|
||||
tr := cmd.NewHTTPTransport()
|
||||
tr.TLSClientConfig.RootCAs, err = certs.GetRootCAs(*ca)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "CA loading failed; check the local CA path")
|
||||
os.Exit(2)
|
||||
}
|
||||
if *h2 {
|
||||
tr.ForceAttemptHTTP2 = true
|
||||
}
|
||||
if *classical {
|
||||
tr.TLSClientConfig.CurvePreferences = []tls.CurveID{tls.CurveP256, tls.X25519, tls.CurveP384, tls.CurveP521}
|
||||
}
|
||||
if *defaultCurves {
|
||||
tr.TLSClientConfig.CurvePreferences = nil
|
||||
}
|
||||
if *tls12 {
|
||||
tr.TLSClientConfig.MinVersion = tls.VersionTLS12
|
||||
tr.TLSClientConfig.MaxVersion = tls.VersionTLS12
|
||||
}
|
||||
if *direct {
|
||||
tr.Proxy = nil
|
||||
}
|
||||
if *ip != "" {
|
||||
base := tr.DialContext
|
||||
tr.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
host, port, e := net.SplitHostPort(address)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if host == u.Hostname() {
|
||||
address = net.JoinHostPort(*ip, port)
|
||||
}
|
||||
return base(ctx, network, address)
|
||||
}
|
||||
}
|
||||
defer tr.CloseIdleConnections()
|
||||
var mu sync.Mutex
|
||||
log := func(format string, args ...any) { mu.Lock(); defer mu.Unlock(); fmt.Printf(format+"\n", args...) }
|
||||
log("go=%s os=%s arch=%s h2=%v classical=%v default_curves=%v tls12=%v", runtime.Version(), runtime.GOOS, runtime.GOARCH, *h2, *classical, *defaultCurves, *tls12)
|
||||
// Deliberately print no URL, headers, body, client ID, secret, or token.
|
||||
req, _ := http.NewRequest(http.MethodGet, u.String(), nil)
|
||||
proxy := "direct"
|
||||
if tr.Proxy != nil {
|
||||
p, e := tr.Proxy(req)
|
||||
if e != nil {
|
||||
log("proxy_selection_error=%T", e)
|
||||
os.Exit(2)
|
||||
}
|
||||
if p != nil {
|
||||
proxy = p.Scheme + " proxy (address omitted)"
|
||||
}
|
||||
}
|
||||
log("route=%s curves=%v", proxy, tr.TLSClientConfig.CurvePreferences)
|
||||
client := &http.Client{Transport: xhttp.WithUserAgent(tr, func() string { return *ua }), Timeout: 20 * time.Second,
|
||||
CheckRedirect: func(_ *http.Request, _ []*http.Request) error { return http.ErrUseLastResponse }}
|
||||
failed := false
|
||||
for i := 0; i < *n; i++ {
|
||||
if *fresh {
|
||||
tr.CloseIdleConnections()
|
||||
}
|
||||
log("request=%d", i+1)
|
||||
trace := &httptrace.ClientTrace{
|
||||
DNSDone: func(d httptrace.DNSDoneInfo) { log("dns_addresses=%v err=%s", d.Addrs, errorClass(d.Err)) },
|
||||
ConnectStart: func(network, addr string) { log("connect=%s %s", network, addr) },
|
||||
ConnectDone: func(_, addr string, e error) { log("connected=%s err=%s", addr, errorClass(e)) },
|
||||
TLSHandshakeStart: func() { log("tls_start") },
|
||||
TLSHandshakeDone: func(s tls.ConnectionState, e error) {
|
||||
log("tls_done=0x%x cipher=%s alpn=%q resumed=%v verified_chains=%d err=%s", s.Version, tls.CipherSuiteName(s.CipherSuite), s.NegotiatedProtocol, s.DidResume, len(s.VerifiedChains), errorClass(e))
|
||||
},
|
||||
GotConn: func(c httptrace.GotConnInfo) { log("got_conn=%s reused=%v", c.Conn.RemoteAddr(), c.Reused) },
|
||||
WroteRequest: func(w httptrace.WroteRequestInfo) { log("wrote_request err=%s", errorClass(w.Err)) },
|
||||
GotFirstResponseByte: func() { log("first_response_byte") },
|
||||
}
|
||||
r := req.Clone(httptrace.WithClientTrace(context.Background(), trace))
|
||||
resp, e := client.Do(r)
|
||||
if e != nil {
|
||||
log("get_error=%s", errorClass(e))
|
||||
failed = true
|
||||
continue
|
||||
}
|
||||
log("status=%d protocol=%s", resp.StatusCode, resp.Proto)
|
||||
_, e = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<20))
|
||||
resp.Body.Close()
|
||||
if e != nil || resp.StatusCode != http.StatusOK {
|
||||
failed = true
|
||||
log("body_error=%s", errorClass(e))
|
||||
}
|
||||
}
|
||||
if failed {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func errorClass(err error) string {
|
||||
if err == nil {
|
||||
return "none"
|
||||
}
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return "deadline"
|
||||
}
|
||||
// Error text may contain a private URL. Emit only category and concrete type.
|
||||
category := "other"
|
||||
s := err.Error()
|
||||
for _, k := range []string{"connection reset by peer", "x509:", "TLS handshake timeout", "connection refused", "EOF"} {
|
||||
if strings.Contains(s, k) {
|
||||
category = k
|
||||
break
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("%s (%T)", category, err)
|
||||
}
|
||||
@@ -1,170 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded, loopback-only full-Server comparison. See the investigation report.
|
||||
|
||||
Run in an isolated generic Linux container with locally built binaries in
|
||||
/lab/bin and a new disposable /lab/out. No customer identities or endpoints.
|
||||
"""
|
||||
import http.cookiejar
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import shutil
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path("/lab")
|
||||
OUT = ROOT / "out"
|
||||
BASE = {k: v for k, v in os.environ.items()
|
||||
if not k.startswith(("MINIO_", "SILO_", "CONSOLE_"))
|
||||
and k.lower() not in {"http_proxy", "https_proxy", "all_proxy", "no_proxy", "godebug"}}
|
||||
|
||||
|
||||
def port():
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
return sock.getsockname()[1]
|
||||
|
||||
|
||||
def request(url, opener=None, payload=None):
|
||||
headers = {"Origin": f"http://{urllib.parse.urlparse(url).netloc}"}
|
||||
if payload is not None:
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(url, data=None if payload is None else json.dumps(payload).encode(), headers=headers)
|
||||
try:
|
||||
response = (opener.open if opener else urllib.request.urlopen)(req, timeout=2)
|
||||
except urllib.error.HTTPError as err:
|
||||
response = err
|
||||
except (urllib.error.URLError, TimeoutError):
|
||||
return 0, {}, b""
|
||||
with response:
|
||||
return response.code, dict(response.headers), response.read(1 << 20)
|
||||
|
||||
|
||||
def stop(proc):
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=4)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait(timeout=2)
|
||||
|
||||
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
return None
|
||||
|
||||
|
||||
def login(console, ca):
|
||||
jar = http.cookiejar.CookieJar()
|
||||
client = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(jar))
|
||||
status, _, raw = request(console + "/api/v1/login", client)
|
||||
details = json.loads(raw)
|
||||
rules = details.get("redirectRules", [])
|
||||
assert status == 200 and len(rules) == 1, (status, details)
|
||||
auth_url = rules[0]["redirect"]
|
||||
provider = urllib.request.build_opener(NoRedirect(), urllib.request.HTTPSHandler(context=ssl.create_default_context(cafile=str(ca))))
|
||||
status, headers, _ = request(auth_url, provider)
|
||||
callback = headers.get("Location", headers.get("location", ""))
|
||||
assert status == 302 and callback.startswith(console + "/oauth_callback?"), (status, callback)
|
||||
values = urllib.parse.parse_qs(urllib.parse.urlparse(callback).query)
|
||||
callback_status, _, _ = request(callback, client)
|
||||
status, _, _ = request(console + "/api/v1/login/oauth2/auth", client,
|
||||
{"code": values["code"][0], "state": values["state"][0]})
|
||||
buckets_status, _, _ = request(console + "/api/v1/buckets", client)
|
||||
# Do not record cookies, codes, JWTs, or the state value.
|
||||
return {"callback_status": callback_status, "login_status": status,
|
||||
"buckets_status": buckets_status, "session_cookie": any(c.name == "token" for c in jar)}
|
||||
|
||||
|
||||
def run(name, binary, mode="normal", debug=None, tls13=False,
|
||||
expected=True, trusted=True, oidc=True, oauth=False, add=False):
|
||||
d = OUT / name
|
||||
(d / "certs/CAs").mkdir(parents=True, exist_ok=False)
|
||||
idp = d / "idp"
|
||||
idp.mkdir()
|
||||
(idp / "mode").write_text(mode)
|
||||
with (d / "fixture.jsonl").open("w") as events, (d / "fixture.stderr").open("w") as errors, (d / "server.log").open("w") as logs:
|
||||
fixture = subprocess.Popen([str(ROOT / "bin/fixture"), "-dir", str(idp), *(["-tls13"] if tls13 else [])], env=BASE, stdout=events, stderr=errors)
|
||||
server = None
|
||||
try:
|
||||
until = time.monotonic() + 5
|
||||
while not (idp / "url").is_file() and time.monotonic() < until:
|
||||
time.sleep(.05)
|
||||
assert (idp / "url").is_file(), "fixture did not initialize"
|
||||
url = (idp / "url").read_text() + "/.well-known/openid-configuration"
|
||||
if trusted:
|
||||
shutil.copyfile(idp / "ca.pem", d / "certs/CAs/lab.pem")
|
||||
sport, cport = port(), port()
|
||||
address = f"127.0.0.1:{sport}"
|
||||
api, console = "http://" + address, f"http://127.0.0.1:{cport}"
|
||||
password = secrets.token_urlsafe(24)
|
||||
env = dict(BASE, MINIO_ROOT_USER="local154", MINIO_ROOT_PASSWORD=password, MINIO_BROWSER="on")
|
||||
if debug:
|
||||
env["GODEBUG"] = debug
|
||||
if oidc:
|
||||
env.update(MINIO_IDENTITY_OPENID_CONFIG_URL=url,
|
||||
MINIO_IDENTITY_OPENID_CLIENT_ID="local154",
|
||||
MINIO_IDENTITY_OPENID_CLIENT_SECRET="local154-placeholder",
|
||||
MINIO_IDENTITY_OPENID_REDIRECT_URI=console + "/oauth_callback")
|
||||
server = subprocess.Popen([str(ROOT / "bin" / binary), "--config-dir", str(d / "config"), "--certs-dir", str(d / "certs"), "server", "--address", address, "--console-address", f"127.0.0.1:{cport}", str(d / "data")], env=env, stdout=logs, stderr=subprocess.STDOUT)
|
||||
until = time.monotonic() + 15
|
||||
while time.monotonic() < until:
|
||||
assert server.poll() is None, "Server exited; inspect its local log"
|
||||
status, _, _ = request(api + "/minio/health/cluster")
|
||||
if status == 200 and request(console)[0] == 200:
|
||||
break
|
||||
if not expected and (d / "server.log").read_text().count("Waiting for OpenID") >= 2:
|
||||
break
|
||||
time.sleep(.1)
|
||||
result = {"case": name, "binary": binary, "mode": mode, "godebug": debug, "tls13": tls13,
|
||||
"cluster": request(api + "/minio/health/cluster")[0],
|
||||
"ready": request(api + "/minio/health/ready")[0], "console": request(console)[0]}
|
||||
assert result["cluster"] == (200 if expected else 503), result
|
||||
aenv = dict(BASE, LAB_SERVER=address, LAB_USER="local154", LAB_PASSWORD=password, LAB_OIDC_URL=url)
|
||||
if expected:
|
||||
admin = subprocess.run([str(ROOT / "bin/admin-check")], env=aenv, capture_output=True, text=True, timeout=7)
|
||||
result["admin_list_ok"] = admin.returncode == 0
|
||||
assert result["admin_list_ok"], admin.stdout
|
||||
curl = subprocess.run(["curl", "--cacert", str(idp / "ca.pem"), "--http2", "--max-time", "3", "-sS", "-o", "/dev/null", "-w", "%{http_code} %{http_version}", url], env=BASE, capture_output=True, text=True, timeout=5)
|
||||
result["curl"] = {"exit": curl.returncode, "status_protocol": curl.stdout}
|
||||
if add:
|
||||
attempt = subprocess.run([str(ROOT / "bin/admin-check"), "add"], env=aenv, capture_output=True, text=True, timeout=7)
|
||||
result["add_ok"] = attempt.returncode == 0
|
||||
result["add_reset"] = "connection reset by peer" in attempt.stdout
|
||||
assert result["add_ok"] == binary.startswith("candidate"), result
|
||||
if oauth:
|
||||
result["oauth"] = login(console, idp / "ca.pem")
|
||||
assert result["oauth"]["login_status"] == 204 and result["oauth"]["buckets_status"] == 200, result
|
||||
for bad in ("bad-signature", "bad-audience"):
|
||||
(idp / "mode").write_text(bad)
|
||||
result[bad] = login(console, idp / "ca.pem")
|
||||
assert result[bad]["login_status"] >= 400 and result[bad]["buckets_status"] >= 400, result
|
||||
# Public handshake metadata only; no authorization parameters.
|
||||
result["events"] = [json.loads(line) for line in (d / "fixture.jsonl").read_text().splitlines()]
|
||||
(d / "result.json").write_text(json.dumps(result, indent=2) + "\n")
|
||||
print(json.dumps({k: v for k, v in result.items() if k != "events"}), flush=True)
|
||||
finally:
|
||||
if server is not None:
|
||||
stop(server)
|
||||
stop(fixture)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run("old126-normal", "old-go126")
|
||||
run("old127-normal", "old-go127")
|
||||
run("old126-compat", "old-go126", "reject-mlkem", "tlsmlkem=0")
|
||||
run("old127-compat", "old-go127", "reject-mlkem", "tlsmlkem=0", expected=False)
|
||||
run("head127-compat", "head-go127", "reject-mlkem", "tlsmlkem=0", expected=False)
|
||||
run("candidate127-compat-login", "candidate-go127", "reject-mlkem", "tlsmlkem=0", oauth=True)
|
||||
run("candidate127-no-optout", "candidate-go127", "reject-mlkem", expected=False)
|
||||
run("candidate127-tls13-login", "candidate-go127", tls13=True, oauth=True)
|
||||
run("candidate127-untrusted", "candidate-go127", trusted=False, expected=False)
|
||||
run("candidate127-mldsa", "candidate-go127", "reject-mldsa", "tlsmlkem=0", expected=False)
|
||||
run("head127-add", "head-go127", "reject-mlkem", "tlsmlkem=0", oidc=False, add=True)
|
||||
run("candidate127-add", "candidate-go127", "reject-mlkem", "tlsmlkem=0", oidc=False, add=True)
|
||||
@@ -1,103 +0,0 @@
|
||||
# Issue #77 当前核验与最小修复建议
|
||||
|
||||
> 历史快照(2026-09-12,实施前)。其中的 Issue 状态、待办和方案约束只描述当时情况;最终实现、修正与验收以 [归档总记录](issue-77.md) 为准。
|
||||
|
||||
核验日期:2026-09-12。对象:[SILO #77](https://github.com/pgsty/silo/issues/77)。代码基准:远端 `main` 的 `5c576581631561c446f30ae5b566f0aa793adc1c`,在独立 detached worktree 中运行测试。用户工作目录仍位于 `12f631b50`;两者差异为 #179 的 federation 修复,不涉及本次复制元数据代码。
|
||||
|
||||
**结论:问题真实,当前主干仍未修完,应保持打开。原方案的 A/B 核心必要,但不能照搬 8 月的实施清单;已有基础设施可以复用,同时必须补上批量复制入口和相同内容的时间戳同步。**
|
||||
|
||||
本次是分析与复现,没有修改产品代码、提交、发布或修改 GitHub Issue。下述测试是本地 ObjectLayer 与进程内 HTTP/RPC 复现,不是线上多站点验收。
|
||||
|
||||
## 1. 当前状态
|
||||
|
||||
GitHub API 实时返回 #77 为 `OPEN`,未分配负责人或 milestone,最后更新为 `2026-09-08T14:45:10Z`。核验时仓库没有打开的 PR。
|
||||
|
||||
| 范围 | 当前事实 | 是否仍属遗留项 |
|
||||
| --- | --- | --- |
|
||||
| #77-C,各站点统计 | [#91](https://github.com/pgsty/silo/pull/91) 于 8 月 29 日合并;本次现有统计回归测试通过 | 否,不应重复实现 |
|
||||
| #77-A,heal 选源与输入 | 仍先用 map 首项初始化,再跳过创建默认值;Tag 远端 heal 仍不携带时间戳 | 是 |
|
||||
| #77-B,源时间与删除状态 | 六类专用处理函数仍调用到达时间写入路径;部分删除状态不导出 | 是 |
|
||||
| #77-D,持续不一致诊断 | 已有损坏配置日志;旧事件静默跳过、默认状态无法选源等诊断未完成 | 是 |
|
||||
| Object Lock 错误 wire 字段、接管桶覆盖已有配置 | [#76](https://github.com/pgsty/silo/issues/76)、[#78](https://github.com/pgsty/silo/issues/78) 已关闭,对应 #89、#90 已合并;相关测试本次通过 | 否 |
|
||||
| 元数据整记录并发写、删除后重建 | [#103](https://github.com/pgsty/silo/pull/103)、[#156](https://github.com/pgsty/silo/pull/156) 已提供桶锁和保存前物理桶检查;删除后排队写测试本次通过 | 基础已具备,但不能代替同字段时间排序 |
|
||||
| 评论提及的 MRF 丢弃观测、delete-marker purge | [#152](https://github.com/pgsty/silo/issues/152)、[#153](https://github.com/pgsty/silo/issues/153) 已于 9 月 9 日随 [#162](https://github.com/pgsty/silo/pull/162) 关闭 | 不再列为 #77 的待实现项 |
|
||||
|
||||
#162 明确保留“405 表示 marker 仍存在”的语义,没有采用报告人建议的“405 一律判定永久删除完成”;它也未声称复现外部报告的请求量或持续 405 风暴。本次只核验其合并/关闭状态与提交说明,没有重跑那组故障实验。
|
||||
|
||||
最新公开 Release 仍是 `RELEASE.2026-09-03T13-18-01Z`。#77 的状态不能用“已有统计修复”或“已有桶锁”推导为已修复,也不能用较晚 PR 的合并推导为已发布。
|
||||
|
||||
## 2. 已复现的真实问题
|
||||
|
||||
最直接的例子:源端 10:00 PUT 配置,10:01 DELETE;目标端积压到 10:05 才接收 PUT,并把配置时间写成 10:05。随后接收源时间为 10:01 的 DELETE,判断它比本地旧,返回 HTTP 200,却保留配置。这个例子不需要机器时钟偏差,只需要事件延迟。
|
||||
|
||||
根因在 [updateAndParse](https://github.com/pgsty/silo/blob/5c576581631561c446f30ae5b566f0aa793adc1c/cmd/bucket-metadata-sys.go#L129):它在锁内统一使用 `UTCNow()`;专用 peer handler 在调用它之前,用 getter 返回的时间做比较。
|
||||
|
||||
| 配置类型 | peer PUT 保留源时间 | 较新源 DELETE | 本地删除后旧 PUT | 删除时间在元数据导出中可见 |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Policy | 否 | HTTP 200,但未删除 | 会复活 | 是 |
|
||||
| Tags | 否 | HTTP 200,但未删除 | 会复活 | 否 |
|
||||
| SSE | 否 | HTTP 200,但未删除 | 会复活 | 否 |
|
||||
| Quota | 否 | HTTP 200,但未删除 | 本次未复活,getter 保留删除时间 | 否 |
|
||||
| Versioning | 否 | nil 正确地不修改配置 | 不适用 | 不应套用可删除配置规则 |
|
||||
| Object Lock | 否 | nil 正确地不修改配置 | 不适用 | 不应套用可删除配置规则 |
|
||||
|
||||
矩阵每项均在 `ErasureSD` 与 16 盘 `Erasure` 两个本地后端执行。六类专用事件通过实际签名 admin 路由调用,检查落盘状态;导出调用实际 `SiteReplicationMetaInfo`。不能把 Quota 与 Policy/Tags/SSE 的 getter 行为写成完全相同。
|
||||
|
||||
另外复现了以下路径:
|
||||
|
||||
1. **检查与写入不在同一临界区。** 阻塞旧 Tag 事件的锁获取,在锁内提交较新 Tag,再释放旧事件;旧事件仍覆盖新内容。#103 解决跨字段丢更新,没有解决 handler 的 getter → Update 竞态。
|
||||
2. **批量元数据入口绕过逐字段排序。** 已持久化较新 Tag 后,发送较旧 bulk 事件,实际 admin 路由返回 HTTP 200,Tag 内容和时间都倒退。该入口被 [桶元数据导入](https://github.com/pgsty/silo/blob/5c576581631561c446f30ae5b566f0aa793adc1c/cmd/admin-bucket-handlers.go#L1106) 使用;[PeerBucketMetadataUpdateHandler](https://github.com/pgsty/silo/blob/5c576581631561c446f30ae5b566f0aa793adc1c/cmd/site-replication.go#L1608) 只检查桶创建时间,非 CORS 字段缺少与已有字段时间的比较。这是旧实施清单漏掉的入口。
|
||||
3. **默认值能够成为错误的 heal 来源。** 两站点状态中,一个有真实 Policy,另一个只有更晚的 `CreatedAt == PolicyUpdatedAt` 默认值。最后一次复现分别有 5/32、2/32 轮错误删除有效 Policy,取决于 Go map 遍历顺序。该计数只是复现样本,不能推断生产发生率。
|
||||
4. **远端 Tag heal 丢失源时间。** 实际 HTTP 捕获的 `UpdatedAt` 为零,见 [发送字段](https://github.com/pgsty/silo/blob/5c576581631561c446f30ae5b566f0aa793adc1c/cmd/site-replication.go#L5011)。
|
||||
5. **同内容、不同时间戳没有同步。** 即便强制 `TagMismatch=true`,内容比较仍使 heal 跳过,较旧排序时间保持不变。两站点看似内容相同,之后却可能对同一个延迟事件做出不同决定。旧方案只统一写入时间、保留全部 heal 跳过条件,仍不完整。
|
||||
6. **Quota 的 nil heal 留下缓存。** 向现有 heal 传入显式较新删除状态,磁盘 `QuotaConfigJSON` 被清空,但 `GetQuotaConfig` 仍返回 1024 字节的旧硬配额。这里显式构造了删除状态,因为当前 exporter 正在隐藏它;这是新增 tombstone 导出不能直接交给旧 heal 的具体证据。
|
||||
|
||||
从当前 `git blame` 和基线比对看,核心错误继承自 MinIO:heal 初始化逻辑来自 2022 年 `3a64580663`,专用 handler 的旧事件检查来自 2022 年 `7cc9286e0f`,bulk 入口来自 2023 年 `0cde37be50`;0806 基线已包含相关逻辑。此次没有发现其由 SILO 最近的 CORS 或统计修改引入。
|
||||
|
||||
对于使用 site replication 的部署,建议按 P1 正确性问题处理:撤销的桶策略可能保留或复活,默认加密和配额也可能与源端不一致。没有启用站点复制的正常单站点请求不触发这些复制路径;本次没有证明对象数据本体丢失。
|
||||
|
||||
## 3. 最小且必要的修复范围
|
||||
|
||||
**第一部分:收束 A 的选源和输入修复。** 提取一个小的内部选择函数,在初始化候选之前排除零时间和创建默认值;无可信候选时不写入。六类 heal 使用它,补齐 Tag 的源 `UpdatedAt`。Versioning/Object Lock 遇到 nil 保持 no-op,Quota 删除使用正确的删除路径。不能重新做 #76/#78 或把 Lifecycle、CORS 一并重构。
|
||||
|
||||
**第二部分:收束 B 的状态更新,包含本次发现的入口遗漏。** 在既有 `metadata.lock` 内完成读取原始字段时间、判旧、写入源时间和保存;普通本地写入保留现有契约。复用 `updateAndParse` 的类型分支及现有 `saveMetadata`,不再从公开 getter 获取删除时间,也不改变 getter 的 S3 错误语义。peer 更新和删除保留各自解析规则,内部返回是否应用,重复且相同的状态不重复保存。
|
||||
|
||||
本地 heal 必须使用同一排序路径,不能选好源后调用普通 `Update/Delete` 再生成到达时间。批量复制入口也必须在其现有整记录锁内逐字段判旧;bulk 的 nil 继续表示“未提供此字段”,不能改成批量删除。heal 需要同步较新的时间戳,即便内容已经一致;这不要求重写 #91 的计数或增加公开 API。
|
||||
|
||||
当前 `saveMetadata` 已在锁内检查物理桶存在,桶删除也使用同一把锁。8 月方案里另加一套 `peer-require-existing` 防幽灵桶机制已没有必要,应保留并复用现有保护。确有 `Created == 0` 的历史桶如何补齐创建时间仍应覆盖,但不能退化当前所有写入都执行的存在性检查。
|
||||
|
||||
**第三部分:单独启用删除时间导出,补足轻量诊断。** Policy 已导出时间;实际新增的是 Tags/SSE/Quota 的 nil payload 时间。先使接收与 heal 能正确处理删除,再开放这些状态。混合旧版 SILO 的风险已经有 Quota 缓存复现支撑;这是产品自身的滚动升级问题,不是要求兼容未经修改的上游 MinIO。
|
||||
|
||||
可以使用仅针对这项导出的明确 opt-in:默认关闭新增导出,全站点升级到具备 A/B 修复的版本后启用。若项目选择自动能力协商,可后续单独实现;不必为了它阻塞没有新增 wire 字段的选源与接收端修复,也不需要为本 Issue 建一个通用能力框架。具体开关尚未实现,不应把这里的建议当成现有配置。
|
||||
|
||||
诊断只需解释“过旧事件”“早于桶创建”“只有默认/未知候选”等跳过原因,使用现有有界去重/限频设施,保持 RPC 的既有成功语义。不需要另建重试队列、每轮日志或大型监控系统。
|
||||
|
||||
有三项边界必须在实现中明确:
|
||||
|
||||
- 旧版 Tag heal 会发送零时间。新实现不能无说明地全部拒绝;可保留明确的兼容降级路径,在旧节点存在时不承诺完整排序收敛。
|
||||
- 相同时间戳、不同内容的冲突:只给 heal 增加 deployment ID 平局规则,并不能让到达顺序不同的 peer apply 本身确定。若承诺此类冲突也收敛,apply/heal 应共用一个类型内比较规则;可参考 CORS 的删除优先和载荷排序,不需要新增持久化源站 ID。若不纳入本轮,必须列为剩余边界,不能称全量收敛审计完成。
|
||||
- 旧版本已经写坏的到达时间无法从现有记录还原。升级不会自动恢复历史操作顺序;应由操作者确认权威状态,并在升级和时钟检查后重新提交相关配置/删除,再验证各站点。
|
||||
|
||||
因此,建议交付为有限的选源修复、统一源时间更新、删除导出与诊断三个可审核部分。已有锁、物理桶保护和 C 的计数修复直接复用。预期产品修改集中在 Server 的 site-replication 与 bucket-metadata 路径,无须为核心修复升级 Console/mcli/silo-pkg、改存储格式或重写复制架构。
|
||||
|
||||
## 4. 验证证据与关闭条件
|
||||
|
||||
本次最后一轮执行:
|
||||
|
||||
```text
|
||||
GOWORK=off go test -tags kqueue,dev ./cmd \
|
||||
-run 'TestIssue77Current|TestSiteReplicationStatusAccountsPerSiteAndSurvivesMalformedConfig|TestPeerBucketObjectLockMetadata|TestPeerBucketAdoption|TestQueuedMetadataUpdateAfterDelete' \
|
||||
-count=1 -timeout=5m -v
|
||||
```
|
||||
|
||||
四组审计复现测试包含 24 个后端/场景子测试,均暴露预期的现存缺陷;八个现有回归测试通过。总命令退出码为 1,原因是上述针对期望正确行为的断言失败,不代表修复验收通过。未运行全量测试、race suite、真实多站点中断/重启或滚动升级实验。
|
||||
|
||||
保留的证据:
|
||||
|
||||
- [完整运行日志](issue-77/current-tests.log)
|
||||
- [源时间、删除和 bulk 审计测试](issue-77/issue77_review_test.go.txt)
|
||||
- [heal 与锁竞态审计测试](issue-77/issue77_heal_review_test.go.txt)
|
||||
|
||||
测试以 `.go.txt` 保存,避免将故意失败的审计用例加入正常 Go 测试集。可在上述 SHA 的独立 worktree 中复制为 `cmd/issue77_review_test.go` 和 `cmd/issue77_heal_review_test.go` 后重跑。创建默认值测试使用重复 map 遍历来观测缺陷,正式回归应在抽出选择函数后改成确定性用例。
|
||||
|
||||
关闭 #77 前需要:上述失败场景转为正确行为;分别覆盖六类配置、落盘重载、漏发/重复/乱序、同字段竞态、同内容新时间、批量导入入口、legacy 桶、删除后的桶及混合版本。全站点升级后的实际断线重连/重启验证应保留独立证据。当前结论是“缺陷与实施范围已核实”,不是“修复完成”。
|
||||
@@ -1,123 +0,0 @@
|
||||
# Issue #77 最小充分修复计划
|
||||
|
||||
> 历史快照(2026-09-12,实施前)。其中的 Issue 状态、待办和方案约束只描述当时情况;最终实现、修正与验收以 [归档总记录](issue-77.md) 为准。
|
||||
|
||||
版本:v4 定稿,2026-09-12。状态:作者复核完成,Claude Code `claude-opus-5 --effort max` 四轮实际评审后,最终结论为 **GO_WITH_NONBLOCKING_NOTES,实施前阻断 0**。非阻断说明已落实到本计划,见 [最终评审汇总](issue-77/review/final-review.md)。
|
||||
|
||||
基准:`pgsty/silo` main `5c576581631561c446f30ae5b566f0aa793adc1c`。本轮交付是方案与评审,不表示已经实现、合并或发布。问题证据见 [当前核验](issue-77-current.md),意见处置见 [首轮处置](issue-77/review/decisions-v2.md)、[第二轮处置](issue-77/review/decisions-v3.md)、[接管边界补查](issue-77/review/decisions-v4.md)。
|
||||
|
||||
## 目标与边界
|
||||
|
||||
修复 Policy、Tags、SSE、Quota、Versioning、Object Lock 六类桶配置的源时间丢失、锁外判旧、heal 错选源和删除传播不完整。覆盖普通本地写入、专用 peer 事件、bulk/import、local/remote heal、元数据导出与初次同步。
|
||||
|
||||
在修复 PGSTY 栈、同一已知桶世代、合法带源时间事件的范围内,使重复、乱序和漏发后的状态能够确定性收敛。开关关闭期间新增删除信息不可见,不承诺完整删除收敛;无时间旧事件、旧版污染时间和桶创建世代冲突需要单独解释,不能自动推断历史真相。
|
||||
|
||||
不重做 #91 的计数、#76 的 Object Lock wire 修复、#78 的桶接管、#103/#156 的锁与删除保护。不改变 CORS、Lifecycle/expiry、notification、对象复制、MRF、resync、IAM 的语义;不改存储 schema、SDK、Console、mcli 或 silo-pkg,不新建能力协商、复制框架、锁或重试系统。
|
||||
|
||||
## 行为契约
|
||||
|
||||
### 1. 明确事件、缺省值与删除
|
||||
|
||||
| 类型 | 专用事件 | bulk 字段未提供 | bulk 字段明确提供 |
|
||||
| --- | --- | --- | --- |
|
||||
| Policy | nil 为删除;沿用现有解析器 `IsEmpty()` 为删除 | 保留 | 非空 RawMessage 按现有解析器处理;语义空策略归一为删除 |
|
||||
| Quota | nil 为删除;非 nil 按现有 quota 解析器处理 | 保留 | 非空 RawMessage 按现有解析器处理,零 quota 仍是 live 文档 |
|
||||
| Tags / SSE | nil 或 base64 解码后空内容为删除 | 保留 | 空字符串为删除,其他内容按原规则解码、校验 |
|
||||
| Versioning / Object Lock | nil/空内容为 no-op | 保留 | nil/空内容仍为 no-op,不能清空配置 |
|
||||
|
||||
“未提供”必须依据真实 wire 类型判定:Policy/Quota 的 `json.RawMessage` 为 nil 或空切片时经 `omitempty` 省略;**显式 JSON `null` 解码后是非 nil 的 `[]byte("null")`**,不能与缺省混淆。按现有解析器,Policy `null` 是语义空策略,Quota `null` 是零值 quota 文档。`*string` 类型的 JSON `null` 则解码为 nil。不得使用统一的 `len(payload)==0 => 删除` 来处理 bulk。
|
||||
|
||||
合法 `{}`/零值 quota 保持 live;取消普通 quota PUT 出站时“零配额改写为 nil”的逻辑,保存与发送同一个语义状态。Policy 保留现有专用 peer 的“空策略=删除”解释,本地 PUT、导入和 bulk 统一归一为同样的删除状态;这是需要写入兼容说明的小范围变化:空策略本地立即按删除处理,GET 返回既有 NotFound 行为,不再先保留空文档、等复制后才被清除。
|
||||
|
||||
**同一次操作的落盘状态与出站事件,经同一归一规则后,必须具有相同的 `(kind, payload key, source time)`。** JSON 的无意义编码次序不要求字节相同;Object Lock 改写后的有效 Versioning 文档必须来自提交结果。
|
||||
|
||||
### 2. 统一状态与排序
|
||||
|
||||
内部仅需一个小状态表示:baseline / live / tombstone,以及比较键和字段源时间。baseline 可携带历史有效配置,但没有真实修改时间;它绝不能被当成删除。复用 CORS 已有设计思路,不改 CORS 本身或扩展为通用框架。
|
||||
|
||||
- 使用 `BucketMetadata` 原始字段时间;字段时间为零时在比较视图中补为 Created,与 `defaultTimestamps()` 一致,不借用会隐藏墓碑时间的 getter 或整记录 `lastUpdate()`。
|
||||
- 在已知 Created 下,零字段时间回退后等于 Created 的状态是 baseline:有有效非空配置的 baseline-live 可作为初始化候选;空 baseline 只是缺省值,不能作为删除或 heal 来源。真正严格早于自身 Created 的字段不是候选;Versioning/Object Lock 的空候选无论时间如何都不参与选源。真实 live/tombstone 的时间必须大于 Created。
|
||||
- 专用 peer 的零时间保留兼容例外:锁内按本地新操作分配时间并限频记录 legacy-zero;不受删除传播开关影响,不在源时间排序保证之内。bulk 零时间仍按现状拒绝。
|
||||
- 非零事件**严格早于**目标桶 Created 才成功/no-op 并记录 before-created,heal 对这种目标跳过。等于目标 Created 的 live 事件可更新仍是 baseline 的目标字段,时间仍保存为源 Created,不盖上到达时间;同时间的空/nil 只算空 baseline,不能删除配置。不同桶世代不能自动合并,需要运维处理,不纳入收敛承诺。
|
||||
- 排序先比较是否为真实状态:任何真实 live/tombstone 都胜 baseline,不能让较晚创建的默认值压过较早的真实修改。真实状态之间再比较源时间,同时间 tombstone 胜 live、live/live 的**稳定比较键字节序较大者胜**。baseline-live 只胜空 baseline,或在 baseline-live 之间按同一稳定键较大者胜;永远不能覆盖真实 live/tombstone。所有候选都是 baseline-live 时仍可确定性初始化并收敛;全部为空 baseline 才安静 found=false。键、状态级别和源时间相同为 no-op。deployment ID 不参与上述比较,也不作为新字段保存。
|
||||
- Quota 使用现有解析结果的 JSON 编码作为比较键。Policy 在 Server 内生成确定性比较表示:既有解析器校验/去重后,对其**完整 JSON 树**的对象键和集合数组递归排序,统一覆盖 Statement、Action/NotAction、Resource/NotResource、Principal、Condition;保留数字类型和精度。比较键必须是已解析策略的纯函数,同一配置从两个站点分别解析也必须得到相同键。普通 `json.Marshal(BucketPolicy)` 不稳定,不能直接作键;不增加依赖不支持的 NotPrincipal 语法,不改 wire/schema。新状态比较仅使用这一套键:不把忽略 Sid 且对 Statement 顺序敏感的 `BucketPolicy.Equals` 再叠加为另一套判等规则,既有公开统计对 Equals 的使用保持不变。
|
||||
- XML 使用有效文档字节,保留大小写和实际内容;Versioning 先应用下述现有 Object Lock 约束。比较器不能依靠字节序方向来补偿保存阶段的隐式改写。
|
||||
|
||||
### 3. 先得到有效状态,再比较与提交
|
||||
|
||||
原始读取、类型处理、比较、修改及保存均在现有 `metadata.lock` 内。内部入口必须传递现有 lock context,避免 legacy migration 再次取锁。
|
||||
|
||||
- 提取并复用 `parseAllConfigs` 已有的 Object Lock→Enabled Versioning 归一规则,使比较视图和真正保存一致;不得扩大 suspend、prefix exclusion、retention 的限制。bulk 先确定实际接受的 Object Lock,再比较该约束下的 Versioning,并在最终提交前应用同一规则。
|
||||
- 以归一化后的有效 `(kind, key, time)` 判定变化;更新了时间也算变化。完全重复不保存、不通知;一次 bulk 校验失败不保存部分结果;成功至多保存一次,解锁后通知。
|
||||
- 可删除字段清空必须基于现有 parse=false 的新加载对象,不能在已经解析且仍持有旧 quota 的对象上执行 Update(nil)。bulk 保留原始读取再解析保存的方式。不要顺手修改 Quota getter 或所有 `parseAllConfigs` 空分支。
|
||||
- 保存函数必须让需要发送 hook 的调用方拿到**本次提交的最终快照**,不能先解锁再读取“最新”状态拼接旧时间。最小做法是让内部 `saveMetadata` 接收元数据指针并回写 Save 的归一化结果,机械更新现有少量调用;对外 `Update/Delete` 签名不变,新增内部提交结果仅供需要该快照的本地 handler/import 使用。不得原地修改已发布到缓存的引用字段。
|
||||
- 保留现有物理桶存在检查、删除锁序、迁移、后台通知上下文。真实历史桶 Created 为零时仅走少见的物理桶 Created 补齐路径;物理桶缺失返回现有错误;创建时间仍未知则不伪造到达时间,报告 indeterminate。
|
||||
|
||||
## 三个实现提交
|
||||
|
||||
### 提交 1:原子 apply、发送一致性与本地时间
|
||||
|
||||
主要文件:`cmd/bucket-metadata-sys.go`、`cmd/bucket-metadata.go`、`cmd/site-replication.go`、`cmd/admin-bucket-handlers.go`,以及实际需要提交快照的本地配置 handler。
|
||||
|
||||
1. 在现有 update/delete 内部路径增加源时间、状态比较和提交结果;公开签名不变。六个 peer handler 移除锁外 getter 判旧,锁内持久化原始源时间。保留 Object Lock 的 legacy Tags 字段载荷回退。
|
||||
2. bulk 对明确提供的六类字段在已有锁内逐字段比较,再原子保存;未提供字段不动,不能循环调用会重入锁的公开 handler。保留 CORS 独立分支与既有行为。
|
||||
3. 六类本地实际写入在锁内分配 `max(UTCNow(), Created+1ns, 当前字段时间+1ns)`。其他类型不变。`enablePeerBucketVersioning` 的实际变更也使用它,只有缺失配置的创建 bootstrap 继续 Created 默认值。
|
||||
接管已有桶时保留原 Created,再执行现有 `SetCreatedAt`;如果 Created 改变,仅将这六类中原本为零或等于原 Created 的默认字段时间调整到新 Created,随后再做既有 versioning/lock bootstrap。原本晚于旧 Created 的真实修改/删除时间及其配置保持不变。不能仅凭 payload 为 nil 判断默认值。该小分支防止 Created 前移时默认值变成假墓碑、后移时历史初值被误判为无效,复用已有接管锁,不重做 #78 的配置保护。代码注释明确限定六类的原因;本轮不改新比较器未读取的其它配置时间。
|
||||
4. quota 本地 PUT 保留零值文档并原样表示该语义;Policy 空策略本地与 peer 一致走删除。需要归一化的本地 handler 从本次提交快照生成 hook;其他内容不发生归一变化的路径可保留既有编码,但必须满足三元组一致性。
|
||||
5. 导入在每桶最终提交锁内,为本次涉及的六类字段生成共同 commitAt,严格大于 Created 和这些字段当前时间且不早于锁内现在。该时间同时用于落盘和 bulk hook,不能沿用 ZIP 开始时间。bulk hook 从最终提交快照构建;若导入的空 Policy 已归一成删除,另外使用现有专用 Policy nil 事件表达它,不能因 `omitempty` 漏发。未导入字段不改,Object Lock 的既有派生 Versioning 修正保留原时间语义;CORS 继续独立时间/事件,其他字段不参与该上界。
|
||||
|
||||
完成条件:六类源时间落盘;旧事件不能越过锁覆盖新状态;四类删除不会被旧 PUT 复活;真实 wire、落盘和出站状态一致;重复无写入;bulk 与 import 没有绕过排序或静默遗漏删除。
|
||||
|
||||
### 提交 2:heal 选源与应用同规则
|
||||
|
||||
主要文件:`cmd/site-replication.go`。
|
||||
|
||||
1. 先过滤空 baseline、无效来源、严格早于自身 Created 的字段和 update-only 空配置,再按上述强弱排序选最大状态;无候选必须显式返回 found=false。保留历史 baseline-live 的初次同步和 heal 能力,消除六处“先 seed map 首项,再过滤默认值”的写法。
|
||||
2. 选源和目标遍历都跳过 `info.Sites` 中不存在的 deployment ID,包括不可达站点的空 ID 占位项;单一 peer 失败记录后继续其它目标,不因 map 顺序放弃健康站点。不改变状态计数或新建重试机制。
|
||||
3. 本地 heal 使用提交 1 的源时间 update/delete;远端仍用原有逐类型 RPC,全部携带源时间,补齐 Tag 的 UpdatedAt。
|
||||
4. 比较完整有效状态,去掉公开 mismatch/payload-only 对写入的门控;同内容较旧时间也同步。对已归一且相同的目标不写入、不发 RPC。Versioning 比较使用与该站点 Object Lock 一致的有效文档;全站点 Lock 状态补齐后不再因旧原始文档产生空转。
|
||||
5. 保留 #91 的计数与公开字段;创建世代冲突、无可用来源通过有限诊断解释。已知 baseline 且各站点无实质差异时安静 no-op。
|
||||
|
||||
完成条件:map 顺序不影响结果;默认空值不再删真配置;历史 baseline-live 可以初始化新站点、不能覆盖真实状态;同内容不同时间、同时间冲突最终一致;Tag 时间完整;Quota 删除后磁盘、缓存、重载一致;不可达占位项不阻断健康目标;完整状态已可见且稳定时,第二轮 heal 无写入/广播。
|
||||
|
||||
### 提交 3:新增删除传播与有界诊断
|
||||
|
||||
只增加一个启动开关,暂定 `MINIO_SITE_REPLICATION_METADATA_TOMBSTONES=off/on`,默认 off;实现沿用现有 env 开关写法,不做能力协商。
|
||||
|
||||
| 行为 | off:升级阶段 | on:所有参与节点修复后 |
|
||||
| --- | --- | --- |
|
||||
| 带时间 peer apply、锁内排序与 heal | 使用提交 1/2 | 同左 |
|
||||
| 专用事件零时间 | 兼容应用并记录 legacy-zero | 同左,不新增协议拒绝 |
|
||||
| Tags/SSE/Quota nil payload 时间导出 | 保留旧版条件导出 | 导出 `time > Created` 的真实删除时间 |
|
||||
| Policy 时间导出 | 保留已有行为 | 保留已有行为 |
|
||||
| 初次同步的真实删除 | 保留已有行为 | Policy/Tags/SSE/Quota 都发送专用 nil + source time 事件 |
|
||||
|
||||
开关只控制**新增**删除信息的导出/初次发送,普通本地删除事件照常复制。**off 不等于禁止删除传播:Policy 墓碑原本已导出,修复后在 off 下也照常参与 heal;Tags/SSE/Quota 的新增墓碑信息才被门控。** off 期间这些字段的隐藏墓碑会使 heal 继续尝试过时 RPC,由接收端排序拒绝;这是状态不可见时的已知代价,不承诺第二轮零 RPC,也不为这种正常拒绝增加每轮日志。
|
||||
|
||||
开关不检测或证明远端能力。启用条件是所有参与站点的全部节点已经修复,同一站点配置一致,旧请求排空;旧节点仍在线时保持 off。此隔离有实证依据:旧版接到新增 Quota heal 墓碑会留下已解析缓存残留。
|
||||
|
||||
日志仅保留三个实际原因:legacy-zero、before-created、indeterminate(未知创建时间、缺失/不可达来源或有实际差异却无可用候选)。精确重复、正常旧事件和成功裁决的同时间冲突不记警告。复用 `LogOnceIf`,以稳定的桶/字段/原因作为 key,**错误正文也必须稳定**;变化的时间与 peer 详情放入日志 ReqInfo,沿用现有每小时清理,不新增限流框架、不输出完整策略。
|
||||
|
||||
Server 文档解释启用顺序和回滚:降级前所有修复节点先关开关,然后滚动降级;旧软件缺陷会恢复。点名旧版 Tag heal 无 UpdatedAt 的来源。旧版到达时间污染、legacy-zero 产生的新本地时间以及创建世代分歧无法自动还原;操作者查看状态后在权威站点重新提交需要纠正的配置/删除。历史世代冲突先处理桶身份,不能靠任意站点强刷绕过创建保护。
|
||||
|
||||
## 最小验收矩阵
|
||||
|
||||
| 组 | 必须覆盖 | 证据方式 |
|
||||
| --- | --- | --- |
|
||||
| T1 | 六类 PUT 源时间;四类 DELETE;旧事件不回退;重复无写入;零 quota 三元组一致;带合法 Version 的空 Policy PUT 成功、GET NotFound、专用删除事件与落盘一致 | 真实 admin/S3 路由、ErasureSD/Erasure16、磁盘重载、RPC 捕获 |
|
||||
| T2 | 同时间两种到达顺序结果相同、删除优先;Policy 多集合及 NotAction/NotResource/Condition 反复编码与排列后,两站点独立解析得到相同键;Sid 差异及大整数不被错误合并 | 确定性比较器测试与代表性真实 handler |
|
||||
| T3 | Versioning/Object Lock nil/空 no-op;legacy Tags 载荷回退;Object Lock + prefix exclusion/ExcludeFolders 在普通写、peer、bulk/import 后有效状态一致,第二轮 heal 无额外写入 | 原有 #76/#78 回归加针对性用例 |
|
||||
| T4 | 锁前旧事件排队、较新写先提交后旧事件不得覆盖;不同字段并发均保留 | 复用已有 `lockBucketMetadataAcquireHook` / RMW 屏障、两种 ObjectLayer、目标 race |
|
||||
| T5 | bulk 新/旧/缺省字段混合;真实 JSON 编解码的 nil、空 RawMessage、显式 null、空字符串、空策略、零 quota;非法字段不部分保存;只导入 tags 不修改 Policy/Quota | 真实 bulk 路由、缓存与磁盘 |
|
||||
| T6 | 本地时间胜过已有未来时间;相邻提交不倒序;ZIP 导入期间插入写入,最终落盘和发出事件的状态与时间一致,空 Policy 删除不漏发 | 本地 API、import 路由与 RPC 捕获 |
|
||||
| T7 | map 排列、空 baseline/全无候选;baseline-live 初始化与同级冲突收敛,但不能覆盖真实 live/tombstone;nil@Created 不删除配置;空 update-only;Tag 时间、Quota 清缓存、空 ID、世代冲突 | 确定性 heal 本地/远端用例 |
|
||||
| T8 | 墓碑经保存/缓存失效/重启仍有效;缺桶、排队写入、零 Created;历史字段时间等于 Created 的桶经初次同步、一轮 heal 后一致,第二轮无写入/广播;接管 Created 前移/后移/不变时默认时间仍是 baseline,真实 PUT/DELETE 时间不变;on/off 与 legacy-zero,off 下 Policy 仍 heal、其它隐藏墓碑允许被拒 RPC 但无每轮警告 | ObjectLayer 与进程内旧版 wire/SRInfo 模拟;复用既有删除/迁移回归 |
|
||||
| T9 | 修复版双站点短暂断线、漏发/重复/乱序后六类合法配置收敛;删除传播启用后第二轮稳定无写入;日志确实有界 | 隔离双站点实验,不把未知桶世代或零时间事件算成通过 |
|
||||
|
||||
固定旧版与修复版混合进程仅作一次性升级冒烟,不新增为长期提交门槛。将已有审计用例改成正式确定性回归,不能把遍历 map 的概率复现直接提交。先记录未修复失败,再验证通过;复用原有 Object Lock、adoption、metadata-lock、计数、CORS 回归,并在实现后运行目标 race、仓库必需检查和完整 Go CI。方案审查、局部测试、双站点结果、合并与发布是不同证据。
|
||||
|
||||
## 作者复核结论
|
||||
|
||||
三个提交分别处理写入正确性、heal 收敛和新增删除信息的升级边界;每项对应已证实缺陷或本次修复直接触及的实际路径。缩减了拒绝零时间的新协议行为、混合版本长期测试门槛和新锁屏障;保留已有持久化、通知与缓存语义。
|
||||
|
||||
Opus 第二轮已确认首轮 R1/R2/R3 实质关闭,第三轮确认历史桶初始化边界,第四轮确认接管默认时间修正必要、充分且未扩大范围;最后两轮均为 0 阻断。四轮原文、版本快照、逐项处置与模型调用证据均保留。方案可以进入实现;当前只完成方案和诊断,仍须通过上述实现期验证,不能据此认定问题已修复或可关闭。
|
||||
@@ -1,85 +0,0 @@
|
||||
# #77:桶配置复制修复与验收归档
|
||||
|
||||
归档日期:2026-09-12。对象为 [SILO #77](https://github.com/pgsty/silo/issues/77)。问题真实;来源时间、删除状态和 heal 选源共同决定是否收敛,不能只补一个删除分支。最终实际 Claude Code Opus 5 Max 实现审查结论为 **GO_WITH_NONBLOCKING_NOTES,阻断 0**,完整 cmd 和最终 lint 随后通过。
|
||||
|
||||
本文固定研究与验收时的事实。合并状态以 Issue 关联 PR 为准;主干包含代码不等于镜像、软件包或生产部署已经更新。研究叙述见伴生站的[中文设计记录](https://github.com/pgsty/silo.pgsty.com/blob/main/content/blog/design/bucket-metadata-convergence.zh.md)和[英文设计记录](https://github.com/pgsty/silo.pgsty.com/blob/main/content/blog/design/bucket-metadata-convergence.md),操作契约见 [Server site-replication README](../site-replication/README.md)。
|
||||
|
||||
## 实现为什么最小、必要且足够
|
||||
|
||||
源端 10:00 PUT、10:01 DELETE,目标到 10:05 才收到 PUT。旧实现把字段时间写成 10:05,随后把源时间为 10:01 的删除当作旧事件跳过,返回成功但保留配置。删除后没有导出时间的字段,在漏发后还会被对端旧值恢复。另有锁外判旧、批量入口绕过排序、默认值抢占来源、Tags heal 丢失时间和同内容不更新时间等独立入口。
|
||||
|
||||
| 必要改动 | 少了它会发生什么 | 复用的边界 |
|
||||
| --- | --- | --- |
|
||||
| 在现有整桶锁内读取、比较并保存来源状态 | 锁外判旧仍可覆盖并发的新状态;到达时间继续污染排序 | 既有 `.metadata.bin`、`metadata.lock` 和物理桶存在检查 |
|
||||
| Policy、Tags、SSE、Quota、Versioning、Object Lock 共用确定性比较 | 等时冲突仍依赖到达顺序;heal 与接收端可能选出不同结果 | 既有载荷、字段时间及 Created,不增加 wire 或持久化字段 |
|
||||
| 专用事件、bulk、导入、本地写和 heal 都使用提交后的状态 | 只改一个 handler 会留下旁路;归一化后的 Versioning/Quota 与出站事件可能不同 | 既有解析、保存与复制钩子,bulk 仍是一次原子保存 |
|
||||
| 删除导出开关默认关闭 | 直接对旧版节点导出新增删除状态有已复现的 Quota 缓存风险 | 全部参与节点升级后统一启用,不引入能力协商框架 |
|
||||
| 物理 Created 恢复与初次同步传播 | 历史无创建时间的真实桶会失去六类配置写入能力 | 现有物理探测;目录 mtime 只是近似值,不推断真实桶世代 |
|
||||
| Policy 状态键与 heal 一致;异常按原因去重 | 永久假 mismatch 无法修复;正常空基线噪声或无来源异常静默 | 既有每站点统计与 logger,不增加后台协调系统 |
|
||||
|
||||
真实状态优先于创建基线;真实状态按来源时间、等时删除优先、规范内容键排序。历史 `live@Created` 可以初始化空目标,但空基线不能删除真实配置。本地纠正时间在锁内取 `max(now, Created+1ns, fieldTime+1ns)`;带时间 peer 事件保留来源时间。Versioning/Object Lock 的 nil 保持 no-op,Quota `null`/`{}` 保持零配额配置,空 Policy 归一为删除。
|
||||
|
||||
Policy 编码器保留在 GET/export/peer 是必要的:既有解析器可以保存的负集合策略必须能读回和复制。PUT/import 继续使用同一编码器,避免额外表示分支。没有为了本轮重写 CORS、Lifecycle、对象复制、IAM 或 #91 已完成的计数,也没有修改 SDK、模块依赖或升级协议。
|
||||
|
||||
充分性限定在同一桶世代、有效且可排序的来源状态、全部参与节点升级并开启删除导出的范围内。历史时间污染、零时间兼容事件和桶身份冲突不属于自动恢复承诺。
|
||||
|
||||
## 计划与实际对抗审查
|
||||
|
||||
- [实施前核验与失败复现](issue-77-current.md)、[最终 v4 计划](issue-77-plan.md)。这些文件保留历史状态,不能当成当前待办。
|
||||
- [四轮计划审查与意见处置](issue-77/review/final-review.md):前两轮 NO_GO,后两轮零阻断;保留各版计划、完整最终意见、调用元数据和必要探针。
|
||||
- 三轮实现审查均使用实际 Claude Code `claude-opus-5 --effort max`。模型身份取自实际 assistant 消息,effort 取自显式调用参数。源码在独立快照中审查,审查者读代码和执行结果,没有代为运行这些验收。
|
||||
|
||||
| 实现审查 | 固定源码 | 结论与处置 |
|
||||
| --- | --- | --- |
|
||||
| [首轮完整意见](issue-77/implementation-review/opus5-max-review.md) / [调用记录](issue-77/implementation-review/session.json) | `4089113e3` | GO_WITH_NONBLOCKING_NOTES,条件性阻断 F1;物理 Created、Policy 假 mismatch、诊断和证据缺口随后修复 |
|
||||
| [第二轮意见](issue-77/implementation-review/round-2/opus5-max-review.md) / [调用记录](issue-77/implementation-review/round-2/session.json) | `62cf066ff` | 条件性和无条件阻断均为 0;修正首轮对 Policy 编码器的过宽质疑,进一步补齐无来源诊断和初次同步回归 |
|
||||
| [最终定向意见](issue-77/implementation-review/round-3/opus5-max-review.md) / [调用记录](issue-77/implementation-review/round-3/session.json) | `fcbb93e89` | 阻断 0,确认后续 `461e9a721` 的测试改写等价;读取时尚在运行的 cmd/lint 后续均退出 0 |
|
||||
|
||||
最初实现审查 SHA `4089113e3` 补签 DCO 后对应 `1ee64a8d8`,两者树完全相同。生产代码最终固定在 `fcbb93e8957275bfa7ad4e6154e93f4d7b0a7025`;验收源码 HEAD 为 `461e9a721047c63e1a95f54ad4b533a6b89def30`,只在两个测试文件有格式和等价条件改写,见 [tree 对照](issue-77/implementation-review/round-3/final-tree-equivalence.json)与 [patch](issue-77/implementation-review/round-3/test-style.diff)。本次归档不改变生产代码或正式回归测试。
|
||||
|
||||
审查原文保留当时的判断,不将后续作者修正倒写成评审者已经观察到的结果。首轮误读 README 中既有的空 Policy / 零 Quota 说明,第二轮修正编码器必要性的判断;最终处置以本文和伴生站的逐项表为准。
|
||||
|
||||
## 验收证据
|
||||
|
||||
| 检查 | 执行对象与结果 |
|
||||
| --- | --- |
|
||||
| [完整 cmd](issue-77/implementation-review/round-3/final-cmd.log) / [命令及退出码](issue-77/implementation-review/round-3/final-cmd.json) | `fcbb93e89`,CGO=0,全部通过,包测试 492.776 秒 |
|
||||
| [最终目标 race](issue-77/implementation-review/round-3/final-target-race.log) / [命令](issue-77/implementation-review/round-3/final-target-race.json) | `fcbb93e89`,真实创建时间、诊断、初次同步、接管、Policy 状态和全部 CORS 命名用例通过 |
|
||||
| [测试改写后的 race](issue-77/implementation-review/round-3/final-style-target-race.log) | `461e9a721`,受影响用例通过 |
|
||||
| [build](issue-77/implementation-review/round-3/final-build.json)、[vet](issue-77/implementation-review/round-3/final-vet.json)、[lint](issue-77/implementation-review/round-3/final-lint.json) | 最终生产树 build/vet 通过,`461e9a721` lint 零问题;可选 typos 未安装,按 Makefile 跳过 |
|
||||
| internal、S3 Select race、生成文件、兼容检查、六平台编译 | `62cf066ff` 的 `ci-*-after.log` 作为较早阶段的补充证据,不冒充最终树在全部平台运行通过;归档中的空生成日志本身不证明退出状态 |
|
||||
| [两个真实站点进程](issue-77/implementation-review/round-3/final-runtime.log) / [执行元数据](issue-77/implementation-review/round-3/final-runtime.json) | 干净 `fcbb93e89` 构建:六类历史配置、真实漏发恢复、乱序、四类删除跨重启、两次各 65 秒零 metadata RPC、异常去重、gate=off 与固定旧版的 PUT/DELETE 冒烟均通过 |
|
||||
| [伴生站构建](issue-77/implementation-review/round-3/docs-check.log) | 提交为 `9fa6248` 的文章内容通过 Hugo 严格构建和站内链接检查,EN 1207 / ZH 1219 页;后续合并状态文案单独检查 |
|
||||
|
||||
[归档清单](issue-77/archive-manifest.json)保存每份材料的原始和归档后 SHA-256,以及全部改动源码的 SHA-256。[二进制身份](issue-77/implementation-review/round-3/binary-identity.json)保留实际版本、Go 构建身份和摘要:最终运行二进制来自干净 `fcbb93e89`,SHA-256 为 `4825a801ce0ac48d636d9428ce4cd5c17a20b6cfec0b569de70a124c9c005049`;旧版来自干净基线 `5c5765816`。第三轮审查的 cmd/lint 条件由上述已完成的 JSON 关闭。
|
||||
|
||||
### 反向复现与排除的证据
|
||||
|
||||
- [confirmed-before.log](issue-77/implementation-review/round-2/confirmed-before.log):正式测试叠加 `1ee64a8d8` 的 `erasure-server-pool.go` / `site-replication.go`,真实 ObjectLayer 创建时间恢复及旧 Policy 顺序失败;[修复后](issue-77/implementation-review/round-2/confirmed-after.log)和 [race](issue-77/implementation-review/round-2/confirmed-after-race.log)通过。
|
||||
- [confirmed-diagnostics-before.log](issue-77/implementation-review/round-2/confirmed-diagnostics-before.log):叠加旧诊断路径,普通空基线误报。第一处断言已经终止测试,不能声称它同时证明后面的 Warning 级别和去重断言在旧码失败。
|
||||
- [no-source-before.log](issue-77/implementation-review/round-2/no-source-before.log):直接运行 `62cf066ff`,三种已有无效状态在没有有效来源时缺少诊断;这一次不是旧码 overlay。
|
||||
- [initial-sync-before.log](issue-77/implementation-review/round-2/initial-sync-before.log):保留物理恢复修复,仅叠加 `1ee64a8d8` 的 `site-replication.go`,首次同步出站仍传零 Created。该单测的 peer 只确认 RPC,不证明真实对端或本地 peer 分支已经落盘。
|
||||
|
||||
`findings-before.log` / `findings-after-1.log` 是早期夹具失败,不能证明产品缺陷;过渡阶段的旧 helper 遗留引用和测试格式失败也不能当成通过记录。这些草稿不进入本归档的验收证据。另一次 Codex 调用因使用限额失败,不计入完成审查次数。
|
||||
|
||||
### 重跑真实双站点验收
|
||||
|
||||
[独立 Go 驱动](issue-77/runtime/main.go)与原执行内容相同,仅添加 `ignore` 构建标记及说明,避免进入普通包测试。它只启动回环地址上的一次性实验实例,每站四个数据目录,使用文件内声明的专用实验凭据。给它一个全新目录和自行从固定提交构建的两个二进制:
|
||||
|
||||
```sh
|
||||
go run docs/investigations/issue-77/runtime/main.go \
|
||||
/absolute/new-lab-directory /absolute/silo-fixed /absolute/silo-before
|
||||
```
|
||||
|
||||
固定构建分别为 `fcbb93e89` 和 `5c5765816`,可在独立干净 worktree 使用 `CGO_ENABLED=0 go build -trimpath`;记录 `--version`、`go version -m` 和 SHA-256。省略旧版二进制参数会跳过混合版本冒烟,不能将其报告为执行过。最终删除收敛快照见 [converged.json](issue-77/runtime/converged.json),数组顺序由驱动的 `states` 函数定义。
|
||||
|
||||
原执行环境为 `go1.27.1 darwin/arm64`。因宿主盘可用空间比例触发存储保留阈值,完整 cmd 和进程实验使用独立 16 GiB APFS 测试卷,没有降低生产阈值。临时卷、审查 worktree 和实例进程均已清理。上述是本地可复核的执行证据,不替代 GitHub Actions、Linux 多节点集群、发布制品或生产验证。
|
||||
|
||||
## 剩余边界
|
||||
|
||||
1. `MINIO_SITE_REPLICATION_METADATA_TOMBSTONES=off` 是默认值。全部站点全部节点升级、配置一致并排空旧请求后,统一开启并重启,才具有新增 Tags/SSE/Quota 删除的漏发自愈能力。普通删除事件与原有 Policy 删除导出仍保留。
|
||||
2. 到达时间污染、零时间旧事件和桶世代冲突无法凭现有数据重建。物理 Created 只是目录 mtime 的近似值;较早事件仍跳过,由操作者核对权威状态后重新提交纠正。
|
||||
3. 日志按每桶/字段/原因去重,仍可能随桶和有值字段数量增长。本轮不增加全站日志调度。
|
||||
4. 最终审查保留三项非阻断改进:解码失败诊断中的时间可能为零;初次同步单测没有证明本地 peer 分支落盘;未来并发后台日志可能需要更强的测试隔离。生产路径已复核,本轮不为这些建议新增 helper 或 hook。
|
||||
|
||||
归档保留最终报告与调用身份,不发布原始模型推理流、二进制、实验数据卷或无效夹具日志。工作站路径、历史文档链接与非必要 JSON 字段经过整理;哈希清单区分原始产物与归档副本,不能将整理后的报告哈希冒充原文件哈希。
|
||||
@@ -1,437 +0,0 @@
|
||||
{
|
||||
"archived_at": "2026-09-12",
|
||||
"issue": "https://github.com/pgsty/silo/issues/77",
|
||||
"base": "5c576581631561c446f30ae5b566f0aa793adc1c",
|
||||
"tested_source_head": "461e9a721047c63e1a95f54ad4b533a6b89def30",
|
||||
"reviewed_production_head": "fcbb93e8957275bfa7ad4e6154e93f4d7b0a7025",
|
||||
"source_sha256": {
|
||||
"buildscripts/rebrand-guard/compat-baseline.json": "7ea6485008dd998373144ebded82483fe2c33ae60712abcf2fba973a7506b3e1",
|
||||
"cmd/admin-bucket-handlers.go": "3aa8fc80cb8f2c4725acb8fbec026d47cd71a60ad58d06f905aa013540b30db7",
|
||||
"cmd/bucket-cors-site-replication_test.go": "2cff52d3857e912526aaae40736b077e6b8bd640e098934b8af0ec76354f3747",
|
||||
"cmd/bucket-metadata-replication.go": "07730f6a19b78a9b51dd9d56696812299b3aedce1a779b23178643e2684ba686",
|
||||
"cmd/bucket-metadata-sys.go": "6d0936f537f104bdae1c460554c5ef6f34e6f057d8eb16569a23f012a9498d60",
|
||||
"cmd/bucket-metadata.go": "2b692e42df36373fb98a74792098e1251ae886f5e2759add91a00909b596d04c",
|
||||
"cmd/bucket-policy-handlers.go": "1ac487ef97f63ab19358212f1d582659b4d6fbd17888f5bb21cd5e5028e783e9",
|
||||
"cmd/bucket-versioning-handler.go": "23ec85d45bacf687ada30ea6a06cb5e6d5d5dd32a81865a24aa52a030759e3ce",
|
||||
"cmd/common-main.go": "c06df6c3051ceb264f7ddbdddea33ebe38a59e0c113b5c4c44906c1c2e08a01b",
|
||||
"cmd/erasure-server-pool.go": "1ebb616ac971a22bd736d1b381845c0011cbc95b18dc273e4a946eec8244c80d",
|
||||
"cmd/site-replication-metadata-gate_test.go": "1f170b84d2d25d67a4386451ce26597a752f96c9215155f3156ac1fd140ead86",
|
||||
"cmd/site-replication-metadata-heal_test.go": "37191261f643dae88fbe5be44da8f08b2c52455245d5c61a2df564ecb7c8bffd",
|
||||
"cmd/site-replication-metadata.go": "5756a5495723a2d9456b1c3ed7875cec789298eefb8230ea57126f9de83c4287",
|
||||
"cmd/site-replication-metadata_test.go": "2898c88d48d9693d1c974ee7fd6846b86fe354f378caa630eac2ad0d9b9edfe8",
|
||||
"cmd/site-replication.go": "5015550411bd4d82119cdb96b7f663fd082e4906baee259b30aedbaa616354c8",
|
||||
"cmd/site-replication_test.go": "d85dd2aa991a6cce1fccfa17205e4a478f4ced00e0585fbe2aee2dbf97884b8a",
|
||||
"docs/site-replication/README.md": "abe39529bf5a864dc7c945fd368b647f3fa2121fb324cea1cf63373eff9683a7"
|
||||
},
|
||||
"original_review_head": "4089113e3edbd21a29be8e6af74662462bcc22af",
|
||||
"equivalent_signed_commit": "1ee64a8d895f0876580d5e27d0ce6facf41c347e",
|
||||
"reviewed_tree": "0d18c940c6dec380dffc977cb5720b4e2384e46a",
|
||||
"environment": "go version go1.27.1 darwin/arm64",
|
||||
"scope": "Historical analysis and plan, final review reports and invocation metadata, selected executed test logs, source/binary identity, and rerunnable loopback lab. Excludes model reasoning streams, binaries, temporary volumes and invalid fixture logs. Paths, historical document links and trailing console whitespace were normalized; JSON billing fields and build-info dependency lists were omitted. Original and archived hashes distinguish these editorial transformations.",
|
||||
"final_verdict": "GO_WITH_NONBLOCKING_NOTES",
|
||||
"blocking_findings": 0,
|
||||
"records": [
|
||||
{
|
||||
"file": "issue-77-current.md",
|
||||
"source_record": "issue-77-current.md",
|
||||
"original_sha256": "702f9fe93703c8d4b79a3e2c4709e7f49614dc20211ed5221f3adbd5e3809e2d",
|
||||
"archived_sha256": "b22b50e584de73cfe2f101d35bb912d314feb7f0e0ce48018e1b4a48dc4cf5c5"
|
||||
},
|
||||
{
|
||||
"file": "issue-77-plan.md",
|
||||
"source_record": "issue-77-plan.md",
|
||||
"original_sha256": "2ee214694f16e5949ed58290364770149aee3748faee3614122fee189ef83add",
|
||||
"archived_sha256": "36ae4f99bbc22827cc9a1a437b413713e61f442a568347eeef836b4245c67081"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/current-tests.log",
|
||||
"source_record": "issue-77/current-tests.log",
|
||||
"original_sha256": "d6ed0ef373a37f8254d387a0757a431c2785b6b9a30c4e06d0034688ca3a41e8",
|
||||
"archived_sha256": "d6ed0ef373a37f8254d387a0757a431c2785b6b9a30c4e06d0034688ca3a41e8"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/issue77_heal_review_test.go.txt",
|
||||
"source_record": "issue-77/issue77_heal_review_test.go.txt",
|
||||
"original_sha256": "f1b164afff783282253660b8fa5c131b0de946a078d370cdedd95c338fa73810",
|
||||
"archived_sha256": "f1b164afff783282253660b8fa5c131b0de946a078d370cdedd95c338fa73810"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/issue77_review_test.go.txt",
|
||||
"source_record": "issue-77/issue77_review_test.go.txt",
|
||||
"original_sha256": "77e557bd098fad1345085b6bca27ff8a733984092c68c6b26d0dc74e5fca8caa",
|
||||
"archived_sha256": "77e557bd098fad1345085b6bca27ff8a733984092c68c6b26d0dc74e5fca8caa"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/adoption-baseline-probe.go.txt",
|
||||
"source_record": "issue-77/review/adoption-baseline-probe.go.txt",
|
||||
"original_sha256": "4cf7aae6763f1ae736af087244dae01e32af1c546923fa782891759967981574",
|
||||
"archived_sha256": "4cf7aae6763f1ae736af087244dae01e32af1c546923fa782891759967981574"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/adoption-baseline-probe.log",
|
||||
"source_record": "issue-77/review/adoption-baseline-probe.log",
|
||||
"original_sha256": "d5ad1bfe7d4faafc7bfc6f55800dc47a7d60b7ec7124cc31ec9ed11b0154bde7",
|
||||
"archived_sha256": "d5ad1bfe7d4faafc7bfc6f55800dc47a7d60b7ec7124cc31ec9ed11b0154bde7"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/adoption-baseline-probe.metadata.json",
|
||||
"source_record": "issue-77/review/adoption-baseline-probe.metadata.json",
|
||||
"original_sha256": "618e81cb5852b08fbc3029d812647b9d2dbe3560349ee2e44b860356e12cffa9",
|
||||
"archived_sha256": "a462ad6b10cb7e2737330eed5d5e0a5f98ac0d664aa6fcf40a1f5f2f073709f6"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/author-followup-findings.md",
|
||||
"source_record": "issue-77/review/author-followup-findings.md",
|
||||
"original_sha256": "b6d60d813702db300313430325046aa89e45071d3cdc20083d82f061b75ab2c4",
|
||||
"archived_sha256": "b6d60d813702db300313430325046aa89e45071d3cdc20083d82f061b75ab2c4"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/decisions-v2.md",
|
||||
"source_record": "issue-77/review/decisions-v2.md",
|
||||
"original_sha256": "52c85f0b45073471e96abe86aa5a182858ff00a5dfe00f290f570f2aad86c5a1",
|
||||
"archived_sha256": "52c85f0b45073471e96abe86aa5a182858ff00a5dfe00f290f570f2aad86c5a1"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/decisions-v3.md",
|
||||
"source_record": "issue-77/review/decisions-v3.md",
|
||||
"original_sha256": "7325864c0a3e93689989612da0427b88f890ef8baf197f609c570636b16a2b85",
|
||||
"archived_sha256": "7325864c0a3e93689989612da0427b88f890ef8baf197f609c570636b16a2b85"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/decisions-v4.md",
|
||||
"source_record": "issue-77/review/decisions-v4.md",
|
||||
"original_sha256": "982b2e802aed883f2b5e211f0e1677388571ccde8e32af878a35b4192c1f4d16",
|
||||
"archived_sha256": "982b2e802aed883f2b5e211f0e1677388571ccde8e32af878a35b4192c1f4d16"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/final-review.md",
|
||||
"source_record": "issue-77/review/final-review.md",
|
||||
"original_sha256": "2410f7b92e2c74b235d54bc155b822675c6b5f9ab0b7226fccca42dc38b0a7ea",
|
||||
"archived_sha256": "2410f7b92e2c74b235d54bc155b822675c6b5f9ab0b7226fccca42dc38b0a7ea"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/opus5-max-v1.md",
|
||||
"source_record": "issue-77/review/opus5-max-v1.md",
|
||||
"original_sha256": "8a3b750a261ef520acbe658395d3303cea0835b07184b96022781a2a5d0d4dcf",
|
||||
"archived_sha256": "8a3b750a261ef520acbe658395d3303cea0835b07184b96022781a2a5d0d4dcf"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/opus5-max-v1.metadata.json",
|
||||
"source_record": "issue-77/review/opus5-max-v1.metadata.json",
|
||||
"original_sha256": "959a9eac3a8449ef27027fbc65aca55a32b1f46063175d7187922296ecd0a957",
|
||||
"archived_sha256": "d5e3be5c5377adeb88fdcb444ad5b6ee338af91d095027e1c9502298eda51d6d"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/opus5-max-v2.md",
|
||||
"source_record": "issue-77/review/opus5-max-v2.md",
|
||||
"original_sha256": "848a476f18ce88208b72d8bf6cb49a2316b2b14dcba64ad66e636fe0a3d70595",
|
||||
"archived_sha256": "848a476f18ce88208b72d8bf6cb49a2316b2b14dcba64ad66e636fe0a3d70595"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/opus5-max-v2.metadata.json",
|
||||
"source_record": "issue-77/review/opus5-max-v2.metadata.json",
|
||||
"original_sha256": "ff63843a993830ea48e1a44987390b94613dea99a796453bc91bebca2f4e119f",
|
||||
"archived_sha256": "3cd83d43f461fb1e13bbc4f4163769fa6daf4d151d8442d8615a3e813da38853"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/opus5-max-v3.md",
|
||||
"source_record": "issue-77/review/opus5-max-v3.md",
|
||||
"original_sha256": "557ef70288a9a3939842a2f76d63694e2c103a8bfcaa99b7c161ee89e1f58061",
|
||||
"archived_sha256": "557ef70288a9a3939842a2f76d63694e2c103a8bfcaa99b7c161ee89e1f58061"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/opus5-max-v3.metadata.json",
|
||||
"source_record": "issue-77/review/opus5-max-v3.metadata.json",
|
||||
"original_sha256": "ad488156f42e33e47ed5d0a267ee69ee82efff69881d0e6400e88cf88a09ef60",
|
||||
"archived_sha256": "1c84da615bccc2f1d7a2aa917ef7af751b0943cc7e413b39f59164a65acc2c98"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/opus5-max-v4.md",
|
||||
"source_record": "issue-77/review/opus5-max-v4.md",
|
||||
"original_sha256": "cf10b9d7963ff2b7d4c0fbb1fe744a0e853549eaf7ce675a94cc000f74fe6682",
|
||||
"archived_sha256": "cf10b9d7963ff2b7d4c0fbb1fe744a0e853549eaf7ce675a94cc000f74fe6682"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/opus5-max-v4.metadata.json",
|
||||
"source_record": "issue-77/review/opus5-max-v4.metadata.json",
|
||||
"original_sha256": "95bae4427322fc42974a9d94ed5dd6d272a46b6eb669f95669a2fe1ef6ee4d42",
|
||||
"archived_sha256": "60d68b2db11557ee09d60b67748844a56d7b1f323803163ef03d420ad1cc8c42"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/plan-v1.md",
|
||||
"source_record": "issue-77/review/plan-v1.md",
|
||||
"original_sha256": "0218a2ca59750acd6a24dc5ece1d55a19762a863f1a2de8edc6ffc9b4ab072e0",
|
||||
"archived_sha256": "97d88bbd010cbd19744c2e2fee0fed8d3713ca553b5d0f06161299772ea8104b"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/plan-v2.md",
|
||||
"source_record": "issue-77/review/plan-v2.md",
|
||||
"original_sha256": "2b2c712b17f757ca1a8501714c6c2472bd442a6be50b29d92f2d87e23c412ba8",
|
||||
"archived_sha256": "b193ee6e47b24576f3f4d06608f1bcd0f7f2b0c27ef782ea4f74ee3706f7aa4d"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/plan-v3.md",
|
||||
"source_record": "issue-77/review/plan-v3.md",
|
||||
"original_sha256": "8c4ab51a397246a07bb1cc967df3377d26ac084b7928191eb8fecacf6e7c3aed",
|
||||
"archived_sha256": "fb3b37ab9dd2f4ab12b8050293603ed589645a730f1e23c81e2f7162ca46f7d2"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/plan-v4.md",
|
||||
"source_record": "issue-77/review/plan-v4.md",
|
||||
"original_sha256": "fbfd37498ab32a589d06254f32182869d5d49fc97722416c46c10b41e5f77816",
|
||||
"archived_sha256": "8624e9b86d3cb6daa497d43ac8cb50318ebabdd7eb4bb7c605e0720522a73bd9"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/policy-encoding-probe.go.txt",
|
||||
"source_record": "issue-77/review/policy-encoding-probe.go.txt",
|
||||
"original_sha256": "4e07cba70bdc329a0003ca0b6711886a6de22323293e8de3625cdc838ccaf537",
|
||||
"archived_sha256": "4e07cba70bdc329a0003ca0b6711886a6de22323293e8de3625cdc838ccaf537"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/wire-state-probe.go.txt",
|
||||
"source_record": "issue-77/review/wire-state-probe.go.txt",
|
||||
"original_sha256": "cafc6e2d482cc8e178ae3e17651410e3070049cd60009f0edd30fbac29e1309c",
|
||||
"archived_sha256": "cafc6e2d482cc8e178ae3e17651410e3070049cd60009f0edd30fbac29e1309c"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/review/wire-state-probe.log",
|
||||
"source_record": "issue-77/review/wire-state-probe.log",
|
||||
"original_sha256": "5077cd264a50cc05fe9dd628a6e98a3b0aaf94e04b65b712563191e11e4ca9f1",
|
||||
"archived_sha256": "5077cd264a50cc05fe9dd628a6e98a3b0aaf94e04b65b712563191e11e4ca9f1"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/opus5-max-review.md",
|
||||
"source_record": "implementation-review/opus5-max-review.md",
|
||||
"original_sha256": "e40dad6e1969898e35c82b7fd24517a5ee714d603a54fe3598b9e69c20922118",
|
||||
"archived_sha256": "e40dad6e1969898e35c82b7fd24517a5ee714d603a54fe3598b9e69c20922118"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/session.json",
|
||||
"source_record": "implementation-review/session.json",
|
||||
"original_sha256": "c347259a0a501242eb787319d43fe16c117d021ea4e418ce74b09b6c9c33ad08",
|
||||
"archived_sha256": "d26e9c3a7f3d0ba3d34de54e95c64e5939b8d5da1d190bb94d80423b6ca79205"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-2/opus5-max-review.md",
|
||||
"source_record": "implementation-review/round-2/opus5-max-review.md",
|
||||
"original_sha256": "306a856010f7f9bd1cbec39aba83236978f5a811c98f7fc3f8a08217f72de111",
|
||||
"archived_sha256": "306a856010f7f9bd1cbec39aba83236978f5a811c98f7fc3f8a08217f72de111"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-2/session.json",
|
||||
"source_record": "implementation-review/round-2/session.json",
|
||||
"original_sha256": "64113634f4e8d0525ed17dd74d092c74419d172aa94b501187daae8ebc27e279",
|
||||
"archived_sha256": "4f4dfe9c50b43b64f15fad830c39c116a296c302655d2c718475249521a68f19"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/opus5-max-review.md",
|
||||
"source_record": "implementation-review/round-3/opus5-max-review.md",
|
||||
"original_sha256": "9106938a7a34cfbc6a11d372dd5ffb71ae7417a77a56d10bbed178f539de8fba",
|
||||
"archived_sha256": "9106938a7a34cfbc6a11d372dd5ffb71ae7417a77a56d10bbed178f539de8fba"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/session.json",
|
||||
"source_record": "implementation-review/round-3/session.json",
|
||||
"original_sha256": "28944efc7a7b2c5451beedf0a05202ffc23b29fad2d9bd04cc89fad3d72184b9",
|
||||
"archived_sha256": "bf31d2dbd8009f0a966c0e820dfdbd5a2820e8b36e71e615f783932fd4ecc9de"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-2/confirmed-before.log",
|
||||
"source_record": "implementation-review/round-2/confirmed-before.log",
|
||||
"original_sha256": "af2df1c296c3377edb6aae45caa9fafd5e620ffb5a09438c15ca50e3d2cb1c10",
|
||||
"archived_sha256": "af2df1c296c3377edb6aae45caa9fafd5e620ffb5a09438c15ca50e3d2cb1c10"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-2/confirmed-after.log",
|
||||
"source_record": "implementation-review/round-2/confirmed-after.log",
|
||||
"original_sha256": "0c8be7325817049afe2bd2d3ed668e61cbde2887a443b988b8defbf12304b5bc",
|
||||
"archived_sha256": "0c8be7325817049afe2bd2d3ed668e61cbde2887a443b988b8defbf12304b5bc"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-2/confirmed-after-race.log",
|
||||
"source_record": "implementation-review/round-2/confirmed-after-race.log",
|
||||
"original_sha256": "da2725133b6c1b41a50a92700ff6adaf6dae7e9ac9d6b5812844715fd2b70e55",
|
||||
"archived_sha256": "da2725133b6c1b41a50a92700ff6adaf6dae7e9ac9d6b5812844715fd2b70e55"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-2/confirmed-diagnostics-before.log",
|
||||
"source_record": "implementation-review/round-2/confirmed-diagnostics-before.log",
|
||||
"original_sha256": "3d2c1325f1612c9ce79172b148d1ae02cf5443087270e9caeb46046fec1ea2c8",
|
||||
"archived_sha256": "3d2c1325f1612c9ce79172b148d1ae02cf5443087270e9caeb46046fec1ea2c8"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-2/no-source-before.log",
|
||||
"source_record": "implementation-review/round-2/no-source-before.log",
|
||||
"original_sha256": "e6bac0d117eb195b0ff67ef55eaff302c5f7ee36abf236fdcb69486330b31185",
|
||||
"archived_sha256": "e6bac0d117eb195b0ff67ef55eaff302c5f7ee36abf236fdcb69486330b31185"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-2/initial-sync-before.log",
|
||||
"source_record": "implementation-review/round-2/initial-sync-before.log",
|
||||
"original_sha256": "f200f9b961baf20e48f22b50cebd82525dd4647ea42203179d140b5474a28427",
|
||||
"archived_sha256": "f200f9b961baf20e48f22b50cebd82525dd4647ea42203179d140b5474a28427"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/docs-check.json",
|
||||
"source_record": "implementation-review/round-3/docs-check.json",
|
||||
"original_sha256": "76747f4d197d250e6bd3aaa1eba5a7013e56b655f10d143b984faed9d22b503b",
|
||||
"archived_sha256": "322f206ca48816c415ba29f9c67935c0eba038823eddfac8fb7e298ff50bda4a"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/docs-check.log",
|
||||
"source_record": "implementation-review/round-3/docs-check.log",
|
||||
"original_sha256": "af8752f3d8a072954dc6de2b8c72a4054cffbe3583aeae225a8d6623fc95e9fa",
|
||||
"archived_sha256": "a1d67bf392e45da04af9d819a5d76cec98006d1fc78c1a4cce486eb305bdc3ac"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/final-build.json",
|
||||
"source_record": "implementation-review/round-3/final-build.json",
|
||||
"original_sha256": "924df2a034af81acfe8961489664fa213791096db637e962ced6b78ba7ea8388",
|
||||
"archived_sha256": "c5de13eceee16b23886290fbe351a5269735a549723ed400ddb3e5f0bf7c88e5"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/final-build.log",
|
||||
"source_record": "implementation-review/round-3/final-build.log",
|
||||
"original_sha256": "d5d78e4d1a5b0dc5bc6042ae94ad64aaf2c500ff3e9091ea1dc9dbdfff04b263",
|
||||
"archived_sha256": "b212590ec72fc2489ea00c36458b3ce900b5b23543c7b141d211ac7b4b06f07b"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/final-cmd.json",
|
||||
"source_record": "implementation-review/round-3/final-cmd.json",
|
||||
"original_sha256": "431f663b748e81b9c311fa1b931483fe0b1ed09dec796b0894bb5d4a37f5f6b9",
|
||||
"archived_sha256": "39a02d590c9615721fbded8e95cc3b04ebacb32ba5c943d8e9a6f55d912c80c7"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/final-cmd.log",
|
||||
"source_record": "implementation-review/round-3/final-cmd.log",
|
||||
"original_sha256": "681576adac6d627140a791a22085fc520b61f4a6e04172bc7fb5d0f3967e9c40",
|
||||
"archived_sha256": "67723bedd3472d22a2e3f043e48fdab9bc5e8a212d932c513d44d61cc13eb09d"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/final-lint.json",
|
||||
"source_record": "implementation-review/round-3/final-lint.json",
|
||||
"original_sha256": "1dc2b38de9c4f37983e45e11ad5f416810f0c96e2cfeeff64b25846544918751",
|
||||
"archived_sha256": "35229aa6e418c54072abf141b4296be491a43cdba87c2f56f853076d78d66559"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/final-lint.log",
|
||||
"source_record": "implementation-review/round-3/final-lint.log",
|
||||
"original_sha256": "40eb5ec64ae64b73489fc7b6e145aabfea6554e0c73fa22be7b4a8201bb63d18",
|
||||
"archived_sha256": "d4b9455691ff7bcce76a0d886f603eda2179e7fa4e408b8e3bb987d693987c3a"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/final-runtime.json",
|
||||
"source_record": "implementation-review/round-3/final-runtime.json",
|
||||
"original_sha256": "6f76d83dbf9749f9dde0d6386f0b8a6b52cda6677c17ef9d739f05ea9379c52f",
|
||||
"archived_sha256": "b57c593e1023a0226e7950c142d5bd9a231832a9e7aefc7d9d584075b70b9aca"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/final-runtime.log",
|
||||
"source_record": "implementation-review/round-3/final-runtime.log",
|
||||
"original_sha256": "4c0137bbe554426026e1d4c8baf48e28368e2450e886c506e0055e0591cd5e5e",
|
||||
"archived_sha256": "2b6228f3c6003e896193be17a7c94970992c1fbcbaba013b1984348bf772bc1f"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/final-style-target-race.json",
|
||||
"source_record": "implementation-review/round-3/final-style-target-race.json",
|
||||
"original_sha256": "09b77aeb0d720de59ea5b1c31f017e8bf32e89c8dc0d4ecc3f35704f21bc0863",
|
||||
"archived_sha256": "075c8d7393164e5b43364d25c163f5da8675f5b69c67e7523a579943a710aaca"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/final-style-target-race.log",
|
||||
"source_record": "implementation-review/round-3/final-style-target-race.log",
|
||||
"original_sha256": "3089ad9a1c7283a23853a137311728598a7e1ae3bdbbf16e5f000e1149a7e12f",
|
||||
"archived_sha256": "a2609e9a889bcc68828e58c15967bb04820a92462c3f94de86ea584e33a6660a"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/final-target-race.json",
|
||||
"source_record": "implementation-review/round-3/final-target-race.json",
|
||||
"original_sha256": "68dfcad5fc63d5981065f86fb00482cc466c46f275c4c340095c26ad719f079c",
|
||||
"archived_sha256": "5fb25fa150e66478b2d170faeed3374d71325311f7e625996fe1653cfc957b66"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/final-target-race.log",
|
||||
"source_record": "implementation-review/round-3/final-target-race.log",
|
||||
"original_sha256": "b437b206ebd12fff304f9cdb59b39f25fd0ecf54e9f510aca98c444b0b516c89",
|
||||
"archived_sha256": "a5b6b6337ebbcf0eca02c2ca15bb1fa562d39ab6fe29f41cd304dd2979976f59"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/final-tree-equivalence.json",
|
||||
"source_record": "implementation-review/round-3/final-tree-equivalence.json",
|
||||
"original_sha256": "f7d1563d2f675ae282b4bec0bcedcc9809b0a0c985332d862c61ff4b8c75a59f",
|
||||
"archived_sha256": "f7d1563d2f675ae282b4bec0bcedcc9809b0a0c985332d862c61ff4b8c75a59f"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/final-vet.json",
|
||||
"source_record": "implementation-review/round-3/final-vet.json",
|
||||
"original_sha256": "934ae2812aef8b8635517f838a07cc81dc4780603d621770f9a150bf00d43561",
|
||||
"archived_sha256": "be1b653811da4f6503f329f7e7c8b321b3426107548f0079c8560b177bc0c37f"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/final-vet.log",
|
||||
"source_record": "implementation-review/round-3/final-vet.log",
|
||||
"original_sha256": "f8166088e218c7c8afa25988c02cee77ec58b552f531121a07959b31bd97ebbb",
|
||||
"archived_sha256": "012d45737551ed5ffdccc3b3535ce0be152a9e9f869f50779d5da04e965a9936"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/test-style.diff",
|
||||
"source_record": "implementation-review/round-3/test-style.diff",
|
||||
"original_sha256": "69a7f7efbb17d8dcdd2a0d7f91098569a48d7f1aeeecb6d2aefe04b6047a9025",
|
||||
"archived_sha256": "69a7f7efbb17d8dcdd2a0d7f91098569a48d7f1aeeecb6d2aefe04b6047a9025"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/ci-crosscompile-after.log",
|
||||
"source_record": "implementation-review/ci-crosscompile-after.log",
|
||||
"original_sha256": "68ef16136fabb16babd8fa26d2e3fcf6ffddac30002f874f7b9dfdd61961e82f",
|
||||
"archived_sha256": "68ef16136fabb16babd8fa26d2e3fcf6ffddac30002f874f7b9dfdd61961e82f"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/ci-gen-after.log",
|
||||
"source_record": "implementation-review/ci-gen-after.log",
|
||||
"original_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"archived_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/ci-internal-after.log",
|
||||
"source_record": "implementation-review/ci-internal-after.log",
|
||||
"original_sha256": "775e5b2b35fda828ffd69f47902e065f58aa2e2ecd915ace9e50f4085aa2d0c4",
|
||||
"archived_sha256": "775e5b2b35fda828ffd69f47902e065f58aa2e2ecd915ace9e50f4085aa2d0c4"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/ci-lint-after.log",
|
||||
"source_record": "implementation-review/ci-lint-after.log",
|
||||
"original_sha256": "cd29668fa61e94079dda874e10920623279083c491798fa9cbdb903ebeee4d82",
|
||||
"archived_sha256": "cd29668fa61e94079dda874e10920623279083c491798fa9cbdb903ebeee4d82"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/ci-s3select-after.log",
|
||||
"source_record": "implementation-review/ci-s3select-after.log",
|
||||
"original_sha256": "30fc4888ba394ef0daa6c76fa497c28086b7307ec30d195fe4f2b725badc59c9",
|
||||
"archived_sha256": "30fc4888ba394ef0daa6c76fa497c28086b7307ec30d195fe4f2b725badc59c9"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/ci-verify-after.log",
|
||||
"source_record": "implementation-review/ci-verify-after.log",
|
||||
"original_sha256": "2b6860f20bdfd453bcc299f8b57a117cd698bab77d6a474913e31794c3937bd3",
|
||||
"archived_sha256": "2b6860f20bdfd453bcc299f8b57a117cd698bab77d6a474913e31794c3937bd3"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/implementation-review/round-3/binary-identity.json",
|
||||
"source_record": "implementation-review/round-3/binary-identity.json",
|
||||
"original_sha256": "ffdb768a1931388f174eaca0980cc840015a8e0995b5509ef68718d10044f5e5",
|
||||
"archived_sha256": "d4ea7abc1f419e834746820b5b2d2f1b3264e42114161e7c6803f4e952fcf17d"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/runtime/converged.json",
|
||||
"source_record": "runtime-reviewed/converged.json",
|
||||
"original_sha256": "d6d97bc01a12fd624af37432e9558356255b5cd9d4333eaa616e39998c2ce94f",
|
||||
"archived_sha256": "d6d97bc01a12fd624af37432e9558356255b5cd9d4333eaa616e39998c2ce94f"
|
||||
},
|
||||
{
|
||||
"file": "issue-77/runtime/main.go",
|
||||
"source_record": "twosite/main.go",
|
||||
"original_sha256": "ec63a5e2553907a826727af416145057215adabfb9fa2839861f0c05b7cc2a8f",
|
||||
"archived_sha256": "023459a3873753d5a373c2ba6c078d690c9a2a18c31efa98ecd5f4976ecaba4a"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,129 +0,0 @@
|
||||
=== RUN TestQueuedMetadataUpdateAfterDelete
|
||||
=== RUN TestQueuedMetadataUpdateAfterDelete/expiry=false
|
||||
=== RUN TestQueuedMetadataUpdateAfterDelete/expiry=true
|
||||
--- PASS: TestQueuedMetadataUpdateAfterDelete (0.43s)
|
||||
--- PASS: TestQueuedMetadataUpdateAfterDelete/expiry=false (0.22s)
|
||||
--- PASS: TestQueuedMetadataUpdateAfterDelete/expiry=true (0.21s)
|
||||
=== RUN TestIssue77CurrentHealInputs
|
||||
=== RUN TestIssue77CurrentHealInputs/ErasureSD/quota-tombstone-cache
|
||||
issue77_heal_review_test.go:58: QUOTA_CACHE: disk has no quota, cache still enforces 1024
|
||||
=== RUN TestIssue77CurrentHealInputs/ErasureSD/same-payload-time-barrier
|
||||
issue77_heal_review_test.go:79: BARRIER_NOT_HEALED: same payload remains at 2026-09-12 06:59:55.50105 +0800 CST, latest is 2026-09-11 23:00:55.50105 +0000 UTC
|
||||
=== RUN TestIssue77CurrentHealInputs/ErasureSD/creation-default-selection
|
||||
issue77_heal_review_test.go:105: DEFAULT_SELECTED: creation-default erased valid policy in 5/32 heal rounds
|
||||
=== RUN TestIssue77CurrentHealInputs/ErasureSD/tag-remote-source-time
|
||||
issue77_heal_review_test.go:144: TAG_WIRE_TIME: sent 0001-01-01 00:00:00 +0000 UTC, want 2026-09-11 23:00:55.50105 +0000 UTC
|
||||
=== RUN TestIssue77CurrentHealInputs/Erasure/quota-tombstone-cache
|
||||
issue77_heal_review_test.go:58: QUOTA_CACHE: disk has no quota, cache still enforces 1024
|
||||
=== RUN TestIssue77CurrentHealInputs/Erasure/same-payload-time-barrier
|
||||
issue77_heal_review_test.go:79: BARRIER_NOT_HEALED: same payload remains at 2026-09-12 06:59:55.71847 +0800 CST, latest is 2026-09-11 23:00:55.71847 +0000 UTC
|
||||
=== RUN TestIssue77CurrentHealInputs/Erasure/creation-default-selection
|
||||
issue77_heal_review_test.go:105: DEFAULT_SELECTED: creation-default erased valid policy in 2/32 heal rounds
|
||||
=== RUN TestIssue77CurrentHealInputs/Erasure/tag-remote-source-time
|
||||
issue77_heal_review_test.go:144: TAG_WIRE_TIME: sent 0001-01-01 00:00:00 +0000 UTC, want 2026-09-11 23:00:55.71847 +0000 UTC
|
||||
--- FAIL: TestIssue77CurrentHealInputs (1.21s)
|
||||
--- FAIL: TestIssue77CurrentHealInputs/ErasureSD/quota-tombstone-cache (0.01s)
|
||||
--- FAIL: TestIssue77CurrentHealInputs/ErasureSD/same-payload-time-barrier (0.00s)
|
||||
--- FAIL: TestIssue77CurrentHealInputs/ErasureSD/creation-default-selection (0.08s)
|
||||
--- FAIL: TestIssue77CurrentHealInputs/ErasureSD/tag-remote-source-time (0.00s)
|
||||
--- FAIL: TestIssue77CurrentHealInputs/Erasure/quota-tombstone-cache (0.04s)
|
||||
--- FAIL: TestIssue77CurrentHealInputs/Erasure/same-payload-time-barrier (0.02s)
|
||||
--- FAIL: TestIssue77CurrentHealInputs/Erasure/creation-default-selection (0.83s)
|
||||
--- FAIL: TestIssue77CurrentHealInputs/Erasure/tag-remote-source-time (0.02s)
|
||||
=== RUN TestIssue77CurrentPeerCheckBeforeLock
|
||||
=== RUN TestIssue77CurrentPeerCheckBeforeLock/ErasureSD
|
||||
issue77_heal_review_test.go:209: CHECK_OUTSIDE_LOCK: queued older event replaced newer committed tags with "<Tagging><TagSet><Tag><Key>key</Key><Value>older</Value></Tag></TagSet></Tagging>"
|
||||
=== RUN TestIssue77CurrentPeerCheckBeforeLock/Erasure
|
||||
issue77_heal_review_test.go:209: CHECK_OUTSIDE_LOCK: queued older event replaced newer committed tags with "<Tagging><TagSet><Tag><Key>key</Key><Value>older</Value></Tag></TagSet></Tagging>"
|
||||
--- FAIL: TestIssue77CurrentPeerCheckBeforeLock (0.30s)
|
||||
--- FAIL: TestIssue77CurrentPeerCheckBeforeLock/ErasureSD (0.01s)
|
||||
--- FAIL: TestIssue77CurrentPeerCheckBeforeLock/Erasure (0.07s)
|
||||
=== RUN TestIssue77CurrentSourceTimeAndDeletion
|
||||
=== RUN TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/policy
|
||||
issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.017159 +0800 CST, want source 2026-09-11 22:59:57.012479 +0000 UTC
|
||||
issue77_review_test.go:91: NEWER_DELETE: HTTP 200, but live config remained after newer source DELETE
|
||||
issue77_review_test.go:110: STALE_RESURRECTION: older source PUT resurrected a locally deleted config
|
||||
=== RUN TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/tags
|
||||
issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.029759 +0800 CST, want source 2026-09-11 22:59:57.012479 +0000 UTC
|
||||
issue77_review_test.go:91: NEWER_DELETE: HTTP 200, but live config remained after newer source DELETE
|
||||
issue77_review_test.go:105: TOMBSTONE_EXPORT: got 0001-01-01 00:00:00 +0000 UTC, want 2026-09-12 07:49:57.03401 +0800 CST
|
||||
issue77_review_test.go:110: STALE_RESURRECTION: older source PUT resurrected a locally deleted config
|
||||
=== RUN TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/sse
|
||||
issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.04395 +0800 CST, want source 2026-09-11 22:59:57.012479 +0000 UTC
|
||||
issue77_review_test.go:91: NEWER_DELETE: HTTP 200, but live config remained after newer source DELETE
|
||||
issue77_review_test.go:105: TOMBSTONE_EXPORT: got 0001-01-01 00:00:00 +0000 UTC, want 2026-09-12 07:49:57.048028 +0800 CST
|
||||
issue77_review_test.go:110: STALE_RESURRECTION: older source PUT resurrected a locally deleted config
|
||||
=== RUN TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/quota
|
||||
issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.057746 +0800 CST, want source 2026-09-11 22:59:57.012479 +0000 UTC
|
||||
issue77_review_test.go:91: NEWER_DELETE: HTTP 200, but live config remained after newer source DELETE
|
||||
issue77_review_test.go:105: TOMBSTONE_EXPORT: got 0001-01-01 00:00:00 +0000 UTC, want 2026-09-12 07:49:57.061453 +0800 CST
|
||||
issue77_review_test.go:112: older source PUT did not resurrect config
|
||||
=== RUN TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/versioning
|
||||
issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.069105 +0800 CST, want source 2026-09-11 22:59:57.012479 +0000 UTC
|
||||
issue77_review_test.go:87: nil payload is correctly a no-op
|
||||
=== RUN TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/objectlock
|
||||
issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.077601 +0800 CST, want source 2026-09-11 22:59:57.012479 +0000 UTC
|
||||
issue77_review_test.go:87: nil payload is correctly a no-op
|
||||
=== RUN TestIssue77CurrentSourceTimeAndDeletion/Erasure/policy
|
||||
issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.247526 +0800 CST, want source 2026-09-11 22:59:57.221624 +0000 UTC
|
||||
issue77_review_test.go:91: NEWER_DELETE: HTTP 200, but live config remained after newer source DELETE
|
||||
issue77_review_test.go:110: STALE_RESURRECTION: older source PUT resurrected a locally deleted config
|
||||
=== RUN TestIssue77CurrentSourceTimeAndDeletion/Erasure/tags
|
||||
issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.369557 +0800 CST, want source 2026-09-11 22:59:57.221624 +0000 UTC
|
||||
issue77_review_test.go:91: NEWER_DELETE: HTTP 200, but live config remained after newer source DELETE
|
||||
issue77_review_test.go:105: TOMBSTONE_EXPORT: got 0001-01-01 00:00:00 +0000 UTC, want 2026-09-12 07:49:57.394147 +0800 CST
|
||||
issue77_review_test.go:110: STALE_RESURRECTION: older source PUT resurrected a locally deleted config
|
||||
=== RUN TestIssue77CurrentSourceTimeAndDeletion/Erasure/sse
|
||||
issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.466111 +0800 CST, want source 2026-09-11 22:59:57.221624 +0000 UTC
|
||||
issue77_review_test.go:91: NEWER_DELETE: HTTP 200, but live config remained after newer source DELETE
|
||||
issue77_review_test.go:105: TOMBSTONE_EXPORT: got 0001-01-01 00:00:00 +0000 UTC, want 2026-09-12 07:49:57.489771 +0800 CST
|
||||
issue77_review_test.go:110: STALE_RESURRECTION: older source PUT resurrected a locally deleted config
|
||||
=== RUN TestIssue77CurrentSourceTimeAndDeletion/Erasure/quota
|
||||
issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.562629 +0800 CST, want source 2026-09-11 22:59:57.221624 +0000 UTC
|
||||
issue77_review_test.go:91: NEWER_DELETE: HTTP 200, but live config remained after newer source DELETE
|
||||
issue77_review_test.go:105: TOMBSTONE_EXPORT: got 0001-01-01 00:00:00 +0000 UTC, want 2026-09-12 07:49:57.590096 +0800 CST
|
||||
issue77_review_test.go:112: older source PUT did not resurrect config
|
||||
=== RUN TestIssue77CurrentSourceTimeAndDeletion/Erasure/versioning
|
||||
issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.640941 +0800 CST, want source 2026-09-11 22:59:57.221624 +0000 UTC
|
||||
issue77_review_test.go:87: nil payload is correctly a no-op
|
||||
=== RUN TestIssue77CurrentSourceTimeAndDeletion/Erasure/objectlock
|
||||
issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.690024 +0800 CST, want source 2026-09-11 22:59:57.221624 +0000 UTC
|
||||
issue77_review_test.go:87: nil payload is correctly a no-op
|
||||
--- FAIL: TestIssue77CurrentSourceTimeAndDeletion (0.78s)
|
||||
--- FAIL: TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/policy (0.01s)
|
||||
--- FAIL: TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/tags (0.01s)
|
||||
--- FAIL: TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/sse (0.01s)
|
||||
--- FAIL: TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/quota (0.01s)
|
||||
--- FAIL: TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/versioning (0.01s)
|
||||
--- FAIL: TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/objectlock (0.01s)
|
||||
--- FAIL: TestIssue77CurrentSourceTimeAndDeletion/Erasure/policy (0.12s)
|
||||
--- FAIL: TestIssue77CurrentSourceTimeAndDeletion/Erasure/tags (0.10s)
|
||||
--- FAIL: TestIssue77CurrentSourceTimeAndDeletion/Erasure/sse (0.10s)
|
||||
--- FAIL: TestIssue77CurrentSourceTimeAndDeletion/Erasure/quota (0.08s)
|
||||
--- FAIL: TestIssue77CurrentSourceTimeAndDeletion/Erasure/versioning (0.05s)
|
||||
--- FAIL: TestIssue77CurrentSourceTimeAndDeletion/Erasure/objectlock (0.05s)
|
||||
=== RUN TestIssue77CurrentBulkApplyOrdering
|
||||
=== RUN TestIssue77CurrentBulkApplyOrdering/ErasureSD
|
||||
issue77_review_test.go:143: BULK_STALE_OVERWRITE: older bulk event overwrote newer config, value="<Tagging><TagSet><Tag><Key>key</Key><Value>old</Value></Tag></TagSet></Tagging>" time=2026-09-12 08:48:57.792608 +0800 CST
|
||||
=== RUN TestIssue77CurrentBulkApplyOrdering/Erasure
|
||||
issue77_review_test.go:143: BULK_STALE_OVERWRITE: older bulk event overwrote newer config, value="<Tagging><TagSet><Tag><Key>key</Key><Value>old</Value></Tag></TagSet></Tagging>" time=2026-09-12 08:48:57.880379 +0800 CST
|
||||
--- FAIL: TestIssue77CurrentBulkApplyOrdering (0.27s)
|
||||
--- FAIL: TestIssue77CurrentBulkApplyOrdering/ErasureSD (0.01s)
|
||||
--- FAIL: TestIssue77CurrentBulkApplyOrdering/Erasure (0.05s)
|
||||
=== RUN TestPeerBucketAdoptionPreservesLockAndVersioningConfigs
|
||||
--- PASS: TestPeerBucketAdoptionPreservesLockAndVersioningConfigs (0.30s)
|
||||
=== RUN TestPeerBucketAdoptionBootstrapsMissingConfigs
|
||||
--- PASS: TestPeerBucketAdoptionBootstrapsMissingConfigs (0.24s)
|
||||
=== RUN TestPeerBucketAdoptionNormalizesVersioningWhenEnablingLock
|
||||
--- PASS: TestPeerBucketAdoptionNormalizesVersioningWhenEnablingLock (0.28s)
|
||||
=== RUN TestPeerBucketAdoptionEnablesSuspendedVersioning
|
||||
--- PASS: TestPeerBucketAdoptionEnablesSuspendedVersioning (0.27s)
|
||||
=== RUN TestPeerBucketObjectLockMetadataCurrentAndLegacyPayloads
|
||||
--- PASS: TestPeerBucketObjectLockMetadataCurrentAndLegacyPayloads (0.30s)
|
||||
=== RUN TestPeerBucketObjectLockMetadataWithoutLockEnabled
|
||||
--- PASS: TestPeerBucketObjectLockMetadataWithoutLockEnabled (0.24s)
|
||||
=== RUN TestSiteReplicationStatusAccountsPerSiteAndSurvivesMalformedConfig
|
||||
--- PASS: TestSiteReplicationStatusAccountsPerSiteAndSurvivesMalformedConfig (0.52s)
|
||||
FAIL
|
||||
FAIL github.com/minio/minio/cmd 7.167s
|
||||
FAIL
|
||||
@@ -1,7 +0,0 @@
|
||||
Testing builds for OS/Arch: linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64
|
||||
--> linux/amd64:github.com/minio/minio
|
||||
--> linux/arm64:github.com/minio/minio
|
||||
--> darwin/amd64:github.com/minio/minio
|
||||
--> darwin/arm64:github.com/minio/minio
|
||||
--> windows/amd64:github.com/minio/minio
|
||||
--> windows/arm64:github.com/minio/minio
|
||||
@@ -1,78 +0,0 @@
|
||||
ok github.com/minio/minio/internal/amztime 5.149s
|
||||
ok github.com/minio/minio/internal/arn 13.830s
|
||||
ok github.com/minio/minio/internal/auth 5.472s
|
||||
ok github.com/minio/minio/internal/bpool 13.244s
|
||||
ok github.com/minio/minio/internal/bucket/bandwidth 12.671s
|
||||
ok github.com/minio/minio/internal/bucket/cors 2.705s
|
||||
ok github.com/minio/minio/internal/bucket/encryption 12.228s
|
||||
ok github.com/minio/minio/internal/bucket/lifecycle 3.223s
|
||||
ok github.com/minio/minio/internal/bucket/object/lock 11.599s
|
||||
ok github.com/minio/minio/internal/bucket/replication 12.817s
|
||||
ok github.com/minio/minio/internal/bucket/versioning 13.157s
|
||||
ok github.com/minio/minio/internal/cachevalue 18.526s
|
||||
? github.com/minio/minio/internal/color [no test files]
|
||||
ok github.com/minio/minio/internal/config 14.365s
|
||||
? github.com/minio/minio/internal/config/api [no test files]
|
||||
? github.com/minio/minio/internal/config/batch [no test files]
|
||||
? github.com/minio/minio/internal/config/browser [no test files]
|
||||
? github.com/minio/minio/internal/config/callhome [no test files]
|
||||
ok github.com/minio/minio/internal/config/compress 15.754s
|
||||
ok github.com/minio/minio/internal/config/dns 17.351s
|
||||
? github.com/minio/minio/internal/config/drive [no test files]
|
||||
ok github.com/minio/minio/internal/config/etcd 18.460s
|
||||
? github.com/minio/minio/internal/config/heal [no test files]
|
||||
ok github.com/minio/minio/internal/config/identity/ldap 19.132s
|
||||
ok github.com/minio/minio/internal/config/identity/openid 19.829s
|
||||
? github.com/minio/minio/internal/config/identity/openid/provider [no test files]
|
||||
? github.com/minio/minio/internal/config/identity/plugin [no test files]
|
||||
? github.com/minio/minio/internal/config/identity/tls [no test files]
|
||||
ok github.com/minio/minio/internal/config/ilm 19.256s
|
||||
? github.com/minio/minio/internal/config/lambda [no test files]
|
||||
ok github.com/minio/minio/internal/config/lambda/event 18.225s
|
||||
? github.com/minio/minio/internal/config/lambda/target [no test files]
|
||||
ok github.com/minio/minio/internal/config/notify 17.465s
|
||||
? github.com/minio/minio/internal/config/policy/opa [no test files]
|
||||
? github.com/minio/minio/internal/config/policy/plugin [no test files]
|
||||
? github.com/minio/minio/internal/config/scanner [no test files]
|
||||
ok github.com/minio/minio/internal/config/storageclass 17.289s
|
||||
ok github.com/minio/minio/internal/config/subnet 17.168s
|
||||
ok github.com/minio/minio/internal/crypto 18.081s
|
||||
ok github.com/minio/minio/internal/deadlineconn 21.703s
|
||||
ok github.com/minio/minio/internal/disk 17.514s
|
||||
ok github.com/minio/minio/internal/dsync 154.956s
|
||||
ok github.com/minio/minio/internal/etag 17.755s
|
||||
ok github.com/minio/minio/internal/event 17.598s
|
||||
ok github.com/minio/minio/internal/event/target 16.108s
|
||||
ok github.com/minio/minio/internal/grid 22.027s
|
||||
ok github.com/minio/minio/internal/handlers 14.030s
|
||||
ok github.com/minio/minio/internal/hash 14.016s
|
||||
? github.com/minio/minio/internal/hash/sha256 [no test files]
|
||||
ok github.com/minio/minio/internal/http 13.661s
|
||||
? github.com/minio/minio/internal/init [no test files]
|
||||
ok github.com/minio/minio/internal/ioutil 14.637s
|
||||
ok github.com/minio/minio/internal/jwt 13.120s
|
||||
ok github.com/minio/minio/internal/kms 12.536s
|
||||
ok github.com/minio/minio/internal/lock 13.029s
|
||||
ok github.com/minio/minio/internal/logger 12.231s
|
||||
? github.com/minio/minio/internal/logger/message/audit [no test files]
|
||||
? github.com/minio/minio/internal/logger/target/console [no test files]
|
||||
? github.com/minio/minio/internal/logger/target/http [no test files]
|
||||
? github.com/minio/minio/internal/logger/target/kafka [no test files]
|
||||
? github.com/minio/minio/internal/logger/target/loggertypes [no test files]
|
||||
? github.com/minio/minio/internal/logger/target/testlogger [no test files]
|
||||
ok github.com/minio/minio/internal/lsync 22.092s
|
||||
? github.com/minio/minio/internal/mcontext [no test files]
|
||||
? github.com/minio/minio/internal/mountinfo [no test files]
|
||||
? github.com/minio/minio/internal/net [no test files]
|
||||
? github.com/minio/minio/internal/once [no test files]
|
||||
ok github.com/minio/minio/internal/pubsub 11.507s
|
||||
ok github.com/minio/minio/internal/rest 10.318s
|
||||
ok github.com/minio/minio/internal/ringbuffer 10.927s
|
||||
ok github.com/minio/minio/internal/s3select 8.665s
|
||||
ok github.com/minio/minio/internal/s3select/csv 8.862s
|
||||
ok github.com/minio/minio/internal/s3select/json 8.547s
|
||||
ok github.com/minio/minio/internal/s3select/jstream 8.936s
|
||||
? github.com/minio/minio/internal/s3select/parquet [no test files]
|
||||
? github.com/minio/minio/internal/s3select/simdj [no test files]
|
||||
ok github.com/minio/minio/internal/s3select/sql 8.637s
|
||||
ok github.com/minio/minio/internal/store 9.594s
|
||||
@@ -1,13 +0,0 @@
|
||||
Installing golangci-lint v2.13.1
|
||||
golangci/golangci-lint info checking GitHub for tag 'v2.13.1'
|
||||
golangci/golangci-lint info found version: 2.13.1 for v2.13.1/darwin/arm64
|
||||
golangci/golangci-lint info installed .bin/golangci/v2.13.1/golangci-lint
|
||||
Running lint check
|
||||
0 issues.
|
||||
typos binary is not found.. skipping..
|
||||
LINT_EXIT=0
|
||||
compatibility manifest: imports=119 env=438 metrics=19 headers=87 routes=223 roots=1 grid=3 storage=15 policy=58 brand=181 sha256=db0d00c26360412a5388be1eb87c2dd763633ae243c558015f7b3ae86c80baaf
|
||||
Silo rebrand compatibility baseline is unchanged
|
||||
Silo delivery and runtime rebrand checks passed
|
||||
docker entrypoint argv compatibility tests passed
|
||||
REBRAND_EXIT=0
|
||||
@@ -1,7 +0,0 @@
|
||||
ok github.com/minio/minio/internal/s3select 2.150s
|
||||
ok github.com/minio/minio/internal/s3select/csv 3.612s
|
||||
ok github.com/minio/minio/internal/s3select/json 2.105s
|
||||
ok github.com/minio/minio/internal/s3select/jstream 2.937s
|
||||
? github.com/minio/minio/internal/s3select/parquet [no test files]
|
||||
? github.com/minio/minio/internal/s3select/simdj [no test files]
|
||||
ok github.com/minio/minio/internal/s3select/sql 2.620s
|
||||
@@ -1,2 +0,0 @@
|
||||
Silo delivery and runtime rebrand checks passed
|
||||
docker entrypoint argv compatibility tests passed
|
||||
@@ -1,292 +0,0 @@
|
||||
# SILO #77 实现深度对抗性代码审查报告
|
||||
|
||||
审查对象 `4089113e3`(基线 `5c5765816`)。只读 worktree,全部结论来自直接阅读 `cmd/` 生产代码、`silo-pkg`/`madmin-go` 依赖源码与调用链,并与证据目录交叉核对。测试 PASS 仅当作"已覆盖的观察",不作正确性证明。
|
||||
|
||||
> 说明:本会话的 Write/ExitPlanMode 工具不可用,报告直接输出于此,未落盘。
|
||||
|
||||
---
|
||||
|
||||
## 0. Verdict
|
||||
|
||||
**GO_WITH_NONBLOCKING_NOTES**
|
||||
|
||||
- **无条件阻断项:0**
|
||||
- **条件性阻断项:1**(F1)——条件:支持矩阵中存在 `.metadata.bin` 缺失或 `Created == 0` 的桶
|
||||
- 其余:F2 (S2)、F3 (S3)、F4 (S3)、F5(最小性)、F6(nit)、F7(范围说明)、F8(证据不足)、F9/F10(残留边界与覆盖缺口)
|
||||
|
||||
我逐条读了六类配置在全部入口的生产路径,**没有**找到在"同一桶世代、Created 已知、合法带源时间事件"范围内会发生回退、删除复活、锁外判旧或重复广播的反例。核心机制是正确的。缺陷集中在两处边缘:零 Created 历史桶被新代码判为不可写(F1,且其自带测试用 stub 掩盖了生产行为),以及 Policy 规范编码排序与既有公开 mismatch 统计口径冲突(F2)。
|
||||
|
||||
---
|
||||
|
||||
## 1. Findings
|
||||
|
||||
### F1 — 零 Created / 无 metadata.bin 的桶:六类配置写入全部硬失败;设计中的"物理桶 Created 补齐"在生产中不可达;配套测试用 stub 掩盖
|
||||
|
||||
**严重度 S1(条件性阻断)/ 否则 S2 · 已证实(源码级链路完整),需运行定量确认**
|
||||
|
||||
位置:
|
||||
- `cmd/bucket-metadata-replication.go:265-278` `ensureBucketMetadataCreated`
|
||||
- `cmd/bucket-metadata-sys.go:181-185`(六类分支入口,失败即 `return err`)
|
||||
- `cmd/erasure-server-pool.go:2280-2285` —— **`GetBucketInfo` 无条件用缓存 `meta.Created` 覆盖物理卷 Created**
|
||||
- `cmd/admin-bucket-handlers.go:1092`(import)、`cmd/site-replication.go:1671`(bulk peer apply)
|
||||
- `cmd/site-replication.go:2156-2166` + `cmd/site-replication-metadata.go:45-55`(初次同步静默跳过)
|
||||
- `cmd/site-replication-metadata-gate_test.go:145-159`(stub)
|
||||
|
||||
最短触发链:
|
||||
1. 桶存在,但 `.minio.sys/buckets/<b>/.metadata.bin` 不存在且无 legacy 配置文件 → `loadBucketMetadataParse` 走 `errConfigNotFound` 分支并**返回 nil error 且 `Created == 0`**(`cmd/bucket-metadata.go:230-297`;注意 `defaultTimestamps()` 只在 `err == nil` 时调用)。
|
||||
2. 启动时 `concurrentLoad`(`cmd/bucket-metadata-sys.go:737-763`)把这个 `Created == 0` 的 meta 写进 `metadataMap`。
|
||||
3. `PUT ?tagging`(或 policy/sse/quota/versioning/object-lock)→ `updateAndParseMetadata` → `ensureBucketMetadataCreated` → `objAPI.GetBucketInfo(..., NoMetadata:true)` → `erasureServerPools.GetBucketInfo` 先取到物理卷时间,**随后被缓存里的 0 覆盖** → `info.Created.IsZero()` → `errors.New("bucket metadata creation time is unknown")`。
|
||||
4. 客户端收到错误。基线版本此处成功(直接赋值 + `saveMetadata`)。
|
||||
|
||||
注:`getAllLegacyConfigs` 在 `cmd/bucket-metadata.go:468` 会用 `info.ModTime` 填 `Created`,所以**有** legacy 配置文件的桶反而没事;**从未设置过任何桶配置**的老桶才是高发场景。
|
||||
|
||||
用户影响:
|
||||
- 六类配置 PUT/DELETE 全部返回错误,**无运维恢复路径**;`mc admin bucket import` 对该桶整体失败(`rpt.SetStatus(bucket, "", err)`);peer bulk apply 返回错误导致该桶复制持续报错。
|
||||
- `syncToAllPeers` 对这类桶**静默跳过全部五类配置**(`state.candidate()` 要求 `valid`,而 `valid = !created.IsZero() && ...`),且**无任何诊断日志** —— 基线版本会发送。
|
||||
|
||||
最小修复(推荐 a):
|
||||
- **(a)** `cmd/erasure-server-pool.go:2282` 改为 `if !meta.Created.IsZero() { bucketInfo.Created = meta.Created }`。一行;恢复计划中"物理桶 Created 补齐路径"的本意;顺带修掉"无 metadata.bin 的桶在 `mc ls` 里创建时间为零"这一既有瑕疵。需复核 `bucketExists`、`hasBucket = !bi.CreatedAt.IsZero()` 等消费点 —— Created 由零变非零对它们都是变好。
|
||||
- **(b)** 另外在 `syncToAllPeers`(`cmd/site-replication.go:2138-2156`)把已在手的 `bucketInfo.Created` 兜底进 `meta.Created`,或在"有内容但 send==false"时打一条 indeterminate,避免静默漏发。
|
||||
|
||||
应补验证:
|
||||
- 把 `TestPeerBucketMetadataUnknownCreated` 的 `"physical-created"` 子用例**去掉 stub**:用真实 ObjectLayer,把 `newBucketMetadata(bucket)`(Created 为零)存盘并让缓存持有它,断言 `globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, tagXML)` 成功且 `Created` 被补齐为物理时间。**当前实现会失败。**
|
||||
- 一个 `syncToAllPeers` 用例:`Created == 0` 且 Tags 非空的桶,断言初次同步仍发 Tags 事件(或至少有诊断)。
|
||||
|
||||
---
|
||||
|
||||
### F2 — Policy 规范编码对 `Statement` 数组排序,与既有公开统计的顺序敏感 `Equals` 冲突:永久假 mismatch,heal 永不修复
|
||||
|
||||
**严重度 S2(公开状态/可观测性,非数据损失)· 已证实(源码级),需运行确认**
|
||||
|
||||
位置:
|
||||
- `cmd/bucket-metadata-replication.go:110` —— `sort.Slice` 作用于**每个**数组,含顶层 `Statement`
|
||||
- `cmd/bucket-metadata-replication.go:124-161` `canonicalBucketPolicy`
|
||||
- `cmd/site-replication.go:3796` `isBktPolicyReplicated` → `prev.Equals(*p)`(`silo-pkg .../policy/bucket-policy.go:190-194` 按 `Statements[i]` **下标**逐一比较,顺序敏感)
|
||||
- `cmd/site-replication-metadata.go:174` vs `:179` —— 本地 heal 写**源站原始字节**,远端 heal 经 `PeerBucketPolicyHandler` → `canonicalBucketPolicy` 写**规范字节**
|
||||
|
||||
最短反例:
|
||||
1. 站点 A 存在升级前写入的桶策略,statement 顺序不是规范字节序(极常见,例如 Deny 在 Allow 之前)。
|
||||
2. 加入新站点 B:`syncToAllPeers` → `initialBucketConfigReplicationEvent` 发送 **A 的原始字节**,`UpdatedAt = A.PolicyConfigUpdatedAt`。
|
||||
3. B 的 admin 入口解析后调 `PeerBucketPolicyHandler` → `canonicalBucketPolicy` → 存**排序后**字节。
|
||||
4. 此后 A/B 的比较键(canonical)完全相同、时间相同 → `compareBucketConfigStates == 0` → heal 永不写、永不发 RPC;而 `Equals` 因 statement 顺序不同返回 false → `mc admin replicate status` **永久**报 bucket policy mismatch,`ReplicatedBucketPolicies` 少计。
|
||||
|
||||
同机制第二条路径:同一轮 heal 中本地目标拿原始字节、远端目标拿规范字节,三站点集群会出现持久分叉。
|
||||
|
||||
为何是新问题:基线 `PeerBucketPolicyHandler` 用 `json.Marshal(policy)`,集合(Action/Resource/Principal)顺序随机但 **statement 切片顺序被 marshal/unmarshal 保持**,`Equals` 一直成立。
|
||||
|
||||
最小修复(三选一,推荐 a 或 b):
|
||||
- (a) `canonicalBucketPolicyJSON` 不对顶层 `Statement` 数组排序(只排集合数组)——顺序在全链路被保留,两站点独立解析同一文档仍得相同键。
|
||||
- (b) `isBktPolicyReplicated` 改用 `canonicalBucketPolicy` 的键比较,而不是 `Equals`。
|
||||
- (c) `healBucketConfig` 本地分支改写 `incoming.data`(与远端同一编码)——只修本地/远端分叉,**不**修 A 与新站点的分叉。
|
||||
|
||||
应补验证:双站点用例——A 侧直接把 legacy 顺序的策略字节写盘(绕过 canonical 编码器),join B,跑两轮 heal,断言 `SiteReplicationStatus` 不报 policy mismatch。
|
||||
|
||||
---
|
||||
|
||||
### F3 — heal 诊断以 ERROR 级、按"桶 × 字段"输出;正常瞬态也报警,且单一 key 会吞掉真实 RPC 失败
|
||||
|
||||
**严重度 S3 · 已证实**
|
||||
|
||||
位置:`cmd/site-replication-metadata.go:134-145`(两个诊断循环)、`:182-186`(peer 错误)、`cmd/logging.go:19-21`(`replLogOnceIf` 无 errKind ⇒ **ErrorKind**)
|
||||
|
||||
1. 第二个循环对 `!state.valid` 的目标打 `indeterminate / unusable peer`。`valid` 要求对端 `CreatedAt != 0` —— 而**对端还没有这个桶**时 `CreatedAt` 就是 0(`cmd/site-replication.go:3096-3103` 给缺桶站点填零值 `SRBucketInfo`)。于是"桶刚建、尚未传播"这种完全正常的瞬态,对每个桶产生 6 条 **ERROR**。
|
||||
2. 第一个循环在对端 `RemoteTargetConnectionErr`(`cmd/site-replication.go:3020-3026` 填空 ID)时,对**每个本地桶 × 6 字段**打 `missing peer`。一个站点掉线 ≈ 每小时 6×桶数 条 ERROR。
|
||||
3. 四种完全不同的情况(缺 peer / peer 不可用 / peer RPC 失败 / Created 未知)共用同一个 key `"bucket-metadata/<bucket>/<file>/indeterminate"` **且错误正文相同**,而 `logOnceIf` 只按 key+正文去重(`internal/logger/logonce.go:100-119`)。结果:**真正的 heal RPC 失败可能被同桶同字段的"缺 peer"消息顶掉而完全不打印**;基线版本这些失败走 `replLogIf`,必然打印。
|
||||
|
||||
最小修复:`target.CreatedAt.IsZero()`(桶不在对端)与 `found == false` 时不打诊断;把 4 种情况拆成不同 reason(`unreachable`/`peer-error`/`unknown-created`)避免 key 撞车;给 `logBucketConfigReplication` 传 `logger.WarningKind`。
|
||||
|
||||
---
|
||||
|
||||
### F4 — 三处用户可见语义变更未写入文档
|
||||
|
||||
**严重度 S3(文档/兼容)· 已证实(逐条比对 `docs/site-replication/README.md:65-118`)**
|
||||
|
||||
1. **空 Bucket Policy PUT 现在立即按删除处理**:`cmd/bucket-policy-handlers.go:102` + `cmd/bucket-metadata-replication.go:176-180` → `canonicalBucketPolicy` 对 `IsEmpty()` 返回 nil → 落盘 nil → `GetBucketPolicy` 从"200 + 空策略文档"变为 **404 NotFound**。计划 §1 明确写了"需要写入兼容说明",文档里没有。
|
||||
2. **零 Quota (`{}`) 在对端从"被删除"变为"保留 live 文档"**:`cmd/admin-bucket-handlers.go:79-96` 删除了出站 `bucketMeta.Quota = nil` 改写。这是正确的对齐(本地原本就保留 `{}`),但对端 `GetBucketQuotaConfig` 的结果会变。
|
||||
3. **`GET ?policy` 与 `mc admin bucket export` 的 JSON 形态改变**:`cmd/bucket-policy-handlers.go:202`、`cmd/admin-bucket-handlers.go:439` 改用 `canonicalBucketPolicy` → 对象键按字母序、集合数组与 **Statement 数组被排序**。语义等价(S3 策略求值与 statement 顺序无关),但字节级比对输出的工具会看到变化。
|
||||
|
||||
---
|
||||
|
||||
### F5 — 最小性:Policy 规范编码器接入 PUT / GET / export / peer 落盘,并非 #77 不变量所必需
|
||||
|
||||
**非缺陷,最小性判断 · 已证实**
|
||||
|
||||
比较键在 `bucketConfigPayload`(`cmd/bucket-metadata-replication.go:172-180`)里**从已解析策略现算**,与落盘字节无关。即使 PUT/peer 仍用 `json.Marshal`(字节不确定),两站点的键依旧相同,收敛性完全不受影响。
|
||||
|
||||
因此这部分改动带来的是两件**额外**的事:(a) 让原本被 `ActionSet.MarshalJSON`(`silo-pkg .../policy/actionset.go:144-148`,空集合报错)挡掉的 `NotAction`/`NotResource` 策略首次可以写入 —— 这是**功能新增**;(b) 直接导致 F2。
|
||||
|
||||
判断:如果作者**有意**支持负集合策略,应作为独立特性声明并单独记录/测试(目前只有 `site-replication-metadata_test.go:530-541` 一个用例);如果只为 #77,最小做法是 `canonicalBucketPolicy` 只用于比较键。我**不**主张必须删除——它确实修掉了"能存不能读"的潜在坑——但必须承认这是范围外的行为扩张,且未在文档中声明。
|
||||
|
||||
其它可删复杂度(都很小,不影响不变量):
|
||||
- `cmd/bucket-metadata-sys.go:151` 在零值结构体上用 `replicatedBucketConfig(&result.meta, configFile)` 做"是不是这六类"的判断,语义晦涩;一个 `isReplicatedBucketConfig(file) bool` 更清楚。
|
||||
- 修掉 F3 后,`healBucketConfig` 的第二个诊断循环(`:139-145`)可并入主循环。
|
||||
|
||||
---
|
||||
|
||||
### F6 — Nit
|
||||
|
||||
- `isBucketMetadataEqual`(`cmd/site-replication.go:5164`)现在只被测试引用,生产死代码。
|
||||
- `cmd/bucket-metadata-replication.go:36-38` 注释"Object Lock is applied before Versioning"对 bulk apply 循环成立,但 `healBuckets`(`cmd/site-replication.go:4745-4746`)先 heal Versioning 再 heal Object Lock。我推演过仍收敛(最多 2 个周期、无写入环),但注释与 heal 顺序不一致,建议补一句。
|
||||
|
||||
---
|
||||
|
||||
### F7 — 范围说明(非缺陷,但必须进入决策)
|
||||
|
||||
**默认 `MINIO_SITE_REPLICATION_METADATA_TOMBSTONES=off` 时,Tags / SSE / Quota 的"漏发删除"不会通过 heal 收敛。** 只有 Policy 墓碑默认导出(`cmd/site-replication.go:3953-3954` 无条件导出;另外三类在 `:3960 / :3980 / :3988` 被 gate 挡住)。
|
||||
|
||||
我完整推演了 off 模式,结论与文档一致:本地真实墓碑在锁内比较时**不会**被旧 PUT 复活(`applyBucketConfig` 用真实落盘状态比较);但持有旧数据的对端会每 30 秒发一次过期 RPC 被拒绝,**状态永不收敛**,直到运维在权威站点重新提交删除,或全站点升级后统一开启开关。
|
||||
|
||||
也就是说:**默认配置下交付的是"顺序正确 + 不复活 + Policy 删除可 heal",不是"四类删除都能自愈"。** 这是 v4 计划的既定取舍(计划 §提交 3 表格),实现与文档都如实写了;我在此只是确保批准时看到这一点。
|
||||
|
||||
---
|
||||
|
||||
### F8 — 证据不足项
|
||||
|
||||
1. **F2 对双站点实验不可见**:`twosite/main.go:141-144` 的所有策略都经 `SRPeerReplicateBucketMeta` → `canonicalBucketPolicy` 写入,两侧都是规范字节,`states()`(`:160-164`)的字节比较自然通过。**从未构造过"升级前旧编码字节"的策略。**
|
||||
2. **F1 被 stub 掩盖**:`cmd/site-replication-metadata-gate_test.go:151-159` 自定义 `GetBucketInfo`,在 `opts.NoMetadata` 时直接返回物理 Created,绕过了 `erasureServerPools.GetBucketInfo` 的缓存覆盖。`"physical-created"` 子用例证明的是一个**生产中不会发生**的行为。
|
||||
3. **"稳态 0 metadata RPC"的边界**:`twosite/main.go:182-190` 只统计 `event.Bucket == bucket` 的单桶、双站点、**gate=on**。结论有效但窄;gate=off 的稳态不为零(文档已声明),多桶/多站点未观测。
|
||||
4. **二进制身份**:`manifest.json` 给了 `silo-final` 的 sha256,但目录里没有 `--version`/build-info 输出(驱动用 `--quiet --json` 启动),`runtime-final.log` 也没有版本行。验收自述"编译信息为 `c8f264f79 + dirty`"是**诚实的**,且日志里的 gate 行为(`gate=on/off`、`off exporter exposed new tombstone`)只能来自第三个提交的生产代码,所以**没有夸大**;但也**不能从证据目录内独立复核**。补一份 `silo-final --version` 或 `go build` 复现即可闭环。
|
||||
5. **baseline.log 行号与最终测试文件不一致**(日志 93/105/117 vs 现文件 96/97/109/121):说明"修复前复现"跑的是测试文件的早期版本。可接受,但严格讲不是同一份用例。
|
||||
|
||||
---
|
||||
|
||||
### F9 — 残留边界:桶世代冲突(计划内已声明,非本轮回归)
|
||||
|
||||
各站点 `Created` 不同时,A 的 baseline-live(`at == A.Created`)在 B 上会被重算为 `real`(`newBucketConfigState` 用**目标**的 `created` 判定,`cmd/bucket-metadata-replication.go:215-219`),可能压过 B 的真实墓碑。计划明确排除在收敛承诺外,且 `AddPeerClusters`(`cmd/site-replication.go:458-466`,"only one cluster may have data")封死了最常见入口。剩余入口:分区期间两站点各自建同名桶,或 #78 接管。基线版本在同场景下是**不确定**的(seed 首个 map 项),所以不算回归。
|
||||
|
||||
---
|
||||
|
||||
### F10 — 覆盖缺口:接管时 `Created` 前移越过真实修改时间
|
||||
|
||||
`rebaseBucketConfigDefaults`(`cmd/bucket-metadata-replication.go:306-318`)只调整"零/等于旧 Created"的默认时间。若 `opts.CreatedAt` 晚于某个**真实**字段时间,该字段变成 `at < Created` ⇒ `valid == false`:作为 heal 源 `candidate()` 为 false;作为 heal 目标 `incoming.valid > current.valid` ⇒ **必被覆盖**,本地真实配置被丢弃。
|
||||
|
||||
在"远端世代胜出"的语义下可以论证这是对的,但 `TestPeerBucketAdoptionRebasesOnlyDefaults`(`cmd/site-replication-metadata_test.go:394-425`)用的 shift 是 ±1h 而真实时间在 `created+2h`,**恰好没有覆盖这一情形**。建议加一个 `shift = +3h` 的子用例,把期望行为固定下来。
|
||||
|
||||
---
|
||||
|
||||
## 2. 六类配置 × 各入口 覆盖判断
|
||||
|
||||
| 配置 | 本地写 | typed peer | bulk | import | initial sync | heal | 接管 | 结论 |
|
||||
|---|---|---|---|---|---|---|---|---|
|
||||
| Policy | ✅ `bucket-policy-handlers.go:108/152`,hook 用 `result.meta`+`result.updatedAt` | ✅ `site-replication.go:1716-1730` | ✅ `len(item.Policy)!=0` 判"已提供" | ✅ 共同 `commitAt` + 空策略另发专用 nil 事件(`admin-bucket-handlers.go:1150-1155`) | ✅ gate 控墓碑 | ✅ | ✅ rebase | 通过(F2/F4 为附带) |
|
||||
| Tags | ✅ `bucket-handlers.go:1940/2021` | ✅ `:1733-1747` | ✅ `*string != nil` | ✅ | ✅ gate | ✅ 补齐 `UpdatedAt`(旧版 heal 缺此字段) | ✅ | 通过 |
|
||||
| SSE | ✅ `bucket-encryption-handlers.go:106/199` | ✅ `:1787-1801` | ✅ | ✅ | ✅ gate | ✅ | ✅ | 通过 |
|
||||
| Quota | ✅ `admin-bucket-handlers.go:85-99`(删掉零值改写) | ✅ `:2032-2047` | ✅ `len(item.Quota)!=0` | ✅ | ✅ gate | ✅ 含缓存清除(`parse=false` 修复) | ✅ | 通过 |
|
||||
| Versioning | ✅ `bucket-versioning-handler.go:100-116`,广播用归一后的 `result.meta` | ✅ 空=no-op | ✅ 空=no-op | ✅ 归一后落盘并广播 | ⛔ 不发(由 MakeBucketHook bootstrap + heal 对齐,与基线一致) | ✅ 按**目标** Lock 状态归一后比较 | ✅ `enablePeerBucketVersioning` 用 `localBucketConfigUpdatedAt` | 通过 |
|
||||
| Object Lock | ✅ `bucket-handlers.go:1841-1852` | ✅ 空=no-op,保留 `item.Tags` legacy 回退 | ✅ 空=no-op | ✅ | ✅ | ✅ | ✅ | 通过 |
|
||||
|
||||
逐条核对结论:
|
||||
|
||||
- **来源时间不会变成本地 now**:六个 typed handler 全部删除了锁外 `GetXConfig()` 判旧,统一走 `updateAndParseMetadata(..., &updatedAt)`;非零 `sourceTime` 直接 `sourceTime.UTC()`(`bucket-metadata-sys.go:191-193`)。只有 `sourceTime == nil`(本地写)或为零(legacy 兼容)才分配本地单调时间。✔
|
||||
- **本地写严格单调**:`localBucketConfigUpdatedAt`(`:255-263`)保证 `> Created` 且 `> 当前字段时间`,涵盖"已有未来时间"。✔
|
||||
- **重复/乱序不保存不广播**:`applyBucketConfig` 在 `compare <= 0` 返回 `changed=false`,据此跳过 `saveMetadata` 与 `LoadBucketMetadata`;bulk 用 `changed` 汇总后一次保存。✔
|
||||
我特别核对了所有使用 `result.meta`/`result.updatedAt` 的本地 handler:在这些路径上 `changed` 恒为 true(`localBucketConfigUpdatedAt` 使 `incoming.at` 严格更大,且 `incoming.candidate()` 恒真),所以**不会**出现"零时间 + 空载荷"被当成删除广播出去。这是一个隐式依赖,建议加一行注释或断言固定住。
|
||||
- **整桶 `.metadata.bin` 读-比较-写全在既有分布式锁内**:`updateAndParseMetadata`、`PeerBucketMetadataUpdateHandler`、import 最终提交、`PeerBucketMakeWithVersioningHandler`、CORS 路径,我逐个确认锁的获取在读之前、释放在 `saveMetadata` 之后、fan-out 在释放之后。✔
|
||||
- **无关字段 / CORS / lifecycle 不被覆盖**:所有写路径都是锁内**重新加载**后只改目标字段;import 用 `applyImportedBucketMetadata` 只按 `fields` 拷贝并 `bytes.Clone`。✔
|
||||
- **缓存快照正确**:`saveMetadata` 改收 `*BucketMetadata`,`meta.Save` 内部 `parseAllConfigs` 回写归一化结果,`sys.Set(name, *meta)` 发布的是提交后快照;调用方拿到的 `result.meta` 是同一份值拷贝,不会原地修改已发布引用。✔
|
||||
- **`parse` 被强制为 false 对缓存无害**:`Save()` 先 `parseAllConfigs`,发布到 `metadataMap` 的解析字段是新的;而"删除后 quota 解析残留"正是靠 `parse=false` 的新加载对象修掉的(`parseAllConfigs` 对空 `QuotaConfigJSON` **不会**把 `quotaConfig` 置 nil,`bucket-metadata.go:400-405`)。✔
|
||||
- **锁/返回值/错误处理**:`unlock()` + `locked=false` 模式保留;`updateAndParse`、`Update`、`Delete` 对外签名不变。✔
|
||||
|
||||
---
|
||||
|
||||
## 3. 六字段 state 比较:传递性、交换序无关、幂等收敛
|
||||
|
||||
`compareBucketConfigStates`(`bucket-metadata-replication.go:227-253`)实际是按 `(valid, real, real?at:—, real?isTombstone:—, key)` 的字典序全序。
|
||||
|
||||
- **传递 / 反对称**:是。`real == false` 时不比较 `at`(只比 key),但空 baseline 的 key 为 nil,`bytes.Compare` 使其恒最小,效果等同"空 baseline 永不获胜"。✔
|
||||
- **交换顺序无关**:`latestBucketConfig` 取全序最大值,与 map 遍历顺序无关;`applyBucketConfig` 同样只依赖全序。`TestLatestBucketConfigCandidates` 跑了全部 6 种排列。✔
|
||||
- **幂等收敛**:写入后目标状态等于源状态 ⇒ 下一轮 `compare == 0` ⇒ 不写不发。我另外手工推演了 Object Lock/Versioning 的"Enabled vs Suspended 同时间"场景,归一后双侧都不再写,稳定无振荡。✔
|
||||
- **baseline Created 与真修改的区分**:`real = valid && at.After(created)`,零字段时间先回填为 `created`(与 `defaultTimestamps()` 一致)。✔
|
||||
- **同时间删除胜出**:`:244-250`,仅在 `a.real` 分支内生效,空 baseline 不会借此删配置。✔
|
||||
- **等时不同载荷的稳定键**:
|
||||
- **Policy**:递归排序的规范 JSON + `json.RawMessage` 保留整数精度(测试 `:308-311` 验证 `9007199254740993` 不失真)。我对照了 `silo-pkg .../policy/bucket-policy-statement.go:27-36`,`BPStatement` 的 8 个字段(SID/Effect/Principal/Actions/NotActions/Resources/NotResources/Conditions)**全部覆盖,无字段静默丢失**;`ParseBucketPolicyConfig` 先做 `Validate` 保证 `Principal.MarshalJSON` 不会失败。✔ 但顶层 Statement 排序引出 F2。
|
||||
- **Quota**:`json.Marshal(parseBucketQuota(...))`。零 quota 仍是 live 文档(`candidate()` 靠 `real`,不靠 `len(data)`)。`{}` 与对端重编码字节不同但键相同,不产生 heal 环;`isBktQuotaCfgReplicated` 按解析值比较,不会误报。✔
|
||||
- **Tags / SSE / Object Lock**:有效文档字节,保留大小写与实际内容。✔
|
||||
- **Versioning**:先按**该站点**的 Object Lock 归一(`effectiveBucketVersioning`),键即归一后的文档,避免"保存阶段隐式改写"造成的空转。✔
|
||||
- **空策略**:`bucketConfigPayload` 对 `cfg.IsEmpty()` 返回 `(nil, nil, nil)`,与 peer 侧既有"空策略=删除"解释统一。✔(副作用见 F4-1)
|
||||
- **Object Lock 强制 Versioning 后的持久化 vs 广播**:本地只广播 Object Lock 事件,派生的 Versioning 由对端在 `Save → parseAllConfigs` 自行推导,**内容一致、时间各自保留**;heal 用归一后的有效文档比较,最多一次写入即收敛。与计划一致。✔
|
||||
|
||||
---
|
||||
|
||||
## 4. Created / 时间的异常输入
|
||||
|
||||
| 输入 | 行为 | 判断 |
|
||||
|---|---|---|
|
||||
| Created 为零 / metadata.bin 缺失 | 六类写入报错;初次同步静默跳过 | **F1,回归** |
|
||||
| 字段时间为零 | 回填为 Created ⇒ baseline | ✔ 与 `defaultTimestamps()` 一致 |
|
||||
| 事件时间 == Created | 可更新仍为 baseline 的字段,时间保存为 Created;同时间 nil/空只算空 baseline,不删除 | ✔ 有用例 |
|
||||
| 事件时间 < Created | `before-created`,跳过 + 单条日志;heal 对该目标 `continue` | ✔ |
|
||||
| 未来时间 | 接受为真实状态;后续本地写用 `max(now, at+1ns)` 压过 | ✔ 有用例 |
|
||||
| 各站点 Created 不同 | 见 F9(计划外,已声明) | 残留 |
|
||||
| 空 deployment ID | `latestBucketConfig` / `healBucketConfig` 均 `id == "" \|\| !known` 跳过 | ✔ 有用例 |
|
||||
| peer 报错 / 缺元数据 | `candidate()` 为 false ⇒ 不参与选源;作为目标 `CreatedAt.IsZero()` ⇒ `continue`。**不会被当成删除** | ✔ |
|
||||
| 单个 peer 不可达 | 记录后继续其它目标(`site-replication-metadata.go:182-186`),不因 map 顺序放弃健康站点 | ✔ 有用例(`"broken"`) |
|
||||
|
||||
**"已知历史不可恢复"与"新实现回归"的区分**:
|
||||
- 历史不可恢复(可接受):旧版到达时间污染、legacy-zero 事件产生的新本地时间、桶世代分歧。
|
||||
- 新实现回归(应修):F1 的零 Created 硬失败与初次同步静默漏发;F2 的永久假 mismatch;F3 的 ERROR 噪声与日志互相顶替。
|
||||
|
||||
---
|
||||
|
||||
## 5. 兼容路径与开关
|
||||
|
||||
- **typed 零时间兼容**:`sourceTime != nil && IsZero()` ⇒ 分配本地时间 + 一条 `legacy-zero`(`bucket-metadata-sys.go:186-190`)。不受 gate 影响,不在源时间排序保证内。✔
|
||||
- **bulk 零时间拒绝**:`PeerBucketMetadataUpdateHandler:1610-1612` ⇒ `errInvalidArgument`,与现状一致;import hook 恒带非零 `commitAt`。✔
|
||||
- **省略 vs 显式 null / 空串**:`madmin.SRBucketMeta` 全字段 `omitempty`(`madmin-go v3.0.110 cluster-commands.go:501-538`),因此
|
||||
- `json.RawMessage`(Policy/Quota):省略 ⇒ nil;显式 `null` ⇒ `[]byte("null")`,走 `len(...)!=0` 判为"已提供",再交既有解析器(Policy `null` ⇒ 语义空 ⇒ 删除;Quota `null` ⇒ 零值文档 ⇒ live)。✔ 与计划 §1 完全一致。
|
||||
- `*string`:省略/`null` ⇒ nil ⇒ 不动;空串 ⇒ 已提供且内容为空 ⇒ 四类删除、两类 no-op。✔
|
||||
- 这些由 `TestPeerBucketMetadataWireAtomicity` 通过**真实 admin 路由 + 真实 JSON 编解码**覆盖(`applySRBucketMetaViaAdmin` 走 `registerAdminRouter`),不是结构体直传。这是这批测试里质量最高的一块。
|
||||
- **direct hook 与 heal 的删除传播**:普通 DELETE 事件在 gate off 下照常复制;heal 依赖导出可见性 ⇒ 见 F7。
|
||||
- **初次同步**:保留原五类范围(不含 Versioning),gate 控制是否发送真实墓碑。✔
|
||||
- **gate 覆盖点是否遗漏**:我检查了所有导出/初次发送点 —— `SiteReplicationMetaInfo` 的 Tags/Quota/SSE 三处(`:3960/:3980/:3988`)+ `initialBucketConfigReplicationEvent:51`。Policy 墓碑**故意**不受 gate 控制(原本已导出);Versioning/Object Lock 无墓碑概念。**未发现遗漏的开关覆盖点。** ✔
|
||||
- **滚动升级/降级**:off 时新增墓碑信息不导出,旧节点不会收到它无法正确处理的 Quota heal 墓碑(文档点名了"旧版会留下已解析 quota 残留"这一实证依据)。✔ 但 off **不等于**与旧版同构:修复端的**接收**行为已经变了(锁内排序、空 baseline 不删除、重复不写),这对旧版发来的事件是更安全的方向,混版冒烟也验证了普通 PUT/DELETE 互通。降级路径文档要求先关开关再滚降;唯一可补的一句是"墓碑时间字段本来就在 schema 内,旧版只是不导出,降级不会产生解析错误"。
|
||||
|
||||
---
|
||||
|
||||
## 6. 三个独立判断
|
||||
|
||||
**最小?—— 基本是,有一处可争议的扩张。**
|
||||
生产 Go 代码净增约 37 行,六份重复的 heal / peer apply 被一套 helper 替代;没有新 schema、没有新 wire 字段、没有新锁、没有能力协商、没有迁移系统。唯一超出必要的是 **Policy 规范编码器接入写/读/导出路径**(F5)——它不是收敛所必需的,并且直接导致 F2。其余(`bucketMetadataUpdate` 提交快照、`ensureBucketMetadataCreated`、`rebaseBucketConfigDefaults`、gate)我都能各自对应到一个已证实的缺陷或本次修复直接触及的路径,**没有**可以无损删除的部分。
|
||||
|
||||
**充分?—— 对"已声明的范围"是;对"缺陷标题"不是。**
|
||||
在同一桶世代、Created 已知、gate=on 的前提下,六类配置的源时间、锁范围、删除状态参与 heal、等时冲突裁决、重复/乱序抑制都成立。但两个口子要明说:(1) 默认 gate=off ⇒ Tags/SSE/Quota 的漏发删除不收敛(F7,计划内取舍);(2) Created 未知的桶从"能写"变成"不能写"(F1,计划外回归)。
|
||||
|
||||
**必要?—— 是。**
|
||||
每一项改动都能对应到 `baseline.log` 里的实测失败(SOURCE_TIME / NEWER_DELETE / STALE_RESURRECTION / BULK_STALE_OVERWRITE / CHECK_OUTSIDE_LOCK)或计划中论证过的路径。`rebaseBucketConfigDefaults` 这种看起来最"多余"的小分支,实际是 `Created` 一旦参与 baseline/tombstone 判定后的必然补丁(不加则接管时默认值会变成假墓碑)。
|
||||
|
||||
---
|
||||
|
||||
## 7. 新增设计记录应准确保留的内容
|
||||
|
||||
**关键决策**
|
||||
1. baseline / live / tombstone 三态由 `(Created, 字段时间, 载荷空否)` 推导,**不新增 schema 字段**;`at == Created` 定义为 baseline。代价:必须知道 Created(⇒ F1 的根因)。
|
||||
2. 全序:`valid > real > at > 同时间墓碑胜 > 稳定内容键`。deployment ID **不**参与比较,也不落盘。
|
||||
3. 比较键是"已解析配置的纯函数":Policy 走递归排序的规范 JSON,Quota 走解析后 `json.Marshal`,XML 走有效文档字节,Versioning 先按本站点 Object Lock 归一。
|
||||
4. Versioning / Object Lock 为 update-only:空事件恒为 no-op,空值不是候选。
|
||||
5. 读-比较-写整体在**既有** `metadata.lock` 内;`saveMetadata` 收指针以便调用方拿到提交后快照,保证"落盘状态 / 出站事件 / 源时间"三元组一致。
|
||||
6. 新增删除信息的**导出**用启动开关 gate,默认 off;开关不探测远端能力,启用条件是全站点全节点已升级且旧请求排空。
|
||||
7. 专用 peer 事件的零时间保留兼容例外(分配本地时间 + 限频 `legacy-zero`);bulk 零时间仍按现状拒绝。
|
||||
|
||||
**被拒绝的方案**(记录以免复议)
|
||||
- 新增 HLC / 向量时钟 / 新 wire 字段 / 能力协商 / 通用复制框架 / 新锁或重试系统。
|
||||
- 用 `len(payload)==0 ⇒ 删除` 统一处理 bulk(会把"省略"误判为删除)。
|
||||
- 把 `BucketPolicy.Equals`(忽略 Sid、对 Statement 顺序敏感)叠加为第二套判等规则。
|
||||
- 把混版长期测试作为提交门槛。
|
||||
- 把未修改上游 MinIO 的兼容性当作必需门禁(AGENTS.md:正式支持 PGSTY 栈,上游兼容为尽力而为)。
|
||||
|
||||
**限制**
|
||||
- gate=off 期间 Tags/SSE/Quota 墓碑不可见,不承诺删除收敛,且存在被拒绝的周期性 RPC。
|
||||
- 旧版到达时间污染、legacy-zero 产生的新本地时间、桶创建世代分歧**无法自动反推**,需运维在权威站点重新提交。
|
||||
- `Created` 未知的桶不在收敛承诺内(并且按当前实现直接不可写,见 F1)。
|
||||
- "比较键相同但落盘字节不同"(Quota 原始 JSON、Policy 旧编码)是允许的稳定状态,公开 mismatch 统计必须与比较键口径一致(见 F2)。
|
||||
|
||||
---
|
||||
|
||||
## 8. 建议处置顺序
|
||||
|
||||
1. 判定 F1 的条件(支持矩阵里有没有 `Created == 0` / 无 `.metadata.bin` 的桶)。有 ⇒ 先修再合。
|
||||
2. 修 F2(推荐:不排序顶层 `Statement`,或让 `isBktPolicyReplicated` 改用规范键)。
|
||||
3. 修 F3(ERROR→Warning、缺桶不报、reason 拆分)。
|
||||
4. 补 F4 的三条文档。
|
||||
5. 补测试:F10 的接管用例、F8-1 的 legacy 策略字节双站点用例、F8-2 去 stub 的 Created 用例、F8-4 的二进制身份记录。
|
||||
6. F5 / F6 由作者判断,可留作后续。
|
||||
@@ -1 +0,0 @@
|
||||
ok github.com/minio/minio/cmd 6.864s
|
||||
@@ -1 +0,0 @@
|
||||
ok github.com/minio/minio/cmd 20.694s
|
||||
@@ -1,57 +0,0 @@
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery (0.57s)
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/ErasureSD/object-lock.xml/missing=false (0.00s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update object-lock.xml: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/ErasureSD/versioning.xml/missing=false (0.00s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update versioning.xml: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/ErasureSD/policy.json/missing=false (0.00s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update policy.json: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/ErasureSD/tagging.xml/missing=false (0.00s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update tagging.xml: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/ErasureSD/bucket-encryption.xml/missing=false (0.00s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update bucket-encryption.xml: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/ErasureSD/quota.json/missing=false (0.00s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update quota.json: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/ErasureSD/object-lock.xml/missing=true (0.00s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update object-lock.xml: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/ErasureSD/versioning.xml/missing=true (0.00s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update versioning.xml: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/ErasureSD/policy.json/missing=true (0.00s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update policy.json: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/ErasureSD/tagging.xml/missing=true (0.00s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update tagging.xml: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/ErasureSD/bucket-encryption.xml/missing=true (0.00s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update bucket-encryption.xml: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/ErasureSD/quota.json/missing=true (0.00s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update quota.json: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/Erasure/object-lock.xml/missing=false (0.02s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update object-lock.xml: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/Erasure/versioning.xml/missing=false (0.02s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update versioning.xml: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/Erasure/policy.json/missing=false (0.02s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update policy.json: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/Erasure/tagging.xml/missing=false (0.02s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update tagging.xml: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/Erasure/bucket-encryption.xml/missing=false (0.06s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update bucket-encryption.xml: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/Erasure/quota.json/missing=false (0.02s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update quota.json: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/Erasure/object-lock.xml/missing=true (0.04s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update object-lock.xml: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/Erasure/versioning.xml/missing=true (0.02s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update versioning.xml: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/Erasure/policy.json/missing=true (0.02s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update policy.json: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/Erasure/tagging.xml/missing=true (0.01s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update tagging.xml: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/Erasure/bucket-encryption.xml/missing=true (0.02s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update bucket-encryption.xml: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketMetadataPhysicalCreatedRecovery/Erasure/quota.json/missing=true (0.02s)
|
||||
site-replication-metadata-gate_test.go:227: bucket without a recorded creation time cannot update quota.json: bucket metadata creation time is unknown
|
||||
--- FAIL: TestBucketPolicyReplicationStatusLegacyOrder (0.31s)
|
||||
--- FAIL: TestBucketPolicyReplicationStatusLegacyOrder/ErasureSD (0.01s)
|
||||
site-replication-metadata_test.go:348: equivalent legacy and received policy reported as permanently mismatched
|
||||
--- FAIL: TestBucketPolicyReplicationStatusLegacyOrder/Erasure (0.06s)
|
||||
site-replication-metadata_test.go:348: equivalent legacy and received policy reported as permanently mismatched
|
||||
FAIL
|
||||
FAIL github.com/minio/minio/cmd 2.940s
|
||||
FAIL
|
||||
-8
@@ -1,8 +0,0 @@
|
||||
--- FAIL: TestHealBucketConfigDiagnostics (0.29s)
|
||||
--- FAIL: TestHealBucketConfigDiagnostics/ErasureSD (0.00s)
|
||||
site-replication-metadata-heal_test.go:80: empty baselines produced diagnostics
|
||||
--- FAIL: TestHealBucketConfigDiagnostics/Erasure (0.01s)
|
||||
site-replication-metadata-heal_test.go:80: empty baselines produced diagnostics
|
||||
FAIL
|
||||
FAIL github.com/minio/minio/cmd 2.721s
|
||||
FAIL
|
||||
@@ -1,8 +0,0 @@
|
||||
--- FAIL: TestBucketMetadataInitialSyncPhysicalCreated (0.47s)
|
||||
--- FAIL: TestBucketMetadataInitialSyncPhysicalCreated/ErasureSD (0.04s)
|
||||
site-replication-metadata-gate_test.go:289: peer creation time "0001-01-01T00:00:00Z", want physical time 2026-09-12 06:03:09 +0000 UTC
|
||||
--- FAIL: TestBucketMetadataInitialSyncPhysicalCreated/Erasure (0.08s)
|
||||
site-replication-metadata-gate_test.go:289: peer creation time "0001-01-01T00:00:00Z", want physical time 2026-09-12 06:03:09 +0000 UTC
|
||||
FAIL
|
||||
FAIL github.com/minio/minio/cmd 2.576s
|
||||
FAIL
|
||||
@@ -1,18 +0,0 @@
|
||||
--- FAIL: TestHealBucketConfigWithoutSourceDiagnostics (0.36s)
|
||||
--- FAIL: TestHealBucketConfigWithoutSourceDiagnostics/ErasureSD (0.00s)
|
||||
--- FAIL: TestHealBucketConfigWithoutSourceDiagnostics/ErasureSD/unknown-created (0.00s)
|
||||
site-replication-metadata-heal_test.go:155: got 0 diagnostics, want 1: []
|
||||
--- FAIL: TestHealBucketConfigWithoutSourceDiagnostics/ErasureSD/malformed (0.00s)
|
||||
site-replication-metadata-heal_test.go:155: got 0 diagnostics, want 1: []
|
||||
--- FAIL: TestHealBucketConfigWithoutSourceDiagnostics/ErasureSD/before-created (0.00s)
|
||||
site-replication-metadata-heal_test.go:155: got 0 diagnostics, want 1: []
|
||||
--- FAIL: TestHealBucketConfigWithoutSourceDiagnostics/Erasure (0.01s)
|
||||
--- FAIL: TestHealBucketConfigWithoutSourceDiagnostics/Erasure/unknown-created (0.00s)
|
||||
site-replication-metadata-heal_test.go:155: got 0 diagnostics, want 1: []
|
||||
--- FAIL: TestHealBucketConfigWithoutSourceDiagnostics/Erasure/malformed (0.00s)
|
||||
site-replication-metadata-heal_test.go:155: got 0 diagnostics, want 1: []
|
||||
--- FAIL: TestHealBucketConfigWithoutSourceDiagnostics/Erasure/before-created (0.00s)
|
||||
site-replication-metadata-heal_test.go:155: got 0 diagnostics, want 1: []
|
||||
FAIL
|
||||
FAIL github.com/minio/minio/cmd 2.430s
|
||||
FAIL
|
||||
@@ -1,174 +0,0 @@
|
||||
# SILO #77 修正提交独立复审(固定 `62cf066ff529c7d281703daa365f555cebba717a`)
|
||||
|
||||
审查方式:只读。以 worktree 生产源码为准(不只读 diff),交叉核对 `silo-pkg v3.13.4-0.20260910091716` 依赖源码、证据目录与设计记录。上一轮报告只作为待验证的命题,不作为已确认结论。
|
||||
|
||||
---
|
||||
|
||||
## 0. 结论
|
||||
|
||||
**GO_WITH_NONBLOCKING_NOTES**
|
||||
|
||||
- **阻断项:0**(无条件 0,条件性 0)
|
||||
- 上一轮的条件性阻断 F1 与 S2 级 F2 均已在生产路径上真正关闭,并有"最终测试 + 旧生产代码覆盖"的复现证据(`round-2/confirmed-before.log`)。
|
||||
- 新增 8 项非阻断发现(NB-1 ~ NB-8),其中 3 项是行为残留(S3/S4),5 项是证据与文档表述准确性问题。
|
||||
- 一处需要纠正上一轮的**判断**(不是纠正实现):F5 中"Policy 规范编码器接入 GET/export/peer 属于超出必要的扩张"这一说法不成立——这三个调用点持有的是**已解析策略**,必须有编码器,而旧编码器对解析器已支持的负集合策略**必然报错**。真正"可选"的只有 PUT/import 两处,且因 `DisallowUnknownFields` 而无损。
|
||||
|
||||
---
|
||||
|
||||
## 1. F1–F10 关闭状态
|
||||
|
||||
| 项 | 上轮判定 | 本轮核实 | 状态 |
|
||||
|---|---|---|---|
|
||||
| F1 零 Created 桶六类配置写入硬失败、初次同步静默漏发 | S1 条件阻断 | 生产路径已修复并有真实 ObjectLayer + 旧码覆盖复现 | **关闭**(残留见 NB-1) |
|
||||
| F2 Policy 状态口径与收敛键冲突导致永久假 mismatch | S2 | `isBktPolicyReplicated` 改用同一 canonical 键,per-site presence 不变,无 SID/ID/Condition/未知值回归 | **关闭** |
|
||||
| F3 heal 诊断 ERROR 级、key 互相顶替、正常瞬态误报 | S3 | ERROR→Warning、四类原因各自持 key、缺桶不报 = 已关闭;"全无有效来源即完全静默"与"按桶×字段扇出"未处理 | **部分关闭**(NB-2、NB-3) |
|
||||
| F4 三处语义变更未文档化 | S3 | 上轮前两项系误读(原提交 README 已有,见 diff 中为 context 行);第三项已补入 `docs/site-replication/README.md:129-134` | **关闭 + 原误读已纠正** |
|
||||
| F5 Policy 规范编码器接入面 | 最小性争议 | 判断更新:GET/export/peer 非可选;PUT/import 可选但无损。建议保留现状 | **关闭(判断修正)** |
|
||||
| F6 死代码与过时注释 | nit | 注释已修(`bucket-metadata-replication.go:35-37`);`isBucketMetadataEqual` 仍为测试专用 | **关闭(保留 nit,见 NB-5)** |
|
||||
| F7 gate=off 不承诺四类删除自愈 | 范围说明 | README:75-89、设计记录 §rollout/§limits 均如实声明,未说成已自动修复 | **保持(如实)** |
|
||||
| F8 证据缺口(stub / 旧顺序策略 / 二进制身份) | 证据不足 | stub 已去除、旧顺序用例已补、二进制身份已闭环(`--version` 含 commit-id + 对照基线二进制) | **关闭**(但 NB-4:两份旧日志不可用作证据) |
|
||||
| F9 桶世代冲突 | 范围外 | 未被声称已修;设计记录 §limits-3 明确 | **保持(如实)** |
|
||||
| F10 接管 Created 越过真实字段时间 | 覆盖缺口 | 补 `shift=3h` 子用例,钉住"旧世代状态不得升级为有效来源" | **关闭(半边,见 NB-7)** |
|
||||
|
||||
---
|
||||
|
||||
## 2. 本轮重点逐项核实
|
||||
|
||||
### F1 —— 真正修好了,机制比上轮建议更完整
|
||||
|
||||
**`NoMetadata=true` 返回物理探测**(`cmd/erasure-server-pool.go:2279-2283`)。我枚举了全仓库 `NoMetadata` 的全部生产调用点,只有三处:
|
||||
|
||||
- `cmd/bucket-metadata-sys.go:272`(`saveMetadata` 存在性重查,只看 error)
|
||||
- `cmd/bucket-metadata.go:306`(迁移锁内存在性重查,只看 error)
|
||||
- `cmd/bucket-metadata-replication.go:281`(新增的 Created 恢复,正是要物理值)
|
||||
|
||||
其余全部是 `BucketOptions{}`(`NoMetadata=false`),行为不变。被跳过的 `globalBucketMetadataSys.Get()`(`bucket-metadata-sys.go:410-424`)是**纯缓存读、无副作用、不触发加载**,因此跳过它不会改变任何缓存预热或懒加载行为。`Versioning`/`ObjectLocking` 两个字段也只在非 NoMetadata 分支填,与 `ListBuckets`(`:2402/2422` 只在 `!NoMetadata` 时覆盖 Created)语义一致。**无调用者回归。**
|
||||
|
||||
**世代一致性链路完整**(这一点比上轮提出的方案更强,值得记录):
|
||||
`syncToAllPeers`(`site-replication.go:2145-2153`)恢复 `meta.Created` → 传入 `MakeBucketOptions.CreatedAt` → `MakeBucketHook`(`:821-826`)在缓存值为零时回落到 `opts.CreatedAt`(此时本地磁盘仍是 0,所以这个回落是必需的)→ **本地**分支 `PeerBucketMakeWithVersioningHandler`(`:943-966`)在 `metadata.lock` 内 `SetCreatedAt(opts.CreatedAt)` + `rebaseBucketConfigDefaults` + `saveMetadata`,**远端**分支通过 `optsMap["createdAt"]` 收到同一个值。也就是说本地并非"只在内存里补一下",而是与全部对端落盘同一个世代值。这条链路 diff 上看不出来,必须读 `MakeBucketHook` 的 `concDo` 第一个闭包才能确认。
|
||||
|
||||
四个恢复点的锁位置也确认无误:写路径(`bucket-metadata-sys.go:186`,锁内)、bulk(`site-replication.go:1674`,锁内)、import(`admin-bucket-handlers.go:1092`,锁内)、初次同步(锁外探测但不落盘,落盘由上面的钩子在锁内完成)。
|
||||
|
||||
**"无法恢复时显式错误"是否正确:正确。** 静默发明 `UTCNow()` 会造出一个不可撤销、且会向全站点传播的假世代;而显式失败保留原数据、可重试、并留一条有界 `indeterminate`。`syncToAllPeers` 因此让整个 `AddPeerClusters` 失败——这在真实文件系统上基本不可达(`StatVol` 用目录 `ModTime`,存在即非零),且 `AddPeerClusters` 是可重试的交互式管理操作,代价可接受。
|
||||
|
||||
**真实用例是否覆盖生产路径:是。** `TestBucketMetadataPhysicalCreatedRecovery`(`site-replication-metadata-gate_test.go:206-237`)通过 `globalBucketMetadataSys.save` 让**缓存持有 Created=0**,再走 `Update` → `updateAndParseMetadata` → `ensureBucketMetadataCreated` → 真实 `erasureServerPools.GetBucketInfo`;`ExecObjectLayerAPITest`(`test-utils_test.go:1766/1795`)两个后端都是真实 `erasureServerPools`,因此缓存覆盖这一段是真跑到的。`setPhysicalBucketCreated` 直接 `os.Chtimes` 每块盘的桶目录,正是 `xl-storage.go:1022-1028` 取值的来源。六类配置 × `missing=true/false` 全覆盖。
|
||||
|
||||
### F2 —— 关闭,且口径现在六类全部自洽
|
||||
|
||||
`site-replication.go:3782-3809`。状态侧的 `policies[i]` 来自 `:3436` 对**各站点原始字节的解析**,与 heal 的 `bucketConfigPayload`→`canonicalBucketPolicy`(`bucket-metadata-replication.go:191`)是同一个纯函数,因此 peer 重排 Statement 后的永久假 mismatch 被彻底关闭。`numPolicies != total` 的 per-site presence 计数原样保留(`:3783-3792`),并有断言覆盖。
|
||||
|
||||
回归面逐条查过:
|
||||
|
||||
- **SID**:旧 `BPStatement.Equals`(silo-pkg `bucket-policy-statement.go:148-171`)**不比较 SID**,新键包含 `Sid`。这是方向正确的收紧——heal 的收敛键同样含 SID,所以状态报出的 mismatch 是**真实且会被 heal 自动消除**的,不是永久噪声。用例已钉住。
|
||||
- **ID / Version**:旧 `Equals` 比较二者,新键在 `ID != ""` 时包含、`Version` 始终包含,等价。
|
||||
- **Condition**:两侧都走同一 `condition.Functions` 编码再递归排序,顺序无关。
|
||||
- **未知值**:`ParseBucketPolicyConfig` 用 `decoder.DisallowUnknownFields()`(`bucket-policy.go:173`),未知字段**根本无法通过解析**,因此不存在"键把未建模字段静默吃掉"的风险。
|
||||
- `isIAMPolicyReplicated`(`:3665-3681`)仍用 `Equals`,那是 IAM 策略、不同子系统,**不应**一并改动。正确地没动。
|
||||
|
||||
新旧行为是"收紧而非放宽",与收敛契约一致,判断合理。
|
||||
|
||||
### F3 —— 关闭了会吃日志的部分,但静默面仍偏大
|
||||
|
||||
已关闭:四种原因各自持 key(`site-replication-metadata.go:145/158/164/189`),真实 heal RPC 失败不再被"缺 peer"顶掉;`logger.WarningKind`(`:42`);对端"尚未拥有该桶"(`CreatedAt=0` 且载荷空、时间零)不再报。`siteReplicationStatus` 对不可达 peer 填 `DeploymentID=""`(`:3029`)落进 `BucketStats[bucket][""]`,而循环 1 按 `info.Sites` 的真实 ID 判缺失、循环 2 跳过 `id==""`,两条路径正交,语义正确。README:112-119 与实现完全一致。
|
||||
|
||||
未关闭的两点见 NB-2、NB-3。另外,"仅报告确实存在、但仍无法排序的状态"(`:157`)这个条件写得准确:`!current.valid && (len(data)!=0 || !at.IsZero())` 恰好把 F9/F10 的世代冲突暴露出来,这是意外的正收益。
|
||||
|
||||
### F5 —— 上一轮的最小性判断需要修正
|
||||
|
||||
`canonicalBucketPolicy` 的六个生产接入点应分成三类,而不是笼统的"超出必要":
|
||||
|
||||
| 接入点 | 性质 |
|
||||
|---|---|
|
||||
| `bucket-metadata-replication.go:191`(比较键) | **必需**,#77 的核心 |
|
||||
| `site-replication.go:3802`(状态键) | **必需**,F2 的修复本体 |
|
||||
| `bucket-policy-handlers.go:202`(GET)、`admin-bucket-handlers.go:438`(export)、`site-replication.go:1723`(peer apply) | **非可选**:这三处拿到的都是 `*policy.BucketPolicy`(`globalPolicySys.Get` / `GetBucketPolicy` / `admin-handlers-site-replication.go:228` 解析后传入),必须有编码器;而旧的 `json.Marshal` 对 `NotAction`/`NotResource` 语句**必然失败**——`BPStatement.Actions` 无 `omitempty`(`bucket-policy-statement.go:31`)+ `ActionSet.MarshalJSON` 空集报错(`actionset.go:144-148`)。bulk/import/peer 都能把这类策略落盘,所以缩回任何一处都会重新制造"写得进、读不出/复制不出" |
|
||||
| `bucket-policy-handlers.go:102`(PUT)、`admin-bucket-handlers.go:922`(import) | **可选的规范化**:比较键不依赖落盘字节。但因为解析器 `DisallowUnknownFields`,规范化**无信息丢失**(唯一差别是重复语句在盘上也被去重,语义等价),而所有读路径都已重新编码,用户观察不到"原样字节"。删掉它只换来与客户端提交字节的一致性,且会让本地写与 peer 写产生两种盘上形态 |
|
||||
|
||||
**结论:不建议缩回任何一处。** 真正可删的只有 PUT/import 两处,收益为零、改动风险非零。上一轮把这部分列为"范围外扩张"的说法应当被本轮判断取代;它是一个被同一改动暴露出来的真实读回缺陷的修复,且已在 README:129-134 声明。
|
||||
|
||||
---
|
||||
|
||||
## 3. 新发现(全部非阻断)
|
||||
|
||||
### NB-1(S3)恢复出的物理 Created 不确定,且可能晚于对端事件的来源时间
|
||||
|
||||
- **位置**:`cmd/bucket-metadata-replication.go:277-290`;取值链 `cmd/peer-s3-client.go:324-328`(取第一个不报错的 peer)→ `cmd/peer-s3-server.go:207-258`(`cloneDrives` 遍历 **Go map**,取第一个不报错的盘)→ `cmd/xl-storage.go:1022-1028`(`Created = st.ModTime()`)。
|
||||
- **最短触发**:桶无 `.metadata.bin`;其顶层目录 mtime 因近期对象写入被推进到接近 now;对端发来源时间早于该 mtime 的事件 → `bucket-metadata-sys.go:198-201` 判 `before-created` → **返回 nil、不写、不报错**,仅一条每小时去重的 warning,直到有人在源站重新提交。
|
||||
- **影响**:(a) 恢复值不是桶的稳定属性(目录 mtime 随顶层条目增删前进,且逐盘不同);(b) 两个站点各自独立恢复时会落进不同世代。二者都落在已声明的 F9 限制内,且相对基线(基线给每个站点各自 `UTCNow()`)是改善;本地侧因 `metadata.lock` 串行化 + 首写落盘,站内不会分裂。真正新的只有"首个跨站事件可能被静默跳过"。
|
||||
- **最小修复(可选)**:让 `ensureBucketMetadataCreated` 返回"本次是否为恢复值",当 `sourceTime` 非零且早于恢复值时取 `min(recovered, sourceTime)`——"在时刻 T 已持有配置的桶必然在 T 已存在",约 4 行,不新增字段。或退一步:只在 README 里把恢复值明确为"物理近似值,可能晚于真实创建"。
|
||||
- **需补证据**:一个用例——缓存 `Created=0`、`os.Chtimes` 把桶目录设为 now、通过 admin 路由投递 `UpdatedAt = now-1h` 的 peer 事件,断言期望行为(写入或明确跳过),把当前语义钉死。
|
||||
|
||||
### NB-2(S3)`!found` 时对畸形/不可排序状态完全静默
|
||||
|
||||
- **位置**:`cmd/site-replication-metadata.go:134-139`。
|
||||
- **最短触发**:全部站点的同一字段都无法成为候选(例如各站点都存着一份 base64 可解但内容非法的 XML,或全部世代未知)。`latestBucketConfig`(`:117-120`)对 `err != nil` 静默 `continue`,`healBucketConfig` 直接 `return nil`——**没有任何诊断**。而状态导出侧的 `logInvalid`(`site-replication.go:3375-3378`)只覆盖 base64 失败和 policy/quota/replication 的解析失败,XML 类内容非法不在其中,于是这种状态在两条路径上都无声。
|
||||
- **影响**:字段永久不可 heal 且无信号。范围窄(要求所有站点同时不可用),但属于"必要诊断"缺口。
|
||||
- **最小修复**:把每目标的 `bucketConfigStateFromInfo` 结果算一遍并缓存(当前每周期每目标实际解码两次:`latestBucketConfig` 一次、主循环一次),把 `currentErr != nil` 的 `indeterminate` 移到 `if !found` 之前。既补诊断又减一次解码,不引入新框架。
|
||||
- **需补证据**:`healBucketConfig` 单测——两个站点都给非法载荷,断言返回 nil 且产生一条 `indeterminate`(可用现有 `srStatusInfo` 夹具,参照 `site-replication-metadata-heal_test.go:136-143`)。
|
||||
|
||||
### NB-3(S4)`unreachable` 诊断按"桶 × 字段"扇出
|
||||
|
||||
- **位置**:`cmd/site-replication-metadata.go:143-147`,被 `cmd/site-replication.go:4750-4760` 每桶调用六次。
|
||||
- **最短触发**:一个站点掉线。每桶产生 6 个不同 key → 10k 桶集群每小时约 6 万条 warning。
|
||||
- **影响**:只是噪声(`logOnceIf` 按小时清理,仍算"有界"),但一个站点级事实被放大成桶×字段级输出。
|
||||
- **最小修复**:把循环 1 上提到 `healBuckets` 里、每桶调用一次(key 去掉 `file` 段),或干脆每周期一次。改动 <15 行。
|
||||
- **需补证据**:不需要新证据,行数/键数可直接核对。
|
||||
|
||||
### NB-4(S4,证据)两份旧日志是草稿测试产物,不能作为产品失败证据;设计记录"三个回归测试"应为两个
|
||||
|
||||
- **位置**:`findings-before.log:3`、`findings-after-1.log:3` 均为 `site-replication-metadata-gate_test.go:230: real physical creation time unavailable`。该消息在最终测试文件中**不存在**,最终断言在 **227** 行、消息是 `bucket without a recorded creation time cannot update %s: %v`。同理 `findings-before.log:52` 失败在 SID 断言(`:353`),而最终文件 SID 断言在 `:355`、`:353` 是 `t.Fatal(err)`。
|
||||
- **判断**:这两份日志跑的是早期草稿;其中 "real physical creation time unavailable" 更像**测试自身的前置检查**失败,不能用来证明产品缺陷。真正成立的是 `round-2/confirmed-before.log`:它用**最终测试**叠加 `before-erasure-server-pool.go` / `before-site-replication.go`(我已核对这两份确实是修正前版本:无 `NoMetadata` 早返回、`MakeBucketHook` 无回落、`syncToAllPeers` 仍用 `bucketInfo.Created`、`isBktPolicyReplicated` 仍是 `prev.Equals`),行号 227/348 与最终文件精确吻合,失败文本是真实产品错误 `bucket metadata creation time is unknown` 与 `equivalent legacy and received policy reported as permanently mismatched`,两后端 × 六类 × missing 双态全覆盖;`confirmed-after.log` 通过。
|
||||
- **另一项**:设计记录第 166 行"三个回归测试在未修复代码上失败、修复后通过"不准确。仓库中只有**两个**这样的测试;F3(heal 诊断)**没有任何自动化回归**(我在全部 `*_test.go` 中检索 `unreachable`/`peer-error`/`unusable peer`/`logBucketConfigReplication`,无命中)。F3 的唯一证据是 `runtime-after-review.log:20` 的驱动自述"40 exceptional events produced exactly one log per reason"。
|
||||
- **最小修复**:把该行改为"两个回归测试 + 一次运行时观察",并在证据索引里把 `findings-before.log` / `findings-after-1.log` 标注为 superseded 草稿。
|
||||
- **需补证据(可选)**:给 F3 补一个日志断言用例(注入 logger target 或计数 hook),否则 F3 的修复在回归套件里是无保护的。
|
||||
|
||||
### NB-5(S4)不要为死代码创造兼容理由
|
||||
|
||||
`isBucketMetadataEqual`(`cmd/site-replication.go:5172-5181`)仅被 `site-replication_test.go:156`、`bucket-cors-site-replication_test.go:484` 引用。代码里没有编造理由(很好),但设计记录第 186 行写"作为上游血缘保留"是一个不成立的说法。**最小修复**:改成"目前仅测试引用,暂不删除以缩小 diff",或连同两处测试一并删除。
|
||||
|
||||
### NB-6(S4)README 对恢复发生位置的描述偏保守
|
||||
|
||||
`docs/site-replication/README.md:106-110` 说"That recovery happens on the write path … 第一次配置写入(本地或复制而来)会记录物理时间"。实际上 `AddPeerClusters` 的初次同步(`site-replication.go:2145` → `MakeBucketHook` → 本地 `PeerBucketMakeWithVersioningHandler`)也会在本地落盘该时间。这是**低估**不是夸大,但会误导运维判断"何时脱离跳过状态"。**最小修复**:补半句"或在把该桶纳入站点复制时"。
|
||||
|
||||
### NB-7(S4)F10 只钉住了"来源"半边
|
||||
|
||||
`cmd/site-replication-metadata_test.go:473-483` 断言接管把 Created 调晚后,越过的真实字段**不能成为来源**(`!state.candidate()`)。但上一轮指出的另一半——该字段作为 **heal 目标**时,`current.valid=false` 使任何合法 incoming 无条件胜出(`site-replication-metadata.go:174` + `bucket-metadata-replication.go:239-245`)——没有钉住。我确认这半边行为存在且在"远端世代胜出"语义下是自洽的,本地用户 PUT 也总能覆盖回来(`localBucketConfigUpdatedAt` 保证 `> Created`),**不是缺陷**,但期望值应当被固定。**最小修复**:同一子用例再加一次 `applyBucketConfig(&got, file, <远端载荷>, created+shift+1m)`,断言返回 `changed=true`。
|
||||
|
||||
### NB-8(S4)恢复失败时的诊断丢掉了事件来源时间
|
||||
|
||||
`cmd/bucket-metadata-sys.go:187` 传 `time.Time{}` 作为 `at`,即使本次是携带 `sourceTime` 的 peer 事件,日志里也显示 `0001-01-01`。**最小修复**:`sourceTime != nil` 时传 `*sourceTime`(一行)。
|
||||
|
||||
---
|
||||
|
||||
## 4. 证据分级
|
||||
|
||||
| 证据 | 评价 |
|
||||
|---|---|
|
||||
| `round-2/confirmed-before.log` + `confirmed-after.log` + `before-*.go` 覆盖文件 | **足够**。最终测试 × 旧生产代码,行号与失败文本可逐条核对,F1/F2 为已证实的产品失败 |
|
||||
| `binary-identity-after-review.json` | **足够**。`silo-final --version` 含 `commit-id=62cf066ff…`,并附基线 `5c5765816` 对照二进制与双方 sha256,上一轮 F8-4 的缺口已闭合 |
|
||||
| `final-cmd-after-review.log`(cmd 全包 622s, EXIT=0)、`ci-lint-after.log`(0 issues, 品牌基线未变)、`ci-verify-after.log`、`ci-gen/internal/s3select/crosscompile-after.log` | **足够**作为"未引入回归"的门禁证据(只截尾行,可接受) |
|
||||
| `findings-before.log`、`findings-after-1.log` | **不可用**。草稿测试产物,行号与失败文本均与最终测试不符(见 NB-4) |
|
||||
| `findings-after.log` | 仅"ok",无信息量,被 `confirmed-after.log` 覆盖 |
|
||||
| `runtime-after-review.log` | **作者自述为主**。日志由作者的双站点驱动产生,断言("six field states equal""zero metadata RPCs""exactly one log per reason")都由驱动自己判定。时间戳 06:01:50 与最终二进制构建时间 06-01-35Z 吻合,身份链成立;但它是端到端观察,不是独立验证。F3 目前**只有**这一条证据 |
|
||||
|
||||
---
|
||||
|
||||
## 5. 设计记录(`bucket-metadata-convergence.zh.md`)的准确性
|
||||
|
||||
总体克制,未发现把限制说成已修复的地方:§rollout(gate=off 不承诺四类删除收敛)、§limits-1/2/3(历史污染、零时间事件、桶世代冲突)都如实写成限制;第 20 行明确"不能用来证明现有下载包或线上实例已具备这些能力";第 161 行主动声明交叉编译 ≠ 六平台运行验收。需要修正的只有三处,均已在上面列出:
|
||||
|
||||
1. 第 166 行"三个回归测试" → 两个回归测试 + 一次运行时观察(NB-4)。
|
||||
2. 第 186 行 `isBucketMetadataEqual`"上游血缘保留"是编造的理由(NB-5)。
|
||||
3. 第 181 行"与 `ListBuckets` 一致"成立(`ListBuckets` 在 `NoMetadata` 时同样不覆盖 Created),但第 185 行把 F5 记为"有意扩张,不是缺陷"仍沿用了上一轮的定性——按本轮判断,GET/export/peer 三处属于**必需**,建议改写,否则会给未来的人留下"这块可以缩回"的错误暗示。
|
||||
|
||||
---
|
||||
|
||||
## 6. 最小性 / 充分性 / 必要性(独立评价)
|
||||
|
||||
**最小性:是。** 本轮修正净增很小,且每一处都能对应到一个已复现的失败或一次被证实的误读:`NoMetadata` 早返回 5 行、`MakeBucketHook` 回落 3 行、`syncToAllPeers` 恢复 4 行、`isBktPolicyReplicated` 换键约 10 行、heal 诊断重排约 15 行、`isReplicatedBucketConfig` 具名化(消除"用一次性零值结构体探测类型"的晦涩写法)、README 两段。没有新 schema、新 wire 字段、新锁、新框架。我找不到可以无损删除的部分——唯一"可删"的 `canonicalBucketPolicy` PUT/import 接入是上一轮遗留,删掉反而制造两种盘上形态。
|
||||
|
||||
**充分性:对已声明契约充分,对两个边缘不充分且已如实声明。** 在"同一桶世代、Created 已知或可恢复、gate=on"范围内,六类配置的来源时间、锁范围、删除参与 heal、等时裁决、重复/乱序抑制、状态口径,我没有找到反例。不充分的两处都写进了文档:gate=off 下 Tags/SSE/Quota 漏发删除不收敛(F7);跨站点桶世代分歧不自动合并(F9,NB-1 属于它的子集)。F3 的静默面(NB-2)是本轮唯一一个"文档声明了、但我认为声明本身偏宽"的地方——"没有任何站点可供传播的字段就不产生诊断"对畸形配置而言不该成立。
|
||||
|
||||
**必要性:是。** F1 与 F2 都有"最终测试 + 修正前生产代码"的复现(`confirmed-before.log`),是确证的产品失败,不是理论推演;F3 的必要性来自 `logOnceIf` 按 key+正文去重这一可验证机制,虽无自动化回归但代码级推理成立;F4/F8/F10 是文档与证据补齐,成本近零。F5 的编码器接入经本轮重新论证后,从"可争议扩张"变为"其中三处必需、两处无损",必要性判断上调。
|
||||
|
||||
**处置建议(均非阻断,可合并后处理)**:NB-4 与 NB-5 是文字修正,建议合并前一并改掉(它们会影响后人对证据强度的判断);NB-2、NB-8 可合成一个约 20 行的提交;NB-1、NB-3、NB-6、NB-7 可进 backlog。
|
||||
@@ -1,49 +0,0 @@
|
||||
{
|
||||
"requested_model": "claude-opus-5",
|
||||
"requested_effort": "max",
|
||||
"head": "62cf066ff529c7d281703daa365f555cebba717a",
|
||||
"prior_review_head": "4089113e3edbd21a29be8e6af74662462bcc22af",
|
||||
"started_at": "2026-09-12T08:46:55.400524+00:00",
|
||||
"status": "completed",
|
||||
"completed_at": "2026-09-12T09:07:16.313535+00:00",
|
||||
"subtype": "success",
|
||||
"is_error": false,
|
||||
"duration_ms": 973153,
|
||||
"num_turns": 71,
|
||||
"assistant_models": [
|
||||
"claude-opus-5"
|
||||
],
|
||||
"used_tools": {
|
||||
"Read": 41,
|
||||
"Grep": 24,
|
||||
"Glob": 5
|
||||
},
|
||||
"stream_sha256": "63046ccfab9fb158178f43b09c75a9809a13ed1ff2da5a626e9e3c3e96c13d37",
|
||||
"review_sha256": "306a856010f7f9bd1cbec39aba83236978f5a811c98f7fc3f8a08217f72de111",
|
||||
"auxiliary_model_note": "CLI model_usage may include auxiliary calls. assistant_models identifies the actual review message model. max was explicitly passed in the recorded command.",
|
||||
"command": [
|
||||
"/opt/homebrew/bin/claude",
|
||||
"--print",
|
||||
"--model",
|
||||
"claude-opus-5",
|
||||
"--effort",
|
||||
"max",
|
||||
"--safe-mode",
|
||||
"--permission-mode",
|
||||
"plan",
|
||||
"--tools",
|
||||
"Read,Grep,Glob",
|
||||
"--strict-mcp-config",
|
||||
"--no-session-persistence",
|
||||
"--add-dir",
|
||||
"$LOCAL_EVIDENCE",
|
||||
"--add-dir",
|
||||
"$DOCS_WORKTREE/content/blog/design",
|
||||
"--output-format",
|
||||
"stream-json",
|
||||
"--verbose"
|
||||
],
|
||||
"command_note": "Same explicit CLI model/effort/read-only options as recorded in the tool invocation; round-2 had its own pinned worktree and prompt.",
|
||||
"session_id": "8743aaa3-5c06-43db-8eba-08302d504c4e",
|
||||
"archive_note": "Workstation paths normalized; billing data and raw model traces omitted. Original hashes are retained in archive-manifest.json."
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
{
|
||||
"head": "fcbb93e8957275bfa7ad4e6154e93f4d7b0a7025",
|
||||
"source_clean": true,
|
||||
"built_at": "2026-09-12T09:12:38.105913+00:00",
|
||||
"binaries": {
|
||||
"silo-reviewed": {
|
||||
"sha256": "4825a801ce0ac48d636d9428ce4cd5c17a20b6cfec0b569de70a124c9c005049",
|
||||
"version": "silo-reviewed version DEVELOPMENT.2026-09-12T09-09-34Z (commit-id=fcbb93e8957275bfa7ad4e6154e93f4d7b0a7025)\nRuntime: go1.27.1 darwin/arm64\nLicense: GNU AGPLv3 - https://www.gnu.org/licenses/agpl-3.0.html\nCopyright: 2015-2025 MinIO, Inc.\nModifications: Copyright 2025-2026 PGSTY\nSource compatibility: based on MinIO technology",
|
||||
"build_info_sha256": "91778f5f2844ada1234ed6a97d97fce7d87dc6334fdd9ef839fbeb7ebdb995e3",
|
||||
"build_info_identity_lines": [
|
||||
"\tpath\tgithub.com/minio/minio",
|
||||
"\tmod\tgithub.com/minio/minio\tv0.0.0-20260912090934-fcbb93e89572\t",
|
||||
"\tbuild\t-buildmode=exe",
|
||||
"\tbuild\t-compiler=gc",
|
||||
"\tbuild\t-trimpath=true",
|
||||
"\tbuild\tCGO_ENABLED=0",
|
||||
"\tbuild\tGOARCH=arm64",
|
||||
"\tbuild\tGOOS=darwin",
|
||||
"\tbuild\tGOARM64=v8.0",
|
||||
"\tbuild\tvcs=git",
|
||||
"\tbuild\tvcs.revision=fcbb93e8957275bfa7ad4e6154e93f4d7b0a7025",
|
||||
"\tbuild\tvcs.time=2026-09-12T09:09:34Z",
|
||||
"\tbuild\tvcs.modified=false"
|
||||
]
|
||||
},
|
||||
"silo-before": {
|
||||
"sha256": "3b07b9604f2ca9f68f33b69f7807f57e0cc9b1efb0310bbf0aa7b3997e4483b9",
|
||||
"version": "silo-before version DEVELOPMENT.2026-09-11T18-00-57Z (commit-id=5c576581631561c446f30ae5b566f0aa793adc1c)\nRuntime: go1.27.1 darwin/arm64\nLicense: GNU AGPLv3 - https://www.gnu.org/licenses/agpl-3.0.html\nCopyright: 2015-2025 MinIO, Inc.\nModifications: Copyright 2025-2026 PGSTY\nSource compatibility: based on MinIO technology",
|
||||
"build_info_sha256": "84fb4540758d7dd56367069e3843a97d1e2c3528905db11d490b9654a2d6205d",
|
||||
"build_info_identity_lines": [
|
||||
"\tpath\tgithub.com/minio/minio",
|
||||
"\tmod\tgithub.com/minio/minio\tv0.0.0-20260911180057-5c5765816315\t",
|
||||
"\tbuild\t-buildmode=exe",
|
||||
"\tbuild\t-compiler=gc",
|
||||
"\tbuild\t-tags=kqueue",
|
||||
"\tbuild\t-trimpath=true",
|
||||
"\tbuild\tCGO_ENABLED=0",
|
||||
"\tbuild\tGOARCH=arm64",
|
||||
"\tbuild\tGOOS=darwin",
|
||||
"\tbuild\tGOARM64=v8.0",
|
||||
"\tbuild\tvcs=git",
|
||||
"\tbuild\tvcs.revision=5c576581631561c446f30ae5b566f0aa793adc1c",
|
||||
"\tbuild\tvcs.time=2026-09-11T18:00:57Z",
|
||||
"\tbuild\tvcs.modified=false"
|
||||
]
|
||||
}
|
||||
},
|
||||
"archive_note": "Dependency listing and machine paths omitted. Actual version, module identity, build settings and original build-info hash retained. Binaries are not included."
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
{
|
||||
"command": [
|
||||
"make",
|
||||
"check"
|
||||
],
|
||||
"cwd": "$DOCS_WORKTREE",
|
||||
"exit_code": 0,
|
||||
"checked_content_commit": "9fa6248a0323838df5aed678b42c4679a7a6f8a4",
|
||||
"committed_after_check": true,
|
||||
"recorded_at": "2026-09-12T09:30:37.223220+00:00",
|
||||
"files_sha256": {
|
||||
"content/blog/design/bucket-metadata-convergence.zh.md": "26807bf8db555c3f0feef5c933b3d5a2da48fc7fd5a65da00d2c9b93ce7b4b21",
|
||||
"content/blog/design/bucket-metadata-convergence.md": "36bc5c7f2310d96447b8130db0f34a99e900d8f7fc6f33d90630b89085769eec"
|
||||
},
|
||||
"rendered_pages": {
|
||||
"en": 1207,
|
||||
"zh": 1219
|
||||
},
|
||||
"checks": [
|
||||
"go mod verify",
|
||||
"strict Hugo build",
|
||||
"internal links"
|
||||
]
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
go mod verify
|
||||
all modules verified
|
||||
hugo build --minify --cleanDestinationDir --printPathWarnings --printI18nWarnings --panicOnWarning
|
||||
Start building sites …
|
||||
hugo v0.165.0+extended+withdeploy darwin/arm64 BuildDate=2026-08-12T14:26:28Z VendorInfo=Homebrew
|
||||
|
||||
|
||||
│ EN │ ZH
|
||||
──────────────────┼──────┼──────
|
||||
Pages │ 1207 │ 1219
|
||||
Paginator pages │ 2 │ 2
|
||||
Non-page files │ 26 │ 0
|
||||
Static files │ 740 │ 740
|
||||
Processed images │ 8 │ 0
|
||||
Aliases │ 45 │ 45
|
||||
Cleaned │ 0 │ 0
|
||||
|
||||
Total in 19619 ms
|
||||
python3 bin/check_internal_links.py public
|
||||
internal link check passed: 485471 rendered internal references across 1429 HTML files
|
||||
@@ -1,16 +0,0 @@
|
||||
{
|
||||
"command": [
|
||||
"go",
|
||||
"build",
|
||||
"./..."
|
||||
],
|
||||
"cwd": "$SILO_WORKTREE",
|
||||
"head": "fcbb93e8957275bfa7ad4e6154e93f4d7b0a7025",
|
||||
"started_at": "2026-09-12T09:11:30.915932+00:00",
|
||||
"environment": {
|
||||
"TMPDIR": "$TMPDIR",
|
||||
"CGO_ENABLED": "0"
|
||||
},
|
||||
"exit_code": 0,
|
||||
"duration_seconds": 24.193
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
{"command": ["go", "build", "./..."], "cwd": "$SILO_WORKTREE", "head": "fcbb93e8957275bfa7ad4e6154e93f4d7b0a7025", "started_at": "2026-09-12T09:11:30.915932+00:00", "environment": {"TMPDIR": "$TMPDIR", "CGO_ENABLED": "0"}}
|
||||
|
||||
EXIT=0
|
||||
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"command": [
|
||||
"go",
|
||||
"test",
|
||||
"./cmd/",
|
||||
"-count=1",
|
||||
"-timeout",
|
||||
"30m"
|
||||
],
|
||||
"cwd": "$SILO_WORKTREE",
|
||||
"head": "fcbb93e8957275bfa7ad4e6154e93f4d7b0a7025",
|
||||
"started_at": "2026-09-12T09:11:30.892660+00:00",
|
||||
"environment": {
|
||||
"TMPDIR": "$LAB_VOLUME/tmp",
|
||||
"CGO_ENABLED": "0",
|
||||
"MINIO_API_REQUESTS_MAX": "10000"
|
||||
},
|
||||
"exit_code": 0,
|
||||
"duration_seconds": 518.124
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
{"command": ["go", "test", "./cmd/", "-count=1", "-timeout", "30m"], "cwd": "$SILO_WORKTREE", "head": "fcbb93e8957275bfa7ad4e6154e93f4d7b0a7025", "started_at": "2026-09-12T09:11:30.892660+00:00", "environment": {"TMPDIR": "$LAB_VOLUME/tmp", "CGO_ENABLED": "0", "MINIO_API_REQUESTS_MAX": "10000"}}
|
||||
ok github.com/minio/minio/cmd 492.776s
|
||||
|
||||
EXIT=0
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"command": [
|
||||
"make",
|
||||
"lint"
|
||||
],
|
||||
"cwd": "$SILO_WORKTREE",
|
||||
"head": "461e9a721047c63e1a95f54ad4b533a6b89def30",
|
||||
"started_at": "2026-09-12T09:17:19.334284+00:00",
|
||||
"environment": {
|
||||
"TMPDIR": "$TMPDIR"
|
||||
},
|
||||
"exit_code": 0,
|
||||
"duration_seconds": 79.064
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
{"command": ["make", "lint"], "cwd": "$SILO_WORKTREE", "head": "461e9a721047c63e1a95f54ad4b533a6b89def30", "started_at": "2026-09-12T09:17:19.334284+00:00", "environment": {"TMPDIR": "$TMPDIR"}}
|
||||
Running lint check
|
||||
0 issues.
|
||||
typos binary is not found.. skipping..
|
||||
|
||||
EXIT=0
|
||||
@@ -1,18 +0,0 @@
|
||||
{
|
||||
"command": [
|
||||
"go",
|
||||
"run",
|
||||
".issue77-evidence/twosite/main.go",
|
||||
"$LAB_VOLUME/runtime-reviewed",
|
||||
".issue77-evidence/silo-reviewed",
|
||||
".issue77-evidence/silo-before"
|
||||
],
|
||||
"cwd": "$SILO_WORKTREE",
|
||||
"head": "fcbb93e8957275bfa7ad4e6154e93f4d7b0a7025",
|
||||
"started_at": "2026-09-12T09:12:41.365111+00:00",
|
||||
"environment": {
|
||||
"TMPDIR": "$LAB_VOLUME/tmp"
|
||||
},
|
||||
"exit_code": 0,
|
||||
"duration_seconds": 190.476
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
{"command": ["go", "run", ".issue77-evidence/twosite/main.go", "$LAB_VOLUME/runtime-reviewed", ".issue77-evidence/silo-reviewed", ".issue77-evidence/silo-before"], "cwd": "$SILO_WORKTREE", "head": "fcbb93e8957275bfa7ad4e6154e93f4d7b0a7025", "started_at": "2026-09-12T09:12:41.365111+00:00", "environment": {"TMPDIR": "$LAB_VOLUME/tmp"}}
|
||||
2026-09-12T09:12:42Z site-a ready gate=on
|
||||
2026-09-12T09:12:43Z site-b ready gate=on
|
||||
2026-09-12T09:12:44Z historical bucket prepared: six live fields at Created
|
||||
2026-09-12T09:12:44Z baseline-initial-sync: waiting for ordinary 30-second heal
|
||||
2026-09-12T09:12:44Z baseline-initial-sync: six field states equal
|
||||
2026-09-12T09:12:44Z initial sync preserved six historical Created timestamps
|
||||
2026-09-12T09:12:44Z real local PUT outgoing replication RPC dropped by partition
|
||||
2026-09-12T09:12:45Z live-reconnect: waiting for ordinary 30-second heal
|
||||
2026-09-12T09:13:13Z live-reconnect: six field states equal
|
||||
2026-09-12T09:13:13Z live-steady: observing two heal intervals
|
||||
2026-09-12T09:14:18Z live-steady: zero metadata RPCs across 65 seconds
|
||||
2026-09-12T09:14:18Z six duplicate/out-of-order events rejected without clock changes
|
||||
2026-09-12T09:14:19Z real local DELETE outgoing replication RPC dropped by partition
|
||||
2026-09-12T09:14:20Z site-a ready gate=on
|
||||
2026-09-12T09:14:20Z four tombstones survived process restart
|
||||
2026-09-12T09:14:20Z delete-reconnect: waiting for ordinary 30-second heal
|
||||
2026-09-12T09:14:44Z delete-reconnect: six field states equal
|
||||
2026-09-12T09:14:44Z delete-steady: observing two heal intervals
|
||||
2026-09-12T09:15:49Z delete-steady: zero metadata RPCs across 65 seconds
|
||||
2026-09-12T09:15:49Z 40 exceptional events produced exactly one log per reason
|
||||
2026-09-12T09:15:49Z starting fixed/previous-server rolling-upgrade smoke with gate off
|
||||
2026-09-12T09:15:50Z site-a ready gate=off
|
||||
2026-09-12T09:15:51Z site-b ready gate=off
|
||||
2026-09-12T09:15:51Z mixed-version ordinary PUT/DELETE and off tombstone visibility passed
|
||||
2026-09-12T09:15:51Z PASS: isolated two-site implementation acceptance
|
||||
|
||||
EXIT=0
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"command": [
|
||||
"go",
|
||||
"test",
|
||||
"-race",
|
||||
"./cmd/",
|
||||
"-run",
|
||||
"^(TestBucketMetadataInitialSyncPhysicalCreated|TestHealBucketConfigDiagnostics|TestHealBucketConfigWithoutSourceDiagnostics)$",
|
||||
"-count=1",
|
||||
"-timeout",
|
||||
"10m"
|
||||
],
|
||||
"cwd": "$SILO_WORKTREE",
|
||||
"head": "461e9a721047c63e1a95f54ad4b533a6b89def30",
|
||||
"started_at": "2026-09-12T09:18:02.407600+00:00",
|
||||
"environment": {
|
||||
"TMPDIR": "$LAB_VOLUME/tmp",
|
||||
"CGO_ENABLED": "1"
|
||||
},
|
||||
"exit_code": 0,
|
||||
"duration_seconds": 45.681
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
{"command": ["go", "test", "-race", "./cmd/", "-run", "^(TestBucketMetadataInitialSyncPhysicalCreated|TestHealBucketConfigDiagnostics|TestHealBucketConfigWithoutSourceDiagnostics)$", "-count=1", "-timeout", "10m"], "cwd": "$SILO_WORKTREE", "head": "461e9a721047c63e1a95f54ad4b533a6b89def30", "started_at": "2026-09-12T09:18:02.407600+00:00", "environment": {"TMPDIR": "$LAB_VOLUME/tmp", "CGO_ENABLED": "1"}}
|
||||
ok github.com/minio/minio/cmd 4.540s
|
||||
|
||||
EXIT=0
|
||||
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"command": [
|
||||
"go",
|
||||
"test",
|
||||
"-race",
|
||||
"./cmd/",
|
||||
"-run",
|
||||
"^(TestBucketMetadataInitialSyncPhysicalCreated|TestPeerBucketMetadataPhysicalCreatedBoundary|TestPeerBucketMetadataUnknownCreated|TestPeerBucketAdoptionRebasesOnlyDefaults|TestHealBucketConfigDiagnostics|TestHealBucketConfigWithoutSourceDiagnostics|TestBucketMetadataPhysicalCreatedRecovery|TestBucketPolicyReplicationStatusLegacyOrder|TestHealBucketConfigSourceAndQuiescence|TestLatestBucketConfigCandidates|Test.*Cors.*|Test.*CORS.*)$",
|
||||
"-count=1",
|
||||
"-timeout",
|
||||
"15m"
|
||||
],
|
||||
"cwd": "$SILO_WORKTREE",
|
||||
"head": "fcbb93e8957275bfa7ad4e6154e93f4d7b0a7025",
|
||||
"started_at": "2026-09-12T09:11:30.873027+00:00",
|
||||
"environment": {
|
||||
"TMPDIR": "$LAB_VOLUME/tmp",
|
||||
"CGO_ENABLED": "1"
|
||||
},
|
||||
"exit_code": 0,
|
||||
"duration_seconds": 90.202
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
{"command": ["go", "test", "-race", "./cmd/", "-run", "^(TestBucketMetadataInitialSyncPhysicalCreated|TestPeerBucketMetadataPhysicalCreatedBoundary|TestPeerBucketMetadataUnknownCreated|TestPeerBucketAdoptionRebasesOnlyDefaults|TestHealBucketConfigDiagnostics|TestHealBucketConfigWithoutSourceDiagnostics|TestBucketMetadataPhysicalCreatedRecovery|TestBucketPolicyReplicationStatusLegacyOrder|TestHealBucketConfigSourceAndQuiescence|TestLatestBucketConfigCandidates|Test.*Cors.*|Test.*CORS.*)$", "-count=1", "-timeout", "15m"], "cwd": "$SILO_WORKTREE", "head": "fcbb93e8957275bfa7ad4e6154e93f4d7b0a7025", "started_at": "2026-09-12T09:11:30.873027+00:00", "environment": {"TMPDIR": "$LAB_VOLUME/tmp", "CGO_ENABLED": "1"}}
|
||||
ok github.com/minio/minio/cmd 20.903s
|
||||
|
||||
EXIT=0
|
||||
-10
@@ -1,10 +0,0 @@
|
||||
{
|
||||
"reviewed_production_head": "fcbb93e8957275bfa7ad4e6154e93f4d7b0a7025",
|
||||
"final_head": "461e9a721047c63e1a95f54ad4b533a6b89def30",
|
||||
"changed_files": [
|
||||
"cmd/site-replication-metadata-gate_test.go",
|
||||
"cmd/site-replication-metadata-heal_test.go"
|
||||
],
|
||||
"difference": "Test-only gofumpt formatting and equivalent if-chain to switch; no production or server documentation changes",
|
||||
"patch": "test-style.diff"
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
{
|
||||
"command": [
|
||||
"go",
|
||||
"vet",
|
||||
"./..."
|
||||
],
|
||||
"cwd": "$SILO_WORKTREE",
|
||||
"head": "fcbb93e8957275bfa7ad4e6154e93f4d7b0a7025",
|
||||
"started_at": "2026-09-12T09:11:55.197305+00:00",
|
||||
"environment": {
|
||||
"TMPDIR": "$TMPDIR",
|
||||
"CGO_ENABLED": "0"
|
||||
},
|
||||
"exit_code": 0,
|
||||
"duration_seconds": 4.024
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user