mirror of
https://github.com/pgsty/minio.git
synced 2026-11-22 15:26:43 +03:00
Compare commits
102 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e069fe9d92 | |||
| d848fb52b5 | |||
| 3ce8319251 | |||
| 9df0f4abaf | |||
| 4d0693cb8c | |||
| bf59e3f222 | |||
| 41aa846097 | |||
| 4093fa0d78 | |||
| 13bf126ebd | |||
| 1cf529ce8a | |||
| 9b76a21675 | |||
| 89637554d6 | |||
| 2dd1e00da4 | |||
| 5d955b5b74 | |||
| f7808a172c | |||
| acc9b514f5 | |||
| 31ba01c5d5 | |||
| 48ec10312f | |||
| 114dc10529 | |||
| 461e9a7210 | |||
| fcbb93e895 | |||
| 62cf066ff5 | |||
| 1ee64a8d89 | |||
| 01aaef2b50 | |||
| bcc62afe3d | |||
| 5c57658163 | |||
| f175e98c34 | |||
| 12f631b502 | |||
| ccb676e60c | |||
| 51d41345f7 | |||
| e59a3d938e | |||
| b32f2d9dd0 | |||
| d63c92e393 | |||
| 68127c5a63 | |||
| c4b5e1cb45 | |||
| 9a303f5096 | |||
| 87d8b5967f | |||
| f760046c44 | |||
| 93e7ef4bcc | |||
| a4229b366f | |||
| 420340bc14 | |||
| e7e87402ed | |||
| a164e1dda1 | |||
| 2f61325d4a | |||
| a6145e1d2e | |||
| 5232546690 | |||
| 0c46cb641b | |||
| 1233254309 | |||
| 8a2fe9b7a0 | |||
| f26ee6bf0a | |||
| d69c4ccfe4 | |||
| 086e619505 | |||
| 48e1846525 | |||
| bcc8871b1d | |||
| 25cb3511c9 | |||
| cfefc049c1 | |||
| af56d17630 | |||
| d1105bbb3d | |||
| 66fe61ff65 | |||
| a1141a43f2 | |||
| 702f113f51 | |||
| 63aace4099 | |||
| 9d7094b770 | |||
| 450dcb8484 | |||
| 4074d00b96 | |||
| 75ba0ce402 | |||
| da142327f2 | |||
| a3df317ae0 | |||
| 2c50d11f72 | |||
| 5ac33e1583 | |||
| d57c4e8407 | |||
| 374de0fa32 | |||
| 80e23dc9f2 | |||
| 2cc0e3c6ed | |||
| f9da3b919d | |||
| 1309853f57 | |||
| 39b8e6c30a | |||
| 9a6e1477f4 | |||
| 49375ed2d3 | |||
| f817b5261c | |||
| 885bd2c20a | |||
| 89b75e7913 | |||
| 079ebb1926 | |||
| 04c29aac11 | |||
| 95e7a190b3 | |||
| 8e2392e48f | |||
| 3abe0d95a5 | |||
| 9c6c9805de | |||
| 5cb900bfad | |||
| 0af5d22286 | |||
| f1687f402b | |||
| ce606df2c4 | |||
| fd44dc4e9b | |||
| 479745e764 | |||
| cc1c54475f | |||
| e7654d470c | |||
| 4b25f7e819 | |||
| 711b092f86 | |||
| 2bc103b80c | |||
| ad873c7357 | |||
| 0db4bf3b00 | |||
| 7a060cab1e |
@@ -1,54 +0,0 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: community, triage
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
## NOTE
|
||||
|
||||
Silo issues are handled by community maintainers on a best-effort basis. There
|
||||
is no SLA, SLO, or emergency production-support channel. Follow the local
|
||||
[Code of Conduct](../code_of_conduct.md) when participating. Report suspected
|
||||
vulnerabilities through the private process in [SECURITY.md](../SECURITY.md),
|
||||
not in a public issue.
|
||||
|
||||
<!--- Provide a general summary of the issue in the Title above -->
|
||||
|
||||
## Expected Behavior
|
||||
<!--- If you're describing a bug, tell us what should happen -->
|
||||
<!--- If you're suggesting a change/improvement, tell us how it should work -->
|
||||
|
||||
## Current Behavior
|
||||
<!--- If describing a bug, tell us what happens instead of the expected behavior -->
|
||||
<!--- If suggesting a change/improvement, explain the difference from current behavior -->
|
||||
|
||||
## Possible Solution
|
||||
<!--- Not obligatory, but suggest a fix/reason for the bug, -->
|
||||
<!--- or ideas how to implement the addition or change -->
|
||||
|
||||
## Steps to Reproduce (for bugs)
|
||||
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
|
||||
<!--- reproduce this bug. Include code to reproduce, if relevant -->
|
||||
<!--- and include relevant Silo logs with secrets and credentials removed -->
|
||||
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
4.
|
||||
|
||||
## Context
|
||||
<!--- How has this issue affected you? What are you trying to accomplish? -->
|
||||
<!--- Providing context helps us come up with a solution that is most useful in the real world -->
|
||||
|
||||
## Regression
|
||||
<!-- Is this issue a regression? (Yes / No) -->
|
||||
<!-- If Yes, optionally include the Silo version, commit id, or PR that caused this regression. -->
|
||||
|
||||
## Your Environment
|
||||
<!--- Include as many relevant details about the environment you experienced the bug in -->
|
||||
* Version used (`silo --version`):
|
||||
* Server setup and configuration:
|
||||
* Operating System and version (`uname -a`):
|
||||
@@ -7,6 +7,14 @@ assignees: ''
|
||||
|
||||
---
|
||||
|
||||
Report bugs in the PGSTY SILO server (`pgsty/silo`) here. Community maintainers
|
||||
handle reports on a best-effort basis. There is no SLA, SLO, or emergency
|
||||
production-support channel. Follow the
|
||||
[Code of Conduct](https://github.com/pgsty/silo/blob/main/code_of_conduct.md).
|
||||
Report suspected vulnerabilities privately through
|
||||
[SECURITY.md](https://github.com/pgsty/silo/blob/main/SECURITY.md).
|
||||
For patches, see the [contribution guide](https://github.com/pgsty/silo/blob/main/CONTRIBUTING.md).
|
||||
|
||||
<!--- Provide a general summary of the issue in the Title above -->
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
@@ -7,6 +7,9 @@ assignees: ''
|
||||
|
||||
---
|
||||
|
||||
Suggest improvements to the PGSTY SILO server (`pgsty/silo`) here.
|
||||
For patches, see the [contribution guide](https://github.com/pgsty/silo/blob/main/CONTRIBUTING.md).
|
||||
|
||||
**Is your feature request related to a problem? Please describe.**
|
||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
||||
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
## Contribution Licensing (no CLA, inbound=outbound, DCO required)
|
||||
|
||||
This project does not use a CLA; contributions are accepted inbound=outbound.
|
||||
This pull request contributes to PGSTY SILO (`pgsty/silo`). Code contributions
|
||||
are accepted under AGPL-3.0-or-later, the same license as the server.
|
||||
This project does not use a CLA or require a separate Apache-2.0 license grant.
|
||||
By submitting this pull request I represent that I have the right to contribute
|
||||
the changes, which are licensed under this repository's
|
||||
the code changes under this repository's
|
||||
[GNU Affero General Public License v3.0 or later](https://www.gnu.org/licenses/agpl-3.0.html)
|
||||
and remain my copyright. Every commit must carry a DCO `Signed-off-by` trailer
|
||||
and retain copyright in my original work. Existing copyright and license
|
||||
notices remain intact; separately licensed material keeps its applicable terms.
|
||||
Every commit must carry a DCO `Signed-off-by` trailer
|
||||
(`git commit -s`) certifying the
|
||||
[Developer Certificate of Origin](https://developercertificate.org/) — see
|
||||
[CONTRIBUTING.md](https://github.com/pgsty/silo/blob/main/CONTRIBUTING.md).
|
||||
|
||||
@@ -140,7 +140,7 @@ jobs:
|
||||
mkdir -p "${context}/dockerscripts"
|
||||
tar -xzf "${archive}" -C "${context}" silo
|
||||
cp Dockerfile.goreleaser Dockerfile.distroless LICENSE NOTICE CREDITS "${context}/"
|
||||
cp dockerscripts/docker-entrypoint.sh dockerscripts/download-static-curl.sh \
|
||||
cp dockerscripts/docker-entrypoint.sh dockerscripts/build-static-curl.sh \
|
||||
"${context}/dockerscripts/"
|
||||
done
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ on:
|
||||
- "Dockerfile.distroless"
|
||||
- "cmd/healthcheck-main.go"
|
||||
- "cmd/main.go"
|
||||
- "dockerscripts/download-static-curl.sh"
|
||||
- "dockerscripts/build-static-curl.sh"
|
||||
- "dockerscripts/docker-entrypoint.sh"
|
||||
- "dockerscripts/docker-entrypoint_test.sh"
|
||||
- "silo.service"
|
||||
@@ -41,6 +41,28 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
curl:
|
||||
name: Static curl (${{ matrix.arch }})
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- arch: amd64
|
||||
runner: ubuntu-latest
|
||||
- arch: arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- name: Build and exercise curl in an empty runtime
|
||||
run: |
|
||||
docker build --build-arg TARGETARCH=${{ matrix.arch }} \
|
||||
--target curl-runtime -f Dockerfile.goreleaser -t silo-curl-test .
|
||||
docker run --rm silo-curl-test --version | tee curl-version.txt
|
||||
grep -F 'curl 8.22.0 ' curl-version.txt
|
||||
grep -F 'HTTP2' curl-version.txt
|
||||
docker run --rm silo-curl-test --fail --silent --show-error \
|
||||
--connect-timeout 15 --max-time 60 https://curl.se/robots.txt
|
||||
|
||||
validate:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
@@ -490,7 +512,7 @@ jobs:
|
||||
bash -n buildscripts/package/lifecycle_test.sh
|
||||
buildscripts/package/lifecycle_test.sh
|
||||
bash -n dockerscripts/docker-entrypoint_test.sh
|
||||
bash -n dockerscripts/download-static-curl.sh
|
||||
bash -n dockerscripts/build-static-curl.sh
|
||||
dockerscripts/docker-entrypoint_test.sh
|
||||
go run ./buildscripts/rebrand-guard
|
||||
buildscripts/verify-rebrand.sh
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
|
||||
- name: Install govulncheck
|
||||
run: |
|
||||
go install golang.org/x/vuln/cmd/govulncheck@v1.7.0
|
||||
go install golang.org/x/vuln/cmd/govulncheck@v1.8.0
|
||||
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Run govulncheck
|
||||
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
# Changelog
|
||||
|
||||
## Unreleased
|
||||
|
||||
The entries below describe source changes on main since the latest published Server.
|
||||
**The latest published Server remains 20260903.** These changes are not in its
|
||||
binaries, packages or images. See the [component matrix](https://silo.pgsty.com/compatibility/versions/)
|
||||
and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-09-03T13-18-01Z...main).
|
||||
|
||||
### Authorization and security
|
||||
|
||||
- Reject unsigned `x-amz-*` request headers that could turn a signed PUT into a
|
||||
copy of another object accessible to the signer (SN-2026-011). The latest
|
||||
public Server is affected; the fix is on main. See [the advisory ledger](docs/security/advisories.md).
|
||||
- Align signed request fields with policy conditions and enforce header-only
|
||||
presigned payload checksums. See [the signed-header review](https://silo.pgsty.com/blog/design/signed-header-coverage/).
|
||||
- **Breaking policy semantics:** separate self-service `admin:ChangeMyPassword`
|
||||
from `admin:CreateUser`. Built-in read-only policies follow the split. Preserve
|
||||
both denies if the previous combined restriction must survive upgrades or
|
||||
rollback. Saved policies are not rewritten. Deploy with the matching Console
|
||||
and pkg; see [the migration guide](docs/iam/password-permissions.md).
|
||||
|
||||
### Object storage and replication
|
||||
|
||||
- Evaluate conditional multipart completion against the logical current object
|
||||
across all pools while holding the existing object lock. A stale `If-Match`
|
||||
can no longer replace newer data in another pool, and the current ETag is no
|
||||
longer rejected because the upload resides next to an older copy. Conditions
|
||||
are evaluated once; a current delete marker counts as an absent object.
|
||||
**Availability change:** if metadata cannot be read from any pool, conditional
|
||||
completion fails even when another pool can still serve GET/HEAD. This also
|
||||
applies when the unreadable pool may not hold the object: absence cannot be
|
||||
verified. Retry after the pool recovers. Unconditional completion and the
|
||||
single-pool path retain their existing behavior.
|
||||
|
||||
- Reconcile ordinary single-object version DELETE across all pools, including
|
||||
null versions, delete markers and unqualified directory-marker DELETE. This
|
||||
applies the deletion to every resolved pool copy under existing quorum
|
||||
rules. Pending outbound delete replication retains versions until the
|
||||
existing replication worker completes their purge; a successful response
|
||||
does not imply immediate physical removal from every drive. Unreadable
|
||||
pools now consistently return 503 instead of depending on pool traversal
|
||||
order; insufficient read quorum returns `SlowDownRead`. This extends the
|
||||
existing failure surface. Retry after recovery.
|
||||
Cleanup failures also return an error. Batch deletion already fans out across
|
||||
pools; replication and scanner cleanup keep their existing contracts. See
|
||||
[scope and limitations](docs/bucket/lifecycle/access-tiering-removal.md#version-deletion-scope).
|
||||
|
||||
- Remove the opt-in GET-frequency pool-tiering feature from PR #60, including
|
||||
its tracker, mover, scanner hooks, configuration, XML actions and metrics.
|
||||
Accept and ignore retired configuration/XML and preserve ordinary statistics
|
||||
when reading v9 caches. See [migration notes](docs/bucket/lifecycle/access-tiering-removal.md).
|
||||
The [decision record](docs/investigations/access-tiering-revert.md) preserves
|
||||
the feature's introduction, subsequent fixes, rollback scope and review history.
|
||||
- Preserve the independent multi-pool write, metadata, healing and conditional
|
||||
deletion fixes from PR #178, including shared remote-tier reference protection.
|
||||
- Enforce `If-Match` on DELETE, preserve retention and independently ordered
|
||||
Object Lock/tag updates, and correctly retransmit encrypted replicas.
|
||||
- Preserve plaintext part sizes and raw SSE-C replicas; prevent SSE-C
|
||||
compression, honor key-rotation checksums, and complete attributes pagination.
|
||||
- Repair federated CopyObject checksums, destination timestamps, reserved
|
||||
metadata, encrypted-object forwarding, legal hold and KMS context.
|
||||
- Make resync counters, target selection, cancellation and worker lifetimes
|
||||
reflect actual work; complete delete-marker purges and report bounded MRF drops.
|
||||
- Converge bucket metadata with deterministic source state, deletion tombstones,
|
||||
creation time recovery and diagnostics. The mixed-version export gate requires
|
||||
coordinated upgrades before tombstones are exported. See [the #77 record](docs/investigations/issue-77-current.md).
|
||||
- Include per-bucket CORS in metadata export/import, close metadata publication
|
||||
and logger races, and report effective bucket quotas in metrics.
|
||||
|
||||
### Console, dependencies and delivery
|
||||
|
||||
- Restore embedded Console login over loopback TLS, trusted-proxy handling and
|
||||
all four WebSocket connection limits. Preserve Go TLS defaults across transports.
|
||||
- Directly require `github.com/pgsty/silo-pkg/v3` v3.14.0; select Console
|
||||
`v0.0.0-20260913015128-417559bb2c97` and MC
|
||||
`v0.0.0-20260913012246-4f609a4da3bb` with explicit PGSTY replacements.
|
||||
- Pin upstream minio-go `v7.3.1-0.20260910142817-60bd07042d49`; refresh Go x/*
|
||||
modules and security fixes including bounded AMQP frame handling. Keep Go
|
||||
1.27.1 and go-systemd v22.6.0's NetBSD compatibility replacement.
|
||||
- Refresh container base digests and build static curl 8.22.0 from verified
|
||||
source for both Linux architectures. Pin the actual mcli 20260913 archives and
|
||||
hashes. Helm's client image follows that release; its Server image still names
|
||||
the latest published Server 20260903.
|
||||
|
||||
The dependency update passed the final candidate's Go, vulnerability and Test
|
||||
Release workflows; native curl builds passed on both architectures. A local
|
||||
ARM64 image passed startup, health, S3 transfer and embedded Console checks.
|
||||
These checks do not publish a Server tag or production image and do not replace
|
||||
cluster upgrade/rollback acceptance for the next release. Dated investigations
|
||||
retain the exact source and runtime boundaries they tested.
|
||||
|
||||
## RELEASE.2026-09-03T13-18-01Z
|
||||
|
||||
Published source: `9b11dc9469e650815b775cb47b039610644f5da4`.
|
||||
[Complete release notes](https://silo.pgsty.com/blog/release/silo-20260903/) ·
|
||||
[GitHub release](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z)
|
||||
|
||||
This release ships Go 1.27.1, silo-pkg v3.13.2, upstream minio-go `0e78d3f18efe`,
|
||||
mcli 20260903 and embedded Console source `464a59d73ada` (v2.3.0 version identity).
|
||||
Installing the newer standalone mcli or Console does not replace components
|
||||
inside this existing Server binary or image.
|
||||
|
||||
Earlier releases: [release archive](https://github.com/pgsty/silo/releases).
|
||||
+14
-8
@@ -79,9 +79,10 @@ documentation is owned by the separate
|
||||
|
||||
## Licensing of Contributions
|
||||
|
||||
Silo is licensed under the [GNU AGPL v3.0 or later](LICENSE). Its core is
|
||||
Copyright (c) MinIO, Inc.; the combined work can never be relicensed, and this
|
||||
fork does not try to.
|
||||
Code contributions to PGSTY SILO (`pgsty/silo`) are accepted under the
|
||||
[GNU AGPL v3.0 or later](LICENSE), the same license as the server. Submit issues
|
||||
and pull requests to this repository's maintainers. No separate Apache-2.0
|
||||
license grant to SILO or upstream MinIO maintainers is required.
|
||||
|
||||
* **No CLA.** We do not ask you to sign a Contributor License Agreement and we
|
||||
do not take your copyright. Contributions are accepted inbound=outbound: you
|
||||
@@ -112,15 +113,20 @@ fork does not try to.
|
||||
`Signed-off-by` trailers) and add your own sign-off as the person passing it
|
||||
along. Never import code from a proprietary distribution.
|
||||
|
||||
* **File headers.** Files derived from upstream keep the original MinIO
|
||||
copyright header unchanged. New files added by this fork use the dual
|
||||
header, followed by the standard AGPL boilerplate:
|
||||
* **File headers.** Preserve existing copyright and license notices in inherited
|
||||
and third-party files. New original files name their actual copyright holders
|
||||
and use AGPL-3.0-or-later. Use a header such as the following, then append the
|
||||
standard AGPL boilerplate:
|
||||
|
||||
```
|
||||
// Copyright (c) 2015-2025 MinIO, Inc.
|
||||
// Copyright (c) 2025-2026 PGSTY
|
||||
// Copyright (c) 2026 Your Name
|
||||
```
|
||||
|
||||
* **Separately licensed material.** Documentation contributions in `docs/`
|
||||
follow its existing [CC BY 4.0 license](docs/LICENSE). Third-party components
|
||||
and earlier Apache-2.0 contributions retain their original licenses and
|
||||
attribution; this policy does not relicense earlier work.
|
||||
|
||||
* **Squash merges** must keep the `Signed-off-by:` trailers in the resulting
|
||||
commit message.
|
||||
|
||||
|
||||
+170
-73
File diff suppressed because one or more lines are too long
+19
-11
@@ -1,4 +1,15 @@
|
||||
FROM golang:1.27.1-alpine AS build
|
||||
FROM golang:1.27.1-alpine@sha256:cf6fca6641884b8433441b2b0652976f975e1d0fdd26d177eaaf8596087f3125 AS curl-build
|
||||
ARG TARGETARCH
|
||||
COPY dockerscripts/build-static-curl.sh /build/build-static-curl
|
||||
RUN /bin/sh /build/build-static-curl
|
||||
|
||||
# Exercise the exact shipped curl without a dynamic loader or shared libraries.
|
||||
FROM scratch AS curl-runtime
|
||||
COPY --from=curl-build /go/bin/curl /curl
|
||||
COPY --from=curl-build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
ENTRYPOINT ["/curl"]
|
||||
|
||||
FROM golang:1.27.1-alpine@sha256:cf6fca6641884b8433441b2b0652976f975e1d0fdd26d177eaaf8596087f3125 AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
|
||||
@@ -6,9 +17,9 @@ ENV GOPATH=/go
|
||||
ENV CGO_ENABLED=0
|
||||
|
||||
ARG MC_REPO=pgsty/mc
|
||||
ARG MC_VERSION=RELEASE.2026-09-03T07-13-05Z
|
||||
ARG MC_AMD64_SHA256=cd7fcd449bb6b52e2eb727431ba6975b1e5d90df011a75869020ea9ac9e2b2a8
|
||||
ARG MC_ARM64_SHA256=7962afc37c3e60e5758b19e819cb62d2f340ee655fad7b067e2ac9bc5716c2e8
|
||||
ARG MC_VERSION=RELEASE.2026-09-13T00-00-00Z
|
||||
ARG MC_AMD64_SHA256=9d2a92de9c7b887d9b944fe9ddce68d23f1b6df3415092e737594e56593f5e2b
|
||||
ARG MC_ARM64_SHA256=3d82e9ea6c601c4cb44fe5dd5f2ad1b7d7d64369378110f9ada9c524688a452a
|
||||
|
||||
RUN apk add -U --no-cache \
|
||||
ca-certificates \
|
||||
@@ -56,18 +67,14 @@ RUN apk add -U --no-cache \
|
||||
chmod +x /go/bin/mcli && \
|
||||
ln -sf mcli /go/bin/mc
|
||||
|
||||
COPY dockerscripts/download-static-curl.sh /build/download-static-curl
|
||||
RUN chmod +x /build/download-static-curl && \
|
||||
/build/download-static-curl
|
||||
|
||||
FROM registry.access.redhat.com/ubi9/ubi:latest AS certs
|
||||
FROM registry.access.redhat.com/ubi9/ubi:latest@sha256:206b65b8ee0f04b992818c9a51b29081b14974630d4850bc358097d0c44ea156 AS certs
|
||||
RUN dnf -y install ca-certificates && \
|
||||
update-ca-trust && \
|
||||
cp /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /tmp/ca-certificates.crt && \
|
||||
dnf clean all && \
|
||||
rm -rf /var/cache/dnf
|
||||
|
||||
FROM registry.access.redhat.com/ubi9/ubi-micro:latest
|
||||
FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:f332c99eb8f798a8486821c91937f10ad64ee83d7e739303be2df051040918f6
|
||||
|
||||
LABEL org.opencontainers.image.title="Silo" \
|
||||
org.opencontainers.image.description="S3-Interface Libre Object Storage" \
|
||||
@@ -88,7 +95,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
COPY --from=certs /tmp/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
COPY silo /usr/bin/silo
|
||||
COPY --from=build /go/bin/mcli /usr/bin/mcli
|
||||
COPY --from=build /go/bin/curl* /usr/bin/
|
||||
COPY --from=curl-build /go/bin/curl /usr/bin/curl
|
||||
COPY --from=curl-build /go/share/curl /licenses/curl
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
COPY LICENSE /licenses/LICENSE
|
||||
COPY NOTICE /licenses/NOTICE
|
||||
|
||||
@@ -216,7 +216,7 @@ docker: checks build-debugging ## builds the local Linux Silo container image
|
||||
--ldflags "$(LDFLAGS)" -o "$$context/silo"; \
|
||||
mkdir -p "$$context/dockerscripts"; \
|
||||
cp Dockerfile.goreleaser LICENSE NOTICE CREDITS "$$context/"; \
|
||||
cp dockerscripts/docker-entrypoint.sh dockerscripts/download-static-curl.sh \
|
||||
cp dockerscripts/docker-entrypoint.sh dockerscripts/build-static-curl.sh \
|
||||
"$$context/dockerscripts/"; \
|
||||
docker build -q --no-cache --platform linux/$(GOARCH) -t $(TAG) --build-arg TARGETARCH=$(GOARCH) \
|
||||
-f "$$context/Dockerfile.goreleaser" "$$context"
|
||||
|
||||
@@ -35,6 +35,14 @@
|
||||
> [!NOTE]
|
||||
> Renamed from `pgsty/minio` to `pgsty/silo`, default branch `master` → `main`, on 2026-08-06. Artifacts under the original MinIO identity stay published on the archived [`minio`](https://github.com/pgsty/silo/tree/minio) branch and in releases up to [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z).
|
||||
|
||||
## Current release and main branch
|
||||
|
||||
The latest published Server is [20260903](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z).
|
||||
As of 2026-09-13, the main branch has newer security, storage, Console and
|
||||
shared-package changes that have not shipped in a Server release. See
|
||||
[CHANGELOG.md](CHANGELOG.md) and the [component version matrix](https://silo.pgsty.com/compatibility/versions/)
|
||||
for the exact release/source boundary, including SN-2026-011 and password-policy migration.
|
||||
|
||||
## Overview
|
||||
|
||||
PGSTY SILO keeps one maintained release line of the open-source MinIO server alive after upstream ended community distribution: builds, packages, multi-arch images, security fixes, and the full web console. Pigsty runs it in production as its PostgreSQL backup repository.
|
||||
@@ -89,76 +97,79 @@ The S3 API, `MINIO_*` variables, `minio_*` metrics, `x-minio-*` headers, `/minio
|
||||
|
||||
Every divergence from upstream is listed in the code-verified [compatibility audit](https://silo.pgsty.com/compatibility/server/). Treat each release as a downstream upgrade: pin versions, read the [release notes](https://silo.pgsty.com/tags/silo/), and keep a rollback path.
|
||||
|
||||
### TLS and Go upgrades
|
||||
|
||||
TLS key exchange follows Go's defaults across the S3 listener, node links,
|
||||
replication, identity providers, etcd, and external HTTP services. If an endpoint
|
||||
cannot accept ML-KEM, `GODEBUG=tlsmlkem=0` disables the default hybrid exchanges
|
||||
for the process; certificate verification remains enabled. This option does not
|
||||
disable ML-DSA signatures or resolve every TLS reset. Prefer updating the
|
||||
incompatible endpoint before removing the temporary setting.
|
||||
If only the new SecP hybrids cause problems, `GODEBUG=tlssecpmlkem=0` disables
|
||||
those groups while retaining X25519MLKEM768.
|
||||
|
||||
For builds targeting Go 1.27, setting either `SSL_CERT_FILE` or `SSL_CERT_DIR`
|
||||
on macOS replaces Keychain trust with on-disk roots and Go's verifier. Stale or
|
||||
incomplete CA paths can break previously trusted connections; unset inherited
|
||||
values to restore Keychain trust. Explicit certificates in the configured `CAs`
|
||||
directory remain additive to the selected root pool.
|
||||
Go 1.27 binaries require macOS 13 or later. See the
|
||||
[Go release notes](https://go.dev/doc/go1.27) and the
|
||||
[SILO stack investigation](docs/investigations/go127-stack.md).
|
||||
|
||||
## Security & Contributing
|
||||
|
||||
Report vulnerabilities privately as described in [`SECURITY.md`](SECURITY.md); every fix ships with a public [advisory](https://silo.pgsty.com/blog/security/). Contributions are accepted inbound=outbound under AGPL-3.0-or-later with no CLA — only DCO sign-off (`git commit -s`) is required; see [`CONTRIBUTING.md`](CONTRIBUTING.md).
|
||||
|
||||
## Contributors
|
||||
|
||||
The cards highlight community changes merged into `main`; the avatar wall includes every other
|
||||
community member who has opened an issue or pull request.
|
||||
**41 community contributors** build SILO, Console, mcli, shared packages, and related projects. The list includes maintainers and every human Issue or PR author, ordered by merged PRs, other PRs, then issue reports. Gold rings highlight significant contributions.
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/h5vx"><img src="https://github.com/h5vx.png?size=100" width="72" alt="h5vx"><br><sub><b>@h5vx</b></sub></a><br><sub>Per-bucket CORS · <a href="https://github.com/pgsty/silo/pull/71">#71</a></sub>
|
||||
</td>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/ycjlin"><img src="https://github.com/ycjlin.png?size=100" width="72" alt="ycjlin"><br><sub><b>@ycjlin</b></sub></a><br><sub>Missing-bucket listing · <a href="https://github.com/pgsty/silo/pull/37">#37</a></sub>
|
||||
</td>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/Dansyuqri"><img src="https://github.com/Dansyuqri.png?size=100" width="72" alt="Dansyuqri"><br><sub><b>@Dansyuqri</b></sub></a><br><sub>Multipart checksum type · <a href="https://github.com/pgsty/silo/pull/57">#57</a></sub>
|
||||
</td>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/pinginfo"><img src="https://github.com/pinginfo.png?size=100" width="72" alt="pinginfo"><br><sub><b>@pinginfo</b></sub></a><br><sub>Notification streaming · <a href="https://github.com/pgsty/silo/pull/34">#34</a></sub>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/ZouhairCharef"><img src="https://github.com/ZouhairCharef.png?size=100" width="72" alt="ZouhairCharef"><br><sub><b>@ZouhairCharef</b></sub></a><br><sub>CVE-2026-34986 · <a href="https://github.com/pgsty/silo/pull/18">#18</a></sub>
|
||||
</td>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/mfredenhagen"><img src="https://github.com/mfredenhagen.png?size=100" width="72" alt="mfredenhagen"><br><sub><b>@mfredenhagen</b></sub></a><br><sub>CVE-2026-39883 · <a href="https://github.com/pgsty/silo/pull/19">#19</a></sub>
|
||||
</td>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/waterkip"><img src="https://github.com/waterkip.png?size=100" width="72" alt="waterkip"><br><sub><b>@waterkip</b></sub></a><br><sub>Documentation links · <a href="https://github.com/pgsty/silo/pull/41">#41</a></sub>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
<p>
|
||||
<a href="https://github.com/magicxor"><img src="https://github.com/magicxor.png?size=64" width="44" alt="magicxor" title="@magicxor"></a>
|
||||
<a href="https://github.com/davinkevin"><img src="https://github.com/davinkevin.png?size=64" width="44" alt="davinkevin" title="@davinkevin"></a>
|
||||
<a href="https://github.com/lem21h"><img src="https://github.com/lem21h.png?size=64" width="44" alt="lem21h" title="@lem21h"></a>
|
||||
<a href="https://github.com/sulin37392"><img src="https://github.com/sulin37392.png?size=64" width="44" alt="sulin37392" title="@sulin37392"></a>
|
||||
<a href="https://github.com/metaneutrons"><img src="https://github.com/metaneutrons.png?size=64" width="44" alt="metaneutrons" title="@metaneutrons"></a>
|
||||
<a href="https://github.com/mrjavadseydi"><img src="https://github.com/mrjavadseydi.png?size=64" width="44" alt="mrjavadseydi" title="@mrjavadseydi"></a>
|
||||
<a href="https://github.com/mosesdd"><img src="https://github.com/mosesdd.png?size=64" width="44" alt="mosesdd" title="@mosesdd"></a>
|
||||
<a href="https://github.com/Xavier-777"><img src="https://github.com/Xavier-777.png?size=64" width="44" alt="Xavier-777" title="@Xavier-777"></a>
|
||||
<a href="https://github.com/jiadzh"><img src="https://github.com/jiadzh.png?size=64" width="44" alt="jiadzh" title="@jiadzh"></a>
|
||||
<a href="https://github.com/TLINDEN"><img src="https://github.com/TLINDEN.png?size=64" width="44" alt="TLINDEN" title="@TLINDEN"></a>
|
||||
<a href="https://github.com/AntonOfTheWoods"><img src="https://github.com/AntonOfTheWoods.png?size=64" width="44" alt="AntonOfTheWoods" title="@AntonOfTheWoods"></a>
|
||||
<a href="https://github.com/zylpsrs"><img src="https://github.com/zylpsrs.png?size=64" width="44" alt="zylpsrs" title="@zylpsrs"></a>
|
||||
<a href="https://github.com/nsanitate"><img src="https://github.com/nsanitate.png?size=64" width="44" alt="nsanitate" title="@nsanitate"></a>
|
||||
<a href="https://github.com/makinikm"><img src="https://github.com/makinikm.png?size=64" width="44" alt="makinikm" title="@makinikm"></a>
|
||||
<a href="https://github.com/spaceg00se-r"><img src="https://github.com/spaceg00se-r.png?size=64" width="44" alt="spaceg00se-r" title="@spaceg00se-r"></a>
|
||||
<a href="https://github.com/heroes1412"><img src="https://github.com/heroes1412.png?size=64" width="44" alt="heroes1412" title="@heroes1412"></a>
|
||||
<a href="https://github.com/vampywiz17"><img src="https://github.com/vampywiz17.png?size=64" width="44" alt="vampywiz17" title="@vampywiz17"></a>
|
||||
<a href="https://github.com/chalukyaj"><img src="https://github.com/chalukyaj.png?size=64" width="44" alt="chalukyaj" title="@chalukyaj"></a>
|
||||
<a href="https://github.com/cbornet"><img src="https://github.com/cbornet.png?size=64" width="44" alt="cbornet" title="@cbornet"></a>
|
||||
<a href="https://github.com/jvasile"><img src="https://github.com/jvasile.png?size=64" width="44" alt="jvasile" title="@jvasile"></a>
|
||||
<a href="https://github.com/Kesavaambati"><img src="https://github.com/Kesavaambati.png?size=64" width="44" alt="Kesavaambati" title="@Kesavaambati"></a>
|
||||
<a href="https://github.com/redfoxfox"><img src="https://github.com/redfoxfox.png?size=64" width="44" alt="redfoxfox" title="@redfoxfox"></a>
|
||||
<a href="https://github.com/kuldeep-link11"><img src="https://github.com/kuldeep-link11.png?size=64" width="44" alt="kuldeep-link11" title="@kuldeep-link11"></a>
|
||||
<a href="https://github.com/meesudzu"><img src="https://github.com/meesudzu.png?size=64" width="44" alt="meesudzu" title="@meesudzu"></a>
|
||||
<a href="https://github.com/pmezhuev"><img src="https://github.com/pmezhuev.png?size=64" width="44" alt="pmezhuev" title="@pmezhuev"></a>
|
||||
<a href="https://github.com/kh0mka"><img src="https://github.com/kh0mka.png?size=64" width="44" alt="kh0mka" title="@kh0mka"></a>
|
||||
<a href="https://github.com/bagutzu"><img src="https://github.com/bagutzu.png?size=64" width="44" alt="bagutzu" title="@bagutzu"></a>
|
||||
<a href="https://github.com/liuhaodongliu990-cmyk"><img src="https://github.com/liuhaodongliu990-cmyk.png?size=64" width="44" alt="liuhaodongliu990-cmyk" title="@liuhaodongliu990-cmyk"></a>
|
||||
<a href="https://github.com/sargarass"><img src="https://github.com/sargarass.png?size=64" width="44" alt="sargarass" title="@sargarass"></a>
|
||||
<a href="https://github.com/mumu-lab"><img src="https://github.com/mumu-lab.png?size=64" width="44" alt="mumu-lab" title="@mumu-lab"></a>
|
||||
<p align="center">
|
||||
<a href="https://github.com/Vonng"><img src="https://silo.pgsty.com/images/contributors/Vonng.svg" width="60" height="60" alt="@Vonng" title="@Vonng — Maintains SILO, Console, mcli, shared packages, releases, and documentation"></a>
|
||||
<a href="https://github.com/h5vx"><img src="https://silo.pgsty.com/images/contributors/h5vx.svg" width="60" height="60" alt="@h5vx" title="@h5vx — Implemented per-bucket CORS configuration and enforcement"></a>
|
||||
<a href="https://github.com/mrjavadseydi"><img src="https://silo.pgsty.com/images/contributors/mrjavadseydi.svg" width="60" height="60" alt="@mrjavadseydi" title="@mrjavadseydi — Fixed effective bucket quota metrics; proposed access-frequency ILM"></a>
|
||||
<a href="https://github.com/Dansyuqri"><img src="https://silo.pgsty.com/images/contributors/Dansyuqri.svg" width="60" height="60" alt="@Dansyuqri" title="@Dansyuqri — Added ChecksumType to multipart completion responses"></a>
|
||||
<a href="https://github.com/ycjlin"><img src="https://silo.pgsty.com/images/contributors/ycjlin.svg" width="60" height="60" alt="@ycjlin" title="@ycjlin — Fixed missing-bucket ListObjects semantics"></a>
|
||||
<a href="https://github.com/pinginfo"><img src="https://silo.pgsty.com/images/contributors/pinginfo.svg" width="60" height="60" alt="@pinginfo" title="@pinginfo — Repaired bucket notification streaming"></a>
|
||||
<a href="https://github.com/ZouhairCharef"><img src="https://silo.pgsty.com/images/contributors/ZouhairCharef.svg" width="60" height="60" alt="@ZouhairCharef" title="@ZouhairCharef — Patched CVE-2026-34986 in go-jose"></a>
|
||||
<a href="https://github.com/mfredenhagen"><img src="https://silo.pgsty.com/images/contributors/mfredenhagen.svg" width="60" height="60" alt="@mfredenhagen" title="@mfredenhagen — Patched CVE-2026-39883 in OpenTelemetry"></a>
|
||||
<a href="https://github.com/waterkip"><img src="https://silo.pgsty.com/images/contributors/waterkip.svg" width="60" height="60" alt="@waterkip" title="@waterkip — Repointed documentation links to the SILO portal"></a>
|
||||
<a href="https://github.com/mikemikimike"><img src="https://silo.pgsty.com/images/contributors/mikemikimike.svg" width="60" height="60" alt="@mikemikimike" title="@mikemikimike — Contributed the replicated SSE-C plaintext part-size fix"></a>
|
||||
<a href="https://github.com/metaneutrons"><img src="https://silo.pgsty.com/images/contributors/metaneutrons.svg" width="60" height="60" alt="@metaneutrons" title="@metaneutrons — Reported and proposed explicit-version delete authorization"></a>
|
||||
<a href="https://github.com/magicxor"><img src="https://silo.pgsty.com/images/contributors/magicxor.svg" width="60" height="60" alt="@magicxor" title="@magicxor — Reported and proposed conditional DELETE support for If-Match"></a>
|
||||
<a href="https://github.com/davinkevin"><img src="https://silo.pgsty.com/images/contributors/davinkevin.svg" width="60" height="60" alt="@davinkevin" title="@davinkevin — Proposed the distroless container image and dependency automation"></a>
|
||||
<a href="https://github.com/lem21h"><img src="https://silo.pgsty.com/images/contributors/lem21h.svg" width="48" height="48" alt="@lem21h" title="@lem21h — Proposed robustness and goroutine improvements"></a>
|
||||
<a href="https://github.com/sulin37392"><img src="https://silo.pgsty.com/images/contributors/sulin37392.svg" width="48" height="48" alt="@sulin37392" title="@sulin37392 — Proposed dependency updates"></a>
|
||||
<a href="https://github.com/cbornet"><img src="https://silo.pgsty.com/images/contributors/cbornet.svg" width="60" height="60" alt="@cbornet" title="@cbornet — Reported multipart and streaming checksum defects and missing-bucket semantics"></a>
|
||||
<a href="https://github.com/vampywiz17"><img src="https://silo.pgsty.com/images/contributors/vampywiz17.svg" width="60" height="60" alt="@vampywiz17" title="@vampywiz17 — Reported LDAP TLS and Console login regressions"></a>
|
||||
<a href="https://github.com/orenyomtov"><img src="https://silo.pgsty.com/images/contributors/orenyomtov.svg" width="60" height="60" alt="@orenyomtov" title="@orenyomtov — Reported the unsigned-header CopyObject cross-object read (SN-2026-011)"></a>
|
||||
<a href="https://github.com/mumu-lab"><img src="https://silo.pgsty.com/images/contributors/mumu-lab.svg" width="48" height="48" alt="@mumu-lab" title="@mumu-lab — Reported bucket quota metrics reading a deprecated field"></a>
|
||||
<a href="https://github.com/jvasile"><img src="https://silo.pgsty.com/images/contributors/jvasile.svg" width="48" height="48" alt="@jvasile" title="@jvasile — Reported missing user, group, and defaults in Debian packages"></a>
|
||||
<a href="https://github.com/pmezhuev"><img src="https://silo.pgsty.com/images/contributors/pmezhuev.svg" width="48" height="48" alt="@pmezhuev" title="@pmezhuev — Reported missing RPM package signatures"></a>
|
||||
<a href="https://github.com/TLINDEN"><img src="https://silo.pgsty.com/images/contributors/TLINDEN.svg" width="48" height="48" alt="@TLINDEN" title="@TLINDEN — Reported the missing client in release tarballs"></a>
|
||||
<a href="https://github.com/makinikm"><img src="https://silo.pgsty.com/images/contributors/makinikm.svg" width="48" height="48" alt="@makinikm" title="@makinikm — Reported the missing client in the container image"></a>
|
||||
<a href="https://github.com/meesudzu"><img src="https://silo.pgsty.com/images/contributors/meesudzu.svg" width="48" height="48" alt="@meesudzu" title="@meesudzu — Requested the migration guide from upstream MinIO"></a>
|
||||
<a href="https://github.com/kuldeep-link11"><img src="https://silo.pgsty.com/images/contributors/kuldeep-link11.svg" width="48" height="48" alt="@kuldeep-link11" title="@kuldeep-link11 — Reported NATS JWT credentials and target reload issues"></a>
|
||||
<a href="https://github.com/sargarass"><img src="https://silo.pgsty.com/images/contributors/sargarass.svg" width="48" height="48" alt="@sargarass" title="@sargarass — Reported ListMultipartUploads prefix and pagination semantics"></a>
|
||||
<a href="https://github.com/liuhaodongliu990-cmyk"><img src="https://silo.pgsty.com/images/contributors/liuhaodongliu990-cmyk.svg" width="48" height="48" alt="@liuhaodongliu990-cmyk" title="@liuhaodongliu990-cmyk — Reported indeterminate progress for prefix downloads"></a>
|
||||
<a href="https://github.com/Xavier-777"><img src="https://silo.pgsty.com/images/contributors/Xavier-777.svg" width="48" height="48" alt="@Xavier-777" title="@Xavier-777 — Reported Console lifecycle management and file preview gaps"></a>
|
||||
<a href="https://github.com/spaceg00se-r"><img src="https://silo.pgsty.com/images/contributors/spaceg00se-r.svg" width="48" height="48" alt="@spaceg00se-r" title="@spaceg00se-r — Requested cpuv1 support and reported a workflow token failure"></a>
|
||||
<a href="https://github.com/kh0mka"><img src="https://silo.pgsty.com/images/contributors/kh0mka.svg" width="48" height="48" alt="@kh0mka" title="@kh0mka — Reported inter-node I/O timeouts in ReadFileStreamHandler"></a>
|
||||
<a href="https://github.com/bagutzu"><img src="https://silo.pgsty.com/images/contributors/bagutzu.svg" width="48" height="48" alt="@bagutzu" title="@bagutzu — Requested KES-compatible external KMS and OpenBao support"></a>
|
||||
<a href="https://github.com/DestroyLee"><img src="https://silo.pgsty.com/images/contributors/DestroyLee.svg" width="48" height="48" alt="@DestroyLee" title="@DestroyLee — Reported the missing documentation navigation"></a>
|
||||
<a href="https://github.com/mosesdd"><img src="https://silo.pgsty.com/images/contributors/mosesdd.svg" width="48" height="48" alt="@mosesdd" title="@mosesdd — Requested a maintained Helm chart"></a>
|
||||
<a href="https://github.com/zylpsrs"><img src="https://silo.pgsty.com/images/contributors/zylpsrs.svg" width="48" height="48" alt="@zylpsrs" title="@zylpsrs — Reported missing Console tiering and site replication"></a>
|
||||
<a href="https://github.com/heroes1412"><img src="https://silo.pgsty.com/images/contributors/heroes1412.svg" width="48" height="48" alt="@heroes1412" title="@heroes1412 — Reported the unusable profiling option"></a>
|
||||
<a href="https://github.com/redfoxfox"><img src="https://silo.pgsty.com/images/contributors/redfoxfox.svg" width="48" height="48" alt="@redfoxfox" title="@redfoxfox — Reported Chinese documentation availability"></a>
|
||||
<a href="https://github.com/jiadzh"><img src="https://silo.pgsty.com/images/contributors/jiadzh.svg" width="48" height="48" alt="@jiadzh" title="@jiadzh — Requested Windows build guidance"></a>
|
||||
<a href="https://github.com/AntonOfTheWoods"><img src="https://silo.pgsty.com/images/contributors/AntonOfTheWoods.svg" width="48" height="48" alt="@AntonOfTheWoods" title="@AntonOfTheWoods — Asked for clarity on Helm chart and operator options"></a>
|
||||
<a href="https://github.com/chalukyaj"><img src="https://silo.pgsty.com/images/contributors/chalukyaj.svg" width="48" height="48" alt="@chalukyaj" title="@chalukyaj — Proposed making the SILO Operator easier to discover"></a>
|
||||
<a href="https://github.com/nsanitate"><img src="https://silo.pgsty.com/images/contributors/nsanitate.svg" width="48" height="48" alt="@nsanitate" title="@nsanitate — Proposed CNCF Sandbox governance"></a>
|
||||
<a href="https://github.com/Kesavaambati"><img src="https://silo.pgsty.com/images/contributors/Kesavaambati.svg" width="48" height="48" alt="@Kesavaambati" title="@Kesavaambati — Asked about community support and image maintenance"></a>
|
||||
</p>
|
||||
|
||||
GitHub does not generate a contributor graph for forks, so [`CONTRIBUTORS.md`](CONTRIBUTORS.md) — not the Insights page — is this project's attribution record. It names all 37 community contributors alongside the change or report they contributed.
|
||||
[View the full contribution record](CONTRIBUTORS.md) for each person's proposals, fixes, and reports.
|
||||
|
||||
## Background
|
||||
|
||||
|
||||
+51
-60
@@ -35,6 +35,13 @@
|
||||
> [!NOTE]
|
||||
> 2026-08-06,本仓库由 `pgsty/minio` 更名为 `pgsty/silo`,默认分支由 `master` 更名为 `main`。以原 MinIO 形态维持的归档构件仍位于归档的 [`minio`](https://github.com/pgsty/silo/tree/minio) 分支,以及截止 [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z) 的历次发布中。
|
||||
|
||||
## 当前发行版与主分支
|
||||
|
||||
最新已发布的 Server 仍为 [20260903](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z)。
|
||||
截至 2026-09-13,主分支已合入更新的安全、存储、Console 与共享包改动,但尚未发布新 Server。
|
||||
准确的已发布/源码边界见 [CHANGELOG.md](CHANGELOG.md) 与[组件版本矩阵](https://silo.pgsty.com/zh/compatibility/versions/),
|
||||
其中包括 SN-2026-011 修复状态与密码权限迁移要求。
|
||||
|
||||
## 概述
|
||||
|
||||
上游停止社区发行后,Silo 为开源 MinIO 服务端维护一条持续可用的版本线:构建、软件包、多架构镜像、安全修复与完整 Web 控制台。Pigsty 在生产环境中用它承载 PostgreSQL 备份存储。
|
||||
@@ -95,69 +102,53 @@ S3 API、`MINIO_*` 环境变量、`minio_*` 指标、`x-minio-*` 头、`/minio/*
|
||||
|
||||
## 贡献者
|
||||
|
||||
上方卡片展示已合入 `main` 的社区贡献;下方头像墙收录其他所有提交过 issue 或 PR 的社区成员。
|
||||
**41 位社区贡献者**共同建设 SILO、Console、mcli、公共包与相关项目。名单包含维护者,以及所有提出 Issue 或 PR 的真人作者;按已合并 PR、其他 PR、Issue 报告排序,黄圈标记显著贡献。
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/h5vx"><img src="https://github.com/h5vx.png?size=100" width="72" alt="h5vx"><br><sub><b>@h5vx</b></sub></a><br><sub>单桶 CORS · <a href="https://github.com/pgsty/silo/pull/71">#71</a></sub>
|
||||
</td>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/ycjlin"><img src="https://github.com/ycjlin.png?size=100" width="72" alt="ycjlin"><br><sub><b>@ycjlin</b></sub></a><br><sub>缺失桶列举语义 · <a href="https://github.com/pgsty/silo/pull/37">#37</a></sub>
|
||||
</td>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/Dansyuqri"><img src="https://github.com/Dansyuqri.png?size=100" width="72" alt="Dansyuqri"><br><sub><b>@Dansyuqri</b></sub></a><br><sub>分片上传校验类型 · <a href="https://github.com/pgsty/silo/pull/57">#57</a></sub>
|
||||
</td>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/pinginfo"><img src="https://github.com/pinginfo.png?size=100" width="72" alt="pinginfo"><br><sub><b>@pinginfo</b></sub></a><br><sub>桶通知流式输出 · <a href="https://github.com/pgsty/silo/pull/34">#34</a></sub>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/ZouhairCharef"><img src="https://github.com/ZouhairCharef.png?size=100" width="72" alt="ZouhairCharef"><br><sub><b>@ZouhairCharef</b></sub></a><br><sub>CVE-2026-34986 · <a href="https://github.com/pgsty/silo/pull/18">#18</a></sub>
|
||||
</td>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/mfredenhagen"><img src="https://github.com/mfredenhagen.png?size=100" width="72" alt="mfredenhagen"><br><sub><b>@mfredenhagen</b></sub></a><br><sub>CVE-2026-39883 · <a href="https://github.com/pgsty/silo/pull/19">#19</a></sub>
|
||||
</td>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/waterkip"><img src="https://github.com/waterkip.png?size=100" width="72" alt="waterkip"><br><sub><b>@waterkip</b></sub></a><br><sub>文档链接修正 · <a href="https://github.com/pgsty/silo/pull/41">#41</a></sub>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
<p>
|
||||
<a href="https://github.com/magicxor"><img src="https://github.com/magicxor.png?size=64" width="44" alt="magicxor" title="@magicxor"></a>
|
||||
<a href="https://github.com/davinkevin"><img src="https://github.com/davinkevin.png?size=64" width="44" alt="davinkevin" title="@davinkevin"></a>
|
||||
<a href="https://github.com/lem21h"><img src="https://github.com/lem21h.png?size=64" width="44" alt="lem21h" title="@lem21h"></a>
|
||||
<a href="https://github.com/sulin37392"><img src="https://github.com/sulin37392.png?size=64" width="44" alt="sulin37392" title="@sulin37392"></a>
|
||||
<a href="https://github.com/metaneutrons"><img src="https://github.com/metaneutrons.png?size=64" width="44" alt="metaneutrons" title="@metaneutrons"></a>
|
||||
<a href="https://github.com/mrjavadseydi"><img src="https://github.com/mrjavadseydi.png?size=64" width="44" alt="mrjavadseydi" title="@mrjavadseydi"></a>
|
||||
<a href="https://github.com/mosesdd"><img src="https://github.com/mosesdd.png?size=64" width="44" alt="mosesdd" title="@mosesdd"></a>
|
||||
<a href="https://github.com/Xavier-777"><img src="https://github.com/Xavier-777.png?size=64" width="44" alt="Xavier-777" title="@Xavier-777"></a>
|
||||
<a href="https://github.com/jiadzh"><img src="https://github.com/jiadzh.png?size=64" width="44" alt="jiadzh" title="@jiadzh"></a>
|
||||
<a href="https://github.com/TLINDEN"><img src="https://github.com/TLINDEN.png?size=64" width="44" alt="TLINDEN" title="@TLINDEN"></a>
|
||||
<a href="https://github.com/AntonOfTheWoods"><img src="https://github.com/AntonOfTheWoods.png?size=64" width="44" alt="AntonOfTheWoods" title="@AntonOfTheWoods"></a>
|
||||
<a href="https://github.com/zylpsrs"><img src="https://github.com/zylpsrs.png?size=64" width="44" alt="zylpsrs" title="@zylpsrs"></a>
|
||||
<a href="https://github.com/nsanitate"><img src="https://github.com/nsanitate.png?size=64" width="44" alt="nsanitate" title="@nsanitate"></a>
|
||||
<a href="https://github.com/makinikm"><img src="https://github.com/makinikm.png?size=64" width="44" alt="makinikm" title="@makinikm"></a>
|
||||
<a href="https://github.com/spaceg00se-r"><img src="https://github.com/spaceg00se-r.png?size=64" width="44" alt="spaceg00se-r" title="@spaceg00se-r"></a>
|
||||
<a href="https://github.com/heroes1412"><img src="https://github.com/heroes1412.png?size=64" width="44" alt="heroes1412" title="@heroes1412"></a>
|
||||
<a href="https://github.com/vampywiz17"><img src="https://github.com/vampywiz17.png?size=64" width="44" alt="vampywiz17" title="@vampywiz17"></a>
|
||||
<a href="https://github.com/chalukyaj"><img src="https://github.com/chalukyaj.png?size=64" width="44" alt="chalukyaj" title="@chalukyaj"></a>
|
||||
<a href="https://github.com/cbornet"><img src="https://github.com/cbornet.png?size=64" width="44" alt="cbornet" title="@cbornet"></a>
|
||||
<a href="https://github.com/jvasile"><img src="https://github.com/jvasile.png?size=64" width="44" alt="jvasile" title="@jvasile"></a>
|
||||
<a href="https://github.com/Kesavaambati"><img src="https://github.com/Kesavaambati.png?size=64" width="44" alt="Kesavaambati" title="@Kesavaambati"></a>
|
||||
<a href="https://github.com/redfoxfox"><img src="https://github.com/redfoxfox.png?size=64" width="44" alt="redfoxfox" title="@redfoxfox"></a>
|
||||
<a href="https://github.com/kuldeep-link11"><img src="https://github.com/kuldeep-link11.png?size=64" width="44" alt="kuldeep-link11" title="@kuldeep-link11"></a>
|
||||
<a href="https://github.com/meesudzu"><img src="https://github.com/meesudzu.png?size=64" width="44" alt="meesudzu" title="@meesudzu"></a>
|
||||
<a href="https://github.com/pmezhuev"><img src="https://github.com/pmezhuev.png?size=64" width="44" alt="pmezhuev" title="@pmezhuev"></a>
|
||||
<a href="https://github.com/kh0mka"><img src="https://github.com/kh0mka.png?size=64" width="44" alt="kh0mka" title="@kh0mka"></a>
|
||||
<a href="https://github.com/bagutzu"><img src="https://github.com/bagutzu.png?size=64" width="44" alt="bagutzu" title="@bagutzu"></a>
|
||||
<a href="https://github.com/liuhaodongliu990-cmyk"><img src="https://github.com/liuhaodongliu990-cmyk.png?size=64" width="44" alt="liuhaodongliu990-cmyk" title="@liuhaodongliu990-cmyk"></a>
|
||||
<a href="https://github.com/sargarass"><img src="https://github.com/sargarass.png?size=64" width="44" alt="sargarass" title="@sargarass"></a>
|
||||
<a href="https://github.com/mumu-lab"><img src="https://github.com/mumu-lab.png?size=64" width="44" alt="mumu-lab" title="@mumu-lab"></a>
|
||||
<p align="center">
|
||||
<a href="https://github.com/Vonng"><img src="https://silo.pgsty.com/images/contributors/Vonng.svg" width="60" height="60" alt="@Vonng" title="@Vonng — 维护 SILO、Console、mcli、公共包、发行与文档"></a>
|
||||
<a href="https://github.com/h5vx"><img src="https://silo.pgsty.com/images/contributors/h5vx.svg" width="60" height="60" alt="@h5vx" title="@h5vx — 实现单桶 CORS 配置与请求执行"></a>
|
||||
<a href="https://github.com/mrjavadseydi"><img src="https://silo.pgsty.com/images/contributors/mrjavadseydi.svg" width="60" height="60" alt="@mrjavadseydi" title="@mrjavadseydi — 修复有效桶配额指标,并提交按访问频率分层的 ILM 方案"></a>
|
||||
<a href="https://github.com/Dansyuqri"><img src="https://silo.pgsty.com/images/contributors/Dansyuqri.svg" width="60" height="60" alt="@Dansyuqri" title="@Dansyuqri — 为分片上传完成响应补充 ChecksumType"></a>
|
||||
<a href="https://github.com/ycjlin"><img src="https://silo.pgsty.com/images/contributors/ycjlin.svg" width="60" height="60" alt="@ycjlin" title="@ycjlin — 修复缺失桶的 ListObjects 语义"></a>
|
||||
<a href="https://github.com/pinginfo"><img src="https://silo.pgsty.com/images/contributors/pinginfo.svg" width="60" height="60" alt="@pinginfo" title="@pinginfo — 修复桶通知的流式输出"></a>
|
||||
<a href="https://github.com/ZouhairCharef"><img src="https://silo.pgsty.com/images/contributors/ZouhairCharef.svg" width="60" height="60" alt="@ZouhairCharef" title="@ZouhairCharef — 修复 go-jose 中的 CVE-2026-34986"></a>
|
||||
<a href="https://github.com/mfredenhagen"><img src="https://silo.pgsty.com/images/contributors/mfredenhagen.svg" width="60" height="60" alt="@mfredenhagen" title="@mfredenhagen — 修复 OpenTelemetry 中的 CVE-2026-39883"></a>
|
||||
<a href="https://github.com/waterkip"><img src="https://silo.pgsty.com/images/contributors/waterkip.svg" width="60" height="60" alt="@waterkip" title="@waterkip — 将文档链接指向 SILO 门户"></a>
|
||||
<a href="https://github.com/mikemikimike"><img src="https://silo.pgsty.com/images/contributors/mikemikimike.svg" width="60" height="60" alt="@mikemikimike" title="@mikemikimike — 提交 SSE-C 复制分片明文尺寸修复"></a>
|
||||
<a href="https://github.com/metaneutrons"><img src="https://silo.pgsty.com/images/contributors/metaneutrons.svg" width="60" height="60" alt="@metaneutrons" title="@metaneutrons — 报告并提交显式版本删除鉴权方案"></a>
|
||||
<a href="https://github.com/magicxor"><img src="https://silo.pgsty.com/images/contributors/magicxor.svg" width="60" height="60" alt="@magicxor" title="@magicxor — 报告并提交 DELETE If-Match 条件请求支持方案"></a>
|
||||
<a href="https://github.com/davinkevin"><img src="https://silo.pgsty.com/images/contributors/davinkevin.svg" width="60" height="60" alt="@davinkevin" title="@davinkevin — 提交 distroless 容器镜像与依赖自动更新方案"></a>
|
||||
<a href="https://github.com/lem21h"><img src="https://silo.pgsty.com/images/contributors/lem21h.svg" width="48" height="48" alt="@lem21h" title="@lem21h — 提交健壮性与 goroutine 改进"></a>
|
||||
<a href="https://github.com/sulin37392"><img src="https://silo.pgsty.com/images/contributors/sulin37392.svg" width="48" height="48" alt="@sulin37392" title="@sulin37392 — 提交依赖更新"></a>
|
||||
<a href="https://github.com/cbornet"><img src="https://silo.pgsty.com/images/contributors/cbornet.svg" width="60" height="60" alt="@cbornet" title="@cbornet — 报告分片与流式校验和缺陷及缺失桶语义问题"></a>
|
||||
<a href="https://github.com/vampywiz17"><img src="https://silo.pgsty.com/images/contributors/vampywiz17.svg" width="60" height="60" alt="@vampywiz17" title="@vampywiz17 — 报告 LDAP TLS 与 Console 登录回归"></a>
|
||||
<a href="https://github.com/orenyomtov"><img src="https://silo.pgsty.com/images/contributors/orenyomtov.svg" width="60" height="60" alt="@orenyomtov" title="@orenyomtov — 报告未签名头导致的 CopyObject 跨对象读取(SN-2026-011)"></a>
|
||||
<a href="https://github.com/mumu-lab"><img src="https://silo.pgsty.com/images/contributors/mumu-lab.svg" width="48" height="48" alt="@mumu-lab" title="@mumu-lab — 报告桶配额指标读取已弃用字段的问题"></a>
|
||||
<a href="https://github.com/jvasile"><img src="https://silo.pgsty.com/images/contributors/jvasile.svg" width="48" height="48" alt="@jvasile" title="@jvasile — 报告 Debian 包缺少用户、用户组与默认配置"></a>
|
||||
<a href="https://github.com/pmezhuev"><img src="https://silo.pgsty.com/images/contributors/pmezhuev.svg" width="48" height="48" alt="@pmezhuev" title="@pmezhuev — 报告 RPM 包缺少 GPG 签名"></a>
|
||||
<a href="https://github.com/TLINDEN"><img src="https://silo.pgsty.com/images/contributors/TLINDEN.svg" width="48" height="48" alt="@TLINDEN" title="@TLINDEN — 报告发布压缩包缺少客户端"></a>
|
||||
<a href="https://github.com/makinikm"><img src="https://silo.pgsty.com/images/contributors/makinikm.svg" width="48" height="48" alt="@makinikm" title="@makinikm — 报告容器镜像缺少客户端"></a>
|
||||
<a href="https://github.com/meesudzu"><img src="https://silo.pgsty.com/images/contributors/meesudzu.svg" width="48" height="48" alt="@meesudzu" title="@meesudzu — 提出从上游 MinIO 迁移的指南需求"></a>
|
||||
<a href="https://github.com/kuldeep-link11"><img src="https://silo.pgsty.com/images/contributors/kuldeep-link11.svg" width="48" height="48" alt="@kuldeep-link11" title="@kuldeep-link11 — 报告 NATS JWT 凭据与通知目标重载问题"></a>
|
||||
<a href="https://github.com/sargarass"><img src="https://silo.pgsty.com/images/contributors/sargarass.svg" width="48" height="48" alt="@sargarass" title="@sargarass — 报告 ListMultipartUploads 前缀与分页语义问题"></a>
|
||||
<a href="https://github.com/liuhaodongliu990-cmyk"><img src="https://silo.pgsty.com/images/contributors/liuhaodongliu990-cmyk.svg" width="48" height="48" alt="@liuhaodongliu990-cmyk" title="@liuhaodongliu990-cmyk — 报告前缀下载进度显示异常"></a>
|
||||
<a href="https://github.com/Xavier-777"><img src="https://silo.pgsty.com/images/contributors/Xavier-777.svg" width="48" height="48" alt="@Xavier-777" title="@Xavier-777 — 报告 Console 生命周期管理与文件预览缺失"></a>
|
||||
<a href="https://github.com/spaceg00se-r"><img src="https://silo.pgsty.com/images/contributors/spaceg00se-r.svg" width="48" height="48" alt="@spaceg00se-r" title="@spaceg00se-r — 提出 cpuv1 支持需求并报告工作流令牌错误"></a>
|
||||
<a href="https://github.com/kh0mka"><img src="https://silo.pgsty.com/images/contributors/kh0mka.svg" width="48" height="48" alt="@kh0mka" title="@kh0mka — 报告 ReadFileStreamHandler 节点间 I/O 超时"></a>
|
||||
<a href="https://github.com/bagutzu"><img src="https://silo.pgsty.com/images/contributors/bagutzu.svg" width="48" height="48" alt="@bagutzu" title="@bagutzu — 提出兼容 KES 的外部 KMS 与 OpenBao 支持需求"></a>
|
||||
<a href="https://github.com/DestroyLee"><img src="https://silo.pgsty.com/images/contributors/DestroyLee.svg" width="48" height="48" alt="@DestroyLee" title="@DestroyLee — 报告文档目录导航缺失"></a>
|
||||
<a href="https://github.com/mosesdd"><img src="https://silo.pgsty.com/images/contributors/mosesdd.svg" width="48" height="48" alt="@mosesdd" title="@mosesdd — 提出维护 Helm Chart 的需求"></a>
|
||||
<a href="https://github.com/zylpsrs"><img src="https://silo.pgsty.com/images/contributors/zylpsrs.svg" width="48" height="48" alt="@zylpsrs" title="@zylpsrs — 报告 Console 缺少分层与站点复制"></a>
|
||||
<a href="https://github.com/heroes1412"><img src="https://silo.pgsty.com/images/contributors/heroes1412.svg" width="48" height="48" alt="@heroes1412" title="@heroes1412 — 报告性能分析选项不可用"></a>
|
||||
<a href="https://github.com/redfoxfox"><img src="https://silo.pgsty.com/images/contributors/redfoxfox.svg" width="48" height="48" alt="@redfoxfox" title="@redfoxfox — 报告中文文档站点不可用"></a>
|
||||
<a href="https://github.com/jiadzh"><img src="https://silo.pgsty.com/images/contributors/jiadzh.svg" width="48" height="48" alt="@jiadzh" title="@jiadzh — 提出 Windows 构建指导需求"></a>
|
||||
<a href="https://github.com/AntonOfTheWoods"><img src="https://silo.pgsty.com/images/contributors/AntonOfTheWoods.svg" width="48" height="48" alt="@AntonOfTheWoods" title="@AntonOfTheWoods — 提出明确 Helm Chart 与 Operator 选项的需求"></a>
|
||||
<a href="https://github.com/chalukyaj"><img src="https://silo.pgsty.com/images/contributors/chalukyaj.svg" width="48" height="48" alt="@chalukyaj" title="@chalukyaj — 提出改善 SILO Operator 可发现性的建议"></a>
|
||||
<a href="https://github.com/nsanitate"><img src="https://silo.pgsty.com/images/contributors/nsanitate.svg" width="48" height="48" alt="@nsanitate" title="@nsanitate — 提出加入 CNCF Sandbox 的治理建议"></a>
|
||||
<a href="https://github.com/Kesavaambati"><img src="https://silo.pgsty.com/images/contributors/Kesavaambati.svg" width="48" height="48" alt="@Kesavaambati" title="@Kesavaambati — 提出社区支持与容器镜像维护问题"></a>
|
||||
</p>
|
||||
|
||||
GitHub 不为 fork 仓库生成贡献者图表,因此 [`CONTRIBUTORS.md`](CONTRIBUTORS.md)(而非 Insights 页面)才是本项目的署名记录,其中逐一记录了全部 37 位社区贡献者对应的改动或报告。
|
||||
[查看完整贡献记录](CONTRIBUTORS.md),了解每位贡献者的提案、修复与问题报告。
|
||||
|
||||
## 背景
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ if [ -n "${MCLI_BIN:-}" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
release=${MCLI_RELEASE:-RELEASE.2026-09-03T07-13-05Z}
|
||||
release=${MCLI_RELEASE:-RELEASE.2026-09-13T00-00-00Z}
|
||||
version_hyphen=${release#RELEASE.}
|
||||
package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')
|
||||
if [ "${package_version}" = "${version_hyphen}" ]; then
|
||||
|
||||
@@ -502,6 +502,7 @@
|
||||
"MINIO_SITE_COMMENT",
|
||||
"MINIO_SITE_NAME",
|
||||
"MINIO_SITE_REGION",
|
||||
"MINIO_SITE_REPLICATION_METADATA_TOMBSTONES",
|
||||
"MINIO_STORAGE_CLASS_COMMENT",
|
||||
"MINIO_STORAGE_CLASS_INLINE_BLOCK",
|
||||
"MINIO_STORAGE_CLASS_OPTIMIZE",
|
||||
@@ -594,6 +595,7 @@
|
||||
"x-minio-internal-encrypted-multipart",
|
||||
"x-minio-internal-encryptedmultipart",
|
||||
"x-minio-internal-erasure-upgraded",
|
||||
"x-minio-internal-ilm-atier",
|
||||
"x-minio-internal-inline-data",
|
||||
"x-minio-internal-objectlock-legalhold-timestamp",
|
||||
"x-minio-internal-replica-status",
|
||||
@@ -616,6 +618,7 @@
|
||||
"x-minio-internal-transitioned-object",
|
||||
"x-minio-internal-xyz",
|
||||
"x-minio-key",
|
||||
"x-minio-last-modified",
|
||||
"x-minio-lifecycleconfig-updatedat",
|
||||
"x-minio-meta",
|
||||
"x-minio-meta-appid",
|
||||
@@ -901,6 +904,7 @@
|
||||
".minio.sys/config/config.json",
|
||||
".minio.sys/config/hello.txt",
|
||||
".minio.sys/config/iam/${username}/identity.json",
|
||||
".minio.sys/config/ilm/access",
|
||||
".minio.sys/format.json",
|
||||
".minio.sys/multipart",
|
||||
".minio.sys/multipart/bucket/object/uploads.json",
|
||||
@@ -1059,6 +1063,7 @@
|
||||
"cmd/metrics.go=\"Version of current MinIO server instance\"",
|
||||
"cmd/metrics.go=\"minio\"",
|
||||
"cmd/object-api-utils.go=\".minio.sys\"",
|
||||
"cmd/object-handlers-common.go=\"X-Minio-Last-Modified\"",
|
||||
"cmd/object-handlers.go=\"minio-federated\"",
|
||||
"cmd/object-handlers.go=\"minio.metadata.\"",
|
||||
"cmd/object-handlers.go=\"minio.versionId\"",
|
||||
|
||||
@@ -115,7 +115,9 @@ func collect(repo string) (manifest, error) {
|
||||
fset := token.NewFileSet()
|
||||
|
||||
for _, rel := range files {
|
||||
// Investigation artifacts contain synthetic routes and archived configurations.
|
||||
if rel == "SILO_REBRANDING_MIGRATION.md" ||
|
||||
strings.HasPrefix(rel, "docs/investigations/") ||
|
||||
strings.HasPrefix(rel, "buildscripts/rebrand-guard/") ||
|
||||
strings.HasPrefix(rel, "buildscripts/helm-migration-guard/") {
|
||||
continue
|
||||
|
||||
@@ -36,7 +36,7 @@ for file in \
|
||||
buildscripts/verify-helm-migration.sh \
|
||||
Dockerfile.goreleaser \
|
||||
Dockerfile.distroless \
|
||||
dockerscripts/download-static-curl.sh \
|
||||
dockerscripts/build-static-curl.sh \
|
||||
dockerscripts/docker-entrypoint.sh \
|
||||
helm/silo/Chart.yaml \
|
||||
helm/silo/values.yaml \
|
||||
@@ -101,7 +101,7 @@ require_text Dockerfile.goreleaser "Published checksum drift"
|
||||
require_text Dockerfile.distroless 'COPY --chmod=0755 silo /usr/bin/silo'
|
||||
require_text Dockerfile.distroless 'ENTRYPOINT ["/usr/bin/silo"]'
|
||||
require_text Dockerfile.distroless '"/usr/bin/silo", "healthcheck", "ready"'
|
||||
require_text dockerscripts/download-static-curl.sh "sha256sum -c"
|
||||
require_text dockerscripts/build-static-curl.sh "sha256sum -c"
|
||||
require_text helm/silo/Chart.yaml "name: silo"
|
||||
require_text helm/silo/values.yaml "repository: pgsty/silo"
|
||||
require_text helm/silo/templates/deployment.yaml "/usr/bin/docker-entrypoint.sh silo server"
|
||||
@@ -159,11 +159,13 @@ fi
|
||||
# The repository and its default branch are pgsty/silo and main. The invariant
|
||||
# is that the old name is never a live target, not that it is never spoken: the
|
||||
# READMEs have to name it to explain the rename and to point at the archived
|
||||
# artifacts, which is the opposite of stranding a reader on it.
|
||||
# artifacts, which is the opposite of stranding a reader on it. CONTRIBUTORS.md
|
||||
# also quotes historical issue titles.
|
||||
#
|
||||
# So two rules. First, no live URL may resolve to the old repository anywhere,
|
||||
# READMEs included.
|
||||
stale_repo_url="$(rg -n -e 'github\.com/pgsty/minio' -e 'hub\.docker\.com/r/pgsty/minio' \
|
||||
# READMEs and CONTRIBUTORS.md included.
|
||||
old_repo_pattern='pgsty/minio(\.git)?([^[:alnum:]_.-]|$)'
|
||||
stale_repo_url="$(rg -n -e "github\.com/${old_repo_pattern}" -e "hub\.docker\.com/r/${old_repo_pattern}" \
|
||||
--glob '!.git/**' --glob '!dist/**' \
|
||||
--glob '!SILO_REBRANDING_MIGRATION.md' \
|
||||
--glob '!buildscripts/rebrand-guard/compat-baseline.json' . |
|
||||
@@ -175,10 +177,10 @@ fi
|
||||
|
||||
# Second, the bare name may only appear where it is deliberate: the pinned
|
||||
# pre-rebrand image digest in the upgrade test, the two guards that refuse a
|
||||
# legacy image, and the two READMEs that document the rename and the archived
|
||||
# minio branch.
|
||||
repo_guard_allowlist='^(buildscripts/minio-upgrade\.sh|buildscripts/verify-rebrand\.sh|buildscripts/helm-migration-guard/main\.go|README\.md|README_ZH\.md):'
|
||||
stale_repo="$(rg -n 'pgsty/minio' --glob '!.git/**' --glob '!dist/**' \
|
||||
# legacy image, the two READMEs that document the rename and the archived
|
||||
# minio branch, and historical issue titles in CONTRIBUTORS.md.
|
||||
repo_guard_allowlist='^(buildscripts/minio-upgrade\.sh|buildscripts/verify-rebrand\.sh|buildscripts/helm-migration-guard/main\.go|README\.md|README_ZH\.md|CONTRIBUTORS\.md):'
|
||||
stale_repo="$(rg -n "${old_repo_pattern}" --glob '!.git/**' --glob '!dist/**' \
|
||||
--glob '!SILO_REBRANDING_MIGRATION.md' \
|
||||
--glob '!buildscripts/rebrand-guard/compat-baseline.json' . |
|
||||
sed 's#^\./##' | grep -Ev "${repo_guard_allowlist}" || true)"
|
||||
|
||||
@@ -76,7 +76,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
|
||||
return
|
||||
}
|
||||
|
||||
quotaConfig, err := parseBucketQuota(bucket, data)
|
||||
_, err = parseBucketQuota(bucket, data)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -94,9 +94,6 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
|
||||
Quota: data,
|
||||
UpdatedAt: updatedAt,
|
||||
}
|
||||
if quotaConfig.Size == 0 && quotaConfig.Quota == 0 {
|
||||
bucketMeta.Quota = nil
|
||||
}
|
||||
|
||||
// Call site replication hook.
|
||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta))
|
||||
@@ -438,7 +435,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
configData, err := json.Marshal(config)
|
||||
configData, err := canonicalBucketPolicy(config)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
@@ -922,7 +919,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
|
||||
configData, err := json.Marshal(bucketPolicy)
|
||||
configData, err := canonicalBucketPolicy(bucketPolicy)
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
@@ -1081,18 +1078,39 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
var merged BucketMetadata
|
||||
var commitAt time.Time
|
||||
err := func() error {
|
||||
lockCtx, unlock, err := lockBucketMetadata(ctx, objectAPI, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer unlock()
|
||||
merged, err = loadBucketMetadataParse(lockCtx, objectAPI, bucket, true)
|
||||
merged, err = loadBucketMetadataParse(lockCtx, objectAPI, bucket, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureBucketMetadataCreated(lockCtx, objectAPI, &merged); err != nil {
|
||||
return err
|
||||
}
|
||||
commitAt = UTCNow()
|
||||
for _, file := range replicatedBucketConfigs {
|
||||
if _, ok := fields[file]; ok {
|
||||
commitAt = localBucketConfigUpdatedAt(merged, file, commitAt)
|
||||
}
|
||||
}
|
||||
applyImportedBucketMetadata(&merged, *meta, fields)
|
||||
return globalBucketMetadataSys.saveMetadata(lockCtx, objectAPI, merged)
|
||||
for _, file := range replicatedBucketConfigs {
|
||||
if _, ok := fields[file]; !ok {
|
||||
continue
|
||||
}
|
||||
data, at := replicatedBucketConfig(&merged, file)
|
||||
payload, _, err := bucketConfigPayload(bucket, file, *data, len(merged.ObjectLockConfigXML) != 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*data, *at = payload, commitAt
|
||||
}
|
||||
return globalBucketMetadataSys.saveMetadata(lockCtx, objectAPI, &merged)
|
||||
}()
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, "", err)
|
||||
@@ -1100,7 +1118,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
}
|
||||
*meta = merged
|
||||
globalNotificationSys.LoadBucketMetadata(bgContext(ctx), bucket)
|
||||
hook := madmin.SRBucketMeta{Bucket: bucket, UpdatedAt: updatedAt}
|
||||
hook := madmin.SRBucketMeta{Bucket: bucket, UpdatedAt: commitAt}
|
||||
var hookNeeded bool
|
||||
if _, ok := fields[bucketQuotaConfigFile]; ok {
|
||||
hook.Quota = meta.QuotaConfigJSON
|
||||
@@ -1108,7 +1126,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
}
|
||||
if _, ok := fields[bucketPolicyConfig]; ok {
|
||||
hook.Policy = meta.PolicyConfigJSON
|
||||
hookNeeded = true
|
||||
hookNeeded = hookNeeded || len(hook.Policy) != 0
|
||||
}
|
||||
if _, ok := fields[bucketVersioningConfig]; ok {
|
||||
hook.Versioning = enc(meta.VersioningConfigXML)
|
||||
@@ -1129,6 +1147,12 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
if hookNeeded {
|
||||
err = globalSiteReplicationSys.BucketMetaHook(ctx, hook)
|
||||
}
|
||||
if _, ok := fields[bucketPolicyConfig]; ok && len(meta.PolicyConfigJSON) == 0 {
|
||||
// An omitted bulk Policy cannot express deletion.
|
||||
err = errors.Join(err, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypePolicy, Bucket: bucket, UpdatedAt: commitAt,
|
||||
}))
|
||||
}
|
||||
if _, ok := fields[bucketCorsConfig]; ok {
|
||||
// CORS carries its own timestamp, so it replicates through the
|
||||
// dedicated event rather than the shared bucket metadata hook. It
|
||||
|
||||
@@ -502,11 +502,15 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
checkDenyOnly := accessKey == cred.AccessKey
|
||||
action := policy.Action(policy.CreateUserAdminAction)
|
||||
if checkDenyOnly {
|
||||
action = policy.ChangeMyPasswordAdminAction
|
||||
}
|
||||
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.CreateUserAdminAction,
|
||||
Action: action,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
|
||||
@@ -243,6 +243,7 @@ func TestIAMInternalIDPServerSuite(t *testing.T) {
|
||||
|
||||
suite.SetUpSuite(c)
|
||||
suite.TestUserCreate(c)
|
||||
suite.TestUserPasswordActionAuthorization(c)
|
||||
suite.TestUserStatusActionAuthorization(c)
|
||||
suite.TestGroupStatusActionAuthorization(c)
|
||||
suite.TestUserPolicyEscalationBug(c)
|
||||
@@ -356,6 +357,106 @@ func (s *TestSuiteIAM) TestUserCreate(c *check) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestUserPasswordActionAuthorization(c *check) {
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
statements string
|
||||
self bool
|
||||
other bool
|
||||
}{
|
||||
{"readonly", "", true, false},
|
||||
{"consolereadonly", "", true, false},
|
||||
{"password grant", `{"Effect":"Allow","Action":"admin:ChangeMyPassword"}`, true, false},
|
||||
{"legacy CreateUser deny", `{"Effect":"Deny","Action":"admin:CreateUser","Resource":"arn:aws:s3:::*"}`, true, false},
|
||||
{"password deny", `{"Effect":"Deny","Action":"admin:ChangeMyPassword"}`, false, false},
|
||||
{"user admin", `{"Effect":"Allow","Action":"admin:CreateUser"}`, true, true},
|
||||
{"user admin with password deny", `{"Effect":"Allow","Action":"admin:CreateUser"},{"Effect":"Deny","Action":"admin:ChangeMyPassword"}`, false, true},
|
||||
{"password deny overrides grant", `{"Effect":"Allow","Action":"admin:ChangeMyPassword"},{"Effect":"Deny","Action":"admin:ChangeMyPassword"}`, false, false},
|
||||
{"wildcard deny", `{"Effect":"Deny","Action":"admin:*"}`, false, false},
|
||||
} {
|
||||
c.Run(tt.name, func(t *testing.T) {
|
||||
c := &check{t, s.serverType}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
var users []string
|
||||
policyName := tt.name
|
||||
defer func() {
|
||||
for _, user := range users {
|
||||
if err := s.adm.RemoveUser(ctx, user); err != nil {
|
||||
c.Errorf("remove test user: %v", err)
|
||||
}
|
||||
}
|
||||
if tt.statements != "" {
|
||||
if err := s.adm.RemoveCannedPolicy(ctx, policyName); err != nil {
|
||||
c.Errorf("remove test policy: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
createUser := func() (string, string) {
|
||||
accessKey, secretKey := mustGenerateCredentials(c)
|
||||
if err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled); err != nil {
|
||||
c.Fatalf("create test user: %v", err)
|
||||
}
|
||||
users = append(users, accessKey)
|
||||
return accessKey, secretKey
|
||||
}
|
||||
client := func(accessKey, secretKey string) *madmin.AdminClient {
|
||||
adm, err := madmin.New(s.endpoint, accessKey, secretKey, s.secure)
|
||||
if err != nil {
|
||||
c.Fatal(err)
|
||||
}
|
||||
adm.SetCustomTransport(s.TestSuiteCommon.client.Transport)
|
||||
return adm
|
||||
}
|
||||
if tt.statements != "" {
|
||||
policyName = getRandomBucketName()
|
||||
doc := []byte(`{"Version":"2012-10-17","Statement":[` + tt.statements + `]}`)
|
||||
if err := s.adm.AddCannedPolicy(ctx, policyName, doc); err != nil {
|
||||
c.Fatalf("save test policy: %v", err)
|
||||
}
|
||||
}
|
||||
accessKey, secretKey := createUser()
|
||||
if _, err := s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
|
||||
User: accessKey, Policies: []string{policyName},
|
||||
}); err != nil {
|
||||
c.Fatalf("attach test policy: %v", err)
|
||||
}
|
||||
adm := client(accessKey, secretKey)
|
||||
_, newSecretKey := mustGenerateCredentials(c)
|
||||
err := adm.SetUser(ctx, accessKey, newSecretKey, madmin.AccountEnabled)
|
||||
if tt.self {
|
||||
if err != nil {
|
||||
c.Fatalf("change own password: %v", err)
|
||||
}
|
||||
if _, err = adm.AccountInfo(ctx, madmin.AccountOpts{}); err == nil {
|
||||
c.Fatal("old password still authenticates")
|
||||
}
|
||||
adm = client(accessKey, newSecretKey)
|
||||
} else if err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("self password change: expected AccessDenied, got %v", err)
|
||||
}
|
||||
if _, err := adm.AccountInfo(ctx, madmin.AccountOpts{}); err != nil {
|
||||
c.Fatalf("current password no longer authenticates: %v", err)
|
||||
}
|
||||
target, _ := createUser()
|
||||
newUser, newUserSecret := mustGenerateCredentials(c)
|
||||
for _, key := range []string{target, newUser} {
|
||||
err := adm.SetUser(ctx, key, newUserSecret, madmin.AccountEnabled)
|
||||
if tt.other {
|
||||
if err != nil {
|
||||
c.Fatalf("create or update another user: %v", err)
|
||||
}
|
||||
if key == newUser {
|
||||
users = append(users, newUser)
|
||||
}
|
||||
} else if err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("create or update another user: expected AccessDenied, got %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestUserStatusActionAuthorization(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
@@ -946,6 +1047,7 @@ func (s *TestSuiteIAM) TestCannedPolicies(c *check) {
|
||||
defaultPolicies := []string{
|
||||
"readwrite",
|
||||
"readonly",
|
||||
"consolereadonly",
|
||||
"writeonly",
|
||||
"diagnostics",
|
||||
"consoleAdmin",
|
||||
|
||||
+4
-11
@@ -632,18 +632,11 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
|
||||
return ErrInvalidDigest
|
||||
}
|
||||
|
||||
// Extract either 'X-Amz-Content-Sha256' header or 'X-Amz-Content-Sha256' query parameter (if V4 presigned)
|
||||
// Do not verify 'X-Amz-Content-Sha256' if skipSHA256.
|
||||
// Honor the selected header/query checksum, including the header fallback
|
||||
// for a presigned request. STS separately hashes its body for its signature.
|
||||
var contentSHA256 []byte
|
||||
if skipSHA256 := skipContentSha256Cksum(r); !skipSHA256 && isRequestPresignedSignatureV4(r) {
|
||||
if sha256Sum, ok := r.Form[xhttp.AmzContentSha256]; ok && len(sha256Sum) > 0 {
|
||||
contentSHA256, err = hex.DecodeString(sha256Sum[0])
|
||||
if err != nil {
|
||||
return ErrContentSHA256Mismatch
|
||||
}
|
||||
}
|
||||
} else if _, ok := r.Header[xhttp.AmzContentSha256]; !skipSHA256 && ok {
|
||||
contentSHA256, err = hex.DecodeString(r.Header.Get(xhttp.AmzContentSha256))
|
||||
if !skipContentSha256Cksum(r) {
|
||||
contentSHA256, err = hex.DecodeString(getContentSha256Cksum(r, serviceS3))
|
||||
if err != nil || len(contentSHA256) == 0 {
|
||||
return ErrContentSHA256Mismatch
|
||||
}
|
||||
|
||||
@@ -467,25 +467,6 @@ func testAdversarialHealCorsPropagatesNewerEqualValueTimestamp(obj ObjectLayer,
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdversarialBucketMetadataComparisonIsBase64CaseSensitive(t *testing.T) {
|
||||
upper := "QQ=="
|
||||
lower := "qQ=="
|
||||
upperBytes, err := base64.StdEncoding.Strict().DecodeString(upper)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lowerBytes, err := base64.StdEncoding.Strict().DecodeString(lower)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(upperBytes) == string(lowerBytes) {
|
||||
t.Fatal("test inputs unexpectedly decode to the same bytes")
|
||||
}
|
||||
if isBucketMetadataEqual(&upper, &lower) {
|
||||
t.Fatal("different decoded payloads were treated as equal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSiteReplicationStatusDetectsCorsTimestampMismatch(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
// Copyright 2026 PGSTY contributors.
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
)
|
||||
|
||||
func TestDeleteBucketMetadataLockCancellation(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testDeleteBucketMetadataLockCancellation})
|
||||
}
|
||||
|
||||
func testDeleteBucketMetadataLockCancellation(obj ObjectLayer, instanceType, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
_, unlock, err := lockBucketMetadata(t.Context(), obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release := sync.OnceFunc(unlock)
|
||||
defer release()
|
||||
|
||||
// Observe DeleteBucket's ACTUAL metadata.lock attempt. Set the hook after
|
||||
// our own acquisition above so it only trips on the delete.
|
||||
delAtLock := make(chan struct{})
|
||||
var once sync.Once
|
||||
hook := func(b string) {
|
||||
if b == bucket {
|
||||
once.Do(func() { close(delAtLock) })
|
||||
}
|
||||
}
|
||||
lockBucketMetadataAcquireHook.Store(&hook)
|
||||
defer lockBucketMetadataAcquireHook.Store(nil)
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- obj.DeleteBucket(ctx, bucket, DeleteBucketOptions{Force: true, NoLock: true}) }()
|
||||
|
||||
select {
|
||||
case <-delAtLock:
|
||||
// Fixed tree: the delete reached metadata.lock and is blocking on the
|
||||
// lock we hold. Cancel it and confirm it fails WITHOUT deleting, while we
|
||||
// still hold the lock (release stays deferred until after the checks).
|
||||
cancel()
|
||||
if err := <-done; err == nil {
|
||||
t.Errorf("%s: canceled deletion succeeded", instanceType)
|
||||
}
|
||||
if _, err := obj.GetBucketInfo(t.Context(), bucket, BucketOptions{}); err != nil {
|
||||
t.Errorf("%s: bucket disappeared while metadata.lock was held: %v", instanceType, err)
|
||||
}
|
||||
if _, err := readBucketMetadata(t.Context(), obj, bucket); err != nil {
|
||||
t.Errorf("%s: canceled deletion removed metadata: %v", instanceType, err)
|
||||
}
|
||||
case err := <-done:
|
||||
// Broken tree: the delete finished without ever taking metadata.lock,
|
||||
// i.e. it did not serialize the destructive operation behind the lock.
|
||||
t.Errorf("%s: delete bypassed metadata.lock (err=%v)", instanceType, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueuedMetadataUpdateAfterDelete(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
for _, expiry := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("expiry=%v", expiry), func(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instanceType, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
previous := newObjectLayerFn()
|
||||
barrier := &lcMergeBarrier{ObjectLayer: obj, bucket: bucket, mAtLock: make(chan struct{}), mProceed: make(chan struct{})}
|
||||
setObjectLayer(barrier)
|
||||
defer setObjectLayer(previous)
|
||||
release := sync.OnceFunc(func() { close(barrier.mProceed) })
|
||||
defer release()
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
ctx = context.WithValue(ctx, lcMergeWriterKey{}, "M")
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
if expiry {
|
||||
done <- globalBucketMetadataSys.UpdateExpiryLCConfig(ctx, bucket, nil, UTCNow())
|
||||
return
|
||||
}
|
||||
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, []byte(`<Tagging><TagSet/></Tagging>`))
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case <-barrier.mAtLock:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("writer did not reach metadata.lock")
|
||||
}
|
||||
if err := obj.DeleteBucket(t.Context(), bucket, DeleteBucketOptions{Force: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release()
|
||||
if err := <-done; !isErrBucketNotFound(err) {
|
||||
t.Errorf("%s: queued update should reject a deleted bucket, got %v", instanceType, err)
|
||||
}
|
||||
if _, err := readBucketMetadata(t.Context(), obj, bucket); !errors.Is(err, errConfigNotFound) && !isErrBucketNotFound(err) {
|
||||
t.Errorf("%s: queued update recreated metadata: %v", instanceType, err)
|
||||
}
|
||||
}})
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
// Copyright (c) 2026 PGSTY
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"net/http"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Target 1 (issue #105): a higher-level lifecycle XML merge racing another
|
||||
// bucket-metadata transition outside BucketMetadataSys.Delete.
|
||||
//
|
||||
// Before the fix, PeerBucketLCConfigHandler / healBucketILMExpiry read the
|
||||
// current lifecycle document without metadata.lock, merged the replicated expiry
|
||||
// rules with the local transition rules, and then persisted the merged blob with
|
||||
// BucketMetadataSys.Update. Update re-read the record under metadata.lock but
|
||||
// overwrote LifecycleConfigXML wholesale with the pre-computed blob, so any
|
||||
// lifecycle transition change committed between the merge read and the merge
|
||||
// write was silently lost on disk. BucketMetadataSys.UpdateExpiryLCConfig now
|
||||
// performs the read, merge, and save under a single metadata.lock. This test
|
||||
// drives PeerBucketLCConfigHandler and asserts the concurrent change survives.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type lcMergeWriterKey struct{}
|
||||
|
||||
// lcMergeBarrier pauses the merge writer (context value "M") exactly when it
|
||||
// tries to take metadata.lock for its persisting Update. By that point the
|
||||
// merge has already read the stale lifecycle document, so the test can commit a
|
||||
// concurrent transition change before releasing the merge write.
|
||||
type lcMergeBarrier struct {
|
||||
ObjectLayer
|
||||
bucket string
|
||||
mAtLock chan struct{}
|
||||
mProceed chan struct{}
|
||||
mOnce sync.Once
|
||||
}
|
||||
|
||||
func (o *lcMergeBarrier) metadataLock() string {
|
||||
return pathJoin(bucketMetaPrefix, o.bucket, "metadata.lock")
|
||||
}
|
||||
|
||||
func (o *lcMergeBarrier) NewNSLock(bucket string, objects ...string) RWLocker {
|
||||
lock := o.ObjectLayer.NewNSLock(bucket, objects...)
|
||||
if bucket != minioMetaBucket || len(objects) != 1 || objects[0] != o.metadataLock() {
|
||||
return lock
|
||||
}
|
||||
return metadataObservedRWLocker{RWLocker: lock, onLock: func(ctx context.Context) {
|
||||
if ctx.Value(lcMergeWriterKey{}) != "M" {
|
||||
return
|
||||
}
|
||||
o.mOnce.Do(func() { close(o.mAtLock) })
|
||||
select {
|
||||
case <-o.mProceed:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}}
|
||||
}
|
||||
|
||||
func TestLifecycleExpiryMergeRaceLosesConcurrentTransition(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testLifecycleExpiryMergeRaceLosesConcurrentTransition,
|
||||
})
|
||||
}
|
||||
|
||||
func testLifecycleExpiryMergeRaceLosesConcurrentTransition(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
// Revision N: a single transition-only rule.
|
||||
baseXML := []byte(`<LifecycleConfiguration><Rule><ID>keep</ID><Filter><Prefix>data/</Prefix></Filter><Status>Enabled</Status><Transition><Days>30</Days><StorageClass>WARM</StorageClass></Transition></Rule></LifecycleConfiguration>`)
|
||||
if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketLifecycleConfig, baseXML); err != nil {
|
||||
t.Fatalf("%s: seed lifecycle: %v", instanceType, err)
|
||||
}
|
||||
|
||||
// A replicated expiry-only rule arriving from a peer site.
|
||||
expXML := `<LifecycleConfiguration><Rule><ID>expire</ID><Filter><Prefix>tmp/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>7</Days></Expiration></Rule></LifecycleConfiguration>`
|
||||
expLCConfig := base64.StdEncoding.EncodeToString([]byte(expXML))
|
||||
|
||||
previousObjectAPI := newObjectLayerFn()
|
||||
barrier := &lcMergeBarrier{
|
||||
ObjectLayer: obj,
|
||||
bucket: bucket,
|
||||
mAtLock: make(chan struct{}),
|
||||
mProceed: make(chan struct{}),
|
||||
}
|
||||
setObjectLayer(barrier)
|
||||
defer setObjectLayer(previousObjectAPI)
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
mCtx := context.WithValue(ctx, lcMergeWriterKey{}, "M")
|
||||
|
||||
mDone := make(chan error, 1)
|
||||
go func() {
|
||||
mDone <- globalSiteReplicationSys.PeerBucketLCConfigHandler(mCtx, bucket, &expLCConfig, UTCNow())
|
||||
}()
|
||||
|
||||
// Wait until the merge writer has read revision N and is about to persist.
|
||||
select {
|
||||
case <-barrier.mAtLock:
|
||||
case err := <-mDone:
|
||||
t.Fatalf("%s: merge writer finished before persisting: %v", instanceType, err)
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("%s: merge writer never reached metadata.lock: %v", instanceType, ctx.Err())
|
||||
}
|
||||
|
||||
// Concurrent local lifecycle transition change commits revision N+1.
|
||||
concurrentXML := []byte(`<LifecycleConfiguration><Rule><ID>keep</ID><Filter><Prefix>data/</Prefix></Filter><Status>Enabled</Status><Transition><Days>10</Days><StorageClass>COLD</StorageClass></Transition></Rule></LifecycleConfiguration>`)
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, concurrentXML); err != nil {
|
||||
t.Fatalf("%s: concurrent transition update: %v", instanceType, err)
|
||||
}
|
||||
|
||||
// Release the merge write so it lands after the concurrent commit.
|
||||
close(barrier.mProceed)
|
||||
if err := <-mDone; err != nil {
|
||||
t.Fatalf("%s: merge writer failed: %v", instanceType, err)
|
||||
}
|
||||
|
||||
// The persisted lifecycle must contain both the replicated expiry rule and
|
||||
// the concurrent transition change.
|
||||
cfg, _, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: read merged lifecycle: %v", instanceType, err)
|
||||
}
|
||||
var keep, expire bool
|
||||
for i := range cfg.Rules {
|
||||
switch cfg.Rules[i].ID {
|
||||
case "keep":
|
||||
keep = true
|
||||
if cfg.Rules[i].Transition.Days != 10 || cfg.Rules[i].Transition.StorageClass != "COLD" {
|
||||
t.Fatalf("%s: lifecycle merge overwrote the concurrent transition change: got Days=%d StorageClass=%q, want Days=10 StorageClass=COLD",
|
||||
instanceType, cfg.Rules[i].Transition.Days, cfg.Rules[i].Transition.StorageClass)
|
||||
}
|
||||
case "expire":
|
||||
expire = true
|
||||
}
|
||||
}
|
||||
if !expire {
|
||||
t.Fatalf("%s: merged lifecycle dropped the replicated expiry rule: %+v", instanceType, cfg.Rules)
|
||||
}
|
||||
if !keep {
|
||||
t.Fatalf("%s: merged lifecycle dropped the transition rule entirely: %+v", instanceType, cfg.Rules)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Target 2 (issue #105): DeleteBucket racing an in-flight metadata writer must
|
||||
// not resurrect a ghost .metadata.bin record.
|
||||
//
|
||||
// Before the fix, erasureServerPools.DeleteBucket took only <bucket>.lck and
|
||||
// purged the metadata prefix while config writers (updateAndParse) took only
|
||||
// metadata.lock, so a writer already MID-SAVE (holding metadata.lock, past
|
||||
// saveMetadata's existence recheck) could persist .metadata.bin after the purge.
|
||||
// DeleteBucket now takes metadata.lock before deleting, so it waits for that
|
||||
// writer and then purges whatever the writer wrote.
|
||||
//
|
||||
// This test isolates the DeleteBucket-lock fix specifically: the writer holds
|
||||
// metadata.lock and is paused at the .metadata.bin PutObject, so the earlier
|
||||
// saveMetadata existence recheck cannot save it — only serializing the delete
|
||||
// behind the writer can. Removing just DeleteBucket's metadata.lock (keeping the
|
||||
// recheck) therefore makes this test fail. The delete's ACTUAL metadata.lock
|
||||
// attempt is observed with lockBucketMetadataAcquireHook (its lock is taken
|
||||
// through the erasureServerPools receiver, invisible to the object-layer
|
||||
// barrier), so the handshake is deterministic with no timing assumption.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestDeleteBucketResurrectsGhostMetadata(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testDeleteBucketResurrectsGhostMetadata,
|
||||
})
|
||||
}
|
||||
|
||||
func testDeleteBucketResurrectsGhostMetadata(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
previousObjectAPI := newObjectLayerFn()
|
||||
// Writer A holds metadata.lock and pauses at the .metadata.bin PutObject,
|
||||
// i.e. already past saveMetadata's existence recheck and mid-save.
|
||||
barrier := &metadataRMWBarrierObjectLayer{
|
||||
ObjectLayer: obj,
|
||||
bucket: bucket,
|
||||
aReady: make(chan struct{}),
|
||||
aRelease: make(chan struct{}),
|
||||
bLockAttempt: make(chan struct{}),
|
||||
}
|
||||
setObjectLayer(barrier)
|
||||
defer setObjectLayer(previousObjectAPI)
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
aCtx := context.WithValue(ctx, metadataRMWWriterKey{}, "A")
|
||||
|
||||
policyJSON := []byte(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::` + bucket + `/*"}]}`)
|
||||
aReleased := sync.OnceFunc(func() { close(barrier.aRelease) })
|
||||
defer aReleased()
|
||||
aDone := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := globalBucketMetadataSys.Update(aCtx, bucket, bucketPolicyConfig, policyJSON)
|
||||
aDone <- err
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-barrier.aReady:
|
||||
case err := <-aDone:
|
||||
t.Fatalf("%s: writer A finished before persisting: %v", instanceType, err)
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("%s: writer A never reached metadata save: %v", instanceType, ctx.Err())
|
||||
}
|
||||
|
||||
// A now holds metadata.lock mid-save. Observe DeleteBucket's ACTUAL
|
||||
// metadata.lock attempt via the acquire hook, set only now so A's earlier
|
||||
// acquisition does not trip it.
|
||||
delAtLock := make(chan struct{})
|
||||
var once sync.Once
|
||||
hook := func(b string) {
|
||||
if b == bucket {
|
||||
once.Do(func() { close(delAtLock) })
|
||||
}
|
||||
}
|
||||
lockBucketMetadataAcquireHook.Store(&hook)
|
||||
defer lockBucketMetadataAcquireHook.Store(nil)
|
||||
|
||||
delDone := make(chan error, 1)
|
||||
go func() {
|
||||
delDone <- obj.DeleteBucket(ctx, bucket, DeleteBucketOptions{Force: true})
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-delAtLock:
|
||||
// Fixed tree: DeleteBucket reached metadata.lock and blocks on A. Release
|
||||
// A so it finishes its save and unlocks; the delete then acquires the
|
||||
// lock and purges the record A wrote.
|
||||
aReleased()
|
||||
if err := <-aDone; err != nil {
|
||||
t.Fatalf("%s: writer A save failed while holding metadata.lock: %v", instanceType, err)
|
||||
}
|
||||
if err := <-delDone; err != nil {
|
||||
t.Fatalf("%s: delete bucket: %v", instanceType, err)
|
||||
}
|
||||
case err := <-delDone:
|
||||
// Broken tree: DeleteBucket purged without taking metadata.lock. Release
|
||||
// A so its mid-save PutObject recreates .metadata.bin (the ghost).
|
||||
if err != nil {
|
||||
t.Fatalf("%s: delete bucket: %v", instanceType, err)
|
||||
}
|
||||
aReleased()
|
||||
if err := <-aDone; err != nil {
|
||||
t.Fatalf("%s: writer A save failed: %v", instanceType, err)
|
||||
}
|
||||
}
|
||||
|
||||
// After DeleteBucket, no .metadata.bin record may remain on disk.
|
||||
if meta, err := readBucketMetadata(ctx, obj, bucket); err == nil {
|
||||
t.Fatalf("%s: ghost .metadata.bin resurrected after DeleteBucket: name=%q created=%s policyLen=%d",
|
||||
instanceType, meta.Name, meta.Created, len(meta.PolicyConfigJSON))
|
||||
} else if !errors.Is(err, errConfigNotFound) && !isErrBucketNotFound(err) && !errors.Is(err, errVolumeNotFound) {
|
||||
t.Fatalf("%s: unexpected error reading deleted bucket metadata: %v", instanceType, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
// Copyright 2026 PGSTY contributors.
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/grid"
|
||||
)
|
||||
|
||||
func TestPeerMetadataReloadWithEqualMaximumTimestamp(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testPeerMetadataReloadWithEqualMaximumTimestamp})
|
||||
}
|
||||
|
||||
func testPeerMetadataReloadWithEqualMaximumTimestamp(obj ObjectLayer, instanceType, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
disk, err := loadBucketMetadata(t.Context(), obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
disk.TaggingConfigXML = []byte(`<Tagging><TagSet><Tag><Key>revision</Key><Value>new</Value></Tag></TagSet></Tagging>`)
|
||||
disk.TaggingConfigUpdatedAt = UTCNow()
|
||||
// An unrelated configuration has the greatest timestamp in both records.
|
||||
disk.PolicyConfigUpdatedAt = disk.TaggingConfigUpdatedAt.Add(time.Hour)
|
||||
if err := disk.Save(t.Context(), obj); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resident := disk
|
||||
resident.TaggingConfigXML = []byte(`<Tagging><TagSet><Tag><Key>revision</Key><Value>old</Value></Tag></TagSet></Tagging>`)
|
||||
resident.TaggingConfigUpdatedAt = disk.TaggingConfigUpdatedAt.Add(-time.Minute)
|
||||
if err := resident.parseAllConfigs(t.Context(), obj); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !resident.lastUpdate().Equal(disk.lastUpdate()) {
|
||||
t.Fatal("fixture must have equal maximum timestamps")
|
||||
}
|
||||
globalBucketMetadataSys.Set(bucket, resident)
|
||||
|
||||
args := grid.MSS{peerRESTBucket: bucket}
|
||||
if _, err := (&peerRESTServer{}).LoadBucketMetadataHandler(&args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tagging, _, err := globalBucketMetadataSys.GetTaggingConfig(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := tagging.ToMap()["revision"]; got != "new" {
|
||||
t.Errorf("%s: peer reload retained tag %q despite a newer tagging configuration", instanceType, got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
bucketsse "github.com/minio/minio/internal/bucket/encryption"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// Only these fields share the site-replication source-time ordering contract.
|
||||
// Bulk apply/import process Object Lock before Versioning, whose effective
|
||||
// document depends on it. Periodic heal retains its existing type order.
|
||||
var replicatedBucketConfigs = [...]string{
|
||||
objectLockConfig, bucketVersioningConfig, bucketPolicyConfig,
|
||||
bucketTaggingConfig, bucketSSEConfig, bucketQuotaConfigFile,
|
||||
}
|
||||
|
||||
func replicatedBucketConfig(meta *BucketMetadata, file string) (*[]byte, *time.Time) {
|
||||
switch file {
|
||||
case bucketPolicyConfig:
|
||||
return &meta.PolicyConfigJSON, &meta.PolicyConfigUpdatedAt
|
||||
case bucketTaggingConfig:
|
||||
return &meta.TaggingConfigXML, &meta.TaggingConfigUpdatedAt
|
||||
case bucketSSEConfig:
|
||||
return &meta.EncryptionConfigXML, &meta.EncryptionConfigUpdatedAt
|
||||
case bucketQuotaConfigFile:
|
||||
return &meta.QuotaConfigJSON, &meta.QuotaConfigUpdatedAt
|
||||
case bucketVersioningConfig:
|
||||
return &meta.VersioningConfigXML, &meta.VersioningConfigUpdatedAt
|
||||
case objectLockConfig:
|
||||
return &meta.ObjectLockConfigXML, &meta.ObjectLockConfigUpdatedAt
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Callers that only need to know whether a file is under the contract must not
|
||||
// probe replicatedBucketConfig with a throwaway BucketMetadata.
|
||||
func isReplicatedBucketConfig(file string) bool {
|
||||
for _, replicated := range replicatedBucketConfigs {
|
||||
if replicated == file {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func bucketConfigUpdateOnly(file string) bool {
|
||||
return file == bucketVersioningConfig || file == objectLockConfig
|
||||
}
|
||||
|
||||
// Reuse the persistence rule before comparison, so an accepted Versioning
|
||||
// event and the document Save actually writes have the same comparison key.
|
||||
func effectiveBucketVersioning(data []byte, lockEnabled bool) []byte {
|
||||
if lockEnabled {
|
||||
config, err := versioning.ParseConfig(bytes.NewReader(data))
|
||||
if err != nil || !config.Enabled() || config.PrefixesExcluded() {
|
||||
return enabledBucketVersioningConfig
|
||||
}
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
// A parsed policy still contains map-backed sets with nondeterministic Marshal
|
||||
// order. Sort every set array recursively, including statements and conditions.
|
||||
// RawMessage keeps integer values intact; decoding through float64 would not.
|
||||
func canonicalBucketPolicyJSON(data json.RawMessage) (json.RawMessage, error) {
|
||||
data = bytes.TrimSpace(data)
|
||||
if len(data) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
switch data[0] {
|
||||
case '{':
|
||||
var obj map[string]json.RawMessage
|
||||
if err := json.Unmarshal(data, &obj); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for key, value := range obj {
|
||||
var err error
|
||||
obj[key], err = canonicalBucketPolicyJSON(value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return json.Marshal(obj)
|
||||
case '[':
|
||||
var arr []json.RawMessage
|
||||
if err := json.Unmarshal(data, &arr); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range arr {
|
||||
var err error
|
||||
arr[i], err = canonicalBucketPolicyJSON(arr[i])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
sort.Slice(arr, func(i, j int) bool { return bytes.Compare(arr[i], arr[j]) < 0 })
|
||||
return json.Marshal(arr)
|
||||
default:
|
||||
var compact bytes.Buffer
|
||||
if err := json.Compact(&compact, data); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return compact.Bytes(), nil
|
||||
}
|
||||
}
|
||||
|
||||
// Encode the validated policy fields explicitly: BPStatement's required
|
||||
// Action/Resource tags otherwise try to marshal empty sets for the supported
|
||||
// NotAction/NotResource alternatives. This stays within the existing schema.
|
||||
func canonicalBucketPolicy(cfg *policy.BucketPolicy) ([]byte, error) {
|
||||
if cfg.IsEmpty() {
|
||||
return nil, nil
|
||||
}
|
||||
doc := map[string]any{"Version": cfg.Version}
|
||||
if cfg.ID != "" {
|
||||
doc["ID"] = cfg.ID
|
||||
}
|
||||
statements := make([]map[string]any, 0, len(cfg.Statements))
|
||||
for _, st := range cfg.Statements {
|
||||
statement := map[string]any{"Effect": st.Effect, "Principal": st.Principal}
|
||||
if st.SID != "" {
|
||||
statement["Sid"] = st.SID
|
||||
}
|
||||
if len(st.Actions) != 0 {
|
||||
statement["Action"] = st.Actions
|
||||
}
|
||||
if len(st.NotActions) != 0 {
|
||||
statement["NotAction"] = st.NotActions
|
||||
}
|
||||
if len(st.Resources) != 0 {
|
||||
statement["Resource"] = st.Resources
|
||||
}
|
||||
if len(st.NotResources) != 0 {
|
||||
statement["NotResource"] = st.NotResources
|
||||
}
|
||||
if len(st.Conditions) != 0 {
|
||||
statement["Condition"] = st.Conditions
|
||||
}
|
||||
statements = append(statements, statement)
|
||||
}
|
||||
doc["Statement"] = statements
|
||||
data, err := json.Marshal(doc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return canonicalBucketPolicyJSON(data)
|
||||
}
|
||||
|
||||
// Validate with the same parsers as Save. Policy's established empty-policy
|
||||
// semantics are deletion; a parsed zero quota is still a live document.
|
||||
func bucketConfigPayload(bucket, file string, data []byte, lockEnabled bool) ([]byte, []byte, error) {
|
||||
if len(data) == 0 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
var err error
|
||||
key := data
|
||||
switch file {
|
||||
case bucketPolicyConfig:
|
||||
var cfg *policy.BucketPolicy
|
||||
cfg, err = policy.ParseBucketPolicyConfig(bytes.NewReader(data), bucket)
|
||||
if err == nil {
|
||||
if cfg.IsEmpty() {
|
||||
return nil, nil, nil
|
||||
}
|
||||
key, err = canonicalBucketPolicy(cfg)
|
||||
}
|
||||
case bucketQuotaConfigFile:
|
||||
cfg, parseErr := parseBucketQuota(bucket, data)
|
||||
err = parseErr
|
||||
if err == nil {
|
||||
key, err = json.Marshal(cfg)
|
||||
}
|
||||
case bucketTaggingConfig:
|
||||
_, err = tags.ParseBucketXML(bytes.NewReader(data))
|
||||
case bucketSSEConfig:
|
||||
_, err = bucketsse.ParseBucketSSEConfig(bytes.NewReader(data))
|
||||
case objectLockConfig:
|
||||
_, err = objectlock.ParseObjectLockConfig(bytes.NewReader(data))
|
||||
case bucketVersioningConfig:
|
||||
data = effectiveBucketVersioning(data, lockEnabled)
|
||||
key = data
|
||||
_, err = versioning.ParseConfig(bytes.NewReader(data))
|
||||
}
|
||||
return data, key, err
|
||||
}
|
||||
|
||||
type bucketConfigState struct {
|
||||
data, key []byte
|
||||
at time.Time
|
||||
real, valid bool
|
||||
}
|
||||
|
||||
func newBucketConfigState(bucket, file string, data []byte, at, created time.Time, lockEnabled bool) (bucketConfigState, error) {
|
||||
data, key, err := bucketConfigPayload(bucket, file, data, lockEnabled)
|
||||
if err != nil {
|
||||
return bucketConfigState{}, err
|
||||
}
|
||||
if at.IsZero() {
|
||||
at = created
|
||||
}
|
||||
valid := !created.IsZero() && !at.Before(created)
|
||||
modified := valid && at.After(created)
|
||||
if bucketConfigUpdateOnly(file) && len(data) == 0 {
|
||||
modified = false
|
||||
}
|
||||
return bucketConfigState{data: data, key: key, at: at, real: modified, valid: valid}, nil
|
||||
}
|
||||
|
||||
func (s bucketConfigState) candidate() bool {
|
||||
return s.valid && (s.real || len(s.data) != 0)
|
||||
}
|
||||
|
||||
func compareBucketConfigStates(a, b bucketConfigState) int {
|
||||
if a.valid != b.valid {
|
||||
if a.valid {
|
||||
return 1
|
||||
}
|
||||
return -1
|
||||
}
|
||||
if a.real != b.real {
|
||||
if a.real {
|
||||
return 1
|
||||
}
|
||||
return -1
|
||||
}
|
||||
if a.real {
|
||||
if n := a.at.Compare(b.at); n != 0 {
|
||||
return n
|
||||
}
|
||||
// At equal source time a real deletion wins, preventing resurrection.
|
||||
if (len(a.data) == 0) != (len(b.data) == 0) {
|
||||
if len(a.data) == 0 {
|
||||
return 1
|
||||
}
|
||||
return -1
|
||||
}
|
||||
}
|
||||
return bytes.Compare(a.key, b.key)
|
||||
}
|
||||
|
||||
func localBucketConfigUpdatedAt(meta BucketMetadata, file string, now time.Time) time.Time {
|
||||
_, at := replicatedBucketConfig(&meta, file)
|
||||
for _, lower := range []time.Time{meta.Created, *at} {
|
||||
if !now.After(lower) {
|
||||
now = lower.Add(time.Nanosecond)
|
||||
}
|
||||
}
|
||||
return now.UTC()
|
||||
}
|
||||
|
||||
func ensureBucketMetadataCreated(ctx context.Context, obj ObjectLayer, meta *BucketMetadata) error {
|
||||
if !meta.Created.IsZero() {
|
||||
return nil
|
||||
}
|
||||
info, err := obj.GetBucketInfo(ctx, meta.Name, BucketOptions{NoMetadata: true})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Created.IsZero() {
|
||||
return errors.New("bucket metadata creation time is unknown")
|
||||
}
|
||||
meta.Created = info.Created.UTC()
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyBucketConfig runs under metadata.lock, on freshly loaded metadata. It
|
||||
// changes only the selected field; the caller persists once after all checks.
|
||||
func applyBucketConfig(meta *BucketMetadata, file string, data []byte, at time.Time) (bool, error) {
|
||||
if bucketConfigUpdateOnly(file) && len(data) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
current, currentAt := replicatedBucketConfig(meta, file)
|
||||
lockEnabled := len(meta.ObjectLockConfigXML) != 0
|
||||
incoming, err := newBucketConfigState(meta.Name, file, data, at, meta.Created, lockEnabled)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !incoming.candidate() {
|
||||
return false, nil
|
||||
}
|
||||
local, err := newBucketConfigState(meta.Name, file, *current, *currentAt, meta.Created, lockEnabled)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if compareBucketConfigStates(incoming, local) <= 0 {
|
||||
return false, nil
|
||||
}
|
||||
*current, *currentAt = bytes.Clone(incoming.data), incoming.at.UTC()
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func rebaseBucketConfigDefaults(meta *BucketMetadata, oldCreated time.Time) {
|
||||
if meta.Created.Equal(oldCreated) {
|
||||
return
|
||||
}
|
||||
// These six fields alone use Created to distinguish a baseline from a
|
||||
// tombstone. Preserve actual source times when adopting an existing bucket.
|
||||
for _, file := range replicatedBucketConfigs {
|
||||
_, at := replicatedBucketConfig(meta, file)
|
||||
if at.IsZero() || at.Equal(oldCreated) {
|
||||
*at = meta.Created
|
||||
}
|
||||
}
|
||||
}
|
||||
+160
-51
@@ -24,6 +24,7 @@ import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
@@ -125,19 +126,42 @@ func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
|
||||
}
|
||||
}
|
||||
|
||||
func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string, configFile string, configData []byte, parse, lifecycleDelete bool) (updatedAt time.Time, err error) {
|
||||
// bucketMetadataUpdate returns the committed snapshot to the caller. meta and
|
||||
// updatedAt hold the saved state only when changed is true. Local writes always
|
||||
// change state, because localBucketConfigUpdatedAt is strictly greater than the
|
||||
// current field time, so their handlers can broadcast meta without rechecking.
|
||||
type bucketMetadataUpdate struct {
|
||||
meta BucketMetadata
|
||||
updatedAt time.Time
|
||||
changed bool
|
||||
}
|
||||
|
||||
func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket, configFile string, configData []byte, parse, lifecycleDelete bool) (time.Time, error) {
|
||||
result, err := sys.updateAndParseMetadata(ctx, bucket, configFile, configData, parse, lifecycleDelete, nil)
|
||||
return result.updatedAt, err
|
||||
}
|
||||
|
||||
func (sys *BucketMetadataSys) updateAndParseMetadata(ctx context.Context, bucket string, configFile string, configData []byte, parse, lifecycleDelete bool, sourceTime *time.Time) (result bucketMetadataUpdate, err error) {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return updatedAt, errServerNotInitialized
|
||||
return result, errServerNotInitialized
|
||||
}
|
||||
|
||||
if isMinioMetaBucketName(bucket) {
|
||||
return updatedAt, errInvalidArgument
|
||||
return result, errInvalidArgument
|
||||
}
|
||||
// Load deletions without parsed caches (notably quota), and compare the
|
||||
// six replicated fields against the raw document under the same lock.
|
||||
if isReplicatedBucketConfig(configFile) {
|
||||
parse = false
|
||||
if bucketConfigUpdateOnly(configFile) && len(configData) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
notifyCtx := ctx
|
||||
ctx, unlock, err := lockBucketMetadata(ctx, objAPI, bucket)
|
||||
if err != nil {
|
||||
return updatedAt, err
|
||||
return result, err
|
||||
}
|
||||
|
||||
err = func() error {
|
||||
@@ -157,55 +181,73 @@ func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string,
|
||||
return err
|
||||
}
|
||||
}
|
||||
updatedAt = UTCNow()
|
||||
switch configFile {
|
||||
case bucketPolicyConfig:
|
||||
meta.PolicyConfigJSON = configData
|
||||
meta.PolicyConfigUpdatedAt = updatedAt
|
||||
case bucketNotificationConfig:
|
||||
meta.NotificationConfigXML = configData
|
||||
meta.NotificationConfigUpdatedAt = updatedAt
|
||||
case bucketLifecycleConfig:
|
||||
meta.LifecycleConfigXML = configData
|
||||
meta.LifecycleConfigUpdatedAt = updatedAt
|
||||
case bucketSSEConfig:
|
||||
meta.EncryptionConfigXML = configData
|
||||
meta.EncryptionConfigUpdatedAt = updatedAt
|
||||
case bucketTaggingConfig:
|
||||
meta.TaggingConfigXML = configData
|
||||
meta.TaggingConfigUpdatedAt = updatedAt
|
||||
case bucketQuotaConfigFile:
|
||||
meta.QuotaConfigJSON = configData
|
||||
meta.QuotaConfigUpdatedAt = updatedAt
|
||||
case objectLockConfig:
|
||||
meta.ObjectLockConfigXML = configData
|
||||
meta.ObjectLockConfigUpdatedAt = updatedAt
|
||||
case bucketVersioningConfig:
|
||||
meta.VersioningConfigXML = configData
|
||||
meta.VersioningConfigUpdatedAt = updatedAt
|
||||
case bucketReplicationConfig:
|
||||
meta.ReplicationConfigXML = configData
|
||||
meta.ReplicationConfigUpdatedAt = updatedAt
|
||||
case bucketTargetsFile:
|
||||
meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(ctx, meta.Name, configData, kms.Context{
|
||||
bucket: meta.Name,
|
||||
bucketTargetsFile: bucketTargetsFile,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error encrypting bucket target metadata %w", err)
|
||||
updatedAt := UTCNow()
|
||||
if isReplicatedBucketConfig(configFile) {
|
||||
if err := ensureBucketMetadataCreated(ctx, objAPI, &meta); err != nil {
|
||||
var at time.Time
|
||||
if sourceTime != nil {
|
||||
at = *sourceTime
|
||||
}
|
||||
logBucketConfigReplication(ctx, bucket, configFile, "indeterminate", at, meta.Created, err.Error())
|
||||
return err
|
||||
}
|
||||
if sourceTime == nil || sourceTime.IsZero() {
|
||||
updatedAt = localBucketConfigUpdatedAt(meta, configFile, updatedAt)
|
||||
if sourceTime != nil {
|
||||
logBucketConfigReplication(ctx, bucket, configFile, "legacy-zero", *sourceTime, meta.Created, "assigned local source time")
|
||||
}
|
||||
} else {
|
||||
updatedAt = sourceTime.UTC()
|
||||
}
|
||||
if updatedAt.Before(meta.Created) {
|
||||
logBucketConfigReplication(ctx, bucket, configFile, "before-created", updatedAt, meta.Created, "peer event")
|
||||
return nil
|
||||
}
|
||||
changed, err := applyBucketConfig(&meta, configFile, configData, updatedAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !changed {
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
switch configFile {
|
||||
case bucketNotificationConfig:
|
||||
meta.NotificationConfigXML = configData
|
||||
meta.NotificationConfigUpdatedAt = updatedAt
|
||||
case bucketLifecycleConfig:
|
||||
meta.LifecycleConfigXML = configData
|
||||
meta.LifecycleConfigUpdatedAt = updatedAt
|
||||
case bucketReplicationConfig:
|
||||
meta.ReplicationConfigXML = configData
|
||||
meta.ReplicationConfigUpdatedAt = updatedAt
|
||||
case bucketTargetsFile:
|
||||
meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(ctx, meta.Name, configData, kms.Context{
|
||||
bucket: meta.Name,
|
||||
bucketTargetsFile: bucketTargetsFile,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error encrypting bucket target metadata %w", err)
|
||||
}
|
||||
meta.BucketTargetsConfigUpdatedAt = updatedAt
|
||||
meta.BucketTargetsConfigMetaUpdatedAt = updatedAt
|
||||
default:
|
||||
return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
|
||||
}
|
||||
meta.BucketTargetsConfigUpdatedAt = updatedAt
|
||||
meta.BucketTargetsConfigMetaUpdatedAt = updatedAt
|
||||
default:
|
||||
return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
|
||||
}
|
||||
return sys.saveMetadata(ctx, objAPI, meta)
|
||||
if err := sys.saveMetadata(ctx, objAPI, &meta); err != nil {
|
||||
return err
|
||||
}
|
||||
result = bucketMetadataUpdate{meta: meta, updatedAt: updatedAt, changed: true}
|
||||
return nil
|
||||
}()
|
||||
if err != nil {
|
||||
return updatedAt, err
|
||||
return result, err
|
||||
}
|
||||
globalNotificationSys.LoadBucketMetadata(bgContext(notifyCtx), bucket) // Do not use caller context here
|
||||
return updatedAt, nil
|
||||
if result.changed {
|
||||
globalNotificationSys.LoadBucketMetadata(bgContext(notifyCtx), bucket)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) error {
|
||||
@@ -218,7 +260,7 @@ func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) err
|
||||
return errInvalidArgument
|
||||
}
|
||||
|
||||
if err := sys.saveMetadata(ctx, objAPI, meta); err != nil {
|
||||
if err := sys.saveMetadata(ctx, objAPI, &meta); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -228,11 +270,16 @@ func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) err
|
||||
|
||||
// saveMetadata persists and publishes metadata locally. Callers performing a
|
||||
// read-modify-write must hold metadata.lock and release it before peer fan-out.
|
||||
func (sys *BucketMetadataSys) saveMetadata(ctx context.Context, objAPI ObjectLayer, meta BucketMetadata) error {
|
||||
func (sys *BucketMetadataSys) saveMetadata(ctx context.Context, objAPI ObjectLayer, meta *BucketMetadata) error {
|
||||
// A writer may have queued for metadata.lock before DeleteBucket completed.
|
||||
// Recheck the physical bucket under that lock, before recreating metadata.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, meta.Name, BucketOptions{NoMetadata: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := meta.Save(ctx, objAPI); err != nil {
|
||||
return err
|
||||
}
|
||||
sys.Set(meta.Name, meta)
|
||||
sys.Set(meta.Name, *meta)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -240,8 +287,19 @@ func lockBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket strin
|
||||
return lockBucketMetadataWithTimeout(ctx, objectAPI, bucket, globalOperationTimeout)
|
||||
}
|
||||
|
||||
// lockBucketMetadataAcquireHook, when set, is invoked at the start of every
|
||||
// metadata.lock acquisition, immediately before the blocking Lock() call. It is
|
||||
// nil in production (a single atomic load, no behavior change) and exists only
|
||||
// so tests can deterministically observe a caller reaching the metadata lock —
|
||||
// notably DeleteBucket, whose lock is taken through its erasureServerPools
|
||||
// receiver and is therefore invisible to an injected object layer.
|
||||
var lockBucketMetadataAcquireHook atomic.Pointer[func(bucket string)]
|
||||
|
||||
func lockBucketMetadataWithTimeout(ctx context.Context, objectAPI ObjectLayer, bucket string, timeout *dynamicTimeout) (context.Context, func(), error) {
|
||||
lock := objectAPI.NewNSLock(minioMetaBucket, pathJoin(bucketMetaPrefix, bucket, "metadata.lock"))
|
||||
if hook := lockBucketMetadataAcquireHook.Load(); hook != nil {
|
||||
(*hook)(bucket)
|
||||
}
|
||||
lkctx, err := lock.GetLock(ctx, timeout)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
@@ -292,6 +350,57 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
|
||||
return sys.updateAndParse(ctx, bucket, configFile, configData, true, false)
|
||||
}
|
||||
|
||||
// UpdateExpiryLCConfig merges a replicated ILM expiry configuration with the
|
||||
// bucket's current lifecycle document and persists the merged result while
|
||||
// holding metadata.lock across the read, merge, and save. The site-replication
|
||||
// expiry heal and peer-apply paths must use this instead of computing the merge
|
||||
// from an unlocked GetConfigFromDisk read and then writing it with Update: that
|
||||
// two-step sequence drops any lifecycle transition change committed in between
|
||||
// (issue #105). Lock order stays <bucket>.lck -> metadata.lock -> .metadata.bin;
|
||||
// the merge and save run under metadata.lock and the peer fan-out runs after it
|
||||
// is released.
|
||||
func (sys *BucketMetadataSys) UpdateExpiryLCConfig(ctx context.Context, bucket string, expLCConfig *string, updatedAt time.Time) error {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
|
||||
if isMinioMetaBucketName(bucket) {
|
||||
return errInvalidArgument
|
||||
}
|
||||
|
||||
notifyCtx := ctx
|
||||
ctx, unlock, err := lockBucketMetadata(ctx, objAPI, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = func() error {
|
||||
defer unlock()
|
||||
meta, err := loadBucketMetadataParse(ctx, objAPI, bucket, true)
|
||||
if err != nil {
|
||||
if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
|
||||
// Only single drive mode needs this fallback.
|
||||
meta = newBucketMetadata(bucket)
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
configData, err := mergeExpiryWithLCConfig(bucket, meta, expLCConfig, updatedAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
meta.LifecycleConfigXML = configData
|
||||
meta.LifecycleConfigUpdatedAt = UTCNow()
|
||||
return sys.saveMetadata(ctx, objAPI, &meta)
|
||||
}()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
globalNotificationSys.LoadBucketMetadata(bgContext(notifyCtx), bucket) // Do not use caller context here
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get metadata for a bucket.
|
||||
// If no metadata exists errConfigNotFound is returned and a new metadata is returned.
|
||||
// Only a shallow copy is returned, so referenced data should not be modified,
|
||||
|
||||
@@ -303,6 +303,9 @@ func loadBucketMetadataParseUnderLock(ctx context.Context, objectAPI ObjectLayer
|
||||
return newBucketMetadata(bucket), fmt.Errorf("%w: %v", errBucketMetadataMigrationLockUnavailable, err)
|
||||
}
|
||||
defer unlock()
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{NoMetadata: true}); err != nil {
|
||||
return newBucketMetadata(bucket), err
|
||||
}
|
||||
return loadBucketMetadataParse(ctx, objectAPI, bucket, parse)
|
||||
}
|
||||
|
||||
@@ -385,15 +388,7 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
|
||||
} else {
|
||||
b.objectLockConfig = nil
|
||||
}
|
||||
if b.objectLockConfig != nil {
|
||||
// Object Lock requires every object to be versioned. Whatever the lock
|
||||
// document contains, a suspended or prefix-excluded versioning document
|
||||
// is replaced by plain Enabled versioning; Save persists the result.
|
||||
config, versioningErr := versioning.ParseConfig(bytes.NewReader(b.VersioningConfigXML))
|
||||
if versioningErr != nil || !config.Enabled() || config.PrefixesExcluded() {
|
||||
b.VersioningConfigXML = enabledBucketVersioningConfig
|
||||
}
|
||||
}
|
||||
b.VersioningConfigXML = effectiveBucketVersioning(b.VersioningConfigXML, b.objectLockConfig != nil)
|
||||
|
||||
if len(b.VersioningConfigXML) != 0 {
|
||||
b.versioningConfig, err = versioning.ParseConfig(bytes.NewReader(b.VersioningConfigXML))
|
||||
|
||||
@@ -39,3 +39,28 @@ func TestBucketMetadataCorsRoundTrip(t *testing.T) {
|
||||
t.Fatalf("CorsConfigUpdatedAt not preserved")
|
||||
}
|
||||
}
|
||||
|
||||
// A persisted retired extension must not prevent the whole bucket's metadata
|
||||
// from loading, including unrelated versioning and ordinary lifecycle rules.
|
||||
func TestBucketMetadataRetiredAccessTiering(t *testing.T) {
|
||||
meta := newBucketMetadata("retired-access")
|
||||
meta.LifecycleConfigXML = []byte(`<LifecycleConfiguration><AccessTierQuota>500GiB</AccessTierQuota><Rule><ID>access</ID><Status>Enabled</Status><Filter><Prefix>logs/</Prefix></Filter><AccessTransition><Window>10m</Window><PromoteAfterAccesses>10</PromoteAfterAccesses></AccessTransition></Rule><Rule><ID>ordinary</ID><Status>Enabled</Status><Filter><Prefix>expired/</Prefix></Filter><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>`)
|
||||
meta.VersioningConfigXML = []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status></VersioningConfiguration>`)
|
||||
data, err := meta.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := newBucketMetadata(meta.Name)
|
||||
if _, err := got.UnmarshalMsg(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := got.parseAllConfigs(t.Context(), nil); err != nil {
|
||||
t.Fatalf("bucket metadata failed to load: %v", err)
|
||||
}
|
||||
if got.lifecycleConfig == nil || got.lifecycleConfig.HasActiveRules("logs/") || !got.lifecycleConfig.HasActiveRules("expired/") {
|
||||
t.Fatal("unexpected lifecycle behavior")
|
||||
}
|
||||
if got.versioningConfig == nil || !got.versioningConfig.Enabled() {
|
||||
t.Fatal("unrelated versioning lost")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -334,11 +334,10 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
||||
return mode, retainDate, legalHold, ErrObjectLocked
|
||||
}
|
||||
|
||||
if !legalHoldRequested && retentionCfg.LockEnabled {
|
||||
// inherit retention from bucket configuration
|
||||
return retentionCfg.Mode, objectlock.RetentionDate{Time: t.Add(retentionCfg.Validity)}, legalHold, ErrNone
|
||||
}
|
||||
return "", objectlock.RetentionDate{}, legalHold, ErrNone
|
||||
// Inherit retention from the bucket configuration. A legal-hold header
|
||||
// on the same request, ON or OFF, is independent of retention and must
|
||||
// not suppress the default (#165).
|
||||
return retentionCfg.Mode, objectlock.RetentionDate{Time: t.Add(retentionCfg.Validity)}, legalHold, ErrNone
|
||||
}
|
||||
return mode, retainDate, legalHold, ErrNone
|
||||
}
|
||||
|
||||
@@ -19,7 +19,6 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
@@ -100,13 +99,13 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
return
|
||||
}
|
||||
|
||||
configData, err := json.Marshal(bucketPolicy)
|
||||
configData, err := canonicalBucketPolicy(bucketPolicy)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
|
||||
result, err := globalBucketMetadataSys.updateAndParseMetadata(ctx, bucket, bucketPolicyConfig, configData, false, false, nil)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -116,8 +115,8 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypePolicy,
|
||||
Bucket: bucket,
|
||||
Policy: bucketPolicyBytes,
|
||||
UpdatedAt: updatedAt,
|
||||
Policy: result.meta.PolicyConfigJSON,
|
||||
UpdatedAt: result.updatedAt,
|
||||
}))
|
||||
|
||||
// Success.
|
||||
@@ -200,7 +199,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
return
|
||||
}
|
||||
|
||||
configData, err := json.Marshal(config)
|
||||
configData, err := canonicalBucketPolicy(config)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
|
||||
+26
-7
@@ -255,13 +255,32 @@ func getConditionValuesWithTags(r *http.Request, lc string, cred auth.Credential
|
||||
}
|
||||
|
||||
cloneHeader := r.Header.Clone()
|
||||
signatureAge := cloneHeader.Get("x-amz-signature-age")
|
||||
cloneHeader.Del("x-amz-signature-age")
|
||||
// The presigned V4 verifier overwrites this internal scratch header after
|
||||
// validating the signature. Ignore a value supplied on every other request
|
||||
// type, where it would otherwise synthesize s3:signatureAge.
|
||||
if authType == authTypePresigned && signatureAge != "" {
|
||||
args["signatureAge"] = []string{signatureAge}
|
||||
|
||||
// s3:signatureAge is derived from the presigned X-Amz-Date rather than from
|
||||
// anything the verifier writes back: PutObject and UploadPart authorize
|
||||
// before they verify the signature, so a post-verification value is not yet
|
||||
// available on the first evaluation. The date is bound by the signature
|
||||
// (doesPresignedSignatureMatch rebuilds and compares it), so a forged date
|
||||
// only changes the authorization outcome of a request that then fails
|
||||
// verification. A date that does not parse leaves the key absent; the
|
||||
// verifier rejects the request as ErrMalformedPresignedDate.
|
||||
if authType == authTypePresigned {
|
||||
if signedDate, err := time.Parse(iso8601Format, r.Form.Get(xhttp.AmzDate)); err == nil {
|
||||
args["signatureAge"] = []string{strconv.FormatInt(currTime.Sub(signedDate).Milliseconds(), 10)}
|
||||
}
|
||||
}
|
||||
|
||||
// s3:x-amz-content-sha256 must name the payload hash the request is actually
|
||||
// verified and enforced against, and only one such value. Presence of the
|
||||
// header controls whether the key exists at all (AWS documents that the
|
||||
// query-string form does not populate it), but the value comes from the same
|
||||
// selection getContentSha256Cksum makes for verification: the presigned query
|
||||
// value takes precedence over the header, and a repeated header contributes
|
||||
// only its first value. Exposing every raw header value instead let a
|
||||
// request satisfy a policy with a value the verifier never checked.
|
||||
if _, ok := cloneHeader[xhttp.AmzContentSha256]; ok {
|
||||
args[xhttp.AmzContentSha256] = []string{getContentSha256Cksum(r, serviceS3)}
|
||||
cloneHeader.Del(xhttp.AmzContentSha256)
|
||||
}
|
||||
|
||||
userTags := cloneHeader.Get(xhttp.AmzObjectTagging)
|
||||
|
||||
@@ -23,8 +23,10 @@ import (
|
||||
"net/url"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
@@ -579,8 +581,20 @@ func TestGetConditionValuesRejectsAbsentInternalKeys(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// s3:signatureAge is derived from the presigned X-Amz-Date, which the signature
|
||||
// binds. A client header under the former scratch name must never supply it on
|
||||
// any auth type, and a presign whose date is missing or malformed leaves the
|
||||
// key absent (the verifier then rejects the request).
|
||||
func TestGetConditionValuesOnlyAcceptsPresignedSignatureAge(t *testing.T) {
|
||||
const signatureAgeHeader = "x-amz-signature-age"
|
||||
signedDate := UTCNow().Add(-90 * time.Second)
|
||||
presignQuery := func(date string) string {
|
||||
q := url.Values{xhttp.AmzCredential: {"access/20260803/us-east-1/s3/aws4_request"}}
|
||||
if date != "" {
|
||||
q.Set(xhttp.AmzDate, date)
|
||||
}
|
||||
return "http://minio.local/bkt/obj?" + q.Encode()
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
@@ -602,19 +616,33 @@ func TestGetConditionValuesOnlyAcceptsPresignedSignatureAge(t *testing.T) {
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "presigned verifier value",
|
||||
target: "http://minio.local/bkt/obj?" + url.Values{
|
||||
xhttp.AmzCredential: {"access/20260803/us-east-1/s3/aws4_request"},
|
||||
}.Encode(),
|
||||
name: "presigned client header without date",
|
||||
target: presignQuery(""),
|
||||
headers: map[string]string{signatureAgeHeader: "250"},
|
||||
},
|
||||
{
|
||||
name: "presigned malformed date",
|
||||
target: presignQuery("yesterday"),
|
||||
},
|
||||
{
|
||||
name: "presigned signed date",
|
||||
target: presignQuery(signedDate.Format(iso8601Format)),
|
||||
headers: map[string]string{signatureAgeHeader: "250"},
|
||||
want: true,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := condValuesForRequest(t, tc.target, tc.headers)
|
||||
_, ok := got["signatureAge"]
|
||||
v, ok := got["signatureAge"]
|
||||
if ok != tc.want {
|
||||
t.Fatalf("signatureAge presence: expected %v, got %v", tc.want, got["signatureAge"])
|
||||
t.Fatalf("signatureAge presence: expected %v, got %v", tc.want, v)
|
||||
}
|
||||
if !tc.want {
|
||||
return
|
||||
}
|
||||
age, err := strconv.ParseInt(strings.Join(v, ""), 10, 64)
|
||||
if err != nil || age < (90*time.Second).Milliseconds() || age > (2*time.Minute).Milliseconds() {
|
||||
t.Fatalf("signatureAge = %v, want about 90s derived from X-Amz-Date rather than the client header", v)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+12
-8
@@ -43,6 +43,17 @@ func NewBucketQuotaSys() *BucketQuotaSys {
|
||||
return &BucketQuotaSys{}
|
||||
}
|
||||
|
||||
// getBucketQuotaSize returns the effective enforced hard-quota size.
|
||||
func getBucketQuotaSize(quota *madmin.BucketQuota) uint64 {
|
||||
if quota == nil || quota.Type != madmin.HardQuota {
|
||||
return 0
|
||||
}
|
||||
if quota.Size > 0 {
|
||||
return quota.Size
|
||||
}
|
||||
return quota.Quota
|
||||
}
|
||||
|
||||
var bucketStorageCache = cachevalue.New[DataUsageInfo]()
|
||||
|
||||
// Init initialize bucket quota.
|
||||
@@ -110,14 +121,7 @@ func (sys *BucketQuotaSys) enforceQuotaHard(ctx context.Context, bucket string,
|
||||
return err
|
||||
}
|
||||
|
||||
var quotaSize uint64
|
||||
if q != nil && q.Type == madmin.HardQuota {
|
||||
if q.Size > 0 {
|
||||
quotaSize = q.Size
|
||||
} else if q.Quota > 0 {
|
||||
quotaSize = q.Quota
|
||||
}
|
||||
}
|
||||
quotaSize := getBucketQuotaSize(q)
|
||||
if quotaSize > 0 {
|
||||
if uint64(size) >= quotaSize { // check if file size already exceeds the quota
|
||||
return BucketQuotaExceeded{Bucket: bucket}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
// Copyright (c) 2015-2025 MinIO, Inc.
|
||||
// Copyright (c) 2025-2026 PGSTY
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
)
|
||||
|
||||
func TestGetBucketQuotaSize(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
quota *madmin.BucketQuota
|
||||
want uint64
|
||||
}{
|
||||
{name: "nil"},
|
||||
{name: "empty", quota: &madmin.BucketQuota{}},
|
||||
{name: "current size", quota: &madmin.BucketQuota{Type: madmin.HardQuota, Size: 1024}, want: 1024},
|
||||
{name: "legacy quota", quota: &madmin.BucketQuota{Type: madmin.HardQuota, Quota: 2048}, want: 2048},
|
||||
{name: "size takes precedence", quota: &madmin.BucketQuota{Type: madmin.HardQuota, Size: 1024, Quota: 2048}, want: 1024},
|
||||
{name: "missing type", quota: &madmin.BucketQuota{Size: 1024}},
|
||||
{name: "unsupported type", quota: &madmin.BucketQuota{Type: "fifo", Size: 1024}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := getBucketQuotaSize(tt.quota); got != tt.want {
|
||||
t.Fatalf("getBucketQuotaSize() = %d, want %d", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsBktQuotaCfgReplicated(t *testing.T) {
|
||||
hardQuota := func(size, legacy uint64) *madmin.BucketQuota {
|
||||
return &madmin.BucketQuota{Type: madmin.HardQuota, Size: size, Quota: legacy}
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
quotas []*madmin.BucketQuota
|
||||
want bool
|
||||
}{
|
||||
{name: "none configured", quotas: []*madmin.BucketQuota{nil, nil}, want: true},
|
||||
{name: "missing from one site", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), nil}},
|
||||
{name: "matching size", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), hardQuota(1024, 0)}, want: true},
|
||||
{name: "different size", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), hardQuota(2048, 0)}},
|
||||
{name: "equivalent representations", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), hardQuota(0, 1024)}, want: true},
|
||||
{name: "different typeless size", quotas: []*madmin.BucketQuota{{Size: 1024}, {Size: 2048}}},
|
||||
{name: "different type", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), {Type: "fifo", Size: 1024}}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := isBktQuotaCfgReplicated(len(tt.quotas), tt.quotas); got != tt.want {
|
||||
t.Fatalf("isBktQuotaCfgReplicated() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -640,10 +640,10 @@ type VersionPurgeStatusType = replication.VersionPurgeStatusType
|
||||
|
||||
type replicationResyncer struct {
|
||||
// map of bucket to their resync status
|
||||
statusMap map[string]BucketReplicationResyncStatus
|
||||
workerSize int
|
||||
resyncCancelCh chan struct{}
|
||||
workerCh chan struct{}
|
||||
statusMap map[string]BucketReplicationResyncStatus
|
||||
workerSize int
|
||||
cancelResyncs map[resyncOpts]context.CancelCauseFunc
|
||||
workerCh chan struct{}
|
||||
sync.RWMutex
|
||||
}
|
||||
|
||||
|
||||
+143
-78
@@ -2354,9 +2354,9 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
|
||||
switch ri.OpType {
|
||||
case replication.HealReplicationType, replication.ExistingObjectReplicationType:
|
||||
ch = p.mrfReplicaCh
|
||||
healCh = p.getWorkerCh(ri.Name, ri.Bucket, ri.Size)
|
||||
healCh = p.getWorkerCh(ri.Bucket, ri.Name, ri.Size)
|
||||
default:
|
||||
ch = p.getWorkerCh(ri.Name, ri.Bucket, ri.Size)
|
||||
ch = p.getWorkerCh(ri.Bucket, ri.Name, ri.Size)
|
||||
}
|
||||
if ch == nil && healCh == nil {
|
||||
return
|
||||
@@ -2964,10 +2964,10 @@ const (
|
||||
|
||||
func newresyncer() *replicationResyncer {
|
||||
rs := replicationResyncer{
|
||||
statusMap: make(map[string]BucketReplicationResyncStatus),
|
||||
workerSize: resyncWorkerCnt,
|
||||
resyncCancelCh: make(chan struct{}, resyncWorkerCnt),
|
||||
workerCh: make(chan struct{}, resyncWorkerCnt),
|
||||
statusMap: make(map[string]BucketReplicationResyncStatus),
|
||||
workerSize: resyncWorkerCnt,
|
||||
cancelResyncs: make(map[resyncOpts]context.CancelCauseFunc),
|
||||
workerCh: make(chan struct{}, resyncWorkerCnt),
|
||||
}
|
||||
for i := 0; i < rs.workerSize; i++ {
|
||||
rs.workerCh <- struct{}{}
|
||||
@@ -2975,13 +2975,67 @@ func newresyncer() *replicationResyncer {
|
||||
return &rs
|
||||
}
|
||||
|
||||
var errResyncCanceled = errors.New("replication resync canceled")
|
||||
|
||||
// Registration and cancellation share the status lock. A queued run therefore
|
||||
// cannot miss cancellation between publishing its status and taking a slot.
|
||||
func (s *replicationResyncer) registerResync(parent context.Context, opts resyncOpts) (context.Context, context.CancelCauseFunc, bool) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
st, ok := s.statusMap[opts.bucket].TargetsMap[opts.arn]
|
||||
if !ok || st.ResyncID != opts.resyncID {
|
||||
return nil, nil, false
|
||||
}
|
||||
if _, running := s.cancelResyncs[opts]; running {
|
||||
return nil, nil, false
|
||||
}
|
||||
ctx, cancel := context.WithCancelCause(parent)
|
||||
if s.cancelResyncs == nil {
|
||||
s.cancelResyncs = make(map[resyncOpts]context.CancelCauseFunc)
|
||||
}
|
||||
s.cancelResyncs[opts] = cancel
|
||||
if st.ResyncStatus == ResyncCanceled {
|
||||
cancel(errResyncCanceled)
|
||||
}
|
||||
return ctx, cancel, true
|
||||
}
|
||||
|
||||
func (s *replicationResyncer) cancelResyncID(resyncID string) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
for bucket, m := range s.statusMap {
|
||||
for arn, st := range m.TargetsMap {
|
||||
if st.ResyncID == resyncID && (st.ResyncStatus == ResyncPending || st.ResyncStatus == ResyncStarted) {
|
||||
st.ResyncStatus = ResyncCanceled
|
||||
st.LastUpdate = UTCNow()
|
||||
m.TargetsMap[arn] = st
|
||||
m.LastUpdate = st.LastUpdate
|
||||
}
|
||||
}
|
||||
s.statusMap[bucket] = m
|
||||
}
|
||||
for opts, cancel := range s.cancelResyncs {
|
||||
if opts.resyncID == resyncID {
|
||||
cancel(errResyncCanceled)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mark status of replication resync on remote target for the bucket
|
||||
func (s *replicationResyncer) markStatus(status ResyncStatusType, opts resyncOpts, objAPI ObjectLayer) {
|
||||
func (s *replicationResyncer) markStatus(status ResyncStatusType, opts resyncOpts, objAPI ObjectLayer) ResyncStatusType {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
m := s.statusMap[opts.bucket]
|
||||
st := m.TargetsMap[opts.arn]
|
||||
st, ok := m.TargetsMap[opts.arn]
|
||||
if !ok || st.ResyncID != opts.resyncID {
|
||||
return NoResync
|
||||
}
|
||||
// A cancel may win the lock after the finalizer checked its context.
|
||||
// Persist that cancellation, never a stale Started/Completed result.
|
||||
if st.ResyncStatus == ResyncCanceled {
|
||||
status = ResyncCanceled
|
||||
}
|
||||
st.LastUpdate = UTCNow()
|
||||
st.ResyncStatus = status
|
||||
m.TargetsMap[opts.arn] = st
|
||||
@@ -2991,14 +3045,18 @@ func (s *replicationResyncer) markStatus(status ResyncStatusType, opts resyncOpt
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
saveResyncStatus(ctx, opts.bucket, m, objAPI)
|
||||
return status
|
||||
}
|
||||
|
||||
// update replication resync stats for bucket's remote target
|
||||
func (s *replicationResyncer) incStats(ts TargetReplicationResyncStatus, opts resyncOpts) {
|
||||
func (s *replicationResyncer) incStats(ts TargetReplicationResyncStatus, opts resyncOpts) bool {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
m := s.statusMap[opts.bucket]
|
||||
st := m.TargetsMap[opts.arn]
|
||||
st, ok := m.TargetsMap[opts.arn]
|
||||
if !ok || st.ResyncID != opts.resyncID || st.ResyncStatus == ResyncCanceled {
|
||||
return false
|
||||
}
|
||||
st.Object = ts.Object
|
||||
st.ReplicatedCount += ts.ReplicatedCount
|
||||
st.FailedCount += ts.FailedCount
|
||||
@@ -3007,6 +3065,7 @@ func (s *replicationResyncer) incStats(ts TargetReplicationResyncStatus, opts re
|
||||
m.TargetsMap[opts.arn] = st
|
||||
m.LastUpdate = UTCNow()
|
||||
s.statusMap[opts.bucket] = m
|
||||
return true
|
||||
}
|
||||
|
||||
// resyncResults consumes the per-object outcomes produced by the resync worker
|
||||
@@ -3023,8 +3082,9 @@ type resyncResults struct {
|
||||
// result into the bucket's resync status.
|
||||
func (s *replicationResyncer) newResyncResults(opts resyncOpts) *resyncResults {
|
||||
return startResyncResults(func(r TargetReplicationResyncStatus) {
|
||||
s.incStats(r, opts)
|
||||
globalSiteResyncMetrics.updateMetric(r, opts.resyncID)
|
||||
if s.incStats(r, opts) {
|
||||
globalSiteResyncMetrics.updateMetric(r, opts.resyncID)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -3063,29 +3123,22 @@ func (rr *resyncResults) finish(workers []chan ReplicateObjectInfo, workerWg *sy
|
||||
}
|
||||
|
||||
// sendResyncResult delivers a worker's computed per-object result to ch,
|
||||
// returning false if the worker must stop first. On the resync-cancel signal it
|
||||
// records the abort - the already-computed result is dropped - so
|
||||
// finalResyncStatus can downgrade a Completed run; on ctx cancellation it stops
|
||||
// without recording, since finalResyncStatus's parent-context check covers that.
|
||||
func (s *replicationResyncer) sendResyncResult(ctx context.Context, ch chan<- TargetReplicationResyncStatus, st TargetReplicationResyncStatus, workerAborted *atomic.Bool) bool {
|
||||
// returning false if the run's context was canceled first.
|
||||
func (s *replicationResyncer) sendResyncResult(ctx context.Context, ch chan<- TargetReplicationResyncStatus, st TargetReplicationResyncStatus) bool {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-s.resyncCancelCh:
|
||||
workerAborted.Store(true)
|
||||
return false
|
||||
case ch <- st:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// finalResyncStatus downgrades a Completed status to Failed when the run could
|
||||
// not have observed every object: the parent context was canceled (workers then
|
||||
// return without sending their computed result) or a worker dropped a result on
|
||||
// the resync-cancel signal. Without this a persisted Completed would misrepresent
|
||||
// an incomplete resync.
|
||||
func finalResyncStatus(status ResyncStatusType, ctxErr error, workerAborted bool) ResyncStatusType {
|
||||
if status == ResyncCompleted && (ctxErr != nil || workerAborted) {
|
||||
// User cancellation is terminal; interrupted completion remains retryable.
|
||||
func finalResyncStatus(status ResyncStatusType, cause error) ResyncStatusType {
|
||||
if errors.Is(cause, errResyncCanceled) {
|
||||
return ResyncCanceled
|
||||
}
|
||||
if status == ResyncCompleted && cause != nil {
|
||||
return ResyncFailed
|
||||
}
|
||||
return status
|
||||
@@ -3156,26 +3209,31 @@ func objectNeedsResyncForARN(roi ReplicateObjectInfo, arn string) bool {
|
||||
// resyncBucket resyncs all qualifying objects as per replication rules for the target
|
||||
// ARN
|
||||
func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI ObjectLayer, heal bool, opts resyncOpts) {
|
||||
ctx, cancel, registered := s.registerResync(ctx, opts)
|
||||
if !registered {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
cancel(nil)
|
||||
s.Lock()
|
||||
delete(s.cancelResyncs, opts)
|
||||
s.Unlock()
|
||||
}()
|
||||
select {
|
||||
case <-s.workerCh: // block till a worker is available
|
||||
case <-ctx.Done():
|
||||
if errors.Is(context.Cause(ctx), errResyncCanceled) {
|
||||
status := s.markStatus(ResyncCanceled, opts, objectAPI)
|
||||
globalSiteResyncMetrics.incBucket(opts, status)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
resyncStatus := ResyncFailed
|
||||
// workerAborted records that a worker dropped an already-computed result on
|
||||
// the resync-cancel signal. With a canceled parent context (which makes
|
||||
// workers return without sending their result), it means a Completed run did
|
||||
// not actually observe every object - see finalResyncStatus below.
|
||||
var workerAborted atomic.Bool
|
||||
defer func() {
|
||||
// Downgrade a Completed status whose counts are incomplete, so the
|
||||
// persisted status is not a misleading Completed. Runs after results.finish
|
||||
// drains (LIFO) and before markStatus persists - markStatus uses its own
|
||||
// background context, so a parent cancellation during the drain would
|
||||
// otherwise still record Completed.
|
||||
resyncStatus = finalResyncStatus(resyncStatus, ctx.Err(), workerAborted.Load())
|
||||
s.markStatus(resyncStatus, opts, objectAPI)
|
||||
// Runs after workers/results drain and before our own deferred cancel.
|
||||
resyncStatus = finalResyncStatus(resyncStatus, context.Cause(ctx))
|
||||
resyncStatus = s.markStatus(resyncStatus, opts, objectAPI)
|
||||
globalSiteResyncMetrics.incBucket(opts, resyncStatus)
|
||||
s.workerCh <- struct{}{}
|
||||
}()
|
||||
@@ -3210,8 +3268,8 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
return
|
||||
}
|
||||
// mark resync status as resync started
|
||||
if !heal {
|
||||
s.markStatus(ResyncStarted, opts, objectAPI)
|
||||
if !heal && s.markStatus(ResyncStarted, opts, objectAPI) != ResyncStarted {
|
||||
return
|
||||
}
|
||||
|
||||
// Walk through all object versions - Walk() is always in ascending order needed to ensure
|
||||
@@ -3237,7 +3295,14 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
// cannot race the last incStats. Registered after the markStatus finalizer, so
|
||||
// LIFO runs finish first.
|
||||
results := s.newResyncResults(opts)
|
||||
defer results.finish(workers, &wg)
|
||||
defer func() {
|
||||
if resyncStatus != ResyncCompleted {
|
||||
cancel(nil)
|
||||
}
|
||||
// Exactly one finish: success drains with a live context; errors stop
|
||||
// blocked workers first. Both drain counts before persisting status.
|
||||
results.finish(workers, &wg)
|
||||
}()
|
||||
for i := range resyncParallelRoutines {
|
||||
wg.Add(1)
|
||||
workers[i] = make(chan ReplicateObjectInfo, 100)
|
||||
@@ -3248,7 +3313,6 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-s.resyncCancelCh:
|
||||
default:
|
||||
}
|
||||
traceFn := s.trace(tgt.ResetID, fmt.Sprintf("%s/%s (%s)", opts.bucket, roi.Name, roi.VersionID))
|
||||
@@ -3295,26 +3359,29 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
}
|
||||
}
|
||||
traceFn(traceSize, traceErr)
|
||||
if !s.sendResyncResult(ctx, results.ch, st, &workerAborted) {
|
||||
if !s.sendResyncResult(ctx, results.ch, st) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}(ctx, i)
|
||||
}
|
||||
for res := range objInfoCh {
|
||||
walkLoop:
|
||||
for {
|
||||
var res itemOrErr[ObjectInfo]
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case item, ok := <-objInfoCh:
|
||||
if !ok {
|
||||
break walkLoop
|
||||
}
|
||||
res = item
|
||||
}
|
||||
if res.Err != nil {
|
||||
resyncStatus = ResyncFailed
|
||||
replLogIf(ctx, res.Err)
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-s.resyncCancelCh:
|
||||
resyncStatus = ResyncCanceled
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
if heal && lastCheckpoint != "" && lastCheckpoint != res.Item.Name {
|
||||
continue
|
||||
}
|
||||
@@ -3328,14 +3395,11 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
if !objectNeedsResyncForARN(roi, opts.arn) {
|
||||
continue
|
||||
}
|
||||
h := xxh3.HashString(roi.Bucket + roi.Name)
|
||||
select {
|
||||
case <-s.resyncCancelCh:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
h := xxh3.HashString(roi.Bucket + roi.Name)
|
||||
workers[h%uint64(resyncParallelRoutines)] <- roi
|
||||
case workers[h%uint64(resyncParallelRoutines)] <- roi:
|
||||
}
|
||||
}
|
||||
resyncStatus = ResyncCompleted
|
||||
@@ -3363,9 +3427,9 @@ func (s *replicationResyncer) start(ctx context.Context, objAPI ObjectLayer, opt
|
||||
if len(tgtArns) == 0 {
|
||||
return fmt.Errorf("arn %s specified for resync not found in replication config", opts.arn)
|
||||
}
|
||||
globalReplicationPool.Get().resyncer.RLock()
|
||||
data, ok := globalReplicationPool.Get().resyncer.statusMap[opts.bucket]
|
||||
globalReplicationPool.Get().resyncer.RUnlock()
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
data, ok := s.statusMap[opts.bucket]
|
||||
if !ok {
|
||||
data, err = loadBucketResyncMetadata(ctx, opts.bucket, objAPI)
|
||||
if err != nil {
|
||||
@@ -3386,23 +3450,14 @@ func (s *replicationResyncer) start(ctx context.Context, objAPI ObjectLayer, opt
|
||||
ResyncStatus: ResyncPending,
|
||||
Bucket: opts.bucket,
|
||||
}
|
||||
data.TargetsMap = data.cloneTgtStats()
|
||||
data.TargetsMap[opts.arn] = status
|
||||
if err = saveResyncStatus(ctx, opts.bucket, data, objAPI); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
globalReplicationPool.Get().resyncer.Lock()
|
||||
defer globalReplicationPool.Get().resyncer.Unlock()
|
||||
brs, ok := globalReplicationPool.Get().resyncer.statusMap[opts.bucket]
|
||||
if !ok {
|
||||
brs = BucketReplicationResyncStatus{
|
||||
Version: resyncMetaVersion,
|
||||
TargetsMap: make(map[string]TargetReplicationResyncStatus),
|
||||
}
|
||||
}
|
||||
brs.TargetsMap[opts.arn] = status
|
||||
globalReplicationPool.Get().resyncer.statusMap[opts.bucket] = brs
|
||||
go globalReplicationPool.Get().resyncer.resyncBucket(GlobalContext, objAPI, false, opts)
|
||||
s.statusMap[opts.bucket] = data
|
||||
go s.resyncBucket(GlobalContext, objAPI, false, opts)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -3476,6 +3531,9 @@ func (p *ReplicationPool) loadResync(ctx context.Context, buckets []string, objA
|
||||
// Make sure only one node running resync on the cluster.
|
||||
ctx, cancel := globalLeaderLock.GetLock(ctx)
|
||||
defer cancel()
|
||||
var workers sync.WaitGroup
|
||||
// Keep the merged leader context alive until every resumed run exits.
|
||||
defer workers.Wait()
|
||||
|
||||
for index := range buckets {
|
||||
bucket := buckets[index]
|
||||
@@ -3489,19 +3547,24 @@ func (p *ReplicationPool) loadResync(ctx context.Context, buckets []string, objA
|
||||
}
|
||||
|
||||
p.resyncer.Lock()
|
||||
p.resyncer.statusMap[bucket] = meta
|
||||
p.resyncer.Unlock()
|
||||
|
||||
if current, ok := p.resyncer.statusMap[bucket]; ok {
|
||||
// A concurrent start/cancel is newer than the disk snapshot.
|
||||
meta = current
|
||||
} else {
|
||||
p.resyncer.statusMap[bucket] = meta
|
||||
}
|
||||
tgts := meta.cloneTgtStats()
|
||||
p.resyncer.Unlock()
|
||||
for arn, st := range tgts {
|
||||
switch st.ResyncStatus {
|
||||
case ResyncFailed, ResyncStarted, ResyncPending:
|
||||
go p.resyncer.resyncBucket(ctx, objAPI, true, resyncOpts{
|
||||
opts := resyncOpts{
|
||||
bucket: bucket,
|
||||
arn: arn,
|
||||
resyncID: st.ResyncID,
|
||||
resyncBefore: st.ResyncBeforeDate,
|
||||
})
|
||||
}
|
||||
workers.Go(func() { p.resyncer.resyncBucket(ctx, objAPI, true, opts) })
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3820,6 +3883,7 @@ func (p *ReplicationPool) queueMRFSave(entry MRFReplicateEntry) {
|
||||
if entry.RetryCount > mrfRetryLimit { // let scanner catch up if retry count exceeded
|
||||
atomic.AddUint64(&p.stats.mrfStats.TotalDroppedCount, 1)
|
||||
atomic.AddUint64(&p.stats.mrfStats.TotalDroppedBytes, uint64(entry.sz))
|
||||
replLogOnceIf(GlobalContext, errors.New("Replication MRF retry limit reached; further repair is deferred to the scanner"), "replication-mrf-retry-limit", logger.WarningKind)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -3834,6 +3898,7 @@ func (p *ReplicationPool) queueMRFSave(entry MRFReplicateEntry) {
|
||||
default:
|
||||
atomic.AddUint64(&p.stats.mrfStats.TotalDroppedCount, 1)
|
||||
atomic.AddUint64(&p.stats.mrfStats.TotalDroppedBytes, uint64(entry.sz))
|
||||
replLogOnceIf(GlobalContext, errors.New("Replication MRF queue is full; dropped entries will need scanner repair"), "replication-mrf-queue-full", logger.WarningKind)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,6 @@ import (
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
@@ -328,20 +327,19 @@ func TestReplicationValidationObjectUsesRulePrefix(t *testing.T) {
|
||||
|
||||
func newTestResyncer(bucket, arn string) (*replicationResyncer, resyncOpts) {
|
||||
s := &replicationResyncer{
|
||||
statusMap: map[string]BucketReplicationResyncStatus{},
|
||||
resyncCancelCh: make(chan struct{}, resyncWorkerCnt),
|
||||
statusMap: map[string]BucketReplicationResyncStatus{},
|
||||
}
|
||||
brs := newBucketResyncStatus(bucket)
|
||||
brs.TargetsMap[arn] = TargetReplicationResyncStatus{ResyncStatus: ResyncStarted}
|
||||
brs.TargetsMap[arn] = TargetReplicationResyncStatus{ResyncStatus: ResyncStarted, ResyncID: "reset-" + bucket}
|
||||
s.statusMap[bucket] = brs
|
||||
return s, resyncOpts{bucket: bucket, arn: arn, resyncID: "reset-" + bucket}
|
||||
}
|
||||
|
||||
// TestResyncBucketFinalize round-trips the terminal status through a real
|
||||
// ObjectLayer: a clean run persists Completed with every result, while a run
|
||||
// whose parent context was canceled during the drain, or in which a worker
|
||||
// dropped a result on the cancel signal, is downgraded to Failed so a persisted
|
||||
// Completed never misrepresents an incomplete resync.
|
||||
// whose parent context was canceled during the drain is downgraded to Failed;
|
||||
// a user-canceled run persists Canceled. Completed never misrepresents an
|
||||
// incomplete resync.
|
||||
func TestResyncBucketFinalize(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
@@ -354,9 +352,9 @@ func TestResyncBucketFinalize(t *testing.T) {
|
||||
// persistTerminal applies resyncBucket's finalizer logic (finalResyncStatus
|
||||
// then markStatus, which persists) and reads the status back the way the
|
||||
// resync status API does.
|
||||
persistTerminal := func(t *testing.T, s *replicationResyncer, opts resyncOpts, status ResyncStatusType, ctxErr error, aborted bool) TargetReplicationResyncStatus {
|
||||
persistTerminal := func(t *testing.T, s *replicationResyncer, opts resyncOpts, status ResyncStatusType, cause error) TargetReplicationResyncStatus {
|
||||
t.Helper()
|
||||
s.markStatus(finalResyncStatus(status, ctxErr, aborted), opts, objAPI)
|
||||
s.markStatus(finalResyncStatus(status, cause), opts, objAPI)
|
||||
brs, err := loadBucketResyncMetadata(ctx, opts.bucket, objAPI)
|
||||
if err != nil {
|
||||
t.Fatalf("load persisted resync metadata: %v", err)
|
||||
@@ -376,7 +374,7 @@ func TestResyncBucketFinalize(t *testing.T) {
|
||||
var wg sync.WaitGroup // no producer workers for this case
|
||||
results.finish(nil, &wg)
|
||||
|
||||
st := persistTerminal(t, s, opts, ResyncCompleted, nil, false)
|
||||
st := persistTerminal(t, s, opts, ResyncCompleted, nil)
|
||||
if st.ResyncStatus != ResyncCompleted {
|
||||
t.Fatalf("persisted status = %s, want Completed", st.ResyncStatus)
|
||||
}
|
||||
@@ -398,29 +396,24 @@ func TestResyncBucketFinalize(t *testing.T) {
|
||||
|
||||
cctx, ccancel := context.WithCancel(context.Background())
|
||||
ccancel()
|
||||
st := persistTerminal(t, s, opts, ResyncCompleted, cctx.Err(), false)
|
||||
st := persistTerminal(t, s, opts, ResyncCompleted, context.Cause(cctx))
|
||||
if st.ResyncStatus != ResyncFailed {
|
||||
t.Fatalf("persisted status = %s, want Failed (parent canceled during drain)", st.ResyncStatus)
|
||||
}
|
||||
})
|
||||
|
||||
// 3. A worker dropped a computed result on the resync-cancel token (parent
|
||||
// still alive) -> sendResyncResult records the abort and Completed is
|
||||
// downgraded to Failed.
|
||||
t.Run("worker abort downgrades to failed", func(t *testing.T) {
|
||||
// 3. A user-canceled worker cannot report a completed resync.
|
||||
t.Run("user cancel persists canceled", func(t *testing.T) {
|
||||
s, opts := newTestResyncer("finalize-worker-abort", "arn1")
|
||||
s.resyncCancelCh <- struct{}{} // cancel token waiting
|
||||
ch := make(chan TargetReplicationResyncStatus) // no reader: the send would block
|
||||
var aborted atomic.Bool
|
||||
if s.sendResyncResult(context.Background(), ch, TargetReplicationResyncStatus{Object: "dropped", ReplicatedCount: 1}, &aborted) {
|
||||
t.Fatal("sendResyncResult reported success despite the cancel token")
|
||||
ctx, cancel := context.WithCancelCause(context.Background())
|
||||
cancel(errResyncCanceled)
|
||||
ch := make(chan TargetReplicationResyncStatus)
|
||||
if s.sendResyncResult(ctx, ch, TargetReplicationResyncStatus{Object: "dropped", ReplicatedCount: 1}) {
|
||||
t.Fatal("sendResyncResult reported success after cancellation")
|
||||
}
|
||||
if !aborted.Load() {
|
||||
t.Fatal("worker abort was not recorded")
|
||||
}
|
||||
st := persistTerminal(t, s, opts, ResyncCompleted, nil, aborted.Load())
|
||||
if st.ResyncStatus != ResyncFailed {
|
||||
t.Fatalf("persisted status = %s, want Failed (worker dropped a result)", st.ResyncStatus)
|
||||
st := persistTerminal(t, s, opts, ResyncCompleted, context.Cause(ctx))
|
||||
if st.ResyncStatus != ResyncCanceled {
|
||||
t.Fatalf("persisted status = %s, want Canceled", st.ResyncStatus)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+6
-2
@@ -319,7 +319,9 @@ func (r *ReplicationStats) getNodeQueueStats(bucket string) (qs ReplQNodeStats)
|
||||
qs.QStats = r.qCache.getBucketStats(bucket)
|
||||
qs.TgtXferStats = make(map[string]map[RMetricName]XferStats)
|
||||
qs.MRFStats = ReplicationMRFStats{
|
||||
LastFailedCount: atomic.LoadUint64(&r.mrfStats.LastFailedCount),
|
||||
LastFailedCount: atomic.LoadUint64(&r.mrfStats.LastFailedCount),
|
||||
TotalDroppedCount: atomic.LoadUint64(&r.mrfStats.TotalDroppedCount),
|
||||
TotalDroppedBytes: atomic.LoadUint64(&r.mrfStats.TotalDroppedBytes),
|
||||
}
|
||||
|
||||
r.RLock()
|
||||
@@ -410,7 +412,9 @@ func (r *ReplicationStats) getNodeQueueStatsSummary() (qs ReplQNodeStats) {
|
||||
qs.XferStats = make(map[RMetricName]XferStats)
|
||||
qs.QStats = r.qCache.getSiteStats()
|
||||
qs.MRFStats = ReplicationMRFStats{
|
||||
LastFailedCount: atomic.LoadUint64(&r.mrfStats.LastFailedCount),
|
||||
LastFailedCount: atomic.LoadUint64(&r.mrfStats.LastFailedCount),
|
||||
TotalDroppedCount: atomic.LoadUint64(&r.mrfStats.TotalDroppedCount),
|
||||
TotalDroppedBytes: atomic.LoadUint64(&r.mrfStats.TotalDroppedBytes),
|
||||
}
|
||||
r.RLock()
|
||||
defer r.RUnlock()
|
||||
|
||||
@@ -97,7 +97,7 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData)
|
||||
result, err := globalBucketMetadataSys.updateAndParseMetadata(ctx, bucket, bucketVersioningConfig, configData, false, false, nil)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -107,12 +107,12 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
|
||||
//
|
||||
// We encode the xml bytes as base64 to ensure there are no encoding
|
||||
// errors.
|
||||
cfgStr := base64.StdEncoding.EncodeToString(configData)
|
||||
cfgStr := base64.StdEncoding.EncodeToString(result.meta.VersioningConfigXML)
|
||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypeVersionConfig,
|
||||
Bucket: bucket,
|
||||
Versioning: &cfgStr,
|
||||
UpdatedAt: updatedAt,
|
||||
UpdatedAt: result.updatedAt,
|
||||
}))
|
||||
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
|
||||
+53
-7
@@ -120,15 +120,40 @@ func init() {
|
||||
|
||||
const consolePrefix = "CONSOLE_"
|
||||
|
||||
// consoleMinIOServerEnv derives the CONSOLE_MINIO_SERVER value the embedded
|
||||
// Console uses to reach the S3/STS API, and whether TLS verification of that
|
||||
// endpoint must be skipped. With no explicit endpoint configured the Console
|
||||
// reaches the API over the loopback address, whose TLS certificate is not
|
||||
// expected to carry a 127.0.0.1 SAN; because Console verifies outbound TLS by
|
||||
// default, the loopback origin has to be exempted or embedded login (local and
|
||||
// LDAP alike) fails at the STS handshake. The exemption is endpoint-scoped in
|
||||
// Console, so every other HTTPS peer stays verified. An explicitly configured
|
||||
// endpoint is always reached under its own verified name and is never exempted.
|
||||
func consoleMinIOServerEnv(endpoint string, isTLS bool, port string) (server string, skipVerify bool) {
|
||||
if endpoint != "" {
|
||||
return endpoint, false
|
||||
}
|
||||
return fmt.Sprintf("%s://127.0.0.1:%s", getURLScheme(isTLS), port), isTLS
|
||||
}
|
||||
|
||||
func minioConfigToConsoleFeatures() {
|
||||
os.Setenv("CONSOLE_PBKDF_SALT", globalDeploymentID())
|
||||
os.Setenv("CONSOLE_PBKDF_PASSPHRASE", globalDeploymentID())
|
||||
if globalMinioEndpoint != "" {
|
||||
os.Setenv("CONSOLE_MINIO_SERVER", globalMinioEndpoint)
|
||||
consoleServer, skipVerify := consoleMinIOServerEnv(globalMinioEndpoint, globalIsTLS, globalMinioPort)
|
||||
os.Setenv("CONSOLE_MINIO_SERVER", consoleServer)
|
||||
if skipVerify {
|
||||
// The embedded Console reaches the loopback S3/STS endpoint above, whose
|
||||
// certificate is not expected to carry a 127.0.0.1 SAN. Console verifies
|
||||
// outbound TLS by default (silo-console v2.3.x), so opt into the
|
||||
// endpoint-scoped compatibility switch to preserve the documented loopback
|
||||
// bypass; every other HTTPS peer (IdP, Prometheus, webhooks, ...) stays
|
||||
// verified. initConsoleServer unsets CONSOLE_* before calling this, so the
|
||||
// switch cannot be supplied by the operator on the embedded path.
|
||||
os.Setenv("CONSOLE_MINIO_SERVER_TLS_SKIP_VERIFY", "on")
|
||||
} else {
|
||||
// Explicitly set 127.0.0.1 so Console will automatically bypass TLS verification to the local S3 API.
|
||||
// This will save users from providing a certificate with IP or FQDN SAN that points to the local host.
|
||||
os.Setenv("CONSOLE_MINIO_SERVER", fmt.Sprintf("%s://127.0.0.1:%s", getURLScheme(globalIsTLS), globalMinioPort))
|
||||
// An explicitly configured endpoint is reached under its own verified name;
|
||||
// never let a loopback exemption apply to it.
|
||||
os.Unsetenv("CONSOLE_MINIO_SERVER_TLS_SKIP_VERIFY")
|
||||
}
|
||||
if value := env.Get(config.EnvMinIOLogQueryURL, ""); value != "" {
|
||||
os.Setenv("CONSOLE_LOG_QUERY_URL", value)
|
||||
@@ -232,11 +257,31 @@ func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
|
||||
return m
|
||||
}
|
||||
|
||||
func initConsoleServer() (*consoleapi.Server, error) {
|
||||
// unset all console_ environment variables.
|
||||
// resetConsoleEnvironment preserves the embedded Console's supported resource
|
||||
// settings verbatim. Server derives all other Console settings itself.
|
||||
func resetConsoleEnvironment() {
|
||||
for _, cenv := range env.List(consolePrefix) {
|
||||
switch cenv {
|
||||
case consoleapi.ConsoleWSMaxConnections,
|
||||
consoleapi.ConsoleWSMaxConnectionsPerClient,
|
||||
consoleapi.ConsoleWSMaxAnonymousConnections,
|
||||
consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient:
|
||||
continue
|
||||
}
|
||||
os.Unsetenv(cenv)
|
||||
}
|
||||
}
|
||||
|
||||
func initConsoleServer() (*consoleapi.Server, error) {
|
||||
resetConsoleEnvironment()
|
||||
// Validate explicitly: ConfigureAPI logs errors, but embedded Console logs
|
||||
// are normally silenced. Return configuration failures to Server startup.
|
||||
if err := consoleapi.ConfigureEmbeddedSourceIPTrust(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := consoleapi.ConfigureWebSocketLimits(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// enable all console environment variables
|
||||
minioConfigToConsoleFeatures()
|
||||
@@ -855,6 +900,7 @@ func serverHandleEnvVars() {
|
||||
}
|
||||
|
||||
globalEnableSyncBoot = env.Get("MINIO_SYNC_BOOT", config.EnableOff) == config.EnableOn
|
||||
globalSiteReplicationMetadataTombstones = env.Get("MINIO_SITE_REPLICATION_METADATA_TOMBSTONES", config.EnableOff) == config.EnableOn
|
||||
}
|
||||
|
||||
func loadRootCredentials() auth.Credentials {
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
consoleapi "github.com/minio/console/api"
|
||||
"github.com/minio/minio/internal/config"
|
||||
)
|
||||
|
||||
@@ -372,3 +373,138 @@ func TestConfigEnvFileNamedTargetDiscovery(t *testing.T) {
|
||||
t.Fatalf("named target %q not discovered from %s: %v", "my-hook", key, targets)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConsoleMinIOServerEnv locks in the loopback TLS exemption that keeps
|
||||
// embedded Console login working (issue #108) while ensuring an explicitly
|
||||
// configured endpoint is never silently exempted from TLS verification.
|
||||
func TestConsoleMinIOServerEnv(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
isTLS bool
|
||||
port string
|
||||
wantServer string
|
||||
wantSkipVerify bool
|
||||
}{
|
||||
{
|
||||
name: "loopback TLS is exempted so embedded login works",
|
||||
isTLS: true,
|
||||
port: "9000",
|
||||
wantServer: "https://127.0.0.1:9000",
|
||||
wantSkipVerify: true,
|
||||
},
|
||||
{
|
||||
name: "loopback plain HTTP needs no exemption",
|
||||
isTLS: false,
|
||||
port: "9000",
|
||||
wantServer: "http://127.0.0.1:9000",
|
||||
},
|
||||
{
|
||||
name: "explicit https endpoint stays verified",
|
||||
endpoint: "https://silo.example:9000",
|
||||
isTLS: true,
|
||||
port: "9000",
|
||||
wantServer: "https://silo.example:9000",
|
||||
},
|
||||
{
|
||||
name: "explicit http endpoint stays verified",
|
||||
endpoint: "http://silo.example:9000",
|
||||
isTLS: false,
|
||||
port: "9000",
|
||||
wantServer: "http://silo.example:9000",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
server, skipVerify := consoleMinIOServerEnv(tt.endpoint, tt.isTLS, tt.port)
|
||||
if server != tt.wantServer {
|
||||
t.Fatalf("server = %q, want %q", server, tt.wantServer)
|
||||
}
|
||||
if skipVerify != tt.wantSkipVerify {
|
||||
t.Fatalf("skipVerify = %v, want %v", skipVerify, tt.wantSkipVerify)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The startup path clears process environment, so preserve all existing Console
|
||||
// variables, including ones unrelated to this test, before exercising it.
|
||||
func preserveConsoleEnvironment(t *testing.T) {
|
||||
t.Helper()
|
||||
for _, entry := range os.Environ() {
|
||||
if strings.HasPrefix(entry, consolePrefix) {
|
||||
name, value, _ := strings.Cut(entry, "=")
|
||||
t.Setenv(name, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetConsoleEnvironment(t *testing.T) {
|
||||
preserveConsoleEnvironment(t)
|
||||
settings := map[string]string{
|
||||
consoleapi.ConsoleWSMaxConnections: "2048",
|
||||
consoleapi.ConsoleWSMaxConnectionsPerClient: "512",
|
||||
consoleapi.ConsoleWSMaxAnonymousConnections: "128",
|
||||
consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient: "16",
|
||||
}
|
||||
for key, value := range settings {
|
||||
t.Setenv(key, value)
|
||||
}
|
||||
decoys := []string{"CONSOLE_MINIO_SERVER_TLS_SKIP_VERIFY", "CONSOLE_MINIO_SERVER", "CONSOLE_PBKDF_SALT", "CONSOLE_TRUSTED_PROXIES", "CONSOLE_WS_MAX_UNKNOWN"}
|
||||
for _, key := range decoys {
|
||||
t.Setenv(key, "operator-value")
|
||||
}
|
||||
resetConsoleEnvironment()
|
||||
for key, want := range settings {
|
||||
if got, present := os.LookupEnv(key); !present || got != want {
|
||||
t.Errorf("%s = %q, present = %v; want %q", key, got, present, want)
|
||||
}
|
||||
}
|
||||
for _, key := range decoys {
|
||||
if _, present := os.LookupEnv(key); present {
|
||||
t.Errorf("unsupported override %s survived", key)
|
||||
}
|
||||
}
|
||||
for _, raw := range []string{"", " 16 ", "env://missing-limit"} {
|
||||
t.Setenv(consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient, raw)
|
||||
resetConsoleEnvironment()
|
||||
if got, present := os.LookupEnv(consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient); !present || got != raw {
|
||||
t.Fatalf("raw value %q was changed to %q (present = %v)", raw, got, present)
|
||||
}
|
||||
}
|
||||
os.Unsetenv(consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient)
|
||||
resetConsoleEnvironment()
|
||||
if _, present := os.LookupEnv(consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient); present {
|
||||
t.Fatal("unset setting became present")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitConsoleServerConfigurationErrors(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name, proxy, limit, want string
|
||||
}{
|
||||
{"proxy error precedes limit error", "proxy.internal", "bad", "MINIO_API_TRUSTED_PROXIES"},
|
||||
{"blank limit", "", "", "CONSOLE_WS_MAX_ANONYMOUS_CONNECTIONS_PER_CLIENT"},
|
||||
{"non-integer limit", "", "bad", "CONSOLE_WS_MAX_ANONYMOUS_CONNECTIONS_PER_CLIENT"},
|
||||
{"out-of-range limit", "", "0", "CONSOLE_WS_MAX_ANONYMOUS_CONNECTIONS_PER_CLIENT"},
|
||||
{"inconsistent limits", "", "256", "must be less than"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Restore the process-wide library configuration after environment cleanup.
|
||||
t.Cleanup(func() {
|
||||
_ = consoleapi.ConfigureEmbeddedSourceIPTrust()
|
||||
_ = consoleapi.ConfigureWebSocketLimits()
|
||||
})
|
||||
preserveConsoleEnvironment(t)
|
||||
t.Setenv(consoleapi.EnvMinIOTrustedProxies, tt.proxy)
|
||||
t.Setenv(consoleapi.ConsoleWSMaxConnections, "1024")
|
||||
t.Setenv(consoleapi.ConsoleWSMaxConnectionsPerClient, "256")
|
||||
t.Setenv(consoleapi.ConsoleWSMaxAnonymousConnections, "64")
|
||||
t.Setenv(consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient, tt.limit)
|
||||
server, err := initConsoleServer()
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) || server != nil {
|
||||
t.Fatalf("initConsoleServer() = %v, %v; want nil server and %q error", server, err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Read a real pre-removal scanner cache with nonzero hot-tier bytes. A v8
|
||||
// payload with a relabeled header would not exercise the ignored hts field.
|
||||
func TestDataUsageCacheReadV9(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/data-usage-v9/data-usage-v9.bin")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(raw) == 0 || raw[0] != 9 {
|
||||
t.Fatal("fixture is not v9")
|
||||
}
|
||||
expected, err := os.ReadFile("testdata/data-usage-v9/data-usage-v9.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var want, got dataUsageCache
|
||||
if err := json.Unmarshal(expected, &want); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := got.deserialize(bytes.NewReader(raw)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !got.Info.LastUpdate.Equal(want.Info.LastUpdate) {
|
||||
t.Fatal("cache timestamp changed")
|
||||
}
|
||||
// msgp restores local time while JSON preserves the UTC representation.
|
||||
want.Info.LastUpdate = got.Info.LastUpdate
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("v9 ordinary cache fields changed:\ngot: %#v\nwant: %#v", got, want)
|
||||
}
|
||||
flat := got.flatten(*got.root())
|
||||
if flat.Size != 74962 || flat.Objects != 3 || flat.Versions != 5 || flat.DeleteMarkers != 2 {
|
||||
t.Fatalf("statistics changed: %+v", flat)
|
||||
}
|
||||
if flat.AllTierStats == nil || flat.AllTierStats.Tiers["COLD"] != (tierStats{TotalSize: 65536, NumVersions: 1, NumObjects: 1}) {
|
||||
t.Fatalf("remote tier statistics changed: %+v", flat.AllTierStats)
|
||||
}
|
||||
buckets := []BucketInfo{{Name: "v9-bucket"}}
|
||||
if gotInfo, wantInfo := got.dui(dataUsageRoot, buckets), want.dui(dataUsageRoot, buckets); !reflect.DeepEqual(gotInfo, wantInfo) {
|
||||
t.Fatalf("bucket usage aggregation changed: %+v != %+v", gotInfo, wantInfo)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := got.serializeTo(&buf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if buf.Bytes()[0] != 8 {
|
||||
t.Fatalf("wrote cache version %d, want 8", buf.Bytes()[0])
|
||||
}
|
||||
var roundtrip dataUsageCache
|
||||
if err := roundtrip.deserialize(&buf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, roundtrip) {
|
||||
t.Fatal("v8 round trip lost ordinary fields")
|
||||
}
|
||||
if err := roundtrip.deserialize(bytes.NewReader(raw[:len(raw)/2])); err == nil {
|
||||
t.Fatal("truncated v9 cache accepted")
|
||||
}
|
||||
raw[0] = 10
|
||||
if err := roundtrip.deserialize(bytes.NewReader(raw)); err == nil {
|
||||
t.Fatal("unknown cache version accepted")
|
||||
}
|
||||
}
|
||||
@@ -981,6 +981,7 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
|
||||
// and write new data with the new version.
|
||||
const (
|
||||
dataUsageCacheVerCurrent = 8
|
||||
dataUsageCacheVerV9 = 9 // Retired access-tier cache; only adds the ignored "hts" entry key.
|
||||
dataUsageCacheVerV7 = 7
|
||||
dataUsageCacheVerV6 = 6
|
||||
dataUsageCacheVerV5 = 5
|
||||
@@ -1182,7 +1183,7 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
|
||||
}
|
||||
|
||||
return nil
|
||||
case dataUsageCacheVerCurrent:
|
||||
case dataUsageCacheVerCurrent, dataUsageCacheVerV9:
|
||||
// Zstd compressed.
|
||||
dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
|
||||
if err != nil {
|
||||
|
||||
@@ -1068,6 +1068,11 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
|
||||
newReqInfo = logger.NewReqInfo("", "", globalDeploymentID(), "", "Heal", bucket, object)
|
||||
}
|
||||
healCtx := logger.SetReqInfo(GlobalContext, newReqInfo)
|
||||
if opts.NoLock {
|
||||
// The caller owns the namespace lock. Stop if that lock's context is
|
||||
// canceled instead of continuing metadata writes after losing it.
|
||||
healCtx = logger.SetReqInfo(ctx, newReqInfo)
|
||||
}
|
||||
|
||||
// Healing directories handle it separately.
|
||||
if HasSuffix(object, SlashSeparator) {
|
||||
|
||||
@@ -157,6 +157,13 @@ func copyPartWithoutChecksumHTTP(t *testing.T, apiRouter http.Handler, creds aut
|
||||
if sourceRange != "" {
|
||||
req.Header.Set(xhttp.AmzCopySourceRange, sourceRange)
|
||||
}
|
||||
// Re-sign so the copy-source x-amz-* headers are covered by the signature,
|
||||
// as real S3 clients send them; the verifier rejects unsigned x-amz-*.
|
||||
if creds.AccessKey != "" && creds.SecretKey != "" {
|
||||
if err := signRequestV4(req, creds.AccessKey, creds.SecretKey); err != nil {
|
||||
t.Fatalf("failed to re-sign UploadPartCopy request: %v", err)
|
||||
}
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
|
||||
+10
-12
@@ -1155,17 +1155,10 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
return oi, toObjectErr(err, bucket, object, uploadID)
|
||||
}
|
||||
|
||||
// A trusted SSE-C replica completion re-orders the Object Lock carried in the
|
||||
// upload metadata against the version it is about to replace, read on this
|
||||
// erasure set under the write lock held above, so a hold or retention that
|
||||
// reached the version after this upload was initiated is not rolled back at
|
||||
// completion (issue #120). Scoped to SSE-C uploads, the only ones this issue
|
||||
// routes through completion.
|
||||
//
|
||||
// Scope: correct for a single erasure set. A multi-pool deployment (duplicate
|
||||
// versions across pools, ModTime ties, cross-pool lock authority) is out of
|
||||
// scope and tracked in pgsty/silo#133.
|
||||
if opts.ReplicaLockReconcile && crypto.SSEC.IsEncrypted(fi.Metadata) {
|
||||
// Reconcile against the upload's persisted version, under the object lock.
|
||||
// Multi-pool callers supply a resolver spanning all pools, including those
|
||||
// draining their contents; the upload itself stays in its original pool.
|
||||
if opts.ReplicaLockReconcile {
|
||||
// A persisted upload records the null version as an empty VersionID; look
|
||||
// it up as the null version so the reconcile reads the addressed version's
|
||||
// stored lock, not the latest version's.
|
||||
@@ -1173,7 +1166,11 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
if lookupVersionID == "" {
|
||||
lookupVersionID = nullVersionID
|
||||
}
|
||||
curr, gerr := er.getObjectInfo(ctx, bucket, object, ObjectOptions{
|
||||
getObjectInfo := er.getObjectInfo
|
||||
if opts.replicaObjectInfo != nil {
|
||||
getObjectInfo = opts.replicaObjectInfo
|
||||
}
|
||||
curr, gerr := getObjectInfo(ctx, bucket, object, ObjectOptions{
|
||||
VersionID: lookupVersionID,
|
||||
Versioned: opts.Versioned,
|
||||
VersionSuspended: opts.VersionSuspended,
|
||||
@@ -1182,6 +1179,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
switch {
|
||||
case gerr == nil:
|
||||
reconcileStoredObjectLock(fi.Metadata, storedObjectLockState(curr.UserDefined))
|
||||
reconcileStoredObjectTags(fi.Metadata, curr.UserTags, curr.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp])
|
||||
case isErrVersionNotFound(gerr) || isErrObjectNotFound(gerr):
|
||||
// No existing version to order against: keep the upload's own accepted
|
||||
// lock, including a pre-upgrade upload that persisted values without
|
||||
|
||||
+14
-10
@@ -133,6 +133,11 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
|
||||
return fi.ToObjectInfo(srcBucket, srcObject, srcOpts.Versioned || srcOpts.VersionSuspended), toObjectErr(errMethodNotAllowed, srcBucket, srcObject)
|
||||
}
|
||||
|
||||
if dstOpts.ReplicaLockReconcile {
|
||||
reconcileStoredObjectLock(srcInfo.UserDefined, storedObjectLockState(fi.Metadata))
|
||||
reconcileStoredObjectTags(srcInfo.UserDefined, fi.Metadata[xhttp.AmzObjectTagging], fi.Metadata[ReservedMetadataPrefixLower+TaggingTimestamp])
|
||||
}
|
||||
|
||||
filterOnlineDisksInplace(fi, metaArr, onlineDisks)
|
||||
|
||||
versionID := srcInfo.VersionID
|
||||
@@ -1280,7 +1285,11 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
opts.NoLock = true
|
||||
}
|
||||
|
||||
obj, err := er.getObjectInfo(ctx, bucket, object, opts)
|
||||
getObjectInfo := er.getObjectInfo
|
||||
if opts.replicaObjectInfo != nil {
|
||||
getObjectInfo = opts.replicaObjectInfo
|
||||
}
|
||||
obj, err := getObjectInfo(ctx, bucket, object, opts)
|
||||
// A destination read that fails for a reason other than not-found must not
|
||||
// be taken as a passed precondition or as absent lock state.
|
||||
if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
@@ -1297,17 +1306,12 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
}
|
||||
}
|
||||
|
||||
// Order this trusted SSE-C replica's Object Lock against the addressed
|
||||
// version's stored state, read on this erasure set under the write lock,
|
||||
// so a value that lost the ordering cannot overwrite a newer one committed
|
||||
// after the handler decided (issue #120). Only reconcile against an
|
||||
// existing version; on not-found the write's own accepted lock is kept.
|
||||
//
|
||||
// Scope: correct for a single erasure set. A multi-pool deployment
|
||||
// (duplicate versions across pools, ModTime ties, cross-pool lock
|
||||
// authority) is out of scope and tracked in pgsty/silo#133.
|
||||
// Re-read under the write lock, using the pools-layer resolver when
|
||||
// present. A missing version keeps the incoming accepted state; an
|
||||
// existing version contributes independently ordered lock and tags.
|
||||
if opts.ReplicaLockReconcile && err == nil {
|
||||
reconcileStoredObjectLock(opts.UserDefined, storedObjectLockState(obj.UserDefined))
|
||||
reconcileStoredObjectTags(opts.UserDefined, obj.UserTags, obj.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp])
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,377 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"maps"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
madmin "github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
// objectPoolInfos reads the addressed version in every pool, including pools
|
||||
// draining their contents. The caller must hold the pools-layer object lock.
|
||||
// An unreadable pool may contain a newer version or metadata; it is not absence.
|
||||
func (z *erasureServerPools) objectPoolInfos(ctx context.Context, bucket, object string, opts ObjectOptions) ([]PoolObjInfo, error) {
|
||||
opts.NoLock = true
|
||||
opts.CheckPrecondFn = nil
|
||||
var copies []PoolObjInfo
|
||||
for i, pool := range z.serverPools {
|
||||
oi, err := pool.GetObjectInfo(ctx, bucket, object, opts)
|
||||
if err == nil || (oi.DeleteMarker && (isErrObjectNotFound(err) || isErrMethodNotAllowed(err))) {
|
||||
copies = append(copies, PoolObjInfo{Index: i, ObjInfo: oi})
|
||||
continue
|
||||
}
|
||||
if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
sort.Slice(copies, func(i, j int) bool {
|
||||
a, b := copies[i], copies[j]
|
||||
if a.ObjInfo.ModTime.Equal(b.ObjInfo.ModTime) {
|
||||
return a.Index < b.Index
|
||||
}
|
||||
return a.ObjInfo.ModTime.After(b.ObjInfo.ModTime)
|
||||
})
|
||||
if len(copies) == 0 {
|
||||
if opts.VersionID != "" {
|
||||
return nil, VersionNotFound{Bucket: bucket, Object: decodeDirObject(object), VersionID: opts.VersionID}
|
||||
}
|
||||
return nil, ObjectNotFound{Bucket: bucket, Object: decodeDirObject(object)}
|
||||
}
|
||||
return copies, nil
|
||||
}
|
||||
|
||||
// Healing also commits object metadata. Both queued healing and drive healing
|
||||
// must use the same namespace as pooled writes, rather than racing them under
|
||||
// a destination set's independent namespace.
|
||||
func (z *erasureServerPools) healObjectInPool(ctx context.Context, er *erasureObjects, bucket, object, versionID string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
|
||||
if !z.SinglePool() && !opts.NoLock {
|
||||
lk := z.NewNSLock(bucket, object)
|
||||
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
return madmin.HealResultItem{}, err
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
opts.NoLock = true
|
||||
}
|
||||
return er.HealObject(ctx, bucket, object, versionID, opts)
|
||||
}
|
||||
|
||||
// mergePoolLockState resolves each independently ordered field. An ordered
|
||||
// removal is a value in its own right; an absent unordered field is not one.
|
||||
func mergePoolLockState(copies []PoolObjInfo) objectLockState {
|
||||
state := storedObjectLockState(copies[0].ObjInfo.UserDefined)
|
||||
for _, copy := range copies[1:] {
|
||||
next := storedObjectLockState(copy.ObjInfo.UserDefined)
|
||||
retentionTime, _ := time.Parse(time.RFC3339Nano, next.retentionTimestamp)
|
||||
if state.retentionIsOlderThan(retentionTime) || (state.retentionTimestamp == "" && state.mode == "" && next.mode != "") {
|
||||
state.mode, state.retainUntil, state.retentionTimestamp = next.mode, next.retainUntil, next.retentionTimestamp
|
||||
}
|
||||
holdTime, _ := time.Parse(time.RFC3339Nano, next.legalHoldTimestamp)
|
||||
if state.legalHoldIsOlderThan(holdTime) || (state.legalHoldTimestamp == "" && state.legalHold == "" && next.legalHold != "") {
|
||||
state.legalHold, state.legalHoldTimestamp = next.legalHold, next.legalHoldTimestamp
|
||||
}
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
func replaceObjectLockMetadata(metadata map[string]string, state objectLockState) {
|
||||
for _, key := range []string{
|
||||
strings.ToLower(xhttp.AmzObjectLockMode), strings.ToLower(xhttp.AmzObjectLockRetainUntilDate),
|
||||
strings.ToLower(xhttp.AmzObjectLockLegalHold),
|
||||
ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp,
|
||||
ReservedMetadataPrefixLower + ObjectLockLegalHoldTimestamp,
|
||||
} {
|
||||
// Metadata updates use maps.Copy at the set layer. Empty values also
|
||||
// clear a previous value there, including timestamp-only removals.
|
||||
if _, exists := metadata[key]; exists {
|
||||
metadata[key] = ""
|
||||
}
|
||||
}
|
||||
state.restoreRetention(metadata)
|
||||
state.restoreLegalHold(metadata)
|
||||
}
|
||||
|
||||
func mergedPoolObjectInfo(copies []PoolObjInfo) ObjectInfo {
|
||||
oi := copies[0].ObjInfo
|
||||
oi.UserDefined = maps.Clone(oi.UserDefined)
|
||||
if oi.UserDefined == nil {
|
||||
oi.UserDefined = make(map[string]string)
|
||||
}
|
||||
replaceObjectLockMetadata(oi.UserDefined, mergePoolLockState(copies))
|
||||
for _, copy := range copies[1:] {
|
||||
ts := copy.ObjInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]
|
||||
stamp, _ := time.Parse(time.RFC3339Nano, ts)
|
||||
if olderThan(oi.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp], stamp) {
|
||||
oi.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = ts
|
||||
oi.UserTags = copy.ObjInfo.UserTags
|
||||
}
|
||||
}
|
||||
return oi
|
||||
}
|
||||
|
||||
func (z *erasureServerPools) replicaObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
|
||||
if opts.VersionID == "" {
|
||||
opts.VersionID = nullVersionID
|
||||
}
|
||||
copies, err := z.objectPoolInfos(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
if copies[0].ObjInfo.DeleteMarker {
|
||||
return copies[0].ObjInfo, MethodNotAllowed{Bucket: bucket, Object: decodeDirObject(object), VersionID: opts.VersionID}
|
||||
}
|
||||
return mergedPoolObjectInfo(copies), nil
|
||||
}
|
||||
|
||||
// metadataPoolInfos resolves an unqualified metadata request to one logical
|
||||
// version before collecting its copies, rather than mixing different versions.
|
||||
func (z *erasureServerPools) metadataPoolInfos(ctx context.Context, bucket, object string, opts ObjectOptions) ([]PoolObjInfo, error) {
|
||||
copies, err := z.objectPoolInfos(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if copies[0].ObjInfo.DeleteMarker {
|
||||
return nil, MethodNotAllowed{Bucket: bucket, Object: decodeDirObject(object), VersionID: opts.VersionID}
|
||||
}
|
||||
if opts.VersionID == "" {
|
||||
opts.VersionID = copies[0].ObjInfo.VersionID
|
||||
if opts.VersionID == "" {
|
||||
opts.VersionID = nullVersionID
|
||||
}
|
||||
return z.objectPoolInfos(ctx, bucket, object, opts)
|
||||
}
|
||||
return copies, nil
|
||||
}
|
||||
|
||||
func (z *erasureServerPools) updatePoolMetadata(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
|
||||
copies, err := z.metadataPoolInfos(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
updated := mergedPoolObjectInfo(copies)
|
||||
before := maps.Clone(updated.UserDefined)
|
||||
if opts.EvalMetadataFn != nil {
|
||||
if _, err := opts.EvalMetadataFn(&updated, nil); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
}
|
||||
changes := make(map[string]string)
|
||||
for key, value := range updated.UserDefined {
|
||||
if prior, ok := before[key]; !ok || prior != value {
|
||||
changes[key] = value
|
||||
}
|
||||
}
|
||||
for key := range before {
|
||||
if _, ok := updated.UserDefined[key]; !ok {
|
||||
changes[key] = ""
|
||||
}
|
||||
}
|
||||
// Metadata callbacks may explicitly replace tags in the raw write map.
|
||||
// Otherwise retain the merged value from the ObjectInfo read model.
|
||||
tags, ok := updated.UserDefined[xhttp.AmzObjectTagging]
|
||||
if !ok {
|
||||
tags = updated.UserTags
|
||||
}
|
||||
state := storedObjectLockState(updated.UserDefined)
|
||||
opts.VersionID = updated.VersionID
|
||||
if opts.VersionID == "" {
|
||||
opts.VersionID = nullVersionID
|
||||
}
|
||||
opts.NoLock = true
|
||||
opts.EvalMetadataFn = func(oi *ObjectInfo, _ error) (ReplicateDecision, error) {
|
||||
maps.Copy(oi.UserDefined, changes)
|
||||
replaceObjectLockMetadata(oi.UserDefined, state)
|
||||
// Reassemble the tag value and its ordering timestamp for storage.
|
||||
if tags != "" || oi.UserTags != "" {
|
||||
oi.UserDefined[xhttp.AmzObjectTagging] = tags
|
||||
}
|
||||
key := ReservedMetadataPrefixLower + TaggingTimestamp
|
||||
if value, exists := updated.UserDefined[key]; exists || oi.UserDefined[key] != "" {
|
||||
oi.UserDefined[key] = value
|
||||
}
|
||||
return ReplicateDecision{}, nil
|
||||
}
|
||||
var primary ObjectInfo
|
||||
for _, copy := range copies {
|
||||
oi, err := z.serverPools[copy.Index].PutObjectMetadata(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
if copy.Index == copies[0].Index {
|
||||
primary = oi
|
||||
}
|
||||
}
|
||||
return primary, nil
|
||||
}
|
||||
|
||||
// Pass the stored tag value explicitly: ObjectInfo.UserDefined excludes it,
|
||||
// whereas FileInfo.Metadata retains the raw storage key.
|
||||
func reconcileStoredObjectTags(metadata map[string]string, storedTags, storedTimestamp string) {
|
||||
key := ReservedMetadataPrefixLower + TaggingTimestamp
|
||||
stamp, err := time.Parse(time.RFC3339Nano, storedTimestamp)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
incoming, err := time.Parse(time.RFC3339Nano, metadata[key])
|
||||
if err != nil || !stamp.Before(incoming) {
|
||||
metadata[key] = storedTimestamp
|
||||
metadata[xhttp.AmzObjectTagging] = storedTags
|
||||
}
|
||||
}
|
||||
|
||||
// A restored version still owns its tier reference even while IsRemote is
|
||||
// false. Only the last copy of a reference may schedule its contents for GC.
|
||||
func sharesTierObject(oi ObjectInfo, copies []PoolObjInfo) bool {
|
||||
ref := oi.TransitionedObject
|
||||
if ref.Status != lifecycle.TransitionComplete {
|
||||
return false
|
||||
}
|
||||
for _, copy := range copies {
|
||||
other := copy.ObjInfo.TransitionedObject
|
||||
if other.Status == lifecycle.TransitionComplete && ref.Tier == other.Tier && ref.Name == other.Name && ref.VersionID == other.VersionID {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// retireReplicaCopies runs only after committing a replacement. Failures are
|
||||
// returned to the caller, so a stale copy cannot be hidden behind a successful
|
||||
// response. Data movement owns its source cleanup and does not use this helper.
|
||||
func (z *erasureServerPools) retireReplicaCopies(ctx context.Context, bucket, object string, keep int, oi ObjectInfo) error {
|
||||
versionID := oi.VersionID
|
||||
if versionID == "" {
|
||||
versionID = nullVersionID
|
||||
}
|
||||
copies, err := z.objectPoolInfos(ctx, bucket, object, ObjectOptions{VersionID: versionID})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var retained []PoolObjInfo
|
||||
for _, copy := range copies {
|
||||
if copy.Index == keep {
|
||||
retained = []PoolObjInfo{copy}
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(retained) == 0 {
|
||||
return VersionNotFound{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID}
|
||||
}
|
||||
for i, copy := range copies {
|
||||
if copy.Index == keep {
|
||||
continue
|
||||
}
|
||||
_, err := z.serverPools[copy.Index].DeleteObject(ctx, bucket, object,
|
||||
ObjectOptions{
|
||||
VersionID: versionID, NoLock: true, NoAuditLog: true,
|
||||
SkipFreeVersion: sharesTierObject(copy.ObjInfo, retained) || sharesTierObject(copy.ObjInfo, copies[i+1:]),
|
||||
})
|
||||
if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteObjectReconciled evaluates any condition once against the logical
|
||||
// version, then removes all its copies under the same lock as pooled writers.
|
||||
func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
|
||||
copies, err := z.objectPoolInfos(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
primary := copies[0]
|
||||
if opts.CheckPrecondFn != nil && opts.CheckPrecondFn(primary.ObjInfo) {
|
||||
return ObjectInfo{}, PreConditionFailed{}
|
||||
}
|
||||
opts.CheckPrecondFn = nil
|
||||
opts.NoLock = true
|
||||
if opts.EvalRetentionBypassFn != nil || opts.EvalMetadataFn != nil {
|
||||
logical := primary.ObjInfo
|
||||
var gerr error
|
||||
switch {
|
||||
case logical.DeleteMarker:
|
||||
// Markers can be deleted by version ID. Match the set layer's
|
||||
// callback inputs instead of rejecting them as metadata updates.
|
||||
gerr = toObjectErr(errMethodNotAllowed, bucket, object)
|
||||
if opts.VersionID == "" || opts.DeleteMarker {
|
||||
gerr = toObjectErr(errFileNotFound, bucket, object)
|
||||
}
|
||||
case opts.VersionID != "":
|
||||
// An addressed version already resolved every copy above.
|
||||
logical = mergedPoolObjectInfo(copies)
|
||||
default:
|
||||
versions, err := z.metadataPoolInfos(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
logical = mergedPoolObjectInfo(versions)
|
||||
}
|
||||
// Keep the retention gate first. These callbacks independently evaluate
|
||||
// the logical version and run once before any deletion; the handler only
|
||||
// sweeps metadata's transition state after a successful delete.
|
||||
if opts.EvalRetentionBypassFn != nil {
|
||||
if err := opts.EvalRetentionBypassFn(logical, gerr); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
opts.EvalRetentionBypassFn = nil
|
||||
}
|
||||
if opts.EvalMetadataFn != nil {
|
||||
decision, err := opts.EvalMetadataFn(&logical, gerr)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
if decision.ReplicateAny() {
|
||||
opts.SetDeleteReplicationState(decision, opts.VersionID)
|
||||
}
|
||||
opts.EvalMetadataFn = nil
|
||||
}
|
||||
}
|
||||
if opts.VersionID == "" && (opts.Versioned || opts.VersionSuspended) {
|
||||
// A single new delete marker hides the current version. Older versions
|
||||
// remain history and must not be removed by an unqualified DELETE.
|
||||
oi, err := z.serverPools[primary.Index].DeleteObject(ctx, bucket, object, opts)
|
||||
oi.Name = decodeDirObject(object)
|
||||
oi.replicationDecision = opts.DeleteReplication.ReplicateDecisionStr
|
||||
return oi, err
|
||||
}
|
||||
// Retire non-authoritative copies first. If cleanup fails, retain the
|
||||
// authoritative version and report the error instead of acknowledging a
|
||||
// deletion that would expose an older copy.
|
||||
for i := 1; i < len(copies); i++ {
|
||||
candidate := copies[i]
|
||||
deleteOpts := opts
|
||||
deleteOpts.SkipFreeVersion = opts.SkipFreeVersion || sharesTierObject(candidate.ObjInfo, copies[:1]) || sharesTierObject(candidate.ObjInfo, copies[i+1:])
|
||||
_, err := z.serverPools[candidate.Index].DeleteObject(ctx, bucket, object, deleteOpts)
|
||||
if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
}
|
||||
oi, err := z.serverPools[primary.Index].DeleteObject(ctx, bucket, object, opts)
|
||||
oi.Name = decodeDirObject(object)
|
||||
oi.replicationDecision = opts.DeleteReplication.ReplicateDecisionStr
|
||||
return oi, err
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -23,6 +23,10 @@ import (
|
||||
)
|
||||
|
||||
func prepareErasurePools() (ObjectLayer, []string, error) {
|
||||
return prepareErasurePoolsWithContext(context.Background())
|
||||
}
|
||||
|
||||
func prepareErasurePoolsWithContext(ctx context.Context) (ObjectLayer, []string, error) {
|
||||
nDisks := 32
|
||||
fsDirs, err := getRandomDisks(nDisks)
|
||||
if err != nil {
|
||||
@@ -32,7 +36,7 @@ func prepareErasurePools() (ObjectLayer, []string, error) {
|
||||
pools := mustGetPoolEndpoints(0, fsDirs[:16]...)
|
||||
pools = append(pools, mustGetPoolEndpoints(1, fsDirs[16:]...)...)
|
||||
|
||||
objLayer, _, err := initObjectLayer(context.Background(), pools)
|
||||
objLayer, _, err := initObjectLayer(ctx, pools)
|
||||
if err != nil {
|
||||
removeRoots(fsDirs)
|
||||
return nil, nil, err
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A multipart completion's If-Match must be evaluated against the logical
|
||||
// latest object across pools, not against the copy local to a pool.
|
||||
//
|
||||
// Multi-pool write placement is not sticky (getPoolIdx picks by available
|
||||
// space even for existing objects), so an upload and a newer overwrite of
|
||||
// the same name routinely end up in different pools. Uploads are pinned to
|
||||
// their pools directly: routing through z.NewMultipartUpload would make the
|
||||
// placement depend on the space-weighted random choice.
|
||||
func TestPoolsMultipartConditionalUsesLogicalLatest(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
ctx := t.Context()
|
||||
|
||||
ifMatch := func(etag string) (opts ObjectOptions) {
|
||||
return ObjectOptions{
|
||||
HasIfMatch: true,
|
||||
CheckPrecondFn: func(oi ObjectInfo) bool {
|
||||
return oi.ETag != etag
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
uploadPart := func(t *testing.T, bucket, object, uploadID string) []CompletePart {
|
||||
t.Helper()
|
||||
pi, err := z.PutObjectPart(ctx, bucket, object, uploadID, 1,
|
||||
mustGetPutObjReader(t, bytes.NewBufferString("part"), 4, "", ""), ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return []CompletePart{{PartNumber: 1, ETag: pi.ETag}}
|
||||
}
|
||||
|
||||
// Scenario A: the uploaded If-Match carries the stale ETag of the pool-0
|
||||
// copy while the logical latest object lives in pool 1. The completion
|
||||
// must fail with 412 instead of shadowing the newer logical state.
|
||||
objectA := "cond-mp-stale-etag"
|
||||
base := time.Now()
|
||||
oldA := putConsistencyObject(t, z, bucket, objectA, 0, "old", ObjectOptions{MTime: base.Add(-2 * time.Minute)})
|
||||
mpA, err := z.serverPools[0].NewMultipartUpload(ctx, bucket, objectA, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newerA := putConsistencyObject(t, z, bucket, objectA, 1, "new", ObjectOptions{
|
||||
MTime: base.Add(-time.Minute),
|
||||
})
|
||||
|
||||
latest, _, err := z.getLatestObjectInfoWithIdx(ctx, bucket, objectA, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if latest.ETag != newerA.ETag {
|
||||
t.Fatalf("logical latest should be the pool-1 copy: got %s want %s", latest.ETag, newerA.ETag)
|
||||
}
|
||||
|
||||
if _, err = z.CompleteMultipartUpload(ctx, bucket, objectA, mpA.UploadID,
|
||||
uploadPart(t, bucket, objectA, mpA.UploadID), ifMatch(oldA.ETag)); err == nil {
|
||||
t.Fatal("If-Match with the stale pool-0 ETag must not complete over the newer pool-1 object")
|
||||
} else if _, ok := err.(PreConditionFailed); !ok {
|
||||
t.Fatalf("expected PreconditionFailed, got %v", err)
|
||||
}
|
||||
|
||||
if latest, _, err = z.getLatestObjectInfoWithIdx(ctx, bucket, objectA, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if latest.ETag != newerA.ETag {
|
||||
t.Fatalf("the newer pool-1 object must remain the logical latest, got %s", latest.ETag)
|
||||
}
|
||||
|
||||
// Scenario B: the uploaded If-Match carries the logical latest ETag (the
|
||||
// pool-1 copy) while the upload sits next to the stale pool-0 copy. The
|
||||
// precondition is satisfied, so completion must succeed and its result
|
||||
// must become the logical latest.
|
||||
objectB := "cond-mp-latest-etag"
|
||||
putConsistencyObject(t, z, bucket, objectB, 0, "old", ObjectOptions{MTime: base.Add(-2 * time.Minute)})
|
||||
mpB, err := z.serverPools[0].NewMultipartUpload(ctx, bucket, objectB, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newerB := putConsistencyObject(t, z, bucket, objectB, 1, "new", ObjectOptions{
|
||||
MTime: base.Add(-time.Minute),
|
||||
})
|
||||
oiB, err := z.CompleteMultipartUpload(ctx, bucket, objectB, mpB.UploadID,
|
||||
uploadPart(t, bucket, objectB, mpB.UploadID), ifMatch(newerB.ETag))
|
||||
if err != nil {
|
||||
t.Fatalf("If-Match with the logical latest ETag must complete, got %v", err)
|
||||
}
|
||||
if oiB.ETag == "" {
|
||||
t.Fatal("completion returned an empty ETag")
|
||||
}
|
||||
if latest, _, err = z.getLatestObjectInfoWithIdx(ctx, bucket, objectB, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if latest.ETag != oiB.ETag {
|
||||
t.Fatalf("the completed object must be the logical latest: got %s want %s", latest.ETag, oiB.ETag)
|
||||
}
|
||||
|
||||
// Scenario C: the upload lives in pool 1 with the newer copy while pool 0
|
||||
// holds the stale one. A set-local evaluation order would let pool 0's
|
||||
// stale copy fail the request before pool 1 is reached; the logical
|
||||
// latest ETag must complete.
|
||||
objectC := "cond-mp-upload-other-pool"
|
||||
putConsistencyObject(t, z, bucket, objectC, 0, "old", ObjectOptions{MTime: base.Add(-2 * time.Minute)})
|
||||
mpC, err := z.serverPools[1].NewMultipartUpload(ctx, bucket, objectC, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newerC := putConsistencyObject(t, z, bucket, objectC, 1, "new", ObjectOptions{
|
||||
MTime: base.Add(-time.Minute),
|
||||
})
|
||||
if _, err = z.CompleteMultipartUpload(ctx, bucket, objectC, mpC.UploadID,
|
||||
uploadPart(t, bucket, objectC, mpC.UploadID), ifMatch(newerC.ETag)); err != nil {
|
||||
t.Fatalf("If-Match with the logical latest ETag must complete regardless of upload pool, got %v", err)
|
||||
}
|
||||
|
||||
// Scenario D: pool 1 holds the newer copy but cannot be read. An
|
||||
// unreadable pool may contain the newest state, so the unverifiable
|
||||
// condition must fail the request rather than pass it against pool 0's
|
||||
// stale ETag.
|
||||
objectD := "cond-mp-unreadable-pool"
|
||||
oldD := putConsistencyObject(t, z, bucket, objectD, 0, "old", ObjectOptions{MTime: base.Add(-2 * time.Minute)})
|
||||
mpD, err := z.serverPools[0].NewMultipartUpload(ctx, bucket, objectD, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newerD := putConsistencyObject(t, z, bucket, objectD, 1, "new", ObjectOptions{
|
||||
MTime: base.Add(-time.Minute),
|
||||
})
|
||||
if latest, _, err = z.getLatestObjectInfoWithIdx(ctx, bucket, objectD, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if latest.ETag != newerD.ETag {
|
||||
t.Fatalf("logical latest before faulting pool 1 should be its copy: got %s want %s", latest.ETag, newerD.ETag)
|
||||
}
|
||||
set := z.serverPools[1].getHashedSet(objectD)
|
||||
getDisks := set.getDisks
|
||||
faulty := append([]StorageAPI(nil), getDisks()...)
|
||||
for i := range faulty {
|
||||
faulty[i] = consistencyReadFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: objectD}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return faulty }
|
||||
defer func() { set.getDisks = getDisks }()
|
||||
|
||||
_, err = z.CompleteMultipartUpload(ctx, bucket, objectD, mpD.UploadID,
|
||||
uploadPart(t, bucket, objectD, mpD.UploadID), ifMatch(oldD.ETag))
|
||||
if !isErrReadQuorum(err) {
|
||||
t.Fatalf("expected an insufficient read quorum error, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
func multipartConditionRequest(t *testing.T, router http.Handler, method, target, body string, headers map[string]string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
req, err := newTestSignedRequestV4(method, target, int64(len(body)), strings.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, headers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
// The server writes the wire spelling ETag directly into Header; unlike a
|
||||
// network response, httptest's map has not canonicalized it to Etag.
|
||||
func multipartConditionResponseETag(rec *httptest.ResponseRecorder) string {
|
||||
for key, values := range rec.Header() {
|
||||
if strings.EqualFold(key, xhttp.ETag) && len(values) > 0 {
|
||||
return strings.Trim(values[0], "\"")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func multipartConditionUpload(t *testing.T, z *erasureServerPools, bucket, object string, owner int, opts ObjectOptions) (string, []CompletePart) {
|
||||
t.Helper()
|
||||
mp, err := z.serverPools[owner].NewMultipartUpload(t.Context(), bucket, object, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
part, err := z.serverPools[owner].PutObjectPart(t.Context(), bucket, object, mp.UploadID, 1, mustGetPutObjReader(t, bytes.NewBufferString("replacement"), 11, "", ""), ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}
|
||||
}
|
||||
|
||||
func multipartConditionCompleteBody(parts []CompletePart) string {
|
||||
data, err := xml.Marshal(CompleteMultipartUpload{Parts: parts})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func multipartConditionError(t *testing.T, rec *httptest.ResponseRecorder, code string) {
|
||||
t.Helper()
|
||||
decoder := xml.NewDecoder(strings.NewReader(rec.Body.String()))
|
||||
var response APIErrorResponse
|
||||
if err := decoder.Decode(&response); err != nil || response.Code != code {
|
||||
t.Fatalf("expected %s error, got %q: %v", code, rec.Body.String(), err)
|
||||
}
|
||||
if err := decoder.Decode(&response); err != io.EOF {
|
||||
t.Fatalf("expected exactly one error response, got %q: %v", rec.Body.String(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// State is deliberately placed per pool; the final request uses signed HTTP
|
||||
// and the real handler, precondition callback, erasure metadata and rename.
|
||||
func TestPoolsMultipartConditionalHTTPMatrix(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for owner := range 2 {
|
||||
for _, localOld := range []bool{false, true} {
|
||||
for _, condition := range []string{"match-old", "match-current", "none-match"} {
|
||||
t.Run(fmt.Sprintf("owner=%d/old=%t/%s", owner, localOld, condition), func(t *testing.T) {
|
||||
object := fmt.Sprintf("http-%d-%t-%s", owner, localOld, condition)
|
||||
oldETag := "old-does-not-exist"
|
||||
if localOld {
|
||||
oldETag = putConsistencyObject(t, z, bucket, object, owner, "old", ObjectOptions{MTime: UTCNow().Add(-time.Hour)}).ETag
|
||||
}
|
||||
id, parts := multipartConditionUpload(t, z, bucket, object, owner, ObjectOptions{})
|
||||
current := putConsistencyObject(t, z, bucket, object, 1-owner, "current", ObjectOptions{MTime: UTCNow().Add(-time.Minute)})
|
||||
head := multipartConditionRequest(t, router, http.MethodHead, getGetObjectURL("", bucket, object), "", nil)
|
||||
if head.Code != 200 || multipartConditionResponseETag(head) != current.ETag {
|
||||
t.Fatalf("bad HEAD: %d %v", head.Code, head.Header())
|
||||
}
|
||||
h := map[string]string{xhttp.IfMatch: "\"" + oldETag + "\""}
|
||||
want := 412
|
||||
if condition == "match-current" {
|
||||
h[xhttp.IfMatch] = "\"" + current.ETag + "\""
|
||||
want = 200
|
||||
}
|
||||
if condition == "none-match" {
|
||||
h = map[string]string{xhttp.IfNoneMatch: "*"}
|
||||
}
|
||||
rec := multipartConditionRequest(t, router, http.MethodPost, getCompleteMultipartUploadURL("", bucket, object, id), multipartConditionCompleteBody(parts), h)
|
||||
t.Logf("HEAD current=%s, requested=%v, complete HTTP=%d", current.ETag, h, rec.Code)
|
||||
if rec.Code != want {
|
||||
t.Errorf("want HTTP %d, got %d: %s", want, rec.Code, rec.Body.String())
|
||||
}
|
||||
if want == 412 {
|
||||
multipartConditionError(t, rec, "PreconditionFailed")
|
||||
got := multipartConditionRequest(t, router, http.MethodGet, getGetObjectURL("", bucket, object), "", nil)
|
||||
if got.Code != 200 || got.Body.String() != "current" {
|
||||
t.Errorf("rejected request must preserve current data: %d %q", got.Code, got.Body.String())
|
||||
}
|
||||
if _, err := z.serverPools[owner].ListObjectParts(t.Context(), bucket, object, id, 0, 10, ObjectOptions{}); err != nil {
|
||||
t.Errorf("rejected request consumed upload: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMultipartConditionalHTTPAbsentObject(t *testing.T) {
|
||||
for _, deleted := range []bool{false, true} {
|
||||
for _, match := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("delete-marker=%t/if-match=%t", deleted, match), func(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{VersioningEnabled: deleted})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
object := "http-absent"
|
||||
if deleted {
|
||||
putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
|
||||
_, err = z.serverPools[1].DeleteObject(t.Context(), bucket, object, ObjectOptions{Versioned: true, VersionID: mustGetUUID(), DeleteMarker: true, MTime: UTCNow().Add(-time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
id, parts := multipartConditionUpload(t, z, bucket, object, 0, ObjectOptions{Versioned: deleted})
|
||||
headers := map[string]string{xhttp.IfNoneMatch: "*"}
|
||||
want := http.StatusOK
|
||||
if match {
|
||||
headers = map[string]string{xhttp.IfMatch: "\"missing\""}
|
||||
want = http.StatusNotFound
|
||||
}
|
||||
rec := multipartConditionRequest(t, router, http.MethodPost, getCompleteMultipartUploadURL("", bucket, object, id), multipartConditionCompleteBody(parts), headers)
|
||||
if rec.Code != want {
|
||||
t.Fatalf("expected HTTP %d, got %d: %s", want, rec.Code, rec.Body.String())
|
||||
}
|
||||
if match {
|
||||
multipartConditionError(t, rec, "NoSuchKey")
|
||||
if _, err := z.serverPools[0].ListObjectParts(t.Context(), bucket, object, id, 0, 10, ObjectOptions{}); err != nil {
|
||||
t.Errorf("rejected request consumed upload: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// All writes below use ordinary signed S3 requests. The result must be correct
|
||||
// for every placement; the matrix above deterministically covers split pools.
|
||||
func TestPoolsMultipartConditionalHTTPNormalRouting(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
object := "normal-routing"
|
||||
url := getPutObjectURL("", bucket, object)
|
||||
old := multipartConditionRequest(t, router, http.MethodPut, url, "old-data", nil)
|
||||
if old.Code != http.StatusOK {
|
||||
t.Fatalf("initial PUT %d: %s", old.Code, old.Body.String())
|
||||
}
|
||||
init := multipartConditionRequest(t, router, http.MethodPost, url+"?uploads", "", nil)
|
||||
if init.Code != http.StatusOK {
|
||||
t.Fatalf("init %d: %s", init.Code, init.Body.String())
|
||||
}
|
||||
var mp InitiateMultipartUploadResponse
|
||||
if err := xml.Unmarshal(init.Body.Bytes(), &mp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
part := multipartConditionRequest(t, router, http.MethodPut, getPutObjectPartURL("", bucket, object, mp.UploadID, "1"), "replacement", nil)
|
||||
if part.Code != http.StatusOK {
|
||||
t.Fatalf("part %d: %s", part.Code, part.Body.String())
|
||||
}
|
||||
parts := []CompletePart{{PartNumber: 1, ETag: multipartConditionResponseETag(part)}}
|
||||
newer := multipartConditionRequest(t, router, http.MethodPut, url, "newer-data", nil)
|
||||
if newer.Code != http.StatusOK {
|
||||
t.Fatalf("new PUT %d: %s", newer.Code, newer.Body.String())
|
||||
}
|
||||
head := multipartConditionRequest(t, router, http.MethodHead, url, "", nil)
|
||||
if head.Code != http.StatusOK || multipartConditionResponseETag(head) != multipartConditionResponseETag(newer) {
|
||||
t.Fatalf("HEAD did not pick new object: %d %v", head.Code, head.Header())
|
||||
}
|
||||
rec := multipartConditionRequest(t, router, http.MethodPost, getCompleteMultipartUploadURL("", bucket, object, mp.UploadID), multipartConditionCompleteBody(parts), map[string]string{xhttp.IfMatch: "\"" + multipartConditionResponseETag(old) + "\""})
|
||||
if rec.Code != http.StatusPreconditionFailed {
|
||||
t.Errorf("stale If-Match should be HTTP 412, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil)
|
||||
if get.Code != http.StatusOK || get.Body.String() != "newer-data" {
|
||||
t.Errorf("conditional completion changed newer data: %d %q", get.Code, get.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMultipartConditionalUnreadablePool(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
object := "quorum-with-readable-copy"
|
||||
old := putConsistencyObject(t, z, bucket, object, 0, "readable", ObjectOptions{MTime: UTCNow().Add(-time.Hour)})
|
||||
putConsistencyObject(t, z, bucket, object, 1, "hidden-newer", ObjectOptions{MTime: UTCNow().Add(-time.Minute)})
|
||||
id, parts := multipartConditionUpload(t, z, bucket, object, 0, ObjectOptions{})
|
||||
set := z.serverPools[1].getHashedSet(object)
|
||||
original := set.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i := range disks {
|
||||
disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: object}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
defer func() { set.getDisks = original }()
|
||||
read, readErr := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||
t.Logf("ordinary GET lookup: etag=%s err=%v", read.ETag, readErr)
|
||||
called := 0
|
||||
_, err := z.CompleteMultipartUpload(t.Context(), bucket, object, id, parts, ObjectOptions{HasIfMatch: true, CheckPrecondFn: func(oi ObjectInfo) bool { called++; return oi.ETag != old.ETag }})
|
||||
if !isErrReadQuorum(err) {
|
||||
t.Errorf("conditional write must fail on unreadable pool, got %v", err)
|
||||
}
|
||||
if called != 0 {
|
||||
t.Errorf("callback evaluated without complete state: %d calls", called)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMultipartConditionalLatestVersionAndCallbackOnce(t *testing.T) {
|
||||
for _, explicit := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("explicit=%t", explicit), func(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
object := "tie-version"
|
||||
opts := ObjectOptions{MTime: UTCNow().Add(-time.Hour), Versioned: explicit}
|
||||
if explicit {
|
||||
opts.VersionID = mustGetUUID()
|
||||
}
|
||||
current := putConsistencyObject(t, z, bucket, object, 0, "first", opts)
|
||||
putConsistencyObject(t, z, bucket, object, 1, "second", opts)
|
||||
if explicit {
|
||||
current = putConsistencyObject(t, z, bucket, object, 1, "latest-other-version", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Minute)})
|
||||
}
|
||||
id, parts := multipartConditionUpload(t, z, bucket, object, 1, opts)
|
||||
called := 0
|
||||
opts.CheckPrecondFn = func(oi ObjectInfo) bool { called++; return oi.ETag != current.ETag }
|
||||
opts.MTime = time.Time{}
|
||||
opts.HasIfMatch = true
|
||||
_, err := z.CompleteMultipartUpload(t.Context(), bucket, object, id, parts, opts)
|
||||
if err != nil {
|
||||
t.Errorf("logical current object should match: %v", err)
|
||||
}
|
||||
if called != 1 {
|
||||
t.Errorf("condition evaluated %d times; want exactly once", called)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMultipartConditionalConcurrentCompletes(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
object := "concurrent-completes"
|
||||
old := putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{MTime: UTCNow().Add(-time.Hour)})
|
||||
putConsistencyObject(t, z, bucket, object, 1, "old", ObjectOptions{MTime: old.ModTime})
|
||||
ids := make([]string, 2)
|
||||
parts := make([][]CompletePart, 2)
|
||||
for i := range 2 {
|
||||
ids[i], parts[i] = multipartConditionUpload(t, z, bucket, object, i, ObjectOptions{})
|
||||
}
|
||||
entered := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
var gate, releaseOnce sync.Once
|
||||
defer releaseOnce.Do(func() { close(release) })
|
||||
errs := make([]error, 2)
|
||||
var wg sync.WaitGroup
|
||||
// Let pool 1's completion hold the object lock before pool 0's starts.
|
||||
// Once pool 1 commits, a set-local read in pool 0 would still see the
|
||||
// old ETag and incorrectly accept the second completion.
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_, errs[1] = z.CompleteMultipartUpload(t.Context(), bucket, object, ids[1], parts[1], ObjectOptions{HasIfMatch: true, CheckPrecondFn: func(oi ObjectInfo) bool {
|
||||
gate.Do(func() { close(entered); <-release })
|
||||
return oi.ETag != old.ETag
|
||||
}})
|
||||
}()
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("first completion did not enter its condition callback")
|
||||
}
|
||||
started := make(chan struct{})
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
close(started)
|
||||
_, errs[0] = z.CompleteMultipartUpload(t.Context(), bucket, object, ids[0], parts[0], ObjectOptions{HasIfMatch: true, CheckPrecondFn: func(oi ObjectInfo) bool { return oi.ETag != old.ETag }})
|
||||
}()
|
||||
<-started
|
||||
releaseOnce.Do(func() { close(release) })
|
||||
wg.Wait()
|
||||
success, failed := 0, 0
|
||||
for _, err := range errs {
|
||||
var p PreConditionFailed
|
||||
switch {
|
||||
case err == nil:
|
||||
success++
|
||||
case errors.As(err, &p):
|
||||
failed++
|
||||
default:
|
||||
t.Errorf("unexpected completion error %v", err)
|
||||
}
|
||||
}
|
||||
if success != 1 || failed != 1 {
|
||||
t.Errorf("CAS writers: success=%d conditional failures=%d errors=%v", success, failed, errs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPoolsMultipartConditionMatrix(t *testing.T) {
|
||||
for owner := range 2 {
|
||||
for _, withOld := range []bool{false, true} {
|
||||
for _, condition := range []string{"match-current", "match-old", "none-match-any"} {
|
||||
t.Run(fmt.Sprintf("upload-pool=%d/old-copy=%t/%s", owner, withOld, condition), func(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
object := "conditional-multipart"
|
||||
oldETag := "arbitrary-old"
|
||||
if withOld {
|
||||
old := putConsistencyObject(t, z, bucket, object, owner, "old", ObjectOptions{MTime: UTCNow().Add(-time.Hour)})
|
||||
oldETag = old.ETag
|
||||
}
|
||||
mp, err := z.serverPools[owner].NewMultipartUpload(t.Context(), bucket, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
part, err := z.serverPools[owner].PutObjectPart(t.Context(), bucket, object, mp.UploadID, 1, mustGetPutObjReader(t, bytes.NewBufferString("replacement"), 11, "", ""), ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current := putConsistencyObject(t, z, bucket, object, 1-owner, "current", ObjectOptions{MTime: UTCNow().Add(-time.Minute)})
|
||||
visible, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||
if err != nil || visible.ETag != current.ETag {
|
||||
t.Fatalf("invalid current state: %v", err)
|
||||
}
|
||||
opts := ObjectOptions{HasIfMatch: condition != "none-match-any", CheckPrecondFn: func(oi ObjectInfo) bool {
|
||||
switch condition {
|
||||
case "match-current":
|
||||
return oi.ETag != current.ETag
|
||||
case "match-old":
|
||||
return oi.ETag != oldETag
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}}
|
||||
_, err = z.CompleteMultipartUpload(t.Context(), bucket, object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, opts)
|
||||
if condition == "match-current" {
|
||||
if err != nil {
|
||||
t.Errorf("correct logical If-Match rejected: %v", err)
|
||||
}
|
||||
} else {
|
||||
var expected PreConditionFailed
|
||||
if !errors.As(err, &expected) {
|
||||
t.Errorf("logical condition must fail, got %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMultipartConditionBoundaries(t *testing.T) {
|
||||
for _, state := range []string{"missing", "latest-delete-marker", "unreadable-other-pool"} {
|
||||
for _, match := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/if-match=%t", state, match), func(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
object := "conditional-boundary"
|
||||
versioned := state == "latest-delete-marker"
|
||||
if versioned {
|
||||
putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
|
||||
if _, err := z.serverPools[1].DeleteObject(t.Context(), bucket, object, ObjectOptions{Versioned: true, VersionID: mustGetUUID(), DeleteMarker: true, MTime: UTCNow().Add(-time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
mp, err := z.serverPools[0].NewMultipartUpload(t.Context(), bucket, object, ObjectOptions{Versioned: versioned})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
part, err := z.serverPools[0].PutObjectPart(t.Context(), bucket, object, mp.UploadID, 1, mustGetPutObjReader(t, bytes.NewBufferString("new"), 3, "", ""), ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state == "unreadable-other-pool" {
|
||||
set := z.serverPools[1].getHashedSet(object)
|
||||
getDisks := set.getDisks
|
||||
faulty := append([]StorageAPI(nil), getDisks()...)
|
||||
for i := range faulty {
|
||||
faulty[i] = consistencyReadFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return faulty }
|
||||
defer func() { set.getDisks = getDisks }()
|
||||
}
|
||||
opts := ObjectOptions{Versioned: versioned, HasIfMatch: match, CheckPrecondFn: func(ObjectInfo) bool { return true }}
|
||||
_, err = z.CompleteMultipartUpload(t.Context(), bucket, object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, opts)
|
||||
switch {
|
||||
case state == "unreadable-other-pool":
|
||||
if !isErrReadQuorum(err) {
|
||||
t.Errorf("unreadable pool must not mean absence: %v", err)
|
||||
}
|
||||
case match:
|
||||
if !isErrObjectNotFound(err) {
|
||||
t.Errorf("If-Match against logical absence should report absence: %v", err)
|
||||
}
|
||||
default:
|
||||
if err != nil {
|
||||
t.Errorf("If-None-Match against logical absence must succeed: %v", err)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if _, lerr := z.serverPools[0].ListObjectParts(t.Context(), bucket, object, mp.UploadID, 0, 10, ObjectOptions{}); lerr != nil {
|
||||
t.Errorf("failed condition consumed upload: %v", lerr)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,459 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
// The fixture places copies directly in real erasure pools. It models a
|
||||
// duplicated version; it does not claim to exercise a rebalance workflow.
|
||||
func TestPoolsMetadataUpdatePreservesTags(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
const (
|
||||
old = "2026-09-09T09:00:00Z"
|
||||
recent = "2026-09-09T10:00:00Z"
|
||||
timestamp = ReservedMetadataPrefixLower + TaggingTimestamp
|
||||
)
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
tags, stamps [2]string
|
||||
winner int
|
||||
single bool
|
||||
}{
|
||||
{name: "newer-secondary", tags: [2]string{"key=old", "key=new"}, stamps: [2]string{old, recent}, winner: 1},
|
||||
{name: "newer-primary", tags: [2]string{"key=new", "key=old"}, stamps: [2]string{recent, old}},
|
||||
{name: "empty-secondary", tags: [2]string{"key=old", ""}, stamps: [2]string{old, recent}, winner: 1},
|
||||
{name: "empty-primary", tags: [2]string{"", "key=old"}, stamps: [2]string{recent, old}},
|
||||
{name: "single-copy-primary", tags: [2]string{"key=only", ""}, stamps: [2]string{recent, ""}, single: true},
|
||||
{name: "single-copy-secondary", tags: [2]string{"", "key=only"}, stamps: [2]string{"", recent}, winner: 1, single: true},
|
||||
{name: "legacy-single-copy", tags: [2]string{"key=legacy", ""}, single: true},
|
||||
{name: "legacy-duplicates", tags: [2]string{"key=legacy", "key=legacy"}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
object := test.name
|
||||
var original ObjectInfo
|
||||
for pool := range 2 {
|
||||
if test.single && pool != test.winner {
|
||||
continue
|
||||
}
|
||||
metadata := map[string]string{
|
||||
xhttp.AmzObjectTagging: test.tags[pool],
|
||||
"copy-local": fmt.Sprint(pool),
|
||||
}
|
||||
if test.stamps[pool] != "" {
|
||||
metadata[timestamp] = test.stamps[pool]
|
||||
}
|
||||
original = putConsistencyObject(t, z, bucket, object, pool, "data", ObjectOptions{
|
||||
Versioned: true, VersionID: original.VersionID, MTime: original.ModTime, UserDefined: metadata,
|
||||
})
|
||||
got, err := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: original.VersionID})
|
||||
if err != nil || got.UserTags != test.tags[pool] || got.UserDefined[timestamp] != test.stamps[pool] {
|
||||
t.Fatalf("pool %d fixture: tags=%q timestamp=%q err=%v", pool, got.UserTags, got.UserDefined[timestamp], err)
|
||||
}
|
||||
if _, exists := got.UserDefined[xhttp.AmzObjectTagging]; exists {
|
||||
t.Fatal("fixture must use the cleaned ObjectInfo representation")
|
||||
}
|
||||
}
|
||||
wantTags, wantStamp := test.tags[test.winner], test.stamps[test.winner]
|
||||
called := 0
|
||||
got, err := z.PutObjectMetadata(t.Context(), bucket, object, ObjectOptions{
|
||||
VersionID: original.VersionID, MTime: original.ModTime,
|
||||
EvalMetadataFn: func(current *ObjectInfo, _ error) (ReplicateDecision, error) {
|
||||
called++
|
||||
if current.UserTags != wantTags || current.UserDefined[timestamp] != wantStamp {
|
||||
t.Errorf("callback tags=%q timestamp=%q; want %q %q", current.UserTags, current.UserDefined[timestamp], wantTags, wantStamp)
|
||||
}
|
||||
current.UserDefined["unrelated-update"] = "preserved"
|
||||
return ReplicateDecision{}, nil
|
||||
},
|
||||
})
|
||||
if err != nil || called != 1 {
|
||||
t.Fatalf("metadata update: %v, callbacks=%d", err, called)
|
||||
}
|
||||
if got.UserTags != wantTags || got.UserDefined[timestamp] != wantStamp {
|
||||
t.Errorf("response tags=%q timestamp=%q; want %q %q", got.UserTags, got.UserDefined[timestamp], wantTags, wantStamp)
|
||||
}
|
||||
for pool := range 2 {
|
||||
got, err := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: original.VersionID})
|
||||
if test.single && pool != test.winner {
|
||||
if !isErrVersionNotFound(err) {
|
||||
t.Errorf("metadata update created another copy: %v", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("pool %d persisted tags=%q timestamp=%q", pool, got.UserTags, got.UserDefined[timestamp])
|
||||
if got.UserTags != wantTags || got.UserDefined[timestamp] != wantStamp {
|
||||
t.Errorf("pool %d persisted tags=%q timestamp=%q; want %q %q", pool, got.UserTags, got.UserDefined[timestamp], wantTags, wantStamp)
|
||||
}
|
||||
if got.UserDefined["unrelated-update"] != "preserved" || got.UserDefined["copy-local"] != fmt.Sprint(pool) {
|
||||
t.Errorf("pool %d lost unrelated metadata: %v", pool, got.UserDefined)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplicaWritesPreserveTagOrdering(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
// Pool allocation checks the host's used-space percentage. Present only
|
||||
// its free space as fixture capacity; all reads and writes still use the
|
||||
// real disks. This keeps unrelated host disk usage out of the tag test.
|
||||
for _, pool := range z.serverPools {
|
||||
for _, set := range pool.sets {
|
||||
getDisks := set.getDisks
|
||||
disks := append([]StorageAPI(nil), getDisks()...)
|
||||
for i := range disks {
|
||||
disks[i] = tagTestCapacityDisk{StorageAPI: disks[i]}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
t.Cleanup(func() { set.getDisks = getDisks })
|
||||
}
|
||||
}
|
||||
const (
|
||||
old = "2026-09-09T09:00:00Z"
|
||||
recent = "2026-09-09T10:00:00Z"
|
||||
timestamp = ReservedMetadataPrefixLower + TaggingTimestamp
|
||||
)
|
||||
for _, path := range []struct {
|
||||
name, operation string
|
||||
direct bool
|
||||
winner int
|
||||
}{
|
||||
{name: "set-put", operation: "put", direct: true},
|
||||
{name: "set-multipart", operation: "multipart", direct: true},
|
||||
{name: "set-copy", operation: "copy", direct: true},
|
||||
{name: "pools-put", operation: "put", winner: 1},
|
||||
{name: "pools-multipart", operation: "multipart", winner: 1},
|
||||
{name: "pools-copy-primary", operation: "copy"},
|
||||
{name: "pools-copy-secondary", operation: "copy", winner: 1},
|
||||
} {
|
||||
for _, test := range []struct {
|
||||
name, storedTags, incomingTags, storedStamp, incomingStamp, wantTags string
|
||||
}{
|
||||
{"stored-newer", "key=stored", "key=incoming", recent, old, "key=stored"},
|
||||
{"stored-deleted", "", "key=incoming", recent, old, ""},
|
||||
{"incoming-newer", "key=stored", "key=incoming", old, recent, "key=incoming"},
|
||||
{"incoming-deleted", "key=stored", "", old, recent, ""},
|
||||
} {
|
||||
t.Run(path.name+"/"+test.name, func(t *testing.T) {
|
||||
object := path.name + "-" + test.name
|
||||
var original ObjectInfo
|
||||
for pool := range 2 {
|
||||
if path.direct && pool != 0 {
|
||||
continue
|
||||
}
|
||||
metadata := map[string]string{
|
||||
xhttp.AmzObjectTagging: "key=older-copy",
|
||||
timestamp: "2026-09-09T08:00:00Z",
|
||||
}
|
||||
if pool == path.winner {
|
||||
metadata[xhttp.AmzObjectTagging] = test.storedTags
|
||||
metadata[timestamp] = test.storedStamp
|
||||
}
|
||||
original = putConsistencyObject(t, z, bucket, object, pool, "data", ObjectOptions{
|
||||
Versioned: true, VersionID: original.VersionID, MTime: original.ModTime, UserDefined: metadata,
|
||||
})
|
||||
}
|
||||
opts := ObjectOptions{
|
||||
Versioned: true, VersionID: original.VersionID, MTime: original.ModTime, ReplicaLockReconcile: true,
|
||||
UserDefined: map[string]string{
|
||||
xhttp.AmzObjectTagging: test.incomingTags,
|
||||
timestamp: test.incomingStamp,
|
||||
},
|
||||
}
|
||||
var got ObjectInfo
|
||||
var err error
|
||||
switch path.operation {
|
||||
case "put":
|
||||
put := z.PutObject
|
||||
if path.direct {
|
||||
put = z.serverPools[0].PutObject
|
||||
}
|
||||
got, err = put(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), opts)
|
||||
case "multipart":
|
||||
// Persist the incoming tags with the upload, before completion
|
||||
// reconciles the destination version through its real resolver.
|
||||
mp, err := z.serverPools[0].NewMultipartUpload(t.Context(), bucket, object, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
part, err := z.serverPools[0].PutObjectPart(t.Context(), bucket, object, mp.UploadID, 1,
|
||||
mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
complete := z.CompleteMultipartUpload
|
||||
if path.direct {
|
||||
complete = z.serverPools[0].CompleteMultipartUpload
|
||||
}
|
||||
got, err = complete(t.Context(), bucket, object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{
|
||||
Versioned: true, MTime: original.ModTime, ReplicaLockReconcile: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "copy":
|
||||
src := original
|
||||
src.metadataOnly = true
|
||||
src.UserDefined = maps.Clone(opts.UserDefined)
|
||||
copyObject := z.CopyObject
|
||||
if path.direct {
|
||||
copyObject = z.serverPools[0].CopyObject
|
||||
}
|
||||
got, err = copyObject(t.Context(), bucket, object, bucket, object, src, ObjectOptions{VersionID: original.VersionID}, opts)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.UserTags != test.wantTags || got.UserDefined[timestamp] != recent {
|
||||
t.Errorf("response tags=%q timestamp=%q; want %q %q", got.UserTags, got.UserDefined[timestamp], test.wantTags, recent)
|
||||
}
|
||||
copies := 0
|
||||
for pool := range 2 {
|
||||
got, err := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: original.VersionID})
|
||||
if isErrVersionNotFound(err) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
copies++
|
||||
if got.UserTags != test.wantTags || got.UserDefined[timestamp] != recent {
|
||||
t.Errorf("pool %d persisted tags=%q timestamp=%q; want %q %q", pool, got.UserTags, got.UserDefined[timestamp], test.wantTags, recent)
|
||||
}
|
||||
}
|
||||
if copies != 1 {
|
||||
t.Errorf("replacement left %d copies; want 1", copies)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type tagTestCapacityDisk struct{ StorageAPI }
|
||||
|
||||
func (d tagTestCapacityDisk) DiskInfo(ctx context.Context, opts DiskInfoOptions) (DiskInfo, error) {
|
||||
info, err := d.StorageAPI.DiskInfo(ctx, opts)
|
||||
info.Total, info.Used = info.Free, 0
|
||||
return info, err
|
||||
}
|
||||
|
||||
func TestMergedPoolObjectInfoTagOrdering(t *testing.T) {
|
||||
const (
|
||||
old = "2026-09-09T09:00:00Z"
|
||||
recent = "2026-09-09T10:00:00Z"
|
||||
timestamp = ReservedMetadataPrefixLower + TaggingTimestamp
|
||||
)
|
||||
for _, test := range []struct {
|
||||
name, firstStamp, secondStamp, secondTags string
|
||||
winner int
|
||||
}{
|
||||
{"newer", old, recent, "key=second", 1},
|
||||
{"newer-removal", old, recent, "", 1},
|
||||
{"equal", recent, recent, "key=second", 0},
|
||||
{"unordered", "", "", "key=second", 0},
|
||||
{"missing-first", "", recent, "key=second", 1},
|
||||
{"missing-second", recent, "", "key=second", 0},
|
||||
{"invalid-first", "invalid", recent, "key=second", 1},
|
||||
{"invalid-second", recent, "invalid", "key=second", 0},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
tagValues := []string{"key=first", test.secondTags}
|
||||
stamps := []string{test.firstStamp, test.secondStamp}
|
||||
copies := make([]PoolObjInfo, 2)
|
||||
before := make([]map[string]string, 2)
|
||||
for i := range copies {
|
||||
fi := FileInfo{Metadata: map[string]string{xhttp.AmzObjectTagging: tagValues[i]}}
|
||||
if stamps[i] != "" {
|
||||
fi.Metadata[timestamp] = stamps[i]
|
||||
}
|
||||
copies[i] = PoolObjInfo{Index: i, ObjInfo: fi.ToObjectInfo("bucket", "object", true)}
|
||||
before[i] = maps.Clone(copies[i].ObjInfo.UserDefined)
|
||||
}
|
||||
got := mergedPoolObjectInfo(copies)
|
||||
if got.UserTags != tagValues[test.winner] || got.UserDefined[timestamp] != stamps[test.winner] {
|
||||
t.Errorf("merged tags=%q timestamp=%q; want %q %q", got.UserTags, got.UserDefined[timestamp], tagValues[test.winner], stamps[test.winner])
|
||||
}
|
||||
if _, exists := got.UserDefined[xhttp.AmzObjectTagging]; exists {
|
||||
t.Error("merged ObjectInfo leaked the raw tagging key into UserDefined")
|
||||
}
|
||||
for i := range copies {
|
||||
if !maps.Equal(copies[i].ObjInfo.UserDefined, before[i]) || copies[i].ObjInfo.UserTags != tagValues[i] {
|
||||
t.Errorf("merge mutated input copy %d", i)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMetadataCallbackReplacesTags(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
const timestamp = ReservedMetadataPrefixLower + TaggingTimestamp
|
||||
for _, test := range []struct{ name, tags string }{
|
||||
{"replace", "key=callback"},
|
||||
{"remove", ""},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var original ObjectInfo
|
||||
for pool := range 2 {
|
||||
original = putConsistencyObject(t, z, bucket, test.name, pool, "data", ObjectOptions{
|
||||
Versioned: true, VersionID: original.VersionID, MTime: original.ModTime,
|
||||
UserDefined: map[string]string{
|
||||
xhttp.AmzObjectTagging: []string{"key=old", "key=new"}[pool],
|
||||
timestamp: []string{"2026-09-09T09:00:00Z", "2026-09-09T10:00:00Z"}[pool],
|
||||
},
|
||||
})
|
||||
}
|
||||
const updatedStamp = "2026-09-09T11:00:00Z"
|
||||
got, err := z.PutObjectMetadata(t.Context(), bucket, test.name, ObjectOptions{
|
||||
VersionID: original.VersionID, MTime: original.ModTime,
|
||||
EvalMetadataFn: func(current *ObjectInfo, _ error) (ReplicateDecision, error) {
|
||||
if current.UserTags != "key=new" {
|
||||
t.Errorf("callback read tags=%q; want key=new", current.UserTags)
|
||||
}
|
||||
if _, exists := current.UserDefined[xhttp.AmzObjectTagging]; exists {
|
||||
t.Error("callback received the raw tagging key")
|
||||
}
|
||||
current.UserDefined[xhttp.AmzObjectTagging] = test.tags
|
||||
current.UserDefined[timestamp] = updatedStamp
|
||||
return ReplicateDecision{}, nil
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.UserTags != test.tags || got.UserDefined[timestamp] != updatedStamp {
|
||||
t.Errorf("callback update response tags=%q timestamp=%q", got.UserTags, got.UserDefined[timestamp])
|
||||
}
|
||||
for pool := range 2 {
|
||||
got, err := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, test.name, ObjectOptions{VersionID: original.VersionID})
|
||||
if err != nil || got.UserTags != test.tags || got.UserDefined[timestamp] != updatedStamp {
|
||||
t.Errorf("pool %d did not persist callback tags: tags=%q timestamp=%q err=%v", pool, got.UserTags, got.UserDefined[timestamp], err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileStoredObjectTagOrdering(t *testing.T) {
|
||||
const (
|
||||
old = "2026-09-09T09:00:00Z"
|
||||
recent = "2026-09-09T10:00:00Z"
|
||||
timestamp = ReservedMetadataPrefixLower + TaggingTimestamp
|
||||
)
|
||||
for _, test := range []struct {
|
||||
name, storedStamp, incomingStamp, storedTags string
|
||||
wantStored bool
|
||||
}{
|
||||
{"stored-newer", recent, old, "key=stored", true},
|
||||
{"incoming-newer", old, recent, "key=stored", false},
|
||||
{"equal", recent, recent, "key=stored", true},
|
||||
{"equal-removal", recent, recent, "", true},
|
||||
{"missing-stored", "", recent, "key=stored", false},
|
||||
{"missing-incoming", recent, "", "key=stored", true},
|
||||
{"invalid-stored", "invalid", recent, "key=stored", false},
|
||||
{"invalid-incoming", recent, "invalid", "key=stored", true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
metadata := map[string]string{
|
||||
xhttp.AmzObjectTagging: "key=incoming",
|
||||
timestamp: test.incomingStamp,
|
||||
"unrelated": "preserved",
|
||||
}
|
||||
reconcileStoredObjectTags(metadata, test.storedTags, test.storedStamp)
|
||||
wantTags, wantStamp := "key=incoming", test.incomingStamp
|
||||
if test.wantStored {
|
||||
wantTags, wantStamp = test.storedTags, test.storedStamp
|
||||
}
|
||||
if metadata[xhttp.AmzObjectTagging] != wantTags || metadata[timestamp] != wantStamp || metadata["unrelated"] != "preserved" {
|
||||
t.Errorf("reconciled metadata=%v; want tags=%q timestamp=%q and unrelated field preserved", metadata, wantTags, wantStamp)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMetadataUpdatePreservesAbsentTags(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
const (
|
||||
old = "2026-09-09T09:00:00Z"
|
||||
recent = "2026-09-09T10:00:00Z"
|
||||
timestamp = ReservedMetadataPrefixLower + TaggingTimestamp
|
||||
)
|
||||
for _, removal := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("timestamp-only-removal=%t", removal), func(t *testing.T) {
|
||||
object := fmt.Sprintf("absent-tags-%t", removal)
|
||||
var original ObjectInfo
|
||||
checkStored := func(pool int, wantKey bool, wantTags, wantStamp string) {
|
||||
t.Helper()
|
||||
infos, errs := readAllFileInfo(t.Context(), z.serverPools[pool].getHashedSet(object).getDisks(), "", bucket, object, original.VersionID, false, false)
|
||||
for disk, info := range infos {
|
||||
if errs[disk] != nil {
|
||||
t.Fatal(errs[disk])
|
||||
}
|
||||
tags, exists := info.Metadata[xhttp.AmzObjectTagging]
|
||||
if exists != wantKey || tags != wantTags || info.Metadata[timestamp] != wantStamp {
|
||||
t.Errorf("pool %d disk %d raw tagging key=%t value=%q stamp=%q; want %t %q %q", pool, disk, exists, tags, info.Metadata[timestamp], wantKey, wantTags, wantStamp)
|
||||
}
|
||||
}
|
||||
}
|
||||
for pool := range 2 {
|
||||
metadata := map[string]string{}
|
||||
if removal {
|
||||
metadata[timestamp] = recent
|
||||
if pool == 1 {
|
||||
metadata[xhttp.AmzObjectTagging] = "key=old"
|
||||
metadata[timestamp] = old
|
||||
}
|
||||
}
|
||||
original = putConsistencyObject(t, z, bucket, object, pool, "data", ObjectOptions{
|
||||
Versioned: true, VersionID: original.VersionID, MTime: original.ModTime, UserDefined: metadata,
|
||||
})
|
||||
checkStored(pool, removal && pool == 1, metadata[xhttp.AmzObjectTagging], metadata[timestamp])
|
||||
}
|
||||
_, err := z.PutObjectMetadata(t.Context(), bucket, object, ObjectOptions{
|
||||
VersionID: original.VersionID, MTime: original.ModTime,
|
||||
EvalMetadataFn: func(current *ObjectInfo, _ error) (ReplicateDecision, error) {
|
||||
current.UserDefined["unrelated-update"] = "preserved"
|
||||
return ReplicateDecision{}, nil
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantStamp := ""
|
||||
if removal {
|
||||
wantStamp = recent
|
||||
}
|
||||
for pool := range 2 {
|
||||
// A previously non-empty key needs an explicit empty value to
|
||||
// propagate deletion; an absent key should remain absent.
|
||||
checkStored(pool, removal && pool == 1, "", wantStamp)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+172
-59
@@ -635,7 +635,14 @@ func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, objec
|
||||
// if none are found falls back to most available space pool, this function is
|
||||
// designed to be only used by PutObject, CopyObject (newObject creation) and NewMultipartUpload.
|
||||
func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64) (idx int, err error) {
|
||||
return z.getWritePoolIdx(ctx, bucket, object, size, false)
|
||||
}
|
||||
|
||||
// getWritePoolIdx keeps the write-allocation policy when the caller already
|
||||
// holds the pools-layer object lock and must not reacquire a set read lock.
|
||||
func (z *erasureServerPools) getWritePoolIdx(ctx context.Context, bucket, object string, size int64, noLock bool) (idx int, err error) {
|
||||
pinfo, _, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, ObjectOptions{
|
||||
NoLock: noLock,
|
||||
SkipDecommissioned: true,
|
||||
SkipRebalancing: true,
|
||||
})
|
||||
@@ -1131,7 +1138,18 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
|
||||
return z.serverPools[0].PutObject(ctx, bucket, object, data, opts)
|
||||
}
|
||||
|
||||
idx, err := z.getPoolIdx(ctx, bucket, object, data.Size())
|
||||
if !opts.NoLock {
|
||||
lk := z.NewNSLock(bucket, object)
|
||||
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
}
|
||||
opts.NoLock = true
|
||||
|
||||
idx, err := z.getWritePoolIdx(ctx, bucket, object, data.Size(), true)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
@@ -1145,7 +1163,15 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
|
||||
}
|
||||
}
|
||||
|
||||
return z.serverPools[idx].PutObject(ctx, bucket, object, data, opts)
|
||||
if opts.ReplicaLockReconcile || opts.DataMovement {
|
||||
opts.ReplicaLockReconcile = true
|
||||
opts.replicaObjectInfo = z.replicaObjectInfo
|
||||
}
|
||||
oi, err := z.serverPools[idx].PutObject(ctx, bucket, object, data, opts)
|
||||
if err == nil && opts.ReplicaLockReconcile && !opts.DataMovement {
|
||||
err = z.retireReplicaCopies(ctx, bucket, object, idx, oi)
|
||||
}
|
||||
return oi, err
|
||||
}
|
||||
|
||||
func (z *erasureServerPools) deletePrefix(ctx context.Context, bucket string, prefix string) error {
|
||||
@@ -1166,15 +1192,7 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
||||
object = encodeDirObject(object)
|
||||
}
|
||||
|
||||
// Acquire a write lock before deleting the object.
|
||||
//
|
||||
// NOTE: this lock is taken at the server-pool level. The conditional
|
||||
// (If-Match) precondition below relies on this lock making the read-check-
|
||||
// delete sequence atomic. That holds for a single erasure set: the write
|
||||
// path (PutObject) locks at the destination set, which shares this lock's
|
||||
// namespace only within one set. Multi-pool conditional-delete atomicity
|
||||
// (concurrent writers across pools, cross-pool version selection) is a
|
||||
// separate concern tracked as a follow-up.
|
||||
// Serialize logical deletion with pooled writes and metadata updates.
|
||||
lk := z.NewNSLock(bucket, object)
|
||||
lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
|
||||
if err != nil {
|
||||
@@ -1187,6 +1205,13 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
||||
return ObjectInfo{}, z.deletePrefix(ctx, bucket, object)
|
||||
}
|
||||
|
||||
// Reconcile ordinary addressed-version deletes independently of pool movement.
|
||||
reconcileVersion := opts.VersionID != "" && !opts.DataMovement &&
|
||||
!opts.ReplicationRequest && !opts.Expiration.Expire && !opts.InclFreeVersions
|
||||
if !z.SinglePool() && (opts.CheckPrecondFn != nil || reconcileVersion) {
|
||||
return z.deleteObjectReconciled(ctx, bucket, object, opts)
|
||||
}
|
||||
|
||||
gopts := opts
|
||||
gopts.NoLock = true
|
||||
|
||||
@@ -1322,8 +1347,12 @@ func (z *erasureServerPools) deleteObjectFromAllPools(ctx context.Context, bucke
|
||||
// the delete call tries to clean up other pools during DeleteObject call.
|
||||
objInfo = dobjects[0]
|
||||
objInfo.Name = decodeDirObject(object)
|
||||
err = derrs[0]
|
||||
return objInfo, err
|
||||
for _, derr := range derrs {
|
||||
if derr != nil && !isErrObjectNotFound(derr) && !isErrVersionNotFound(derr) {
|
||||
return objInfo, derr
|
||||
}
|
||||
}
|
||||
return objInfo, derrs[0]
|
||||
}
|
||||
|
||||
func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
|
||||
@@ -1411,11 +1440,38 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
|
||||
dstOpts.NoLock = true
|
||||
}
|
||||
|
||||
if !z.SinglePool() && cpSrcDstSame && srcInfo.metadataOnly && dstOpts.ReplicaLockReconcile && srcOpts.VersionID == dstOpts.VersionID {
|
||||
copies, err := z.metadataPoolInfos(ctx, dstBucket, dstObject, dstOpts)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
stored := mergedPoolObjectInfo(copies)
|
||||
reconcileStoredObjectLock(srcInfo.UserDefined, storedObjectLockState(stored.UserDefined))
|
||||
reconcileStoredObjectTags(srcInfo.UserDefined, stored.UserTags, stored.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp])
|
||||
idx := copies[0].Index
|
||||
oi, err := z.serverPools[idx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
|
||||
if err == nil {
|
||||
err = z.retireReplicaCopies(ctx, dstBucket, dstObject, idx, oi)
|
||||
}
|
||||
return oi, err
|
||||
}
|
||||
|
||||
poolIdx, err := z.getPoolIdxNoLock(ctx, dstBucket, dstObject, srcInfo.Size)
|
||||
if err != nil {
|
||||
return objInfo, err
|
||||
}
|
||||
|
||||
if !z.SinglePool() && dstOpts.ReplicaLockReconcile {
|
||||
dstOpts.replicaObjectInfo = z.replicaObjectInfo
|
||||
if !dstOpts.DataMovement {
|
||||
defer func() {
|
||||
if err == nil {
|
||||
err = z.retireReplicaCopies(ctx, dstBucket, dstObject, poolIdx, objInfo)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
// CopyObjectHandler predicts the outcome of this decision in
|
||||
// copyRewritesObjectData(); keep the two in sync.
|
||||
if cpSrcDstSame && srcInfo.metadataOnly {
|
||||
@@ -1450,6 +1506,8 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
|
||||
EncryptFn: dstOpts.EncryptFn,
|
||||
WantChecksum: dstOpts.WantChecksum,
|
||||
WantServerSideChecksumType: dstOpts.WantServerSideChecksumType,
|
||||
ReplicaLockReconcile: dstOpts.ReplicaLockReconcile,
|
||||
replicaObjectInfo: dstOpts.replicaObjectInfo,
|
||||
}
|
||||
|
||||
return z.serverPools[poolIdx].PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
|
||||
@@ -2085,6 +2143,60 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
|
||||
}
|
||||
}()
|
||||
|
||||
if !z.SinglePool() {
|
||||
if !opts.NoLock {
|
||||
lk := z.NewNSLock(bucket, encodeDirObject(object))
|
||||
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
return objInfo, err
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
}
|
||||
opts.NoLock = true
|
||||
|
||||
// A conditional completion must be evaluated against the logical
|
||||
// latest object across pools, under the object write lock held for
|
||||
// this operation. The pool hosting the upload may only hold a stale
|
||||
// duplicate, so its set-local check would both accept an outdated
|
||||
// ETag and reject the current one. An unreadable pool is not
|
||||
// absence: it may hold the newest copy, so a read that cannot be
|
||||
// verified fails the request instead of passing the condition.
|
||||
// Once satisfied, the callback is cleared so the set layer does not
|
||||
// re-evaluate it against its local copy.
|
||||
if opts.CheckPrecondFn != nil {
|
||||
copies, lerr := z.objectPoolInfos(ctx, bucket, encodeDirObject(object), ObjectOptions{
|
||||
// Conditions always compare the logical current object,
|
||||
// independently of the completion's destination version.
|
||||
VersionID: "",
|
||||
Versioned: opts.Versioned,
|
||||
VersionSuspended: opts.VersionSuspended,
|
||||
NoAuditLog: true,
|
||||
})
|
||||
var latest ObjectInfo
|
||||
if lerr == nil {
|
||||
latest = copies[0].ObjInfo
|
||||
if latest.DeleteMarker {
|
||||
// A delete-marker latest reads as an absent key, matching
|
||||
// the set layer's getObjectInfo.
|
||||
lerr = toObjectErr(errFileNotFound, bucket, object)
|
||||
}
|
||||
}
|
||||
if lerr == nil && opts.CheckPrecondFn(latest) {
|
||||
return ObjectInfo{}, PreConditionFailed{}
|
||||
}
|
||||
if lerr != nil && !isErrVersionNotFound(lerr) && !isErrObjectNotFound(lerr) {
|
||||
return ObjectInfo{}, lerr
|
||||
}
|
||||
// if object doesn't exist return error for If-Match conditional requests
|
||||
// If-None-Match should be allowed to proceed for non-existent objects
|
||||
if lerr != nil && opts.HasIfMatch && (isErrObjectNotFound(lerr) || isErrVersionNotFound(lerr)) {
|
||||
return ObjectInfo{}, lerr
|
||||
}
|
||||
opts.CheckPrecondFn = nil
|
||||
}
|
||||
}
|
||||
|
||||
// Hold write locks to verify uploaded parts, also disallows any
|
||||
// parallel PutObjectPart() requests.
|
||||
uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID))
|
||||
@@ -2099,13 +2211,21 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
|
||||
return z.serverPools[0].CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
|
||||
}
|
||||
|
||||
if opts.ReplicaLockReconcile || opts.DataMovement {
|
||||
opts.ReplicaLockReconcile = true
|
||||
opts.replicaObjectInfo = z.replicaObjectInfo
|
||||
}
|
||||
|
||||
for idx, pool := range z.serverPools {
|
||||
if z.IsSuspended(idx) {
|
||||
continue
|
||||
}
|
||||
objInfo, err = pool.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
|
||||
if err == nil {
|
||||
return objInfo, nil
|
||||
if opts.ReplicaLockReconcile && !opts.DataMovement {
|
||||
err = z.retireReplicaCopies(ctx, bucket, encodeDirObject(object), idx, objInfo)
|
||||
}
|
||||
return objInfo, err
|
||||
}
|
||||
if _, ok := err.(InvalidUploadID); ok {
|
||||
// upload id not found move to next pool
|
||||
@@ -2127,6 +2247,11 @@ func (z *erasureServerPools) GetBucketInfo(ctx context.Context, bucket string, o
|
||||
if err != nil {
|
||||
return bucketInfo, toObjectErr(err, bucket)
|
||||
}
|
||||
// Physical existence/creation probes must not be overwritten by cached
|
||||
// metadata, which can legitimately lack Created on an unmigrated bucket.
|
||||
if opts.NoMetadata {
|
||||
return bucketInfo, nil
|
||||
}
|
||||
|
||||
meta, err := globalBucketMetadataSys.Get(bucket)
|
||||
if err == nil {
|
||||
@@ -2195,7 +2320,15 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
|
||||
opts.Force = true
|
||||
}
|
||||
|
||||
err := z.s3Peer.DeleteBucket(ctx, bucket, opts)
|
||||
// Take the metadata writer lock before deleting anything. Failure or
|
||||
// cancellation must leave both the bucket and its metadata intact.
|
||||
ctx, unlock, err := lockBucketMetadata(ctx, z, bucket)
|
||||
if err != nil {
|
||||
return toObjectErr(err, bucket)
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
err = z.s3Peer.DeleteBucket(ctx, bucket, opts)
|
||||
if err == nil || isErrBucketNotFound(err) {
|
||||
// If site replication is configured, hold on to deleted bucket state until sites sync
|
||||
if opts.SRDeleteOp == MarkDelete {
|
||||
@@ -2204,8 +2337,9 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
// Purge the entire bucket metadata entirely.
|
||||
z.deleteAll(context.Background(), minioMetaBucket, pathJoin(bucketMetaPrefix, bucket))
|
||||
// Finish cleanup after a committed delete even if the client disconnects.
|
||||
// Both the bucket-name and metadata locks remain held until return.
|
||||
z.deleteAll(context.WithoutCancel(ctx), minioMetaBucket, pathJoin(bucketMetaPrefix, bucket))
|
||||
}
|
||||
|
||||
return toObjectErr(err, bucket)
|
||||
@@ -2623,7 +2757,7 @@ func (z *erasureServerPools) HealObject(ctx context.Context, bucket, object, ver
|
||||
wg.Add(1)
|
||||
go func(idx int, pool *erasureSets) {
|
||||
defer wg.Done()
|
||||
result, err := pool.HealObject(ctx, bucket, object, versionID, opts)
|
||||
result, err := z.healObjectInPool(ctx, pool.getHashedSet(object), bucket, object, versionID, opts)
|
||||
result.Object = decodeDirObject(result.Object)
|
||||
errs[idx] = err
|
||||
results[idx] = result
|
||||
@@ -2892,15 +3026,7 @@ func (z *erasureServerPools) PutObjectMetadata(ctx context.Context, bucket, obje
|
||||
defer lk.Unlock(lkctx)
|
||||
}
|
||||
|
||||
opts.MetadataChg = true
|
||||
opts.NoLock = true
|
||||
// We don't know the size here set 1GiB at least.
|
||||
idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
|
||||
return z.serverPools[idx].PutObjectMetadata(ctx, bucket, object, opts)
|
||||
return z.updatePoolMetadata(ctx, bucket, object, opts)
|
||||
}
|
||||
|
||||
// PutObjectTags - replace or add tags to an existing object
|
||||
@@ -2921,44 +3047,31 @@ func (z *erasureServerPools) PutObjectTags(ctx context.Context, bucket, object s
|
||||
defer lk.Unlock(lkctx)
|
||||
}
|
||||
|
||||
opts.MetadataChg = true
|
||||
opts.NoLock = true
|
||||
|
||||
// We don't know the size here set 1GiB at least.
|
||||
idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
|
||||
copies, err := z.metadataPoolInfos(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
|
||||
return z.serverPools[idx].PutObjectTags(ctx, bucket, object, tags, opts)
|
||||
opts.NoLock = true
|
||||
opts.VersionID = copies[0].ObjInfo.VersionID
|
||||
if opts.VersionID == "" {
|
||||
opts.VersionID = nullVersionID
|
||||
}
|
||||
var primary ObjectInfo
|
||||
for _, copy := range copies {
|
||||
oi, err := z.serverPools[copy.Index].PutObjectTags(ctx, bucket, object, tags, opts)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
if copy.Index == copies[0].Index {
|
||||
primary = oi
|
||||
}
|
||||
}
|
||||
return primary, nil
|
||||
}
|
||||
|
||||
// DeleteObjectTags - delete object tags from an existing object
|
||||
func (z *erasureServerPools) DeleteObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
|
||||
object = encodeDirObject(object)
|
||||
if z.SinglePool() {
|
||||
return z.serverPools[0].DeleteObjectTags(ctx, bucket, object, opts)
|
||||
}
|
||||
|
||||
if !opts.NoLock {
|
||||
// Lock the object before deleting tags.
|
||||
lk := z.NewNSLock(bucket, object)
|
||||
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
}
|
||||
|
||||
opts.MetadataChg = true
|
||||
opts.NoLock = true
|
||||
idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
|
||||
return z.serverPools[idx].DeleteObjectTags(ctx, bucket, object, opts)
|
||||
return z.PutObjectTags(ctx, bucket, object, "", opts)
|
||||
}
|
||||
|
||||
// GetObjectTags - get object tags from an existing object
|
||||
@@ -2968,7 +3081,7 @@ func (z *erasureServerPools) GetObjectTags(ctx context.Context, bucket, object s
|
||||
return z.serverPools[0].GetObjectTags(ctx, bucket, object, opts)
|
||||
}
|
||||
|
||||
oi, _, err := z.getLatestObjectInfoWithIdx(ctx, bucket, object, opts)
|
||||
oi, err := z.GetObjectInfo(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+8
-2
@@ -166,6 +166,12 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
|
||||
if objAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
healInPool := er.HealObject
|
||||
if z, ok := objAPI.(*erasureServerPools); ok && !z.SinglePool() {
|
||||
healInPool = func(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
|
||||
return z.healObjectInPool(ctx, er, bucket, object, versionID, opts)
|
||||
}
|
||||
}
|
||||
|
||||
started := tracker.Started
|
||||
if started.IsZero() || started.Equal(timeSentinel) {
|
||||
@@ -419,7 +425,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
|
||||
var result healEntryResult
|
||||
fivs, err := entry.fileInfoVersions(bucket)
|
||||
if err != nil {
|
||||
res, err := er.HealObject(ctx, bucket, encodedEntryName, "",
|
||||
res, err := healInPool(ctx, bucket, encodedEntryName, "",
|
||||
madmin.HealOpts{
|
||||
ScanMode: scanMode,
|
||||
Remove: healDeleteDangling,
|
||||
@@ -455,7 +461,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
|
||||
continue
|
||||
}
|
||||
|
||||
res, err := er.HealObject(ctx, bucket, encodedEntryName,
|
||||
res, err := healInPool(ctx, bucket, encodedEntryName,
|
||||
version.VersionID, madmin.HealOpts{
|
||||
ScanMode: scanMode,
|
||||
Remove: healDeleteDangling,
|
||||
|
||||
+4
-6
@@ -51,9 +51,8 @@ func initGlobalGrid(ctx context.Context, eps EndpointServerPools) error {
|
||||
grid.ContextDialer(xhttp.DialContextWithLookupHost(lookupHost, xhttp.NewInternodeDialContext(rest.DefaultTimeout, globalTCPOptions.ForWebsocket()))),
|
||||
newCachedAuthToken(),
|
||||
&tls.Config{
|
||||
RootCAs: globalRootCAs,
|
||||
CipherSuites: crypto.TLSCiphers(),
|
||||
CurvePreferences: crypto.TLSCurveIDs(),
|
||||
RootCAs: globalRootCAs,
|
||||
CipherSuites: crypto.TLSCiphers(),
|
||||
}),
|
||||
Local: local,
|
||||
Hosts: hosts,
|
||||
@@ -84,9 +83,8 @@ func initGlobalLockGrid(ctx context.Context, eps EndpointServerPools) error {
|
||||
grid.ContextDialer(xhttp.DialContextWithLookupHost(lookupHost, xhttp.NewInternodeDialContext(rest.DefaultTimeout, globalTCPOptions.ForWebsocket()))),
|
||||
newCachedAuthToken(),
|
||||
&tls.Config{
|
||||
RootCAs: globalRootCAs,
|
||||
CipherSuites: crypto.TLSCiphers(),
|
||||
CurvePreferences: crypto.TLSCurveIDs(),
|
||||
RootCAs: globalRootCAs,
|
||||
CipherSuites: crypto.TLSCiphers(),
|
||||
}, grid.RouteLockPath),
|
||||
Local: local,
|
||||
Hosts: hosts,
|
||||
|
||||
+24
-2
@@ -992,6 +992,26 @@ func getClusterReplMRFFailedOperationsMD() MetricDescription {
|
||||
}
|
||||
}
|
||||
|
||||
func getClusterReplMRFDroppedOperationsMD() MetricDescription {
|
||||
return MetricDescription{
|
||||
Namespace: nodeMetricNamespace,
|
||||
Subsystem: replicationSubsystem,
|
||||
Name: "mrf_dropped_operations_total",
|
||||
Help: "Total number of replication MRF entries dropped since server start; entries may refer to the same object",
|
||||
Type: counterMetric,
|
||||
}
|
||||
}
|
||||
|
||||
func getClusterReplMRFDroppedBytesMD() MetricDescription {
|
||||
return MetricDescription{
|
||||
Namespace: nodeMetricNamespace,
|
||||
Subsystem: replicationSubsystem,
|
||||
Name: "mrf_dropped_bytes_total",
|
||||
Help: "Total known bytes of replication MRF entries dropped since server start; delete entries count as zero bytes",
|
||||
Type: counterMetric,
|
||||
}
|
||||
}
|
||||
|
||||
func getClusterRepCredentialErrorsMD(namespace MetricNamespace) MetricDescription {
|
||||
return MetricDescription{
|
||||
Namespace: namespace,
|
||||
@@ -2423,6 +2443,8 @@ func getReplicationNodeMetrics(opts MetricsGroupOpts) *MetricsGroupV2 {
|
||||
avgTransferRate,
|
||||
maxTransferRate,
|
||||
mrfCount,
|
||||
{Description: getClusterReplMRFDroppedOperationsMD(), Value: float64(qs.MRFStats.TotalDroppedCount)},
|
||||
{Description: getClusterReplMRFDroppedBytesMD(), Value: float64(qs.MRFStats.TotalDroppedBytes)},
|
||||
}
|
||||
}
|
||||
for ep, health := range globalBucketTargetSys.healthStats() {
|
||||
@@ -3314,10 +3336,10 @@ func getBucketUsageMetrics(opts MetricsGroupOpts) *MetricsGroupV2 {
|
||||
VariableLabels: map[string]string{"bucket": bucket},
|
||||
})
|
||||
|
||||
if quota != nil && quota.Quota > 0 {
|
||||
if quotaSize := getBucketQuotaSize(quota); quotaSize > 0 {
|
||||
metrics = append(metrics, MetricV2{
|
||||
Description: getBucketUsageQuotaTotalBytesMD(),
|
||||
Value: float64(quota.Quota),
|
||||
Value: float64(quotaSize),
|
||||
VariableLabels: map[string]string{"bucket": bucket},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -167,8 +167,8 @@ func loadClusterUsageBucketMetrics(ctx context.Context, m MetricValues, c *metri
|
||||
m.Set(usageBucketVersionsCount, float64(usage.VersionsCount), "bucket", bucket)
|
||||
m.Set(usageBucketDeleteMarkersCount, float64(usage.DeleteMarkersCount), "bucket", bucket)
|
||||
|
||||
if quota != nil && quota.Quota > 0 {
|
||||
m.Set(usageBucketQuotaTotalBytes, float64(quota.Quota), "bucket", bucket)
|
||||
if quotaSize := getBucketQuotaSize(quota); quotaSize > 0 {
|
||||
m.Set(usageBucketQuotaTotalBytes, float64(quotaSize), "bucket", bucket)
|
||||
}
|
||||
|
||||
for k, v := range usage.ObjectSizesHistogram {
|
||||
|
||||
@@ -35,6 +35,8 @@ const (
|
||||
replicationMaxQueuedCount = "max_queued_count"
|
||||
replicationMaxDataTransferRate = "max_data_transfer_rate"
|
||||
replicationRecentBacklogCount = "recent_backlog_count"
|
||||
replicationMRFDroppedOperations = "mrf_dropped_operations_total"
|
||||
replicationMRFDroppedBytes = "mrf_dropped_bytes_total"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -64,6 +66,10 @@ var (
|
||||
"Maximum replication data transfer rate in bytes/sec seen since server start")
|
||||
replicationRecentBacklogCountMD = NewGaugeMD(replicationRecentBacklogCount,
|
||||
"Total number of objects seen in replication backlog in the last 5 minutes")
|
||||
replicationMRFDroppedOperationsMD = NewCounterMD(replicationMRFDroppedOperations,
|
||||
"Total number of replication MRF entries dropped since server start; entries may refer to the same object")
|
||||
replicationMRFDroppedBytesMD = NewCounterMD(replicationMRFDroppedBytes,
|
||||
"Total known bytes of replication MRF entries dropped since server start; delete entries count as zero bytes")
|
||||
)
|
||||
|
||||
// loadClusterReplicationMetrics - `MetricsLoaderFn` for cluster replication metrics
|
||||
@@ -96,6 +102,8 @@ func loadClusterReplicationMetrics(ctx context.Context, m MetricValues, c *metri
|
||||
m.Set(replicationMaxDataTransferRate, tots.Peak)
|
||||
}
|
||||
m.Set(replicationRecentBacklogCount, float64(qs.MRFStats.LastFailedCount))
|
||||
m.Set(replicationMRFDroppedOperations, float64(qs.MRFStats.TotalDroppedCount))
|
||||
m.Set(replicationMRFDroppedBytes, float64(qs.MRFStats.TotalDroppedBytes))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -342,6 +342,8 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
|
||||
replicationMaxQueuedCountMD,
|
||||
replicationMaxDataTransferRateMD,
|
||||
replicationRecentBacklogCountMD,
|
||||
replicationMRFDroppedOperationsMD,
|
||||
replicationMRFDroppedBytesMD,
|
||||
},
|
||||
loadClusterReplicationMetrics,
|
||||
)
|
||||
|
||||
@@ -99,10 +99,13 @@ type ObjectOptions struct {
|
||||
ReplicationSourceTaggingTimestamp time.Time // set if MinIOSourceTaggingTimestamp received
|
||||
ReplicationSourceLegalholdTimestamp time.Time // set if MinIOSourceObjectLegalholdTimestamp received
|
||||
ReplicationSourceRetentionTimestamp time.Time // set if MinIOSourceObjectRetentionTimestamp received
|
||||
ReplicaLockReconcile bool // set for a trusted SSE-C replica full write/completion: re-order Object Lock against the destination version read under the write lock (single erasure set; see pgsty/silo#133)
|
||||
ReplicaLockReconcile bool // re-order a trusted replica write against the destination version under the object write lock
|
||||
DeletePrefix bool // set true to enforce a prefix deletion, only application for DeleteObject API,
|
||||
DeletePrefixObject bool // set true when object's erasure set is resolvable by object name (using getHashedSetIndex)
|
||||
|
||||
// Pools-layer version resolver; the caller holds the shared object lock.
|
||||
replicaObjectInfo GetObjectInfoFn
|
||||
|
||||
Speedtest bool // object call specifically meant for SpeedTest code, set to 'true' when invoked by SpeedtestHandler.
|
||||
|
||||
// Use the maximum parity (N/2), used when saving server configuration files
|
||||
@@ -132,8 +135,8 @@ type ObjectOptions struct {
|
||||
// when looking up a version by fi.VersionID
|
||||
InclFreeVersions bool
|
||||
// SkipFreeVersion skips adding a free version when a tiered version is
|
||||
// being 'replaced'
|
||||
// Note: Used only when a tiered object is being expired.
|
||||
// being replaced. Used when expiring tiered content or retiring a copy
|
||||
// whose tier reference is still owned by another copy.
|
||||
SkipFreeVersion bool
|
||||
|
||||
MetadataChg bool // is true if it is a metadata update operation.
|
||||
|
||||
@@ -433,7 +433,16 @@ func putOptsFromHeaders(ctx context.Context, hdr http.Header, metadata map[strin
|
||||
if err != nil {
|
||||
return ObjectOptions{}, err
|
||||
}
|
||||
sseKms, err := encrypt.NewSSEKMS(keyID, context)
|
||||
// kms.Context implements encoding.TextMarshaler, so handing it to the
|
||||
// SDK's interface{} parameter would serialize the context as a JSON
|
||||
// string, which the receiving ParseHTTP rejects; a nil Context is a
|
||||
// typed nil there and would be sent as "{}". Pass a plain map, or
|
||||
// nothing when no context was requested.
|
||||
var sdkContext any
|
||||
if context != nil {
|
||||
sdkContext = map[string]string(context)
|
||||
}
|
||||
sseKms, err := encrypt.NewSSEKMS(keyID, sdkContext)
|
||||
if err != nil {
|
||||
return ObjectOptions{}, err
|
||||
}
|
||||
|
||||
@@ -62,6 +62,13 @@ func copyChecksumRequest(t *testing.T, apiRouter http.Handler, credentials auth.
|
||||
t.Fatalf("failed to build CopyObject request: %v", err)
|
||||
}
|
||||
req.Header.Set(xhttp.AmzCopySource, SlashSeparator+pathJoin(bucket, source))
|
||||
// Re-sign so x-amz-copy-source is covered by the signature, as real S3
|
||||
// clients send it; the verifier rejects unsigned x-amz-* headers.
|
||||
if credentials.AccessKey != "" && credentials.SecretKey != "" {
|
||||
if err := signRequestV4(req, credentials.AccessKey, credentials.SecretKey); err != nil {
|
||||
t.Fatalf("failed to re-sign CopyObject request: %v", err)
|
||||
}
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
return rec
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
)
|
||||
|
||||
// enableBucketObjectLock puts a lock-enabled configuration on an existing
|
||||
// bucket so checkPutObjectLockAllowed accepts a legal-hold header instead of
|
||||
// rejecting the request with ErrInvalidBucketObjectLockConfiguration.
|
||||
func enableBucketObjectLock(t *testing.T, bucket string) {
|
||||
t.Helper()
|
||||
meta, err := globalBucketMetadataSys.Get(bucket)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to read bucket metadata for %s: %v", bucket, err)
|
||||
}
|
||||
updated := meta
|
||||
updated.ObjectLockConfigXML = enabledBucketObjectLockConfig
|
||||
updated.VersioningConfigXML = enabledBucketVersioningConfig
|
||||
// The XML alone is not enough: BucketMetadata keeps a parsed copy that the
|
||||
// lookups actually read, and it is only populated by parseAllConfigs.
|
||||
if err := updated.parseAllConfigs(t.Context(), newObjectLayerFn()); err != nil {
|
||||
t.Fatalf("unable to parse bucket metadata for %s: %v", bucket, err)
|
||||
}
|
||||
globalBucketMetadataSys.Set(bucket, updated)
|
||||
}
|
||||
|
||||
// Exercise #165 and #166 together, including hold OFF, default retention and
|
||||
// explicit subsecond retention. Ordinary copies must not inherit source locks.
|
||||
func TestAPIFederatedCopyObjectLockParity(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
endpoints: []string{"CopyObject", "PutObject", "HeadObject", "GetObject"},
|
||||
objAPITest: func(obj ObjectLayer, instanceType, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
testKMS, err := kms.NewBuiltin(federationTestKMSKeyID, bytes.Repeat([]byte{0x58}, 32))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previousKMS := GlobalKMS
|
||||
GlobalKMS = testKMS
|
||||
defer func() { GlobalKMS = previousKMS }()
|
||||
remoteBucket, capture, cleanup := setupCopyObjectFederation(t, obj, router, instanceType, bucket)
|
||||
defer cleanup()
|
||||
enableBucketObjectLock(t, bucket)
|
||||
until := UTCNow().Add(7 * 24 * time.Hour).Truncate(time.Second).Add(789 * time.Millisecond).Format(time.RFC3339Nano)
|
||||
for _, sourceType := range []string{"plain", "s3"} {
|
||||
source := sourceType + "-held-source"
|
||||
headers := federationSSEHeaders(sourceType, 0, false)
|
||||
headers[xhttp.AmzObjectLockLegalHold] = "ON"
|
||||
putCopyChecksumSource(t, router, cred, bucket, source, []byte("held source"), headers)
|
||||
for _, tc := range []struct {
|
||||
name, hold, defaultMode, explicitMode string
|
||||
replace bool
|
||||
}{
|
||||
{name: "on", hold: "ON"},
|
||||
{name: "off", hold: "OFF"},
|
||||
{name: "no source inheritance"},
|
||||
{name: "on with default", hold: "ON", defaultMode: "GOVERNANCE"},
|
||||
{name: "off with default", hold: "OFF", defaultMode: "COMPLIANCE"},
|
||||
{name: "explicit retention", explicitMode: "GOVERNANCE"},
|
||||
{name: "hold and explicit retention", hold: "ON", explicitMode: "COMPLIANCE"},
|
||||
{name: "replace metadata", hold: "ON", explicitMode: "GOVERNANCE", replace: true},
|
||||
} {
|
||||
t.Run(instanceType+"/"+sourceType+"/"+tc.name, func(t *testing.T) {
|
||||
setTestBucketDefaultRetention(t, bucket, tc.defaultMode)
|
||||
setTestBucketDefaultRetention(t, remoteBucket, tc.defaultMode)
|
||||
headers := map[string]string{}
|
||||
if tc.hold != "" {
|
||||
headers[xhttp.AmzObjectLockLegalHold] = tc.hold
|
||||
}
|
||||
if tc.explicitMode != "" {
|
||||
headers[xhttp.AmzObjectLockMode] = tc.explicitMode
|
||||
headers[xhttp.AmzObjectLockRetainUntilDate] = until
|
||||
}
|
||||
if tc.replace {
|
||||
headers[xhttp.AmzMetadataDirective] = "REPLACE"
|
||||
headers["X-Amz-Meta-Origin"] = "replacement"
|
||||
}
|
||||
capture.mu.Lock()
|
||||
capture.headers = nil
|
||||
capture.mu.Unlock()
|
||||
for _, destinationBucket := range []string{bucket, remoteBucket} {
|
||||
destination := sourceType + "-copy-" + strings.ReplaceAll(tc.name, " ", "-")
|
||||
rec := federatedCopyRequest(t, router, cred, bucket, source, destinationBucket, destination, headers)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("copy to %s: %d %s", destinationBucket, rec.Code, rec.Body.String())
|
||||
}
|
||||
info, err := obj.GetObjectInfo(t.Context(), destinationBucket, destination, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := objectlock.GetObjectLegalHoldMeta(info.UserDefined).Status; string(got) != tc.hold {
|
||||
t.Errorf("stored hold = %q, want %q", got, tc.hold)
|
||||
}
|
||||
retention := objectlock.GetObjectRetentionMeta(info.UserDefined)
|
||||
mode := tc.explicitMode
|
||||
if mode == "" {
|
||||
mode = tc.defaultMode
|
||||
}
|
||||
if string(retention.Mode) != mode {
|
||||
t.Errorf("stored retention = %q, want %q", retention.Mode, mode)
|
||||
}
|
||||
if tc.explicitMode != "" && retention.RetainUntilDate.Format(time.RFC3339Nano) != until {
|
||||
t.Errorf("retention date lost precision: %s, want %s", retention.RetainUntilDate, until)
|
||||
}
|
||||
for key := range info.UserDefined {
|
||||
if stringsHasPrefixFold(key, "X-Amz-Meta-X-Amz-Object-Lock-") {
|
||||
t.Errorf("lock state became user metadata: %s", key)
|
||||
}
|
||||
}
|
||||
if tc.replace && info.UserDefined["X-Amz-Meta-Origin"] != "replacement" {
|
||||
t.Errorf("replacement metadata was lost: %v", info.UserDefined)
|
||||
}
|
||||
}
|
||||
typed, asMetadata := capture.legalHoldHeaders()
|
||||
if len(asMetadata) != 0 || (tc.hold != "" && strings.Join(typed, "") != tc.hold) || (tc.hold == "" && len(typed) != 0) {
|
||||
t.Errorf("forwarded hold headers = %v, metadata = %v; want %q", typed, asMetadata, tc.hold)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// legalHoldHeaders returns the forwarded legal-hold headers the remote saw,
|
||||
// separated into the real Object Lock header and the user-metadata spelling
|
||||
// minio-go produces for an unrecognized UserMetadata key.
|
||||
func (c *federationRemoteCapture) legalHoldHeaders() (typed, asMetadata []string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
metaKey := "X-Amz-Meta-" + xhttp.AmzObjectLockLegalHold
|
||||
for _, h := range c.headers {
|
||||
for k, v := range h {
|
||||
switch {
|
||||
case strings.EqualFold(k, xhttp.AmzObjectLockLegalHold):
|
||||
typed = append(typed, strings.Join(v, ","))
|
||||
case strings.EqualFold(k, metaKey):
|
||||
asMetadata = append(asMetadata, strings.Join(v, ","))
|
||||
}
|
||||
}
|
||||
}
|
||||
return typed, asMetadata
|
||||
}
|
||||
|
||||
// TestAPIFederatedCopyObjectLegalHold drives the legacy etcd federation branch
|
||||
// of CopyObjectHandler with an explicit legal hold on the copy.
|
||||
//
|
||||
// Before the fix the resolved hold was forwarded inside
|
||||
// PutObjectOptions.UserMetadata. minio-go's Header() prefixes every
|
||||
// UserMetadata key it does not recognize with "x-amz-meta-", and
|
||||
// x-amz-object-lock-legal-hold is in neither supportedHeaders nor isAmzHeader,
|
||||
// so the hold reached the remote as X-Amz-Meta-X-Amz-Object-Lock-Legal-Hold.
|
||||
// The destination stored no hold and the copy still answered 200 -- a silent
|
||||
// loss of a WORM control (#166). Retention requested on the same copy survived,
|
||||
// which is what made it easy to miss.
|
||||
func TestAPIFederatedCopyObjectLegalHold(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPIFederatedCopyObjectLegalHold,
|
||||
endpoints: []string{"CopyObject", "PutObject", "HeadObject", "GetObject"},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPIFederatedCopyObjectLegalHold(objectAPI ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
data := []byte("federated copy with a legal hold")
|
||||
srcObject := "federation/legal-hold-source"
|
||||
putCopyChecksumSource(t, apiRouter, credentials, bucketName, srcObject, data, nil)
|
||||
|
||||
remoteBucket, capture, cleanup := setupCopyObjectFederation(t, objectAPI, apiRouter, instanceType, bucketName)
|
||||
defer cleanup()
|
||||
|
||||
// Both roles read bucket metadata from the shared backend in this fixture,
|
||||
// so one configuration covers the proxy's own check and the remote's.
|
||||
enableBucketObjectLock(t, bucketName)
|
||||
enableBucketObjectLock(t, remoteBucket)
|
||||
|
||||
dstObject := "federation/legal-hold-destination"
|
||||
rec := federatedCopyRequest(t, apiRouter, credentials, bucketName, srcObject, remoteBucket, dstObject,
|
||||
map[string]string{xhttp.AmzObjectLockLegalHold: string(objectlock.LegalHoldOn)})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: federated CopyObject with a legal hold failed: %d %s",
|
||||
instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// The wire is the point: the hold must arrive as the Object Lock header,
|
||||
// never as user metadata. A 200 with the metadata spelling is exactly the
|
||||
// silent loss this test exists for.
|
||||
typed, asMetadata := capture.legalHoldHeaders()
|
||||
if len(asMetadata) != 0 {
|
||||
t.Fatalf("%s: legal hold forwarded as user metadata %v; the destination stores no hold",
|
||||
instanceType, asMetadata)
|
||||
}
|
||||
if len(typed) == 0 {
|
||||
t.Fatalf("%s: no %s header reached the remote deployment", instanceType, xhttp.AmzObjectLockLegalHold)
|
||||
}
|
||||
for _, got := range typed {
|
||||
if !strings.EqualFold(got, string(objectlock.LegalHoldOn)) {
|
||||
t.Fatalf("%s: forwarded legal hold = %q, want %q", instanceType, got, objectlock.LegalHoldOn)
|
||||
}
|
||||
}
|
||||
|
||||
// And it must actually be stored on the destination version.
|
||||
oi, err := objectAPI.GetObjectInfo(t.Context(), remoteBucket, dstObject, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: unable to stat the federated copy destination: %v", instanceType, err)
|
||||
}
|
||||
if hold := objectlock.GetObjectLegalHoldMeta(oi.UserDefined); hold.Status != objectlock.LegalHoldOn {
|
||||
t.Fatalf("%s: destination legal hold = %q, want %q (metadata: %v)",
|
||||
instanceType, hold.Status, objectlock.LegalHoldOn, oi.UserDefined)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/amztime"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
sse "github.com/minio/minio/internal/bucket/encryption"
|
||||
"github.com/minio/minio/internal/event"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/pubsub"
|
||||
)
|
||||
|
||||
func TestAPIFederatedCopyObjectVersionAndEvent(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
endpoints: []string{"CopyObject", "PutObject", "GetObject", "HeadObject"},
|
||||
objAPITest: func(obj ObjectLayer, instanceType, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
testKMS, err := kms.NewBuiltin(federationTestKMSKeyID, bytes.Repeat([]byte{0x58}, 32))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previousKMS := GlobalKMS
|
||||
GlobalKMS = testKMS
|
||||
defer func() { GlobalKMS = previousKMS }()
|
||||
restore := setCopyChecksumCompression(true)
|
||||
defer restore()
|
||||
remoteBucket, _, cleanup := setupCopyObjectFederation(t, obj, router, instanceType, bucket)
|
||||
defer cleanup()
|
||||
enableBucketObjectLock(t, remoteBucket)
|
||||
events := make(chan event.Event, 8)
|
||||
done := make(chan struct{})
|
||||
defer close(done)
|
||||
if err := globalHTTPListen.Subscribe(pubsub.MaskFromMaskable(event.ObjectCreatedCopy), events, done, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, kind := range []string{"plain", "compressed", "encrypted"} {
|
||||
t.Run(instanceType+"/"+kind, func(t *testing.T) {
|
||||
data := []byte("logical object size")
|
||||
source := kind + "-source.bin"
|
||||
var headers map[string]string
|
||||
if kind == "compressed" {
|
||||
data = bytes.Repeat(data, 8192)
|
||||
source = kind + "-source.txt"
|
||||
}
|
||||
if kind == "encrypted" {
|
||||
headers = federationSSEHeaders("s3", 0, false)
|
||||
}
|
||||
putCopyChecksumSource(t, router, cred, bucket, source, data, headers)
|
||||
destination := "result/" + kind + " with space.bin"
|
||||
rec := federatedCopyRequest(t, router, cred, bucket, source, remoteBucket, destination, nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("copy: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
versionID := strings.Join(rec.Header()[xhttp.AmzVersionID], "")
|
||||
if versionID == "" {
|
||||
t.Error("copy response omitted destination version ID")
|
||||
} else {
|
||||
info, err := obj.GetObjectInfo(t.Context(), remoteBucket, destination, ObjectOptions{VersionID: versionID})
|
||||
if err != nil || info.VersionID != versionID {
|
||||
t.Fatalf("response does not name the written version: %v, %q", err, info.VersionID)
|
||||
}
|
||||
var response CopyObjectResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := amztime.ISO8601Format(info.ModTime.UTC()); response.LastModified != want {
|
||||
t.Errorf("copy time = %q, want written version time %q", response.LastModified, want)
|
||||
}
|
||||
}
|
||||
select {
|
||||
case evt := <-events:
|
||||
key, err := url.QueryUnescape(evt.S3.Object.Key)
|
||||
if err != nil || key != destination || evt.S3.Bucket.Name != remoteBucket || evt.S3.Object.Size != int64(len(data)) || evt.S3.Object.VersionID == "" || evt.S3.Object.VersionID != versionID {
|
||||
t.Errorf("copy event does not describe the written object: %+v", evt.S3)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("copy event was not emitted")
|
||||
}
|
||||
})
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func TestAPIFederatedCopyObjectDestinationSSEDefaults(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
endpoints: []string{"CopyObject", "PutObject", "GetObject", "HeadObject"},
|
||||
objAPITest: func(obj ObjectLayer, instanceType, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
data := []byte("destination chooses default encryption")
|
||||
putCopyChecksumSource(t, router, cred, bucket, "source", data, nil)
|
||||
testKMS, err := kms.NewBuiltin(federationTestKMSKeyID, bytes.Repeat([]byte{0x58}, 32))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previousKMS, previousAuto := GlobalKMS, globalAutoEncryption
|
||||
GlobalKMS, globalAutoEncryption = testKMS, true
|
||||
defer func() { GlobalKMS, globalAutoEncryption = previousKMS, previousAuto }()
|
||||
remoteBucket, capture, cleanup := setupCopyObjectFederationRemote(t, obj, router, instanceType, bucket, true)
|
||||
defer cleanup()
|
||||
destinationConfig, err := sse.ParseBucketSSEConfig(strings.NewReader(`<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>aws:kms</SSEAlgorithm><KMSMasterKeyID>destination-bucket-key</KMSMasterKeyID></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, kind := range []string{"plain", "s3", "kms", "kms-default", "c"} {
|
||||
t.Run(instanceType+"/"+kind, func(t *testing.T) {
|
||||
var headers map[string]string
|
||||
if kind == "kms-default" {
|
||||
headers = map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionKMS}
|
||||
} else {
|
||||
headers = federationSSEHeaders(kind, 0x22, false)
|
||||
}
|
||||
rec := federatedCopyRequest(t, router, cred, bucket, "source", remoteBucket, kind, headers)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("copy: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
capture.mu.Lock()
|
||||
forwarded := capture.headers[len(capture.headers)-1].Clone()
|
||||
capture.mu.Unlock()
|
||||
if kind == "plain" {
|
||||
if forwarded.Get(xhttp.AmzServerSideEncryption) != "" {
|
||||
t.Errorf("proxy injected SSE %q into a request with no client SSE", forwarded.Get(xhttp.AmzServerSideEncryption))
|
||||
}
|
||||
// With nothing injected, the KMS encryption the destination
|
||||
// stores can only be the remote applying its own defaults.
|
||||
info, err := obj.GetObjectInfo(t.Context(), remoteBucket, kind, ObjectOptions{})
|
||||
if err != nil || federationStoredSSE(info.UserDefined) != "kms" {
|
||||
t.Errorf("remote destination did not apply its own auto-encryption: %v, %v", err, info.UserDefined)
|
||||
}
|
||||
}
|
||||
// Replay the actual inbound headers against an independent bucket
|
||||
// configuration. No handler flips shared globals while serving.
|
||||
destinationConfig.Apply(forwarded, sse.ApplyOptions{})
|
||||
wantKey := headers[xhttp.AmzServerSideEncryptionKmsID]
|
||||
if kind == "plain" {
|
||||
wantKey = "destination-bucket-key"
|
||||
}
|
||||
if got := forwarded.Get(xhttp.AmzServerSideEncryptionKmsID); got != wantKey {
|
||||
t.Errorf("destination selected key %q, want %q", got, wantKey)
|
||||
}
|
||||
})
|
||||
}
|
||||
// A local destination still inherits the server's auto-encryption.
|
||||
rec := federatedCopyRequest(t, router, cred, bucket, "source", bucket, "local-copy", nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("local copy: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
info, err := obj.GetObjectInfo(t.Context(), bucket, "local-copy", ObjectOptions{})
|
||||
if err != nil || federationStoredSSE(info.UserDefined) != "kms" {
|
||||
t.Errorf("local copy lost auto-encryption: %v, %v", err, info.UserDefined)
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,956 @@
|
||||
// Copyright (c) 2015-2025 MinIO, Inc.
|
||||
// Copyright (c) 2025-2026 PGSTY
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
miniogo "github.com/minio/minio-go/v7"
|
||||
miniocredentials "github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio/internal/amztime"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/config/dns"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
)
|
||||
|
||||
// federationRemoteCapture records the exact request headers the remote
|
||||
// deployment receives on each forwarded request, so a test can assert what
|
||||
// actually crossed the wire rather than what the destination ends up storing.
|
||||
type federationRemoteCapture struct {
|
||||
mu sync.Mutex
|
||||
headers []http.Header
|
||||
}
|
||||
|
||||
type federationResponseFilter struct {
|
||||
http.ResponseWriter
|
||||
filter func(http.Header)
|
||||
}
|
||||
|
||||
func (w federationResponseFilter) WriteHeader(status int) {
|
||||
w.filter(w.Header())
|
||||
w.ResponseWriter.WriteHeader(status)
|
||||
}
|
||||
|
||||
func (c *federationRemoteCapture) record(h http.Header) {
|
||||
c.mu.Lock()
|
||||
c.headers = append(c.headers, h.Clone())
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// reservedKeys returns every reserved-prefix header key seen across all
|
||||
// forwarded requests.
|
||||
func (c *federationRemoteCapture) reservedKeys() []string {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
var keys []string
|
||||
for _, h := range c.headers {
|
||||
for k := range h {
|
||||
if stringsHasPrefixFold(k, ReservedMetadataPrefix) {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// setupCopyObjectFederation makes the current process play both federation
|
||||
// roles for a whole-object CopyObject. The destination bucket really exists in
|
||||
// the shared backend, but the proxy's own bucket lookup is told it lives on a
|
||||
// remote deployment, so CopyObjectHandler takes the legacy etcd federation
|
||||
// branch and forwards the write through getRemoteInstanceClient and minio-go
|
||||
// into a second HTTP endpoint that serves the real PutObjectHandler. That
|
||||
// endpoint records the inbound headers first so a caller can inspect the wire.
|
||||
//
|
||||
// GetBucketLocation is deliberately left unregistered on that endpoint: the
|
||||
// remoteBucketObjectLayer reports remoteBucket as missing (so the proxy takes
|
||||
// the federation branch), which would also fail a real GetBucketLocation the
|
||||
// minio-go client probes for. Leaving it unregistered makes the probe fall back
|
||||
// to the default region, exactly as TestAPIFederatedCopyObjectPartChecksum does.
|
||||
func setupCopyObjectFederation(t *testing.T, objectAPI ObjectLayer, apiRouter http.Handler,
|
||||
instanceType, srcBucket string, responseFilters ...func(http.Header),
|
||||
) (remoteBucket string, capture *federationRemoteCapture, cleanup func()) {
|
||||
t.Helper()
|
||||
return setupCopyObjectFederationRemote(t, objectAPI, apiRouter, instanceType, srcBucket, false, responseFilters...)
|
||||
}
|
||||
|
||||
// setupCopyObjectFederationTLS is setupCopyObjectFederation with a TLS remote
|
||||
// endpoint and globalIsTLS set, so an SSE-C copy is accepted on both hops: the
|
||||
// proxy's own SSE-C transport gate and the minio-go client's SSE-C policy. The
|
||||
// proxy client is built exactly as in production, trusting the test
|
||||
// certificate for the duration.
|
||||
func setupCopyObjectFederationTLS(t *testing.T, objectAPI ObjectLayer, apiRouter http.Handler,
|
||||
instanceType, srcBucket string,
|
||||
) (remoteBucket string, cleanup func()) {
|
||||
t.Helper()
|
||||
remoteBucket, _, cleanup = setupCopyObjectFederationRemote(t, objectAPI, apiRouter, instanceType, srcBucket, true)
|
||||
return remoteBucket, cleanup
|
||||
}
|
||||
|
||||
func setupCopyObjectFederationRemote(t *testing.T, objectAPI ObjectLayer, apiRouter http.Handler,
|
||||
instanceType, srcBucket string, secure bool, responseFilters ...func(http.Header),
|
||||
) (remoteBucket string, capture *federationRemoteCapture, cleanup func()) {
|
||||
t.Helper()
|
||||
remoteBucket = getRandomBucketName()
|
||||
if err := objectAPI.MakeBucket(t.Context(), remoteBucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatalf("%s: unable to create the remote bucket: %v", instanceType, err)
|
||||
}
|
||||
|
||||
capture = &federationRemoteCapture{}
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
capture.record(r.Header)
|
||||
if len(responseFilters) != 0 && r.Method == http.MethodPut {
|
||||
w = federationResponseFilter{ResponseWriter: w, filter: responseFilters[0]}
|
||||
}
|
||||
apiRouter.ServeHTTP(w, r)
|
||||
})
|
||||
var remote *httptest.Server
|
||||
if secure {
|
||||
remote = httptest.NewTLSServer(handler)
|
||||
} else {
|
||||
remote = httptest.NewServer(handler)
|
||||
}
|
||||
host, port, _ := strings.Cut(remote.Listener.Addr().String(), ":")
|
||||
|
||||
globalObjLayerMutex.Lock()
|
||||
previousLayer := globalObjectAPI
|
||||
globalObjectAPI = remoteBucketObjectLayer{ObjectLayer: previousLayer, remoteBucket: remoteBucket}
|
||||
globalObjLayerMutex.Unlock()
|
||||
previousDNS, previousFederation, previousIPs := globalDNSConfig, globalBucketFederation, globalDomainIPs
|
||||
previousTLS, previousClient := globalIsTLS, getRemoteInstanceClient
|
||||
globalDNSConfig = federationTestDNS{records: map[string][]dns.SrvRecord{
|
||||
srcBucket: {{Host: host, Port: json.Number(port)}},
|
||||
remoteBucket: {{Host: host, Port: json.Number(port)}},
|
||||
}}
|
||||
// Every DNS record resolves to this process, so the bucket forwarding
|
||||
// middleware always serves locally and only the handler proxies.
|
||||
globalDomainIPs = set.CreateStringSet(remote.Listener.Addr().String())
|
||||
globalBucketFederation = true
|
||||
if secure {
|
||||
globalIsTLS = true
|
||||
transport := remote.Client().Transport
|
||||
getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core, error) {
|
||||
cred := getReqAccessCred(r, globalSite.Region())
|
||||
core, err := miniogo.NewCore(host, &miniogo.Options{
|
||||
Creds: miniocredentials.NewStaticV4(cred.AccessKey, cred.SecretKey, ""),
|
||||
Secure: true,
|
||||
Transport: federatedWriteTransport{transport},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
core.SetAppInfo(federatedInternalAppName, ReleaseTag)
|
||||
return core, nil
|
||||
}
|
||||
}
|
||||
|
||||
cleanup = func() {
|
||||
remote.Close()
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = previousLayer
|
||||
globalObjLayerMutex.Unlock()
|
||||
globalDNSConfig, globalBucketFederation, globalDomainIPs = previousDNS, previousFederation, previousIPs
|
||||
globalIsTLS, getRemoteInstanceClient = previousTLS, previousClient
|
||||
}
|
||||
return remoteBucket, capture, cleanup
|
||||
}
|
||||
|
||||
// federatedCopyRequest drives a whole-object CopyObject at the proxy that
|
||||
// forwards across deployments, and returns the recorded response.
|
||||
func federatedCopyRequest(t *testing.T, apiRouter http.Handler, credentials auth.Credentials,
|
||||
srcBucket, srcObject, dstBucket, dstObject string, headers map[string]string,
|
||||
) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, getCopyObjectURL("", dstBucket, dstObject),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, headers)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to build federated CopyObject request: %v", err)
|
||||
}
|
||||
req.Header.Set(xhttp.AmzCopySource, SlashSeparator+pathJoin(srcBucket, srcObject))
|
||||
// Re-sign so x-amz-copy-source is covered by the signature, as real S3
|
||||
// clients send it; the verifier rejects unsigned x-amz-* headers.
|
||||
if credentials.AccessKey != "" && credentials.SecretKey != "" {
|
||||
if err := signRequestV4(req, credentials.AccessKey, credentials.SecretKey); err != nil {
|
||||
t.Fatalf("failed to re-sign federated CopyObject request: %v", err)
|
||||
}
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
func TestAPIFederatedCopyObjectRejectsRawSSECReplica(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
endpoints: []string{"CopyObject", "PutObject"},
|
||||
objAPITest: func(obj ObjectLayer, instanceType, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
remoteBucket, capture, cleanup := setupCopyObjectFederationRemote(t, obj, router, instanceType, bucket, true)
|
||||
defer cleanup()
|
||||
for _, body := range []string{"", "raw SSE-C replica"} {
|
||||
for _, withKey := range []bool{false, true} {
|
||||
t.Run("size="+strconv.Itoa(len(body))+"/key="+strconv.FormatBool(withKey), func(t *testing.T) {
|
||||
putCopyChecksumSource(t, router, cred, bucket, "source", []byte(body), federationSSEHeaders("c", 0x11, false))
|
||||
headers := map[string]string{
|
||||
xhttp.MinIOSourceReplicationRequest: "true",
|
||||
xhttp.AmzBucketReplicationStatus: "REPLICA",
|
||||
}
|
||||
if withKey {
|
||||
maps.Copy(headers, federationSSEHeaders("c", 0x11, true))
|
||||
}
|
||||
destination := "destination-" + strconv.Itoa(len(body)) + "-" + strconv.FormatBool(withKey)
|
||||
rec := federatedCopyRequest(t, router, cred, bucket, "source", remoteBucket, destination, headers)
|
||||
var response APIErrorResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec.Code != http.StatusNotImplemented || response.Code != "NotImplemented" ||
|
||||
!strings.Contains(response.Message, "federated raw SSE-C replica CopyObject") {
|
||||
t.Errorf("copy = %d %s, want explicit 501 rejection", rec.Code, rec.Body.String())
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(t.Context(), remoteBucket, destination, ObjectOptions{}); !isErrObjectNotFound(err) {
|
||||
t.Errorf("rejected copy created a destination: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
capture.mu.Lock()
|
||||
defer capture.mu.Unlock()
|
||||
if len(capture.headers) != 0 {
|
||||
t.Errorf("rejected copies made %d remote requests", len(capture.headers))
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// TestAPIFederatedCopyObjectInlineSource drives the legacy etcd federation
|
||||
// branch of CopyObjectHandler end to end for a source object stored inline.
|
||||
// Such a source carries x-minio-internal-inline-data in its stored metadata,
|
||||
// which the remote deployment rejects as a reserved-prefix header. Before the
|
||||
// fix the forwarded write failed with 400 InvalidArgument; the copy must now
|
||||
// succeed and forward no reserved-prefix metadata at all.
|
||||
func TestAPIFederatedCopyObjectInlineSource(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPIFederatedCopyObjectInlineSource,
|
||||
endpoints: []string{"CopyObject", "PutObject", "HeadObject", "GetObject"},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPIFederatedCopyObjectInlineSource(objectAPI ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
// A small object is stored inline, so its stored metadata carries
|
||||
// x-minio-internal-inline-data. A user metadata key rides along to prove
|
||||
// the fix strips only the reserved class, never ordinary metadata.
|
||||
data := []byte("federated inline copy 26b!")
|
||||
srcObject := "federation/inline-source"
|
||||
putCopyChecksumSource(t, apiRouter, credentials, bucketName, srcObject, data,
|
||||
map[string]string{"X-Amz-Meta-Origin": "inline-source"})
|
||||
|
||||
remoteBucket, capture, cleanup := setupCopyObjectFederation(t, objectAPI, apiRouter, instanceType, bucketName)
|
||||
defer cleanup()
|
||||
|
||||
dstObject := "federation/inline-destination"
|
||||
rec := federatedCopyRequest(t, apiRouter, credentials, bucketName, srcObject, remoteBucket, dstObject, nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: federated CopyObject of an inline source failed: %d %s",
|
||||
instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// No reserved-prefix header may reach the remote deployment on any of the
|
||||
// forwarded requests.
|
||||
if leaked := capture.reservedKeys(); len(leaked) != 0 {
|
||||
t.Fatalf("%s: forwarded reserved metadata to the remote: %v", instanceType, leaked)
|
||||
}
|
||||
|
||||
// The destination object must be readable and byte-identical, and it must
|
||||
// still carry the copied user metadata.
|
||||
gr, err := objectAPI.GetObjectNInfo(t.Context(), remoteBucket, dstObject, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: unable to read the federated copy destination: %v", instanceType, err)
|
||||
}
|
||||
got, err := io.ReadAll(gr)
|
||||
closeErr := gr.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("%s: reading the federated copy destination failed: %v", instanceType, err)
|
||||
}
|
||||
if closeErr != nil {
|
||||
t.Fatalf("%s: closing the federated copy destination failed: %v", instanceType, closeErr)
|
||||
}
|
||||
if !bytes.Equal(got, data) {
|
||||
t.Fatalf("%s: federated copy destination body = %q, want %q", instanceType, got, data)
|
||||
}
|
||||
if origin, ok := gr.ObjInfo.UserDefined["X-Amz-Meta-Origin"]; !ok || origin != "inline-source" {
|
||||
t.Fatalf("%s: destination lost copied user metadata: %v", instanceType, gr.ObjInfo.UserDefined)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPIFederatedCopyObjectRequestedChecksum drives the legacy etcd federation
|
||||
// branch of CopyObjectHandler and verifies that a server-side checksum is both
|
||||
// returned and persisted, matching the local CopyObject path. Before the fix
|
||||
// the federated copy forwarded the write without asking for a checksum and
|
||||
// discarded whatever the remote returned, so the response carried an empty
|
||||
// checksum even when the client requested one (#99).
|
||||
//
|
||||
// The no-algorithm case is included deliberately: a checksum-less source gains
|
||||
// the S3 default CRC-64NVME full-object checksum on the local path, so the
|
||||
// federated path must return the same. "No requested algorithm" does not mean
|
||||
// "no checksum".
|
||||
func TestAPIFederatedCopyObjectRequestedChecksum(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPIFederatedCopyObjectRequestedChecksum,
|
||||
endpoints: []string{"CopyObject", "PutObject", "HeadObject", "GetObject"},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPIFederatedCopyObjectRequestedChecksum(objectAPI ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
data := []byte("federated copy checksum body")
|
||||
srcObject := "federation/checksum-source"
|
||||
putCopyChecksumSource(t, apiRouter, credentials, bucketName, srcObject, data, nil)
|
||||
|
||||
remoteBucket, _, cleanup := setupCopyObjectFederation(t, objectAPI, apiRouter, instanceType, bucketName)
|
||||
defer cleanup()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
typ hash.ChecksumType
|
||||
explicit bool
|
||||
}{
|
||||
{name: "CRC32", typ: hash.ChecksumCRC32, explicit: true},
|
||||
{name: "CRC32C", typ: hash.ChecksumCRC32C, explicit: true},
|
||||
{name: "SHA1", typ: hash.ChecksumSHA1, explicit: true},
|
||||
{name: "SHA256", typ: hash.ChecksumSHA256, explicit: true},
|
||||
{name: "CRC64NVME", typ: hash.ChecksumCRC64NVME, explicit: true},
|
||||
// Default: no requested algorithm still yields the S3 CRC-64NVME.
|
||||
{name: "default", typ: hash.ChecksumCRC64NVME},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var headers map[string]string
|
||||
if tc.explicit {
|
||||
headers = map[string]string{xhttp.AmzChecksumAlgo: tc.typ.String()}
|
||||
}
|
||||
dstObject := "federation/checksum-destination-" + tc.name
|
||||
rec := federatedCopyRequest(t, apiRouter, credentials, bucketName, srcObject, remoteBucket, dstObject, headers)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: federated CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
// The CopyObjectResult must carry the checksum of the copied bytes.
|
||||
assertCopyChecksumResponse(t, rec, tc.typ, data)
|
||||
// The remote must have persisted that same checksum.
|
||||
assertCopyChecksum(t, objectAPI, remoteBucket, dstObject, tc.typ, data, false, nil)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIFederatedCopyObjectRejectsInvalidRemoteChecksum(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: func(obj ObjectLayer, instanceType, bucket string, router http.Handler, credentials auth.Credentials, t *testing.T) {
|
||||
data := []byte("remote checksum response fixture")
|
||||
putCopyChecksumSource(t, router, credentials, bucket, "source", data, nil)
|
||||
// "" missing, "invalid-base64" unparseable, "YQ==" wrong digest
|
||||
// length, and "<valid>-0" a multipart-marked value that a single
|
||||
// forwarded PutObject must never yield (it would mislabel the
|
||||
// destination as composite).
|
||||
for _, value := range []string{"", "invalid-base64", "YQ==", mustChecksum(t, hash.ChecksumCRC32, data) + "-0"} {
|
||||
t.Run("checksum="+value, func(t *testing.T) {
|
||||
remoteBucket, _, cleanup := setupCopyObjectFederation(t, obj, router, instanceType, bucket, func(header http.Header) {
|
||||
header.Set(xhttp.AmzChecksumCRC32, value)
|
||||
})
|
||||
defer cleanup()
|
||||
rec := federatedCopyRequest(t, router, credentials, bucket, "source", remoteBucket, "destination",
|
||||
map[string]string{xhttp.AmzChecksumAlgo: "CRC32"})
|
||||
if rec.Code < 500 {
|
||||
t.Fatalf("invalid remote checksum must fail the copy, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
},
|
||||
endpoints: []string{"CopyObject", "PutObject", "HeadObject", "GetObject"},
|
||||
})
|
||||
}
|
||||
|
||||
// TestAPIFederatedCopyObjectChecksumIsBoundToWrite guards the checksum
|
||||
// representation: a federated copy that requests one algorithm must return only
|
||||
// that algorithm, and a copy of a checksum-less source without a requested
|
||||
// algorithm must never fabricate one other than the S3 default.
|
||||
func TestAPIFederatedCopyObjectChecksumIsBoundToWrite(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPIFederatedCopyObjectChecksumIsBoundToWrite,
|
||||
endpoints: []string{"CopyObject", "PutObject", "HeadObject", "GetObject"},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPIFederatedCopyObjectChecksumIsBoundToWrite(objectAPI ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
data := []byte("federated copy single checksum body")
|
||||
srcObject := "federation/single-checksum-source"
|
||||
putCopyChecksumSource(t, apiRouter, credentials, bucketName, srcObject, data, nil)
|
||||
|
||||
remoteBucket, _, cleanup := setupCopyObjectFederation(t, objectAPI, apiRouter, instanceType, bucketName)
|
||||
defer cleanup()
|
||||
|
||||
dstObject := "federation/single-checksum-destination"
|
||||
rec := federatedCopyRequest(t, apiRouter, credentials, bucketName, srcObject, remoteBucket, dstObject,
|
||||
map[string]string{xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String()})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: federated CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
var response CopyObjectResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
|
||||
t.Fatalf("%s: unable to decode CopyObjectResult: %v", instanceType, err)
|
||||
}
|
||||
if response.ChecksumCRC32 == "" {
|
||||
t.Fatalf("%s: requested CRC32 checksum missing from response: %s", instanceType, rec.Body.String())
|
||||
}
|
||||
// Only the requested algorithm may be present.
|
||||
if response.ChecksumCRC32C != "" || response.ChecksumSHA1 != "" ||
|
||||
response.ChecksumSHA256 != "" || response.ChecksumCRC64NVME != "" {
|
||||
t.Fatalf("%s: response carried checksums beyond the requested CRC32: %s", instanceType, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPIFederatedCopyObjectEmptySource guards the empty-body regression: a
|
||||
// checksum-less object gains the S3 default CRC-64NVME, but minio-go streams no
|
||||
// trailing checksum for a 0-byte body, so the remote returned none, the bind
|
||||
// found nothing, and every empty-object federated copy 500'd. An empty source
|
||||
// must now copy with 200 and carry the empty-content checksum, both when a
|
||||
// checksum is requested explicitly and via the default.
|
||||
func TestAPIFederatedCopyObjectEmptySource(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPIFederatedCopyObjectEmptySource,
|
||||
endpoints: []string{"CopyObject", "PutObject", "HeadObject", "GetObject"},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPIFederatedCopyObjectEmptySource(objectAPI ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
srcObject := "federation/empty-source"
|
||||
putCopyChecksumSource(t, apiRouter, credentials, bucketName, srcObject, nil, nil)
|
||||
|
||||
remoteBucket, _, cleanup := setupCopyObjectFederation(t, objectAPI, apiRouter, instanceType, bucketName)
|
||||
defer cleanup()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
typ hash.ChecksumType
|
||||
explicit bool
|
||||
}{
|
||||
{name: "explicit-CRC32", typ: hash.ChecksumCRC32, explicit: true},
|
||||
{name: "explicit-SHA256", typ: hash.ChecksumSHA256, explicit: true},
|
||||
// No requested algorithm: the S3 default CRC-64NVME still applies.
|
||||
{name: "default-CRC64NVME", typ: hash.ChecksumCRC64NVME},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var headers map[string]string
|
||||
if tc.explicit {
|
||||
headers = map[string]string{xhttp.AmzChecksumAlgo: tc.typ.String()}
|
||||
}
|
||||
dstObject := "federation/empty-destination-" + tc.name
|
||||
rec := federatedCopyRequest(t, apiRouter, credentials, bucketName, srcObject, remoteBucket, dstObject, headers)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: federated CopyObject of an empty source failed: %d %s",
|
||||
instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
// The empty-content digest must be returned and persisted.
|
||||
assertCopyChecksumResponse(t, rec, tc.typ, nil)
|
||||
assertCopyChecksum(t, objectAPI, remoteBucket, dstObject, tc.typ, nil, false, nil)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPIFederatedCopyObjectInheritedChecksum guards that a full-object checksum
|
||||
// already stored on the source is preserved across a federated copy that
|
||||
// requests no algorithm. That checksum sets dstOpts.WantChecksum (not
|
||||
// WantServerSideChecksumType), which the federated branch previously ignored,
|
||||
// silently dropping the checksum the local path keeps.
|
||||
func TestAPIFederatedCopyObjectInheritedChecksum(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPIFederatedCopyObjectInheritedChecksum,
|
||||
endpoints: []string{"CopyObject", "PutObject", "HeadObject", "GetObject"},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPIFederatedCopyObjectInheritedChecksum(objectAPI ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
data := []byte("abc")
|
||||
want := mustChecksum(t, hash.ChecksumCRC32, data) // "NSRBwg=="
|
||||
srcObject := "federation/inherited-checksum-source"
|
||||
putCopyChecksumSource(t, apiRouter, credentials, bucketName, srcObject, data,
|
||||
map[string]string{xhttp.AmzChecksumCRC32: want})
|
||||
|
||||
remoteBucket, _, cleanup := setupCopyObjectFederation(t, objectAPI, apiRouter, instanceType, bucketName)
|
||||
defer cleanup()
|
||||
|
||||
// No algorithm header: the source's stored CRC32 must survive the copy.
|
||||
dstObject := "federation/inherited-checksum-destination"
|
||||
rec := federatedCopyRequest(t, apiRouter, credentials, bucketName, srcObject, remoteBucket, dstObject, nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: federated CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, data)
|
||||
assertCopyChecksum(t, objectAPI, remoteBucket, dstObject, hash.ChecksumCRC32, data, false, nil)
|
||||
}
|
||||
|
||||
// federationTestKMSKeyID names the single key of the builtin KMS the SSE test
|
||||
// installs; SSE-S3 seals with it implicitly, SSE-KMS names it by ID.
|
||||
const federationTestKMSKeyID = "federation-test-key"
|
||||
|
||||
// federationSSEHeaders returns the request headers that select one server-side
|
||||
// encryption kind: "plain", "s3", "kms", "kms-context" (SSE-KMS with an
|
||||
// explicit encryption context), "kms-empty-context" or "c". SSE-C keys derive from keyByte so a
|
||||
// case can name two distinct customer keys. With copySource the SSE-C key is
|
||||
// returned in its x-amz-copy-source-* form, the only kind a copy has to name
|
||||
// for its source; the other kinds then return nothing.
|
||||
func federationSSEHeaders(kind string, keyByte byte, copySource bool) map[string]string {
|
||||
h := map[string]string{}
|
||||
if copySource && kind != "c" {
|
||||
return h
|
||||
}
|
||||
switch kind {
|
||||
case "plain":
|
||||
case "s3":
|
||||
h[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
|
||||
case "kms", "kms-context", "kms-empty-context":
|
||||
h[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
|
||||
h[xhttp.AmzServerSideEncryptionKmsID] = federationTestKMSKeyID
|
||||
switch kind {
|
||||
case "kms-context":
|
||||
h[xhttp.AmzServerSideEncryptionKmsContext] = base64.StdEncoding.EncodeToString([]byte(`{"tenant":"federation"}`))
|
||||
case "kms-empty-context":
|
||||
h[xhttp.AmzServerSideEncryptionKmsContext] = base64.StdEncoding.EncodeToString([]byte(`{}`))
|
||||
}
|
||||
case "c":
|
||||
key := bytes.Repeat([]byte{keyByte}, 32)
|
||||
sum := md5.Sum(key)
|
||||
algorithm, customerKey, keyMD5 := xhttp.AmzServerSideEncryptionCustomerAlgorithm,
|
||||
xhttp.AmzServerSideEncryptionCustomerKey, xhttp.AmzServerSideEncryptionCustomerKeyMD5
|
||||
if copySource {
|
||||
algorithm, customerKey, keyMD5 = xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm,
|
||||
xhttp.AmzServerSideEncryptionCopyCustomerKey, xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5
|
||||
}
|
||||
h[algorithm] = xhttp.AmzEncryptionAES
|
||||
h[customerKey] = base64.StdEncoding.EncodeToString(key)
|
||||
h[keyMD5] = base64.StdEncoding.EncodeToString(sum[:])
|
||||
default:
|
||||
panic("unknown SSE kind " + kind)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// federationStoredSSE reports which SSE kind stored object metadata declares.
|
||||
func federationStoredSSE(metadata map[string]string) string {
|
||||
switch {
|
||||
case crypto.SSEC.IsEncrypted(metadata):
|
||||
return "c"
|
||||
case crypto.S3KMS.IsEncrypted(metadata):
|
||||
return "kms"
|
||||
case crypto.S3.IsEncrypted(metadata):
|
||||
return "s3"
|
||||
}
|
||||
return "plain"
|
||||
}
|
||||
|
||||
// federationGetObject reads an object through GetObjectHandler.
|
||||
func federationGetObject(t *testing.T, apiRouter http.Handler, credentials auth.Credentials,
|
||||
bucket, object string, headers map[string]string,
|
||||
) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
req, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucket, object),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, headers)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to build GetObject request: %v", err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
// assertFederationCopyReadback checks storage and the public GET/HEAD checksum
|
||||
// surface with the destination key, including the committed write's time.
|
||||
func assertFederationCopyReadback(t *testing.T, obj ObjectLayer, router http.Handler, cred auth.Credentials,
|
||||
bucket, object string, rec *httptest.ResponseRecorder, typ hash.ChecksumType, data []byte, compressed bool, keyHeaders map[string]string,
|
||||
) ObjectInfo {
|
||||
t.Helper()
|
||||
assertCopyChecksumResponse(t, rec, typ, data)
|
||||
headers := make(http.Header)
|
||||
for k, v := range keyHeaders {
|
||||
headers.Set(k, v)
|
||||
}
|
||||
info := assertCopyChecksum(t, obj, bucket, object, typ, data, compressed, headers)
|
||||
var response CopyObjectResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := amztime.ISO8601Format(info.ModTime.UTC()); info.ModTime.IsZero() || response.LastModified != want {
|
||||
t.Errorf("copy time = %q, want stored time %q", response.LastModified, want)
|
||||
}
|
||||
readHeaders := map[string]string{xhttp.AmzChecksumMode: "ENABLED"}
|
||||
maps.Copy(readHeaders, keyHeaders)
|
||||
for _, method := range []string{http.MethodGet, http.MethodHead} {
|
||||
req, err := newTestSignedRequestV4(method, getGetObjectURL("", bucket, object), 0, nil, cred.AccessKey, cred.SecretKey, readHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := httptest.NewRecorder()
|
||||
router.ServeHTTP(got, req)
|
||||
if got.Code != http.StatusOK {
|
||||
t.Fatalf("destination %s = %d %s", method, got.Code, got.Body.String())
|
||||
}
|
||||
if method == http.MethodGet && !bytes.Equal(got.Body.Bytes(), data) {
|
||||
t.Fatalf("destination GET differs from the %d-byte plaintext", len(data))
|
||||
}
|
||||
if got.Header().Get(xhttp.ContentLength) != strconv.Itoa(len(data)) ||
|
||||
got.Header().Get(typ.Key()) != mustChecksum(t, typ, data) ||
|
||||
got.Header().Get(xhttp.AmzChecksumType) != xhttp.AmzChecksumTypeFullObject {
|
||||
t.Fatalf("destination %s length/checksum headers = %v", method, got.Header())
|
||||
}
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
func assertFederationKMSContext(t *testing.T, info ObjectInfo, requested string) {
|
||||
t.Helper()
|
||||
decode := func(encoded string) map[string]string {
|
||||
t.Helper()
|
||||
value, err := base64.StdEncoding.DecodeString(encoded)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var context map[string]string
|
||||
if err := json.Unmarshal(value, &context); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return context
|
||||
}
|
||||
want := map[string]string{}
|
||||
if requested != "" {
|
||||
want = decode(requested)
|
||||
}
|
||||
stored, present := info.UserDefined[crypto.MetaContext]
|
||||
if len(want) == 0 {
|
||||
if present {
|
||||
t.Errorf("absent/empty destination context stored client metadata %q", stored)
|
||||
}
|
||||
} else if !present || !maps.Equal(decode(stored), want) {
|
||||
t.Errorf("stored KMS context = %q, want %v", stored, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPIFederatedCopyObjectSSE guards the legacy etcd federation branch of
|
||||
// CopyObjectHandler for encrypted sources and destinations (#158). The proxy
|
||||
// reads its source through getObjectNInfo, which yields the decrypted and
|
||||
// decompressed bytes, but it used to run the destination encryption locally
|
||||
// as well and then forward that stream with the source's stored size under
|
||||
// the destination SSE option. SSE to plain and plain to SSE failed on the
|
||||
// length mismatch; SSE to SSE matched by coincidence, so the remote encrypted
|
||||
// the ciphertext a second time and a destination GET returned the inner
|
||||
// ciphertext with HTTP 200. The proxy must forward the logical bytes with
|
||||
// their logical size and let the remote encrypt exactly once.
|
||||
func TestAPIFederatedCopyObjectSSE(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPIFederatedCopyObjectSSE,
|
||||
// Register PutObjectPart and CopyObject before the query-less
|
||||
// PutObject PUT route; otherwise PutObject shadows them.
|
||||
endpoints: []string{
|
||||
"NewMultipart", "PutObjectPart", "CompleteMultipart",
|
||||
"CopyObject", "PutObject", "HeadObject", "GetObject",
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// putFederationMultipartSource uploads parts as one multipart object through
|
||||
// the API router. Headers select the upload's encryption/checksum; SSE-C keys
|
||||
// and checksums are also supplied on the parts and completion as required.
|
||||
func putFederationMultipartSource(t *testing.T, apiRouter http.Handler, credentials auth.Credentials,
|
||||
bucket, object string, parts [][]byte, headers map[string]string,
|
||||
) {
|
||||
t.Helper()
|
||||
do := func(method, url string, body []byte, headers map[string]string) *httptest.ResponseRecorder {
|
||||
req, err := newTestSignedRequestV4(method, url, int64(len(body)), bytes.NewReader(body),
|
||||
credentials.AccessKey, credentials.SecretKey, headers)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to build %s %s request: %v", method, url, err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s %s failed: %d %s", method, url, rec.Code, rec.Body.String())
|
||||
}
|
||||
return rec
|
||||
}
|
||||
var upload InitiateMultipartUploadResponse
|
||||
rec := do(http.MethodPost, getNewMultipartURL("", bucket, object), nil, headers)
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &upload); err != nil {
|
||||
t.Fatalf("failed to decode NewMultipartUpload response: %v", err)
|
||||
}
|
||||
completion := CompleteMultipartUpload{}
|
||||
partHeaders := map[string]string{}
|
||||
for _, key := range []string{xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzServerSideEncryptionCustomerKey, xhttp.AmzServerSideEncryptionCustomerKeyMD5} {
|
||||
if value := headers[key]; value != "" {
|
||||
partHeaders[key] = value
|
||||
}
|
||||
}
|
||||
checksumType := hash.NewChecksumType(headers[xhttp.AmzChecksumAlgo], headers[xhttp.AmzChecksumType])
|
||||
for i, part := range parts {
|
||||
if checksumType.IsSet() {
|
||||
partHeaders[checksumType.Key()] = mustChecksum(t, checksumType, part)
|
||||
}
|
||||
rec := do(http.MethodPut, getPutObjectPartURL("", bucket, object, upload.UploadID, strconv.Itoa(i+1)), part, partHeaders)
|
||||
completion.Parts = append(completion.Parts, completePartWithChecksum(checksumType, i+1,
|
||||
canonicalizeETag(rec.Header()[xhttp.ETag][0]), partHeaders[checksumType.Key()]))
|
||||
}
|
||||
body, err := xml.Marshal(completion)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to encode CompleteMultipartUpload: %v", err)
|
||||
}
|
||||
delete(partHeaders, checksumType.Key())
|
||||
do(http.MethodPost, getCompleteMultipartUploadURL("", bucket, object, upload.UploadID), body, partHeaders)
|
||||
}
|
||||
|
||||
func testAPIFederatedCopyObjectSSE(objectAPI ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
testKMS, err := kms.NewBuiltin(federationTestKMSKeyID, bytes.Repeat([]byte{0x58}, 32))
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create the test KMS: %v", err)
|
||||
}
|
||||
previousKMS := GlobalKMS
|
||||
GlobalKMS = testKMS
|
||||
defer func() { GlobalKMS = previousKMS }()
|
||||
// .txt sources are stored compressed, SSE-S3 and SSE-KMS ones included;
|
||||
// SSE-C data is never compressed.
|
||||
restoreCompression := setCopyChecksumCompression(true)
|
||||
defer restoreCompression()
|
||||
|
||||
remoteBucket, cleanup := setupCopyObjectFederationTLS(t, objectAPI, apiRouter, instanceType, bucketName)
|
||||
defer cleanup()
|
||||
|
||||
// A DARE package holds 64 KiB of plaintext, so the larger bodies span two
|
||||
// packages and catch a size that accounts for only one.
|
||||
large := bytes.Repeat([]byte("federated sse copy body "), 64*1024/24+1)[:64*1024+1]
|
||||
bodies := []struct {
|
||||
name, ext string
|
||||
data []byte
|
||||
requested, inherited hash.ChecksumType
|
||||
}{
|
||||
{name: "empty", ext: ".bin"},
|
||||
{name: "empty-explicit", ext: ".bin", requested: hash.ChecksumSHA256},
|
||||
{name: "inherited", ext: ".bin", data: []byte("stored encrypted checksum"), inherited: hash.ChecksumCRC32},
|
||||
{name: "small", ext: ".bin", data: []byte("abc")},
|
||||
{name: "two-packages", ext: ".bin", data: large},
|
||||
{name: "compressed", ext: ".txt", data: large},
|
||||
}
|
||||
pairs := []struct{ src, dst string }{
|
||||
{"plain", "s3"},
|
||||
{"s3", "plain"},
|
||||
{"s3", "s3"},
|
||||
{"plain", "c"},
|
||||
{"c", "plain"},
|
||||
{"c", "c"},
|
||||
{"s3", "c"},
|
||||
{"plain", "kms"},
|
||||
{"kms", "plain"},
|
||||
{"kms", "kms"},
|
||||
{"plain", "kms-context"},
|
||||
{"kms-context", "kms-context"},
|
||||
{"kms-context", "kms"},
|
||||
{"kms-context", "kms-empty-context"},
|
||||
}
|
||||
const srcKeyByte, dstKeyByte = 0x11, 0x22
|
||||
|
||||
for _, body := range bodies {
|
||||
for _, pair := range pairs {
|
||||
// Focus empty and inherited-checksum cases on each encrypted kind.
|
||||
if body.name == "empty" || body.name == "empty-explicit" || body.name == "inherited" {
|
||||
if pair.src != pair.dst || pair.src == "kms-context" {
|
||||
continue
|
||||
}
|
||||
}
|
||||
t.Run(body.name+"/"+pair.src+"-to-"+pair.dst, func(t *testing.T) {
|
||||
prefix := "federation/sse-" + body.name + "-" + pair.src + "-to-" + pair.dst
|
||||
srcObject, dstObject := prefix+"-source"+body.ext, prefix+"-destination"+body.ext
|
||||
|
||||
sourceHeaders := federationSSEHeaders(pair.src, srcKeyByte, false)
|
||||
if pair.src == "kms-context" {
|
||||
sourceHeaders[xhttp.AmzServerSideEncryptionKmsContext] = base64.StdEncoding.EncodeToString([]byte(`{"tenant":"source"}`))
|
||||
}
|
||||
if body.inherited.IsSet() {
|
||||
sourceHeaders[body.inherited.Key()] = mustChecksum(t, body.inherited, body.data)
|
||||
}
|
||||
putCopyChecksumSource(t, apiRouter, credentials, bucketName, srcObject, body.data, sourceHeaders)
|
||||
before, err := objectAPI.GetObjectInfo(t.Context(), bucketName, srcObject, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: GetObjectInfo(source) failed: %v", instanceType, err)
|
||||
}
|
||||
if got, want := federationStoredSSE(before.UserDefined), strings.Split(pair.src, "-")[0]; got != want {
|
||||
t.Fatalf("%s: source stored as %s, want %s", instanceType, got, want)
|
||||
}
|
||||
if compressed := body.ext == ".txt" && pair.src != "c"; before.IsCompressed() != compressed {
|
||||
t.Fatalf("%s: source compressed=%v, want %v", instanceType, before.IsCompressed(), compressed)
|
||||
}
|
||||
assertFederationKMSContext(t, before, sourceHeaders[xhttp.AmzServerSideEncryptionKmsContext])
|
||||
|
||||
if body.inherited.IsSet() {
|
||||
sourceKeys := make(http.Header)
|
||||
for k, v := range federationSSEHeaders(pair.src, srcKeyByte, true) {
|
||||
sourceKeys.Set(k, v)
|
||||
}
|
||||
assertCopyChecksum(t, objectAPI, bucketName, srcObject, body.inherited, body.data, false, sourceKeys)
|
||||
}
|
||||
headers := federationSSEHeaders(pair.dst, dstKeyByte, false)
|
||||
if body.requested.IsSet() {
|
||||
headers[xhttp.AmzChecksumAlgo] = body.requested.String()
|
||||
}
|
||||
maps.Copy(headers, federationSSEHeaders(pair.src, srcKeyByte, true))
|
||||
rec := federatedCopyRequest(t, apiRouter, credentials, bucketName, srcObject, remoteBucket, dstObject, headers)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: federated CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
wantChecksum := hash.ChecksumCRC64NVME
|
||||
if body.requested.IsSet() {
|
||||
wantChecksum = body.requested
|
||||
} else if body.inherited.IsSet() {
|
||||
wantChecksum = body.inherited
|
||||
}
|
||||
var getHeaders map[string]string
|
||||
if pair.dst == "c" {
|
||||
getHeaders = federationSSEHeaders("c", dstKeyByte, false)
|
||||
}
|
||||
after := assertFederationCopyReadback(t, objectAPI, apiRouter, credentials, remoteBucket, dstObject,
|
||||
rec, wantChecksum, body.data, body.ext == ".txt" && pair.dst != "c", getHeaders)
|
||||
assertFederationKMSContext(t, after, headers[xhttp.AmzServerSideEncryptionKmsContext])
|
||||
// A second encryption layer would add its own DARE overhead.
|
||||
if got, want := federationStoredSSE(after.UserDefined), strings.Split(pair.dst, "-")[0]; got != want {
|
||||
t.Fatalf("%s: destination stored as %s, want %s", instanceType, got, want)
|
||||
}
|
||||
if pair.dst != "plain" && !after.IsCompressed() {
|
||||
once := ObjectInfo{Size: int64(len(body.data))}
|
||||
if want := once.EncryptedSize(); after.Size != want {
|
||||
t.Fatalf("%s: destination stored %d bytes, want %d for %d bytes encrypted once",
|
||||
instanceType, after.Size, want, len(body.data))
|
||||
}
|
||||
}
|
||||
|
||||
// The source must be untouched.
|
||||
source, err := objectAPI.GetObjectInfo(t.Context(), bucketName, srcObject, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: GetObjectInfo(source) after the copy failed: %v", instanceType, err)
|
||||
}
|
||||
if source.Size != before.Size || source.ETag != before.ETag || !source.ModTime.Equal(before.ModTime) {
|
||||
t.Fatalf("%s: federated copy modified its source: size %d->%d etag %s->%s",
|
||||
instanceType, before.Size, source.Size, before.ETag, source.ETag)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Multipart sources cross an encryption boundary per part. SHA256 also
|
||||
// exercises promotion from a stored composite to a full-object checksum.
|
||||
parts := [][]byte{bytes.Repeat([]byte("p"), globalMinPartSize+1), []byte("q!")}
|
||||
data := bytes.Join(parts, nil)
|
||||
for _, pair := range []struct {
|
||||
src, dst string
|
||||
composite bool
|
||||
}{
|
||||
{src: "s3", dst: "plain"},
|
||||
{src: "s3", dst: "s3"},
|
||||
{src: "c", dst: "c"},
|
||||
{src: "kms", dst: "kms", composite: true},
|
||||
} {
|
||||
t.Run("multipart/"+pair.src+"-to-"+pair.dst, func(t *testing.T) {
|
||||
srcObject, dstObject := "federation/multipart-source-"+pair.src+"-"+pair.dst+".bin", "federation/multipart-destination-"+pair.src+"-"+pair.dst+".bin"
|
||||
sourceHeaders := federationSSEHeaders(pair.src, srcKeyByte, false)
|
||||
wantChecksum := hash.ChecksumCRC64NVME
|
||||
if pair.composite {
|
||||
wantChecksum = hash.ChecksumSHA256
|
||||
sourceHeaders[xhttp.AmzChecksumAlgo] = wantChecksum.String()
|
||||
sourceHeaders[xhttp.AmzChecksumType] = xhttp.AmzChecksumTypeComposite
|
||||
}
|
||||
putFederationMultipartSource(t, apiRouter, credentials, bucketName, srcObject, parts, sourceHeaders)
|
||||
before, err := objectAPI.GetObjectInfo(t.Context(), bucketName, srcObject, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(before.Parts) != len(parts) || federationStoredSSE(before.UserDefined) != pair.src {
|
||||
t.Fatalf("source has %d parts stored as %s, want %d parts as %s", len(before.Parts), federationStoredSSE(before.UserDefined), len(parts), pair.src)
|
||||
}
|
||||
if pair.composite {
|
||||
checksums, _ := before.decryptChecksums(0, nil)
|
||||
if checksums[xhttp.AmzChecksumType] != xhttp.AmzChecksumTypeComposite || !strings.HasSuffix(checksums[wantChecksum.String()], "-2") {
|
||||
t.Fatalf("source did not persist a two-part composite checksum: %v", checksums)
|
||||
}
|
||||
}
|
||||
headers := federationSSEHeaders(pair.dst, dstKeyByte, false)
|
||||
maps.Copy(headers, federationSSEHeaders(pair.src, srcKeyByte, true))
|
||||
rec := federatedCopyRequest(t, apiRouter, credentials, bucketName, srcObject, remoteBucket, dstObject, headers)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: federated CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
var getHeaders map[string]string
|
||||
if pair.dst == "c" {
|
||||
getHeaders = federationSSEHeaders("c", dstKeyByte, false)
|
||||
}
|
||||
after := assertFederationCopyReadback(t, objectAPI, apiRouter, credentials, remoteBucket, dstObject, rec, wantChecksum, data, false, getHeaders)
|
||||
if got := federationStoredSSE(after.UserDefined); got != pair.dst {
|
||||
t.Fatalf("destination stored as %s, want %s", got, pair.dst)
|
||||
}
|
||||
if once := (ObjectInfo{Size: int64(len(data))}); pair.dst != "plain" && after.Size != once.EncryptedSize() {
|
||||
t.Fatalf("destination stored %d bytes, want %d for %d bytes encrypted once", after.Size, once.EncryptedSize(), len(data))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
miniogo "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio/internal/amztime"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
)
|
||||
|
||||
// Exercise the selected SDK's actual retry loop and concurrent calls sharing a
|
||||
// transport. Timestamps must follow the final PUT response, never a location
|
||||
// probe, another operation, a failed attempt, or the HTTP Date header.
|
||||
func TestFederatedWriteTimeTransport(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
attempts := make(map[string]int)
|
||||
written := time.Date(2001, 2, 3, 4, 5, 6, 123456789, time.UTC)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
w.Header().Set(federatedLastModified, written.Add(-time.Hour).Format(time.RFC3339Nano))
|
||||
if r.Method == http.MethodGet && r.URL.Query().Has("location") {
|
||||
_, _ = io.WriteString(w, `<LocationConstraint>us-east-1</LocationConstraint>`)
|
||||
return
|
||||
}
|
||||
if r.Method != http.MethodPut {
|
||||
t.Errorf("unexpected follow-up read: %s %s", r.Method, r.URL)
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
attempts[r.URL.Path]++
|
||||
attempt := attempts[r.URL.Path]
|
||||
mu.Unlock()
|
||||
if strings.Contains(r.URL.Path, "retry-") && attempt == 1 {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_, _ = io.WriteString(w, `<Error><Code>SlowDown</Code></Error>`)
|
||||
return
|
||||
}
|
||||
stamp := written.Add(time.Duration(len(r.URL.Path)) * time.Nanosecond)
|
||||
if r.URL.Query().Has("partNumber") {
|
||||
stamp = stamp.Add(time.Second)
|
||||
}
|
||||
w.Header().Set(federatedLastModified, stamp.Format(time.RFC3339Nano))
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "missing"):
|
||||
w.Header().Del(federatedLastModified)
|
||||
case strings.HasSuffix(r.URL.Path, "malformed"):
|
||||
w.Header().Set(federatedLastModified, "invalid-time")
|
||||
}
|
||||
w.Header().Set("ETag", `"`+r.URL.Path+`"`)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
core, err := miniogo.NewCore(server.Listener.Addr().String(), &miniogo.Options{
|
||||
Creds: credentials.NewStaticV4("federation", "federation-secret", ""),
|
||||
Transport: federatedWriteTransport{server.Client().Transport},
|
||||
BucketLookup: miniogo.BucketLookupPath,
|
||||
MaxRetries: 2,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, part := range []bool{false, true} {
|
||||
for _, scenario := range []string{"present", "missing", "malformed", "retry-present", "retry-missing", "retry-malformed"} {
|
||||
t.Run(fmt.Sprintf("part=%v/%s", part, scenario), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx, modified := withFederatedWriteTime(t.Context())
|
||||
object := fmt.Sprintf("part-%v/%s", part, scenario)
|
||||
var etag string
|
||||
if part {
|
||||
info, err := core.PutObjectPart(ctx, "bucket", object, "upload-id", 1, bytes.NewReader([]byte("abc")), 3, miniogo.PutObjectPartOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
etag = info.ETag
|
||||
} else {
|
||||
info, err := core.PutObject(ctx, "bucket", object, bytes.NewReader([]byte("abc")), 3, "", "", miniogo.PutObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
etag = info.ETag
|
||||
}
|
||||
want := time.Time{}
|
||||
if strings.HasSuffix(scenario, "present") {
|
||||
want = written.Add(time.Duration(len("/bucket/"+object)) * time.Nanosecond)
|
||||
if part {
|
||||
want = want.Add(time.Second)
|
||||
}
|
||||
}
|
||||
if !modified.Equal(want) || etag != "/bucket/"+object {
|
||||
t.Errorf("write result: time=%s ETag=%s, want time=%s ETag=/bucket/%s", modified, etag, want, object)
|
||||
}
|
||||
mu.Lock()
|
||||
count := attempts["/bucket/"+object]
|
||||
mu.Unlock()
|
||||
wantAttempts := 1
|
||||
if strings.HasPrefix(scenario, "retry-") {
|
||||
wantAttempts = 2
|
||||
}
|
||||
if count != wantAttempts {
|
||||
t.Errorf("PUT attempts = %d, want %d", count, wantAttempts)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIFederatedCopyWriteTimeSSEAndCompatibility(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
endpoints: []string{"CopyObjectPart", "NewMultipart", "PutObjectPart", "ListObjectParts", "CompleteMultipart", "CopyObject", "PutObject", "GetObject"},
|
||||
objAPITest: func(obj ObjectLayer, instanceType, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
testKMS, err := kms.NewBuiltin(federationTestKMSKeyID, bytes.Repeat([]byte{0x58}, 32))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previous := GlobalKMS
|
||||
GlobalKMS = testKMS
|
||||
defer func() { GlobalKMS = previous }()
|
||||
for _, mode := range []string{"current", "missing", "malformed"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
remote, _, cleanup := setupCopyObjectFederationRemote(t, obj, router, instanceType, bucket, true, func(h http.Header) {
|
||||
switch mode {
|
||||
case "missing":
|
||||
h.Del(federatedLastModified)
|
||||
case "malformed":
|
||||
h.Set(federatedLastModified, "invalid-time")
|
||||
}
|
||||
})
|
||||
defer cleanup()
|
||||
for _, kind := range []string{"plain", "s3", "c", "kms"} {
|
||||
if mode != "current" && kind != "plain" && kind != "c" {
|
||||
continue
|
||||
}
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
data := []byte("federated write timestamp")
|
||||
putCopyChecksumSource(t, router, cred, bucket, "source", data, federationSSEHeaders(kind, 0x11, false))
|
||||
headers := federationSSEHeaders(kind, 0x22, false)
|
||||
maps.Copy(headers, federationSSEHeaders(kind, 0x11, true))
|
||||
rec := federatedCopyRequest(t, router, cred, bucket, "source", remote, kind+"-copy", headers)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("CopyObject: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
info, err := obj.GetObjectInfo(t.Context(), remote, kind+"-copy", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var copied CopyObjectResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &copied); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertTime := func(got string, stored time.Time) {
|
||||
t.Helper()
|
||||
if stored.IsZero() {
|
||||
t.Fatal("storage returned a zero timestamp")
|
||||
}
|
||||
if mode != "current" {
|
||||
stored = time.Time{}
|
||||
}
|
||||
if got != amztime.ISO8601Format(stored.UTC()) {
|
||||
t.Errorf("copy time = %q, want %q", got, amztime.ISO8601Format(stored.UTC()))
|
||||
}
|
||||
}
|
||||
assertTime(copied.LastModified, info.ModTime)
|
||||
|
||||
object := kind + "-multipart"
|
||||
req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", remote, object), 0, nil,
|
||||
cred.AccessKey, cred.SecretKey, federationSSEHeaders(kind, 0x22, false))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("NewMultipartUpload: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var upload InitiateMultipartUploadResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &upload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// UploadPartCopy selects KMS/S3 encryption from the upload;
|
||||
// only SSE-C keys accompany the part request. The replica
|
||||
// markers must still take the ordinary decrypting copy path.
|
||||
headers = federationSSEHeaders(kind, 0x11, true)
|
||||
var keys map[string]string
|
||||
if kind == "c" {
|
||||
keys = federationSSEHeaders("c", 0x22, false)
|
||||
maps.Copy(headers, keys)
|
||||
}
|
||||
headers[xhttp.AmzCopySource] = SlashSeparator + pathJoin(bucket, "source")
|
||||
headers[xhttp.MinIOSourceReplicationRequest] = "true"
|
||||
headers[xhttp.AmzBucketReplicationStatus] = "REPLICA"
|
||||
req, err = newTestSignedRequestV4(http.MethodPut, getCopyObjectPartURL("", remote, object, upload.UploadID, "1"), 0, nil, cred.AccessKey, cred.SecretKey, headers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("UploadPartCopy: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var part CopyObjectPartResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &part); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parts, err := obj.ListObjectParts(t.Context(), remote, object, upload.UploadID, 0, 100, ObjectOptions{})
|
||||
if err != nil || len(parts.Parts) != 1 {
|
||||
t.Fatalf("stored part: %v, %v", parts, err)
|
||||
}
|
||||
assertTime(part.LastModified, parts.Parts[0].LastModified)
|
||||
rec = completePartsHTTP(t, router, cred, remote, object, upload.UploadID, []CompletePart{{PartNumber: 1, ETag: canonicalizeETag(part.ETag)}}, keys)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("CompleteMultipartUpload: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
for _, name := range []string{kind + "-copy", object} {
|
||||
got := federationGetObject(t, router, cred, remote, name, keys)
|
||||
if got.Code != http.StatusOK || !bytes.Equal(got.Body.Bytes(), data) {
|
||||
t.Fatalf("readback %s: %d %s", name, got.Code, got.Body.String())
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -36,6 +36,44 @@ import (
|
||||
|
||||
var etagRegex = regexp.MustCompile("\"*?([^\"]*?)\"*?$")
|
||||
|
||||
// federatedLastModified carries the committed object/part time on the write
|
||||
// response. HTTP Date is a response time and Last-Modified loses subsecond
|
||||
// precision, so neither can supply CopyObject/UploadPartCopy's timestamp.
|
||||
const federatedLastModified = "X-Minio-Last-Modified"
|
||||
|
||||
type federatedWriteTimeKey struct{}
|
||||
|
||||
// withFederatedWriteTime allocates one result per SDK operation. SDK retries
|
||||
// run sequentially with this context; concurrent copies never share the result.
|
||||
func withFederatedWriteTime(ctx context.Context) (context.Context, *time.Time) {
|
||||
modified := new(time.Time)
|
||||
return context.WithValue(ctx, federatedWriteTimeKey{}, modified), modified
|
||||
}
|
||||
|
||||
type federatedWriteTransport struct {
|
||||
http.RoundTripper
|
||||
}
|
||||
|
||||
func (t federatedWriteTransport) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
resp, err := t.RoundTripper.RoundTrip(r)
|
||||
if modified, ok := r.Context().Value(federatedWriteTimeKey{}).(*time.Time); ok && r.Method == http.MethodPut {
|
||||
// Replace on every attempt, including a success from an older target
|
||||
// without the header. Never retain a failed attempt's timestamp or fail
|
||||
// a committed write just because its timestamp is unavailable.
|
||||
*modified = time.Time{}
|
||||
if err == nil && resp != nil && resp.StatusCode == http.StatusOK {
|
||||
*modified, _ = time.Parse(time.RFC3339Nano, resp.Header.Get(federatedLastModified))
|
||||
}
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
|
||||
func setFederatedWriteTime(w http.ResponseWriter, r *http.Request, modified time.Time) {
|
||||
if isFederatedInternalRequest(r.UserAgent()) && !modified.IsZero() {
|
||||
w.Header().Set(federatedLastModified, modified.UTC().Format(time.RFC3339Nano))
|
||||
}
|
||||
}
|
||||
|
||||
// Validates the preconditions for CopyObjectPart, returns true if CopyObjectPart
|
||||
// operation should not proceed. Preconditions supported are:
|
||||
//
|
||||
|
||||
+184
-28
@@ -1203,12 +1203,20 @@ const federatedInternalAppName = "minio-federated"
|
||||
// Applicable only in a federated deployment
|
||||
var getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core, error) {
|
||||
cred := getReqAccessCred(r, globalSite.Region())
|
||||
transport := getRemoteInstanceTransport()
|
||||
if transport == nil {
|
||||
var err error
|
||||
transport, err = miniogo.DefaultTransport(globalIsTLS)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// In a federated deployment, all the instances share config files
|
||||
// and hence expected to have same credentials.
|
||||
core, err := miniogo.NewCore(host, &miniogo.Options{
|
||||
Creds: credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, ""),
|
||||
Secure: globalIsTLS,
|
||||
Transport: getRemoteInstanceTransport(),
|
||||
Transport: federatedWriteTransport{transport},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1217,6 +1225,43 @@ var getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core,
|
||||
return core, nil
|
||||
}
|
||||
|
||||
// federatedChecksumType maps a requested server-side checksum type to the
|
||||
// minio-go checksum the federation proxy asks the remote deployment to compute
|
||||
// on the forwarded PutObject. Returns ChecksumNone for an unset/unknown type.
|
||||
func federatedChecksumType(t hash.ChecksumType) miniogo.ChecksumType {
|
||||
switch t.Base() {
|
||||
case hash.ChecksumCRC32:
|
||||
return miniogo.ChecksumCRC32
|
||||
case hash.ChecksumCRC32C:
|
||||
return miniogo.ChecksumCRC32C
|
||||
case hash.ChecksumSHA1:
|
||||
return miniogo.ChecksumSHA1
|
||||
case hash.ChecksumSHA256:
|
||||
return miniogo.ChecksumSHA256
|
||||
case hash.ChecksumCRC64NVME:
|
||||
return miniogo.ChecksumCRC64NVME
|
||||
}
|
||||
return miniogo.ChecksumNone
|
||||
}
|
||||
|
||||
// federatedChecksumValue returns the base64 checksum the remote deployment
|
||||
// reported for the requested type on the forwarded PutObject.
|
||||
func federatedChecksumValue(t hash.ChecksumType, info miniogo.UploadInfo) string {
|
||||
switch t.Base() {
|
||||
case hash.ChecksumCRC32:
|
||||
return info.ChecksumCRC32
|
||||
case hash.ChecksumCRC32C:
|
||||
return info.ChecksumCRC32C
|
||||
case hash.ChecksumSHA1:
|
||||
return info.ChecksumSHA1
|
||||
case hash.ChecksumSHA256:
|
||||
return info.ChecksumSHA256
|
||||
case hash.ChecksumCRC64NVME:
|
||||
return info.ChecksumCRC64NVME
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Check if the destination bucket is on a remote site, this code only gets executed
|
||||
// when federation is enabled, ie when globalDNSConfig is non 'nil'.
|
||||
//
|
||||
@@ -1365,11 +1410,27 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
allowReplicationMetadata := replicaTrusted
|
||||
|
||||
// Check if bucket encryption is enabled
|
||||
sseConfig, _ := globalBucketSSEConfigSys.Get(dstBucket)
|
||||
sseConfig.Apply(r.Header, sse.ApplyOptions{
|
||||
AutoEncrypt: globalAutoEncryption,
|
||||
})
|
||||
// Federation only: the destination bucket lives on another deployment and
|
||||
// the copy is forwarded to it as a PutObject. That remote write owns the
|
||||
// destination's storage transformations, so this handler hands it the
|
||||
// logical (decompressed, decrypted) bytes at their logical size and lets
|
||||
// the remote compress and encrypt once. Encrypting here as well would
|
||||
// forward ciphertext under the destination's own SSE option, which either
|
||||
// fails the length check or, for SSE to SSE, has the remote encrypt the
|
||||
// ciphertext a second time and store an unreadable object (#158).
|
||||
remoteCallRequired := isRemoteCopyRequired(ctx, srcBucket, dstBucket, objectAPI)
|
||||
|
||||
// Apply the destination bucket's default encryption only when this
|
||||
// deployment writes the destination. For a remote destination the proxy
|
||||
// has no authority over that bucket's defaults: applying its own here
|
||||
// would forward an explicit SSE header that the remote then honors in
|
||||
// place of the destination's configuration (#167).
|
||||
if !remoteCallRequired {
|
||||
sseConfig, _ := globalBucketSSEConfigSys.Get(dstBucket)
|
||||
sseConfig.Apply(r.Header, sse.ApplyOptions{
|
||||
AutoEncrypt: globalAutoEncryption,
|
||||
})
|
||||
}
|
||||
var srcOpts, dstOpts ObjectOptions
|
||||
srcOpts, err = copySrcOpts(ctx, r, srcBucket, srcObject)
|
||||
if err != nil {
|
||||
@@ -1434,6 +1495,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
defer gr.Close()
|
||||
srcInfo := gr.ObjInfo
|
||||
// A trusted SSE-C replica reads raw ciphertext. Federation's ordinary PUT
|
||||
// expects plaintext and does not forward the replica's sealed-key metadata.
|
||||
if remoteCallRequired && replicaTrusted && crypto.SSEC.IsEncrypted(srcInfo.UserDefined) {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{Message: "federated raw SSE-C replica CopyObject is not supported"}), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// maximum Upload size for object in a single CopyObject operation.
|
||||
if isMaxObjectSize(srcInfo.Size) {
|
||||
@@ -1484,7 +1551,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
// Pass the decompressed stream to such calls.
|
||||
isDstCompressed := isCompressible(r.Header, dstObject) &&
|
||||
length > minCompressibleSize &&
|
||||
!isRemoteCopyRequired(ctx, srcBucket, dstBucket, objectAPI)
|
||||
!remoteCallRequired
|
||||
if isDstCompressed {
|
||||
compressMetadata = make(map[string]string, 2)
|
||||
// Preserving the compression metadata.
|
||||
@@ -1613,6 +1680,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
var targetSize int64
|
||||
|
||||
switch {
|
||||
case remoteCallRequired:
|
||||
// The remote receives the logical bytes, see above.
|
||||
targetSize = actualSize
|
||||
case isDstCompressed:
|
||||
targetSize = -1
|
||||
case !isSourceEncrypted && !isTargetEncrypted:
|
||||
@@ -1683,7 +1753,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
pReader.setChecksumReader(checksumReader)
|
||||
}
|
||||
|
||||
if isTargetEncrypted {
|
||||
if isTargetEncrypted && !remoteCallRequired {
|
||||
var encReader io.Reader
|
||||
kind, _ := crypto.IsRequested(r.Header)
|
||||
encReader, objEncKey, err = newEncryptReader(ctx, srcInfo.Reader, kind, newKeyID, newKey, dstBucket, dstObject, encMetadata, kmsCtx)
|
||||
@@ -1707,7 +1777,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
return
|
||||
}
|
||||
|
||||
if isTargetEncrypted {
|
||||
if isTargetEncrypted && !remoteCallRequired {
|
||||
pReader, err = pReader.WithEncryption(srcInfo.Reader, &objEncKey)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
@@ -1774,6 +1844,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
applyReplicatedObjectLock(srcInfo.UserDefined, storedLock, replicaTrusted,
|
||||
retentionMode, retentionDate, legalHold,
|
||||
dstOpts.ReplicationSourceRetentionTimestamp, dstOpts.ReplicationSourceLegalholdTimestamp)
|
||||
dstOpts.ReplicaLockReconcile = replicaTrusted && dstOpts.VersionID != ""
|
||||
|
||||
if replicaTrusted {
|
||||
// An SSE-C key rotation snapshots every stored reserved key into
|
||||
@@ -1860,9 +1931,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
srcInfo.metadataOnly = false
|
||||
}
|
||||
|
||||
// Federation only.
|
||||
remoteCallRequired := isRemoteCopyRequired(ctx, srcBucket, dstBucket, objectAPI)
|
||||
|
||||
var objInfo ObjectInfo
|
||||
var os *objSweeper
|
||||
if remoteCallRequired {
|
||||
@@ -1884,23 +1952,109 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
// Remove the metadata for remote calls.
|
||||
delete(srcInfo.UserDefined, ReservedMetadataPrefix+"compression")
|
||||
delete(srcInfo.UserDefined, ReservedMetadataPrefix+"actual-size")
|
||||
// A plain federated PutObject must not carry any internal storage
|
||||
// metadata. The remote rejects every reserved-prefix header as a class
|
||||
// (containsReservedMetadata), so strip the whole class here rather than
|
||||
// an enumerated subset: an inline source object also carries
|
||||
// inline-data, and replication bookkeeping adds still more, so removing
|
||||
// only compression/actual-size just defers the next rejected key. Match
|
||||
// the remote's case-insensitive detection.
|
||||
for k := range srcInfo.UserDefined {
|
||||
if stringsHasPrefixFold(k, ReservedMetadataPrefix) {
|
||||
delete(srcInfo.UserDefined, k)
|
||||
}
|
||||
}
|
||||
// Forward a clone: request-only keys are removed or added below and
|
||||
// srcInfo.UserDefined stays the resolved record for the response.
|
||||
forwardedMeta := cloneMSS(srcInfo.UserDefined)
|
||||
// minio-go prefixes any UserMetadata key it does not recognize with
|
||||
// x-amz-meta-, and it recognizes the retention headers but not
|
||||
// x-amz-object-lock-legal-hold, so a hold left in the map reaches the
|
||||
// remote as user metadata and is silently dropped (#166). Carry it on
|
||||
// the typed option instead. Retention stays in the map: the typed
|
||||
// RetainUntilDate would truncate the date to whole seconds.
|
||||
legalHoldKey := strings.ToLower(xhttp.AmzObjectLockLegalHold)
|
||||
forwardedLegalHold := forwardedMeta[legalHoldKey]
|
||||
delete(forwardedMeta, legalHoldKey)
|
||||
opts := miniogo.PutObjectOptions{
|
||||
UserMetadata: srcInfo.UserDefined,
|
||||
UserMetadata: forwardedMeta,
|
||||
ServerSideEncryption: dstOpts.ServerSideEncryption,
|
||||
UserTags: tag.ToMap(),
|
||||
LegalHold: miniogo.LegalHoldStatus(forwardedLegalHold),
|
||||
}
|
||||
remoteObjInfo, rerr := core.PutObject(ctx, dstBucket, dstObject, srcInfo.Reader,
|
||||
srcInfo.Size, "", "", opts)
|
||||
// The destination must carry the same checksum the local path would
|
||||
// produce; the federated path has the remote compute, validate, persist
|
||||
// and return it. Without this the federated copy silently returns an
|
||||
// empty checksum (#99).
|
||||
wantChecksumType := dstOpts.WantServerSideChecksumType
|
||||
var checksumHeaderValue string
|
||||
switch {
|
||||
case wantChecksumType.IsSet():
|
||||
if actualSize == 0 {
|
||||
// minio-go streams no trailing checksum for an empty body, so the
|
||||
// remote would compute none, the bind below would fail, and every
|
||||
// empty-object federated copy would 500. Forward the empty-content
|
||||
// digest as an ordinary checksum header instead, so the remote
|
||||
// validates, persists and returns it (parity with the local path,
|
||||
// e.g. CRC32 "AAAAAA==").
|
||||
if empty := hash.NewChecksumFromData(wantChecksumType, nil); empty != nil {
|
||||
checksumHeaderValue = empty.Encoded
|
||||
}
|
||||
} else {
|
||||
// Stream a trailing checksum of the requested type so the remote
|
||||
// computes and persists it over the copied bytes.
|
||||
opts.Checksum = federatedChecksumType(wantChecksumType)
|
||||
}
|
||||
case dstOpts.WantChecksum != nil && dstOpts.WantChecksum.Type.IsSet():
|
||||
// A full-object checksum inherited from a checksum-bearing source: the
|
||||
// local path persists it without recomputation (only multipart
|
||||
// composite sources are promoted to WantServerSideChecksumType above,
|
||||
// so this is never composite). Forward the value as an ordinary
|
||||
// checksum header so the remote validates and persists it, and bind
|
||||
// the returned value below. WantChecksum.Encoded is always a plain
|
||||
// digest, never a "-N" multipart form.
|
||||
wantChecksumType = dstOpts.WantChecksum.Type.Base()
|
||||
checksumHeaderValue = dstOpts.WantChecksum.Encoded
|
||||
}
|
||||
if checksumHeaderValue != "" {
|
||||
opts.UserMetadata[wantChecksumType.Key()] = checksumHeaderValue
|
||||
}
|
||||
// For an ordinary federated copy, actualSize is the logical plaintext
|
||||
// length to declare. srcInfo.Size is not a reliable wire length because
|
||||
// the read path may already have adjusted it.
|
||||
writeCtx, modified := withFederatedWriteTime(ctx)
|
||||
remoteObjInfo, rerr := core.PutObject(writeCtx, dstBucket, dstObject, srcInfo.Reader,
|
||||
actualSize, "", "", opts)
|
||||
if rerr != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, rerr), r.URL)
|
||||
return
|
||||
}
|
||||
objInfo.UserDefined = cloneMSS(opts.UserMetadata)
|
||||
// Keep the resolved legal hold in response and event metadata; the
|
||||
// request-only checksum header exists only in the forwarding map.
|
||||
objInfo.UserDefined = cloneMSS(srcInfo.UserDefined)
|
||||
// The response headers and the ObjectCreated:Copy event describe the
|
||||
// object this handler wrote, so name it: without these the response
|
||||
// carries no x-amz-version-id and the event has an empty key and zero
|
||||
// size (#170). Size is the logical size the remote was handed, which is
|
||||
// what it reports back.
|
||||
objInfo.Bucket = dstBucket
|
||||
objInfo.Name = dstObject
|
||||
objInfo.Size = actualSize
|
||||
objInfo.VersionID = remoteObjInfo.VersionID
|
||||
objInfo.ETag = remoteObjInfo.ETag
|
||||
objInfo.ModTime = remoteObjInfo.LastModified
|
||||
objInfo.ModTime = *modified
|
||||
// Bind the checksum the remote computed for this exact write. A single
|
||||
// forwarded PutObject must yield a full-object digest, so reject a
|
||||
// missing, malformed, or multipart-marked ("-N") value rather than
|
||||
// mislabel the destination as composite.
|
||||
if wantChecksumType.IsSet() {
|
||||
cs := hash.NewChecksumWithType(wantChecksumType, federatedChecksumValue(wantChecksumType, remoteObjInfo))
|
||||
if cs == nil || cs.Type.Is(hash.ChecksumMultipart) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
|
||||
return
|
||||
}
|
||||
objInfo.Checksum = cs.AppendTo(nil, nil)
|
||||
}
|
||||
} else {
|
||||
os = newObjSweeper(dstBucket, dstObject).WithVersioning(dstOpts.Versioned, dstOpts.VersionSuspended)
|
||||
// Get appropriate object info to identify the remote object to delete
|
||||
@@ -2274,10 +2428,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
// The decision above orders against the version as read here; let the
|
||||
// object layer re-run it against the version read under the write lock
|
||||
// that guards the replacement, so a newer hold or retention committed in
|
||||
// between is not rolled back (issue #120). Scoped to the SSE-C replica
|
||||
// retransmit this issue enables, keyed on the incoming write's restored
|
||||
// SSE-C seal, the same predicate as the duplicate-version exemption.
|
||||
opts.ReplicaLockReconcile = isReplicaTrusted(ctx) && opts.VersionID != "" && crypto.SSEC.IsEncrypted(metadata)
|
||||
// between is not rolled back, across all pools and encryption modes.
|
||||
opts.ReplicaLockReconcile = isReplicaTrusted(ctx) && opts.VersionID != ""
|
||||
}
|
||||
if s3Err != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
@@ -2392,6 +2544,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
|
||||
setPutObjHeaders(w, objInfo, false, r.Header)
|
||||
setFederatedWriteTime(w, r, objInfo.ModTime)
|
||||
|
||||
// Notify object created event.
|
||||
evt := eventArgs{
|
||||
@@ -3069,7 +3222,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
|
||||
if objInfo.ReplicationStatus == replication.Pending || objInfo.VersionPurgeStatus == replication.VersionPurgePending {
|
||||
dmVersionID := ""
|
||||
versionID := ""
|
||||
if objInfo.DeleteMarker {
|
||||
if objInfo.DeleteMarker && objInfo.VersionPurgeStatus.Empty() {
|
||||
dmVersionID = objInfo.VersionID
|
||||
} else {
|
||||
versionID = objInfo.VersionID
|
||||
@@ -3544,9 +3697,6 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
|
||||
}
|
||||
tagsStr := tags.String()
|
||||
|
||||
// Set this such that authorization policies can be applied on the object tags.
|
||||
r.Header.Set(xhttp.AmzObjectTagging, tagsStr)
|
||||
|
||||
logger.GetReqInfo(ctx).BucketName = bucket
|
||||
logger.GetReqInfo(ctx).ObjectName = object
|
||||
if s3Error := authenticateRequest(ctx, r, policy.PutObjectTaggingAction); s3Error != ErrNone {
|
||||
@@ -3554,6 +3704,12 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
|
||||
// Set this such that authorization policies can be applied on the object
|
||||
// tags. This is derived from the request body, so it must be injected only
|
||||
// after signature verification: the SigV4 verifier now rejects unsigned
|
||||
// x-amz-* request headers, and this synthesized header is never signed.
|
||||
r.Header.Set(xhttp.AmzObjectTagging, tagsStr)
|
||||
|
||||
opts, err := getOpts(ctx, r, bucket, object)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
|
||||
@@ -1799,6 +1799,12 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam
|
||||
req.Header.Set("X-Amz-Copy-Source", url.QueryEscape(pathJoin(bucketName, objectName)))
|
||||
req.Header.Set("X-Amz-Copy-Source-Range", fmt.Sprintf("bytes=%d-%d", a, b))
|
||||
|
||||
// Re-sign so the copy-source x-amz-* headers are covered by the
|
||||
// signature, as real clients do; the verifier rejects unsigned x-amz-*.
|
||||
if err = signRequestV4(req, credentials.AccessKey, credentials.SecretKey); err != nil {
|
||||
t.Fatalf("Test failed to re-sign HTTP request for copy object part: <ERROR> %v", err)
|
||||
}
|
||||
|
||||
// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
|
||||
// Call the ServeHTTP to execute the handler, `func (api objectAPIHandlers) CopyObjectHandler` handles the request.
|
||||
a = globalMinPartSize + 1
|
||||
@@ -2200,6 +2206,15 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
|
||||
}
|
||||
}
|
||||
|
||||
// Re-sign so the copy-source x-amz-* headers set above are covered by
|
||||
// the signature, as real clients do; the verifier rejects unsigned
|
||||
// x-amz-* headers.
|
||||
if testCase.accessKey != "" && testCase.secretKey != "" {
|
||||
if err = signRequestV4(req, testCase.accessKey, testCase.secretKey); err != nil {
|
||||
t.Fatalf("Test %d: Failed to re-sign HTTP request for copy Object: <ERROR> %v", i+1, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
|
||||
// Call the ServeHTTP to execute the handler, `func (api objectAPIHandlers) CopyObjectHandler` handles the request.
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
@@ -2626,6 +2641,16 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
|
||||
if testCase.metadataGarbage {
|
||||
req.Header.Set("X-Amz-Metadata-Directive", "Unknown")
|
||||
}
|
||||
// The x-amz-copy-source and related x-amz-* headers set above must be
|
||||
// part of the SigV4 signature, exactly as real S3 clients send them.
|
||||
// Re-sign now that they are present; the verifier rejects unsigned
|
||||
// x-amz-* headers (an unsigned x-amz-copy-source could otherwise turn a
|
||||
// PUT grant into a server-side copy).
|
||||
if testCase.accessKey != "" && testCase.secretKey != "" {
|
||||
if err = signRequestV4(req, testCase.accessKey, testCase.secretKey); err != nil {
|
||||
t.Fatalf("Test %d: Failed to re-sign HTTP request for copy Object: <ERROR> %v", i, err)
|
||||
}
|
||||
}
|
||||
// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
|
||||
// Call the ServeHTTP to execute the handler, `func (api objectAPIHandlers) CopyObjectHandler` handles the request.
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
func setTestBucketDefaultRetention(t *testing.T, bucket, mode string) {
|
||||
t.Helper()
|
||||
enableBucketObjectLock(t, bucket)
|
||||
if mode == "" {
|
||||
return
|
||||
}
|
||||
meta, err := globalBucketMetadataSys.Get(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta.ObjectLockConfigXML = fmt.Appendf(nil, `<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>%s</Mode><Days>2</Days></DefaultRetention></Rule></ObjectLockConfiguration>`, mode)
|
||||
if err := meta.parseAllConfigs(t.Context(), newObjectLayerFn()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
globalBucketMetadataSys.Set(bucket, meta)
|
||||
}
|
||||
|
||||
func TestAPIObjectLockDefaultRetentionWithLegalHold(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
endpoints: []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "CopyObject", "PutObject", "DeleteObject"},
|
||||
objAPITest: func(obj ObjectLayer, instanceType, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
data := []byte("default retention and legal hold are independent")
|
||||
putCopyChecksumSource(t, router, cred, bucket, "source", data, nil)
|
||||
for _, mode := range []string{"GOVERNANCE", "COMPLIANCE", ""} {
|
||||
setTestBucketDefaultRetention(t, bucket, mode)
|
||||
for _, hold := range []string{"ON", "OFF"} {
|
||||
for _, operation := range []string{"put", "copy", "multipart"} {
|
||||
t.Run(instanceType+"/"+mode+"/"+hold+"/"+operation, func(t *testing.T) {
|
||||
object := mode + "-" + hold + "-" + operation
|
||||
headers := map[string]string{xhttp.AmzObjectLockLegalHold: hold}
|
||||
// The stored retention date carries millisecond precision.
|
||||
before := UTCNow().Truncate(time.Millisecond)
|
||||
switch operation {
|
||||
case "put":
|
||||
putCopyChecksumSource(t, router, cred, bucket, object, data, headers)
|
||||
case "copy":
|
||||
rec := federatedCopyRequest(t, router, cred, bucket, "source", bucket, object, headers)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("copy: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
case "multipart":
|
||||
putFederationMultipartSource(t, router, cred, bucket, object, [][]byte{data}, headers)
|
||||
}
|
||||
info, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ret := objectlock.GetObjectRetentionMeta(info.UserDefined)
|
||||
if string(ret.Mode) != mode {
|
||||
t.Errorf("stored retention mode = %q, want %q", ret.Mode, mode)
|
||||
}
|
||||
if mode != "" && (ret.RetainUntilDate.Before(before.Add(48*time.Hour)) || ret.RetainUntilDate.After(UTCNow().Add(48*time.Hour))) {
|
||||
t.Errorf("default retention date = %s, want write time + 2 days", ret.RetainUntilDate)
|
||||
}
|
||||
if got := objectlock.GetObjectLegalHoldMeta(info.UserDefined).Status; string(got) != hold {
|
||||
t.Errorf("stored legal hold = %q, want %q", got, hold)
|
||||
}
|
||||
if mode == "COMPLIANCE" && hold == "OFF" {
|
||||
req, err := newTestSignedRequestV4(http.MethodDelete, getPutObjectURL("", bucket, object)+"?versionId="+info.VersionID, 0, nil, cred.AccessKey, cred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "InvalidRequest") {
|
||||
t.Errorf("version DELETE must enforce retention: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func TestObjectLockDefaultRetentionBoundaries(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: func(obj ObjectLayer, instanceType, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
setTestBucketDefaultRetention(t, bucket, "COMPLIANCE")
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
replica, marker, explicit bool
|
||||
retentionErr, holdErr APIErrorCode
|
||||
wantMode objectlock.RetMode
|
||||
wantErr APIErrorCode
|
||||
}{
|
||||
{name: "trusted replica", replica: true},
|
||||
{name: "marker-only ordinary write", marker: true, wantMode: objectlock.RetCompliance},
|
||||
{name: "explicit retention", explicit: true, wantMode: objectlock.RetGovernance},
|
||||
{name: "retention permission denied", retentionErr: ErrAccessDenied, wantErr: ErrAccessDenied},
|
||||
{name: "legal hold permission denied", holdErr: ErrAccessDenied, wantErr: ErrAccessDenied},
|
||||
} {
|
||||
t.Run(instanceType+"/"+tc.name, func(t *testing.T) {
|
||||
r := httptest.NewRequest(http.MethodPut, "http://minio.local/"+bucket+"/object", nil)
|
||||
r.Header.Set(xhttp.AmzObjectLockLegalHold, "OFF")
|
||||
if tc.marker {
|
||||
r.Header.Set(xhttp.MinIOSourceReplicationRequest, "true")
|
||||
}
|
||||
until := UTCNow().Add(24 * time.Hour).Truncate(time.Second).Add(789 * time.Millisecond)
|
||||
if tc.explicit {
|
||||
r.Header.Set(xhttp.AmzObjectLockMode, "GOVERNANCE")
|
||||
r.Header.Set(xhttp.AmzObjectLockRetainUntilDate, until.Format(time.RFC3339Nano))
|
||||
}
|
||||
mode, date, _, code := checkPutObjectLockAllowed(t.Context(), r, bucket, "object", obj.GetObjectInfo, tc.retentionErr, tc.holdErr, tc.replica)
|
||||
if code != tc.wantErr || mode != tc.wantMode {
|
||||
t.Fatalf("got mode %s error %s, want mode %s error %s", mode, niceError(code), tc.wantMode, niceError(tc.wantErr))
|
||||
}
|
||||
if tc.explicit && !date.Equal(until) {
|
||||
t.Errorf("explicit retention lost precision: %s != %s", date, until)
|
||||
}
|
||||
if tc.replica && !date.IsZero() {
|
||||
t.Errorf("replica acquired a destination default: %s", date)
|
||||
}
|
||||
})
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -30,6 +30,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
miniogo "github.com/minio/minio-go/v7"
|
||||
miniocredentials "github.com/minio/minio-go/v7/pkg/credentials"
|
||||
@@ -51,7 +52,7 @@ func TestAPIFederatedUploadPartChecksumResponse(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func testAPIFederatedUploadPartChecksumResponse(_ ObjectLayer, instanceType, bucketName string,
|
||||
func testAPIFederatedUploadPartChecksumResponse(objectAPI ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
algorithms := []struct {
|
||||
@@ -101,6 +102,18 @@ func testAPIFederatedUploadPartChecksumResponse(_ ObjectLayer, instanceType, buc
|
||||
if got := rec.Header().Get(xhttp.AmzChecksumType); got != "" {
|
||||
t.Fatalf("%s: UploadPart returned checksum type %q", instanceType, got)
|
||||
}
|
||||
modified := rec.Header().Get(federatedLastModified)
|
||||
if userAgent.want {
|
||||
parts, err := objectAPI.ListObjectParts(t.Context(), bucketName, object, uploadID, 0, 100, ObjectOptions{})
|
||||
if err != nil || len(parts.Parts) != 1 {
|
||||
t.Fatalf("stored part: %v, %v", parts, err)
|
||||
}
|
||||
if modified != parts.Parts[0].LastModified.UTC().Format(time.RFC3339Nano) {
|
||||
t.Errorf("response time %q does not match committed part time %s", modified, parts.Parts[0].LastModified)
|
||||
}
|
||||
} else if modified != "" {
|
||||
t.Errorf("ordinary UploadPart exposed federation timestamp %q", modified)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -158,7 +171,7 @@ func TestAPIFederatedUploadPartChecksumConcurrentOverwrite(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func testAPIFederatedUploadPartChecksumConcurrentOverwrite(_ ObjectLayer, instanceType, bucketName string,
|
||||
func testAPIFederatedUploadPartChecksumConcurrentOverwrite(objectAPI ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
object := "federation/concurrent-overwrite"
|
||||
@@ -194,6 +207,10 @@ func testAPIFederatedUploadPartChecksumConcurrentOverwrite(_ ObjectLayer, instan
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
parts, err := objectAPI.ListObjectParts(t.Context(), bucketName, object, uploadID, 0, 100, ObjectOptions{})
|
||||
if err != nil || len(parts.Parts) != 1 {
|
||||
t.Fatalf("stored part: %v, %v", parts, err)
|
||||
}
|
||||
|
||||
for i, rec := range recorders {
|
||||
if rec.Code != http.StatusOK {
|
||||
@@ -213,6 +230,13 @@ func testAPIFederatedUploadPartChecksumConcurrentOverwrite(_ ObjectLayer, instan
|
||||
if want := hex.EncodeToString(md5sum[:]); canonicalizeETag(etags[0]) != want {
|
||||
t.Fatalf("%s: concurrent request %d ETag %q, want %q", instanceType, i, etags[0], want)
|
||||
}
|
||||
modified, err := time.Parse(time.RFC3339Nano, rec.Header().Get(federatedLastModified))
|
||||
if err != nil || modified.IsZero() {
|
||||
t.Fatalf("concurrent request %d returned invalid time %v: %v", i, modified, err)
|
||||
}
|
||||
if canonicalizeETag(etags[0]) == parts.Parts[0].ETag && !modified.Equal(parts.Parts[0].LastModified) {
|
||||
t.Errorf("surviving part has time %s, its writer returned %s", parts.Parts[0].LastModified, modified)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -352,6 +376,9 @@ func testAPIFederatedCopyObjectPartChecksum(objectAPI ObjectLayer, instanceType,
|
||||
if len(parts.Parts) != 1 {
|
||||
t.Fatalf("%s: ListParts returned %d parts, want 1", instanceType, len(parts.Parts))
|
||||
}
|
||||
if response.LastModified != parts.Parts[0].LastModified {
|
||||
t.Errorf("CopyPartResult time = %q, want stored part time %q", response.LastModified, parts.Parts[0].LastModified)
|
||||
}
|
||||
if got := partChecksum(algorithm.typ, parts.Parts[0]); got != want {
|
||||
t.Fatalf("%s: persisted part %s is %q, want %q", instanceType, algorithm.typ.String(), got, want)
|
||||
}
|
||||
|
||||
@@ -59,8 +59,8 @@ import (
|
||||
//
|
||||
// This is only a response-shape hint. User-Agent is not authenticated and must
|
||||
// never gate authorization, object visibility, or request validation. It is
|
||||
// safe here because the only effect is returning the checksum of the body the
|
||||
// caller was already authorized to upload.
|
||||
// safe here because the only effect is returning the checksum and modification
|
||||
// time of the body the caller was already authorized to upload.
|
||||
func isFederatedInternalRequest(userAgent string) bool {
|
||||
for _, product := range strings.Fields(userAgent) {
|
||||
name, version, ok := strings.Cut(product, "/")
|
||||
@@ -572,7 +572,8 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
||||
SSE: dstOpts.ServerSideEncryption,
|
||||
}
|
||||
|
||||
partInfo, err := core.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, gr, length, popts)
|
||||
writeCtx, modified := withFederatedWriteTime(ctx)
|
||||
partInfo, err := core.PutObjectPart(writeCtx, dstBucket, dstObject, uploadID, partID, gr, length, popts)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -580,7 +581,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
||||
|
||||
response := generateCopyObjectPartResponse(PartInfo{
|
||||
ETag: partInfo.ETag,
|
||||
LastModified: partInfo.LastModified,
|
||||
LastModified: *modified,
|
||||
ChecksumCRC32: partInfo.ChecksumCRC32,
|
||||
ChecksumCRC32C: partInfo.ChecksumCRC32C,
|
||||
ChecksumSHA1: partInfo.ChecksumSHA1,
|
||||
@@ -1082,6 +1083,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
|
||||
// checksum cannot be mixed with a concurrent overwrite.
|
||||
hash.AddChecksumHeader(w, partChecksumMap(partInfo))
|
||||
}
|
||||
setFederatedWriteTime(w, r, partInfo.LastModified)
|
||||
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
@@ -165,6 +165,11 @@ func testAPIZeroByteSSECAuthenticatesKey(obj ObjectLayer, instanceType, bucketNa
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Header.Set(xhttp.AmzCopySource, SlashSeparator+pathJoin(bucketName, object))
|
||||
// Re-sign so x-amz-copy-source is covered by the signature, as real S3
|
||||
// clients send it; the verifier rejects unsigned x-amz-* headers.
|
||||
if err = signRequestV4(req, credentials.AccessKey, credentials.SecretKey); err != nil {
|
||||
t.Fatalf("%s: failed to re-sign UploadPartCopy request: %v", instanceType, err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusForbidden {
|
||||
|
||||
@@ -544,6 +544,17 @@ func (s *peerRESTServer) LoadBucketMetadataHandler(mss *grid.MSS) (np grid.NoPay
|
||||
return np, grid.NewRemoteErr(err)
|
||||
}
|
||||
|
||||
// Publish the reloaded metadata unconditionally. Overlapping peer reloads
|
||||
// may briefly leave the resident cache a revision behind, or momentarily
|
||||
// apply an older reload's derived registries. The resident cache may lag
|
||||
// until another authoritative update advances it; the periodic metadata
|
||||
// refresh is only best-effort here and cannot repair a divergence whose
|
||||
// maximum per-config timestamp already equals the resident record's (its
|
||||
// staleness check compares the same lastUpdate() and would see no change).
|
||||
// This acceptable-until-refresh behavior is deliberate: lastUpdate() is the
|
||||
// max of per-config timestamps and cannot order whole-record revisions, so a
|
||||
// publication guard keyed on it would wrongly reject legitimately newer
|
||||
// records (issue #105 follow-up dropped that guard).
|
||||
globalBucketMetadataSys.Set(bucketName, meta)
|
||||
|
||||
if meta.notificationConfig != nil {
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
func TestPresignedHeaderPayloadHashIsEnforced(t *testing.T) {
|
||||
setupSignatureBoundaryTest(t)
|
||||
digest := getSHA256Hash([]byte("expected"))
|
||||
for _, tc := range []struct {
|
||||
name, query, header string
|
||||
unsigned bool
|
||||
}{
|
||||
{name: "header only", header: digest},
|
||||
{name: "query only", query: digest},
|
||||
{name: "query wins", query: digest, header: getSHA256Hash([]byte("different"))},
|
||||
{name: "unsigned query wins", query: unsignedPayload, header: digest, unsigned: true},
|
||||
{name: "unsigned header", header: unsignedPayload, unsigned: true},
|
||||
{name: "absent", unsigned: true},
|
||||
} {
|
||||
for _, body := range []string{"expected", "modified"} {
|
||||
t.Run(tc.name+"/"+body, func(t *testing.T) {
|
||||
r := httptest.NewRequest(http.MethodPut, "http://minio.local/bucket/object", strings.NewReader(body))
|
||||
if tc.header != "" {
|
||||
r.Header.Set(xhttp.AmzContentSha256, tc.header)
|
||||
}
|
||||
if tc.query != "" {
|
||||
query := r.URL.Query()
|
||||
query.Set(xhttp.AmzContentSha256, tc.query)
|
||||
r.URL.RawQuery = query.Encode()
|
||||
}
|
||||
presignBoundaryRequest(t, r, UTCNow(), []string{"host"}, globalActiveCred)
|
||||
if code := isReqAuthenticated(t.Context(), r, globalSite.Region(), serviceS3); code != ErrNone {
|
||||
t.Fatal(niceError(code))
|
||||
}
|
||||
_, err := io.ReadAll(r.Body)
|
||||
wantMismatch := body == "modified" && !tc.unsigned
|
||||
if wantMismatch {
|
||||
if toAPIErrorCode(t.Context(), err) != ErrContentSHA256Mismatch {
|
||||
t.Errorf("modified signed payload: got %v, want SHA-256 mismatch", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Errorf("valid payload: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIPresignedBucketPolicyPayloadHash(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: func(objectAPI ObjectLayer, instanceType, bucket string, apiRouter http.Handler, credentials auth.Credentials, t *testing.T) {
|
||||
server := httptest.NewServer(apiRouter)
|
||||
defer server.Close()
|
||||
client := server.Client()
|
||||
client.Timeout = 10 * time.Second
|
||||
expected := fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/*"]}]}`, bucket)
|
||||
modified := strings.Replace(expected, "s3:GetObject", "s3:PutObject", 1)
|
||||
for _, tc := range []struct {
|
||||
name, body string
|
||||
status int
|
||||
}{
|
||||
{"original", expected, http.StatusNoContent},
|
||||
{"modified", modified, http.StatusBadRequest},
|
||||
} {
|
||||
t.Run(instanceType+"/"+tc.name, func(t *testing.T) {
|
||||
r, err := http.NewRequestWithContext(t.Context(), http.MethodPut, server.URL+"/"+bucket+"?policy=", strings.NewReader(tc.body))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.Header.Set(xhttp.AmzContentSha256, getSHA256Hash([]byte(expected)))
|
||||
presignBoundaryRequest(t, r, UTCNow(), []string{"host"}, credentials)
|
||||
response, err := client.Do(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
body, err := io.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.StatusCode != tc.status {
|
||||
t.Fatalf("PutBucketPolicy: got %d %s, want %d", response.StatusCode, body, tc.status)
|
||||
}
|
||||
if tc.status == http.StatusBadRequest && !strings.Contains(string(body), "XAmzContentSHA256Mismatch") {
|
||||
t.Fatalf("expected payload checksum rejection, got %s", body)
|
||||
}
|
||||
})
|
||||
}
|
||||
meta, err := globalBucketMetadataSys.Get(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(meta.PolicyConfigJSON), "s3:PutObject") {
|
||||
t.Fatal("tampered presigned request replaced the bucket policy")
|
||||
}
|
||||
},
|
||||
endpoints: []string{"PutBucketPolicy"},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/once"
|
||||
)
|
||||
|
||||
// Exercise the actual single-object DELETE handler and erasure metadata. A
|
||||
// marker purge must delete the target version and finish the source purge;
|
||||
// merely seeing a 405 on HEAD is not evidence of a completed permanent delete.
|
||||
func TestReplicateDeleteMarkerPurge(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
for _, legacy := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("recover_legacy_%v", legacy), func(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t, endpoints: []string{"DeleteObject"},
|
||||
objAPITest: func(obj ObjectLayer, instanceType, bucket string, router http.Handler, creds auth.Credentials, t *testing.T) {
|
||||
testReplicateDeleteMarkerPurge(obj, instanceType, bucket, router, creds, t, legacy)
|
||||
},
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplicateDeleteMarkerTargetSemantics(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
purge bool
|
||||
deleteCode int
|
||||
wantDelete bool
|
||||
wantFailed bool
|
||||
}{
|
||||
{name: "existing marker is idempotent"},
|
||||
{name: "purge removes an existing marker", purge: true, deleteCode: 204, wantDelete: true},
|
||||
{name: "purge forbidden", purge: true, deleteCode: 403, wantDelete: true, wantFailed: true},
|
||||
{name: "purge method rejected", purge: true, deleteCode: 405, wantDelete: true, wantFailed: true},
|
||||
{name: "purge unavailable", purge: true, deleteCode: 503, wantDelete: true, wantFailed: true},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var deletes atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodHead {
|
||||
w.Header().Set(xhttp.AmzDeleteMarker, "true")
|
||||
w.Header().Set(xhttp.AmzVersionID, r.URL.Query().Get("versionId"))
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
deletes.Add(1)
|
||||
w.WriteHeader(tt.deleteCode)
|
||||
if tt.wantFailed {
|
||||
code := map[int]string{403: "AccessDenied", 405: "MethodNotAllowed", 503: "ServiceUnavailable"}[tt.deleteCode]
|
||||
fmt.Fprintf(w, `<Error><Code>%s</Code><Message>injected delete rejection</Message></Error>`, code)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := minio.New(strings.TrimPrefix(server.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := globalBucketTargetSys
|
||||
globalBucketTargetSys = &BucketTargetSys{hc: map[string]epHealth{client.EndpointURL().Host: {Online: true}}}
|
||||
defer func() { globalBucketTargetSys = old }()
|
||||
deletion := DeletedObjectReplicationInfo{Bucket: "source", DeletedObject: DeletedObject{
|
||||
ObjectName: "marker", DeleteMarker: true, DeleteMarkerVersionID: mustGetUUID(),
|
||||
}}
|
||||
if tt.purge {
|
||||
deletion.VersionID = deletion.DeleteMarkerVersionID
|
||||
deletion.DeleteMarkerVersionID = ""
|
||||
deletion.ReplicationState.PurgeTargets = map[string]VersionPurgeStatusType{"arn1": replication.VersionPurgePending}
|
||||
}
|
||||
result := replicateDeleteToTarget(t.Context(), deletion, &TargetClient{Client: client, ARN: "arn1", Bucket: "target"})
|
||||
if (deletes.Load() != 0) != tt.wantDelete {
|
||||
t.Fatalf("remote DELETE count = %d, want delete %v", deletes.Load(), tt.wantDelete)
|
||||
}
|
||||
if tt.purge {
|
||||
want := replication.VersionPurgeComplete
|
||||
if tt.wantFailed {
|
||||
want = replication.VersionPurgeFailed
|
||||
}
|
||||
if result.VersionPurgeStatus != want || (result.Err != nil) != tt.wantFailed {
|
||||
t.Errorf("purge result = %+v, want %s", result, want)
|
||||
}
|
||||
} else if result.ReplicationStatus != replication.Completed {
|
||||
t.Errorf("existing marker result = %+v, want Completed", result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func testReplicateDeleteMarkerPurge(obj ObjectLayer, instanceType, bucket string, router http.Handler, creds auth.Credentials, t *testing.T, legacy bool) {
|
||||
ctx := t.Context()
|
||||
const arn = "arn:minio:replication::af470089-d354-4473-934c-9e1f52f6da89:bucket"
|
||||
const name = "marker"
|
||||
version := mustGetUUID()
|
||||
remoteBucket := getRandomBucketName()
|
||||
if err := obj.MakeBucket(ctx, remoteBucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, b := range []string{bucket, remoteBucket} {
|
||||
if _, err := obj.PutObject(ctx, b, name, mustGetPutObjReader(t, bytes.NewReader([]byte("data")), 4, "", ""), ObjectOptions{Versioned: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts := ObjectOptions{VersionID: version, Versioned: true, DeleteMarker: true, ReplicationRequest: true, MTime: UTCNow()}
|
||||
opts.SetReplicaStatus(replication.Replica)
|
||||
if _, err := obj.DeleteObject(ctx, b, name, opts); err != nil {
|
||||
t.Fatalf("%s: seed marker in %s: %v", instanceType, b, err)
|
||||
}
|
||||
}
|
||||
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
opts := ObjectOptions{VersionID: r.URL.Query().Get("versionId"), Versioned: true}
|
||||
switch r.Method {
|
||||
case http.MethodHead:
|
||||
oi, err := obj.GetObjectInfo(r.Context(), remoteBucket, name, opts)
|
||||
if oi.DeleteMarker {
|
||||
w.Header().Set(xhttp.AmzDeleteMarker, "true")
|
||||
w.Header().Set(xhttp.AmzVersionID, oi.VersionID)
|
||||
}
|
||||
if err != nil {
|
||||
writeErrorResponseHeadersOnly(w, toAPIError(r.Context(), err))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case http.MethodDelete:
|
||||
opts.DeleteMarker = r.Header.Get(xhttp.MinIOSourceDeleteMarker) == "true"
|
||||
opts.SetReplicaStatus(replication.Replica)
|
||||
_, err := obj.DeleteObject(r.Context(), remoteBucket, name, opts)
|
||||
if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
default:
|
||||
t.Errorf("unexpected remote method %s", r.Method)
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
}
|
||||
}))
|
||||
defer remote.Close()
|
||||
client, err := minio.New(strings.TrimPrefix(remote.URL, "http://"), &minio.Options{Region: "us-east-1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target := &TargetClient{Client: client, ARN: arn, Bucket: remoteBucket}
|
||||
globalBucketTargetSys.Lock()
|
||||
globalBucketTargetSys.arnRemotesMap[arn] = arnTarget{Client: target, lastRefresh: UTCNow()}
|
||||
globalBucketTargetSys.targetsMap[bucket] = []madmin.BucketTarget{{Arn: arn, TargetBucket: remoteBucket}}
|
||||
globalBucketTargetSys.Unlock()
|
||||
globalBucketTargetSys.hMutex.Lock()
|
||||
globalBucketTargetSys.hc[client.EndpointURL().Host] = epHealth{Online: true}
|
||||
globalBucketTargetSys.hMutex.Unlock()
|
||||
meta, err := globalBucketMetadataSys.Get(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := configs[0]
|
||||
cfg.RoleArn = arn
|
||||
meta.replicationConfig = &cfg
|
||||
globalBucketMetadataSys.Set(bucket, meta)
|
||||
worker := make(chan ReplicationWorkerOperation, 1)
|
||||
p := &ReplicationPool{
|
||||
ctx: ctx,
|
||||
objLayer: obj,
|
||||
workers: []chan ReplicationWorkerOperation{worker},
|
||||
stats: globalReplicationStats.Load(),
|
||||
mrfSaveCh: make(chan MRFReplicateEntry, 1),
|
||||
}
|
||||
oldPool := globalReplicationPool
|
||||
globalReplicationPool = once.NewSingleton[ReplicationPool]()
|
||||
globalReplicationPool.Set(p)
|
||||
defer func() { globalReplicationPool = oldPool }()
|
||||
|
||||
req, err := newTestSignedRequestV4(http.MethodDelete, "/"+bucket+"/"+name+"?versionId="+version, 0, nil, creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusNoContent {
|
||||
t.Fatalf("DELETE status %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
var deletion DeletedObjectReplicationInfo
|
||||
select {
|
||||
case op := <-worker:
|
||||
deletion = op.(DeletedObjectReplicationInfo)
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("DELETE did not schedule replication")
|
||||
}
|
||||
if deletion.VersionID != version || deletion.DeleteMarkerVersionID != "" {
|
||||
t.Errorf("purge scheduled as marker creation: version=%q marker=%q", deletion.VersionID, deletion.DeleteMarkerVersionID)
|
||||
}
|
||||
if legacy {
|
||||
// Reproduce the old producer's state and let the existing scanner/heal
|
||||
// path recover it. Upgrades must also finish purges already left pending.
|
||||
deletion.VersionID, deletion.DeleteMarkerVersionID = "", version
|
||||
replicateDelete(ctx, deletion, obj)
|
||||
oi, _ := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: version, Versioned: true})
|
||||
if oi.VersionPurgeStatus != replication.VersionPurgePending {
|
||||
t.Fatalf("legacy source purge = %s, want PENDING", oi.VersionPurgeStatus)
|
||||
}
|
||||
targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queueReplicationHeal(ctx, bucket, oi, replicationConfig{Config: &cfg, remotes: targets}, 0)
|
||||
select {
|
||||
case op := <-worker:
|
||||
deletion = op.(DeletedObjectReplicationInfo)
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("legacy pending purge was not scheduled for healing")
|
||||
}
|
||||
}
|
||||
result := replicateDelete(context.Background(), deletion, obj)
|
||||
if result.VersionPurgeStatus() != replication.VersionPurgeComplete {
|
||||
t.Errorf("remote purge result = %s, want COMPLETE", result.VersionPurgeStatus())
|
||||
}
|
||||
for _, b := range []string{bucket, remoteBucket} {
|
||||
oi, err := obj.GetObjectInfo(ctx, b, name, ObjectOptions{VersionID: version, Versioned: true})
|
||||
if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Errorf("%s: marker remains in %s: %+v, err=%v", instanceType, b, oi, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
// A full MRF queue and an exhausted retry budget drop queue entries, not the
|
||||
// source objects. Both drops must remain visible in the admin API snapshots.
|
||||
func TestReplicationMRFDropsVisible(t *testing.T) {
|
||||
stats := NewReplicationStats(t.Context(), nil)
|
||||
old := globalReplicationStats.Swap(stats)
|
||||
t.Cleanup(func() { globalReplicationStats.Store(old) })
|
||||
p := &ReplicationPool{
|
||||
objLayer: &replicationMRFTestObjectLayer{},
|
||||
stats: stats,
|
||||
mrfSaveCh: make(chan MRFReplicateEntry, 1),
|
||||
mrfStopCh: make(chan struct{}),
|
||||
}
|
||||
p.queueMRFSave(MRFReplicateEntry{sz: 10})
|
||||
p.queueMRFSave(MRFReplicateEntry{sz: 20})
|
||||
p.queueMRFSave(MRFReplicateEntry{sz: 30, RetryCount: mrfRetryLimit + 1})
|
||||
if len(p.mrfSaveCh) != 1 {
|
||||
t.Fatalf("queue length = %d, want 1", len(p.mrfSaveCh))
|
||||
}
|
||||
if got := atomic.LoadUint64(&stats.mrfStats.TotalDroppedCount); got != 2 {
|
||||
t.Fatalf("dropped entries = %d, want 2", got)
|
||||
}
|
||||
for name, qs := range map[string]ReplQNodeStats{
|
||||
"bucket": stats.getNodeQueueStats("test-bucket"),
|
||||
"node": stats.getNodeQueueStatsSummary(),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
data, err := json.Marshal(qs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var decoded ReplQNodeStats
|
||||
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := decoded.MRFStats; got.TotalDroppedCount != 2 || got.TotalDroppedBytes != 50 {
|
||||
t.Fatalf("API MRF stats = %+v, want dropped count 2 and bytes 50", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
oldTargets := globalBucketTargetSys
|
||||
globalBucketTargetSys = &BucketTargetSys{}
|
||||
t.Cleanup(func() { globalBucketTargetSys = oldTargets })
|
||||
want := map[string]float64{"mrf_dropped_operations_total": 2, "mrf_dropped_bytes_total": 50}
|
||||
seen := make(map[string]bool)
|
||||
for _, metric := range getReplicationNodeMetrics(MetricsGroupOpts{}).Get() {
|
||||
name := string(metric.Description.Name)
|
||||
if value, ok := want[name]; ok {
|
||||
seen[name] = true
|
||||
if metric.Value != value || metric.Description.Type != counterMetric {
|
||||
t.Errorf("v2 %s = %+v, want counter %v", name, metric, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
for name := range want {
|
||||
if !seen[name] {
|
||||
t.Errorf("v2 does not expose %s", name)
|
||||
}
|
||||
}
|
||||
groups := newMetricGroups(prometheus.NewRegistry())
|
||||
families, err := groups.mgGatherers[replicationCollectorPath].Gather()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seen = make(map[string]bool)
|
||||
for _, family := range families {
|
||||
for name, value := range want {
|
||||
if family.GetName() != replicationCollectorPath.metricPrefix()+"_"+name {
|
||||
continue
|
||||
}
|
||||
seen[name] = true
|
||||
if len(family.Metric) != 1 || family.Metric[0].Counter == nil || family.Metric[0].GetCounter().GetValue() != value {
|
||||
t.Errorf("v3 %s = %v, want counter %v", name, family, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
for name := range want {
|
||||
if !seen[name] {
|
||||
t.Errorf("v3 registry does not expose %s", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type replicationMRFTestObjectLayer struct{ ObjectLayer }
|
||||
|
||||
func TestReplicationObjectDeleteWorkerAffinity(t *testing.T) {
|
||||
p := &ReplicationPool{ctx: t.Context(), workers: make([]chan ReplicationWorkerOperation, 8)}
|
||||
for i := range p.workers {
|
||||
p.workers[i] = make(chan ReplicationWorkerOperation, 2)
|
||||
}
|
||||
for _, op := range []replication.Type{replication.ObjectReplicationType, replication.HealReplicationType, replication.ExistingObjectReplicationType} {
|
||||
for i := range 10 {
|
||||
name := fmt.Sprintf("object-%d", i)
|
||||
p.queueReplicaTask(ReplicateObjectInfo{Bucket: "bucket", Name: name, OpType: op})
|
||||
p.queueReplicaDeleteTask(DeletedObjectReplicationInfo{Bucket: "bucket", DeletedObject: DeletedObject{ObjectName: name}})
|
||||
objectWorker, deleteWorker := -1, -1
|
||||
for idx, ch := range p.workers {
|
||||
for len(ch) > 0 {
|
||||
switch (<-ch).(type) {
|
||||
case ReplicateObjectInfo:
|
||||
objectWorker = idx
|
||||
case DeletedObjectReplicationInfo:
|
||||
deleteWorker = idx
|
||||
}
|
||||
}
|
||||
}
|
||||
if objectWorker < 0 || objectWorker != deleteWorker {
|
||||
t.Fatalf("operation %d, %s: object worker %d != delete worker %d", op, name, objectWorker, deleteWorker)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,442 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio/internal/once"
|
||||
)
|
||||
|
||||
func TestSiteResyncCancelState(t *testing.T) {
|
||||
globalSiteReplicationSys.Lock()
|
||||
old := globalSiteReplicationSys.enabled
|
||||
globalSiteReplicationSys.enabled = true
|
||||
globalSiteReplicationSys.Unlock()
|
||||
t.Cleanup(func() {
|
||||
globalSiteReplicationSys.Lock()
|
||||
globalSiteReplicationSys.enabled = old
|
||||
globalSiteReplicationSys.Unlock()
|
||||
})
|
||||
rs := newSiteResyncStatus("peer", []BucketInfo{{Name: "one"}, {Name: "two"}})
|
||||
sm := &siteResyncMetrics{
|
||||
resyncStatus: map[string]SiteResyncStatus{rs.ResyncID: rs.clone()},
|
||||
peerResyncMap: map[string]resyncState{"peer": {resyncID: rs.ResyncID}},
|
||||
}
|
||||
rs.Status = ResyncCanceled
|
||||
if err := sm.updateState(rs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := sm.status("peer")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Status != ResyncCanceled {
|
||||
t.Fatalf("cancel returned without updating site state: got %s", got.Status)
|
||||
}
|
||||
for _, bucket := range []string{"one", "two"} {
|
||||
sm.incBucket(resyncOpts{bucket: bucket, resyncID: rs.ResyncID}, ResyncCanceled)
|
||||
}
|
||||
got, err = sm.status("peer")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for bucket, status := range got.BucketStatuses {
|
||||
if status != ResyncCanceled {
|
||||
t.Errorf("bucket %s = %s, want Canceled", bucket, status)
|
||||
}
|
||||
}
|
||||
// An already-running worker must not resurrect a canceled site.
|
||||
sm.incBucket(resyncOpts{bucket: "one", resyncID: rs.ResyncID}, ResyncCompleted)
|
||||
got, _ = sm.status("peer")
|
||||
if got.Status != ResyncCanceled || got.BucketStatuses["one"] != ResyncCanceled {
|
||||
t.Fatalf("late completion overwrote canceled state: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The fixture runs the real resyncBucket control flow with in-memory metadata
|
||||
// persistence. Walk is deliberately controllable so cancellation does not
|
||||
// depend on disk speed, network timing or the walker closing its output.
|
||||
type resyncCancelObjectLayer struct {
|
||||
ObjectLayer
|
||||
walk func(context.Context, chan<- itemOrErr[ObjectInfo]) error
|
||||
lock RWLocker
|
||||
mu sync.Mutex
|
||||
saved map[string]BucketReplicationResyncStatus
|
||||
}
|
||||
|
||||
func (o *resyncCancelObjectLayer) NewNSLock(bucket string, objects ...string) RWLocker {
|
||||
return o.lock
|
||||
}
|
||||
|
||||
func (o *resyncCancelObjectLayer) Walk(ctx context.Context, bucket, prefix string, results chan<- itemOrErr[ObjectInfo], opts WalkOptions) error {
|
||||
return o.walk(ctx, results)
|
||||
}
|
||||
|
||||
func (o *resyncCancelObjectLayer) PutObject(_ context.Context, bucket, object string, r *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
|
||||
data, err := io.ReadAll(r)
|
||||
if err == nil && o.saved != nil {
|
||||
var status BucketReplicationResyncStatus
|
||||
_, err = status.UnmarshalMsg(data[4:])
|
||||
o.mu.Lock()
|
||||
o.saved[object] = status
|
||||
o.mu.Unlock()
|
||||
}
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
|
||||
func (o *resyncCancelObjectLayer) GetObjectNInfo(_ context.Context, bucket, object string, _ *HTTPRangeSpec, _ http.Header, opts ObjectOptions) (*GetObjectReader, error) {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
status, ok := o.saved[object]
|
||||
if !ok {
|
||||
return nil, ObjectNotFound{Bucket: bucket, Object: object}
|
||||
}
|
||||
data := make([]byte, 4)
|
||||
binary.LittleEndian.PutUint16(data[:2], resyncMetaFormat)
|
||||
binary.LittleEndian.PutUint16(data[2:], resyncMetaVersion)
|
||||
data, err := status.MarshalMsg(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewGetObjectReaderFromReader(bytes.NewReader(data), ObjectInfo{Size: int64(len(data))}, opts)
|
||||
}
|
||||
|
||||
func TestResyncRecoveryOwnsLeaderContext(t *testing.T) {
|
||||
for _, loseLeader := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("lose_leader_%v", loseLeader), func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
var walkCtx context.Context
|
||||
var output chan<- itemOrErr[ObjectInfo]
|
||||
obj := &resyncCancelObjectLayer{saved: make(map[string]BucketReplicationResyncStatus), walk: func(ctx context.Context, ch chan<- itemOrErr[ObjectInfo]) error {
|
||||
walkCtx, output = ctx, ch
|
||||
return nil
|
||||
}}
|
||||
s, opts := setupResyncCancelTest(t, obj)
|
||||
if err := saveResyncStatus(t.Context(), opts.bucket, s.statusMap[opts.bucket], obj); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
leaderCtx, lose := context.WithCancel(t.Context())
|
||||
defer lose()
|
||||
leader := &sharedLock{lockContext: make(chan LockContext, 1)}
|
||||
leader.lockContext <- LockContext{ctx: leaderCtx}
|
||||
oldLeader := globalLeaderLock
|
||||
globalLeaderLock = leader
|
||||
defer func() { globalLeaderLock = oldLeader }()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- (&ReplicationPool{resyncer: s}).loadResync(t.Context(), []string{opts.bucket}, obj) }()
|
||||
synctest.Wait()
|
||||
if walkCtx == nil || walkCtx.Err() != nil {
|
||||
t.Fatal("recovery canceled its worker at startup")
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("recovery released its leader context before the run finished")
|
||||
default:
|
||||
}
|
||||
want := ResyncCompleted
|
||||
if loseLeader {
|
||||
want = ResyncFailed
|
||||
lose()
|
||||
} else {
|
||||
close(output)
|
||||
}
|
||||
synctest.Wait()
|
||||
if err := <-done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if walkCtx.Err() == nil {
|
||||
t.Fatal("finished recovery leaked the Walk context")
|
||||
}
|
||||
if got := s.statusMap[opts.bucket].TargetsMap[opts.arn].ResyncStatus; got != want {
|
||||
t.Fatalf("recovery status = %s, want %s", got, want)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResyncCancelRouting(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
parent, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
walks := make(chan context.Context, 4)
|
||||
obj := &resyncCancelObjectLayer{walk: func(ctx context.Context, output chan<- itemOrErr[ObjectInfo]) error {
|
||||
walks <- ctx
|
||||
return nil
|
||||
}}
|
||||
s, opts := setupResyncCancelTest(t, obj)
|
||||
add := func(bucket, id string) resyncOpts {
|
||||
o := opts
|
||||
o.bucket, o.resyncID = bucket, id
|
||||
s.Lock()
|
||||
m := newBucketResyncStatus(bucket)
|
||||
m.TargetsMap[o.arn] = TargetReplicationResyncStatus{ResyncID: id, ResyncStatus: ResyncPending}
|
||||
s.statusMap[bucket] = m
|
||||
s.Unlock()
|
||||
meta, _ := globalBucketMetadataSys.Get(opts.bucket)
|
||||
globalBucketMetadataSys.Set(bucket, meta)
|
||||
globalBucketTargetSys.Lock()
|
||||
globalBucketTargetSys.targetsMap[bucket] = []madmin.BucketTarget{{Arn: o.arn}}
|
||||
globalBucketTargetSys.Unlock()
|
||||
return o
|
||||
}
|
||||
run := func(o resyncOpts) chan struct{} {
|
||||
done := make(chan struct{})
|
||||
go func() { s.resyncBucket(parent, obj, false, o); close(done) }()
|
||||
return done
|
||||
}
|
||||
first := run(opts)
|
||||
synctest.Wait()
|
||||
firstCtx := <-walks
|
||||
queuedOpts := add("queued", opts.resyncID)
|
||||
queued := run(queuedOpts)
|
||||
otherOpts := add("other", "other-id")
|
||||
other := run(otherOpts)
|
||||
synctest.Wait()
|
||||
s.cancelResyncID(opts.resyncID)
|
||||
synctest.Wait()
|
||||
for name, done := range map[string]chan struct{}{"active": first, "queued": queued} {
|
||||
select {
|
||||
case <-done:
|
||||
default:
|
||||
t.Fatalf("%s matching run survived cancellation", name)
|
||||
}
|
||||
}
|
||||
if !errors.Is(context.Cause(firstCtx), errResyncCanceled) {
|
||||
t.Fatal("active Walk did not receive user cancellation")
|
||||
}
|
||||
select {
|
||||
case <-other:
|
||||
t.Fatal("unrelated run was canceled")
|
||||
default:
|
||||
}
|
||||
otherCtx := <-walks
|
||||
if otherCtx.Err() != nil {
|
||||
t.Fatal("unrelated Walk was canceled")
|
||||
}
|
||||
// There is no token/tombstone left for a subsequently registered run.
|
||||
freshOpts := add("fresh", opts.resyncID)
|
||||
fresh := run(freshOpts)
|
||||
synctest.Wait()
|
||||
select {
|
||||
case <-fresh:
|
||||
t.Fatal("fresh run inherited an old cancellation")
|
||||
default:
|
||||
}
|
||||
stop()
|
||||
synctest.Wait()
|
||||
<-other
|
||||
<-fresh
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
if len(s.cancelResyncs) != 0 || len(s.workerCh) != 1 {
|
||||
t.Fatal("run registrations or worker slot leaked")
|
||||
}
|
||||
if s.statusMap[queuedOpts.bucket].TargetsMap[opts.arn].ResyncStatus != ResyncCanceled {
|
||||
t.Fatal("queued user cancellation was not recorded")
|
||||
}
|
||||
if s.statusMap[freshOpts.bucket].TargetsMap[opts.arn].ResyncStatus != ResyncPending {
|
||||
t.Fatal("shutdown changed a queued run's resumable Pending status")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestResyncCancellationWinsFinalization(t *testing.T) {
|
||||
s, opts := newTestResyncer("finalize-cancel", "arn1")
|
||||
obj := &resyncCancelObjectLayer{saved: make(map[string]BucketReplicationResyncStatus)}
|
||||
ctx, cancel, registered := s.registerResync(t.Context(), opts)
|
||||
if !registered {
|
||||
t.Fatal("registration failed")
|
||||
}
|
||||
defer cancel(nil)
|
||||
// Simulate cancellation after the finalizer computed Completed but before
|
||||
// it acquired the persistence lock.
|
||||
status := finalResyncStatus(ResyncCompleted, context.Cause(ctx))
|
||||
s.cancelResyncID(opts.resyncID)
|
||||
if got := s.markStatus(status, opts, obj); got != ResyncCanceled {
|
||||
t.Fatalf("final status = %s, want Canceled", got)
|
||||
}
|
||||
for _, saved := range obj.saved {
|
||||
if saved.TargetsMap[opts.arn].ResyncStatus != ResyncCanceled {
|
||||
t.Fatal("persisted Completed over cancellation")
|
||||
}
|
||||
}
|
||||
if len(obj.saved) != 1 {
|
||||
t.Fatal("canceled status was not persisted")
|
||||
}
|
||||
m := s.statusMap[opts.bucket]
|
||||
m.TargetsMap[opts.arn] = TargetReplicationResyncStatus{ResyncID: "new-run", ResyncStatus: ResyncStarted}
|
||||
s.statusMap[opts.bucket] = m
|
||||
if s.markStatus(ResyncCompleted, opts, obj) != NoResync || s.incStats(TargetReplicationResyncStatus{ReplicatedCount: 1}, opts) {
|
||||
t.Fatal("old run changed the replacement run")
|
||||
}
|
||||
if st := s.statusMap[opts.bucket].TargetsMap[opts.arn]; st.ResyncStatus != ResyncStarted || st.ReplicatedCount != 0 {
|
||||
t.Fatalf("replacement state changed: %+v", st)
|
||||
}
|
||||
delete(s.statusMap, opts.bucket)
|
||||
if s.markStatus(ResyncFailed, opts, obj) != NoResync {
|
||||
t.Fatal("deleted bucket was recreated")
|
||||
}
|
||||
}
|
||||
|
||||
func setupResyncCancelTest(t *testing.T, obj *resyncCancelObjectLayer) (*replicationResyncer, resyncOpts) {
|
||||
t.Helper()
|
||||
s, opts := newTestResyncer("cancel-bucket", "arn1")
|
||||
s.workerCh = make(chan struct{}, 1)
|
||||
s.workerCh <- struct{}{}
|
||||
cfg := configs[0]
|
||||
cfg.RoleArn = opts.arn
|
||||
meta := newBucketMetadata(opts.bucket)
|
||||
meta.replicationConfig = &cfg
|
||||
oldMeta, oldTargets, oldObj := globalBucketMetadataSys, globalBucketTargetSys, newObjectLayerFn()
|
||||
oldPool := globalReplicationPool
|
||||
oldNotifier := globalEventNotifier
|
||||
globalEventNotifier = &EventNotifier{}
|
||||
globalReplicationPool = once.NewSingleton[ReplicationPool]()
|
||||
globalReplicationPool.Set(&ReplicationPool{})
|
||||
globalBucketMetadataSys = NewBucketMetadataSys()
|
||||
globalBucketMetadataSys.Set(opts.bucket, meta)
|
||||
client, err := minio.New("127.0.0.1:1", &minio.Options{Region: "us-east-1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
globalBucketTargetSys = &BucketTargetSys{
|
||||
arnRemotesMap: map[string]arnTarget{opts.arn: {Client: &TargetClient{Client: client, ARN: opts.arn}}},
|
||||
targetsMap: map[string][]madmin.BucketTarget{opts.bucket: {{Arn: opts.arn}}},
|
||||
}
|
||||
setObjectLayer(obj)
|
||||
t.Cleanup(func() {
|
||||
globalBucketMetadataSys, globalBucketTargetSys = oldMeta, oldTargets
|
||||
globalReplicationPool = oldPool
|
||||
globalEventNotifier = oldNotifier
|
||||
setObjectLayer(oldObj)
|
||||
})
|
||||
return s, opts
|
||||
}
|
||||
|
||||
type blockedResyncLock struct {
|
||||
RWLocker
|
||||
started chan struct{}
|
||||
}
|
||||
|
||||
func (l *blockedResyncLock) GetLock(ctx context.Context, _ *dynamicTimeout) (LockContext, error) {
|
||||
close(l.started)
|
||||
<-ctx.Done()
|
||||
return LockContext{}, ctx.Err()
|
||||
}
|
||||
|
||||
func TestResyncCancelFullWorkerQueue(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
lock := &blockedResyncLock{started: make(chan struct{})}
|
||||
produced := 0
|
||||
walkerDone := make(chan struct{})
|
||||
obj := &resyncCancelObjectLayer{lock: lock}
|
||||
s, opts := setupResyncCancelTest(t, obj)
|
||||
obj.walk = func(ctx context.Context, output chan<- itemOrErr[ObjectInfo]) error {
|
||||
go func() {
|
||||
defer close(walkerDone)
|
||||
defer close(output)
|
||||
for range 120 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case output <- itemOrErr[ObjectInfo]{Item: ObjectInfo{Bucket: opts.bucket, Name: "same-key", VersionID: mustGetUUID(), DeleteMarker: true, ModTime: time.Now()}}:
|
||||
produced++
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() { s.resyncBucket(ctx, obj, false, opts); close(done) }()
|
||||
synctest.Wait()
|
||||
select {
|
||||
case <-lock.started:
|
||||
default:
|
||||
t.Fatal("worker did not enter replication")
|
||||
}
|
||||
if produced < 101 || produced >= 120 {
|
||||
t.Fatalf("expected a full 100-entry worker queue to block dispatch; Walk produced %d", produced)
|
||||
}
|
||||
cancel()
|
||||
synctest.Wait()
|
||||
select {
|
||||
case <-done:
|
||||
default:
|
||||
t.Fatal("canceled dispatcher deadlocked sending to a full worker queue")
|
||||
}
|
||||
select {
|
||||
case <-walkerDone:
|
||||
default:
|
||||
t.Fatal("canceled Walk producer leaked")
|
||||
}
|
||||
if len(s.workerCh) != 1 {
|
||||
t.Fatal("resync worker slot was not returned")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestResyncCancelBlockedWalkReceive(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
parent, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
var output chan<- itemOrErr[ObjectInfo]
|
||||
obj := &resyncCancelObjectLayer{walk: func(ctx context.Context, ch chan<- itemOrErr[ObjectInfo]) error {
|
||||
output = ch
|
||||
return nil
|
||||
}}
|
||||
s, opts := setupResyncCancelTest(t, obj)
|
||||
done := make(chan struct{})
|
||||
go func() { s.resyncBucket(parent, obj, false, opts); close(done) }()
|
||||
synctest.Wait()
|
||||
if output == nil {
|
||||
t.Fatal("resync did not reach Walk")
|
||||
}
|
||||
cancel()
|
||||
synctest.Wait()
|
||||
select {
|
||||
case <-done:
|
||||
default:
|
||||
t.Error("resync remained blocked on Walk output after cancellation")
|
||||
}
|
||||
close(output) // release the old implementation on failure
|
||||
synctest.Wait()
|
||||
if len(s.workerCh) != 1 {
|
||||
t.Error("resync worker slot was not released")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestResyncCancelsOwnedWalkOnError(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
var walkCtx context.Context
|
||||
obj := &resyncCancelObjectLayer{walk: func(ctx context.Context, ch chan<- itemOrErr[ObjectInfo]) error {
|
||||
walkCtx = ctx
|
||||
return errors.New("injected walk failure")
|
||||
}}
|
||||
s, opts := setupResyncCancelTest(t, obj)
|
||||
s.resyncBucket(t.Context(), obj, false, opts)
|
||||
if walkCtx == nil || walkCtx.Err() == nil {
|
||||
t.Fatal("resync exit did not cancel the context it passed to Walk")
|
||||
}
|
||||
if t.Context().Err() != nil {
|
||||
t.Fatal("resync canceled its caller's context")
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"hash"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio-go/v7/pkg/signer"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
)
|
||||
|
||||
type sdkStreamingHasher struct{ hash.Hash }
|
||||
|
||||
func (sdkStreamingHasher) Close() {}
|
||||
|
||||
func TestAPIUpstreamSDKStreamingContentType(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: func(obj ObjectLayer, instanceType, bucketName string, router http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
payload := bytes.Repeat([]byte("sdk-streaming-"), 8192)
|
||||
for _, tamper := range []bool{false, true} {
|
||||
req, err := http.NewRequest(http.MethodPut, getPutObjectURL("http://localhost", bucketName, "sdk-streaming"), bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-silo-test")
|
||||
hasher := sdkStreamingHasher{sha256.New()}
|
||||
req = signer.StreamingSignV4(req, cred.AccessKey, cred.SecretKey, "", globalSite.Region(), int64(len(payload)), UTCNow(), hasher)
|
||||
_, signedHeaders, _ := strings.Cut(req.Header.Get("Authorization"), "SignedHeaders=")
|
||||
signedHeaders, _, _ = strings.Cut(signedHeaders, ",")
|
||||
if !slices.Contains(strings.Split(signedHeaders, ";"), "content-type") {
|
||||
t.Fatal("upstream SDK did not sign Content-Type")
|
||||
}
|
||||
if tamper {
|
||||
req.Header.Set("Content-Type", "application/x-tampered")
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
want := http.StatusOK
|
||||
if tamper {
|
||||
want = http.StatusForbidden
|
||||
}
|
||||
if rec.Code != want {
|
||||
t.Fatalf("%s tamper=%v: status %d, want %d: %s", instanceType, tamper, rec.Code, want, rec.Body.String())
|
||||
}
|
||||
}
|
||||
info, err := obj.GetObjectInfo(t.Context(), bucketName, "sdk-streaming", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Size != int64(len(payload)) || info.ContentType != "application/x-silo-test" {
|
||||
t.Fatalf("stored size/type = %d/%q", info.Size, info.ContentType)
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// forgedSignatureAgeHeader is the name of the scratch header the presigned
|
||||
// verifier once wrote s3:signatureAge through. Production code no longer knows
|
||||
// it; a client that sends it is sending an ordinary unsigned x-amz-* header.
|
||||
const forgedSignatureAgeHeader = "X-Amz-Signature-Age"
|
||||
|
||||
// Presign at a chosen time, including exactly the supplied operation headers.
|
||||
func presignBoundaryRequest(t *testing.T, r *http.Request, date time.Time, signedHeaders []string, cred auth.Credentials) {
|
||||
t.Helper()
|
||||
query := r.URL.Query()
|
||||
query.Del(xhttp.AmzSignature)
|
||||
query.Set(xhttp.AmzAlgorithm, signV4Algorithm)
|
||||
query.Set(xhttp.AmzDate, date.Format(iso8601Format))
|
||||
query.Set(xhttp.AmzExpires, "3600")
|
||||
query.Set(xhttp.AmzSignedHeaders, strings.Join(signedHeaders, ";"))
|
||||
query.Set(xhttp.AmzCredential, cred.AccessKey+"/"+getScope(date, globalSite.Region()))
|
||||
r.Form = query
|
||||
headers, code := extractSignedHeaders(signedHeaders, r)
|
||||
if code != ErrNone {
|
||||
t.Fatal(niceError(code))
|
||||
}
|
||||
canonical := getCanonicalRequest(headers, getContentSha256Cksum(r, serviceS3), query.Encode(), r.URL.Path, r.Method)
|
||||
key := getSigningKey(cred.SecretKey, date, globalSite.Region(), serviceS3)
|
||||
query.Set(xhttp.AmzSignature, getSignature(key, getStringToSign(canonical, date, getScope(date, globalSite.Region()))))
|
||||
r.URL.RawQuery = query.Encode()
|
||||
r.Form = query
|
||||
}
|
||||
|
||||
func setupSignatureBoundaryTest(t *testing.T) {
|
||||
t.Helper()
|
||||
obj, fsDir, err := prepareFS(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(fsDir) })
|
||||
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Canonicalization comma-joins repeated header fields, so a signature over one
|
||||
// x-amz-copy-source value containing a comma also covers the same text split
|
||||
// into two fields, while the copy handlers act on Header.Get alone. A single
|
||||
// value may contain a literal or percent-encoded comma; a repeated header is
|
||||
// rejected at the shared SigV4 boundary for both signed and presigned requests.
|
||||
func TestV4CopySourceMultiplicity(t *testing.T) {
|
||||
setupSignatureBoundaryTest(t)
|
||||
for _, presigned := range []bool{false, true} {
|
||||
for _, source := range []string{"/source/allowed,tail", "/source/allowed%2Ctail"} {
|
||||
t.Run(fmt.Sprintf("presigned=%v/%s", presigned, source), func(t *testing.T) {
|
||||
r := httptest.NewRequest(http.MethodPut, "http://minio.local/destination/object", nil)
|
||||
r.Header.Set(xhttp.AmzContentSha256, emptySHA256)
|
||||
r.Header.Set(xhttp.AmzCopySource, source)
|
||||
if presigned {
|
||||
presignBoundaryRequest(t, r, UTCNow(), []string{"host", "x-amz-copy-source"}, globalActiveCred)
|
||||
} else {
|
||||
if err := signRequestV4(r, globalActiveCred.AccessKey, globalActiveCred.SecretKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.Form = r.URL.Query()
|
||||
}
|
||||
if code := reqSignatureV4Verify(r, globalSite.Region(), serviceS3); code != ErrNone {
|
||||
t.Fatalf("a single source key containing a comma must remain valid: %s", niceError(code))
|
||||
}
|
||||
if source != "/source/allowed,tail" {
|
||||
return
|
||||
}
|
||||
r.Header[xhttp.AmzCopySource] = []string{"/source/allowed", "tail"}
|
||||
if code := reqSignatureV4Verify(r, globalSite.Region(), serviceS3); code != ErrInvalidCopySource {
|
||||
t.Fatalf("split copy source: got %s, want %s; handler would copy %q",
|
||||
niceError(code), niceError(ErrInvalidCopySource), r.Header.Get(xhttp.AmzCopySource))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// PutObject and UploadPart authorize before they verify the signature, so
|
||||
// s3:signatureAge must come from the signed X-Amz-Date on the first policy
|
||||
// evaluation. A client header under the former scratch name must neither
|
||||
// supply the value nor survive verification, and verifying the same request
|
||||
// twice must give the same answer.
|
||||
func TestGetConditionValuesPresignedAgeFromDate(t *testing.T) {
|
||||
setupSignatureBoundaryTest(t)
|
||||
for _, tc := range []struct {
|
||||
name, header string
|
||||
age time.Duration
|
||||
wantVerify APIErrorCode
|
||||
}{
|
||||
{name: "old", age: 10 * time.Minute},
|
||||
{name: "old forged 0", age: 10 * time.Minute, header: "0", wantVerify: ErrUnsignedHeaders},
|
||||
{name: "old forged 1", age: 10 * time.Minute, header: "1", wantVerify: ErrUnsignedHeaders},
|
||||
// A signer slightly ahead of the server stays within globalMaxSkewTime.
|
||||
{name: "future within skew", age: -2 * time.Minute},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r := httptest.NewRequest(http.MethodPut, "http://minio.local/bucket/object", nil)
|
||||
presignBoundaryRequest(t, r, UTCNow().Add(-tc.age), []string{"host"}, globalActiveCred)
|
||||
if tc.header != "" {
|
||||
r.Header.Set(forgedSignatureAgeHeader, tc.header)
|
||||
}
|
||||
values := getConditionValues(r, "", globalActiveCred)
|
||||
age, err := strconv.ParseInt(strings.Join(values["signatureAge"], ""), 10, 64)
|
||||
lo, hi := (tc.age - time.Minute).Milliseconds(), (tc.age + time.Minute).Milliseconds()
|
||||
if err != nil || age < lo || age > hi {
|
||||
t.Errorf("pre-verification policy got age %v; want the age of the signed date (about %d ms)", values["signatureAge"], tc.age.Milliseconds())
|
||||
}
|
||||
code := reqSignatureV4Verify(r, globalSite.Region(), serviceS3)
|
||||
if code != tc.wantVerify {
|
||||
t.Fatalf("verification: got %s, want %s", niceError(code), niceError(tc.wantVerify))
|
||||
}
|
||||
if again := reqSignatureV4Verify(r, globalSite.Region(), serviceS3); again != code {
|
||||
t.Fatalf("second verification changed the outcome: %s -> %s", niceError(code), niceError(again))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The s3:x-amz-content-sha256 policy value must be the single payload hash the
|
||||
// request is verified and enforced against. Header presence decides whether the
|
||||
// key exists at all; the value is the one getContentSha256Cksum selects, so a
|
||||
// presigned query value wins over the header and a repeated header contributes
|
||||
// only its first value. None of these requests is rejected at the protocol
|
||||
// level; the policy simply sees what verification bound.
|
||||
func TestGetConditionValuesPayloadHashMatchesVerifiedValue(t *testing.T) {
|
||||
setupSignatureBoundaryTest(t)
|
||||
hashA, hashB := getSHA256Hash([]byte("a")), getSHA256Hash([]byte("b"))
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
presigned bool
|
||||
query string
|
||||
header []string
|
||||
want []string
|
||||
}{
|
||||
{name: "signed header", header: []string{hashA}, want: []string{hashA}},
|
||||
{name: "signed absent"},
|
||||
{name: "signed present empty", header: []string{""}, want: []string{""}},
|
||||
{name: "signed duplicate header", header: []string{hashA, hashB}, want: []string{hashA}},
|
||||
{name: "signed streaming with second value", header: []string{streamingContentSHA256, hashA}, want: []string{streamingContentSHA256}},
|
||||
{name: "presigned query only", presigned: true, query: hashA},
|
||||
{name: "presigned header only", presigned: true, header: []string{hashA}, want: []string{hashA}},
|
||||
{name: "presigned matching query and header", presigned: true, query: hashA, header: []string{hashA}, want: []string{hashA}},
|
||||
{name: "presigned unsigned query with forged header", presigned: true, query: unsignedPayload, header: []string{hashA}, want: []string{unsignedPayload}},
|
||||
{name: "presigned duplicate header", presigned: true, header: []string{hashA, hashB}, want: []string{hashA}},
|
||||
{name: "presigned query with empty header", presigned: true, query: hashA, header: []string{""}, want: []string{hashA}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
target := "http://minio.local/bucket/object"
|
||||
if tc.query != "" {
|
||||
target += "?" + xhttp.AmzContentSha256 + "=" + tc.query
|
||||
}
|
||||
r := httptest.NewRequest(http.MethodPut, target, nil)
|
||||
if tc.header != nil {
|
||||
r.Header[xhttp.AmzContentSha256] = tc.header
|
||||
}
|
||||
if tc.presigned {
|
||||
presignBoundaryRequest(t, r, UTCNow(), []string{"host"}, globalActiveCred)
|
||||
} else {
|
||||
r.Header.Set(xhttp.Authorization, signV4Algorithm+" Credential=x/20260910/us-east-1/s3/aws4_request, SignedHeaders=host, Signature=x")
|
||||
r.Form = r.URL.Query()
|
||||
}
|
||||
got, ok := getConditionValues(r, "", globalActiveCred)[xhttp.AmzContentSha256]
|
||||
if ok != (tc.want != nil) || !slices.Equal(got, tc.want) {
|
||||
t.Fatalf("policy value = %v (present=%v), want %v (present=%v)", got, ok, tc.want, tc.want != nil)
|
||||
}
|
||||
if tc.presigned {
|
||||
if code := reqSignatureV4Verify(r, globalSite.Region(), serviceS3); code != ErrNone {
|
||||
t.Fatalf("the presigned request must still verify: %s", niceError(code))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func newSignatureBoundaryUser(t *testing.T, bucket, statements string) auth.Credentials {
|
||||
t.Helper()
|
||||
cred, err := auth.GetNewCredentials()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := globalIAMSys.CreateUser(t.Context(), cred.AccessKey, madmin.AddOrUpdateUserReq{SecretKey: cred.SecretKey, Status: madmin.AccountEnabled}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, err := policy.ParseConfig(strings.NewReader(fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::%s/*"},%s]}`, bucket, statements)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
name := "signature-boundary-" + mustGetUUID()
|
||||
if _, err := globalIAMSys.SetPolicy(t.Context(), name, *p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := globalIAMSys.PolicyDBSet(t.Context(), cred.AccessKey, name, regUser, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return cred
|
||||
}
|
||||
|
||||
func TestAPIPresignedSignatureAgeBeforeAuthorization(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
endpoints: []string{"NewMultipart", "PutObjectPart", "PutObject"},
|
||||
objAPITest: func(obj ObjectLayer, instanceType, bucket string, router http.Handler, root auth.Credentials, t *testing.T) {
|
||||
user := newSignatureBoundaryUser(t, bucket, fmt.Sprintf(`{"Effect":"Deny","Action":"s3:PutObject","Resource":"arn:aws:s3:::%s/*","Condition":{"NumericGreaterThan":{"s3:signatureAge":"60000"}}}`, bucket))
|
||||
initReq, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucket, "multipart"), 0, nil, root.AccessKey, root.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
initRec := httptest.NewRecorder()
|
||||
router.ServeHTTP(initRec, initReq)
|
||||
var upload InitiateMultipartUploadResponse
|
||||
if initRec.Code != http.StatusOK || xml.Unmarshal(initRec.Body.Bytes(), &upload) != nil {
|
||||
t.Fatalf("multipart initiation: %d %s", initRec.Code, initRec.Body.String())
|
||||
}
|
||||
for _, operation := range []string{"put", "part"} {
|
||||
for _, tc := range []struct {
|
||||
name, header string
|
||||
age time.Duration
|
||||
want int
|
||||
}{
|
||||
{name: "fresh", want: http.StatusOK},
|
||||
{name: "old without header", age: 10 * time.Minute, want: http.StatusForbidden},
|
||||
{name: "old forged header", age: 10 * time.Minute, header: "0", want: http.StatusForbidden},
|
||||
// Policy allows a fresh signature; the unsigned header then fails
|
||||
// verification with the existing ErrUnsignedHeaders (HTTP 400).
|
||||
{name: "fresh with unsigned header", header: "0", want: http.StatusBadRequest},
|
||||
} {
|
||||
t.Run(instanceType+"/"+operation+"/"+tc.name, func(t *testing.T) {
|
||||
target := getPutObjectURL("", bucket, "put-"+strings.ReplaceAll(tc.name, " ", "-"))
|
||||
if operation == "part" {
|
||||
target = getPutObjectPartURL("", bucket, "multipart", upload.UploadID, "1")
|
||||
}
|
||||
r, err := newTestRequest(http.MethodPut, target, 4, bytes.NewReader([]byte("body")))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.Header.Del(xhttp.AmzContentSha256)
|
||||
presignBoundaryRequest(t, r, UTCNow().Add(-tc.age), []string{"host"}, user)
|
||||
if tc.header != "" {
|
||||
r.Header.Set(forgedSignatureAgeHeader, tc.header)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, r)
|
||||
if rec.Code != tc.want {
|
||||
t.Errorf("got %d %s, want %d", rec.Code, rec.Body.String(), tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func TestAPIPayloadHashPolicyMatchesVerifiedValue(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
endpoints: []string{"PutObject"},
|
||||
objAPITest: func(_ ObjectLayer, instanceType, bucket string, router http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
allowedHash := getSHA256Hash([]byte("expected"))
|
||||
user := newSignatureBoundaryUser(t, bucket, fmt.Sprintf(`{"Effect":"Deny","Action":"s3:PutObject","Resource":"arn:aws:s3:::%s/*","Condition":{"StringNotEquals":{"s3:x-amz-content-sha256":"%s"}}}`, bucket, allowedHash))
|
||||
for _, kind := range []string{"signed control", "presigned control", "unsigned query forged header", "signed duplicate header", "presigned duplicate header"} {
|
||||
t.Run(instanceType+"/"+kind, func(t *testing.T) {
|
||||
control := strings.HasSuffix(kind, "control")
|
||||
body := "modified"
|
||||
if control {
|
||||
body = "expected"
|
||||
}
|
||||
r, err := newTestRequest(http.MethodPut, getPutObjectURL("", bucket, strings.ReplaceAll(kind, " ", "-")), int64(len(body)), strings.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(kind, "duplicate") {
|
||||
r.Header.Add(xhttp.AmzContentSha256, allowedHash)
|
||||
}
|
||||
if kind == "unsigned query forged header" {
|
||||
q := r.URL.Query()
|
||||
q.Set(xhttp.AmzContentSha256, unsignedPayload)
|
||||
r.URL.RawQuery = q.Encode()
|
||||
r.Header.Set(xhttp.AmzContentSha256, allowedHash)
|
||||
}
|
||||
if strings.HasPrefix(kind, "signed") {
|
||||
if err := signRequestV4(r, user.AccessKey, user.SecretKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
presignBoundaryRequest(t, r, UTCNow(), []string{"host"}, user)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, r)
|
||||
if control && rec.Code != http.StatusOK {
|
||||
t.Errorf("control rejected: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !control && rec.Code < http.StatusBadRequest {
|
||||
t.Errorf("payload-hash policy bypass returned %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -211,7 +211,16 @@ func extractSignedHeaders(signedHeaders []string, r *http.Request) (http.Header,
|
||||
// `host` will not be found in the headers, can be found in r.Host.
|
||||
// but its always necessary that the list of signed headers containing host in it.
|
||||
val, ok := reqHeaders[http.CanonicalHeaderKey(header)]
|
||||
if !ok {
|
||||
if ok {
|
||||
// Canonicalization comma-joins repeated header fields, so a signature
|
||||
// over the single value "/src/a,tail" also verifies a request carrying
|
||||
// ["/src/a", "tail"]. The copy handlers read only Header.Get, so that
|
||||
// rewrite would copy a different source than the one signed. Reject a
|
||||
// repeated x-amz-copy-source; a single value may still contain commas.
|
||||
if len(val) > 1 && strings.EqualFold(header, strings.ToLower(xhttp.AmzCopySource)) {
|
||||
return nil, ErrInvalidCopySource
|
||||
}
|
||||
} else {
|
||||
// try to set headers from Query String
|
||||
val, ok = reqQueries[header]
|
||||
}
|
||||
@@ -264,14 +273,40 @@ func signV4TrimAll(input string) string {
|
||||
return strings.Join(strings.Fields(input), " ")
|
||||
}
|
||||
|
||||
// checkMetaHeaders will check if the metadata from header/url is the same with the one from signed headers
|
||||
func checkMetaHeaders(signedHeadersMap http.Header, r *http.Request) APIErrorCode {
|
||||
// check values from http header
|
||||
for k, val := range r.Header {
|
||||
if stringsHasPrefixFold(k, "X-Amz-Meta-") {
|
||||
if signedHeadersMap.Get(k) == val[0] {
|
||||
continue
|
||||
}
|
||||
// checkUnsignedHeaders rejects any x-amz-* request header that is not covered by
|
||||
// the SigV4 signed-headers list. AWS S3 requires every x-amz-* header to be
|
||||
// signed and returns AccessDenied ("There were headers present in the request
|
||||
// which were not signed") otherwise. Enforcing the same here prevents an
|
||||
// unsigned x-amz-* header (for example x-amz-copy-source) from changing the
|
||||
// semantics of an already-signed or presigned request: without this check a
|
||||
// presigned PUT grant could be turned into a server-side copy that reads any
|
||||
// object the signing key can reach.
|
||||
//
|
||||
// Only headers actually sent by the client are inspected. Server-synthesized
|
||||
// x-amz-* headers (e.g. x-amz-tagging derived from a request body) are set
|
||||
// after signature verification and therefore never reach this walk.
|
||||
func checkUnsignedHeaders(signedHeadersMap http.Header, r *http.Request) APIErrorCode {
|
||||
// check headers that arrived on the request
|
||||
for k := range r.Header {
|
||||
if !stringsHasPrefixFold(k, "X-Amz-") {
|
||||
continue
|
||||
}
|
||||
// X-Amz-Content-Sha256 carries the payload hash, not an operation or
|
||||
// authorization input, and is handled specially: for presigned requests
|
||||
// it is read from the query string (getContentSha256Cksum) and any
|
||||
// header copy is ignored, while for signed requests it is bound into the
|
||||
// string-to-sign as the payload hash, so a tampered value fails
|
||||
// signature verification regardless of the signed-headers list. Some
|
||||
// clients send it as an unsigned header, so exempt it to preserve
|
||||
// compatibility without weakening the operation-header protection.
|
||||
if strings.EqualFold(k, xhttp.AmzContentSha256) {
|
||||
continue
|
||||
}
|
||||
// The header must be a member of the signed-headers list. Testing
|
||||
// membership (not value equality) is essential: an unsigned header whose
|
||||
// first value is empty would otherwise compare equal to the empty string
|
||||
// returned for an absent key and slip through.
|
||||
if _, ok := signedHeadersMap[http.CanonicalHeaderKey(k)]; !ok {
|
||||
return ErrUnsignedHeaders
|
||||
}
|
||||
}
|
||||
|
||||
@@ -363,8 +363,8 @@ func TestGetContentSha256Cksum(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Test TestCheckMetaHeaders tests the logic of checkMetaHeaders() function
|
||||
func TestCheckMetaHeaders(t *testing.T) {
|
||||
// Test TestCheckUnsignedHeaders tests the logic of checkUnsignedHeaders() function
|
||||
func TestCheckUnsignedHeaders(t *testing.T) {
|
||||
signedHeadersMap := map[string][]string{
|
||||
"X-Amz-Meta-Test": {"test"},
|
||||
"X-Amz-Meta-Extension": {"png"},
|
||||
@@ -384,7 +384,7 @@ func TestCheckMetaHeaders(t *testing.T) {
|
||||
inputHeader.Set("X-Amz-Meta-Extension", expectedMetaExtension)
|
||||
inputHeader.Set("X-Amz-Meta-Name", expectedMetaName)
|
||||
// calling the function being tested.
|
||||
errCode := checkMetaHeaders(signedHeadersMap, r)
|
||||
errCode := checkUnsignedHeaders(signedHeadersMap, r)
|
||||
if errCode != ErrNone {
|
||||
t.Fatalf("Expected the APIErrorCode to be %d, but got %d", ErrNone, errCode)
|
||||
}
|
||||
@@ -392,7 +392,7 @@ func TestCheckMetaHeaders(t *testing.T) {
|
||||
// Add new metadata in inputHeader
|
||||
inputHeader.Set("X-Amz-Meta-Clone", "fail")
|
||||
// calling the function being tested.
|
||||
errCode = checkMetaHeaders(signedHeadersMap, r)
|
||||
errCode = checkUnsignedHeaders(signedHeadersMap, r)
|
||||
if errCode != ErrUnsignedHeaders {
|
||||
t.Fatalf("Expected the APIErrorCode to be %d, but got %d", ErrUnsignedHeaders, errCode)
|
||||
}
|
||||
@@ -400,7 +400,7 @@ func TestCheckMetaHeaders(t *testing.T) {
|
||||
// Delete extra metadata from header to don't affect other test
|
||||
inputHeader.Del("X-Amz-Meta-Clone")
|
||||
// calling the function being tested.
|
||||
errCode = checkMetaHeaders(signedHeadersMap, r)
|
||||
errCode = checkUnsignedHeaders(signedHeadersMap, r)
|
||||
if errCode != ErrNone {
|
||||
t.Fatalf("Expected the APIErrorCode to be %d, but got %d", ErrNone, errCode)
|
||||
}
|
||||
@@ -413,8 +413,72 @@ func TestCheckMetaHeaders(t *testing.T) {
|
||||
|
||||
r.ParseForm()
|
||||
// calling the function being tested.
|
||||
errCode = checkMetaHeaders(signedHeadersMap, r)
|
||||
errCode = checkUnsignedHeaders(signedHeadersMap, r)
|
||||
if errCode != ErrNone {
|
||||
t.Fatalf("Expected the APIErrorCode to be %d, but got %d", ErrNone, errCode)
|
||||
}
|
||||
|
||||
// Regression for the unsigned x-amz-copy-source coverage gap: an x-amz-*
|
||||
// header outside the signed-headers list (here x-amz-copy-source, which the
|
||||
// router uses to select CopyObjectHandler) must be rejected. Previously only
|
||||
// x-amz-meta-* headers were inspected, so this header slipped through and a
|
||||
// presigned/authorized PUT could be turned into a server-side copy.
|
||||
r, err = http.NewRequest(http.MethodPut, "http://play.min.io:9000", nil)
|
||||
if err != nil {
|
||||
t.Fatal("Unable to create http.Request :", err)
|
||||
}
|
||||
r.Header.Set("X-Amz-Copy-Source", "/src/secret.txt")
|
||||
if errCode = checkUnsignedHeaders(signedHeadersMap, r); errCode != ErrUnsignedHeaders {
|
||||
t.Fatalf("unsigned x-amz-copy-source: expected %d, got %d", ErrUnsignedHeaders, errCode)
|
||||
}
|
||||
|
||||
// When the same header is part of the signed-headers list with a matching
|
||||
// value it is allowed through, exactly as for x-amz-meta-*.
|
||||
signedWithCopy := http.Header{}
|
||||
for k, v := range signedHeadersMap {
|
||||
signedWithCopy[k] = v
|
||||
}
|
||||
signedWithCopy.Set("X-Amz-Copy-Source", "/src/secret.txt")
|
||||
if errCode = checkUnsignedHeaders(signedWithCopy, r); errCode != ErrNone {
|
||||
t.Fatalf("signed x-amz-copy-source: expected %d, got %d", ErrNone, errCode)
|
||||
}
|
||||
|
||||
// Membership, not value equality: an unsigned x-amz-* header whose first
|
||||
// value is empty must still be rejected. A value-equality check would
|
||||
// compare "" against the empty string returned for an absent signed header
|
||||
// and wrongly let it through, so a multi-value header like
|
||||
// {"", "/src/secret.txt"} could smuggle an unsigned copy-source.
|
||||
r, err = http.NewRequest(http.MethodPut, "http://play.min.io:9000", nil)
|
||||
if err != nil {
|
||||
t.Fatal("Unable to create http.Request :", err)
|
||||
}
|
||||
r.Header["X-Amz-Copy-Source"] = []string{"", "/src/secret.txt"}
|
||||
if errCode = checkUnsignedHeaders(signedHeadersMap, r); errCode != ErrUnsignedHeaders {
|
||||
t.Fatalf("empty-first unsigned x-amz-copy-source: expected %d, got %d", ErrUnsignedHeaders, errCode)
|
||||
}
|
||||
|
||||
// X-Amz-Content-Sha256 is exempt: it carries the payload hash (handled from
|
||||
// the query for presigned and bound into the string-to-sign for signed
|
||||
// requests), so it is allowed even when it is not in the signed-headers map.
|
||||
r, err = http.NewRequest(http.MethodPut, "http://play.min.io:9000", nil)
|
||||
if err != nil {
|
||||
t.Fatal("Unable to create http.Request :", err)
|
||||
}
|
||||
r.Header.Set(xhttp.AmzContentSha256, unsignedPayload)
|
||||
if errCode = checkUnsignedHeaders(signedHeadersMap, r); errCode != ErrNone {
|
||||
t.Fatalf("unsigned x-amz-content-sha256 must be exempt: expected %d, got %d", ErrNone, errCode)
|
||||
}
|
||||
|
||||
// X-Amz-Signature-Age was once a scratch header the presigned verifier wrote
|
||||
// back and this check exempted. s3:signatureAge is now derived from the
|
||||
// signed date, so a client that sends the header is sending an ordinary
|
||||
// unsigned x-amz-* header and must be rejected like any other.
|
||||
r, err = http.NewRequest(http.MethodPut, "http://play.min.io:9000", nil)
|
||||
if err != nil {
|
||||
t.Fatal("Unable to create http.Request :", err)
|
||||
}
|
||||
r.Header.Set("X-Amz-Signature-Age", "1234")
|
||||
if errCode = checkUnsignedHeaders(signedHeadersMap, r); errCode != ErrUnsignedHeaders {
|
||||
t.Fatalf("unsigned x-amz-signature-age must be rejected: expected %d, got %d", ErrUnsignedHeaders, errCode)
|
||||
}
|
||||
}
|
||||
|
||||
+13
-6
@@ -229,10 +229,11 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
|
||||
return errCode
|
||||
}
|
||||
|
||||
// Check if the metadata headers are equal with signedheaders
|
||||
errMetaCode := checkMetaHeaders(extractedSignedHeaders, r)
|
||||
if errMetaCode != ErrNone {
|
||||
return errMetaCode
|
||||
// Reject any x-amz-* header that the client did not sign. Without this an
|
||||
// unsigned header (e.g. x-amz-copy-source) could alter the request that the
|
||||
// presigned URL actually authorized.
|
||||
if errUnsigned := checkUnsignedHeaders(extractedSignedHeaders, r); errUnsigned != ErrNone {
|
||||
return errUnsigned
|
||||
}
|
||||
|
||||
// If the host which signed the request is slightly ahead in time (by less than globalMaxSkewTime) the
|
||||
@@ -335,8 +336,6 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
|
||||
return ErrSignatureDoesNotMatch
|
||||
}
|
||||
|
||||
r.Header.Set("x-amz-signature-age", strconv.FormatInt(UTCNow().Sub(pSignValues.Date).Milliseconds(), 10))
|
||||
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
@@ -363,6 +362,14 @@ func doesSignatureMatch(hashedPayload string, r *http.Request, region string, st
|
||||
return errCode
|
||||
}
|
||||
|
||||
// Reject any x-amz-* header that the client did not sign. The Authorization
|
||||
// header path shares extractSignedHeaders with the presigned path but, prior
|
||||
// to this, never inspected the headers that actually arrived, so an unsigned
|
||||
// x-amz-copy-source could redirect a signed PUT into a server-side copy.
|
||||
if errUnsigned := checkUnsignedHeaders(extractedSignedHeaders, r); errUnsigned != ErrNone {
|
||||
return errUnsigned
|
||||
}
|
||||
|
||||
cred, _, s3Err := checkKeyValid(r, signV4Values.Credential.accessKey)
|
||||
if s3Err != ErrNone {
|
||||
return s3Err
|
||||
|
||||
@@ -313,3 +313,44 @@ func TestDoesPresignedSignatureMatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPresignedVerifyIdempotent guards against a regression where verifying the
|
||||
// same presigned request twice began to fail. The verifier must not write
|
||||
// anything back to the request (it once recorded an internal
|
||||
// x-amz-signature-age header that the unsigned-header check then had to
|
||||
// exempt), and an unsigned x-amz-content-sha256 the client may carry stays
|
||||
// exempt, so a second verification of the same *http.Request still succeeds.
|
||||
func TestPresignedVerifyIdempotent(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
|
||||
obj, fsDir, err := prepareFS(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(fsDir)
|
||||
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
req, err := newTestRequest(http.MethodGet, "http://127.0.0.1:9000/bucket/object", 0, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = preSignV4(req, globalActiveCred.AccessKey, globalActiveCred.SecretKey, int64(10*60)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = req.ParseForm(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got := reqSignatureV4Verify(req, globalSite.Region(), serviceS3); got != ErrNone {
|
||||
t.Fatalf("first verification: expected ErrNone, got %s", niceError(got))
|
||||
}
|
||||
if got := reqSignatureV4Verify(req, globalSite.Region(), serviceS3); got != ErrNone {
|
||||
t.Fatalf("second verification of the same request: expected ErrNone, got %s", niceError(got))
|
||||
}
|
||||
if _, ok := req.Header["X-Amz-Signature-Age"]; ok {
|
||||
t.Fatal("the verifier wrote a scratch header back to the request")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,337 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
)
|
||||
|
||||
func TestBucketMetadataTombstoneExportAndInitialSync(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
t.Run(backend, func(t *testing.T) {
|
||||
recordBucketConfigPeer(t, cred)
|
||||
old := globalSiteReplicationMetadataTombstones
|
||||
defer func() { globalSiteReplicationMetadataTombstones = old }()
|
||||
created := UTCNow().Add(-time.Hour)
|
||||
deletedAt := created.Add(time.Minute)
|
||||
for _, enabled := range []bool{false, true} {
|
||||
globalSiteReplicationMetadataTombstones = enabled
|
||||
meta := newBucketMetadata(bucket)
|
||||
meta.Created = created
|
||||
meta.defaultTimestamps()
|
||||
for _, file := range []string{bucketPolicyConfig, bucketTaggingConfig, bucketSSEConfig, bucketQuotaConfigFile} {
|
||||
_, at := replicatedBucketConfig(&meta, file)
|
||||
*at = deletedAt
|
||||
}
|
||||
if err := globalBucketMetadataSys.save(t.Context(), meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Force a disk reload instead of accepting the just-published cache.
|
||||
globalBucketMetadataSys.Remove(bucket)
|
||||
info, err := globalSiteReplicationSys.SiteReplicationMetaInfo(t.Context(), obj, madmin.SRStatusOptions{Buckets: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exported := info.Buckets[bucket]
|
||||
if !exported.PolicyUpdatedAt.Equal(deletedAt) {
|
||||
t.Fatal("Policy tombstone hidden by gate")
|
||||
}
|
||||
for _, at := range []time.Time{exported.TagConfigUpdatedAt, exported.SSEConfigUpdatedAt, exported.QuotaConfigUpdatedAt} {
|
||||
if enabled && !at.Equal(deletedAt) || !enabled && !at.IsZero() {
|
||||
t.Fatalf("gate=%v timestamp=%v", enabled, at)
|
||||
}
|
||||
}
|
||||
for file, data := range bucketConfigTestData(bucket) {
|
||||
event, send, err := initialBucketConfigReplicationEvent(meta, file)
|
||||
wantSend := enabled && !bucketConfigUpdateOnly(file)
|
||||
if err != nil || send != wantSend || send && !event.UpdatedAt.Equal(deletedAt) {
|
||||
t.Fatalf("%s initial gate=%v: %+v %v %v", file, enabled, event, send, err)
|
||||
}
|
||||
baseline := newBucketMetadata(bucket)
|
||||
baseline.Created = created
|
||||
baseline.defaultTimestamps()
|
||||
if _, send, err := initialBucketConfigReplicationEvent(baseline, file); err != nil || send {
|
||||
t.Fatalf("empty baseline sent: %s", file)
|
||||
}
|
||||
value, _ := replicatedBucketConfig(&baseline, file)
|
||||
*value = data
|
||||
event, send, err = initialBucketConfigReplicationEvent(baseline, file)
|
||||
if err != nil || !send || !event.UpdatedAt.Equal(created) {
|
||||
t.Fatalf("historical baseline-live omitted: %s %v", file, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}})
|
||||
}
|
||||
|
||||
func TestPeerBucketMetadataLegacyAndGeneration(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
t.Run(backend, func(t *testing.T) {
|
||||
old := globalSiteReplicationMetadataTombstones
|
||||
defer func() { globalSiteReplicationMetadataTombstones = old }()
|
||||
created := UTCNow().Add(-time.Hour)
|
||||
for _, enabled := range []bool{false, true} {
|
||||
globalSiteReplicationMetadataTombstones = enabled
|
||||
for file, data := range bucketConfigTestData(bucket) {
|
||||
meta := newBucketMetadata(bucket)
|
||||
meta.Created = created
|
||||
meta.defaultTimestamps()
|
||||
if err := globalBucketMetadataSys.save(t.Context(), meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
counter := &bucketConfigWriteCounter{ObjectLayer: obj}
|
||||
setObjectLayer(counter)
|
||||
event := newBucketConfigReplicationEvent(bucket, file, bucketConfigState{data: data, at: created.Add(-time.Second)})
|
||||
rec := applySRBucketMetaViaAdmin(t, cred, event)
|
||||
if rec.Code != http.StatusOK || counter.writes.Load() != 0 {
|
||||
t.Fatalf("pre-creation apply: %s %d writes=%d", file, rec.Code, counter.writes.Load())
|
||||
}
|
||||
// A live baseline may initialize. The same-time nil cannot delete it.
|
||||
event.UpdatedAt = created
|
||||
rec = applySRBucketMetaViaAdmin(t, cred, event)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("baseline apply: %s %s", file, rec.Body.String())
|
||||
}
|
||||
before := counter.writes.Load()
|
||||
rec = applySRBucketMetaViaAdmin(t, cred, madmin.SRBucketMeta{Type: event.Type, Bucket: bucket, UpdatedAt: created})
|
||||
if rec.Code != http.StatusOK || counter.writes.Load() != before {
|
||||
t.Fatalf("nil baseline cleared configuration: %s", file)
|
||||
}
|
||||
event.UpdatedAt = time.Time{}
|
||||
for range 2 {
|
||||
rec = applySRBucketMetaViaAdmin(t, cred, event)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("legacy-zero rejected: %s %s", file, rec.Body.String())
|
||||
}
|
||||
}
|
||||
meta, err := loadBucketMetadata(t.Context(), obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, at := replicatedBucketConfig(&meta, file)
|
||||
if len(*value) == 0 || !at.After(created) {
|
||||
t.Fatalf("legacy-zero not reclocked: %s %v", file, *at)
|
||||
}
|
||||
setObjectLayer(obj)
|
||||
}
|
||||
}
|
||||
})
|
||||
}})
|
||||
}
|
||||
|
||||
type bucketMetadataCreatedObjectLayer struct {
|
||||
ObjectLayer
|
||||
missing bool
|
||||
}
|
||||
|
||||
func (o bucketMetadataCreatedObjectLayer) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) {
|
||||
if opts.NoMetadata {
|
||||
if o.missing {
|
||||
return BucketInfo{}, BucketNotFound{Bucket: bucket}
|
||||
}
|
||||
// A physical bucket that reports no creation time either.
|
||||
return BucketInfo{Name: bucket}, nil
|
||||
}
|
||||
return o.ObjectLayer.GetBucketInfo(ctx, bucket, opts)
|
||||
}
|
||||
|
||||
func TestPeerBucketMetadataUnknownCreated(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
t.Run(backend, func(t *testing.T) {
|
||||
defer setObjectLayer(obj)
|
||||
data := bucketConfigTestData(bucket)[bucketTaggingConfig]
|
||||
for _, mode := range []string{"unknown", "missing"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
setObjectLayer(obj)
|
||||
if err := globalBucketMetadataSys.save(t.Context(), newBucketMetadata(bucket)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
counter := &bucketConfigWriteCounter{ObjectLayer: bucketMetadataCreatedObjectLayer{ObjectLayer: obj, missing: mode == "missing"}}
|
||||
setObjectLayer(counter)
|
||||
stamp := UTCNow()
|
||||
_, err := globalBucketMetadataSys.updateAndParseMetadata(t.Context(), bucket, bucketTaggingConfig, data, false, false, &stamp)
|
||||
if err == nil || counter.writes.Load() != 0 {
|
||||
t.Fatalf("unknown generation was invented: %v writes=%d", err, counter.writes.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}})
|
||||
}
|
||||
|
||||
// setPhysicalBucketCreated stamps the bucket directory on every local drive,
|
||||
// which is what StatVol reports as the physical creation time.
|
||||
func setPhysicalBucketCreated(t *testing.T, bucket string, at time.Time) {
|
||||
t.Helper()
|
||||
globalLocalDrivesMu.RLock()
|
||||
drives := cloneDrives(globalLocalDrivesMap)
|
||||
globalLocalDrivesMu.RUnlock()
|
||||
if len(drives) == 0 {
|
||||
t.Fatal("no local drives registered")
|
||||
}
|
||||
for _, drive := range drives {
|
||||
if err := os.Chtimes(pathJoin(drive.Endpoint().Path, bucket), at, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Recovery has to run against the real ObjectLayer: a stub GetBucketInfo
|
||||
// returning the expected time would hide the cached zero creation time
|
||||
// overwriting it, which is what a bucket that never held a configuration has.
|
||||
func TestBucketMetadataPhysicalCreatedRecovery(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
// One known time on every drive, so the recovered value can be neither
|
||||
// confused with UTCNow() nor dependent on which drive answers first.
|
||||
physical := UTCNow().Add(-3 * time.Hour).Truncate(time.Second)
|
||||
for _, missing := range []bool{false, true} {
|
||||
for _, file := range replicatedBucketConfigs {
|
||||
t.Run(backend+"/"+file+"/missing="+strconv.FormatBool(missing), func(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
setPhysicalBucketCreated(t, bucket, physical)
|
||||
if err := globalBucketMetadataSys.save(ctx, newBucketMetadata(bucket)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if missing {
|
||||
if err := deleteConfig(ctx, obj, pathJoin(bucketMetaPrefix, bucket, bucketMetadataFile)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
data := bucketConfigTestData(bucket)[file]
|
||||
at, err := globalBucketMetadataSys.Update(ctx, bucket, file, data)
|
||||
if err != nil {
|
||||
t.Fatalf("bucket without a recorded creation time cannot update %s: %v", file, err)
|
||||
}
|
||||
got, err := readBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil || !got.Created.Equal(physical) || !at.After(physical) {
|
||||
t.Fatalf("physical creation not persisted: created=%v physical=%v updated=%v err=%v", got.Created, physical, at, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}})
|
||||
}
|
||||
|
||||
func TestBucketMetadataInitialSyncPhysicalCreated(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
t.Run(backend, func(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, cred.AccessKey, nil, newServiceAccountOpts{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { globalIAMSys.DeleteServiceAccount(context.Background(), serviceCred.AccessKey, false) })
|
||||
physical := UTCNow().Add(-3 * time.Hour).Truncate(time.Second)
|
||||
setPhysicalBucketCreated(t, bucket, physical)
|
||||
meta := newBucketMetadata(bucket)
|
||||
meta.TaggingConfigXML = bucketConfigTestData(bucket)[bucketTaggingConfig]
|
||||
if err := globalBucketMetadataSys.save(ctx, meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var createdAt string
|
||||
var events []madmin.SRBucketMeta
|
||||
peer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if r.URL.Query().Get("operation") == string(madmin.MakeWithVersioningBktOp) {
|
||||
createdAt = r.URL.Query().Get("createdAt")
|
||||
}
|
||||
if strings.HasSuffix(r.URL.Path, "/bucket-meta") {
|
||||
var event madmin.SRBucketMeta
|
||||
if err := json.NewDecoder(r.Body).Decode(&event); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
events = append(events, event)
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer peer.Close()
|
||||
// Exercise the complete outgoing sync sequence with real source
|
||||
// storage. This peer acknowledges RPCs; it is not a second ObjectLayer.
|
||||
c := &SiteReplicationSys{enabled: true, state: srState{
|
||||
ServiceAccountAccessKey: serviceCred.AccessKey,
|
||||
Peers: map[string]madmin.PeerInfo{"initial-peer": {DeploymentID: "initial-peer", Endpoint: peer.URL}},
|
||||
}}
|
||||
if err := c.syncToAllPeers(ctx, madmin.SRAddOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if createdAt != physical.Format(time.RFC3339Nano) {
|
||||
t.Fatalf("peer creation time %q, want physical time %s", createdAt, physical)
|
||||
}
|
||||
for _, event := range events {
|
||||
if event.Bucket == bucket && event.Type == madmin.SRBucketMetaTypeTags {
|
||||
if event.Tags == nil || *event.Tags != base64.StdEncoding.EncodeToString(meta.TaggingConfigXML) || !event.UpdatedAt.Equal(physical) {
|
||||
t.Fatalf("historical tags lost baseline: %+v", event)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("initial sync silently skipped historical tags")
|
||||
})
|
||||
}})
|
||||
}
|
||||
|
||||
func TestPeerBucketMetadataPhysicalCreatedBoundary(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
t.Run(backend, func(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
physical := UTCNow().Truncate(time.Second)
|
||||
setPhysicalBucketCreated(t, bucket, physical)
|
||||
if err := globalBucketMetadataSys.save(ctx, newBucketMetadata(bucket)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
counter := &bucketConfigWriteCounter{ObjectLayer: obj}
|
||||
setObjectLayer(counter)
|
||||
defer setObjectLayer(obj)
|
||||
data := bucketConfigTestData(bucket)[bucketTaggingConfig]
|
||||
at := physical.Add(-time.Hour)
|
||||
_, err := globalBucketMetadataSys.updateAndParseMetadata(ctx, bucket, bucketTaggingConfig, data, false, false, &at)
|
||||
if err != nil || counter.writes.Load() != 0 {
|
||||
t.Fatalf("event before recovered creation must be skipped: err=%v writes=%d", err, counter.writes.Load())
|
||||
}
|
||||
// Physical mtime is only an approximation. A local correction can
|
||||
// establish it; an earlier peer event cannot lower the bucket identity.
|
||||
at, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := readBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil || !got.Created.Equal(physical) || !at.After(physical) || !bytes.Equal(got.TaggingConfigXML, data) {
|
||||
t.Fatalf("local correction did not establish physical creation: %+v %v", got, err)
|
||||
}
|
||||
})
|
||||
}})
|
||||
}
|
||||
@@ -0,0 +1,323 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/logger/target/testlogger"
|
||||
)
|
||||
|
||||
func bucketConfigTestData(bucket string) map[string][]byte {
|
||||
return map[string][]byte{
|
||||
bucketPolicyConfig: []byte(fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket)),
|
||||
bucketTaggingConfig: []byte(`<Tagging><TagSet><Tag><Key>key</Key><Value>value</Value></Tag></TagSet></Tagging>`),
|
||||
bucketSSEConfig: []byte(`<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`),
|
||||
bucketQuotaConfigFile: []byte(`{"quota":1024,"quotatype":"hard"}`),
|
||||
bucketVersioningConfig: enabledBucketVersioningConfig,
|
||||
objectLockConfig: enabledBucketObjectLockConfig,
|
||||
}
|
||||
}
|
||||
|
||||
type bucketConfigLogCapture struct {
|
||||
testing.TB
|
||||
mu sync.Mutex
|
||||
lines []string
|
||||
}
|
||||
|
||||
func (c *bucketConfigLogCapture) Logf(format string, args ...any) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.lines = append(c.lines, fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
func TestHealBucketConfigDiagnostics(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
t.Run(backend, func(t *testing.T) {
|
||||
recordBucketConfigPeer(t, cred)
|
||||
capture := &bucketConfigLogCapture{TB: t}
|
||||
defer testlogger.T.SetLogTB(capture)()
|
||||
disabled := logger.DisableLog
|
||||
logger.DisableLog = false
|
||||
defer func() { logger.DisableLog = disabled }()
|
||||
created := UTCNow().Add(-time.Hour)
|
||||
local := globalDeploymentID()
|
||||
bs := map[string]srBucketStatsSummary{
|
||||
local: bucketConfigTestInfo(bucket, bucketTaggingConfig, nil, created, created),
|
||||
"missing-bucket": {},
|
||||
"broken-rpc": bucketConfigTestInfo(bucket, bucketTaggingConfig, nil, created, created),
|
||||
}
|
||||
info := srStatusInfo{Sites: map[string]madmin.PeerInfo{local: {}, "missing-bucket": {}, "unreachable": {}, "broken-rpc": {}}, BucketStats: map[string]map[string]srBucketStatsSummary{bucket: bs}}
|
||||
if err := globalSiteReplicationSys.healBucketConfig(t.Context(), bucket, bucketTaggingConfig, info); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(capture.lines) != 0 {
|
||||
t.Fatal("empty baselines produced diagnostics")
|
||||
}
|
||||
bs[local] = bucketConfigTestInfo(bucket, bucketTaggingConfig, bucketConfigTestData(bucket)[bucketTaggingConfig], created.Add(time.Minute), created)
|
||||
for range 2 {
|
||||
if err := globalSiteReplicationSys.healBucketConfig(t.Context(), bucket, bucketTaggingConfig, info); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
capture.mu.Lock()
|
||||
defer capture.mu.Unlock()
|
||||
var unreachable, peerError int
|
||||
for _, line := range capture.lines {
|
||||
switch {
|
||||
case strings.Contains(line, "bucket metadata replication: unreachable"):
|
||||
unreachable++
|
||||
case strings.Contains(line, "bucket metadata replication: peer-error"):
|
||||
peerError++
|
||||
default:
|
||||
t.Fatalf("unexpected diagnostic: %s", line)
|
||||
}
|
||||
if !strings.HasPrefix(line, "WARNING:") {
|
||||
t.Fatalf("diagnostic is not a warning: %s", line)
|
||||
}
|
||||
}
|
||||
if unreachable != 1 || peerError != 1 {
|
||||
t.Fatalf("distinct reasons were lost or not deduplicated: unreachable=%d peer-error=%d", unreachable, peerError)
|
||||
}
|
||||
})
|
||||
}})
|
||||
}
|
||||
|
||||
func TestHealBucketConfigWithoutSourceDiagnostics(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
t.Run(backend, func(t *testing.T) {
|
||||
events := recordBucketConfigPeer(t, cred)
|
||||
disabled := logger.DisableLog
|
||||
logger.DisableLog = false
|
||||
defer func() { logger.DisableLog = disabled }()
|
||||
created := UTCNow().Add(-time.Hour)
|
||||
data := bucketConfigTestData(bucket)[bucketTaggingConfig]
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
data []byte
|
||||
at time.Time
|
||||
created time.Time
|
||||
wantLog bool
|
||||
}{
|
||||
{"unknown-created", data, created.Add(time.Minute), time.Time{}, true},
|
||||
{"malformed", []byte("<Tagging>"), created.Add(time.Minute), created, true},
|
||||
{"before-created", data, created.Add(-time.Minute), created, true},
|
||||
{"empty-baseline", nil, created, created, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
capture := &bucketConfigLogCapture{TB: t}
|
||||
defer testlogger.T.SetLogTB(capture)()
|
||||
// Each case has its own diagnostic key; no source can be selected
|
||||
// and neither local storage nor the recording peer may be written.
|
||||
name := bucket + "-" + tc.name
|
||||
local := globalDeploymentID()
|
||||
info := srStatusInfo{
|
||||
Sites: map[string]madmin.PeerInfo{local: {}, "metadata-peer": {}, "unreachable": {}},
|
||||
BucketStats: map[string]map[string]srBucketStatsSummary{name: {
|
||||
local: bucketConfigTestInfo(name, bucketTaggingConfig, tc.data, tc.at, tc.created),
|
||||
"metadata-peer": {},
|
||||
}},
|
||||
}
|
||||
for range 2 {
|
||||
if err := globalSiteReplicationSys.healBucketConfig(t.Context(), name, bucketTaggingConfig, info); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
capture.mu.Lock()
|
||||
defer capture.mu.Unlock()
|
||||
want := 0
|
||||
if tc.wantLog {
|
||||
want = 1
|
||||
}
|
||||
if len(capture.lines) != want {
|
||||
t.Fatalf("got %d diagnostics, want %d: %v", len(capture.lines), want, capture.lines)
|
||||
}
|
||||
for _, line := range capture.lines {
|
||||
if !strings.HasPrefix(line, "WARNING:") || !strings.Contains(line, "bucket metadata replication: indeterminate") {
|
||||
t.Fatalf("unexpected diagnostic: %s", line)
|
||||
}
|
||||
}
|
||||
if len(events()) != 0 {
|
||||
t.Fatal("healing without a source sent a metadata RPC")
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}})
|
||||
}
|
||||
|
||||
// Construct independent wire fixtures, including timestamps hidden by legacy
|
||||
// exporters, without going through the production event or state helpers.
|
||||
func bucketConfigTestInfo(bucket, file string, data []byte, at, created time.Time) srBucketStatsSummary {
|
||||
info := madmin.SRBucketInfo{Bucket: bucket, CreatedAt: created}
|
||||
var payload *string
|
||||
if len(data) != 0 {
|
||||
encoded := base64.StdEncoding.EncodeToString(data)
|
||||
payload = &encoded
|
||||
}
|
||||
switch file {
|
||||
case bucketPolicyConfig:
|
||||
info.Policy, info.PolicyUpdatedAt = data, at
|
||||
case bucketTaggingConfig:
|
||||
info.Tags, info.TagConfigUpdatedAt = payload, at
|
||||
case bucketSSEConfig:
|
||||
info.SSEConfig, info.SSEConfigUpdatedAt = payload, at
|
||||
case bucketQuotaConfigFile:
|
||||
info.QuotaConfig, info.QuotaConfigUpdatedAt = payload, at
|
||||
case bucketVersioningConfig:
|
||||
info.Versioning, info.VersioningConfigUpdatedAt = payload, at
|
||||
case objectLockConfig:
|
||||
info.ObjectLockConfig, info.ObjectLockConfigUpdatedAt = payload, at
|
||||
}
|
||||
return srBucketStatsSummary{meta: srBucketMetaInfo{SRBucketInfo: info}}
|
||||
}
|
||||
|
||||
func TestLatestBucketConfigCandidates(t *testing.T) {
|
||||
created := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
for file, data := range bucketConfigTestData("bucket") {
|
||||
t.Run(file, func(t *testing.T) {
|
||||
info := srStatusInfo{Sites: map[string]madmin.PeerInfo{"a": {}, "b": {}, "c": {}}, BucketStats: map[string]map[string]srBucketStatsSummary{"bucket": {}}}
|
||||
bs := info.BucketStats["bucket"]
|
||||
baseline := bucketConfigTestInfo("bucket", file, nil, created, created)
|
||||
bs["a"], bs["b"], bs["c"] = baseline, baseline, baseline
|
||||
if _, found := latestBucketConfig("bucket", file, info); found {
|
||||
t.Fatal("empty baseline chosen as source")
|
||||
}
|
||||
// Unknown and unavailable IDs cannot contribute, even with future clocks.
|
||||
bs[""], bs["unknown"] = bucketConfigTestInfo("bucket", file, data, created.Add(10*time.Hour), created), bucketConfigTestInfo("bucket", file, data, created.Add(20*time.Hour), created)
|
||||
if _, found := latestBucketConfig("bucket", file, info); found {
|
||||
t.Fatal("unknown source chosen")
|
||||
}
|
||||
liveBaseline := bucketConfigTestInfo("bucket", file, data, created, created)
|
||||
modified := bucketConfigTestInfo("bucket", file, data, created.Add(time.Hour), created)
|
||||
oldGeneration := bucketConfigTestInfo("bucket", file, data, created, created.Add(time.Second))
|
||||
states := []srBucketStatsSummary{liveBaseline, modified, oldGeneration}
|
||||
for _, order := range [][3]int{{0, 1, 2}, {0, 2, 1}, {1, 0, 2}, {1, 2, 0}, {2, 0, 1}, {2, 1, 0}} {
|
||||
bs["a"], bs["b"], bs["c"] = states[order[0]], states[order[1]], states[order[2]]
|
||||
winner, found := latestBucketConfig("bucket", file, info)
|
||||
if !found || !winner.at.Equal(created.Add(time.Hour)) || !bytes.Equal(winner.data, data) {
|
||||
t.Fatalf("permutation %v chose %+v found=%v", order, winner, found)
|
||||
}
|
||||
}
|
||||
bs["a"], bs["b"], bs["c"] = baseline, liveBaseline, baseline
|
||||
if winner, found := latestBucketConfig("bucket", file, info); !found || !bytes.Equal(winner.data, data) || winner.real {
|
||||
t.Fatal("historical baseline-live cannot initialize")
|
||||
}
|
||||
if !bucketConfigUpdateOnly(file) {
|
||||
// A late-created empty baseline cannot beat a real earlier write.
|
||||
bs["a"], bs["b"], bs["c"] = modified, bucketConfigTestInfo("bucket", file, nil, created.Add(2*time.Hour), created.Add(2*time.Hour)), baseline
|
||||
if winner, found := latestBucketConfig("bucket", file, info); !found || len(winner.data) == 0 {
|
||||
t.Fatal("default became a deletion")
|
||||
}
|
||||
bs["c"] = bucketConfigTestInfo("bucket", file, nil, created.Add(time.Hour), created)
|
||||
if winner, found := latestBucketConfig("bucket", file, info); !found || len(winner.data) != 0 || !winner.real {
|
||||
t.Fatal("real tombstone lost equal-time tie")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealBucketConfigSourceAndQuiescence(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
t.Run(backend, func(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
events := recordBucketConfigPeer(t, cred)
|
||||
local := globalDeploymentID()
|
||||
created := UTCNow().Add(-time.Hour)
|
||||
oldAt, newAt := created.Add(time.Minute), created.Add(2*time.Minute)
|
||||
counter := &bucketConfigWriteCounter{ObjectLayer: obj}
|
||||
setObjectLayer(counter)
|
||||
defer setObjectLayer(obj)
|
||||
for file, data := range bucketConfigTestData(bucket) {
|
||||
t.Run(file, func(t *testing.T) {
|
||||
meta := newBucketMetadata(bucket)
|
||||
meta.Created = created
|
||||
meta.defaultTimestamps()
|
||||
value, at := replicatedBucketConfig(&meta, file)
|
||||
*value, *at = data, newAt
|
||||
if err := globalBucketMetadataSys.save(ctx, meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bs := map[string]srBucketStatsSummary{
|
||||
local: bucketConfigTestInfo(bucket, file, data, newAt, created),
|
||||
"metadata-peer": bucketConfigTestInfo(bucket, file, data, oldAt, created),
|
||||
"": {}, "unknown": bucketConfigTestInfo(bucket, file, nil, newAt.Add(time.Hour), created),
|
||||
// Known but absent from c.state.Peers, forcing getAdminClient failure.
|
||||
"broken": bucketConfigTestInfo(bucket, file, nil, created, created),
|
||||
}
|
||||
info := srStatusInfo{Sites: map[string]madmin.PeerInfo{local: {}, "metadata-peer": {}, "broken": {}}, BucketStats: map[string]map[string]srBucketStatsSummary{bucket: bs}}
|
||||
before := len(events())
|
||||
writes := counter.writes.Load()
|
||||
if err := globalSiteReplicationSys.healBucketConfig(ctx, bucket, file, info); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := events()[before:]
|
||||
if len(got) != 1 || !got[0].UpdatedAt.Equal(newAt) {
|
||||
t.Fatalf("same-payload timestamp heal did not reach healthy peer: %+v", got)
|
||||
}
|
||||
if file == bucketTaggingConfig && got[0].Tags == nil {
|
||||
t.Fatal("tag event missing payload")
|
||||
}
|
||||
if counter.writes.Load() != writes {
|
||||
t.Fatal("matching local state was rewritten")
|
||||
}
|
||||
bs["metadata-peer"], bs["broken"] = bs[local], bs[local]
|
||||
if err := globalSiteReplicationSys.healBucketConfig(ctx, bucket, file, info); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(events()) != before+1 || counter.writes.Load() != writes {
|
||||
t.Fatal("stable second heal wrote or broadcast")
|
||||
}
|
||||
if bucketConfigUpdateOnly(file) {
|
||||
return
|
||||
}
|
||||
bs["metadata-peer"] = bucketConfigTestInfo(bucket, file, nil, newAt.Add(time.Minute), created)
|
||||
delete(bs, "broken")
|
||||
if err := globalSiteReplicationSys.healBucketConfig(ctx, bucket, file, info); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, err := loadBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, at = replicatedBucketConfig(&after, file)
|
||||
if len(*value) != 0 || !at.Equal(newAt.Add(time.Minute)) {
|
||||
t.Fatalf("local heal lost deletion/source time: %s %v", *value, *at)
|
||||
}
|
||||
if file == bucketQuotaConfigFile {
|
||||
q, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket)
|
||||
if err != nil || q.Quota != 0 {
|
||||
t.Fatalf("quota cache not cleared: %+v %v", q, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}})
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
)
|
||||
|
||||
// Read once at startup. Enable only after every participating node is fixed.
|
||||
var globalSiteReplicationMetadataTombstones bool
|
||||
|
||||
func logBucketConfigReplication(ctx context.Context, bucket, file, reason string, at, created time.Time, detail string) {
|
||||
// LogOnceIf compares error text as well as its key. Keep both stable; changing
|
||||
// times and peer errors belong in ReqInfo, not in the error's message.
|
||||
req := &logger.ReqInfo{API: "SiteReplicationMetadata", BucketName: bucket}
|
||||
req.AppendTags("field", file)
|
||||
req.AppendTags("sourceTime", at.UTC().Format(time.RFC3339Nano))
|
||||
req.AppendTags("created", created.UTC().Format(time.RFC3339Nano))
|
||||
req.AppendTags("detail", detail)
|
||||
replLogOnceIf(logger.SetReqInfo(ctx, req), errors.New("bucket metadata replication: "+reason),
|
||||
"bucket-metadata/"+bucket+"/"+file+"/"+reason, logger.WarningKind)
|
||||
}
|
||||
|
||||
func initialBucketConfigReplicationEvent(meta BucketMetadata, file string) (madmin.SRBucketMeta, bool, error) {
|
||||
data, at := replicatedBucketConfig(&meta, file)
|
||||
state, err := newBucketConfigState(meta.Name, file, *data, *at, meta.Created, len(meta.ObjectLockConfigXML) != 0)
|
||||
if err != nil {
|
||||
return madmin.SRBucketMeta{}, false, err
|
||||
}
|
||||
if !state.candidate() || (len(state.data) == 0 && !globalSiteReplicationMetadataTombstones) {
|
||||
return madmin.SRBucketMeta{}, false, nil
|
||||
}
|
||||
return newBucketConfigReplicationEvent(meta.Name, file, state), true, nil
|
||||
}
|
||||
|
||||
func bucketConfigStateFromInfo(bucket, file string, meta madmin.SRBucketInfo) (bucketConfigState, error) {
|
||||
var payload *string
|
||||
var at time.Time
|
||||
switch file {
|
||||
case bucketPolicyConfig:
|
||||
return newBucketConfigState(bucket, file, meta.Policy, meta.PolicyUpdatedAt, meta.CreatedAt, false)
|
||||
case bucketTaggingConfig:
|
||||
payload, at = meta.Tags, meta.TagConfigUpdatedAt
|
||||
case bucketSSEConfig:
|
||||
payload, at = meta.SSEConfig, meta.SSEConfigUpdatedAt
|
||||
case bucketQuotaConfigFile:
|
||||
payload, at = meta.QuotaConfig, meta.QuotaConfigUpdatedAt
|
||||
case bucketVersioningConfig:
|
||||
payload, at = meta.Versioning, meta.VersioningConfigUpdatedAt
|
||||
case objectLockConfig:
|
||||
payload, at = meta.ObjectLockConfig, meta.ObjectLockConfigUpdatedAt
|
||||
}
|
||||
var data []byte
|
||||
if payload != nil {
|
||||
var err error
|
||||
data, err = base64.StdEncoding.DecodeString(*payload)
|
||||
if err != nil {
|
||||
return bucketConfigState{}, err
|
||||
}
|
||||
}
|
||||
lockEnabled := meta.ObjectLockConfig != nil && len(*meta.ObjectLockConfig) != 0
|
||||
return newBucketConfigState(bucket, file, data, at, meta.CreatedAt, lockEnabled)
|
||||
}
|
||||
|
||||
func newBucketConfigReplicationEvent(bucket, file string, state bucketConfigState) madmin.SRBucketMeta {
|
||||
event := madmin.SRBucketMeta{Bucket: bucket, UpdatedAt: state.at}
|
||||
var payload *string
|
||||
if len(state.data) != 0 {
|
||||
encoded := base64.StdEncoding.EncodeToString(state.data)
|
||||
payload = &encoded
|
||||
}
|
||||
switch file {
|
||||
case bucketPolicyConfig:
|
||||
event.Type, event.Policy = madmin.SRBucketMetaTypePolicy, state.data
|
||||
case bucketTaggingConfig:
|
||||
event.Type, event.Tags = madmin.SRBucketMetaTypeTags, payload
|
||||
case bucketSSEConfig:
|
||||
event.Type, event.SSEConfig = madmin.SRBucketMetaTypeSSEConfig, payload
|
||||
case bucketQuotaConfigFile:
|
||||
event.Type, event.Quota = madmin.SRBucketMetaTypeQuotaConfig, state.data
|
||||
case bucketVersioningConfig:
|
||||
event.Type, event.Versioning = madmin.SRBucketMetaTypeVersionConfig, payload
|
||||
case objectLockConfig:
|
||||
event.Type, event.ObjectLockConfig = madmin.SRBucketMetaTypeObjectLockConfig, payload
|
||||
}
|
||||
return event
|
||||
}
|
||||
|
||||
func latestBucketConfig(bucket, file string, info srStatusInfo) (bucketConfigState, bool) {
|
||||
var latest bucketConfigState
|
||||
found := false
|
||||
for id, status := range info.BucketStats[bucket] {
|
||||
if _, known := info.Sites[id]; !known || id == "" {
|
||||
continue
|
||||
}
|
||||
state, err := bucketConfigStateFromInfo(bucket, file, status.meta.SRBucketInfo)
|
||||
if err != nil || !state.candidate() {
|
||||
continue
|
||||
}
|
||||
if !found || compareBucketConfigStates(state, latest) > 0 {
|
||||
latest, found = state, true
|
||||
}
|
||||
}
|
||||
return latest, found
|
||||
}
|
||||
|
||||
func (c *SiteReplicationSys) healBucketConfig(ctx context.Context, bucket, file string, info srStatusInfo) error {
|
||||
c.RLock()
|
||||
defer c.RUnlock()
|
||||
if !c.enabled {
|
||||
return nil
|
||||
}
|
||||
latest, found := latestBucketConfig(bucket, file, info)
|
||||
// Every reason keeps its own log key, so a site that did not report cannot
|
||||
// deduplicate away an unusable peer state or a real heal RPC failure for
|
||||
// the same bucket and field.
|
||||
if found {
|
||||
for id := range info.Sites {
|
||||
if _, present := info.BucketStats[bucket][id]; !present {
|
||||
logBucketConfigReplication(ctx, bucket, file, "unreachable", latest.at, time.Time{}, "peer "+id+" did not report")
|
||||
}
|
||||
}
|
||||
}
|
||||
for id, status := range info.BucketStats[bucket] {
|
||||
if _, known := info.Sites[id]; !known || id == "" {
|
||||
continue
|
||||
}
|
||||
target := status.meta.SRBucketInfo
|
||||
current, currentErr := bucketConfigStateFromInfo(bucket, file, target)
|
||||
// A peer without the bucket reports neither a field nor a creation
|
||||
// time; bucket healing covers that normal transient. Report only a
|
||||
// state that exists and still cannot be ordered.
|
||||
if currentErr != nil || (!current.valid && (len(current.data) != 0 || !current.at.IsZero())) {
|
||||
logBucketConfigReplication(ctx, bucket, file, "indeterminate", current.at, target.CreatedAt, "unusable peer "+id)
|
||||
}
|
||||
// Invalid existing state still needs a diagnosis when no source can
|
||||
// be selected. Only propagation depends on having a valid source.
|
||||
if !found || target.CreatedAt.IsZero() {
|
||||
continue
|
||||
}
|
||||
if latest.at.Before(target.CreatedAt) {
|
||||
logBucketConfigReplication(ctx, bucket, file, "before-created", latest.at, target.CreatedAt, "peer "+id)
|
||||
continue
|
||||
}
|
||||
// Versioning can be normalized differently until Object Lock itself has
|
||||
// converged. Compare what this target would actually persist.
|
||||
incoming, err := newBucketConfigState(bucket, file, latest.data, latest.at, target.CreatedAt,
|
||||
target.ObjectLockConfig != nil && len(*target.ObjectLockConfig) != 0)
|
||||
if err != nil || !incoming.candidate() {
|
||||
continue
|
||||
}
|
||||
if currentErr == nil && compareBucketConfigStates(incoming, current) <= 0 {
|
||||
continue
|
||||
}
|
||||
if id == globalDeploymentID() {
|
||||
_, err = globalBucketMetadataSys.updateAndParseMetadata(ctx, bucket, file, latest.data, false, false, &latest.at)
|
||||
} else {
|
||||
var client *madmin.AdminClient
|
||||
client, err = c.getAdminClient(ctx, id)
|
||||
if err == nil {
|
||||
err = client.SRPeerReplicateBucketMeta(ctx, newBucketConfigReplicationEvent(bucket, file, incoming))
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
// A missing credential or unreachable peer must not abandon the other
|
||||
// targets simply because it happened to be visited first in this map.
|
||||
logBucketConfigReplication(ctx, bucket, file, "peer-error", latest.at, target.CreatedAt, "peer "+id+": "+err.Error())
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,779 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
func TestPeerBucketMetadataSourceTimeAndDeletion(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testPeerBucketMetadataSourceTimeAndDeletion})
|
||||
}
|
||||
|
||||
func testPeerBucketMetadataSourceTimeAndDeletion(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
ctx := t.Context()
|
||||
created := UTCNow().Add(-time.Hour)
|
||||
putAt := created.Add(10 * time.Minute)
|
||||
delAt := putAt.Add(time.Minute)
|
||||
enc := func(s string) *string { v := base64.StdEncoding.EncodeToString([]byte(s)); return &v }
|
||||
base := newBucketMetadata(bucket)
|
||||
base.SetCreatedAt(created)
|
||||
base.defaultTimestamps()
|
||||
quotaJSON, err := json.Marshal(madmin.BucketQuota{Quota: 1024, Type: madmin.HardQuota})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cases := []struct {
|
||||
name, file string
|
||||
put madmin.SRBucketMeta
|
||||
value func(BucketMetadata) []byte
|
||||
stamp func(BucketMetadata) time.Time
|
||||
deletable bool
|
||||
}{
|
||||
{"policy", bucketPolicyConfig, madmin.SRBucketMeta{Type: madmin.SRBucketMetaTypePolicy, Policy: []byte(fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket))}, func(m BucketMetadata) []byte { return m.PolicyConfigJSON }, func(m BucketMetadata) time.Time { return m.PolicyConfigUpdatedAt }, true},
|
||||
{"tags", bucketTaggingConfig, madmin.SRBucketMeta{Type: madmin.SRBucketMetaTypeTags, Tags: enc(`<Tagging><TagSet><Tag><Key>key</Key><Value>old</Value></Tag></TagSet></Tagging>`)}, func(m BucketMetadata) []byte { return m.TaggingConfigXML }, func(m BucketMetadata) time.Time { return m.TaggingConfigUpdatedAt }, true},
|
||||
{"sse", bucketSSEConfig, madmin.SRBucketMeta{Type: madmin.SRBucketMetaTypeSSEConfig, SSEConfig: enc(`<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`)}, func(m BucketMetadata) []byte { return m.EncryptionConfigXML }, func(m BucketMetadata) time.Time { return m.EncryptionConfigUpdatedAt }, true},
|
||||
{"quota", bucketQuotaConfigFile, madmin.SRBucketMeta{Type: madmin.SRBucketMetaTypeQuotaConfig, Quota: quotaJSON}, func(m BucketMetadata) []byte { return m.QuotaConfigJSON }, func(m BucketMetadata) time.Time { return m.QuotaConfigUpdatedAt }, true},
|
||||
{"versioning", bucketVersioningConfig, madmin.SRBucketMeta{Type: madmin.SRBucketMetaTypeVersionConfig, Versioning: enc(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status></VersioningConfiguration>`)}, func(m BucketMetadata) []byte { return m.VersioningConfigXML }, func(m BucketMetadata) time.Time { return m.VersioningConfigUpdatedAt }, false},
|
||||
{"objectlock", objectLockConfig, newSRBucketObjectLockMeta(bucket, enc(`<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>30</Days></DefaultRetention></Rule></ObjectLockConfiguration>`), putAt), func(m BucketMetadata) []byte { return m.ObjectLockConfigXML }, func(m BucketMetadata) time.Time { return m.ObjectLockConfigUpdatedAt }, false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(backend+"/"+tc.name, func(t *testing.T) {
|
||||
if err := globalBucketMetadataSys.save(ctx, base); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
item := tc.put
|
||||
item.Bucket, item.UpdatedAt = bucket, putAt
|
||||
apply := func(item madmin.SRBucketMeta) {
|
||||
t.Helper()
|
||||
rec := applySRBucketMetaViaAdmin(t, cred, item)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("admin apply returned %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
read := func() BucketMetadata {
|
||||
t.Helper()
|
||||
m, err := loadBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
apply(item)
|
||||
put := read()
|
||||
if len(tc.value(put)) == 0 {
|
||||
t.Fatal("PUT did not establish live config")
|
||||
}
|
||||
if !tc.stamp(put).Equal(putAt) {
|
||||
t.Errorf("SOURCE_TIME: persisted %s, want source %s", tc.stamp(put), putAt)
|
||||
}
|
||||
apply(madmin.SRBucketMeta{Type: item.Type, Bucket: bucket, UpdatedAt: delAt})
|
||||
after := read()
|
||||
if !tc.deletable {
|
||||
if !bytes.Equal(tc.value(put), tc.value(after)) || !tc.stamp(put).Equal(tc.stamp(after)) {
|
||||
t.Error("NIL_NOOP: update-only config changed")
|
||||
}
|
||||
t.Log("nil payload is correctly a no-op")
|
||||
return
|
||||
}
|
||||
if len(tc.value(after)) != 0 {
|
||||
t.Error("NEWER_DELETE: HTTP 200, but live config remained after newer source DELETE")
|
||||
}
|
||||
if _, err := globalBucketMetadataSys.Delete(ctx, bucket, tc.file); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tombstone := read()
|
||||
if len(tc.value(tombstone)) != 0 {
|
||||
t.Fatal("local DELETE failed to establish tombstone")
|
||||
}
|
||||
apply(item)
|
||||
after = read()
|
||||
if len(tc.value(after)) != 0 {
|
||||
t.Error("STALE_RESURRECTION: older source PUT resurrected a locally deleted config")
|
||||
} else {
|
||||
t.Log("older source PUT did not resurrect config")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPeerBucketMetadataBulkOrdering(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
t.Run(backend, func(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
meta, err := loadBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newAt := meta.Created.Add(time.Hour)
|
||||
newXML := `<Tagging><TagSet><Tag><Key>key</Key><Value>new</Value></Tag></TagSet></Tagging>`
|
||||
meta.TaggingConfigXML, meta.TaggingConfigUpdatedAt = []byte(newXML), newAt
|
||||
if err := globalBucketMetadataSys.save(ctx, meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldXML := base64.StdEncoding.EncodeToString([]byte(`<Tagging><TagSet><Tag><Key>key</Key><Value>old</Value></Tag></TagSet></Tagging>`))
|
||||
item := madmin.SRBucketMeta{Bucket: bucket, Tags: &oldXML, UpdatedAt: newAt.Add(-time.Minute)}
|
||||
rec := applySRBucketMetaViaAdmin(t, cred, item)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("bulk apply returned %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
after, err := loadBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(after.TaggingConfigXML, []byte(newXML)) || !after.TaggingConfigUpdatedAt.Equal(newAt) {
|
||||
t.Errorf("BULK_STALE_OVERWRITE: older bulk event overwrote newer config, value=%q time=%s", after.TaggingConfigXML, after.TaggingConfigUpdatedAt)
|
||||
}
|
||||
})
|
||||
}})
|
||||
}
|
||||
|
||||
func TestPeerBucketMetadataOrderingUnderLock(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
t.Run(backend, func(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
meta, err := loadBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldAt, newAt := meta.Created.Add(time.Hour), meta.Created.Add(2*time.Hour)
|
||||
oldXML := base64.StdEncoding.EncodeToString([]byte(`<Tagging><TagSet><Tag><Key>key</Key><Value>older</Value></Tag></TagSet></Tagging>`))
|
||||
newXML := []byte(`<Tagging><TagSet><Tag><Key>key</Key><Value>newest</Value></Tag></TagSet></Tagging>`)
|
||||
lockCtx, unlock, err := lockBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
locked := true
|
||||
defer func() {
|
||||
if locked {
|
||||
unlock()
|
||||
}
|
||||
}()
|
||||
ready := make(chan struct{}, 1)
|
||||
hook := func(name string) {
|
||||
if name == bucket {
|
||||
select {
|
||||
case ready <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
lockBucketMetadataAcquireHook.Store(&hook)
|
||||
defer lockBucketMetadataAcquireHook.Store(nil)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, bucket, &oldXML, oldAt) }()
|
||||
select {
|
||||
case <-ready:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("older event never reached metadata lock")
|
||||
}
|
||||
// A newer writer commits while holding the existing metadata lock.
|
||||
// The old peer event has already checked the pre-commit cache.
|
||||
meta.TaggingConfigXML, meta.TaggingConfigUpdatedAt = newXML, newAt
|
||||
if err := globalBucketMetadataSys.saveMetadata(lockCtx, obj, &meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unlock()
|
||||
locked = false
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("older event did not finish")
|
||||
}
|
||||
after, err := loadBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(after.TaggingConfigXML) != string(newXML) {
|
||||
t.Errorf("CHECK_OUTSIDE_LOCK: queued older event replaced newer committed tags with %q", after.TaggingConfigXML)
|
||||
}
|
||||
})
|
||||
}})
|
||||
}
|
||||
|
||||
// Count actual metadata persistence, including writes that would be invisible
|
||||
// in a value-only assertion after a duplicate or rejected event.
|
||||
type bucketConfigWriteCounter struct {
|
||||
ObjectLayer
|
||||
writes atomic.Int64
|
||||
}
|
||||
|
||||
func (o *bucketConfigWriteCounter) PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
|
||||
if bucket == minioMetaBucket && path.Base(object) == bucketMetadataFile {
|
||||
o.writes.Add(1)
|
||||
}
|
||||
return o.ObjectLayer.PutObject(ctx, bucket, object, data, opts)
|
||||
}
|
||||
|
||||
func TestBucketConfigStateOrdering(t *testing.T) {
|
||||
created := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
live := []byte(`<Tagging><TagSet><Tag><Key>k</Key><Value>a</Value></Tag></TagSet></Tagging>`)
|
||||
other := bytes.ReplaceAll(live, []byte("<Value>a"), []byte("<Value>b"))
|
||||
state := func(data []byte, at, birth time.Time) bucketConfigState {
|
||||
t.Helper()
|
||||
s, err := newBucketConfigState("bucket", bucketTaggingConfig, data, at, birth, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
baseline := state(nil, created, created)
|
||||
baselineLive := state(live, created, created)
|
||||
put := state(live, created.Add(time.Second), created)
|
||||
deleted := state(nil, put.at, created)
|
||||
lateBaseline := state(other, created.Add(time.Hour), created.Add(time.Hour))
|
||||
tests := []struct {
|
||||
name string
|
||||
lower, higher bucketConfigState
|
||||
}{
|
||||
{"baseline initialization", baseline, baselineLive},
|
||||
{"real before later baseline", lateBaseline, put},
|
||||
{"delete wins tie", put, deleted},
|
||||
{"live key tie", put, state(other, put.at, created)},
|
||||
{"new PUT after deletion", deleted, state(live, put.at.Add(time.Second), created)},
|
||||
{"baseline key ignores creation time", state(live, created.Add(time.Hour), created.Add(time.Hour)), state(other, created, created)},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if compareBucketConfigStates(tc.lower, tc.higher) >= 0 || compareBucketConfigStates(tc.higher, tc.lower) <= 0 {
|
||||
t.Fatal("ordering is not antisymmetric or chose the wrong winner")
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, s := range []bucketConfigState{baseline, state(live, created.Add(-time.Second), created), state(live, created, time.Time{})} {
|
||||
if s.candidate() {
|
||||
t.Fatal("default, pre-creation, or unknown-generation state became a source")
|
||||
}
|
||||
}
|
||||
for _, file := range []string{bucketVersioningConfig, objectLockConfig} {
|
||||
s, err := newBucketConfigState("bucket", file, nil, put.at, created, false)
|
||||
if err != nil || s.candidate() {
|
||||
t.Fatalf("empty update-only candidate: %+v %v", s, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketPolicyReplicationKey(t *testing.T) {
|
||||
// Permute independent set arrays and parse separately, as two sites would.
|
||||
a := []byte(`{"Version":"2012-10-17","Statement":[{"Sid":"one","Effect":"Allow","Principal":{"AWS":["b","a"]},"Action":["s3:GetObject","s3:PutObject"],"Resource":["arn:aws:s3:::bucket/b*","arn:aws:s3:::bucket/a*"],"Condition":{"StringLike":{"s3:prefix":["b*","a*"]}}},{"Sid":"two","Effect":"Deny","Principal":"*","NotAction":["s3:GetObject","s3:PutObject"],"NotResource":["arn:aws:s3:::bucket/d*","arn:aws:s3:::bucket/c*"]}]}`)
|
||||
// Use a condition valid for object actions.
|
||||
a = bytes.ReplaceAll(a, []byte("s3:prefix"), []byte("aws:UserAgent"))
|
||||
b := []byte(`{"Statement":[{"NotResource":["arn:aws:s3:::bucket/c*","arn:aws:s3:::bucket/d*"],"NotAction":["s3:PutObject","s3:GetObject"],"Principal":"*","Effect":"Deny","Sid":"two"},{"Condition":{"StringLike":{"aws:UserAgent":["a*","b*"]}},"Resource":["arn:aws:s3:::bucket/a*","arn:aws:s3:::bucket/b*"],"Action":["s3:PutObject","s3:GetObject"],"Principal":{"AWS":["a","b"]},"Effect":"Allow","Sid":"one"}],"Version":"2012-10-17"}`)
|
||||
_, key, err := bucketConfigPayload("bucket", bucketPolicyConfig, a, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range 30 {
|
||||
_, next, err := bucketConfigPayload("bucket", bucketPolicyConfig, b, false)
|
||||
if err != nil || !bytes.Equal(key, next) {
|
||||
t.Fatalf("independent parse %d: %s != %s (%v)", i, key, next, err)
|
||||
}
|
||||
}
|
||||
_, sid, err := bucketConfigPayload("bucket", bucketPolicyConfig, bytes.ReplaceAll(a, []byte(`"one"`), []byte(`"other"`)), false)
|
||||
if err != nil || bytes.Equal(key, sid) {
|
||||
t.Fatalf("Sid difference lost: %v", err)
|
||||
}
|
||||
n, err := canonicalBucketPolicyJSON([]byte(`{"n":[9007199254740993,9007199254740992]}`))
|
||||
if err != nil || string(n) != `{"n":[9007199254740992,9007199254740993]}` {
|
||||
t.Fatalf("integer precision lost: %s %v", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketPolicyReplicationStatusLegacyOrder(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
t.Run(backend, func(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
// Statement order here is the reverse of the canonical encoder's,
|
||||
// which is what an upgraded peer stores: the permutation must not
|
||||
// be reported as a permanent mismatch.
|
||||
legacy := []byte(fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Sid":"allow","Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"},{"Sid":"deny","Effect":"Deny","Principal":"*","Action":"s3:DeleteObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket, bucket))
|
||||
source, err := policy.ParseBucketPolicyConfig(bytes.NewReader(legacy), bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta := newBucketMetadata(bucket)
|
||||
meta.Created = UTCNow().Add(-time.Hour)
|
||||
meta.PolicyConfigJSON = legacy
|
||||
meta.PolicyConfigUpdatedAt = meta.Created.Add(time.Minute)
|
||||
event, send, err := initialBucketConfigReplicationEvent(meta, bucketPolicyConfig)
|
||||
if err != nil || !send {
|
||||
t.Fatalf("legacy initial event: %v send=%v", err, send)
|
||||
}
|
||||
target := newBucketMetadata(bucket)
|
||||
target.Created = meta.Created
|
||||
if err := globalBucketMetadataSys.save(ctx, target); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec := applySRBucketMetaViaAdmin(t, cred, event); rec.Code != http.StatusOK {
|
||||
t.Fatalf("peer apply: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
received, _, err := globalBucketMetadataSys.GetPolicyConfig(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !isBktPolicyReplicated(2, []*policy.BucketPolicy{source, received}) {
|
||||
t.Fatal("equivalent legacy and received policy reported as permanently mismatched")
|
||||
}
|
||||
changed, err := policy.ParseBucketPolicyConfig(bytes.NewReader(legacy), bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
changed.Statements[0].SID = "different"
|
||||
if isBktPolicyReplicated(2, []*policy.BucketPolicy{source, changed}) {
|
||||
t.Fatal("distinct policy state reported as replicated")
|
||||
}
|
||||
if isBktPolicyReplicated(2, []*policy.BucketPolicy{source, nil}) || !isBktPolicyReplicated(2, []*policy.BucketPolicy{nil, nil}) {
|
||||
t.Fatal("per-site presence accounting changed")
|
||||
}
|
||||
})
|
||||
}})
|
||||
}
|
||||
|
||||
func TestPeerBucketMetadataWireAtomicity(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
t.Run(backend, func(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
meta, err := loadBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stamp := meta.Created.Add(time.Hour)
|
||||
policyData := []byte(fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket))
|
||||
meta.PolicyConfigJSON, meta.PolicyConfigUpdatedAt = policyData, stamp
|
||||
meta.QuotaConfigJSON, meta.QuotaConfigUpdatedAt = []byte(`{"quota":1024,"quotatype":"hard"}`), stamp
|
||||
meta.TaggingConfigXML, meta.TaggingConfigUpdatedAt = []byte(`<Tagging><TagSet><Tag><Key>k</Key><Value>a</Value></Tag></TagSet></Tagging>`), stamp
|
||||
if err = globalBucketMetadataSys.save(ctx, meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
counter := &bucketConfigWriteCounter{ObjectLayer: obj}
|
||||
setObjectLayer(counter)
|
||||
defer setObjectLayer(obj)
|
||||
read := func() BucketMetadata {
|
||||
t.Helper()
|
||||
m, err := loadBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
apply := func(fragment string, at time.Time, ok bool) {
|
||||
t.Helper()
|
||||
raw := fmt.Sprintf(`{"bucket":%q,"updatedAt":%q,%s}`, bucket, at.Format(time.RFC3339Nano), fragment)
|
||||
var item madmin.SRBucketMeta
|
||||
if err := json.Unmarshal([]byte(raw), &item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := applySRBucketMetaViaAdmin(t, cred, item)
|
||||
if (rec.Code == http.StatusOK) != ok {
|
||||
t.Fatalf("%s: %d %s", raw, rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
// Omitted/null *string and empty update-only payloads are all no-ops.
|
||||
apply(`"policy":null,"quota":null,"tags":null,"versioningConfig":"","objectLockConfig":""`, stamp.Add(time.Minute), true)
|
||||
current := read()
|
||||
if len(current.PolicyConfigJSON) != 0 || len(current.QuotaConfigJSON) == 0 {
|
||||
t.Fatal("explicit null semantics lost")
|
||||
}
|
||||
if q, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket); err != nil || q.Quota != 0 {
|
||||
t.Fatalf("zero quota cache: %+v %v", q, err)
|
||||
}
|
||||
before := counter.writes.Load()
|
||||
apply(`"tags":null,"versioningConfig":"","objectLockConfig":""`, stamp.Add(2*time.Minute), true)
|
||||
if counter.writes.Load() != before {
|
||||
t.Fatal("omitted fields or empty update-only wrote metadata")
|
||||
}
|
||||
// An invalid second field cannot persist the valid deletion in this bulk.
|
||||
apply(`"tags":"","quota":{"quota":1,"quotatype":"invalid"}`, stamp.Add(2*time.Minute), false)
|
||||
if counter.writes.Load() != before || len(read().TaggingConfigXML) == 0 {
|
||||
t.Fatal("invalid bulk partially committed")
|
||||
}
|
||||
apply(`"tags":""`, stamp.Add(2*time.Minute), true)
|
||||
before = counter.writes.Load()
|
||||
apply(`"tags":""`, stamp.Add(2*time.Minute), true)
|
||||
apply(`"tags":""`, stamp.Add(time.Minute), true)
|
||||
if counter.writes.Load() != before {
|
||||
t.Fatal("duplicate or old event wrote metadata")
|
||||
}
|
||||
localAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, []byte(`{}`))
|
||||
if err != nil || !localAt.After(current.QuotaConfigUpdatedAt) {
|
||||
t.Fatalf("local future monotonic time: %v %v", localAt, err)
|
||||
}
|
||||
deletedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketQuotaConfigFile)
|
||||
if err != nil || !deletedAt.After(localAt) {
|
||||
t.Fatalf("adjacent local time: %v %v", deletedAt, err)
|
||||
}
|
||||
if q, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket); err != nil || q.Quota != 0 {
|
||||
t.Fatalf("deleted quota cache: %+v %v", q, err)
|
||||
}
|
||||
})
|
||||
}})
|
||||
}
|
||||
|
||||
func TestPeerBucketAdoptionRebasesOnlyDefaults(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
for _, shift := range []time.Duration{-time.Hour, 0, time.Hour, 3 * time.Hour} {
|
||||
t.Run(fmt.Sprintf("%s/%s", backend, shift), func(t *testing.T) {
|
||||
created := UTCNow().Add(-3 * time.Hour)
|
||||
meta := newBucketMetadata(bucket)
|
||||
meta.Created = created
|
||||
meta.defaultTimestamps()
|
||||
meta.QuotaConfigUpdatedAt = time.Time{}
|
||||
meta.TaggingConfigUpdatedAt = created.Add(2 * time.Hour) // real deletion
|
||||
meta.EncryptionConfigXML = []byte(`<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`)
|
||||
meta.EncryptionConfigUpdatedAt = created.Add(2 * time.Hour)
|
||||
if err := globalBucketMetadataSys.save(t.Context(), meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(t.Context(), bucket, MakeBucketOptions{CreatedAt: created.Add(shift)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := readBucketMetadata(t.Context(), obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !got.PolicyConfigUpdatedAt.Equal(got.Created) || !got.QuotaConfigUpdatedAt.Equal(got.Created) {
|
||||
t.Fatal("default turned into tombstone or invalid source")
|
||||
}
|
||||
if !got.TaggingConfigUpdatedAt.Equal(meta.TaggingConfigUpdatedAt) || !got.EncryptionConfigUpdatedAt.Equal(meta.EncryptionConfigUpdatedAt) || !bytes.Equal(got.EncryptionConfigXML, meta.EncryptionConfigXML) {
|
||||
t.Fatal("actual state changed during adoption")
|
||||
}
|
||||
if shift > 2*time.Hour {
|
||||
// Preserve history, but do not promote state from an earlier
|
||||
// bucket generation to a new valid source by retimestamping it.
|
||||
for _, file := range []string{bucketTaggingConfig, bucketSSEConfig} {
|
||||
data, at := replicatedBucketConfig(&got, file)
|
||||
state, err := newBucketConfigState(bucket, file, *data, *at, got.Created, false)
|
||||
if err != nil || state.candidate() {
|
||||
t.Fatalf("pre-generation history became a source: %s %v", file, err)
|
||||
}
|
||||
// As a target, that invalid history must yield to a valid
|
||||
// state from the adopted generation, including a live tag
|
||||
// replacing the preserved earlier-generation deletion.
|
||||
incoming := bucketConfigTestData(bucket)[file]
|
||||
incomingAt := got.Created.Add(time.Minute)
|
||||
changed, err := applyBucketConfig(&got, file, incoming, incomingAt)
|
||||
if err != nil || !changed || !at.Equal(incomingAt) || !bytes.Equal(*data, incoming) {
|
||||
t.Fatalf("adopted generation did not replace invalid target: %s %v", file, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}})
|
||||
}
|
||||
|
||||
func recordBucketConfigPeer(t *testing.T, cred auth.Credentials) func() []madmin.SRBucketMeta {
|
||||
t.Helper()
|
||||
var mu sync.Mutex
|
||||
var events []madmin.SRBucketMeta
|
||||
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var event madmin.SRBucketMeta
|
||||
if err := json.NewDecoder(r.Body).Decode(&event); err != nil {
|
||||
t.Error(err)
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
events = append(events, event)
|
||||
mu.Unlock()
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
t.Cleanup(remote.Close)
|
||||
serviceCred, err := auth.CreateCredentials("metadata-source-time-svc", "metadata-source-time-service-secret")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serviceCred.ParentUser = cred.AccessKey
|
||||
globalSiteReplicatorCred.Set(serviceCred.SecretKey)
|
||||
t.Cleanup(func() { globalSiteReplicatorCred.Set("") })
|
||||
if _, err = globalIAMSys.store.AddServiceAccount(t.Context(), serviceCred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { globalIAMSys.DeleteServiceAccount(context.Background(), serviceCred.AccessKey, false) })
|
||||
globalSiteReplicationSys.Lock()
|
||||
oldEnabled, oldState := globalSiteReplicationSys.enabled, globalSiteReplicationSys.state
|
||||
globalSiteReplicationSys.enabled = true
|
||||
globalSiteReplicationSys.state = srState{Name: "metadata-source-test", ServiceAccountAccessKey: serviceCred.AccessKey, Peers: map[string]madmin.PeerInfo{
|
||||
globalDeploymentID(): {Name: "local", DeploymentID: globalDeploymentID()},
|
||||
"metadata-peer": {Name: "remote", DeploymentID: "metadata-peer", Endpoint: remote.URL},
|
||||
}}
|
||||
globalSiteReplicationSys.Unlock()
|
||||
t.Cleanup(func() {
|
||||
globalSiteReplicationSys.Lock()
|
||||
globalSiteReplicationSys.enabled, globalSiteReplicationSys.state = oldEnabled, oldState
|
||||
globalSiteReplicationSys.Unlock()
|
||||
})
|
||||
return func() []madmin.SRBucketMeta {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return append([]madmin.SRBucketMeta(nil), events...)
|
||||
}
|
||||
}
|
||||
|
||||
// The final metadata lock is taken after reading the ZIP. Commit another
|
||||
// writer immediately before that acquisition, using the existing lock wrapper.
|
||||
type importInterleavingObjectLayer struct {
|
||||
ObjectLayer
|
||||
once atomic.Bool
|
||||
write func()
|
||||
}
|
||||
|
||||
func (o *importInterleavingObjectLayer) NewNSLock(bucket string, objects ...string) RWLocker {
|
||||
lock := o.ObjectLayer.NewNSLock(bucket, objects...)
|
||||
if bucket != minioMetaBucket || len(objects) != 1 || path.Base(objects[0]) != "metadata.lock" {
|
||||
return lock
|
||||
}
|
||||
return metadataObservedRWLocker{RWLocker: lock, onLock: func(context.Context) {
|
||||
if o.once.CompareAndSwap(false, true) {
|
||||
o.write()
|
||||
}
|
||||
}}
|
||||
}
|
||||
|
||||
func TestLocalBucketMetadataCommittedEvents(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, endpoints: []string{"PutBucketPolicy", "GetBucketPolicy", "PutBucketVersioning"}, objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
t.Run(backend, func(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
events := recordBucketConfigPeer(t, cred)
|
||||
putPolicy := func(data []byte) {
|
||||
t.Helper()
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, getPutPolicyURL("", bucket), int64(len(data)), bytes.NewReader(data), cred.AccessKey, cred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("empty policy PUT: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
putPolicy([]byte(`{"Version":"2012-10-17","Statement":[]}`))
|
||||
meta, err := loadBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := events()
|
||||
if len(got) != 1 || got[0].Type != madmin.SRBucketMetaTypePolicy || len(got[0].Policy) != 0 || len(meta.PolicyConfigJSON) != 0 || !meta.PolicyConfigUpdatedAt.Equal(got[0].UpdatedAt) {
|
||||
t.Fatalf("empty policy event/disk mismatch: %+v", got)
|
||||
}
|
||||
req, err := newTestSignedRequestV4(http.MethodGet, getGetPolicyURL("", bucket), 0, nil, cred.AccessKey, cred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("empty policy GET: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
// Supported negative sets must survive the same serializer on PUT,
|
||||
// peer apply, GET and export instead of failing on empty Action.
|
||||
putPolicy([]byte(fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","NotAction":["s3:DeleteObject"],"NotResource":["arn:aws:s3:::%s/private/*"]}]}`, bucket)))
|
||||
req, err = newTestSignedRequestV4(http.MethodGet, getGetPolicyURL("", bucket), 0, nil, cred.AccessKey, cred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK || !bytes.Contains(rec.Body.Bytes(), []byte(`"NotAction"`)) || !bytes.Contains(rec.Body.Bytes(), []byte(`"NotResource"`)) {
|
||||
t.Fatalf("negative policy set GET: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
corsAdminRequest(t, cred, http.MethodPut, "/set-bucket-quota?bucket="+bucket, []byte(`{}`))
|
||||
meta, err = loadBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got = events()
|
||||
last := got[len(got)-1]
|
||||
if last.Type != madmin.SRBucketMetaTypeQuotaConfig || len(last.Quota) == 0 || !last.UpdatedAt.Equal(meta.QuotaConfigUpdatedAt) {
|
||||
t.Fatalf("zero quota event: %+v", last)
|
||||
}
|
||||
|
||||
injectedAt := UTCNow().Add(2 * time.Hour)
|
||||
interleaving := &importInterleavingObjectLayer{ObjectLayer: obj, write: func() {
|
||||
data := []byte(`{"quota":2048,"quotatype":"hard"}`)
|
||||
_, err := globalBucketMetadataSys.updateAndParseMetadata(ctx, bucket, bucketQuotaConfigFile, data, false, false, &injectedAt)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}}
|
||||
setObjectLayer(interleaving)
|
||||
defer setObjectLayer(obj)
|
||||
before := len(events())
|
||||
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata", corsZip(t, map[string][]byte{
|
||||
bucket + "/" + bucketPolicyConfig: []byte(`{"Version":"2012-10-17","Statement":[]}`),
|
||||
bucket + "/quota.json": []byte(`{}`),
|
||||
bucket + "/" + bucketTaggingConfig: []byte(`<Tagging><TagSet><Tag><Key>imported</Key><Value>yes</Value></Tag></TagSet></Tagging>`),
|
||||
bucket + "/" + objectLockConfig: enabledBucketObjectLockConfig,
|
||||
bucket + "/" + bucketVersioningConfig: []byte(`<VersioningConfiguration><Status>Enabled</Status><ExcludeFolders>true</ExcludeFolders></VersioningConfiguration>`),
|
||||
}))
|
||||
report := corsImportReport(t, rec).Buckets[bucket]
|
||||
if report.Err != "" || report.Policy.Err != "" || report.Quota.Err != "" || report.Tagging.Err != "" {
|
||||
t.Fatalf("import: %+v", report)
|
||||
}
|
||||
meta, err = loadBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !meta.QuotaConfigUpdatedAt.After(injectedAt) || !meta.TaggingConfigUpdatedAt.Equal(meta.QuotaConfigUpdatedAt) || !meta.PolicyConfigUpdatedAt.Equal(meta.QuotaConfigUpdatedAt) {
|
||||
t.Fatalf("import time %v must exceed intervening %v; policy=%v tags=%v", meta.QuotaConfigUpdatedAt, injectedAt, meta.PolicyConfigUpdatedAt, meta.TaggingConfigUpdatedAt)
|
||||
}
|
||||
if !bytes.Equal(meta.VersioningConfigXML, enabledBucketVersioningConfig) || !meta.VersioningConfigUpdatedAt.Equal(meta.QuotaConfigUpdatedAt) || !meta.ObjectLockConfigUpdatedAt.Equal(meta.QuotaConfigUpdatedAt) {
|
||||
t.Fatal("import normalization or common source time lost")
|
||||
}
|
||||
got = events()[before:]
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("import events: %+v", got)
|
||||
}
|
||||
for _, event := range got {
|
||||
if !event.UpdatedAt.Equal(meta.QuotaConfigUpdatedAt) {
|
||||
t.Fatal("import hook used ZIP start time")
|
||||
}
|
||||
if event.Type == madmin.SRBucketMetaTypePolicy {
|
||||
if len(event.Policy) != 0 {
|
||||
t.Fatal("empty policy import is not deletion")
|
||||
}
|
||||
} else if event.Tags == nil || len(event.Quota) == 0 {
|
||||
t.Fatalf("bulk omitted imported state: %+v", event)
|
||||
}
|
||||
if event.Type == "" {
|
||||
if event.Versioning == nil || event.ObjectLockConfig == nil {
|
||||
t.Fatal("normalized import fields omitted")
|
||||
}
|
||||
payload, err := base64.StdEncoding.DecodeString(*event.Versioning)
|
||||
if err != nil || !bytes.Equal(payload, enabledBucketVersioningConfig) {
|
||||
t.Fatal("import sent pre-normalization versioning")
|
||||
}
|
||||
}
|
||||
}
|
||||
beforeMeta, beforeEvents := meta, len(events())
|
||||
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata", corsZip(t, map[string][]byte{
|
||||
bucket + "/" + bucketTaggingConfig: []byte(`<Tagging><TagSet><Tag><Key>only</Key><Value>tags</Value></Tag></TagSet></Tagging>`),
|
||||
}))
|
||||
if status := corsImportReport(t, rec).Buckets[bucket]; status.Err != "" || status.Tagging.Err != "" {
|
||||
t.Fatalf("tags-only import: %+v", status)
|
||||
}
|
||||
meta, err = loadBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(meta.PolicyConfigJSON, beforeMeta.PolicyConfigJSON) || !meta.PolicyConfigUpdatedAt.Equal(beforeMeta.PolicyConfigUpdatedAt) || !bytes.Equal(meta.QuotaConfigJSON, beforeMeta.QuotaConfigJSON) || !meta.QuotaConfigUpdatedAt.Equal(beforeMeta.QuotaConfigUpdatedAt) {
|
||||
t.Fatal("tags-only import changed Policy or Quota")
|
||||
}
|
||||
got = events()[beforeEvents:]
|
||||
if len(got) != 1 || got[0].Tags == nil || got[0].Policy != nil || got[0].Quota != nil || got[0].Versioning != nil || got[0].ObjectLockConfig != nil || !got[0].UpdatedAt.Equal(meta.TaggingConfigUpdatedAt) {
|
||||
t.Fatalf("tags-only import hook leaked unspecified fields: %+v", got)
|
||||
}
|
||||
})
|
||||
}})
|
||||
}
|
||||
|
||||
func TestPeerBucketMetadataNormalizesBeforeComparison(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
t.Run(backend, func(t *testing.T) {
|
||||
meta, err := loadBucketMetadata(t.Context(), obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
versioning := base64.StdEncoding.EncodeToString([]byte(`<VersioningConfiguration><Status>Enabled</Status><ExcludedPrefixes><Prefix>skip/</Prefix></ExcludedPrefixes><ExcludeFolders>true</ExcludeFolders></VersioningConfiguration>`))
|
||||
lock := base64.StdEncoding.EncodeToString(enabledBucketObjectLockConfig)
|
||||
item := madmin.SRBucketMeta{Bucket: bucket, ObjectLockConfig: &lock, Versioning: &versioning, UpdatedAt: meta.Created.Add(time.Hour)}
|
||||
counter := &bucketConfigWriteCounter{ObjectLayer: obj}
|
||||
setObjectLayer(counter)
|
||||
defer setObjectLayer(obj)
|
||||
for range 2 {
|
||||
rec := applySRBucketMetaViaAdmin(t, cred, item)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("bulk: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
if counter.writes.Load() != 1 {
|
||||
t.Fatalf("normalized duplicate persisted %d times", counter.writes.Load())
|
||||
}
|
||||
meta, err = loadBucketMetadata(t.Context(), obj, bucket)
|
||||
if err != nil || !bytes.Equal(meta.VersioningConfigXML, enabledBucketVersioningConfig) || !meta.VersioningConfigUpdatedAt.Equal(item.UpdatedAt) {
|
||||
t.Fatalf("normalization: %s %v", meta.VersioningConfigXML, err)
|
||||
}
|
||||
// Ordinary local updates use the identical effective document in their
|
||||
// commit result, even if Object Lock changed since the handler's precheck.
|
||||
data, _ := base64.StdEncoding.DecodeString(versioning)
|
||||
result, err := globalBucketMetadataSys.updateAndParseMetadata(t.Context(), bucket, bucketVersioningConfig, data, false, false, nil)
|
||||
if err != nil || !bytes.Equal(result.meta.VersioningConfigXML, enabledBucketVersioningConfig) {
|
||||
t.Fatalf("local commit snapshot: %s %v", result.meta.VersioningConfigXML, err)
|
||||
}
|
||||
})
|
||||
}})
|
||||
}
|
||||
|
||||
func TestPeerBucketMetadataEqualTimeArrivalOrders(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
t.Run(backend, func(t *testing.T) {
|
||||
created := UTCNow().Add(-time.Hour)
|
||||
a := []byte(`<Tagging><TagSet><Tag><Key>key</Key><Value>a</Value></Tag></TagSet></Tagging>`)
|
||||
b := bytes.ReplaceAll(a, []byte("<Value>a"), []byte("<Value>b"))
|
||||
for _, pair := range [][2][]byte{{a, b}, {b, a}, {a, nil}, {nil, a}} {
|
||||
meta := newBucketMetadata(bucket)
|
||||
meta.Created = created
|
||||
meta.defaultTimestamps()
|
||||
if err := globalBucketMetadataSys.save(t.Context(), meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, data := range pair {
|
||||
var payload *string
|
||||
if data != nil {
|
||||
encoded := base64.StdEncoding.EncodeToString(data)
|
||||
payload = &encoded
|
||||
}
|
||||
rec := applySRBucketMetaViaAdmin(t, cred, madmin.SRBucketMeta{Bucket: bucket, Type: madmin.SRBucketMetaTypeTags, Tags: payload, UpdatedAt: created.Add(time.Minute)})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("equal time apply: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
got, err := readBucketMetadata(t.Context(), obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := b
|
||||
if pair[0] == nil || pair[1] == nil {
|
||||
want = nil
|
||||
}
|
||||
if !bytes.Equal(got.TaggingConfigXML, want) || !got.TaggingConfigUpdatedAt.Equal(created.Add(time.Minute)) {
|
||||
t.Fatalf("arrival order changed winner: %q at %v", got.TaggingConfigXML, got.TaggingConfigUpdatedAt)
|
||||
}
|
||||
}
|
||||
})
|
||||
}})
|
||||
}
|
||||
@@ -204,16 +204,24 @@ func (sm *siteResyncMetrics) updateState(s SiteResyncStatus) error {
|
||||
switch s.Status {
|
||||
case ResyncStarted:
|
||||
sm.peerResyncMap[s.DeplID] = resyncState{resyncID: s.ResyncID, LastSaved: time.Time{}}
|
||||
sm.resyncStatus[s.ResyncID] = s
|
||||
sm.resyncStatus[s.ResyncID] = s.clone()
|
||||
case ResyncCompleted, ResyncCanceled, ResyncFailed:
|
||||
st, ok := sm.resyncStatus[s.ResyncID]
|
||||
if ok {
|
||||
st.LastUpdate = s.LastUpdate
|
||||
st.Status = s.Status
|
||||
if s.Status == ResyncCanceled {
|
||||
for bucket, status := range st.BucketStatuses {
|
||||
if status == ResyncPending || status == ResyncStarted {
|
||||
st.BucketStatuses[bucket] = ResyncCanceled
|
||||
}
|
||||
}
|
||||
}
|
||||
sm.resyncStatus[s.ResyncID] = st
|
||||
return nil
|
||||
}
|
||||
sm.resyncStatus[s.ResyncID] = st
|
||||
return saveSiteResyncMetadata(GlobalContext, st, newObjectLayerFn())
|
||||
sm.resyncStatus[s.ResyncID] = s.clone()
|
||||
return saveSiteResyncMetadata(GlobalContext, s, newObjectLayerFn())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -230,7 +238,15 @@ func (sm *siteResyncMetrics) incBucket(o resyncOpts, bktStatus ResyncStatusType)
|
||||
if st.BucketStatuses == nil {
|
||||
st.BucketStatuses = map[string]ResyncStatusType{}
|
||||
}
|
||||
if st.BucketStatuses[o.bucket] == ResyncCanceled {
|
||||
return
|
||||
}
|
||||
switch bktStatus {
|
||||
case ResyncCanceled:
|
||||
st.BucketStatuses[o.bucket] = ResyncCanceled
|
||||
st.Status = ResyncCanceled
|
||||
st.LastUpdate = UTCNow()
|
||||
sm.resyncStatus[o.resyncID] = st
|
||||
case ResyncCompleted:
|
||||
st.BucketStatuses[o.bucket] = ResyncCompleted
|
||||
st.Status = siteResyncStatus(st.Status, st.BucketStatuses)
|
||||
|
||||
+159
-683
File diff suppressed because it is too large
Load Diff
@@ -133,28 +133,3 @@ func TestSRBucketMetaCorsRoundTrip(t *testing.T) {
|
||||
t.Fatalf("expected nil Cors for deletion, got %q", *gotDel.Cors)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsBucketMetadataEqualCors covers the pointer-comparison helper used by
|
||||
// the CORS heal path to decide whether a peer already holds the latest config.
|
||||
func TestIsBucketMetadataEqualCors(t *testing.T) {
|
||||
a := base64.StdEncoding.EncodeToString([]byte("config-a"))
|
||||
b := base64.StdEncoding.EncodeToString([]byte("config-b"))
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
one *string
|
||||
two *string
|
||||
want bool
|
||||
}{
|
||||
{"both nil", nil, nil, true},
|
||||
{"one nil", &a, nil, false},
|
||||
{"other nil", nil, &b, false},
|
||||
{"equal", &a, &a, true},
|
||||
{"different", &a, &b, false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := isBucketMetadataEqual(tc.one, tc.two); got != tc.want {
|
||||
t.Errorf("%s: got %v want %v", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+27
@@ -0,0 +1,27 @@
|
||||
# Data-usage v9 retirement fixture
|
||||
|
||||
Generated using unmodified `scanDataFolder` and `dataUsageCache.serializeTo` from
|
||||
Server commit `89637554d60c27cfc51d2281d0a4fe15e415f06d` (Go 1.27.1, darwin/arm64).
|
||||
The scanner visits three real local files; its size callback supplies synthetic
|
||||
version/delete-marker/remote-tier summaries, following `TestDataUsageCacheSerialize`.
|
||||
It is a scanner/cache compatibility fixture, not a distributed object-store test.
|
||||
|
||||
The old scanner counts 74,962 bytes, 3 objects, 5 versions and 2 delete markers.
|
||||
Its nonzero retired `hts` totals 9,426 bytes. Remote tier `COLD` contains 65,536
|
||||
bytes, one version and one object. The cache includes nested children, both
|
||||
histograms, a fixed timestamp and scanner cycle 42. The JSON is the old scanner's
|
||||
complete expected cache (including `HotTierSize`, which the new reader ignores).
|
||||
|
||||
Binary SHA-256: `4c9c7e94cea7758fe9b498b19f639188764b2787582783e6cc950f2c44d52a8b`.
|
||||
|
||||
To regenerate, create a detached worktree at that exact source commit, copy
|
||||
`generate.go.txt` to `cmd/retirement-fixture_test.go`, and run:
|
||||
|
||||
```sh
|
||||
SILO_RETIRE_FIXTURE_DIR=/absolute/output/directory go test ./cmd -run '^TestGenerateAccessRetirementV9Fixture$' -count=1 -v
|
||||
```
|
||||
|
||||
The historical production writer adds the version byte, compresses with zstd
|
||||
and encodes msgp, including the nonzero `hts` field. Do not regenerate using the
|
||||
retired implementation or by changing the header of a v8 payload. Map order may
|
||||
change serialized bytes across regeneration; compare the decoded full cache.
|
||||
BIN
Binary file not shown.
+118
@@ -0,0 +1,118 @@
|
||||
{
|
||||
"Info": {
|
||||
"Name": "/",
|
||||
"NextCycle": 42,
|
||||
"LastUpdate": "2026-09-15T00:00:00Z",
|
||||
"SkipHealing": true
|
||||
},
|
||||
"Cache": {
|
||||
"/": {
|
||||
"Children": {
|
||||
"v9-bucket": {}
|
||||
},
|
||||
"Size": 0,
|
||||
"HotTierSize": 0,
|
||||
"Objects": 0,
|
||||
"Versions": 0,
|
||||
"DeleteMarkers": 0,
|
||||
"ObjSizes": [
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
],
|
||||
"ObjVersions": [
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
],
|
||||
"AllTierStats": null,
|
||||
"Compacted": false
|
||||
},
|
||||
"v9-bucket": {
|
||||
"Children": {
|
||||
"v9-bucket/nested": {}
|
||||
},
|
||||
"Size": 1234,
|
||||
"HotTierSize": 1234,
|
||||
"Objects": 1,
|
||||
"Versions": 1,
|
||||
"DeleteMarkers": 0,
|
||||
"ObjSizes": [
|
||||
0,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
],
|
||||
"ObjVersions": [
|
||||
0,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
],
|
||||
"AllTierStats": null,
|
||||
"Compacted": false
|
||||
},
|
||||
"v9-bucket/nested": {
|
||||
"Children": null,
|
||||
"Size": 73728,
|
||||
"HotTierSize": 8192,
|
||||
"Objects": 2,
|
||||
"Versions": 4,
|
||||
"DeleteMarkers": 2,
|
||||
"ObjSizes": [
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
],
|
||||
"ObjVersions": [
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
],
|
||||
"AllTierStats": {
|
||||
"Tiers": {
|
||||
"COLD": {
|
||||
"TotalSize": 65536,
|
||||
"NumVersions": 1,
|
||||
"NumObjects": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"Compacted": true
|
||||
}
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/cachevalue"
|
||||
)
|
||||
|
||||
// Run only on 89637554d, before access-tiering is removed. This uses the real
|
||||
// folder scanner and v9 serializer with synthetic file-size/tier summaries,
|
||||
// following TestDataUsageCacheSerialize; it does not relabel a v8 payload.
|
||||
func TestGenerateAccessRetirementV9Fixture(t *testing.T) {
|
||||
if dataUsageCacheVerCurrent != 9 { t.Fatal("requires the historical v9 writer") }
|
||||
base := t.TempDir()
|
||||
const bucket = "v9-bucket"
|
||||
createUsageTestFiles(t, base, bucket, []usageTestFile{
|
||||
{name: "root", size: 1234},
|
||||
{name: "nested/versions", size: 8192},
|
||||
{name: "nested/remote", size: 65536},
|
||||
})
|
||||
getSize := func(item scannerItem) (s sizeSummary, err error) {
|
||||
if item.Typ&os.ModeDir != 0 { return s, nil }
|
||||
info, err := os.Stat(item.Path)
|
||||
if err != nil { return s, err }
|
||||
s.totalSize, s.hotTierSize, s.versions = info.Size(), info.Size(), 1
|
||||
if filepath.Base(item.Path) == "versions" { s.versions, s.deleteMarkers = 3, 2 }
|
||||
if filepath.Base(item.Path) == "remote" {
|
||||
s.hotTierSize = 0
|
||||
s.tiers = map[string]tierStats{"COLD": {TotalSize: uint64(info.Size()), NumVersions: 1, NumObjects: 1}}
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
xls := xlStorage{drivePath: base, diskInfoCache: cachevalue.New[DiskInfo]()}
|
||||
xls.diskInfoCache.InitOnce(time.Second, cachevalue.Opts{}, func(context.Context) (DiskInfo,error) {
|
||||
return DiskInfo{Total: 1<<40, Free: 1<<40}, nil
|
||||
})
|
||||
cache, err := scanDataFolder(t.Context(), nil, &xls, dataUsageCache{Info:dataUsageCacheInfo{Name:bucket, SkipHealing:true}}, getSize, 0, func()bool{return false})
|
||||
if err != nil { t.Fatal(err) }
|
||||
root := *cache.find(bucket)
|
||||
cache.replace(dataUsageRoot, "", dataUsageEntry{})
|
||||
cache.replace(bucket, dataUsageRoot, root)
|
||||
cache.Info.Name = dataUsageRoot
|
||||
cache.Info.LastUpdate = time.Date(2026, 9, 15, 0, 0, 0, 0, time.UTC)
|
||||
cache.Info.NextCycle = 42
|
||||
flat := cache.flatten(*cache.root())
|
||||
if flat.Size != 74962 || flat.Objects != 3 || flat.Versions != 5 || flat.DeleteMarkers != 2 || flat.HotTierSize != 9426 {
|
||||
t.Fatalf("unexpected scanner fixture: %+v", flat)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := cache.serializeTo(&buf); err != nil { t.Fatal(err) }
|
||||
dir := os.Getenv("SILO_RETIRE_FIXTURE_DIR")
|
||||
if dir == "" { t.Fatal("SILO_RETIRE_FIXTURE_DIR required") }
|
||||
if err := os.MkdirAll(dir, 0755); err != nil { t.Fatal(err) }
|
||||
if err := os.WriteFile(filepath.Join(dir,"data-usage-v9.bin"),buf.Bytes(),0644);err != nil{t.Fatal(err)}
|
||||
expected, err := json.MarshalIndent(cache,""," ")
|
||||
if err != nil { t.Fatal(err) }
|
||||
if err := os.WriteFile(filepath.Join(dir,"data-usage-v9.json"),append(expected,'\n'),0644);err != nil{t.Fatal(err)}
|
||||
t.Logf("old scanner/v9 writer: bytes=%d size=%d objects=%d versions=%d markers=%d hts=%d",buf.Len(),flat.Size,flat.Objects,flat.Versions,flat.DeleteMarkers,flat.HotTierSize)
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
// Copyright (c) 2026 Pigsty
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/config/etcd"
|
||||
)
|
||||
|
||||
func TestOutboundTLSKeyExchangeDefaults(t *testing.T) {
|
||||
for _, debug := range []string{"tlsmlkem=0", "tlsmlkem=1"} {
|
||||
t.Run(debug, func(t *testing.T) {
|
||||
t.Setenv("GODEBUG", debug)
|
||||
for _, version := range []uint16{tls.VersionTLS12, tls.VersionTLS13} {
|
||||
t.Run(tls.VersionName(version), func(t *testing.T) {
|
||||
hellos := make(chan []tls.CurveID, 1)
|
||||
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = io.WriteString(w, "ok")
|
||||
}))
|
||||
server.TLS = &tls.Config{
|
||||
MinVersion: version, MaxVersion: version,
|
||||
GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) {
|
||||
select {
|
||||
case hellos <- slices.Clone(hello.SupportedCurves):
|
||||
default:
|
||||
}
|
||||
return nil, nil
|
||||
},
|
||||
}
|
||||
server.StartTLS()
|
||||
t.Cleanup(server.Close)
|
||||
roots := x509.NewCertPool()
|
||||
roots.AddCert(server.Certificate())
|
||||
|
||||
dir := t.TempDir()
|
||||
certFile, keyFile := filepath.Join(dir, "public.crt"), filepath.Join(dir, "private.key")
|
||||
cert := server.TLS.Certificates[0]
|
||||
key, err := x509.MarshalPKCS8PrivateKey(cert.PrivateKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for path, block := range map[string]*pem.Block{
|
||||
certFile: {Type: "CERTIFICATE", Bytes: cert.Certificate[0]},
|
||||
keyFile: {Type: "PRIVATE KEY", Bytes: key},
|
||||
} {
|
||||
if err := os.WriteFile(path, pem.EncodeToMemory(block), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
t.Setenv(etcd.EnvEtcdEndpoints, server.URL)
|
||||
t.Setenv(etcd.EnvEtcdClientCert, "")
|
||||
t.Setenv(etcd.EnvEtcdClientCertKey, "")
|
||||
etcdConfig, err := etcd.LookupConfig(etcd.DefaultKVS, roots)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, transport := range map[string]*http.Transport{
|
||||
"external": NewHTTPTransport(),
|
||||
"internode": NewInternodeHTTPTransport(1)().(*http.Transport),
|
||||
"replication": NewRemoteTargetHTTPTransport(false)(),
|
||||
"cloud-client-cert": NewHTTPTransportWithClientCerts(certFile, keyFile).(*http.Transport),
|
||||
"etcd": {TLSClientConfig: etcdConfig.TLS},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
defer transport.CloseIdleConnections()
|
||||
transport.Proxy = nil
|
||||
transport.TLSClientConfig.RootCAs = roots
|
||||
client := &http.Client{Transport: transport, Timeout: 5 * time.Second}
|
||||
resp, err := client.Get(server.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK || resp.TLS.Version != version {
|
||||
t.Fatalf("status %d, TLS %s", resp.StatusCode, tls.VersionName(resp.TLS.Version))
|
||||
}
|
||||
curves := <-hellos
|
||||
if got, want := slices.Contains(curves, tls.X25519MLKEM768), debug == "tlsmlkem=1"; got != want {
|
||||
t.Errorf("ML-KEM offered = %v, want %v; curves %v", got, want, curves)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerTLSKeyExchangeDefaults(t *testing.T) {
|
||||
for _, debug := range []string{"tlsmlkem=0", "tlsmlkem=1"} {
|
||||
t.Run(debug, func(t *testing.T) {
|
||||
t.Setenv("GODEBUG", debug)
|
||||
// Reuse httptest's certificate with the actual Server TLS constructor.
|
||||
seed := httptest.NewTLSServer(http.NotFoundHandler())
|
||||
cert := seed.TLS.Certificates[0]
|
||||
seed.Close()
|
||||
server := httptest.NewUnstartedServer(http.NotFoundHandler())
|
||||
server.TLS = newTLSConfig(func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return &cert, nil })
|
||||
server.StartTLS()
|
||||
defer server.Close()
|
||||
roots := x509.NewCertPool()
|
||||
leaf, err := x509.ParseCertificate(cert.Certificate[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
roots.AddCert(leaf)
|
||||
for _, curve := range []tls.CurveID{tls.X25519MLKEM768, tls.CurveP256} {
|
||||
conn, err := tls.Dial("tcp", server.Listener.Addr().String(), &tls.Config{
|
||||
RootCAs: roots, MinVersion: tls.VersionTLS13, CurvePreferences: []tls.CurveID{curve},
|
||||
})
|
||||
wantSuccess := curve == tls.CurveP256 || debug == "tlsmlkem=1"
|
||||
if (err == nil) != wantSuccess {
|
||||
t.Errorf("curve %v: error %v, want success %v", curve, err, wantSuccess)
|
||||
}
|
||||
if conn != nil {
|
||||
_ = conn.Close()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user