mirror of
https://github.com/pgsty/minio.git
synced 2026-10-17 15:16:00 +03:00
Compare commits
97 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 86fb2b833f | |||
| fbcb511978 | |||
| 87c1c6b3dc | |||
| 1e0e5ac53c | |||
| 7e9454456e | |||
| 4ed86dfe62 | |||
| 4e55fc7db4 | |||
| 2616d43fbc | |||
| 2a4db6a4f9 | |||
| e90cb2067d | |||
| 00a375c4e2 | |||
| ab4c97e607 | |||
| 721dbfd1b7 | |||
| 2926474688 | |||
| a7370aa9ac | |||
| a592d69f45 | |||
| bfc7906064 | |||
| eeff112491 | |||
| a70156e45b | |||
| 99245e3ef8 | |||
| 7a3ca0f821 | |||
| d94b3b3c92 | |||
| 86c5a92eef | |||
| cdc3b8cb10 | |||
| 0d3ba5d4cb | |||
| 71a1631f13 | |||
| 2d57b975d1 | |||
| 14042b5aab | |||
| 38a5ab549c | |||
| fa565ec9fc | |||
| 3196480da9 | |||
| cb85461c42 | |||
| ba6e8c8d69 | |||
| a54d0c4354 | |||
| 00d4a206f7 | |||
| f480086b6b | |||
| 968c30238c | |||
| bcb5fc9566 | |||
| 394d9db212 | |||
| 8c784f6133 | |||
| 258b167b1c | |||
| 69d5e92791 | |||
| efd5698597 | |||
| 2fde3cf535 | |||
| 2a4d51406b | |||
| 0596685ae7 | |||
| 358ab38fb0 | |||
| 254b19ac07 | |||
| eb4f5e5b31 | |||
| 8d06424b12 | |||
| fced863036 | |||
| 5af0865aab | |||
| 83821f0f1f | |||
| bf053386a4 | |||
| e7963381ba | |||
| 37c0edc7ca | |||
| a2fe70424e | |||
| 027d43b4eb | |||
| f99ed829b5 | |||
| 956a1a8e33 | |||
| 82f0a9828e | |||
| 584b5a3a8f | |||
| 39f49d548f | |||
| a168576adb | |||
| e85c4c8dfb | |||
| 0e3c43778e | |||
| 9101fe78db | |||
| 82948d6306 | |||
| dfb4b2a1d2 | |||
| 143f6970d8 | |||
| ea5dac5a99 | |||
| 3c26a8b0b5 | |||
| 2fabd436c0 | |||
| 07c68d6054 | |||
| 15090dc4fd | |||
| e791640dac | |||
| 9b4ae82a29 | |||
| 4620be394b | |||
| 5e7d603083 | |||
| fb7c406ddc | |||
| 416826f61f | |||
| a2e2f3ee82 | |||
| 70c7ec4a9f | |||
| 2b722b7e92 | |||
| 4cbb074ccd | |||
| 40220bd836 | |||
| df0dfa0a34 | |||
| 80684fed59 | |||
| 055030ea53 | |||
| aea3882c95 | |||
| 0c61128d23 | |||
| 680eac66e4 | |||
| 9f3037e941 | |||
| 5f00f6762f | |||
| af2b1794d3 | |||
| d371f77dcc | |||
| 4fcdf37ce6 |
@@ -90,6 +90,29 @@ jobs:
|
||||
- name: Run S3 Select tests under race detector
|
||||
run: go test -race ./internal/s3select/... -count=1
|
||||
|
||||
- name: Run conditional PUT tests under race detector
|
||||
run: go test -race ./cmd -run '^Test(PoolsConditionalPut|SinglePoolConditionalPutHTTP)' -count=1 -timeout=5m
|
||||
|
||||
- name: Run multipart listing and cancellation tests under race detector
|
||||
run: go test -race ./cmd -run '^Test(MultipartListing|MultipartAbort|PaginateMultipartUploads|ListMultipartUploads)' -count=1 -timeout=5m
|
||||
|
||||
- name: Run listing metadata recovery, pagination and cancellation tests under race detector
|
||||
run: go test -race ./cmd -run '^Test(ListFallback|ListingCache|ListingMerge|ListingRecovery|ListingPending|ListingEntry|ListingSecondLock|StreamMetadataParts|LocalLockerListing|DistributedListing|MergeXLV2LatestNullCrossParity|CountXLV2MixedECGroups|ListCrossParity)' -count=1 -timeout=5m
|
||||
|
||||
- name: Run lock cancellation and late-grant cleanup tests under race detector
|
||||
run: go test -race ./internal/dsync ./internal/lsync -run '^Test(DRWMutex|LRWMutex)' -count=1 -timeout=5m
|
||||
|
||||
- name: Run CPU metrics tests under race detector
|
||||
run: go test -race ./cmd -run '^TestLoadCPUMetrics' -count=1 -timeout=5m
|
||||
|
||||
- name: Run tag replication tests under race detector
|
||||
run: go test -race ./cmd -run '^TestAPITagging' -count=1 -timeout=5m
|
||||
|
||||
- name: Run Object Lock authorization and overwrite tests under race detector
|
||||
env:
|
||||
MINIO_API_REQUESTS_MAX: "10000"
|
||||
run: go test -race -tags kqueue,dev ./cmd -run '^Test(APIPutObjectRetention|APIVersionAddressedWrite|PutObjectRetention|DirectoryObjectLock|ObjectLockOverwrite|APICopyObjectPreservesVersionAfterSuspension|MultipartNullObjectLock|PoolsObjectLockOverwrite|MultipartVersioningTransitions|PlanReview|RecoveredMultipartUpload)' -count=1 -timeout=5m
|
||||
|
||||
crosscompile:
|
||||
name: Cross Compile
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -70,6 +70,12 @@ jobs:
|
||||
buildscripts/verify-rebrand.sh
|
||||
dockerscripts/docker-entrypoint_test.sh
|
||||
|
||||
# Keep this gate aligned with the Object Lock race regression in go.yml.
|
||||
- name: Gate release on Object Lock authorization and overwrite tests under race detector
|
||||
env:
|
||||
MINIO_API_REQUESTS_MAX: "10000"
|
||||
run: go test -race -tags kqueue,dev ./cmd -run '^Test(APIPutObjectRetention|APIVersionAddressedWrite|PutObjectRetention|DirectoryObjectLock|ObjectLockOverwrite|APICopyObjectPreservesVersionAfterSuspension|MultipartNullObjectLock|PoolsObjectLockOverwrite|MultipartVersioningTransitions|PlanReview|RecoveredMultipartUpload)' -count=1 -timeout=5m
|
||||
|
||||
- name: Compute release variables
|
||||
env:
|
||||
# Passed through the environment, never interpolated into the script
|
||||
@@ -132,9 +138,9 @@ jobs:
|
||||
cosign-release: v3.1.2
|
||||
|
||||
- name: Build Draft release with GoReleaser
|
||||
uses: goreleaser/goreleaser-action@v7
|
||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
|
||||
with:
|
||||
version: "~> v2"
|
||||
version: v2.18.3
|
||||
args: release --clean --skip=validate --config .github/goreleaser.yml
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -150,7 +156,7 @@ jobs:
|
||||
- name: Install nFPM
|
||||
run: |
|
||||
set -euo pipefail
|
||||
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.47.0
|
||||
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.48.0
|
||||
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Build nFPM packages
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
name: Repository Cards
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 00:00 UTC = 08:00 Asia/Shanghai. GitHub may queue scheduled runs.
|
||||
- cron: "0 0 * * *"
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- .github/workflows/repository-cards.yml
|
||||
- .github/silo.svg
|
||||
- buildscripts/repository-cards/**
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- .github/workflows/repository-cards.yml
|
||||
- .github/silo.svg
|
||||
- buildscripts/repository-cards/**
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: repository-cards-${{ github.event.pull_request.number || 'publish' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Validate repository cards
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
|
||||
with:
|
||||
python-version: "3.13"
|
||||
- run: python -m pip install -r buildscripts/repository-cards/requirements.txt
|
||||
- run: python -m unittest discover -s buildscripts/repository-cards -p 'test_*.py' -v
|
||||
|
||||
publish:
|
||||
name: Update README images
|
||||
needs: check
|
||||
if: github.repository == 'pgsty/silo' && github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: write
|
||||
issues: read
|
||||
pull-requests: read
|
||||
env:
|
||||
OUTPUT_BRANCH: codex/repository-cards
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
|
||||
with:
|
||||
python-version: "3.13"
|
||||
- run: python -m pip install -r buildscripts/repository-cards/requirements.txt
|
||||
|
||||
- name: Load the generated-assets branch
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
artifacts="$RUNNER_TEMP/repository-cards"
|
||||
if git ls-remote --exit-code --heads origin "$OUTPUT_BRANCH"; then
|
||||
git fetch --depth=1 origin "$OUTPUT_BRANCH"
|
||||
git worktree add --detach "$artifacts" FETCH_HEAD
|
||||
else
|
||||
status=$?
|
||||
# Exit 2 means no matching ref; transport/auth failures must stop.
|
||||
if [ "$status" -ne 2 ]; then exit "$status"; fi
|
||||
git worktree add --detach "$artifacts" HEAD
|
||||
git -C "$artifacts" checkout --orphan "$OUTPUT_BRANCH"
|
||||
git -C "$artifacts" rm -rf .
|
||||
fi
|
||||
|
||||
- name: Refresh contributor and star cards
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: python buildscripts/repository-cards/update.py --output "$RUNNER_TEMP/repository-cards"
|
||||
|
||||
- name: Publish changed assets
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cd "$RUNNER_TEMP/repository-cards"
|
||||
git add README.md history.json curated.json contributors.json \
|
||||
contributors-light.svg contributors-dark.svg \
|
||||
star-history-light.svg star-history-dark.svg
|
||||
if git diff --cached --quiet; then
|
||||
echo "Repository cards are already current."
|
||||
exit 0
|
||||
fi
|
||||
git config user.name 'github-actions[bot]'
|
||||
git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
|
||||
git commit -s -m "chore: update repository cards $(date -u +%F)"
|
||||
# A normal push preserves history and refuses concurrent overwrites.
|
||||
git push origin "HEAD:refs/heads/$OUTPUT_BRANCH"
|
||||
@@ -4,6 +4,8 @@ on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
paths:
|
||||
- "go.mod"
|
||||
- "go.sum"
|
||||
- ".github/goreleaser.yml"
|
||||
- ".github/nfpm.yml"
|
||||
- "Dockerfile.goreleaser"
|
||||
@@ -93,9 +95,9 @@ jobs:
|
||||
echo "LDFLAGS: ${LDFLAGS}"
|
||||
|
||||
- name: GoReleaser config check
|
||||
uses: goreleaser/goreleaser-action@v7
|
||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
|
||||
with:
|
||||
version: "~> v2"
|
||||
version: v2.18.3
|
||||
args: check --config .github/goreleaser.yml
|
||||
|
||||
- name: Validate Helm chart and legacy upgrade identity
|
||||
@@ -107,9 +109,9 @@ jobs:
|
||||
syft-version: v1.50.0
|
||||
|
||||
- name: Build snapshot artifacts
|
||||
uses: goreleaser/goreleaser-action@v7
|
||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
|
||||
with:
|
||||
version: "~> v2"
|
||||
version: v2.18.3
|
||||
# A pull-request snapshot has no trusted release identity. Exercise
|
||||
# the SBOM/checksum pipeline here, and reserve keyless signing for
|
||||
# the tag-triggered release workflow with GitHub OIDC.
|
||||
@@ -141,7 +143,7 @@ jobs:
|
||||
- name: Install package validation tools
|
||||
run: |
|
||||
set -euo pipefail
|
||||
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.47.0
|
||||
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.48.0
|
||||
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
|
||||
sudo apt-get update
|
||||
sudo apt-get install --yes rpm binutils
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
|
||||
- name: Install govulncheck
|
||||
run: |
|
||||
go install golang.org/x/vuln/cmd/govulncheck@v1.8.0
|
||||
go install golang.org/x/vuln/cmd/govulncheck@v1.9.0
|
||||
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Run govulncheck
|
||||
|
||||
+2
-2
@@ -62,10 +62,10 @@ dist/
|
||||
|
||||
.claude/
|
||||
.codex/
|
||||
AGENTS.md
|
||||
CLAUDE.md
|
||||
_bmad/
|
||||
_bmad-output/
|
||||
# Investigation and editorial working files belong outside this repository.
|
||||
docs/investigations/
|
||||
docs/security/
|
||||
docs/rebranding.md
|
||||
.release-sign/
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
# SILO Repository Guide
|
||||
|
||||
## Project map
|
||||
|
||||
- Server: `pgsty/silo` (this checkout, default branch `main`)
|
||||
- Console: `pgsty/silo-console` (usual sibling checkout `../silo-console`)
|
||||
- Client: `pgsty/mc`, shipped as `mcli` (usual sibling checkout `../mc`)
|
||||
- Shared packages: `pgsty/silo-pkg` (usual sibling checkout `../silo-pkg`)
|
||||
- Documentation: `pgsty/silo.pgsty.com` (usual sibling checkout `../silo.pgsty.com`),
|
||||
published at <https://silo.pgsty.com/>
|
||||
|
||||
The server executable, package, systemd service, and container image use `silo`;
|
||||
the public Docker image is `docker.io/pgsty/silo`. The server module remains
|
||||
`github.com/minio/minio`. Consult the current `go.mod` and release configuration
|
||||
for selected component versions and compatibility replacements.
|
||||
|
||||
## Supported stack and compatibility policy
|
||||
|
||||
The maintained, release-gating product graph is the coordinated PGSTY stack:
|
||||
`silo` + `silo-console` + `mc` + `silo-pkg`.
|
||||
|
||||
Keep compatibility with upstream MinIO/MC on a best-effort basis. Preserve
|
||||
inexpensive wire, configuration, CLI, migration, and import compatibility when
|
||||
it helps users, and document known differences. Do not infer from source
|
||||
lineage, MinIO-compatible protocols, retained `MINIO_*`/`MC_*` names, or an
|
||||
inherited upstream test that unmodified upstream MinIO is a supported release
|
||||
target.
|
||||
|
||||
Upstream-only compatibility checks may remain as advisory evidence, but they
|
||||
must not force a downgrade of a maintained SILO component, a fork-only API
|
||||
workaround, or a release block. Make upstream compatibility a hard gate only
|
||||
when the user explicitly requests that scope.
|
||||
|
||||
## Dependency selection
|
||||
|
||||
- When PGSTY maintains a component, import and require it directly under its own
|
||||
module path. In particular, prefer `github.com/pgsty/silo-pkg/v3` over
|
||||
`github.com/minio/pkg/v3` in maintained SILO source.
|
||||
- Do not use an upstream package merely to make unmodified upstream MinIO or MC
|
||||
compile. Unavoidable transitive upstream modules should be documented and
|
||||
kept separate from the maintained product dependency.
|
||||
- `github.com/minio/minio-go/v7` is the explicit exception: use the verified
|
||||
upstream module/commit while it contains the required fixes; do not recreate
|
||||
a SILO fork without a concrete functional divergence.
|
||||
- Check the current `go.mod` before changing versions. Coordinate breaking
|
||||
import-path changes across package, client, Console, and server releases.
|
||||
|
||||
## Documentation and delivery
|
||||
|
||||
The companion site owns product, operations, migration, security-advisory,
|
||||
design, and release documentation. Maintain English and Chinese content in
|
||||
that repository and link to its canonical URLs from this one. The site's
|
||||
`content/docs/` is the documentation entry point; detailed pages live under
|
||||
`content/operations/`, `administration/`, `reference/`, `compatibility/`,
|
||||
`about/`, and `blog/`, following the existing structure.
|
||||
|
||||
Keep repository entry points and contributor instructions here. Retained
|
||||
upstream documents, examples, and test fixtures under `docs/` may be updated
|
||||
when a code or tooling change requires it; do not build a second documentation
|
||||
site in this tree.
|
||||
|
||||
Investigation plans, prompts, AI session transcripts, execution logs, temporary
|
||||
reports, and private security material belong outside the checkout, for example
|
||||
in a task-specific directory under `~/tmp/`. Do not recreate
|
||||
`docs/investigations/`, `docs/security/`, or `docs/rebranding.md`, or move their
|
||||
working material to another repository directory. Extract reusable, verified
|
||||
knowledge into the companion site; keep private evidence outside public Git.
|
||||
|
||||
Compatibility pages may continue to describe similarities with upstream, but
|
||||
must label that compatibility as best effort. The supported and tested server
|
||||
for Console and mcli administration is `pgsty/silo`.
|
||||
|
||||
Before removing or moving documentation, check repository links and scripts,
|
||||
the companion site's links and anchors, and references from this guide. Use
|
||||
fixed commit URLs for historical evidence that should survive deletion from
|
||||
the current tree. Run the site's `make check` for site changes and this
|
||||
repository's `make rebrand-guard` for documentation cleanup that changes the
|
||||
identifier inventory. When new public URLs are involved, publish the site
|
||||
before publishing source changes that depend on those URLs.
|
||||
|
||||
Keep changes in the repository that owns the affected surface. Treat every
|
||||
repository commit, tag, release, image, documentation update, and deployment as
|
||||
a separate deliverable. Follow `CONTRIBUTING.md`, including DCO sign-off
|
||||
(`git commit -s`). `CLAUDE.md` imports this guide so both agents use one policy.
|
||||
+171
-24
@@ -2,16 +2,67 @@
|
||||
|
||||
## Unreleased
|
||||
|
||||
The entries below describe source changes on main since the latest published Server.
|
||||
**The latest published Server remains 20260903.** These changes are not in its
|
||||
binaries, packages or images. See the [component matrix](https://silo.pgsty.com/compatibility/versions/)
|
||||
and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-09-03T13-18-01Z...main).
|
||||
Preparation target: `RELEASE.2026-10-18T00-00-00Z` (package version
|
||||
`20261018000000.0.0`). **This is a tag and Draft preparation; it is not a public
|
||||
Server release.** The latest published Server remains 20260916. Server 20261015
|
||||
also remains a Draft. Existing binaries, packages and images do not change.
|
||||
See the [component matrix](https://silo.pgsty.com/compatibility/versions/) and
|
||||
[changes since the public release](https://github.com/pgsty/silo/compare/RELEASE.2026-09-16T00-00-00Z...main).
|
||||
|
||||
### October 18 dependency update
|
||||
|
||||
- Select silo-pkg v3.14.2, upstream minio-go v7.4.1, published mcli 20261018
|
||||
and Console v2.4.2 source (`ef2dd2c41d21`). The SDK includes cancellation cleanup, UTC retention
|
||||
timestamps and preserved STS duration; parallel GET/Compose remain opt-in.
|
||||
- Build with Go 1.27.2 and refresh Go x/* modules. Apply go-jose v4.1.6,
|
||||
LDAP v3.4.15, compress v1.20.3, msgp v1.6.5, etcd v3.7.2, fwd v1.2.1 and
|
||||
cpuid v2.4.1 bug/security fixes. OIDC JWK RSA keys must not exceed 4096 bits.
|
||||
- Update the bundled mcli archive hashes, installation helper and Helm client
|
||||
image to the published 20261018 release. Server Helm image defaults retain the
|
||||
public 20260916 version until a Server image is published.
|
||||
- Refresh pinned UBI 9 images and use GoReleaser v2.18.3, nFPM v2.48.0,
|
||||
golangci-lint v2.15.0 and govulncheck v1.9.0. Keep the NetBSD go-systemd
|
||||
v22.6.0 compatibility replacement.
|
||||
|
||||
The sections below retain the cumulative maintenance history, including changes
|
||||
already shipped in 20260916; the source comparison above identifies this
|
||||
candidate's exact delta.
|
||||
|
||||
### Authorization and security
|
||||
|
||||
- Synchronize CPU metrics reads with resource-metrics updates (#210), preventing
|
||||
concurrent map access from terminating the server during Prometheus scraping.
|
||||
Metric names, values and authentication requirements are unchanged.
|
||||
- Restrict embedded Console's anonymous sharing proxy to object-content GETs
|
||||
at the configured S3 origin, and reject every redirect. Internal metrics,
|
||||
system paths and non-download S3 operations cannot be reached through it.
|
||||
Normal public, presigned and versioned downloads remain available without a
|
||||
new setting; a full sharing-disable switch is not introduced. See
|
||||
[Console #56](https://github.com/pgsty/silo-console/pull/56) and the
|
||||
[design record](https://github.com/pgsty/silo-console/issues/52).
|
||||
Thanks to Jiri Pejchal (@jiri-pejchal) for the report.
|
||||
- Persist IAM deletion revisions and parent revocation boundaries so stale site
|
||||
events cannot restore deleted identities, policies or their older grants
|
||||
(#191, #192). Peer deletion notifications reload committed storage; deliberate
|
||||
recreation requires a newer revision, and credentials issued before the
|
||||
parent's revocation remain invalid.
|
||||
**Coordinated upgrade required:** upgrade every participating node and site.
|
||||
Mixed old/new nodes sharing an IAM backend and rolling downgrade are
|
||||
unsupported. Back up complete IAM storage and encryption material; an admin
|
||||
export of live records omits deletion history. Reissue credentials for
|
||||
recreated parents and explicitly reconcile pre-upgrade revocations whose
|
||||
history is already lost. Restoring an older backup can lose later revocations;
|
||||
keep affected sites isolated until reconciliation/rekeying is complete. See
|
||||
[the operator runbook](https://silo.pgsty.com/operations/replication/iam-upgrade/).
|
||||
- Enforce an absolute HTTP/1 request-header deadline through the connection
|
||||
wrapper (#196). Repeated small reads no longer extend that deadline, and
|
||||
`--read-header-timeout` / `MINIO_READ_HEADER_TIMEOUT` now reaches the HTTP
|
||||
server. HTTP/1 request bodies retain the rolling idle timeout; this does not
|
||||
impose a total upload/download duration. A shorter setting also constrains
|
||||
TLS handshake reads. The wrapper's strict header mode is not applied to HTTP/2.
|
||||
- Reject unsigned `x-amz-*` request headers that could turn a signed PUT into a
|
||||
copy of another object accessible to the signer (SN-2026-011). The latest
|
||||
public Server is affected; the fix is on main. See [the advisory ledger](docs/security/advisories.md).
|
||||
public Server is affected; the fix is on main. See [the advisory ledger](https://silo.pgsty.com/about/security-advisories/).
|
||||
- Align signed request fields with policy conditions and enforce header-only
|
||||
presigned payload checksums. See [the signed-header review](https://silo.pgsty.com/blog/design/signed-header-coverage/).
|
||||
- **Breaking policy semantics:** separate self-service `admin:ChangeMyPassword`
|
||||
@@ -22,17 +73,108 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0
|
||||
|
||||
### Object storage and replication
|
||||
|
||||
- Make durable `ListMultipartUploads` scanning the only listing path (#79):
|
||||
apply prefix, delimiter, markers, ordering and the 1,000-entry page limit
|
||||
globally across pools and sets, independently of node-local caches. Omit
|
||||
readable historical uploads only when both bucket/key identity fields are
|
||||
absent; malformed identities, uncertain coverage and resource limits remain
|
||||
errors. Old uploads still support part upload, completion and cancellation
|
||||
with a known bucket/key/upload ID. No backfill or new storage layout is added.
|
||||
`MINIO_API_MULTIPART_LISTING` is retired and ignored with a startup warning;
|
||||
historical shared `api multipart_listing` keys remain tolerated. The read-only
|
||||
admin `multipart-preflight` reports `mode=persistent`; `ready` diagnoses the
|
||||
absence of old records and no longer gates listing. Each page still scans
|
||||
durable state within the existing 100,000-directory-entry, 30-second and
|
||||
per-process concurrency budgets. See the
|
||||
[design and limits](https://silo.pgsty.com/blog/design/list-multipart-uploads/#implementation).
|
||||
Thanks to mr javad seydi (@mrjavadseydi) for the original metadata-and-scan
|
||||
implementation (#198).
|
||||
- Keep the default read-quorum, best-effort multipart cancellation and
|
||||
first-matching-pool behavior when retiring the listing switch. The optional
|
||||
strict cancellation branch is removed. Wrong-target protection, upload-ID
|
||||
path validation and peer notification with the request context remain.
|
||||
HTTP 204 for an absent upload remains idempotent and does not prove physical
|
||||
cleanup. **Known boundary:** delayed creation writes can still restore a
|
||||
readable upload after cancellation; this change adds no creation fence.
|
||||
- Preserve object tags during multi-pool metadata reconciliation by reading the
|
||||
resolved tag field together with its revision (#189). Previously, reconciliation
|
||||
could replace existing tags with an empty value.
|
||||
- Preserve the tag revision on SSE-KMS metadata replication (#193), and advance
|
||||
tag revisions monotonically on local PUT/DELETE tagging (#196). Empty tags
|
||||
participate in reconciliation as an ordered deletion, preventing older
|
||||
events from restoring removed tags. SSE-C key rotation also retains the tag
|
||||
revision. Malformed historical revisions can fail and retry; their missing
|
||||
history is not reconstructed by the upgrade.
|
||||
- Complete delete-marker version purges and preserve their identity and retry
|
||||
state through MRF recovery (#196). Recovery accepts a 405 marker response only
|
||||
when its version, bucket, object name and modification time match the task.
|
||||
Purge audit status is normalized from `COMPLETE` to `COMPLETED`.
|
||||
Thanks to Julien Laurenceau (@julienlau) for the investigation and proposed
|
||||
fix in #184 that helped shape this follow-up.
|
||||
- Restore only the six replication-specific metadata fields after ordinary
|
||||
request metadata extraction (#194). This prevents transport-only `aws-chunked`
|
||||
from being stored as Content-Encoding while preserving the signed-header
|
||||
protections. Trusted Snowball entries no longer inherit the outer archive's
|
||||
ordinary metadata. Thanks to Mikhail Khadarenka (@chodorenko) for the fix in #187.
|
||||
**Existing data:** these repairs prevent new errors; they do not scan or rewrite
|
||||
historical object metadata, recover lost tags or prove that old purge work has
|
||||
converged. Follow the [read-only audit procedure](https://silo.pgsty.com/operations/replication/replica-metadata-audit/)
|
||||
before planning any repair of stored state.
|
||||
|
||||
- Make exact-version delete-marker purges converge in replicated buckets
|
||||
(`eb4f5e5b3`, `254b19ac0`, `358ab38fb`). A purge no longer creates the
|
||||
marker on drives that lacked it; a retried purge of a missing version is
|
||||
acknowledged only when a write-quorum majority of drives report it absent;
|
||||
purge results merge removed and reliably absent replies; healing a marker
|
||||
preserves its stored replication and purge metadata; and a queued marker
|
||||
creation is re-checked against the source under the replication lock before
|
||||
it is sent, so a purge that already reached the targets is not undone by a
|
||||
stale task from another frontend, a GET/LIST heal, the scanner or MRF.
|
||||
Purging a data version whose earlier purge is still pending reports it as a
|
||||
data version. **Known limitations:** creations already in flight or replayed
|
||||
from another site, and minority marker copies left by a crash after a
|
||||
majority-acknowledged purge, are tracked in #217. See
|
||||
[the replication reliability record](https://silo.pgsty.com/blog/design/replication-reliability/).
|
||||
- Keep a null object version that has listing quorum when a newer minority of
|
||||
drives sorts first (`8d06424b1`). The resolver recounts per header only when
|
||||
the original selection lacks quorum, every non-empty drive stream holds
|
||||
exactly one ordinary null version and all share the same erasure layout;
|
||||
mixed histories keep their previous behavior. **Known limitation:** a
|
||||
successful ListObjects can still omit readable keys during rolling restarts
|
||||
with concurrent overwrites (#218). Do not run destination-deleting sync tools
|
||||
against a listing taken during a rolling restart; list again once the
|
||||
cluster is stable.
|
||||
- Carry object tags through rebalance and decommission for ordinary and
|
||||
multipart writes (`fced86303`). Both migration entry points restore the tags
|
||||
and their revision fields when rewriting the object in the destination pool.
|
||||
Tags dropped by earlier migrations are not recovered; audit tag-dependent
|
||||
lifecycle and policy rules for pools migrated with an older build.
|
||||
|
||||
- Evaluate conditional multipart completion against the logical current object
|
||||
across all pools while holding the existing object lock. A stale `If-Match`
|
||||
can no longer replace newer data in another pool, and the current ETag is no
|
||||
longer rejected because the upload resides next to an older copy. Conditions
|
||||
are evaluated once; a current delete marker counts as an absent object.
|
||||
**Availability change:** if metadata cannot be read from any pool, conditional
|
||||
**Availability change:** if any pool's metadata cannot be read, conditional
|
||||
completion fails even when another pool can still serve GET/HEAD. This also
|
||||
applies when the unreadable pool may not hold the object: absence cannot be
|
||||
verified. Retry after the pool recovers. Unconditional completion and the
|
||||
single-pool path retain their existing behavior.
|
||||
|
||||
- Evaluate ordinary multi-pool conditional PUT against the logical current
|
||||
object across all pools, including draining pools, under the existing object
|
||||
lock (#207). A stale destination copy no longer accepts a stale ETag or rejects
|
||||
the current one; a current delete marker is treated as absence.
|
||||
**Availability change:** if any pool's object metadata cannot be verified,
|
||||
the condition fails even when GET can use another pool; read-quorum failures
|
||||
return 503. Restore readability or heal before retrying. Unconditional PUT,
|
||||
single-pool conditions and internal replication retain their existing behavior.
|
||||
A public condition with a destination `versionId` compares the current object
|
||||
while preserving the requested write version. This change does not retire
|
||||
stale copies in other pools, undo historical accepted overwrites or provide
|
||||
a new global clock-ordering guarantee. The multipart-completion repair in #190
|
||||
neither introduced nor repaired this separate PUT defect.
|
||||
|
||||
- Reconcile ordinary single-object version DELETE across all pools, including
|
||||
null versions, delete markers and unqualified directory-marker DELETE. This
|
||||
applies the deletion to every resolved pool copy under existing quorum
|
||||
@@ -50,7 +192,7 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0
|
||||
its tracker, mover, scanner hooks, configuration, XML actions and metrics.
|
||||
Accept and ignore retired configuration/XML and preserve ordinary statistics
|
||||
when reading v9 caches. See [migration notes](docs/bucket/lifecycle/access-tiering-removal.md).
|
||||
The [decision record](docs/investigations/access-tiering-revert.md) preserves
|
||||
The [decision record](https://silo.pgsty.com/compatibility/access-tiering-removal/) preserves
|
||||
the feature's introduction, subsequent fixes, rollback scope and review history.
|
||||
- Preserve the independent multi-pool write, metadata, healing and conditional
|
||||
deletion fixes from PR #178, including shared remote-tier reference protection.
|
||||
@@ -61,10 +203,10 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0
|
||||
- Repair federated CopyObject checksums, destination timestamps, reserved
|
||||
metadata, encrypted-object forwarding, legal hold and KMS context.
|
||||
- Make resync counters, target selection, cancellation and worker lifetimes
|
||||
reflect actual work; complete delete-marker purges and report bounded MRF drops.
|
||||
reflect actual work, and report bounded MRF drops.
|
||||
- Converge bucket metadata with deterministic source state, deletion tombstones,
|
||||
creation time recovery and diagnostics. The mixed-version export gate requires
|
||||
coordinated upgrades before tombstones are exported. See [the #77 record](docs/investigations/issue-77-current.md).
|
||||
coordinated upgrades before tombstones are exported. See [the #77 record](https://silo.pgsty.com/blog/design/bucket-metadata-convergence/).
|
||||
- Include per-bucket CORS in metadata export/import, close metadata publication
|
||||
and logger races, and report effective bucket quotas in metrics.
|
||||
|
||||
@@ -72,23 +214,28 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0
|
||||
|
||||
- Restore embedded Console login over loopback TLS, trusted-proxy handling and
|
||||
all four WebSocket connection limits. Preserve Go TLS defaults across transports.
|
||||
- Directly require `github.com/pgsty/silo-pkg/v3` v3.14.0; select Console
|
||||
`v0.0.0-20260913015128-417559bb2c97` and MC
|
||||
`v0.0.0-20260913012246-4f609a4da3bb` with explicit PGSTY replacements.
|
||||
- Pin upstream minio-go `v7.3.1-0.20260910142817-60bd07042d49`; refresh Go x/*
|
||||
modules and security fixes including bounded AMQP frame handling. Keep Go
|
||||
1.27.1 and go-systemd v22.6.0's NetBSD compatibility replacement.
|
||||
- Directly require `github.com/pgsty/silo-pkg/v3` v3.14.1; select released Console
|
||||
v2.4.1 (`v0.0.0-20260916075814-1360e26d976d`) and mcli 20260916
|
||||
(`v0.0.0-20260916070421-e952aa78f10a`) with explicit PGSTY replacements.
|
||||
The embedded frontend identifies itself as Console v2.4.1.
|
||||
- Pin upstream minio-go `v7.3.1-0.20260915093545-32e1f32cb176` to handle
|
||||
CopyObject errors embedded in HTTP 200 responses. Update JWX to v3.3.0 for
|
||||
JSON field-name escaping, strfmt to v0.27.2 for Go 1.27 hostname validation,
|
||||
and LZ4 to v4.1.30 for frame-reader, partial-read and concurrency fixes.
|
||||
Retain the earlier Go x/* and bounded AMQP frame updates, Go 1.27.1, and
|
||||
go-systemd v22.6.0's NetBSD compatibility replacement.
|
||||
- Refresh container base digests and build static curl 8.22.0 from verified
|
||||
source for both Linux architectures. Pin the actual mcli 20260913 archives and
|
||||
hashes. Helm's client image follows that release; its Server image still names
|
||||
the latest published Server 20260903.
|
||||
source for both Linux architectures. Pin the published mcli 20260916 archives
|
||||
and hashes in the container and update the client installer default.
|
||||
- Prepare Helm chart 7.0.3 with Server and client defaults for the September 16
|
||||
batch. Publish the chart only after the corresponding Server image exists.
|
||||
- Pin GoReleaser v2.18.1 and its action commit identically in snapshot and release
|
||||
workflows. Dependency-only PRs now run the Test Release Pipeline too.
|
||||
|
||||
The dependency update passed the final candidate's Go, vulnerability and Test
|
||||
Release workflows; native curl builds passed on both architectures. A local
|
||||
ARM64 image passed startup, health, S3 transfer and embedded Console checks.
|
||||
These checks do not publish a Server tag or production image and do not replace
|
||||
cluster upgrade/rollback acceptance for the next release. Dated investigations
|
||||
retain the exact source and runtime boundaries they tested.
|
||||
Validation of earlier source revisions does not establish acceptance of this
|
||||
candidate. Final source, package, image and multi-process checks are tracked
|
||||
separately in [#203](https://github.com/pgsty/silo/issues/203). No Server release
|
||||
or production rollout is implied by this preparation target.
|
||||
|
||||
## RELEASE.2026-09-03T13-18-01Z
|
||||
|
||||
|
||||
@@ -77,6 +77,16 @@ test evidence, compatibility notes, and documentation impact. Public product
|
||||
documentation is owned by the separate
|
||||
[`pgsty/silo.pgsty.com`](https://github.com/pgsty/silo.pgsty.com) repository.
|
||||
|
||||
## Contributor recognition
|
||||
|
||||
Every human issue or pull-request author is recognized in [CONTRIBUTORS.md](CONTRIBUTORS.md),
|
||||
including open issues, draft PRs, and PRs closed without merging. Merged fixes,
|
||||
adopted proposals, and reports that lead to fixes receive greater prominence;
|
||||
first participation guides the remaining order. Work incorporated through a
|
||||
later PR retains credit without changing the original PR's recorded status.
|
||||
Security disclosures are credited with the reporter's agreement. DCO sign-off
|
||||
applies to code commits, not to opening an issue.
|
||||
|
||||
## Licensing of Contributions
|
||||
|
||||
Code contributions to PGSTY SILO (`pgsty/silo`) are accepted under the
|
||||
|
||||
+116
-120
File diff suppressed because one or more lines are too long
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.27.1-alpine@sha256:cf6fca6641884b8433441b2b0652976f975e1d0fdd26d177eaaf8596087f3125 AS curl-build
|
||||
FROM golang:1.27.2-alpine@sha256:f92b6ef800e499660581efdabdf25d9d817a9d124eaf900924f0504e7e27e12d AS curl-build
|
||||
ARG TARGETARCH
|
||||
COPY dockerscripts/build-static-curl.sh /build/build-static-curl
|
||||
RUN /bin/sh /build/build-static-curl
|
||||
@@ -9,7 +9,7 @@ COPY --from=curl-build /go/bin/curl /curl
|
||||
COPY --from=curl-build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
ENTRYPOINT ["/curl"]
|
||||
|
||||
FROM golang:1.27.1-alpine@sha256:cf6fca6641884b8433441b2b0652976f975e1d0fdd26d177eaaf8596087f3125 AS build
|
||||
FROM golang:1.27.2-alpine@sha256:f92b6ef800e499660581efdabdf25d9d817a9d124eaf900924f0504e7e27e12d AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
|
||||
@@ -17,9 +17,9 @@ ENV GOPATH=/go
|
||||
ENV CGO_ENABLED=0
|
||||
|
||||
ARG MC_REPO=pgsty/mc
|
||||
ARG MC_VERSION=RELEASE.2026-09-13T00-00-00Z
|
||||
ARG MC_AMD64_SHA256=9d2a92de9c7b887d9b944fe9ddce68d23f1b6df3415092e737594e56593f5e2b
|
||||
ARG MC_ARM64_SHA256=3d82e9ea6c601c4cb44fe5dd5f2ad1b7d7d64369378110f9ada9c524688a452a
|
||||
ARG MC_VERSION=RELEASE.2026-10-18T00-00-00Z
|
||||
ARG MC_AMD64_SHA256=b5392a642dbc9685b517ee901f032d1d5e88f41704436f0f08dde6060b577db3
|
||||
ARG MC_ARM64_SHA256=17f3eb694ed34e155b331b1215832140749656d8d4b60e2e3a4d0573b8f320d7
|
||||
|
||||
RUN apk add -U --no-cache \
|
||||
ca-certificates \
|
||||
@@ -67,14 +67,14 @@ RUN apk add -U --no-cache \
|
||||
chmod +x /go/bin/mcli && \
|
||||
ln -sf mcli /go/bin/mc
|
||||
|
||||
FROM registry.access.redhat.com/ubi9/ubi:latest@sha256:206b65b8ee0f04b992818c9a51b29081b14974630d4850bc358097d0c44ea156 AS certs
|
||||
FROM registry.access.redhat.com/ubi9/ubi:latest@sha256:16e9e43d99e181463513cc84a514859b1abb9460f1c4d9e208da4b4e51cd5cc3 AS certs
|
||||
RUN dnf -y install ca-certificates && \
|
||||
update-ca-trust && \
|
||||
cp /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /tmp/ca-certificates.crt && \
|
||||
dnf clean all && \
|
||||
rm -rf /var/cache/dnf
|
||||
|
||||
FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:f332c99eb8f798a8486821c91937f10ad64ee83d7e739303be2df051040918f6
|
||||
FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:5be877a2a280ae36eb9ab80ca4efb28bbf77c57abbba38137cdc37a03b8465e6
|
||||
|
||||
LABEL org.opencontainers.image.title="Silo" \
|
||||
org.opencontainers.image.description="S3-Interface Libre Object Storage" \
|
||||
|
||||
@@ -4,7 +4,7 @@ LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
|
||||
|
||||
GOOS ?= $(shell go env GOOS)
|
||||
GOARCH ?= $(shell go env GOARCH)
|
||||
GOLANGCI_VERSION ?= v2.13.1
|
||||
GOLANGCI_VERSION ?= v2.15.0
|
||||
|
||||
VERSION ?= $(shell git describe --tags)
|
||||
REPO ?= docker.io/pgsty
|
||||
|
||||
@@ -38,7 +38,7 @@
|
||||
## Current release and main branch
|
||||
|
||||
The latest published Server is [20260903](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z).
|
||||
As of 2026-09-13, the main branch has newer security, storage, Console and
|
||||
As of 2026-09-16, the main branch has newer security, storage, Console and
|
||||
shared-package changes that have not shipped in a Server release. See
|
||||
[CHANGELOG.md](CHANGELOG.md) and the [component version matrix](https://silo.pgsty.com/compatibility/versions/)
|
||||
for the exact release/source boundary, including SN-2026-011 and password-policy migration.
|
||||
@@ -93,13 +93,16 @@ Every release ships checksums, SPDX SBOMs, Sigstore-signed manifests, and GitHub
|
||||
|
||||
## Compatibility
|
||||
|
||||
The S3 API, `MINIO_*` variables, `minio_*` metrics, `x-minio-*` headers, `/minio/*` routes, the `github.com/minio/*` import paths, and the on-disk format (including `.minio.sys`) are preserved and held in place by a CI compatibility check. Only Silo-owned delivery surfaces change: the `silo` executable, package, service, Helm chart, and container image — no `minio` binary alias is installed.
|
||||
Silo preserves S3 and storage-format compatibility, including existing `MINIO_*` variables, `minio_*` metrics, `x-minio-*` headers, `/minio/*` routes, and `.minio.sys` data. CI guards selected compatibility identifiers; release notes document intentional security and behavior changes. Silo-owned delivery surfaces use the `silo` executable, package, service, Helm chart, and container image; no `minio` server binary alias is installed.
|
||||
|
||||
The supported release stack is `pgsty/silo` + `pgsty/silo-console` + `pgsty/mc` + `pgsty/silo-pkg`; compatibility with unmodified upstream MinIO/MC is best effort. The Server, Console, and client retain their historical module paths where needed, while maintained code imports `github.com/pgsty/silo-pkg/v3` directly. The SDK `github.com/minio/minio-go/v7` is an explicit upstream dependency. See the current [go.mod](go.mod) for versions and replacements.
|
||||
|
||||
Every divergence from upstream is listed in the code-verified [compatibility audit](https://silo.pgsty.com/compatibility/server/). Treat each release as a downstream upgrade: pin versions, read the [release notes](https://silo.pgsty.com/tags/silo/), and keep a rollback path.
|
||||
|
||||
### TLS and Go upgrades
|
||||
|
||||
TLS key exchange follows Go's defaults across the S3 listener, node links,
|
||||
The following TLS repair is on main and is not included in Server 20260903.
|
||||
With that repair, TLS key exchange follows Go's defaults across the S3 listener, node links,
|
||||
replication, identity providers, etcd, and external HTTP services. If an endpoint
|
||||
cannot accept ML-KEM, `GODEBUG=tlsmlkem=0` disables the default hybrid exchanges
|
||||
for the process; certificate verification remains enabled. This option does not
|
||||
@@ -115,7 +118,16 @@ values to restore Keychain trust. Explicit certificates in the configured `CAs`
|
||||
directory remain additive to the selected root pool.
|
||||
Go 1.27 binaries require macOS 13 or later. See the
|
||||
[Go release notes](https://go.dev/doc/go1.27) and the
|
||||
[SILO stack investigation](docs/investigations/go127-stack.md).
|
||||
[Go 1.27 TLS and OIDC discovery guide](https://silo.pgsty.com/blog/design/go127-tls-oidc-discovery/).
|
||||
|
||||
## Documentation ownership
|
||||
|
||||
User documentation is maintained at [silo.pgsty.com](https://silo.pgsty.com/docs/),
|
||||
with source in [pgsty/silo.pgsty.com](https://github.com/pgsty/silo.pgsty.com).
|
||||
The remaining `docs/` tree contains inherited references, examples, and tooling
|
||||
fixtures. Investigation logs, AI work records, and temporary reports are kept
|
||||
outside this repository; reusable findings belong in the companion site.
|
||||
See [AGENTS.md](AGENTS.md) for repository ownership and maintenance rules.
|
||||
|
||||
## Security & Contributing
|
||||
|
||||
@@ -123,53 +135,25 @@ Report vulnerabilities privately as described in [`SECURITY.md`](SECURITY.md); e
|
||||
|
||||
## Contributors
|
||||
|
||||
**41 community contributors** build SILO, Console, mcli, shared packages, and related projects. The list includes maintainers and every human Issue or PR author, ordered by merged PRs, other PRs, then issue reports. Gold rings highlight significant contributions.
|
||||
<!-- Generated by silo.pgsty.com/bin/contributors.py. -->
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/Vonng"><img src="https://silo.pgsty.com/images/contributors/Vonng.svg" width="60" height="60" alt="@Vonng" title="@Vonng — Maintains SILO, Console, mcli, shared packages, releases, and documentation"></a>
|
||||
<a href="https://github.com/h5vx"><img src="https://silo.pgsty.com/images/contributors/h5vx.svg" width="60" height="60" alt="@h5vx" title="@h5vx — Implemented per-bucket CORS configuration and enforcement"></a>
|
||||
<a href="https://github.com/mrjavadseydi"><img src="https://silo.pgsty.com/images/contributors/mrjavadseydi.svg" width="60" height="60" alt="@mrjavadseydi" title="@mrjavadseydi — Fixed effective bucket quota metrics; proposed access-frequency ILM"></a>
|
||||
<a href="https://github.com/Dansyuqri"><img src="https://silo.pgsty.com/images/contributors/Dansyuqri.svg" width="60" height="60" alt="@Dansyuqri" title="@Dansyuqri — Added ChecksumType to multipart completion responses"></a>
|
||||
<a href="https://github.com/ycjlin"><img src="https://silo.pgsty.com/images/contributors/ycjlin.svg" width="60" height="60" alt="@ycjlin" title="@ycjlin — Fixed missing-bucket ListObjects semantics"></a>
|
||||
<a href="https://github.com/pinginfo"><img src="https://silo.pgsty.com/images/contributors/pinginfo.svg" width="60" height="60" alt="@pinginfo" title="@pinginfo — Repaired bucket notification streaming"></a>
|
||||
<a href="https://github.com/ZouhairCharef"><img src="https://silo.pgsty.com/images/contributors/ZouhairCharef.svg" width="60" height="60" alt="@ZouhairCharef" title="@ZouhairCharef — Patched CVE-2026-34986 in go-jose"></a>
|
||||
<a href="https://github.com/mfredenhagen"><img src="https://silo.pgsty.com/images/contributors/mfredenhagen.svg" width="60" height="60" alt="@mfredenhagen" title="@mfredenhagen — Patched CVE-2026-39883 in OpenTelemetry"></a>
|
||||
<a href="https://github.com/waterkip"><img src="https://silo.pgsty.com/images/contributors/waterkip.svg" width="60" height="60" alt="@waterkip" title="@waterkip — Repointed documentation links to the SILO portal"></a>
|
||||
<a href="https://github.com/mikemikimike"><img src="https://silo.pgsty.com/images/contributors/mikemikimike.svg" width="60" height="60" alt="@mikemikimike" title="@mikemikimike — Contributed the replicated SSE-C plaintext part-size fix"></a>
|
||||
<a href="https://github.com/metaneutrons"><img src="https://silo.pgsty.com/images/contributors/metaneutrons.svg" width="60" height="60" alt="@metaneutrons" title="@metaneutrons — Reported and proposed explicit-version delete authorization"></a>
|
||||
<a href="https://github.com/magicxor"><img src="https://silo.pgsty.com/images/contributors/magicxor.svg" width="60" height="60" alt="@magicxor" title="@magicxor — Reported and proposed conditional DELETE support for If-Match"></a>
|
||||
<a href="https://github.com/davinkevin"><img src="https://silo.pgsty.com/images/contributors/davinkevin.svg" width="60" height="60" alt="@davinkevin" title="@davinkevin — Proposed the distroless container image and dependency automation"></a>
|
||||
<a href="https://github.com/lem21h"><img src="https://silo.pgsty.com/images/contributors/lem21h.svg" width="48" height="48" alt="@lem21h" title="@lem21h — Proposed robustness and goroutine improvements"></a>
|
||||
<a href="https://github.com/sulin37392"><img src="https://silo.pgsty.com/images/contributors/sulin37392.svg" width="48" height="48" alt="@sulin37392" title="@sulin37392 — Proposed dependency updates"></a>
|
||||
<a href="https://github.com/cbornet"><img src="https://silo.pgsty.com/images/contributors/cbornet.svg" width="60" height="60" alt="@cbornet" title="@cbornet — Reported multipart and streaming checksum defects and missing-bucket semantics"></a>
|
||||
<a href="https://github.com/vampywiz17"><img src="https://silo.pgsty.com/images/contributors/vampywiz17.svg" width="60" height="60" alt="@vampywiz17" title="@vampywiz17 — Reported LDAP TLS and Console login regressions"></a>
|
||||
<a href="https://github.com/orenyomtov"><img src="https://silo.pgsty.com/images/contributors/orenyomtov.svg" width="60" height="60" alt="@orenyomtov" title="@orenyomtov — Reported the unsigned-header CopyObject cross-object read (SN-2026-011)"></a>
|
||||
<a href="https://github.com/mumu-lab"><img src="https://silo.pgsty.com/images/contributors/mumu-lab.svg" width="48" height="48" alt="@mumu-lab" title="@mumu-lab — Reported bucket quota metrics reading a deprecated field"></a>
|
||||
<a href="https://github.com/jvasile"><img src="https://silo.pgsty.com/images/contributors/jvasile.svg" width="48" height="48" alt="@jvasile" title="@jvasile — Reported missing user, group, and defaults in Debian packages"></a>
|
||||
<a href="https://github.com/pmezhuev"><img src="https://silo.pgsty.com/images/contributors/pmezhuev.svg" width="48" height="48" alt="@pmezhuev" title="@pmezhuev — Reported missing RPM package signatures"></a>
|
||||
<a href="https://github.com/TLINDEN"><img src="https://silo.pgsty.com/images/contributors/TLINDEN.svg" width="48" height="48" alt="@TLINDEN" title="@TLINDEN — Reported the missing client in release tarballs"></a>
|
||||
<a href="https://github.com/makinikm"><img src="https://silo.pgsty.com/images/contributors/makinikm.svg" width="48" height="48" alt="@makinikm" title="@makinikm — Reported the missing client in the container image"></a>
|
||||
<a href="https://github.com/meesudzu"><img src="https://silo.pgsty.com/images/contributors/meesudzu.svg" width="48" height="48" alt="@meesudzu" title="@meesudzu — Requested the migration guide from upstream MinIO"></a>
|
||||
<a href="https://github.com/kuldeep-link11"><img src="https://silo.pgsty.com/images/contributors/kuldeep-link11.svg" width="48" height="48" alt="@kuldeep-link11" title="@kuldeep-link11 — Reported NATS JWT credentials and target reload issues"></a>
|
||||
<a href="https://github.com/sargarass"><img src="https://silo.pgsty.com/images/contributors/sargarass.svg" width="48" height="48" alt="@sargarass" title="@sargarass — Reported ListMultipartUploads prefix and pagination semantics"></a>
|
||||
<a href="https://github.com/liuhaodongliu990-cmyk"><img src="https://silo.pgsty.com/images/contributors/liuhaodongliu990-cmyk.svg" width="48" height="48" alt="@liuhaodongliu990-cmyk" title="@liuhaodongliu990-cmyk — Reported indeterminate progress for prefix downloads"></a>
|
||||
<a href="https://github.com/Xavier-777"><img src="https://silo.pgsty.com/images/contributors/Xavier-777.svg" width="48" height="48" alt="@Xavier-777" title="@Xavier-777 — Reported Console lifecycle management and file preview gaps"></a>
|
||||
<a href="https://github.com/spaceg00se-r"><img src="https://silo.pgsty.com/images/contributors/spaceg00se-r.svg" width="48" height="48" alt="@spaceg00se-r" title="@spaceg00se-r — Requested cpuv1 support and reported a workflow token failure"></a>
|
||||
<a href="https://github.com/kh0mka"><img src="https://silo.pgsty.com/images/contributors/kh0mka.svg" width="48" height="48" alt="@kh0mka" title="@kh0mka — Reported inter-node I/O timeouts in ReadFileStreamHandler"></a>
|
||||
<a href="https://github.com/bagutzu"><img src="https://silo.pgsty.com/images/contributors/bagutzu.svg" width="48" height="48" alt="@bagutzu" title="@bagutzu — Requested KES-compatible external KMS and OpenBao support"></a>
|
||||
<a href="https://github.com/DestroyLee"><img src="https://silo.pgsty.com/images/contributors/DestroyLee.svg" width="48" height="48" alt="@DestroyLee" title="@DestroyLee — Reported the missing documentation navigation"></a>
|
||||
<a href="https://github.com/mosesdd"><img src="https://silo.pgsty.com/images/contributors/mosesdd.svg" width="48" height="48" alt="@mosesdd" title="@mosesdd — Requested a maintained Helm chart"></a>
|
||||
<a href="https://github.com/zylpsrs"><img src="https://silo.pgsty.com/images/contributors/zylpsrs.svg" width="48" height="48" alt="@zylpsrs" title="@zylpsrs — Reported missing Console tiering and site replication"></a>
|
||||
<a href="https://github.com/heroes1412"><img src="https://silo.pgsty.com/images/contributors/heroes1412.svg" width="48" height="48" alt="@heroes1412" title="@heroes1412 — Reported the unusable profiling option"></a>
|
||||
<a href="https://github.com/redfoxfox"><img src="https://silo.pgsty.com/images/contributors/redfoxfox.svg" width="48" height="48" alt="@redfoxfox" title="@redfoxfox — Reported Chinese documentation availability"></a>
|
||||
<a href="https://github.com/jiadzh"><img src="https://silo.pgsty.com/images/contributors/jiadzh.svg" width="48" height="48" alt="@jiadzh" title="@jiadzh — Requested Windows build guidance"></a>
|
||||
<a href="https://github.com/AntonOfTheWoods"><img src="https://silo.pgsty.com/images/contributors/AntonOfTheWoods.svg" width="48" height="48" alt="@AntonOfTheWoods" title="@AntonOfTheWoods — Asked for clarity on Helm chart and operator options"></a>
|
||||
<a href="https://github.com/chalukyaj"><img src="https://silo.pgsty.com/images/contributors/chalukyaj.svg" width="48" height="48" alt="@chalukyaj" title="@chalukyaj — Proposed making the SILO Operator easier to discover"></a>
|
||||
<a href="https://github.com/nsanitate"><img src="https://silo.pgsty.com/images/contributors/nsanitate.svg" width="48" height="48" alt="@nsanitate" title="@nsanitate — Proposed CNCF Sandbox governance"></a>
|
||||
<a href="https://github.com/Kesavaambati"><img src="https://silo.pgsty.com/images/contributors/Kesavaambati.svg" width="48" height="48" alt="@Kesavaambati" title="@Kesavaambati — Asked about community support and image maintenance"></a>
|
||||
</p>
|
||||
Every human issue or pull-request author is part of the SILO community, including open and unmerged work. Merged fixes, adopted proposals, and actionable reports receive priority, with first participation guiding the remaining order. Gold rings highlight reviewed significant contributions.
|
||||
|
||||
[View the full contribution record](CONTRIBUTORS.md) for each person's proposals, fixes, and reports.
|
||||
<a href="CONTRIBUTORS.md">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/contributors-dark.svg">
|
||||
<img src="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/contributors-light.svg" alt="SILO community contributors">
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
[View contribution notes and actual PR status](CONTRIBUTORS.md).
|
||||
|
||||
## Star History
|
||||
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/star-history-dark.svg">
|
||||
<img src="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/star-history-light.svg" alt="SILO GitHub star history">
|
||||
</picture>
|
||||
|
||||
## Background
|
||||
|
||||
|
||||
+34
-47
@@ -38,7 +38,7 @@
|
||||
## 当前发行版与主分支
|
||||
|
||||
最新已发布的 Server 仍为 [20260903](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z)。
|
||||
截至 2026-09-13,主分支已合入更新的安全、存储、Console 与共享包改动,但尚未发布新 Server。
|
||||
截至 2026-09-16,主分支已合入更新的安全、存储、Console 与共享包改动,但尚未发布新 Server。
|
||||
准确的已发布/源码边界见 [CHANGELOG.md](CHANGELOG.md) 与[组件版本矩阵](https://silo.pgsty.com/zh/compatibility/versions/),
|
||||
其中包括 SN-2026-011 修复状态与密码权限迁移要求。
|
||||
|
||||
@@ -92,63 +92,50 @@ docker exec silo mcli mb local/demo && docker exec silo mcli ls local
|
||||
|
||||
## 兼容性
|
||||
|
||||
S3 API、`MINIO_*` 环境变量、`minio_*` 指标、`x-minio-*` 头、`/minio/*` 路由、`github.com/minio/*` 导入路径与磁盘格式(含 `.minio.sys`)原样保留,并由 CI 兼容性门禁冻结。只有 Silo 自有交付面改名:`silo` 可执行文件、软件包、服务、Helm Chart 与容器镜像 —— 原生交付物不会安装 `minio` 二进制别名。
|
||||
Silo 保留 S3 与存储格式兼容性,包括既有 `MINIO_*` 环境变量、`minio_*` 指标、`x-minio-*` 头、`/minio/*` 路由与 `.minio.sys` 数据。CI 守卫检查选定的兼容性标识,有意的安全与行为变化在发布说明中记录。Silo 自有交付面使用 `silo` 可执行文件、软件包、服务、Helm Chart 与容器镜像;原生交付物不会安装 `minio` 服务端二进制别名。
|
||||
|
||||
正式支持和发布验收的组合为 `pgsty/silo` + `pgsty/silo-console` + `pgsty/mc` + `pgsty/silo-pkg`,对未修改的上游 MinIO/MC 尽最大努力保持兼容。Server、Console 与客户端按需保留历史模块路径,维护源码直接导入 `github.com/pgsty/silo-pkg/v3`;SDK `github.com/minio/minio-go/v7` 是明确保留的上游依赖。具体版本与 replace 以当前 [go.mod](go.mod) 为准。
|
||||
|
||||
与上游的全部分歧,以逐项核验代码的[兼容性审计](https://silo.pgsty.com/zh/compatibility/server/)形式维护。每个版本仍应视为下游升级:锁定版本,阅读[版本说明](https://silo.pgsty.com/zh/tags/silo/),并保留回滚路径。
|
||||
|
||||
### TLS 与 Go 升级
|
||||
|
||||
Server 恢复 Go 默认密钥交换策略的修复已在 main,尚未包含在 Server 20260903。
|
||||
`GODEBUG=tlsmlkem=0`、`tlssecpmlkem=0` 的适用范围、macOS 根证书来源变化,
|
||||
以及 OIDC discovery 的诊断方法见 [Go 1.27 TLS 与 OIDC 指南](https://silo.pgsty.com/zh/blog/design/go127-tls-oidc-discovery/)。
|
||||
|
||||
## 文档归属
|
||||
|
||||
用户文档统一维护在 [silo.pgsty.com](https://silo.pgsty.com/zh/docs/),源码位于
|
||||
[pgsty/silo.pgsty.com](https://github.com/pgsty/silo.pgsty.com)。本仓库保留的 `docs/`
|
||||
主要是继承的参考材料、示例和工具测试夹具。调查日志、AI 工作记录与临时报告放在仓库外;
|
||||
可复用的结论应整理进伴生文档站。仓库职责与维护规则见 [AGENTS.md](AGENTS.md)。
|
||||
|
||||
## 安全与贡献
|
||||
|
||||
请按照 [`SECURITY.md`](SECURITY.md) 私密报告漏洞;每项修复都会发布公开[安全公告](https://silo.pgsty.com/zh/blog/security/)。本项目不要求签署 CLA:贡献按 AGPL-3.0-or-later(inbound=outbound)接收,只需 DCO 签署(`git commit -s`),详见 [`CONTRIBUTING.md`](CONTRIBUTING.md)。
|
||||
|
||||
## 贡献者
|
||||
|
||||
**41 位社区贡献者**共同建设 SILO、Console、mcli、公共包与相关项目。名单包含维护者,以及所有提出 Issue 或 PR 的真人作者;按已合并 PR、其他 PR、Issue 报告排序,黄圈标记显著贡献。
|
||||
<!-- Generated by silo.pgsty.com/bin/contributors.py. -->
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/Vonng"><img src="https://silo.pgsty.com/images/contributors/Vonng.svg" width="60" height="60" alt="@Vonng" title="@Vonng — 维护 SILO、Console、mcli、公共包、发行与文档"></a>
|
||||
<a href="https://github.com/h5vx"><img src="https://silo.pgsty.com/images/contributors/h5vx.svg" width="60" height="60" alt="@h5vx" title="@h5vx — 实现单桶 CORS 配置与请求执行"></a>
|
||||
<a href="https://github.com/mrjavadseydi"><img src="https://silo.pgsty.com/images/contributors/mrjavadseydi.svg" width="60" height="60" alt="@mrjavadseydi" title="@mrjavadseydi — 修复有效桶配额指标,并提交按访问频率分层的 ILM 方案"></a>
|
||||
<a href="https://github.com/Dansyuqri"><img src="https://silo.pgsty.com/images/contributors/Dansyuqri.svg" width="60" height="60" alt="@Dansyuqri" title="@Dansyuqri — 为分片上传完成响应补充 ChecksumType"></a>
|
||||
<a href="https://github.com/ycjlin"><img src="https://silo.pgsty.com/images/contributors/ycjlin.svg" width="60" height="60" alt="@ycjlin" title="@ycjlin — 修复缺失桶的 ListObjects 语义"></a>
|
||||
<a href="https://github.com/pinginfo"><img src="https://silo.pgsty.com/images/contributors/pinginfo.svg" width="60" height="60" alt="@pinginfo" title="@pinginfo — 修复桶通知的流式输出"></a>
|
||||
<a href="https://github.com/ZouhairCharef"><img src="https://silo.pgsty.com/images/contributors/ZouhairCharef.svg" width="60" height="60" alt="@ZouhairCharef" title="@ZouhairCharef — 修复 go-jose 中的 CVE-2026-34986"></a>
|
||||
<a href="https://github.com/mfredenhagen"><img src="https://silo.pgsty.com/images/contributors/mfredenhagen.svg" width="60" height="60" alt="@mfredenhagen" title="@mfredenhagen — 修复 OpenTelemetry 中的 CVE-2026-39883"></a>
|
||||
<a href="https://github.com/waterkip"><img src="https://silo.pgsty.com/images/contributors/waterkip.svg" width="60" height="60" alt="@waterkip" title="@waterkip — 将文档链接指向 SILO 门户"></a>
|
||||
<a href="https://github.com/mikemikimike"><img src="https://silo.pgsty.com/images/contributors/mikemikimike.svg" width="60" height="60" alt="@mikemikimike" title="@mikemikimike — 提交 SSE-C 复制分片明文尺寸修复"></a>
|
||||
<a href="https://github.com/metaneutrons"><img src="https://silo.pgsty.com/images/contributors/metaneutrons.svg" width="60" height="60" alt="@metaneutrons" title="@metaneutrons — 报告并提交显式版本删除鉴权方案"></a>
|
||||
<a href="https://github.com/magicxor"><img src="https://silo.pgsty.com/images/contributors/magicxor.svg" width="60" height="60" alt="@magicxor" title="@magicxor — 报告并提交 DELETE If-Match 条件请求支持方案"></a>
|
||||
<a href="https://github.com/davinkevin"><img src="https://silo.pgsty.com/images/contributors/davinkevin.svg" width="60" height="60" alt="@davinkevin" title="@davinkevin — 提交 distroless 容器镜像与依赖自动更新方案"></a>
|
||||
<a href="https://github.com/lem21h"><img src="https://silo.pgsty.com/images/contributors/lem21h.svg" width="48" height="48" alt="@lem21h" title="@lem21h — 提交健壮性与 goroutine 改进"></a>
|
||||
<a href="https://github.com/sulin37392"><img src="https://silo.pgsty.com/images/contributors/sulin37392.svg" width="48" height="48" alt="@sulin37392" title="@sulin37392 — 提交依赖更新"></a>
|
||||
<a href="https://github.com/cbornet"><img src="https://silo.pgsty.com/images/contributors/cbornet.svg" width="60" height="60" alt="@cbornet" title="@cbornet — 报告分片与流式校验和缺陷及缺失桶语义问题"></a>
|
||||
<a href="https://github.com/vampywiz17"><img src="https://silo.pgsty.com/images/contributors/vampywiz17.svg" width="60" height="60" alt="@vampywiz17" title="@vampywiz17 — 报告 LDAP TLS 与 Console 登录回归"></a>
|
||||
<a href="https://github.com/orenyomtov"><img src="https://silo.pgsty.com/images/contributors/orenyomtov.svg" width="60" height="60" alt="@orenyomtov" title="@orenyomtov — 报告未签名头导致的 CopyObject 跨对象读取(SN-2026-011)"></a>
|
||||
<a href="https://github.com/mumu-lab"><img src="https://silo.pgsty.com/images/contributors/mumu-lab.svg" width="48" height="48" alt="@mumu-lab" title="@mumu-lab — 报告桶配额指标读取已弃用字段的问题"></a>
|
||||
<a href="https://github.com/jvasile"><img src="https://silo.pgsty.com/images/contributors/jvasile.svg" width="48" height="48" alt="@jvasile" title="@jvasile — 报告 Debian 包缺少用户、用户组与默认配置"></a>
|
||||
<a href="https://github.com/pmezhuev"><img src="https://silo.pgsty.com/images/contributors/pmezhuev.svg" width="48" height="48" alt="@pmezhuev" title="@pmezhuev — 报告 RPM 包缺少 GPG 签名"></a>
|
||||
<a href="https://github.com/TLINDEN"><img src="https://silo.pgsty.com/images/contributors/TLINDEN.svg" width="48" height="48" alt="@TLINDEN" title="@TLINDEN — 报告发布压缩包缺少客户端"></a>
|
||||
<a href="https://github.com/makinikm"><img src="https://silo.pgsty.com/images/contributors/makinikm.svg" width="48" height="48" alt="@makinikm" title="@makinikm — 报告容器镜像缺少客户端"></a>
|
||||
<a href="https://github.com/meesudzu"><img src="https://silo.pgsty.com/images/contributors/meesudzu.svg" width="48" height="48" alt="@meesudzu" title="@meesudzu — 提出从上游 MinIO 迁移的指南需求"></a>
|
||||
<a href="https://github.com/kuldeep-link11"><img src="https://silo.pgsty.com/images/contributors/kuldeep-link11.svg" width="48" height="48" alt="@kuldeep-link11" title="@kuldeep-link11 — 报告 NATS JWT 凭据与通知目标重载问题"></a>
|
||||
<a href="https://github.com/sargarass"><img src="https://silo.pgsty.com/images/contributors/sargarass.svg" width="48" height="48" alt="@sargarass" title="@sargarass — 报告 ListMultipartUploads 前缀与分页语义问题"></a>
|
||||
<a href="https://github.com/liuhaodongliu990-cmyk"><img src="https://silo.pgsty.com/images/contributors/liuhaodongliu990-cmyk.svg" width="48" height="48" alt="@liuhaodongliu990-cmyk" title="@liuhaodongliu990-cmyk — 报告前缀下载进度显示异常"></a>
|
||||
<a href="https://github.com/Xavier-777"><img src="https://silo.pgsty.com/images/contributors/Xavier-777.svg" width="48" height="48" alt="@Xavier-777" title="@Xavier-777 — 报告 Console 生命周期管理与文件预览缺失"></a>
|
||||
<a href="https://github.com/spaceg00se-r"><img src="https://silo.pgsty.com/images/contributors/spaceg00se-r.svg" width="48" height="48" alt="@spaceg00se-r" title="@spaceg00se-r — 提出 cpuv1 支持需求并报告工作流令牌错误"></a>
|
||||
<a href="https://github.com/kh0mka"><img src="https://silo.pgsty.com/images/contributors/kh0mka.svg" width="48" height="48" alt="@kh0mka" title="@kh0mka — 报告 ReadFileStreamHandler 节点间 I/O 超时"></a>
|
||||
<a href="https://github.com/bagutzu"><img src="https://silo.pgsty.com/images/contributors/bagutzu.svg" width="48" height="48" alt="@bagutzu" title="@bagutzu — 提出兼容 KES 的外部 KMS 与 OpenBao 支持需求"></a>
|
||||
<a href="https://github.com/DestroyLee"><img src="https://silo.pgsty.com/images/contributors/DestroyLee.svg" width="48" height="48" alt="@DestroyLee" title="@DestroyLee — 报告文档目录导航缺失"></a>
|
||||
<a href="https://github.com/mosesdd"><img src="https://silo.pgsty.com/images/contributors/mosesdd.svg" width="48" height="48" alt="@mosesdd" title="@mosesdd — 提出维护 Helm Chart 的需求"></a>
|
||||
<a href="https://github.com/zylpsrs"><img src="https://silo.pgsty.com/images/contributors/zylpsrs.svg" width="48" height="48" alt="@zylpsrs" title="@zylpsrs — 报告 Console 缺少分层与站点复制"></a>
|
||||
<a href="https://github.com/heroes1412"><img src="https://silo.pgsty.com/images/contributors/heroes1412.svg" width="48" height="48" alt="@heroes1412" title="@heroes1412 — 报告性能分析选项不可用"></a>
|
||||
<a href="https://github.com/redfoxfox"><img src="https://silo.pgsty.com/images/contributors/redfoxfox.svg" width="48" height="48" alt="@redfoxfox" title="@redfoxfox — 报告中文文档站点不可用"></a>
|
||||
<a href="https://github.com/jiadzh"><img src="https://silo.pgsty.com/images/contributors/jiadzh.svg" width="48" height="48" alt="@jiadzh" title="@jiadzh — 提出 Windows 构建指导需求"></a>
|
||||
<a href="https://github.com/AntonOfTheWoods"><img src="https://silo.pgsty.com/images/contributors/AntonOfTheWoods.svg" width="48" height="48" alt="@AntonOfTheWoods" title="@AntonOfTheWoods — 提出明确 Helm Chart 与 Operator 选项的需求"></a>
|
||||
<a href="https://github.com/chalukyaj"><img src="https://silo.pgsty.com/images/contributors/chalukyaj.svg" width="48" height="48" alt="@chalukyaj" title="@chalukyaj — 提出改善 SILO Operator 可发现性的建议"></a>
|
||||
<a href="https://github.com/nsanitate"><img src="https://silo.pgsty.com/images/contributors/nsanitate.svg" width="48" height="48" alt="@nsanitate" title="@nsanitate — 提出加入 CNCF Sandbox 的治理建议"></a>
|
||||
<a href="https://github.com/Kesavaambati"><img src="https://silo.pgsty.com/images/contributors/Kesavaambati.svg" width="48" height="48" alt="@Kesavaambati" title="@Kesavaambati — 提出社区支持与容器镜像维护问题"></a>
|
||||
</p>
|
||||
每位 issue 或 PR 的作者都是 SILO 社区的一员,包括尚未合并的工作。已合并的修复、被采纳的方案和有效报告优先展示,其余参考首次参与时间;金色圆环突出经过审核的显著贡献。
|
||||
|
||||
[查看完整贡献记录](CONTRIBUTORS.md),了解每位贡献者的提案、修复与问题报告。
|
||||
<a href="CONTRIBUTORS.md">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/contributors-dark.svg">
|
||||
<img src="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/contributors-light.svg" alt="SILO 社区贡献者">
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
[查看贡献记录与实际 PR 状态](CONTRIBUTORS.md)。
|
||||
|
||||
## Star History
|
||||
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/star-history-dark.svg">
|
||||
<img src="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/star-history-light.svg" alt="SILO GitHub 星标历史">
|
||||
</picture>
|
||||
|
||||
## 背景
|
||||
|
||||
|
||||
+3
-3
@@ -7,7 +7,7 @@ Silo-specific fixes or release notes.
|
||||
## Supported Versions
|
||||
|
||||
Security fixes are tracked on the active development branch and summarized in
|
||||
[docs/security/advisories.md](docs/security/advisories.md). Only the current
|
||||
[the security advisory ledger](https://silo.pgsty.com/about/security-advisories/). Only the current
|
||||
Silo release line is supported unless an advisory says otherwise.
|
||||
|
||||
## Inherited Fix Evidence
|
||||
@@ -26,7 +26,7 @@ separately even when the fork preserves the original commit object and SHA.
|
||||
The inherited [service-account](https://github.com/pgsty/silo/blob/c1a49490c78e9c3ebcad86ba0662319138ace190/cmd/admin-handlers-users_test.go#L211-L212)
|
||||
and [STS](https://github.com/pgsty/silo/blob/c1a49490c78e9c3ebcad86ba0662319138ace190/cmd/sts-handlers_test.go#L45-L46)
|
||||
regression groups remain part of `go test ./cmd`; see the
|
||||
[canonical ledger](docs/security/advisories.md#inherited-upstream-advisory-baseline)
|
||||
[canonical ledger](https://silo.pgsty.com/about/security-advisories/#inherited)
|
||||
for the operator-facing record.
|
||||
|
||||
## Reporting a Vulnerability
|
||||
@@ -42,4 +42,4 @@ For vulnerabilities in this fork:
|
||||
|
||||
## Disclosure Process
|
||||
|
||||
Fork-specific fixes and user-visible upgrade notes are published in [docs/security/advisories.md](docs/security/advisories.md). The fork-specific triage and remediation process is described in [VULNERABILITY_REPORT.md](VULNERABILITY_REPORT.md).
|
||||
Fork-specific fixes and user-visible upgrade notes are published in [the security advisory ledger](https://silo.pgsty.com/about/security-advisories/). The fork-specific triage and remediation process is described in [VULNERABILITY_REPORT.md](VULNERABILITY_REPORT.md).
|
||||
|
||||
@@ -34,4 +34,4 @@ Based on the report, the Silo maintainers investigate:
|
||||
If the vulnerability exists in this fork itself, the maintainers will, when
|
||||
feasible, fix the issue or implement reasonable countermeasures such that the
|
||||
vulnerability can no longer be exploited. Fork-specific upgrade notes and
|
||||
security advisories are published in `docs/security/advisories.md`.
|
||||
security advisories are published in [the security advisory ledger](https://silo.pgsty.com/about/security-advisories/).
|
||||
|
||||
@@ -7,7 +7,7 @@ _init() {
|
||||
|
||||
## Minimum required versions for build dependencies
|
||||
GIT_VERSION="1.0"
|
||||
GO_VERSION="1.27.1"
|
||||
GO_VERSION="1.27.2"
|
||||
OSX_VERSION="10.8"
|
||||
KNAME=$(uname -s)
|
||||
ARCH=$(uname -m)
|
||||
|
||||
@@ -40,7 +40,7 @@ if [ -n "${MCLI_BIN:-}" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
release=${MCLI_RELEASE:-RELEASE.2026-09-13T00-00-00Z}
|
||||
release=${MCLI_RELEASE:-RELEASE.2026-10-18T00-00-00Z}
|
||||
version_hyphen=${release#RELEASE.}
|
||||
package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')
|
||||
if [ "${package_version}" = "${version_hyphen}" ]; then
|
||||
|
||||
@@ -132,8 +132,8 @@
|
||||
"MINIO_API_DELETE_CLEANUP_INTERVAL",
|
||||
"MINIO_API_DISABLE_ODIRECT",
|
||||
"MINIO_API_GZIP_OBJECTS",
|
||||
"MINIO_API_LEGACY_BUCKET_RESOURCE_MATCH",
|
||||
"MINIO_API_LIST_QUORUM",
|
||||
"MINIO_API_MULTIPART_LISTING",
|
||||
"MINIO_API_OBJECT_MAX_VERSIONS",
|
||||
"MINIO_API_ODIRECT",
|
||||
"MINIO_API_REMOTE_TRANSPORT_DEADLINE",
|
||||
@@ -571,6 +571,14 @@
|
||||
"minio_resource_stats",
|
||||
"minio_s3",
|
||||
"minio_stats",
|
||||
"minio_system_cpu_avg_idle",
|
||||
"minio_system_cpu_avg_iowait",
|
||||
"minio_system_cpu_load",
|
||||
"minio_system_cpu_load_perc",
|
||||
"minio_system_cpu_nice",
|
||||
"minio_system_cpu_steal",
|
||||
"minio_system_cpu_system",
|
||||
"minio_system_cpu_user",
|
||||
"minio_test"
|
||||
],
|
||||
"headers": [
|
||||
@@ -627,7 +635,6 @@
|
||||
"x-minio-origin-endpoint",
|
||||
"x-minio-prefixes-total",
|
||||
"x-minio-read-quorum",
|
||||
"x-minio-replication",
|
||||
"x-minio-replication-actual-object-size",
|
||||
"x-minio-replication-delete-status",
|
||||
"x-minio-replication-deletemarker-status",
|
||||
@@ -766,6 +773,7 @@
|
||||
"/metrics/v3",
|
||||
"/minio/grid/",
|
||||
"/minio/grid/lock/",
|
||||
"/multipart-preflight",
|
||||
"/netperf",
|
||||
"/notification",
|
||||
"/oauth2/callback",
|
||||
@@ -934,6 +942,7 @@
|
||||
"arn:minio:kms:::this-is-disregarded",
|
||||
"arn:minio:kms:::xyz-test-key",
|
||||
"arn:minio:replication:",
|
||||
"arn:minio:replication::",
|
||||
"arn:minio:replication::8320b6d18f9032b4700f1f03b50d8d1853de8f22cab86931ee794e12f190852c:destinationbucket",
|
||||
"arn:minio:replication:::",
|
||||
"arn:minio:replication:::dest-bucket",
|
||||
|
||||
@@ -115,9 +115,8 @@ func collect(repo string) (manifest, error) {
|
||||
fset := token.NewFileSet()
|
||||
|
||||
for _, rel := range files {
|
||||
// Investigation artifacts contain synthetic routes and archived configurations.
|
||||
// Migration notes and guard fixtures contain archived identifiers.
|
||||
if rel == "SILO_REBRANDING_MIGRATION.md" ||
|
||||
strings.HasPrefix(rel, "docs/investigations/") ||
|
||||
strings.HasPrefix(rel, "buildscripts/rebrand-guard/") ||
|
||||
strings.HasPrefix(rel, "buildscripts/helm-migration-guard/") {
|
||||
continue
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
__pycache__/
|
||||
@@ -0,0 +1,163 @@
|
||||
# Copyright (c) 2026 Feng Ruohang
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
"""Pure, self-contained SVG rendering for SILO's README cards."""
|
||||
from datetime import date, timedelta
|
||||
from html import escape
|
||||
import math
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
themes = {
|
||||
'light': dict(bg='#ffffff', wash='#f2f7fc', edge='#d9e3ee', ink='#16222e',
|
||||
muted='#62758a', blue='#1d588c', copper='#b4762e', grid='#e5edf5',
|
||||
line='#2b6ca3', ring='#dce5ef', field='#f7f9fc', label='#3d4e61'),
|
||||
'dark': dict(bg='#101923', wash='#152738', edge='#2b3c50', ink='#e8eef6',
|
||||
muted='#93a3b8', blue='#7fb8e8', copper='#e0a35c', grid='#263749',
|
||||
line='#5da2dd', ring='#3a4e63', field='#0b1119', label='#b6c2d2'),
|
||||
}
|
||||
|
||||
def read_emblem(path):
|
||||
emblem = ET.parse(path).getroot()
|
||||
body = ''.join(ET.tostring(child, encoding='unicode') for child in emblem
|
||||
if child.tag.rsplit('}', 1)[-1] in ('defs', 'g'))
|
||||
return '\n'.join(line.rstrip() for line in body.splitlines()).strip()
|
||||
|
||||
|
||||
def txt(x, y, value, size=14, color=None, weight=400, anchor='start', mono=False, spacing=None):
|
||||
family = 'Menlo,Consolas,monospace' if mono else 'Arial,Helvetica,sans-serif'
|
||||
extra = f' letter-spacing="{spacing}"' if spacing is not None else ''
|
||||
return (f'<text x="{x}" y="{y}" font-family="{family}" font-size="{size}" '
|
||||
f'font-weight="{weight}" fill="{color}" text-anchor="{anchor}"{extra}>'
|
||||
f'{escape(str(value))}</text>')
|
||||
|
||||
|
||||
def start(height, theme, title, description, emblem_body):
|
||||
t = themes[theme]
|
||||
return [f'<svg xmlns="http://www.w3.org/2000/svg" width="1000" height="{height}" '
|
||||
f'viewBox="0 0 1000 {height}" role="img" aria-labelledby="title desc">',
|
||||
f'<title id="title">{escape(title)}</title><desc id="desc">{escape(description)}</desc>',
|
||||
'<defs><linearGradient id="surface" x1="0" y1="1" x2="1" y2="0">'
|
||||
f'<stop offset="0" stop-color="{t["bg"]}"/>'
|
||||
f'<stop offset="1" stop-color="{t["wash"]}"/></linearGradient>'
|
||||
'<linearGradient id="accent" x1="0" y1="0" x2="1" y2="0">'
|
||||
f'<stop offset="0" stop-color="{t["blue"]}"/>'
|
||||
f'<stop offset="1" stop-color="{t["copper"]}"/></linearGradient>'
|
||||
'<linearGradient id="area" x1="0" y1="0" x2="0" y2="1">'
|
||||
f'<stop offset="0" stop-color="{t["line"]}" stop-opacity=".22"/>'
|
||||
f'<stop offset="1" stop-color="{t["line"]}" stop-opacity=".015"/>'
|
||||
'</linearGradient></defs>',
|
||||
f'<rect x=".75" y=".75" width="998.5" height="{height-1.5}" rx="22" '
|
||||
f'fill="url(#surface)" stroke="{t["edge"]}" stroke-width="1.5"/>',
|
||||
f'<svg x="40" y="25" width="25" height="25" viewBox="230 213 570 570">{emblem_body}</svg>']
|
||||
|
||||
|
||||
def heading(parts, t, eyebrow, title, subtitle, value, value_label):
|
||||
parts.extend([
|
||||
txt(76, 43, eyebrow, 11, t['muted'], 600, mono=True, spacing=1.7),
|
||||
txt(40, 94, title, 32, t['ink'], 700),
|
||||
txt(41, 123, subtitle, 14, t['muted']),
|
||||
txt(958, 89, f'{value:,}', 45, t['ink'], 700, anchor='end'),
|
||||
txt(957, 114, value_label, 10, t['muted'], 600, anchor='end', mono=True, spacing=1.5),
|
||||
f'<path d="M40 146 H960" stroke="{t["edge"]}"/>',
|
||||
])
|
||||
|
||||
|
||||
def contributors(theme, people, snapshot, emblem_body):
|
||||
height = 206 + 76 * math.ceil(len(people) / 10)
|
||||
t = themes[theme]
|
||||
parts = start(height, theme, f'SILO community — {len(people)} contributors',
|
||||
f'The existing SILO community roll, including code, proposals and reports across related projects. '
|
||||
f'Gold rings retain the existing significant-contribution designation. Snapshot {snapshot}.', emblem_body)
|
||||
heading(parts, t, 'SILO / COMMUNITY', 'Contributors',
|
||||
'Code, proposals & reports across SILO and related projects', len(people), 'COMMUNITY CONTRIBUTORS')
|
||||
for row in range(math.ceil(len(people) / 10)):
|
||||
group = people[row * 10:(row + 1) * 10]
|
||||
row_width = len(group) * 91
|
||||
for col, person in enumerate(group):
|
||||
x = (1000 - row_width) / 2 + col * 91 + 45.5
|
||||
y = 199 + row * 76
|
||||
identifier = f'avatar-{row}-{col}'
|
||||
featured = bool(person.get('featured'))
|
||||
parts.append(f'<g><title>@{escape(person["handle"])} — {escape(person["what"])}</title>')
|
||||
parts.append(f'<defs><clipPath id="{identifier}"><circle cx="{x}" cy="{y}" r="29"/></clipPath></defs>')
|
||||
if featured:
|
||||
parts.append(f'<circle cx="{x}" cy="{y}" r="34" fill="{t["copper"]}" opacity=".09"/>')
|
||||
if person.get('avatarDataUrl'):
|
||||
parts.append(f'<image x="{x-29}" y="{y-29}" width="58" height="58" '
|
||||
f'clip-path="url(#{identifier})" href="{escape(person["avatarDataUrl"])}"/>')
|
||||
else:
|
||||
parts.append(f'<circle cx="{x}" cy="{y}" r="29" fill="{t["ring"]}"/>')
|
||||
parts.append(txt(x, y + 9, person['handle'][0].upper(), 26, t['ink'], 700, 'middle'))
|
||||
parts.append(f'<circle cx="{x}" cy="{y}" r="30.5" fill="none" '
|
||||
f'stroke="{t["copper"] if featured else t["ring"]}" stroke-width="{2 if featured else 1.25}"/></g>')
|
||||
parts.extend([
|
||||
f'<path d="M40 {height-42} H960" stroke="{t["edge"]}"/>',
|
||||
f'<circle cx="47" cy="{height-21}" r="4" fill="none" stroke="{t["copper"]}" stroke-width="1.5"/>',
|
||||
txt(61, height-17, 'Gold rings mark significant contributions', 12, t['muted']),
|
||||
txt(959, height-17, f'AS OF {snapshot}', 10, t['muted'], 500, 'end', mono=True, spacing=.6),
|
||||
'</svg>',
|
||||
])
|
||||
return ''.join(parts)
|
||||
|
||||
|
||||
def stars(theme, history, snapshot, emblem_body):
|
||||
points = history['points']
|
||||
star_count = points[-1]['stars']
|
||||
t = themes[theme]
|
||||
provenance = ('Initial history reconstructed · Daily totals since ' + history['bootstrap']['through'] + ' · UTC'
|
||||
if history['bootstrap']['reconstructed'] else 'Observed daily star totals · UTC')
|
||||
parts = start(558, theme, f'SILO star history — {star_count:,} stars',
|
||||
f'GitHub repository pgsty/silo. {star_count:,} stars as of {snapshot}. ' +
|
||||
provenance, emblem_body)
|
||||
heading(parts, t, 'SILO / GITHUB', 'Star History', 'pgsty/silo', star_count, 'GITHUB STARS')
|
||||
left, right, top, bottom = 76, 958, 177, 440
|
||||
begin = date.fromisoformat(points[0]['date'])
|
||||
end = date.fromisoformat(points[-1]['date'])
|
||||
days = max(1, (end - begin).days)
|
||||
maximum = max(500, math.ceil(max(p['stars'] for p in points) / 500) * 500)
|
||||
tick_step = 10 ** max(0, int(math.log10(maximum)))
|
||||
xy = lambda day, n: (left + (right-left)*(date.fromisoformat(day)-begin).days / days,
|
||||
bottom-(bottom-top)*n/maximum)
|
||||
for value in range(0, maximum+1, tick_step):
|
||||
y = xy(points[0]['date'], value)[1]
|
||||
parts.append(f'<path d="M{left} {y:.2f} H{right}" stroke="{t["grid"]}" stroke-dasharray="4 6"/>')
|
||||
parts.append(txt(left-16, round(y+4, 2), f'{value / 1000:g}k' if value >= 1000 else str(value), 12, t['muted'], anchor='end'))
|
||||
dates = sorted({begin + timedelta(days=round((end-begin).days*i/5)) for i in range(6)})
|
||||
ticks = [(d.isoformat(), d.strftime('%b %Y') if days > 90 else d.strftime('%b %d')) for d in dates]
|
||||
for day, label in ticks:
|
||||
x = xy(day, 0)[0]
|
||||
parts.append(f'<path d="M{x:.2f} {top} V{bottom}" stroke="{t["grid"]}" stroke-opacity=".65"/>')
|
||||
anchor = 'start' if day == points[0]['date'] else 'end' if day == snapshot else 'middle'
|
||||
parts.append(txt(round(x, 2), 466, label, 12, t['muted'], anchor=anchor))
|
||||
coords = [xy(p['date'], p['stars']) for p in points]
|
||||
line = 'M' + ' L'.join(f'{x:.2f} {y:.2f}' for x, y in coords)
|
||||
area = line + f' L{coords[-1][0]:.2f} {bottom} L{left} {bottom} Z'
|
||||
parts.extend([
|
||||
f'<path d="{area}" fill="url(#area)"/>',
|
||||
f'<path d="{line}" fill="none" stroke="url(#accent)" stroke-width="3" '
|
||||
'stroke-linecap="round" stroke-linejoin="round"/>',
|
||||
f'<path d="M{left} {bottom} H{right}" stroke="{t["edge"]}"/>',
|
||||
])
|
||||
x, y = coords[-1]
|
||||
parts.extend([
|
||||
f'<circle cx="{x}" cy="{y}" r="10" fill="{t["copper"]}" opacity=".12"/>',
|
||||
f'<circle cx="{x}" cy="{y}" r="5" fill="{t["copper"]}" stroke="{t["bg"]}" stroke-width="2"/>',
|
||||
f'<path d="M40 491 H960" stroke="{t["edge"]}"/>',
|
||||
txt(40, 516, f'{begin:%b %Y} — {end:%b %Y}'.upper(), 10, t['muted'], 500, mono=True, spacing=.7),
|
||||
txt(959, 516, f'SNAPSHOT {snapshot}', 10, t['muted'], 500, 'end', mono=True, spacing=.6),
|
||||
txt(40, 539, provenance, 11, t['muted']),
|
||||
'</svg>',
|
||||
])
|
||||
return ''.join(parts)
|
||||
@@ -0,0 +1 @@
|
||||
PyYAML==6.0.3
|
||||
@@ -0,0 +1,178 @@
|
||||
# Copyright (c) 2026 Feng Ruohang
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
"""Regression checks for historical accuracy, contributor scope and SVG safety."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from urllib.error import URLError
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
import render
|
||||
import update
|
||||
|
||||
NS = {'s': 'http://www.w3.org/2000/svg'}
|
||||
PNG = base64.b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+a7mgAAAAASUVORK5CYII=')
|
||||
|
||||
|
||||
def person(handle='Alice', group='reports', featured=False):
|
||||
return {'handle': handle, 'group': group, 'featured': featured,
|
||||
'what': 'A reviewed contribution', 'firstContribution': '2026-09-01'}
|
||||
|
||||
|
||||
def history():
|
||||
return {'repository': 'pgsty/silo',
|
||||
'bootstrap': {'through': '2026-09-15', 'reconstructed': True},
|
||||
'points': [{'date': '2026-09-14', 'stars': 100}, {'date': '2026-09-15', 'stars': 105}]}
|
||||
|
||||
|
||||
class HistoryTests(unittest.TestCase):
|
||||
def test_new_day_preserves_old_counts_and_unstars(self):
|
||||
before = history()
|
||||
after = update.update_history(before, '2026-09-16', 103)
|
||||
self.assertEqual(after['points'][:-1], before['points'])
|
||||
self.assertEqual(after['points'][-1], {'date': '2026-09-16', 'stars': 103})
|
||||
self.assertEqual(before, history())
|
||||
|
||||
def test_same_day_rerun_replaces_instead_of_appending(self):
|
||||
first = update.update_history(history(), '2026-09-15', 107)
|
||||
self.assertEqual(len(first['points']), 2)
|
||||
self.assertEqual(first, update.update_history(first, '2026-09-15', 107))
|
||||
|
||||
def test_missing_days_are_not_invented(self):
|
||||
result = update.update_history(history(), '2026-09-18', 106)
|
||||
self.assertEqual([p['date'] for p in result['points']], ['2026-09-14', '2026-09-15', '2026-09-18'])
|
||||
|
||||
def test_rejects_wrong_repository_and_corrupt_history(self):
|
||||
cases = []
|
||||
wrong = history(); wrong['repository'] = 'someone/else'; cases.append(wrong)
|
||||
duplicate = history(); duplicate['points'].append(duplicate['points'][-1]); cases.append(duplicate)
|
||||
unordered = history(); unordered['points'].reverse(); cases.append(unordered)
|
||||
negative = history(); negative['points'][0]['stars'] = -1; cases.append(negative)
|
||||
future = history(); future['points'][-1]['date'] = '2026-09-20'; cases.append(future)
|
||||
for case in cases:
|
||||
with self.subTest(case=case), self.assertRaises(ValueError):
|
||||
update.update_history(case, '2026-09-16', 100)
|
||||
|
||||
def test_first_run_has_no_fabricated_history(self):
|
||||
result = update.update_history(None, '2026-09-16', 10)
|
||||
self.assertFalse(result['bootstrap']['reconstructed'])
|
||||
self.assertEqual(result['points'], [{'date': '2026-09-16', 'stars': 10}])
|
||||
|
||||
|
||||
class ContributorTests(unittest.TestCase):
|
||||
def test_retries_truncated_json_before_using_it(self):
|
||||
with patch('update.request', side_effect=[b'{"partial":', b'{"ok":true}']), patch('update.time.sleep'):
|
||||
self.assertEqual(update.GitHub('').get('repos/pgsty/silo'), {'ok': True})
|
||||
|
||||
def test_paginates_past_one_full_page(self):
|
||||
class API(update.GitHub):
|
||||
def __init__(self): self.calls = []
|
||||
def get(self, path):
|
||||
self.calls.append(path)
|
||||
return list(range(100)) if 'page=1&' in path else [100]
|
||||
api = API()
|
||||
self.assertEqual(len(list(api.issues('pgsty/silo'))), 101)
|
||||
self.assertIn('state=all', api.calls[0])
|
||||
self.assertIn('page=2&', api.calls[1])
|
||||
|
||||
def test_bots_deduplication_unmerged_work_and_reviewed_credit(self):
|
||||
def issue(login, kind='issue', user_type='User'):
|
||||
item = {'user': {'login': login, 'type': user_type, 'avatar_url': ''}, 'created_at': '2026-09-02T00:00:00Z'}
|
||||
if kind != 'issue': item['pull_request'] = {'merged_at': None if kind == 'open' else '2026-09-03T00:00:00Z'}
|
||||
return item
|
||||
class API:
|
||||
def issues(self, _repo):
|
||||
return [issue('alice'), issue('Bob', 'open'), issue('Bob', 'merged'),
|
||||
issue('Carol', 'open'), issue('Copilot'), issue('robot', user_type='Bot')]
|
||||
curated = {'repositories': ['pgsty/silo', 'pgsty/mc'], 'bots': ['Copilot'],
|
||||
'people': [person('Alice', featured=True), person('Reporter'), person('Copilot')]}
|
||||
result = update.collect_people(API(), curated)
|
||||
self.assertEqual({p['handle'] for p in result}, {'Alice', 'Bob', 'Carol', 'Reporter'})
|
||||
self.assertEqual(result[0]['handle'], 'Bob')
|
||||
self.assertEqual(result[1]['handle'], 'Carol')
|
||||
self.assertTrue(next(p for p in result if p['handle'] == 'Alice')['featured'])
|
||||
self.assertEqual(next(p for p in result if p['handle'] == 'Bob')['group'], 'code')
|
||||
self.assertFalse(next(p for p in result if p['handle'] == 'Carol')['featured'])
|
||||
|
||||
def test_newer_reviewed_preview_survives_until_site_catches_up(self):
|
||||
remote = {'updated': '2026-09-16T03:00:00+00:00'}
|
||||
cached = {'updated': '2026-09-16T04:00:00+00:00'}
|
||||
self.assertIs(update.select_curated(remote, cached), cached)
|
||||
newer = {'updated': '2026-09-17T03:00:00+00:00'}
|
||||
self.assertIs(update.select_curated(newer, cached), newer)
|
||||
|
||||
def test_avatar_failure_reuses_raster_cache(self):
|
||||
previous = {**person(), 'avatarDataUrl': update.raster_data_url(PNG)}
|
||||
with patch('update.request', side_effect=URLError('unavailable')):
|
||||
result = update.add_avatars(None, [{**person(), 'avatarUrl': 'https://avatars.githubusercontent.com/u/1'}], [previous])
|
||||
self.assertEqual(result[0]['avatarDataUrl'], previous['avatarDataUrl'])
|
||||
with self.assertRaises(ValueError): update.raster_data_url(b'<svg onload="bad()"/>')
|
||||
with self.assertRaises(ValueError): update.cached_avatar({'avatarDataUrl': 'data:image/svg+xml;base64,PHN2Zy8+'})
|
||||
|
||||
def test_fetch_failure_leaves_published_assets_untouched(self):
|
||||
class API:
|
||||
def get(self, path):
|
||||
if path == 'repos/pgsty/silo': return {'full_name': 'pgsty/silo', 'stargazers_count': 106}
|
||||
raise URLError('roster unavailable')
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
out = Path(directory)
|
||||
original = json.dumps(history())
|
||||
(out / 'history.json').write_text(original)
|
||||
(out / 'contributors-light.svg').write_text('previous image')
|
||||
with self.assertRaises(URLError): update.refresh(out, API(), Path('.'))
|
||||
self.assertEqual((out / 'history.json').read_text(), original)
|
||||
self.assertEqual((out / 'contributors-light.svg').read_text(), 'previous image')
|
||||
|
||||
|
||||
class RenderTests(unittest.TestCase):
|
||||
def test_real_emblem_generates_clean_xml(self):
|
||||
emblem = render.read_emblem(Path(__file__).resolve().parents[2] / '.github/silo.svg')
|
||||
svg = render.contributors('light', [person()], '2026-09-16', emblem)
|
||||
ET.fromstring(svg)
|
||||
self.assertTrue(all(line == line.rstrip() for line in svg.splitlines()))
|
||||
|
||||
def test_all_avatars_fit_when_the_roster_grows(self):
|
||||
people = [{**person(f'person-{i}'), 'avatarDataUrl': update.raster_data_url(PNG)} for i in range(151)]
|
||||
for theme in ('light', 'dark'):
|
||||
root = ET.fromstring(render.contributors(theme, people, '2026-09-16', ''))
|
||||
images = root.findall('.//s:image', NS)
|
||||
self.assertEqual(len(images), 151)
|
||||
footer = float(root.attrib['height']) - 42
|
||||
self.assertTrue(all(float(i.attrib['y']) + float(i.attrib['height']) < footer for i in images))
|
||||
self.assertTrue(all(i.attrib['href'].startswith('data:image/png;base64,') for i in images))
|
||||
|
||||
def test_untrusted_text_is_escaped(self):
|
||||
data = [{**person(), 'what': '<script>alert("x")</script> & contributions'}]
|
||||
svg = render.contributors('light', data, '2026-09-16', '')
|
||||
root = ET.fromstring(svg)
|
||||
self.assertEqual(root.findall('.//s:script', NS), [])
|
||||
self.assertIn('<script>', svg)
|
||||
|
||||
def test_single_point_and_decreasing_star_history_render(self):
|
||||
for data in (update.update_history(None, '2026-09-16', 0), update.update_history(history(), '2026-09-16', 90)):
|
||||
for theme in ('light', 'dark'):
|
||||
svg = render.stars(theme, data, '2026-09-16', '')
|
||||
ET.fromstring(svg)
|
||||
self.assertNotIn('nan', svg.lower())
|
||||
self.assertNotIn('inf', svg.lower())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,295 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) 2026 Feng Ruohang
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
"""Refresh the generated-asset checkout; publishing is handled by the workflow."""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import date, datetime, timezone
|
||||
import json
|
||||
from http.client import IncompleteRead
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlparse
|
||||
from urllib.request import Request, urlopen
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
import yaml
|
||||
|
||||
import render
|
||||
|
||||
REPOSITORY = 'pgsty/silo'
|
||||
SOURCE = 'repos/pgsty/silo.pgsty.com/contents/data/home/contributors.yaml?ref=main'
|
||||
GROUPS = ('code', 'proposed', 'reports')
|
||||
HANDLE = re.compile(r'[A-Za-z0-9][A-Za-z0-9-]{0,38}\Z')
|
||||
|
||||
|
||||
def request(url, token='', limit=8 * 1024 * 1024):
|
||||
headers = {'User-Agent': 'silo-repository-cards', 'Accept': 'application/vnd.github+json'}
|
||||
if urlparse(url).netloc == 'api.github.com':
|
||||
headers['X-GitHub-Api-Version'] = '2022-11-28'
|
||||
if token:
|
||||
headers['Authorization'] = f'Bearer {token}'
|
||||
for attempt in range(3):
|
||||
try:
|
||||
with urlopen(Request(url, headers=headers), timeout=25) as response:
|
||||
data = response.read(limit + 1)
|
||||
if len(data) > limit:
|
||||
raise ValueError('Response exceeds the size limit')
|
||||
expected = response.headers.get('Content-Length')
|
||||
if expected is not None and len(data) != int(expected):
|
||||
raise URLError('Incomplete response body')
|
||||
return data
|
||||
except HTTPError as exc:
|
||||
if exc.code < 500 or attempt == 2:
|
||||
raise
|
||||
except (URLError, TimeoutError, IncompleteRead):
|
||||
if attempt == 2:
|
||||
raise
|
||||
time.sleep(attempt + 1)
|
||||
|
||||
|
||||
class GitHub:
|
||||
def __init__(self, token):
|
||||
self.token = token
|
||||
|
||||
def get(self, path):
|
||||
for attempt in range(3):
|
||||
try:
|
||||
return json.loads(request('https://api.github.com/' + path, self.token))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||
if attempt == 2:
|
||||
raise ValueError(f'Incomplete or invalid GitHub JSON: {path}') from exc
|
||||
time.sleep(attempt + 1)
|
||||
|
||||
def issues(self, repository):
|
||||
page = 1
|
||||
while True:
|
||||
batch = self.get(f'repos/{repository}/issues?state=all&per_page=100&page={page}&sort=created&direction=asc')
|
||||
if not isinstance(batch, list):
|
||||
raise ValueError(f'Invalid issues response for {repository}')
|
||||
yield from batch
|
||||
if len(batch) < 100:
|
||||
return
|
||||
page += 1
|
||||
|
||||
|
||||
def curated_snapshot(data, revision):
|
||||
updated = str(data['updated'])
|
||||
datetime.fromisoformat(updated)
|
||||
repositories = [item['repo'] for item in data['repositories']]
|
||||
if not repositories or any(not re.fullmatch(r'pgsty/[A-Za-z0-9_.-]+', repo) for repo in repositories):
|
||||
raise ValueError('Invalid contributor repository scope')
|
||||
people = []
|
||||
for group in GROUPS:
|
||||
for entry in data[group]:
|
||||
if not HANDLE.fullmatch(entry['handle']):
|
||||
raise ValueError('Invalid GitHub contributor handle')
|
||||
people.append({
|
||||
'handle': entry['handle'], 'group': group,
|
||||
'featured': bool(entry.get('featured')), 'what': entry['what'],
|
||||
'firstContribution': str(entry.get('firstContribution', '9999-12-31')),
|
||||
})
|
||||
if not people or len({p['handle'].lower() for p in people}) != len(people):
|
||||
raise ValueError('Empty or duplicate contributor roster')
|
||||
return {'updated': updated, 'revision': revision, 'repositories': repositories,
|
||||
'bots': data.get('bots', ['Copilot', 'dependabot[bot]']), 'people': people}
|
||||
|
||||
|
||||
def select_curated(remote, cached):
|
||||
# The initial, approved preview can contain reviewed credit not published by
|
||||
# the companion site yet. Keep that newer snapshot until the site catches up.
|
||||
if cached and datetime.fromisoformat(cached['updated']) > datetime.fromisoformat(remote['updated']):
|
||||
return cached
|
||||
return remote
|
||||
|
||||
|
||||
def collect_people(api, curated):
|
||||
bots = {name.lower() for name in curated['bots']}
|
||||
people = {p['handle'].lower(): dict(p) for p in curated['people']
|
||||
if p['handle'].lower() not in bots and not p['handle'].lower().endswith('[bot]')}
|
||||
order = {p['handle'].lower(): index for index, p in enumerate(curated['people'])}
|
||||
for repository in curated['repositories']:
|
||||
print(f'Reading issue and PR authors: {repository}', flush=True)
|
||||
for issue in api.issues(repository):
|
||||
user = issue.get('user') or {}
|
||||
handle = user.get('login', '')
|
||||
key = handle.lower()
|
||||
if user.get('type') != 'User' or key in bots or key.endswith('[bot]'):
|
||||
continue
|
||||
if not HANDLE.fullmatch(handle):
|
||||
raise ValueError('Invalid issue author')
|
||||
pr = issue.get('pull_request')
|
||||
group = 'code' if pr and pr.get('merged_at') else 'proposed' if pr else 'reports'
|
||||
first = issue['created_at'][:10]
|
||||
date.fromisoformat(first)
|
||||
person = people.setdefault(key, {
|
||||
'handle': handle, 'group': group, 'featured': False,
|
||||
'what': 'Contributed an issue or pull request to SILO and related projects',
|
||||
'firstContribution': first,
|
||||
})
|
||||
person['avatarUrl'] = user.get('avatar_url', '')
|
||||
person['firstContribution'] = min(person['firstContribution'], first)
|
||||
if GROUPS.index(group) < GROUPS.index(person['group']):
|
||||
person['group'] = group
|
||||
if not people:
|
||||
raise ValueError('No human contributors were collected')
|
||||
return sorted(people.values(), key=lambda p: (
|
||||
GROUPS.index(p['group']), not p['featured'],
|
||||
order.get(p['handle'].lower(), len(order)), p['firstContribution'], p['handle'].lower()))
|
||||
|
||||
|
||||
def raster_data_url(data):
|
||||
if data.startswith(b'\x89PNG\r\n\x1a\n'):
|
||||
mime = 'image/png'
|
||||
elif data.startswith(b'\xff\xd8\xff'):
|
||||
mime = 'image/jpeg'
|
||||
elif data.startswith((b'GIF87a', b'GIF89a')):
|
||||
mime = 'image/gif'
|
||||
elif data[:4] == b'RIFF' and data[8:12] == b'WEBP':
|
||||
mime = 'image/webp'
|
||||
else:
|
||||
raise ValueError('Avatar is not a raster image')
|
||||
return f'data:{mime};base64,' + base64.b64encode(data).decode('ascii')
|
||||
|
||||
|
||||
def cached_avatar(person):
|
||||
value = person.get('avatarDataUrl', '')
|
||||
if not value:
|
||||
return ''
|
||||
prefix, encoded = value.split(',', 1)
|
||||
if prefix not in ('data:image/png;base64', 'data:image/jpeg;base64', 'data:image/gif;base64', 'data:image/webp;base64'):
|
||||
raise ValueError('Invalid cached avatar format')
|
||||
raw = base64.b64decode(encoded, validate=True)
|
||||
if len(raw) > 512 * 1024 or raster_data_url(raw) != value:
|
||||
raise ValueError('Invalid cached avatar')
|
||||
return value
|
||||
|
||||
|
||||
def add_avatars(api, people, previous):
|
||||
cached = {p['handle'].lower(): cached_avatar(p) for p in previous}
|
||||
|
||||
def update(person):
|
||||
person = dict(person)
|
||||
try:
|
||||
url = person.pop('avatarUrl', '') or api.get('users/' + person['handle'])['avatar_url']
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme != 'https' or parsed.netloc != 'avatars.githubusercontent.com':
|
||||
raise ValueError('Unexpected avatar host')
|
||||
data = request(url + ('&' if '?' in url else '?') + 's=96', limit=512 * 1024)
|
||||
person['avatarDataUrl'] = raster_data_url(data)
|
||||
except (HTTPError, URLError, TimeoutError, IncompleteRead, ValueError, KeyError) as exc:
|
||||
person.pop('avatarUrl', None)
|
||||
person['avatarDataUrl'] = cached.get(person['handle'].lower(), '')
|
||||
print(f'Avatar fallback for @{person["handle"]}: {type(exc).__name__}', file=sys.stderr)
|
||||
return person
|
||||
|
||||
with ThreadPoolExecutor(max_workers=6) as pool:
|
||||
return list(pool.map(update, people))
|
||||
|
||||
|
||||
def update_history(history, day, stars):
|
||||
date.fromisoformat(day)
|
||||
if type(stars) is not int or stars < 0:
|
||||
raise ValueError('Invalid repository star count')
|
||||
if history is None:
|
||||
history = {'repository': REPOSITORY, 'bootstrap': {'through': day, 'reconstructed': False}, 'points': []}
|
||||
if history['repository'] != REPOSITORY:
|
||||
raise ValueError('Star history belongs to a different repository')
|
||||
date.fromisoformat(history['bootstrap']['through'])
|
||||
dates = []
|
||||
for point in history['points']:
|
||||
date.fromisoformat(point['date'])
|
||||
if type(point['stars']) is not int or point['stars'] < 0:
|
||||
raise ValueError('Invalid historical star count')
|
||||
dates.append(point['date'])
|
||||
if dates != sorted(set(dates)) or any(d > day for d in dates):
|
||||
raise ValueError('History contains duplicate, unordered, or future dates')
|
||||
# Replace today's observation, preserve previous days, and allow unstars.
|
||||
points = [dict(p) for p in history['points'] if p['date'] != day]
|
||||
points.append({'date': day, 'stars': stars})
|
||||
return {**history, 'points': points}
|
||||
|
||||
|
||||
def read_json(path, default=None):
|
||||
return json.loads(path.read_text()) if path.exists() else default
|
||||
|
||||
|
||||
def refresh(output, api, source_root):
|
||||
day = datetime.now(timezone.utc).date().isoformat()
|
||||
metadata = api.get('repos/' + REPOSITORY)
|
||||
if metadata['full_name'].lower() != REPOSITORY:
|
||||
raise ValueError('Unexpected repository metadata')
|
||||
history = update_history(read_json(output / 'history.json'), day, metadata['stargazers_count'])
|
||||
source = api.get(SOURCE)
|
||||
reviewed = yaml.safe_load(base64.b64decode(source['content'], validate=False))
|
||||
curated = select_curated(curated_snapshot(reviewed, source['sha']), read_json(output / 'curated.json'))
|
||||
people = collect_people(api, curated)
|
||||
previous = read_json(output / 'contributors.json', {}).get('people', [])
|
||||
people = add_avatars(api, people, previous)
|
||||
emblem = render.read_emblem(source_root / '.github/silo.svg')
|
||||
payloads = {}
|
||||
for theme in ('light', 'dark'):
|
||||
payloads[f'contributors-{theme}.svg'] = render.contributors(theme, people, day, emblem) + '\n'
|
||||
payloads[f'star-history-{theme}.svg'] = render.stars(theme, history, day, emblem) + '\n'
|
||||
for svg in payloads.values():
|
||||
ET.fromstring(svg)
|
||||
for name, data in {
|
||||
'history.json': history,
|
||||
'curated.json': curated,
|
||||
'contributors.json': {'repository': REPOSITORY, 'updated': day, 'people': people},
|
||||
}.items():
|
||||
payloads[name] = json.dumps(data, indent=2, ensure_ascii=False) + '\n'
|
||||
payloads['README.md'] = f'''# SILO repository cards
|
||||
|
||||
Generated by [Repository Cards](https://github.com/pgsty/silo/actions/workflows/repository-cards.yml)
|
||||
at 00:00 UTC daily (08:00 Asia/Shanghai). GitHub may queue scheduled runs.
|
||||
|
||||
Snapshot: {day}. {metadata['stargazers_count']:,} stars; {len(people)} community contributors.
|
||||
|
||||
- `contributors-light.svg` / `contributors-dark.svg`: human issue and PR authors across the SILO project scope, plus reviewed acknowledgements. Bots are excluded. Gold rings follow the reviewed companion-site roster; new authors are collected automatically.
|
||||
- `star-history-light.svg` / `star-history-dark.svg`: initial history reconstructed from the then-current stargazers; later points are daily observed totals, including decreases. Missing days are not fabricated.
|
||||
- `curated.json`: a cache of reviewed contributor credit from `pgsty/silo.pgsty.com/data/home/contributors.yaml`. The approved initial preview may be newer than the published site; a newer reviewed snapshot is retained until the site catches up.
|
||||
- `contributors.json`: generated contributor data and embedded raster avatars. Failed avatar refreshes use the previous image, or an initial when no image is available.
|
||||
- `history.json`: persistent daily totals. Keep this file when regenerating images.
|
||||
|
||||
The SVGs are self-contained. Source and instructions live on the default branch;
|
||||
this branch contains generated assets only. Do not merge it into `main`.
|
||||
'''
|
||||
# Collect and validate everything before touching the publication checkout.
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
for filename, text in payloads.items():
|
||||
(output / filename).write_text(text)
|
||||
print(f'{day}: {len(people)} contributors; {metadata["stargazers_count"]:,} stars; {len(history["points"])} history points')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
configured = os.environ.get('GITHUB_REPOSITORY', REPOSITORY)
|
||||
if configured.lower() != REPOSITORY:
|
||||
raise SystemExit('This workflow is scoped to pgsty/silo')
|
||||
refresh(args.output, GitHub(os.environ.get('GH_TOKEN', '')), Path(__file__).resolve().parents[2])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -113,7 +113,7 @@ helm_run template my-release "${new_chart}" \
|
||||
go run ./buildscripts/helm-migration-guard "${old_render}" "${new_render}"
|
||||
|
||||
helm_run package "${new_chart}" --destination "${output_dir}" >/dev/null
|
||||
test -s "${work_dir}/silo-7.0.2.tgz"
|
||||
test -s "${work_dir}/silo-7.0.3.tgz"
|
||||
if find "${work_dir}" -maxdepth 1 -type f -name 'minio-*.tgz' | grep -q .; then
|
||||
echo "Helm packaging emitted a legacy MinIO chart name" >&2
|
||||
exit 1
|
||||
|
||||
@@ -249,8 +249,10 @@ func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.R
|
||||
|
||||
// In case of LDAP/OIDC we need to set `opts.claims` to ensure
|
||||
// it is associated with the LDAP/OIDC user properly.
|
||||
// The new credential's policy comes from the request, not the caller.
|
||||
for k, v := range cred.Claims {
|
||||
if k == expClaim {
|
||||
switch k {
|
||||
case expClaim, policy.SessionPolicyName, sessionPolicyNameExtracted, iamPolicyClaimNameSA():
|
||||
continue
|
||||
}
|
||||
opts.claims[k] = v
|
||||
|
||||
@@ -0,0 +1,507 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
polplugin "github.com/minio/minio/internal/config/policy/plugin"
|
||||
"github.com/minio/mux"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
func iamAdminFixture(t *testing.T) (context.Context, *IAMSys) {
|
||||
t.Helper()
|
||||
ctx, sys, obj := prepareIAMRevisionFixture(t)
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = obj
|
||||
globalObjLayerMutex.Unlock()
|
||||
return ctx, sys
|
||||
}
|
||||
|
||||
func iamAdminUser(ctx context.Context, t *testing.T, sys *IAMSys, name, policyName string) auth.Credentials {
|
||||
t.Helper()
|
||||
cred := auth.Credentials{AccessKey: name, SecretKey: "review-user-valid-secret"}
|
||||
_, err := sys.CreateUser(ctx, name, madmin.AddOrUpdateUserReq{SecretKey: cred.SecretKey, Status: madmin.AccountEnabled})
|
||||
mustIAM(t, err)
|
||||
if policyName != "" {
|
||||
_, err = sys.PolicyDBSet(ctx, name, policyName, regUser, false)
|
||||
mustIAM(t, err)
|
||||
}
|
||||
return cred
|
||||
}
|
||||
|
||||
func iamAdminServiceAccount(ctx context.Context, t *testing.T, sys *IAMSys, parent, key, document string) auth.Credentials {
|
||||
t.Helper()
|
||||
var sp *policy.Policy
|
||||
var err error
|
||||
if document != "" {
|
||||
sp, err = policy.ParseConfig(strings.NewReader(document))
|
||||
mustIAM(t, err)
|
||||
}
|
||||
cred, _, err := sys.NewServiceAccount(ctx, parent, nil, newServiceAccountOpts{accessKey: key, secretKey: "review-service-valid-secret", sessionPolicy: sp})
|
||||
mustIAM(t, err)
|
||||
return cred
|
||||
}
|
||||
|
||||
func iamAdminRequest(t *testing.T, cred auth.Credentials, method, target string, payload any) *http.Request {
|
||||
t.Helper()
|
||||
var body []byte
|
||||
if payload != nil {
|
||||
raw, err := json.Marshal(payload)
|
||||
mustIAM(t, err)
|
||||
body, err = madmin.EncryptData(cred.SecretKey, raw)
|
||||
mustIAM(t, err)
|
||||
}
|
||||
headers := map[string]string{}
|
||||
if cred.IsTemp() && !cred.IsServiceAccount() {
|
||||
headers["X-Amz-Security-Token"] = cred.SessionToken
|
||||
}
|
||||
req, err := newTestSignedRequestV4(method, "http://localhost"+target, int64(len(body)), bytes.NewReader(body), cred.AccessKey, cred.SecretKey, headers)
|
||||
mustIAM(t, err)
|
||||
mustIAM(t, req.ParseForm())
|
||||
return req
|
||||
}
|
||||
|
||||
func iamAdminCall(t *testing.T, cred auth.Credentials, method, target string, payload any, vars map[string]string, handler http.HandlerFunc) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
req := iamAdminRequest(t, cred, method, target, payload)
|
||||
req = mux.SetURLVars(req, vars)
|
||||
rec := httptest.NewRecorder()
|
||||
req = req.WithContext(newContext(req, rec, "IAMAdminAuthorization"))
|
||||
handler(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
func assertIAMAdminStatus(t *testing.T, operation string, rec *httptest.ResponseRecorder, want int) {
|
||||
t.Helper()
|
||||
if rec.Code != want {
|
||||
t.Errorf("%s status=%d want=%d body=%s", operation, rec.Code, want, rec.Body.String())
|
||||
return
|
||||
}
|
||||
if want == http.StatusForbidden || want == http.StatusNotFound {
|
||||
var response APIErrorResponse
|
||||
mustIAM(t, json.Unmarshal(rec.Body.Bytes(), &response))
|
||||
wantCode := errorCodes.ToAPIErr(ErrAccessDenied).Code
|
||||
if want == http.StatusNotFound {
|
||||
wantCode = errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound).Code
|
||||
}
|
||||
if response.Code != wantCode {
|
||||
t.Errorf("%s error code=%s want=%s", operation, response.Code, wantCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func iamAdminPlugin(t *testing.T, allow bool) {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]bool{"result": allow})
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
u, err := xnet.ParseHTTPURL(server.URL)
|
||||
mustIAM(t, err)
|
||||
previous := newGlobalAuthZPluginFn()
|
||||
setGlobalAuthZPlugin(polplugin.New(polplugin.Args{URL: u, CloseRespFn: func(r io.ReadCloser) { r.Close() }}))
|
||||
t.Cleanup(func() { setGlobalAuthZPlugin(previous) })
|
||||
}
|
||||
|
||||
func TestIAMAdminRootEmptyPolicy(t *testing.T) {
|
||||
for _, doc := range []string{"", "null", "{}", `{"Statement":null}`} {
|
||||
t.Run("policy="+doc, func(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
caller := iamAdminServiceAccount(ctx, t, sys, globalActiveCred.AccessKey, "review-root-empty", doc)
|
||||
sibling := iamAdminServiceAccount(ctx, t, sys, globalActiveCred.AccessKey, "review-root-sibling", "")
|
||||
req := iamAdminRequest(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil)
|
||||
checked, owner, code := checkKeyValid(req, caller.AccessKey)
|
||||
if code != ErrNone {
|
||||
t.Fatalf("checkKeyValid: %v", code)
|
||||
}
|
||||
active, _ := isAllowedBySessionPolicyForServiceAccount(policy.Args{Claims: checked.Claims})
|
||||
t.Logf("owner=%v parsed_active_session_policy=%v", owner, active)
|
||||
api := adminAPIHandlers{}
|
||||
assertIAMAdminStatus(t, "create", iamAdminCall(t, caller, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{AccessKey: "review-root-new", SecretKey: "review-service-valid-secret"}, nil, api.AddServiceAccount), 200)
|
||||
assertIAMAdminStatus(t, "update", iamAdminCall(t, caller, "POST", "/minio/admin/v3/update-service-account?accessKey="+sibling.AccessKey, madmin.UpdateServiceAccountReq{NewName: "review"}, map[string]string{"accessKey": sibling.AccessKey}, api.UpdateServiceAccount), 204)
|
||||
assertIAMAdminStatus(t, "delete", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+sibling.AccessKey, nil, map[string]string{"accessKey": sibling.AccessKey}, api.DeleteServiceAccount), 204)
|
||||
assertIAMAdminStatus(t, "revoke", iamAdminCall(t, caller, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true", nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 204)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminEmptyPolicyDelegation(t *testing.T) {
|
||||
for _, parentType := range []string{"root", "user"} {
|
||||
for _, document := range []string{"null", "{}", `{"Statement":null}`} {
|
||||
t.Run(parentType+"/"+document, func(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
parent := globalActiveCred
|
||||
if parentType == "user" {
|
||||
parent = iamAdminUser(ctx, t, sys, "empty-policy-parent", "readwrite")
|
||||
}
|
||||
sp, err := policy.ParseConfig(strings.NewReader(document))
|
||||
mustIAM(t, err)
|
||||
caller, _, err := sys.NewServiceAccount(ctx, parent.AccessKey, nil, newServiceAccountOpts{
|
||||
accessKey: "empty-policy-caller", secretKey: "empty-policy-caller-secret", sessionPolicy: sp,
|
||||
claims: map[string]any{"identity-attribute": "preserved"},
|
||||
})
|
||||
mustIAM(t, err)
|
||||
const limitedPolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:ListServiceAccounts"]}]}`
|
||||
for _, tc := range []struct {
|
||||
policy string
|
||||
listStatus int
|
||||
allowObject bool
|
||||
}{
|
||||
{"", http.StatusOK, true},
|
||||
{"null", http.StatusOK, true},
|
||||
{"{}", http.StatusOK, true},
|
||||
{`{"Statement":null}`, http.StatusOK, true},
|
||||
{limitedPolicy, http.StatusOK, false},
|
||||
{`{"Version":"2012-10-17"}`, http.StatusForbidden, false},
|
||||
{`{"Statement":[]}`, http.StatusForbidden, false},
|
||||
} {
|
||||
expires := UTCNow().Add(time.Hour)
|
||||
req := madmin.AddServiceAccountReq{Expiration: &expires}
|
||||
if tc.policy != "" {
|
||||
req.Policy = json.RawMessage(tc.policy)
|
||||
}
|
||||
api := adminAPIHandlers{}
|
||||
rec := iamAdminCall(t, caller, "PUT", "/minio/admin/v3/add-service-account", req, nil, api.AddServiceAccount)
|
||||
assertIAMAdminStatus(t, "create expiring child", rec, http.StatusOK)
|
||||
data, err := madmin.DecryptData(caller.SecretKey, rec.Body)
|
||||
mustIAM(t, err)
|
||||
var response madmin.AddServiceAccountResp
|
||||
mustIAM(t, json.Unmarshal(data, &response))
|
||||
child, ok := sys.store.GetUser(response.Credentials.AccessKey)
|
||||
if !ok {
|
||||
t.Fatal("new service account missing")
|
||||
}
|
||||
// Exercise the issued key through authentication and authorization.
|
||||
assertIAMAdminStatus(t, "use expiring child policy="+tc.policy, iamAdminCall(t, child.Credentials, "GET", "/minio/admin/v3/list-service-accounts", nil, nil, api.ListServiceAccounts), tc.listStatus)
|
||||
signed := iamAdminRequest(t, child.Credentials, "GET", "/example/object", nil)
|
||||
checked, owner, code := checkKeyValid(signed, child.Credentials.AccessKey)
|
||||
if code != ErrNone {
|
||||
t.Fatalf("child authentication: %v", code)
|
||||
}
|
||||
if checked.Claims["identity-attribute"] != "preserved" {
|
||||
t.Fatal("creation lost the caller's identity context")
|
||||
}
|
||||
allowed := sys.IsAllowed(policy.Args{AccountName: checked.AccessKey, Claims: checked.Claims, IsOwner: owner, Action: policy.GetObjectAction, BucketName: "example", ObjectName: "object"})
|
||||
if allowed != tc.allowObject {
|
||||
t.Errorf("GetObject allowed=%v with requested policy=%s", allowed, tc.policy)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminCorruptServiceCleanup(t *testing.T) {
|
||||
for _, status := range []string{auth.AccountOn, auth.AccountOff, "expired"} {
|
||||
t.Run(status, func(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
caller := iamAdminUser(ctx, t, sys, "review-admin", "consoleAdmin")
|
||||
parent := iamAdminUser(ctx, t, sys, "review-parent", "readwrite")
|
||||
svc := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "rv-corrupt", "")
|
||||
if status == auth.AccountOff {
|
||||
_, err := sys.UpdateServiceAccount(ctx, svc.AccessKey, updateServiceAccountOpts{status: status})
|
||||
mustIAM(t, err)
|
||||
}
|
||||
cache := sys.store.lock()
|
||||
u := cache.iamUsersMap[svc.AccessKey]
|
||||
u.Credentials.SessionToken = "broken-session-token"
|
||||
if status == "expired" {
|
||||
u.Credentials.Expiration = UTCNow().Add(-time.Minute)
|
||||
}
|
||||
cache.iamUsersMap[svc.AccessKey] = u
|
||||
sys.store.unlock()
|
||||
_, _, err := sys.GetServiceAccount(ctx, svc.AccessKey)
|
||||
if err == nil {
|
||||
t.Fatal("fault injection failed")
|
||||
}
|
||||
raw, exists := sys.store.GetUser(svc.AccessKey)
|
||||
_, loginOK := sys.GetUser(ctx, svc.AccessKey)
|
||||
t.Logf("metadata_exists=%v parent_known=%v GetUser_ok=%v", exists, raw.Credentials.ParentUser == parent.AccessKey, loginOK)
|
||||
api := adminAPIHandlers{}
|
||||
outsider := iamAdminUser(ctx, t, sys, "cleanup-outsider", "readwrite")
|
||||
assertIAMAdminStatus(t, "unauthorized cleanup", iamAdminCall(t, outsider, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.DeleteServiceAccount), 404)
|
||||
assertIAMAdminStatus(t, "admin cleanup", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.DeleteServiceAccount), 204)
|
||||
if _, ok := sys.store.GetUser(svc.AccessKey); ok {
|
||||
t.Fatal("deleted service account remains in the IAM cache")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminPolicylessSelf(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
caller := iamAdminUser(ctx, t, sys, "review-policyless", "")
|
||||
svc := iamAdminServiceAccount(ctx, t, sys, caller.AccessKey, "rv-policyless-child", "")
|
||||
api := adminAPIHandlers{}
|
||||
assertIAMAdminStatus(t, "list", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil, nil, api.ListServiceAccounts), 200)
|
||||
assertIAMAdminStatus(t, "info service", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.InfoServiceAccount), 200)
|
||||
assertIAMAdminStatus(t, "info access key", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-access-key?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.InfoAccessKey), 200)
|
||||
assertIAMAdminStatus(t, "delete", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.DeleteServiceAccount), 204)
|
||||
assertIAMAdminStatus(t, "revoke", iamAdminCall(t, caller, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true", nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 204)
|
||||
assertIAMAdminStatus(t, "list another user", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts?user="+globalActiveCred.AccessKey, nil, nil, api.ListServiceAccounts), 403)
|
||||
assertIAMAdminStatus(t, "revoke another user", iamAdminCall(t, caller, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true&user="+globalActiveCred.AccessKey, nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 403)
|
||||
}
|
||||
|
||||
func TestIAMAdminExpiringInheritedPolicy(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
parent := iamAdminUser(ctx, t, sys, "review-legacy-parent", "consoleAdmin")
|
||||
document := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:CreateServiceAccount","admin:UpdateServiceAccount"]}]}`
|
||||
// Model a credential persisted by the old handler: it copied its caller's
|
||||
// policy claims but set the newly created service's type to inherited.
|
||||
expires := UTCNow().Add(time.Hour)
|
||||
caller, _, err := sys.NewServiceAccount(ctx, parent.AccessKey, nil, newServiceAccountOpts{
|
||||
accessKey: "rv-exp-inherited", secretKey: "review-service-valid-secret", expiration: &expires,
|
||||
claims: map[string]any{policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(document))},
|
||||
})
|
||||
mustIAM(t, err)
|
||||
req := iamAdminRequest(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil)
|
||||
checked, owner, code := checkKeyValid(req, caller.AccessKey)
|
||||
if code != ErrNone {
|
||||
t.Fatalf("checkKeyValid: %v", code)
|
||||
}
|
||||
args := policy.Args{AccountName: caller.AccessKey, Claims: checked.Claims, IsOwner: owner, Action: policy.CreateServiceAccountAdminAction, DenyOnly: true}
|
||||
if !sys.IsAllowed(args) {
|
||||
t.Fatal("fixture cannot create")
|
||||
}
|
||||
args.Action = policy.CreateUserAdminAction
|
||||
args.DenyOnly = false
|
||||
if sys.IsAllowed(args) {
|
||||
t.Fatal("fixture not constrained")
|
||||
}
|
||||
t.Logf("service=%v temp=%v inherited=%v; create allowed, create-user denied", checked.IsServiceAccount(), checked.IsTemp(), checked.Claims[iamPolicyClaimNameSA()] == inheritedPolicyType)
|
||||
api := adminAPIHandlers{}
|
||||
rec := iamAdminCall(t, caller, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{AccessKey: "review-escaped", SecretKey: "review-service-valid-secret"}, nil, api.AddServiceAccount)
|
||||
if rec.Code == 200 {
|
||||
child, ok := sys.GetUser(ctx, "review-escaped")
|
||||
if !ok {
|
||||
t.Fatal("new key missing")
|
||||
}
|
||||
childReq := iamAdminRequest(t, child.Credentials, "GET", "/minio/admin/v3/list-service-accounts", nil)
|
||||
childChecked, childOwner, childCode := checkKeyValid(childReq, child.Credentials.AccessKey)
|
||||
if childCode != ErrNone {
|
||||
t.Fatalf("child auth=%v", childCode)
|
||||
}
|
||||
allowed := sys.IsAllowed(policy.Args{AccountName: childChecked.AccessKey, Claims: childChecked.Claims, IsOwner: childOwner, Action: policy.CreateUserAdminAction})
|
||||
t.Errorf("constrained inherited service created a new key; new key CreateUser allowed=%v", allowed)
|
||||
} else {
|
||||
assertIAMAdminStatus(t, "reject derived creation", rec, 403)
|
||||
}
|
||||
updated := iamAdminCall(t, caller, "POST", "/minio/admin/v3/update-service-account?accessKey="+caller.AccessKey, madmin.UpdateServiceAccountReq{NewName: "metadata-only"}, map[string]string{"accessKey": caller.AccessKey}, api.UpdateServiceAccount)
|
||||
if updated.Code == 204 {
|
||||
after, ok := sys.GetUser(ctx, caller.AccessKey)
|
||||
if !ok {
|
||||
t.Fatal("updated key missing")
|
||||
}
|
||||
afterReq := iamAdminRequest(t, after.Credentials, "GET", "/minio/admin/v3/list-service-accounts", nil)
|
||||
afterChecked, afterOwner, afterCode := checkKeyValid(afterReq, caller.AccessKey)
|
||||
if afterCode != ErrNone {
|
||||
t.Fatalf("updated auth=%v", afterCode)
|
||||
}
|
||||
allowed := sys.IsAllowed(policy.Args{AccountName: caller.AccessKey, Claims: afterChecked.Claims, IsOwner: afterOwner, Action: policy.CreateUserAdminAction})
|
||||
t.Errorf("constrained inherited service updated itself; CreateUser allowed after update=%v", allowed)
|
||||
} else {
|
||||
assertIAMAdminStatus(t, "reject inherited self update", updated, 403)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminSessionPolicyCannotDelegate(t *testing.T) {
|
||||
for _, temp := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "service", true: "sts"}[temp], func(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
parent := iamAdminUser(ctx, t, sys, "rv-parent", "consoleAdmin")
|
||||
doc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:CreateServiceAccount","admin:UpdateServiceAccount"]}]}`
|
||||
var caller auth.Credentials
|
||||
if !temp {
|
||||
caller = iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "review-restricted", doc)
|
||||
} else {
|
||||
var err error
|
||||
caller, err = auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(), policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(doc))}, globalActiveCred.SecretKey)
|
||||
mustIAM(t, err)
|
||||
caller.ParentUser = parent.AccessKey
|
||||
_, err = sys.SetTempUser(ctx, caller.AccessKey, caller, "")
|
||||
mustIAM(t, err)
|
||||
}
|
||||
sibling := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "rv-sibling", "")
|
||||
api := adminAPIHandlers{}
|
||||
assertIAMAdminStatus(t, "restrict create", iamAdminCall(t, caller, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{AccessKey: "review-bad-new", SecretKey: "review-service-valid-secret"}, nil, api.AddServiceAccount), 403)
|
||||
assertIAMAdminStatus(t, "restrict LDAP create", iamAdminCall(t, caller, "PUT", "/minio/admin/v3/idp/ldap/add-service-account", madmin.AddServiceAccountReq{AccessKey: "review-bad-new", SecretKey: "review-service-valid-secret"}, nil, api.AddServiceAccountLDAP), 403)
|
||||
assertIAMAdminStatus(t, "restrict update", iamAdminCall(t, caller, "POST", "/minio/admin/v3/update-service-account?accessKey="+sibling.AccessKey, madmin.UpdateServiceAccountReq{NewSecretKey: "review-known-new-secret"}, map[string]string{"accessKey": sibling.AccessKey}, api.UpdateServiceAccount), 403)
|
||||
if _, ok := sys.store.GetUser("review-bad-new"); ok {
|
||||
t.Fatal("denied creation persisted a credential")
|
||||
}
|
||||
u, ok := sys.store.GetUser(sibling.AccessKey)
|
||||
if !ok || u.Credentials.SecretKey != sibling.SecretKey {
|
||||
t.Fatal("denied update changed the target credential")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminSelfDenials(t *testing.T) {
|
||||
for _, mode := range []string{"parent_deny", "group_deny", "session_implicit_deny", "session_explicit_deny", "plugin_deny"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
parent := iamAdminUser(ctx, t, sys, "rv-self-parent", "readwrite")
|
||||
target := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "rv-self-target", "")
|
||||
caller := parent
|
||||
doc := `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":["admin:ListServiceAccounts","admin:RemoveServiceAccount"]}]}`
|
||||
switch mode {
|
||||
case "parent_deny", "group_deny":
|
||||
p, err := policy.ParseConfig(strings.NewReader(doc))
|
||||
mustIAM(t, err)
|
||||
_, err = sys.SetPolicy(ctx, "review-self-deny", *p)
|
||||
mustIAM(t, err)
|
||||
if mode == "group_deny" {
|
||||
_, err = sys.AddUsersToGroup(ctx, "self-deny-group", []string{parent.AccessKey})
|
||||
mustIAM(t, err)
|
||||
_, err = sys.PolicyDBSet(ctx, "self-deny-group", "review-self-deny", regUser, true)
|
||||
} else {
|
||||
_, err = sys.PolicyDBSet(ctx, parent.AccessKey, "readwrite,review-self-deny", regUser, false)
|
||||
}
|
||||
mustIAM(t, err)
|
||||
case "session_implicit_deny":
|
||||
caller = iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "rv-self-caller", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::example/*"]}]}`)
|
||||
case "session_explicit_deny":
|
||||
caller = iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "rv-self-caller", doc)
|
||||
case "plugin_deny":
|
||||
iamAdminPlugin(t, false)
|
||||
}
|
||||
api := adminAPIHandlers{}
|
||||
assertIAMAdminStatus(t, "list", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil, nil, api.ListServiceAccounts), 403)
|
||||
assertIAMAdminStatus(t, "info service", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.InfoServiceAccount), 403)
|
||||
assertIAMAdminStatus(t, "info access key", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-access-key?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.InfoAccessKey), 403)
|
||||
assertIAMAdminStatus(t, "delete", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.DeleteServiceAccount), 403)
|
||||
assertIAMAdminStatus(t, "revoke", iamAdminCall(t, caller, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true", nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 403)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminNonRootCannotManageRoot(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
caller := iamAdminUser(ctx, t, sys, "rv-console-admin", "consoleAdmin")
|
||||
target := iamAdminServiceAccount(ctx, t, sys, globalActiveCred.AccessKey, "rv-root-target", "")
|
||||
api := adminAPIHandlers{}
|
||||
assertIAMAdminStatus(t, "create root", iamAdminCall(t, caller, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{AccessKey: "rv-new-root", SecretKey: "review-service-valid-secret", TargetUser: globalActiveCred.AccessKey}, nil, api.AddServiceAccount), 403)
|
||||
assertIAMAdminStatus(t, "update root", iamAdminCall(t, caller, "POST", "/minio/admin/v3/update-service-account?accessKey="+target.AccessKey, madmin.UpdateServiceAccountReq{NewSecretKey: "review-known-new-secret"}, map[string]string{"accessKey": target.AccessKey}, api.UpdateServiceAccount), 403)
|
||||
assertIAMAdminStatus(t, "delete root", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.DeleteServiceAccount), 404)
|
||||
assertIAMAdminStatus(t, "revoke root", iamAdminCall(t, caller, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true&user="+globalActiveCred.AccessKey, nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 403)
|
||||
// Restricted root-derived credentials and the site replicator are not root
|
||||
// administrators, even when their effective policy allows these actions.
|
||||
limited := iamAdminServiceAccount(ctx, t, sys, globalActiveCred.AccessKey, "limited-root", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:*"]}]}`)
|
||||
assertIAMAdminStatus(t, "limited root delete", iamAdminCall(t, limited, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.DeleteServiceAccount), 404)
|
||||
assertIAMAdminStatus(t, "limited root revoke", iamAdminCall(t, limited, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true", nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 403)
|
||||
replicator, _, err := sys.NewServiceAccount(ctx, globalActiveCred.AccessKey, nil, newServiceAccountOpts{accessKey: siteReplicatorSvcAcc, secretKey: "replicator-valid-secret", allowSiteReplicatorAccount: true})
|
||||
mustIAM(t, err)
|
||||
assertIAMAdminStatus(t, "replicator create root", iamAdminCall(t, replicator, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{}, nil, api.AddServiceAccount), 403)
|
||||
|
||||
// An external plugin remains authoritative, including for root targets.
|
||||
iamAdminPlugin(t, true)
|
||||
assertIAMAdminStatus(t, "plugin update root", iamAdminCall(t, limited, "POST", "/minio/admin/v3/update-service-account?accessKey="+target.AccessKey, madmin.UpdateServiceAccountReq{NewName: "plugin-authorized"}, map[string]string{"accessKey": target.AccessKey}, api.UpdateServiceAccount), 204)
|
||||
assertIAMAdminStatus(t, "plugin create root", iamAdminCall(t, limited, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{}, nil, api.AddServiceAccount), 200)
|
||||
}
|
||||
|
||||
func TestIAMAdminDeleteCorruptServiceReplicates(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
caller := iamAdminUser(ctx, t, sys, "cleanup-admin", "consoleAdmin")
|
||||
parent := iamAdminUser(ctx, t, sys, "cleanup-parent", "readwrite")
|
||||
target := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "cleanup-target", "")
|
||||
cache := sys.store.lock()
|
||||
u := cache.iamUsersMap[target.AccessKey]
|
||||
u.Credentials.SessionToken = "broken-session-token"
|
||||
u.Credentials.Status = auth.AccountOff
|
||||
cache.iamUsersMap[target.AccessKey] = u
|
||||
sys.store.unlock()
|
||||
|
||||
var deletions atomic.Int32
|
||||
peer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/minio/health/live" {
|
||||
return
|
||||
}
|
||||
var batch iamRevisionBatch
|
||||
if err := json.NewDecoder(r.Body).Decode(&batch); err != nil {
|
||||
t.Error(err)
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if len(batch.Items) != 1 || batch.Items[0].SvcAccChange == nil || batch.Items[0].SvcAccChange.Delete == nil || batch.Items[0].SvcAccChange.Delete.AccessKey != target.AccessKey {
|
||||
t.Error("replication did not carry the service account deletion")
|
||||
} else {
|
||||
deletions.Add(1)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: "peer", Instance: "peer-boot", Digest: "deleted"}})
|
||||
}))
|
||||
t.Cleanup(peer.Close)
|
||||
globalSiteReplicatorCred.RLock()
|
||||
previousSecret := globalSiteReplicatorCred.secretKey
|
||||
globalSiteReplicatorCred.RUnlock()
|
||||
globalSiteReplicatorCred.Set(caller.SecretKey)
|
||||
t.Cleanup(func() { globalSiteReplicatorCred.Set(previousSecret) })
|
||||
globalSiteReplicationSys.Lock()
|
||||
globalSiteReplicationSys.enabled = true
|
||||
globalSiteReplicationSys.state = srState{ServiceAccountAccessKey: caller.AccessKey, Peers: map[string]madmin.PeerInfo{
|
||||
globalDeploymentID(): {DeploymentID: globalDeploymentID(), Name: "local"},
|
||||
"peer": {DeploymentID: "peer", Name: "peer", Endpoint: peer.URL},
|
||||
}}
|
||||
globalSiteReplicationSys.Unlock()
|
||||
t.Cleanup(func() {
|
||||
globalSiteReplicationSys.Lock()
|
||||
globalSiteReplicationSys.enabled = false
|
||||
globalSiteReplicationSys.Unlock()
|
||||
})
|
||||
api := adminAPIHandlers{}
|
||||
assertIAMAdminStatus(t, "replicated cleanup", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.DeleteServiceAccount), 204)
|
||||
if deletions.Load() != 1 {
|
||||
t.Fatalf("expected one immediate deletion notification, got %d", deletions.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminListSelfAliases(t *testing.T) {
|
||||
for _, kind := range []string{"service", "sts"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
parent := iamAdminUser(ctx, t, sys, "alias-parent", "readwrite")
|
||||
target := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "alias-target", "")
|
||||
caller := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "alias-caller", "")
|
||||
if kind == "sts" {
|
||||
var err error
|
||||
caller, err = auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix()}, globalActiveCred.SecretKey)
|
||||
mustIAM(t, err)
|
||||
caller.ParentUser = parent.AccessKey
|
||||
_, err = sys.SetTempUser(ctx, caller.AccessKey, caller, "")
|
||||
mustIAM(t, err)
|
||||
}
|
||||
api := adminAPIHandlers{}
|
||||
for _, alias := range []string{"", caller.AccessKey, parent.AccessKey} {
|
||||
rec := iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts?user="+alias, nil, nil, api.ListServiceAccounts)
|
||||
assertIAMAdminStatus(t, "list self alias", rec, 200)
|
||||
data, err := madmin.DecryptData(caller.SecretKey, rec.Body)
|
||||
mustIAM(t, err)
|
||||
var response madmin.ListServiceAccountsResp
|
||||
mustIAM(t, json.Unmarshal(data, &response))
|
||||
found := false
|
||||
for _, account := range response.Accounts {
|
||||
found = found || account.AccessKey == target.AccessKey
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("self alias %q did not list the parent's service accounts", alias)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+139
-82
@@ -714,8 +714,10 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
// In case of LDAP/OIDC we need to set `opts.claims` to ensure
|
||||
// it is associated with the LDAP/OIDC user properly.
|
||||
// The new credential's policy comes from the request, not the caller.
|
||||
for k, v := range cred.Claims {
|
||||
if k == expClaim {
|
||||
switch k {
|
||||
case expClaim, policy.SessionPolicyName, sessionPolicyNameExtracted, iamPolicyClaimNameSA():
|
||||
continue
|
||||
}
|
||||
opts.claims[k] = v
|
||||
@@ -855,10 +857,9 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
|
||||
// Permission checks:
|
||||
//
|
||||
// 1. Any type of account (i.e. access keys (previously/still called service
|
||||
// accounts), STS accounts, internal IDP accounts, etc) with the
|
||||
// policy.UpdateServiceAccountAdminAction permission can update any service
|
||||
// account.
|
||||
// 1. Updating requires policy.UpdateServiceAccountAdminAction. Under built-in
|
||||
// IAM, session-limited callers cannot update credentials, and only unrestricted
|
||||
// root credentials can manage root's service accounts.
|
||||
//
|
||||
// 2. We would like to let a user update their own access keys, however it
|
||||
// is currently blocked pending a re-design. Users are still able to delete
|
||||
@@ -874,6 +875,11 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
if newGlobalAuthZPluginFn() == nil && (hasActiveSessionPolicy(cred) ||
|
||||
(svcAccount.ParentUser == globalActiveCred.AccessKey && !canManageRootCredentials(cred))) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var sp *policy.Policy
|
||||
if len(updateReq.NewPolicy) > 0 {
|
||||
@@ -951,23 +957,21 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
requestUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
requestUser = cred.ParentUser
|
||||
}
|
||||
if !isAllowedServiceAccountAction(cred, policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: requestUser == svcAccount.ParentUser,
|
||||
}) {
|
||||
requestUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
requestUser = cred.ParentUser
|
||||
}
|
||||
|
||||
if requestUser != svcAccount.ParentUser {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// if session policy is nil or empty, then it is implied policy
|
||||
@@ -1039,29 +1043,27 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
|
||||
return
|
||||
}
|
||||
|
||||
var targetAccount string
|
||||
|
||||
// If listing is requested for a specific user (who is not the request
|
||||
// sender), check that the user has permissions.
|
||||
selfUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
selfUser = cred.ParentUser
|
||||
}
|
||||
user := r.Form.Get("user")
|
||||
if user != "" && user != cred.AccessKey {
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
targetAccount = user
|
||||
} else {
|
||||
targetAccount = cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
targetAccount = cred.ParentUser
|
||||
}
|
||||
selfOnly := user == "" || user == cred.AccessKey || user == selfUser
|
||||
if !isAllowedServiceAccountAction(cred, policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: selfOnly,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
targetAccount := user
|
||||
if selfOnly {
|
||||
targetAccount = selfUser
|
||||
}
|
||||
|
||||
serviceAccounts, err := globalIAMSys.ListServiceAccounts(ctx, targetAccount)
|
||||
@@ -1110,7 +1112,7 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil || globalNotificationSys == nil {
|
||||
if objectAPI == nil || globalNotificationSys == nil || !globalIAMSys.Initialized() {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -1131,35 +1133,41 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidArgument), r.URL)
|
||||
return
|
||||
}
|
||||
// We do not care if service account is readable or not at this point,
|
||||
// since this is a delete call we shall allow it to be deleted if possible.
|
||||
svcAccount, _, err := globalIAMSys.GetServiceAccount(ctx, serviceAccount)
|
||||
if errors.Is(err, errNoSuchServiceAccount) {
|
||||
// Deletion needs the stored identity, not a valid token or session policy.
|
||||
// Keep disabled, expired and damaged service accounts removable, and retain
|
||||
// their parent for authorization and the site replication hook below.
|
||||
u, ok := globalIAMSys.store.GetUser(serviceAccount)
|
||||
if !ok || !u.Credentials.IsServiceAccount() {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound), r.URL)
|
||||
return
|
||||
}
|
||||
svcAccount := u.Credentials
|
||||
|
||||
adminPrivilege := globalIAMSys.IsAllowed(policy.Args{
|
||||
parentUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
parentUser = cred.ParentUser
|
||||
}
|
||||
isSelf := parentUser == svcAccount.ParentUser
|
||||
if !isAllowedServiceAccountAction(cred, policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.RemoveServiceAccountAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
})
|
||||
|
||||
if !adminPrivilege {
|
||||
parentUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
parentUser = cred.ParentUser
|
||||
}
|
||||
if svcAccount.ParentUser != "" && parentUser != svcAccount.ParentUser {
|
||||
// The service account belongs to another user but return not
|
||||
// found error to mitigate brute force attacks. or the
|
||||
// serviceAccount doesn't exist.
|
||||
DenyOnly: isSelf,
|
||||
}) {
|
||||
if !isSelf {
|
||||
// Hide the existence of another user's service account.
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound), r.URL)
|
||||
return
|
||||
}
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
if newGlobalAuthZPluginFn() == nil && svcAccount.ParentUser == globalActiveCred.AccessKey && !canManageRootCredentials(cred) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err := globalIAMSys.DeleteServiceAccount(ctx, serviceAccount, true); err != nil {
|
||||
@@ -2046,26 +2054,30 @@ func (a adminAPIHandlers) RevokeTokens(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
adminPrivilege := globalIAMSys.IsAllowed(policy.Args{
|
||||
parentUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
parentUser = cred.ParentUser
|
||||
}
|
||||
isSelfTarget := isTokenSelfRevoke || user == parentUser
|
||||
if !isAllowedServiceAccountAction(cred, policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.RemoveServiceAccountAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
})
|
||||
|
||||
if !adminPrivilege || isTokenSelfRevoke {
|
||||
parentUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
parentUser = cred.ParentUser
|
||||
}
|
||||
if !isTokenSelfRevoke && user != parentUser {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
DenyOnly: isSelfTarget,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
if isSelfTarget {
|
||||
user = parentUser
|
||||
}
|
||||
if newGlobalAuthZPluginFn() == nil && user == globalActiveCred.AccessKey && !canManageRootCredentials(cred) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Infer token revoke type from the request if requestor is STS.
|
||||
if isTokenSelfRevoke && tokenRevokeType == "" && !fullRevoke {
|
||||
@@ -2112,29 +2124,22 @@ func (a adminAPIHandlers) InfoAccessKey(w http.ResponseWriter, r *http.Request)
|
||||
u, ok := globalIAMSys.GetUser(ctx, accessKey)
|
||||
targetCred := u.Credentials
|
||||
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
requestUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
requestUser = cred.ParentUser
|
||||
}
|
||||
isSelf := ok && targetCred.ParentUser != "" && requestUser == targetCred.ParentUser
|
||||
if !isAllowedServiceAccountAction(cred, policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: isSelf,
|
||||
}) {
|
||||
// If requested user does not exist and requestor is not allowed to list service accounts, return access denied.
|
||||
if !ok {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
requestUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
requestUser = cred.ParentUser
|
||||
}
|
||||
|
||||
if requestUser != targetCred.ParentUser {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if !ok {
|
||||
@@ -2908,6 +2913,44 @@ func addExpirationToCondValues(exp *time.Time, condValues map[string][]string) e
|
||||
return nil
|
||||
}
|
||||
|
||||
// hasActiveSessionPolicy follows the same credential dispatch as IAMSys.IsAllowed.
|
||||
// Expiring service accounts also satisfy IsTemp and are evaluated as STS there.
|
||||
// Callers use this only after authorization under built-in IAM.
|
||||
func hasActiveSessionPolicy(cred auth.Credentials) bool {
|
||||
args := policy.Args{Claims: cred.Claims}
|
||||
if cred.IsTemp() {
|
||||
hasSP, _ := isAllowedBySessionPolicy(args)
|
||||
return hasSP
|
||||
}
|
||||
if cred.IsServiceAccount() {
|
||||
if cred.Claims[iamPolicyClaimNameSA()] == inheritedPolicyType {
|
||||
return false
|
||||
}
|
||||
hasSP, _ := isAllowedBySessionPolicyForServiceAccount(args)
|
||||
return hasSP
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// canManageRootCredentials is an additional guard after built-in IAM authorization.
|
||||
// The signature's owner flag is insufficient: historical empty session policies
|
||||
// clear it even though the service account still inherits root's permissions.
|
||||
func canManageRootCredentials(cred auth.Credentials) bool {
|
||||
return cred.AccessKey == globalActiveCred.AccessKey ||
|
||||
(cred.ParentUser == globalActiveCred.AccessKey && cred.AccessKey != siteReplicatorSvcAcc && !hasActiveSessionPolicy(cred))
|
||||
}
|
||||
|
||||
// isAllowedServiceAccountAction preserves regular users' self inspection and
|
||||
// cleanup even without a policy attachment, while enforcing explicit denies.
|
||||
// Derived credentials and external authorization plugins use full IAM evaluation.
|
||||
func isAllowedServiceAccountAction(cred auth.Credentials, args policy.Args) bool {
|
||||
if args.DenyOnly && !args.IsOwner && !cred.IsTemp() && !cred.IsServiceAccount() && newGlobalAuthZPluginFn() == nil {
|
||||
policies, err := globalIAMSys.PolicyDBGet(args.AccountName, args.Groups...)
|
||||
return err == nil && globalIAMSys.GetCombinedPolicy(policies...).IsAllowed(args)
|
||||
}
|
||||
return globalIAMSys.IsAllowed(args)
|
||||
}
|
||||
|
||||
func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.Credentials, newServiceAccountOpts, madmin.AddServiceAccountReq, string, APIError) {
|
||||
ctx := r.Context()
|
||||
|
||||
@@ -2996,6 +3039,14 @@ func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.
|
||||
}) {
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAccessDenied)
|
||||
}
|
||||
effectiveTarget := targetUser
|
||||
if (targetUser == cred.AccessKey || targetUser == cred.ParentUser) && cred.ParentUser != "" {
|
||||
effectiveTarget = cred.ParentUser
|
||||
}
|
||||
if newGlobalAuthZPluginFn() == nil && (hasActiveSessionPolicy(cred) ||
|
||||
(effectiveTarget == globalActiveCred.AccessKey && !canManageRootCredentials(cred))) {
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAccessDenied)
|
||||
}
|
||||
|
||||
var sp *policy.Policy
|
||||
if len(createReq.Policy) > 0 {
|
||||
@@ -3003,6 +3054,12 @@ func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.
|
||||
if err != nil {
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err)
|
||||
}
|
||||
// Empty service policies mean inheritance, including for expiring keys.
|
||||
// Normalize the replication request as well as the local credential.
|
||||
if sp.Version == "" && sp.Statements == nil && sp.ID == "" {
|
||||
sp = nil
|
||||
createReq.Policy = nil
|
||||
}
|
||||
}
|
||||
|
||||
opts.sessionPolicy = sp
|
||||
|
||||
@@ -163,6 +163,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
|
||||
// StorageInfo operations
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(adminMiddleware(adminAPI.StorageInfoHandler, traceAllFlag))
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/multipart-preflight").HandlerFunc(adminMiddleware(adminAPI.MultipartPreflightHandler, traceAllFlag))
|
||||
// DataUsageInfo operations
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(adminMiddleware(adminAPI.DataUsageInfoHandler, traceAllFlag))
|
||||
// Metrics operation
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
//
|
||||
// This file is part of SILO Object Storage.
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/hash"
|
||||
)
|
||||
|
||||
func TestChecksumMismatchAPIMessage(t *testing.T) {
|
||||
for _, algorithm := range []string{"CRC32", "CRC32C", "SHA1", "SHA256", "CRC64NVME", ""} {
|
||||
t.Run(algorithm, func(t *testing.T) {
|
||||
mismatch := hash.ChecksumMismatch{Algorithm: algorithm, Want: "expected", Got: "actual"}
|
||||
want := errorCodes.ToAPIErr(ErrContentChecksumMismatch).Description
|
||||
if algorithm != "" {
|
||||
want = fmt.Sprintf("The %s you specified did not match the calculated checksum.", algorithm)
|
||||
}
|
||||
for _, err := range []error{mismatch, fmt.Errorf("upload: %w", mismatch)} {
|
||||
got := toAPIError(t.Context(), err)
|
||||
if got.HTTPStatusCode != http.StatusBadRequest || got.Code != "XAmzContentChecksumMismatch" || got.Description != want {
|
||||
t.Fatalf("toAPIError(%v) = %+v, want 400 XAmzContentChecksumMismatch: %s", err, got, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+31
-1
@@ -450,6 +450,9 @@ const (
|
||||
ErrAdminNoSecretKey
|
||||
|
||||
ErrIAMNotInitialized
|
||||
ErrMultipartListingLegacy // Reserved: legacy uploads no longer block listing.
|
||||
ErrMultipartListingIdentity
|
||||
ErrSlowDown
|
||||
|
||||
apiErrCodeEnd // This is used only for the testing code
|
||||
)
|
||||
@@ -547,7 +550,7 @@ var errorCodes = errorCodeMap{
|
||||
},
|
||||
ErrBadDigest: {
|
||||
Code: "BadDigest",
|
||||
Description: "The Content-Md5 you specified did not match what we received.",
|
||||
Description: "The Content-MD5 you specified did not match what we received.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrEntityTooSmall: {
|
||||
@@ -1336,6 +1339,21 @@ var errorCodes = errorCodeMap{
|
||||
Description: "IAM sub-system not initialized yet, please try again.",
|
||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||
},
|
||||
ErrMultipartListingLegacy: {
|
||||
Code: "MultipartListingNotReady",
|
||||
Description: "Legacy multipart uploads prevent a complete listing. Upgrade all writers, drain old uploads and run the multipart preflight check.",
|
||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||
},
|
||||
ErrMultipartListingIdentity: {
|
||||
Code: "MultipartListingMetadataInvalid",
|
||||
Description: "Multipart upload metadata is inconsistent. Run the multipart preflight check to locate the affected storage set.",
|
||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||
},
|
||||
ErrSlowDown: {
|
||||
Code: "SlowDown",
|
||||
Description: "Please reduce your request rate",
|
||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||
},
|
||||
ErrBucketMetadataNotInitialized: {
|
||||
Code: "XMinioBucketMetadataNotInitialized",
|
||||
Description: "Bucket metadata not initialized yet, please try again.",
|
||||
@@ -2173,6 +2191,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
err = unwrapAll(err)
|
||||
|
||||
switch err {
|
||||
case errMultipartListingIdentity:
|
||||
apiErr = ErrMultipartListingIdentity
|
||||
case errCompleteMultipartChecksumMismatch, errCompleteMultipartChecksumTypeMismatch:
|
||||
apiErr = ErrBadDigest
|
||||
case errMissingPartChecksum:
|
||||
@@ -2303,6 +2323,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
}
|
||||
|
||||
switch err.(type) {
|
||||
case SlowDown:
|
||||
apiErr = ErrSlowDown
|
||||
case StorageFull:
|
||||
apiErr = ErrStorageFull
|
||||
case hash.BadDigest:
|
||||
@@ -2481,6 +2503,14 @@ func toAPIError(ctx context.Context, err error) APIError {
|
||||
case errors.Is(err, errMissingPartChecksum):
|
||||
apiErr.Description = strings.TrimPrefix(err.Error(), errMissingPartChecksum.Error()+": ")
|
||||
}
|
||||
// PutObject/UploadPart checksum mismatches name the algorithm in the
|
||||
// message, matching S3 (e.g. "The CRC32 you specified did not match the
|
||||
// calculated checksum."). Falls through to the generic table description
|
||||
// when the algorithm is unknown.
|
||||
var checksumMismatch hash.ChecksumMismatch
|
||||
if errors.As(err, &checksumMismatch) && checksumMismatch.Algorithm != "" {
|
||||
apiErr.Description = fmt.Sprintf("The %s you specified did not match the calculated checksum.", checksumMismatch.Algorithm)
|
||||
}
|
||||
switch apiErr.Code {
|
||||
case "NotImplemented":
|
||||
apiErr = APIError{
|
||||
|
||||
+1
-1
@@ -41,7 +41,7 @@ import (
|
||||
const (
|
||||
maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
|
||||
maxDeleteList = 1000 // Limit number of objects deleted in a delete call.
|
||||
maxUploadsList = 10000 // Limit number of uploads in a listUploadsResponse.
|
||||
maxUploadsList = 1000 // Limit number of uploads in a listUploadsResponse.
|
||||
maxPartsList = 10000 // Limit number of parts in a listPartsResponse.
|
||||
)
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -28,13 +28,10 @@ import (
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
"github.com/minio/minio/internal/etag"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
@@ -733,48 +730,6 @@ func setAuthMiddleware(h http.Handler) http.Handler {
|
||||
})
|
||||
}
|
||||
|
||||
func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate time.Time, retMode objectlock.RetMode, byPassSet bool, r *http.Request, cred auth.Credentials, owner bool) (s3Err APIErrorCode) {
|
||||
var retSet bool
|
||||
if cred.AccessKey == "" {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
conditions := getConditionValues(r, "", cred)
|
||||
conditions["object-lock-mode"] = []string{string(retMode)}
|
||||
conditions["object-lock-retain-until-date"] = []string{retDate.UTC().Format(time.RFC3339)}
|
||||
if retDays > 0 {
|
||||
conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(retDays)}
|
||||
}
|
||||
if retMode == objectlock.RetGovernance && byPassSet {
|
||||
byPassSet = globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.BypassGovernanceRetentionAction,
|
||||
BucketName: bucketName,
|
||||
ObjectName: objectName,
|
||||
ConditionValues: conditions,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
})
|
||||
}
|
||||
if globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.PutObjectRetentionAction,
|
||||
BucketName: bucketName,
|
||||
ConditionValues: conditions,
|
||||
ObjectName: objectName,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
retSet = true
|
||||
}
|
||||
if byPassSet || retSet {
|
||||
return ErrNone
|
||||
}
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
// isPutActionAllowed - check if PUT operation is allowed on the resource, this
|
||||
// call verifies bucket policies and IAM policies, supports multi user
|
||||
// checks etc.
|
||||
|
||||
@@ -583,7 +583,11 @@ func toObjectInfo(bucket, object string, objInfo minio.ObjectInfo) ObjectInfo {
|
||||
func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLayer, remoteClnt *minio.Client, entries []ObjectInfo, prefix string) error {
|
||||
input := make(chan minio.SnowballObject, 1)
|
||||
opts := minio.SnowballOptions{
|
||||
Opts: minio.PutObjectOptions{},
|
||||
Opts: minio.PutObjectOptions{
|
||||
Internal: minio.AdvancedPutOptions{
|
||||
ReplicationRequest: true,
|
||||
},
|
||||
},
|
||||
InMemory: *r.Source.Snowball.InMemory,
|
||||
Compress: *r.Source.Snowball.Compress,
|
||||
SkipErrs: *r.Source.Snowball.SkipErrs,
|
||||
|
||||
@@ -277,14 +277,6 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
|
||||
return
|
||||
}
|
||||
|
||||
if keyMarker != "" {
|
||||
// Marker not common with prefix is not implemented.
|
||||
if !HasPrefix(keyMarker, prefix) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
listMultipartsInfo, err := objectAPI.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
@@ -1281,6 +1273,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
|
||||
return
|
||||
}
|
||||
if isDirObject(object) {
|
||||
if _, _, _, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, objectAPI.GetObjectInfo, ErrNone, ErrNone, false); s3Err != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
opts.WantChecksum = checksum
|
||||
|
||||
fanOutOpts := fanOutOptions{Checksum: checksum}
|
||||
|
||||
@@ -425,7 +425,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
|
||||
shouldPass: true,
|
||||
},
|
||||
// Test case - 4.
|
||||
// Setting Invalid prefix and marker combination.
|
||||
// A key marker outside the prefix is valid and produces an empty page.
|
||||
{
|
||||
bucket: bucketName,
|
||||
prefix: "asia",
|
||||
@@ -435,8 +435,8 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
|
||||
maxUploads: "0",
|
||||
accessKey: credentials.AccessKey,
|
||||
secretKey: credentials.SecretKey,
|
||||
expectedRespStatus: http.StatusNotImplemented,
|
||||
shouldPass: false,
|
||||
expectedRespStatus: http.StatusOK,
|
||||
shouldPass: true,
|
||||
},
|
||||
// Test case - 5.
|
||||
// Invalid upload id and marker combination.
|
||||
|
||||
+122
-61
@@ -20,8 +20,8 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -164,76 +164,129 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
|
||||
return nil
|
||||
}
|
||||
|
||||
// enforceRetentionBypassForPut enforces whether an existing object under governance can be overwritten
|
||||
// with governance bypass headers set in the request.
|
||||
// Objects under site wide WORM cannot be overwritten.
|
||||
// For objects in "Governance" mode, overwrite is allowed if a) object retention date is past OR
|
||||
// governance bypass headers are set and user has governance bypass permissions.
|
||||
// Objects in compliance mode can be overwritten only if retention date is being extended. No mode change is permitted.
|
||||
// isRetentionActionAllowed evaluates action for a PutObjectRetention request
|
||||
// with the object-lock condition keys derived from the requested retention.
|
||||
// The keys come from the request, not from the stored version, so the
|
||||
// handler can decide s3:PutObjectRetention before reading any object state.
|
||||
func isRetentionActionAllowed(action policy.Action, bucket, object string, objRetention *objectlock.ObjectRetention, r *http.Request, cred auth.Credentials, owner bool) bool {
|
||||
if cred.AccessKey == "" {
|
||||
return false
|
||||
}
|
||||
t, err := objectlock.UTCNowNTP()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
conditions := getConditionValues(r, "", cred)
|
||||
conditions["object-lock-mode"] = []string{string(objRetention.Mode)}
|
||||
conditions["object-lock-retain-until-date"] = []string{objRetention.RetainUntilDate.UTC().Format(time.RFC3339)}
|
||||
// Relative days from now, for "object-lock-remaining-retention-days".
|
||||
// Clearing retention (<Retention/>) has a zero RetainUntilDate, which must
|
||||
// evaluate as 0 remaining days rather than a large positive math.Abs delta
|
||||
// from year 0001.
|
||||
days := 0
|
||||
if !objRetention.RetainUntilDate.IsZero() && objRetention.RetainUntilDate.After(t) {
|
||||
// Time.Sub saturates after about 292 years. Count seconds instead so
|
||||
// maximum-retention policies also apply to distant RFC3339 dates.
|
||||
seconds := objRetention.RetainUntilDate.Unix() - t.Unix()
|
||||
days = int(seconds / 86400)
|
||||
if seconds%86400 != 0 || objRetention.RetainUntilDate.Nanosecond() > t.Nanosecond() {
|
||||
days++
|
||||
}
|
||||
}
|
||||
conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(days)}
|
||||
return globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: action,
|
||||
BucketName: bucket,
|
||||
ConditionValues: conditions,
|
||||
ObjectName: object,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
})
|
||||
}
|
||||
|
||||
// enforceRetentionBypassForPut enforces the stored retention of oi against a
|
||||
// PutObjectRetention request. The caller must already hold
|
||||
// s3:PutObjectRetention; this only decides changes that depend on the stored
|
||||
// state. Expired or absent retention may be changed freely. Unexpired
|
||||
// GOVERNANCE retention may be shortened, removed or have its mode changed only
|
||||
// with the bypass header and s3:BypassGovernanceRetention. Unexpired
|
||||
// COMPLIANCE retention may only be extended.
|
||||
func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi ObjectInfo, objRetention *objectlock.ObjectRetention, cred auth.Credentials, owner bool) error {
|
||||
byPassSet := objectlock.IsObjectLockGovernanceBypassSet(r.Header)
|
||||
ret := objectlock.GetObjectRetentionMeta(oi.UserDefined)
|
||||
if !ret.Mode.Valid() {
|
||||
return nil
|
||||
}
|
||||
|
||||
t, err := objectlock.UTCNowNTP()
|
||||
if err != nil {
|
||||
internalLogIf(ctx, err, logger.WarningKind)
|
||||
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
||||
}
|
||||
if ret.RetainUntilDate.Before(t) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Pass in relative days from current time, to additionally
|
||||
// to verify "object-lock-remaining-retention-days" policy if any.
|
||||
days := int(math.Ceil(math.Abs(objRetention.RetainUntilDate.Sub(t).Hours()) / 24))
|
||||
|
||||
ret := objectlock.GetObjectRetentionMeta(oi.UserDefined)
|
||||
if ret.Mode.Valid() {
|
||||
// Retention has expired you may change whatever you like.
|
||||
if ret.RetainUntilDate.Before(t) {
|
||||
apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
|
||||
days, objRetention.RetainUntilDate.Time,
|
||||
objRetention.Mode, byPassSet, r, cred,
|
||||
owner)
|
||||
if apiErr == ErrAccessDenied {
|
||||
return errAuthentication
|
||||
}
|
||||
return nil
|
||||
weakens := objRetention.Mode != ret.Mode || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time)
|
||||
if !weakens {
|
||||
return nil
|
||||
}
|
||||
switch ret.Mode {
|
||||
case objectlock.RetGovernance:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html#object-lock-bypass-governance
|
||||
if !objectlock.IsObjectLockGovernanceBypassSet(r.Header) {
|
||||
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
||||
}
|
||||
|
||||
switch ret.Mode {
|
||||
case objectlock.RetGovernance:
|
||||
govPerm := isPutRetentionAllowed(oi.Bucket, oi.Name, days,
|
||||
objRetention.RetainUntilDate.Time, objRetention.Mode,
|
||||
byPassSet, r, cred, owner)
|
||||
// Governance mode retention period cannot be shortened, if x-amz-bypass-governance is not set.
|
||||
if !byPassSet {
|
||||
if objRetention.Mode != objectlock.RetGovernance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) {
|
||||
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
||||
}
|
||||
}
|
||||
if govPerm == ErrAccessDenied {
|
||||
return errAuthentication
|
||||
}
|
||||
return nil
|
||||
case objectlock.RetCompliance:
|
||||
// Compliance retention mode cannot be changed or shortened.
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
|
||||
if objRetention.Mode != objectlock.RetCompliance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) {
|
||||
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
||||
}
|
||||
apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
|
||||
days, objRetention.RetainUntilDate.Time, objRetention.Mode,
|
||||
false, r, cred, owner)
|
||||
if apiErr == ErrAccessDenied {
|
||||
return errAuthentication
|
||||
}
|
||||
return nil
|
||||
if !isRetentionActionAllowed(policy.BypassGovernanceRetentionAction, oi.Bucket, oi.Name, objRetention, r, cred, owner) {
|
||||
return errAuthentication
|
||||
}
|
||||
return nil
|
||||
} // No pre-existing retention metadata present.
|
||||
default:
|
||||
// Compliance retention mode cannot be changed or shortened.
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
|
||||
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
||||
}
|
||||
}
|
||||
|
||||
apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
|
||||
days, objRetention.RetainUntilDate.Time,
|
||||
objRetention.Mode, byPassSet, r, cred, owner)
|
||||
if apiErr == ErrAccessDenied {
|
||||
return errAuthentication
|
||||
// checkObjectLockedForOverwrite re-checks under the object write lock whether an
|
||||
// in-place write to the null version (such as a directory object) would
|
||||
// overwrite a locked version in an Object Lock bucket.
|
||||
func checkObjectLockedForOverwrite(ctx context.Context, bucket, object string, getObjectInfoFn GetObjectInfoFn) error {
|
||||
if isMinioMetaBucketName(bucket) {
|
||||
return nil
|
||||
}
|
||||
retentionCfg, err := globalBucketObjectLockSys.Get(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !retentionCfg.LockEnabled {
|
||||
return nil
|
||||
}
|
||||
objInfo, err := getObjectInfoFn(ctx, bucket, object, ObjectOptions{
|
||||
VersionID: nullVersionID,
|
||||
NoLock: true,
|
||||
NoAuditLog: true,
|
||||
})
|
||||
if err != nil {
|
||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) || errors.As(err, &MethodNotAllowed{}) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined).Status == objectlock.LegalHoldOn {
|
||||
return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: nullVersionID}
|
||||
}
|
||||
r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
|
||||
if r.Mode == objectlock.RetCompliance || r.Mode == objectlock.RetGovernance {
|
||||
t, err := objectlock.UTCNowNTP()
|
||||
if err != nil {
|
||||
internalLogIf(ctx, err, logger.WarningKind)
|
||||
return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: nullVersionID}
|
||||
}
|
||||
if r.RetainUntilDate.After(t) {
|
||||
return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: nullVersionID}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -274,16 +327,24 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
||||
if err != nil {
|
||||
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
|
||||
}
|
||||
if isDirObject(object) && opts.VersionID == "" {
|
||||
opts.VersionID = nullVersionID
|
||||
}
|
||||
|
||||
if opts.VersionID != "" && !replicaTrusted {
|
||||
if objInfo, err := getObjectInfoFn(ctx, bucket, object, opts); err == nil {
|
||||
objInfo, err := getObjectInfoFn(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) && !errors.As(err, &MethodNotAllowed{}) {
|
||||
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
|
||||
}
|
||||
} else {
|
||||
r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
|
||||
t, err := objectlock.UTCNowNTP()
|
||||
if err != nil {
|
||||
internalLogIf(ctx, err, logger.WarningKind)
|
||||
return mode, retainDate, legalHold, ErrObjectLocked
|
||||
}
|
||||
if r.Mode == objectlock.RetCompliance && r.RetainUntilDate.After(t) {
|
||||
if (r.Mode == objectlock.RetCompliance || r.Mode == objectlock.RetGovernance) && r.RetainUntilDate.After(t) {
|
||||
return mode, retainDate, legalHold, ErrObjectLocked
|
||||
}
|
||||
mode = r.Mode
|
||||
|
||||
@@ -418,6 +418,9 @@ func getReplicationState(rinfos replicatedInfos, prevState ReplicationState, vID
|
||||
|
||||
for _, rinfo := range rinfos.Targets {
|
||||
if rinfo.ResyncTimestamp != "" {
|
||||
if rs.ResetStatusesMap == nil {
|
||||
rs.ResetStatusesMap = make(map[string]string)
|
||||
}
|
||||
rs.ResetStatusesMap[targetResetHeader(rinfo.Arn)] = rinfo.ResyncTimestamp
|
||||
}
|
||||
}
|
||||
|
||||
+175
-63
@@ -425,6 +425,7 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
|
||||
// the mere presence or absence of the target version.
|
||||
func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer) replicatedInfos {
|
||||
var replicationStatus replication.StatusType
|
||||
isPurge := dobj.isVersionPurge()
|
||||
bucket := dobj.Bucket
|
||||
versionID := dobj.DeleteMarkerVersionID
|
||||
if versionID == "" {
|
||||
@@ -484,6 +485,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
||||
lk := objectAPI.NewNSLock(bucket, "/[replicate]/"+dobj.ObjectName)
|
||||
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
dobj.RetryCount++
|
||||
globalReplicationPool.Get().queueMRFSave(dobj.ToMRFEntry())
|
||||
sendEvent(eventArgs{
|
||||
BucketName: bucket,
|
||||
@@ -502,6 +504,35 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
|
||||
if !isPurge && dobj.DeleteMarkerVersionID != "" {
|
||||
// A creation task carries the marker as it looked when it was queued
|
||||
// by the DELETE handler, a GET/HEAD/LIST heal, the scanner or MRF.
|
||||
// While it waited for this lock, another frontend may have purged the
|
||||
// marker and replicated that purge. The targets no longer hold the
|
||||
// marker, so the queued creation would recreate it from the stale
|
||||
// snapshot. Confirm the source version under the lock first.
|
||||
switch deleteMarkerCreationState(ctx, objectAPI, dobj) {
|
||||
case creationStale:
|
||||
return replicatedInfos{}
|
||||
case creationUnverified:
|
||||
dobj.RetryCount++
|
||||
globalReplicationPool.Get().queueMRFSave(dobj.ToMRFEntry())
|
||||
sendEvent(eventArgs{
|
||||
BucketName: bucket,
|
||||
Object: ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: dobj.ObjectName,
|
||||
VersionID: versionID,
|
||||
DeleteMarker: dobj.DeleteMarker,
|
||||
},
|
||||
UserAgent: "Internal: [Replication]",
|
||||
Host: globalLocalNodeName,
|
||||
EventName: event.ObjectReplicationNotTracked,
|
||||
})
|
||||
return replicatedInfos{}
|
||||
}
|
||||
}
|
||||
|
||||
rinfos := replicatedInfos{Targets: make([]replicatedTargetInfo, 0, len(dsc.targetsMap))}
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
@@ -546,25 +577,38 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
||||
replicationStatus = rinfos.ReplicationStatus()
|
||||
prevStatus := dobj.DeleteMarkerReplicationStatus()
|
||||
|
||||
if dobj.VersionID != "" {
|
||||
prevStatus = replication.StatusType(dobj.VersionPurgeStatus())
|
||||
replicationStatus = replication.StatusType(rinfos.VersionPurgeStatus())
|
||||
if isPurge {
|
||||
prevStatus = purgeReplicationStatus(dobj.VersionPurgeStatus())
|
||||
replicationStatus = purgeReplicationStatus(rinfos.VersionPurgeStatus())
|
||||
}
|
||||
|
||||
// to decrement pending count later.
|
||||
for _, rinfo := range rinfos.Targets {
|
||||
if rinfo.ReplicationStatus != rinfo.PrevReplicationStatus {
|
||||
globalReplicationStats.Load().Update(dobj.Bucket, rinfo, replicationStatus,
|
||||
prevStatus)
|
||||
status, previous := rinfo.ReplicationStatus, rinfo.PrevReplicationStatus
|
||||
if isPurge {
|
||||
status = purgeReplicationStatus(rinfo.VersionPurgeStatus)
|
||||
previous = purgeReplicationStatus(dobj.ReplicationState.PurgeTargets[rinfo.Arn])
|
||||
}
|
||||
if status != previous {
|
||||
globalReplicationStats.Load().Update(dobj.Bucket, rinfo, status, previous)
|
||||
}
|
||||
}
|
||||
|
||||
eventName := event.ObjectReplicationComplete
|
||||
if replicationStatus == replication.Failed {
|
||||
eventName = event.ObjectReplicationFailed
|
||||
dobj.RetryCount++
|
||||
globalReplicationPool.Get().queueMRFSave(dobj.ToMRFEntry())
|
||||
}
|
||||
drs := getReplicationState(rinfos, dobj.ReplicationState, dobj.VersionID)
|
||||
if isPurge {
|
||||
// A purge must not rewrite the marker's creation/replica metadata.
|
||||
// Multiple serialized empty target statuses can parse as nonempty,
|
||||
// so explicitly send an empty creation update to the metadata writer.
|
||||
drs.ReplicationStatusInternal = ""
|
||||
drs.Targets = nil
|
||||
drs.ReplicaStatus = ""
|
||||
}
|
||||
if replicationStatus != prevStatus {
|
||||
drs.ReplicationTimeStamp = UTCNow()
|
||||
}
|
||||
@@ -606,6 +650,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
||||
}
|
||||
|
||||
func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationInfo, tgt *TargetClient) (rinfo replicatedTargetInfo) {
|
||||
isPurge := dobj.isVersionPurge()
|
||||
versionID := dobj.DeleteMarkerVersionID
|
||||
if versionID == "" {
|
||||
versionID = dobj.VersionID
|
||||
@@ -615,42 +660,50 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
|
||||
rinfo.OpType = dobj.OpType
|
||||
rinfo.endpoint = tgt.EndpointURL().Host
|
||||
rinfo.secure = tgt.EndpointURL().Scheme == "https"
|
||||
// A purge leaves ReplicationStatus empty: the metadata writer interprets
|
||||
// that as preserving the entire creation-status block, including targets
|
||||
// outside this fan-out. Only VersionPurgeStatus records the purge outcome.
|
||||
defer func() {
|
||||
if rinfo.ReplicationStatus == replication.Completed && tgt.ResetID != "" && dobj.OpType == replication.ExistingObjectReplicationType {
|
||||
completed := rinfo.ReplicationStatus == replication.Completed
|
||||
if isPurge {
|
||||
completed = rinfo.VersionPurgeStatus == replication.VersionPurgeComplete
|
||||
}
|
||||
if completed && tgt.ResetID != "" && dobj.OpType == replication.ExistingObjectReplicationType {
|
||||
rinfo.ResyncTimestamp = fmt.Sprintf("%s;%s", UTCNow().Format(http.TimeFormat), tgt.ResetID)
|
||||
}
|
||||
}()
|
||||
|
||||
if dobj.VersionID == "" && rinfo.PrevReplicationStatus == replication.Completed && dobj.OpType != replication.ExistingObjectReplicationType {
|
||||
if !isPurge && rinfo.PrevReplicationStatus == replication.Completed && dobj.OpType != replication.ExistingObjectReplicationType {
|
||||
rinfo.ReplicationStatus = rinfo.PrevReplicationStatus
|
||||
return rinfo
|
||||
}
|
||||
if dobj.VersionID != "" && rinfo.VersionPurgeStatus == replication.VersionPurgeComplete {
|
||||
if isPurge && rinfo.VersionPurgeStatus == replication.VersionPurgeComplete {
|
||||
return rinfo
|
||||
}
|
||||
if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
|
||||
replLogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN), "replication-target-offline-delete-"+tgt.ARN)
|
||||
rinfo.Err = fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN)
|
||||
replLogOnceIf(ctx, rinfo.Err, "replication-target-offline-delete-"+tgt.ARN)
|
||||
sendEvent(eventArgs{
|
||||
BucketName: dobj.Bucket,
|
||||
Object: ObjectInfo{
|
||||
Bucket: dobj.Bucket,
|
||||
Name: dobj.ObjectName,
|
||||
VersionID: dobj.VersionID,
|
||||
VersionID: versionID,
|
||||
DeleteMarker: dobj.DeleteMarker,
|
||||
},
|
||||
UserAgent: "Internal: [Replication]",
|
||||
Host: globalLocalNodeName,
|
||||
EventName: event.ObjectReplicationNotTracked,
|
||||
})
|
||||
if dobj.VersionID == "" {
|
||||
rinfo.ReplicationStatus = replication.Failed
|
||||
} else {
|
||||
if isPurge {
|
||||
rinfo.VersionPurgeStatus = replication.VersionPurgeFailed
|
||||
} else {
|
||||
rinfo.ReplicationStatus = replication.Failed
|
||||
}
|
||||
return rinfo
|
||||
}
|
||||
// early return if already replicated delete marker for existing object replication/ healing delete markers
|
||||
if dobj.DeleteMarkerVersionID != "" {
|
||||
if !isPurge && dobj.DeleteMarkerVersionID != "" {
|
||||
toi, err := tgt.StatObject(ctx, tgt.Bucket, dobj.ObjectName, minio.StatObjectOptions{
|
||||
VersionID: versionID,
|
||||
Internal: minio.AdvancedGetOptions{
|
||||
@@ -662,16 +715,10 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
|
||||
switch {
|
||||
case isErrMethodNotAllowed(serr):
|
||||
// delete marker already replicated
|
||||
if dobj.VersionID == "" && rinfo.VersionPurgeStatus.Empty() {
|
||||
rinfo.ReplicationStatus = replication.Completed
|
||||
return rinfo
|
||||
}
|
||||
rinfo.ReplicationStatus = replication.Completed
|
||||
return rinfo
|
||||
case isErrObjectNotFound(serr), isErrVersionNotFound(serr):
|
||||
// version being purged is already not found on target.
|
||||
if !rinfo.VersionPurgeStatus.Empty() {
|
||||
rinfo.VersionPurgeStatus = replication.VersionPurgeComplete
|
||||
return rinfo
|
||||
}
|
||||
// The marker still needs to be created on the target.
|
||||
case isErrReadQuorum(serr), isErrWriteQuorum(serr):
|
||||
// destination has some quorum issues, perform removeObject() anyways
|
||||
// to complete the operation.
|
||||
@@ -691,7 +738,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
|
||||
rmErr := tgt.RemoveObject(ctx, tgt.Bucket, dobj.ObjectName, minio.RemoveObjectOptions{
|
||||
VersionID: versionID,
|
||||
Internal: minio.AdvancedRemoveOptions{
|
||||
ReplicationDeleteMarker: dobj.DeleteMarkerVersionID != "",
|
||||
ReplicationDeleteMarker: !isPurge && dobj.DeleteMarkerVersionID != "",
|
||||
ReplicationMTime: dobj.DeleteMarkerMTime.Time,
|
||||
ReplicationStatus: minio.ReplicationStatusReplica,
|
||||
ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside
|
||||
@@ -699,20 +746,20 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
|
||||
})
|
||||
if rmErr != nil {
|
||||
rinfo.Err = rmErr
|
||||
if dobj.VersionID == "" {
|
||||
rinfo.ReplicationStatus = replication.Failed
|
||||
} else {
|
||||
if isPurge {
|
||||
rinfo.VersionPurgeStatus = replication.VersionPurgeFailed
|
||||
} else {
|
||||
rinfo.ReplicationStatus = replication.Failed
|
||||
}
|
||||
replLogIf(ctx, fmt.Errorf("unable to replicate delete marker to %s: %s/%s(%s): %w", tgt.EndpointURL(), tgt.Bucket, dobj.ObjectName, versionID, rmErr))
|
||||
if rmErr != nil && minio.IsNetworkOrHostDown(rmErr, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
|
||||
globalBucketTargetSys.markOffline(tgt.EndpointURL())
|
||||
}
|
||||
} else {
|
||||
if dobj.VersionID == "" {
|
||||
rinfo.ReplicationStatus = replication.Completed
|
||||
} else {
|
||||
if isPurge {
|
||||
rinfo.VersionPurgeStatus = replication.VersionPurgeComplete
|
||||
} else {
|
||||
rinfo.ReplicationStatus = replication.Completed
|
||||
}
|
||||
}
|
||||
return rinfo
|
||||
@@ -781,6 +828,18 @@ func (m caseInsensitiveMap) Lookup(key string) (string, bool) {
|
||||
return "", false
|
||||
}
|
||||
|
||||
// replicationTaggingTimestamp carries a recorded removal even when tags are
|
||||
// empty. Only legacy nonempty tags use ModTime; absence is not a tombstone.
|
||||
func replicationTaggingTimestamp(objInfo ObjectInfo) (time.Time, error) {
|
||||
if stamp, ok := caseInsensitiveMap(objInfo.UserDefined).Lookup(ReservedMetadataPrefixLower + TaggingTimestamp); ok {
|
||||
return time.Parse(time.RFC3339Nano, stamp)
|
||||
}
|
||||
if objInfo.UserTags != "" {
|
||||
return objInfo.ModTime, nil
|
||||
}
|
||||
return time.Time{}, nil
|
||||
}
|
||||
|
||||
func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts minio.PutObjectOptions, isMP bool, err error) {
|
||||
meta := make(map[string]string)
|
||||
isSSEC := crypto.SSEC.IsEncrypted(objInfo.UserDefined)
|
||||
@@ -850,17 +909,12 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (put
|
||||
tag, _ := tags.ParseObjectTags(objInfo.UserTags)
|
||||
if tag != nil {
|
||||
putOpts.UserTags = tag.ToMap()
|
||||
// set tag timestamp in opts
|
||||
tagTimestamp := objInfo.ModTime
|
||||
if tagTmstampStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]; ok {
|
||||
tagTimestamp, err = time.Parse(time.RFC3339Nano, tagTmstampStr)
|
||||
if err != nil {
|
||||
return putOpts, false, err
|
||||
}
|
||||
}
|
||||
putOpts.Internal.TaggingTimestamp = tagTimestamp
|
||||
}
|
||||
}
|
||||
putOpts.Internal.TaggingTimestamp, err = replicationTaggingTimestamp(objInfo)
|
||||
if err != nil {
|
||||
return putOpts, false, err
|
||||
}
|
||||
|
||||
lkMap := caseInsensitiveMap(objInfo.UserDefined)
|
||||
if lang, ok := lkMap.Lookup(xhttp.ContentLanguage); ok {
|
||||
@@ -1003,6 +1057,12 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
|
||||
if (oi2.UserTagCount > 0 && !reflect.DeepEqual(oi2Map, t.ToMap())) || (oi2.UserTagCount != len(t.ToMap())) {
|
||||
return replicateMetadata
|
||||
}
|
||||
// HEAD does not report the tag revision. Equal values can hide a newer
|
||||
// deletion or re-addition, so scheduled metadata/heal work must deliver it.
|
||||
// Completed objects are still excluded by the existing scanner gates.
|
||||
if _, ok := caseInsensitiveMap(oi1.UserDefined).Lookup(ReservedMetadataPrefixLower + TaggingTimestamp); ok {
|
||||
return replicateMetadata
|
||||
}
|
||||
|
||||
// Compare only necessary headers
|
||||
compareKeys := []string{
|
||||
@@ -1269,9 +1329,6 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
|
||||
oi.UserDefined[targetResetHeader(rinfo.Arn)] = rinfo.ResyncTimestamp
|
||||
}
|
||||
}
|
||||
if ri.UserTags != "" {
|
||||
oi.UserDefined[xhttp.AmzObjectTagging] = ri.UserTags
|
||||
}
|
||||
return dsc, nil
|
||||
},
|
||||
}
|
||||
@@ -1689,14 +1746,11 @@ applyAction:
|
||||
if _, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok {
|
||||
dstOpts.Internal.RetentionTimestamp = objInfo.ModTime
|
||||
}
|
||||
if objInfo.UserTags != "" {
|
||||
dstOpts.Internal.TaggingTimestamp = objInfo.ModTime
|
||||
}
|
||||
if tagTmStr, ok := lkMap.Lookup(ReservedMetadataPrefixLower + TaggingTimestamp); ok {
|
||||
ondiskTimestamp, err := time.Parse(time.RFC3339, tagTmStr)
|
||||
if err == nil {
|
||||
dstOpts.Internal.TaggingTimestamp = ondiskTimestamp
|
||||
}
|
||||
dstOpts.Internal.TaggingTimestamp, rinfo.Err = replicationTaggingTimestamp(objInfo)
|
||||
if rinfo.Err != nil {
|
||||
rinfo.ReplicationStatus = replication.Failed
|
||||
replLogIf(ctx, fmt.Errorf("invalid tagging timestamp for object %s/%s(%s): %w", bucket, object, objInfo.VersionID, rinfo.Err))
|
||||
return rinfo
|
||||
}
|
||||
if retTmStr, ok := lkMap.Lookup(ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp); ok {
|
||||
ondiskTimestamp, err := time.Parse(time.RFC3339, retTmStr)
|
||||
@@ -1916,11 +1970,61 @@ func filterReplicationStatusMetadata(metadata map[string]string) map[string]stri
|
||||
// DeletedObjectReplicationInfo has info on deleted object
|
||||
type DeletedObjectReplicationInfo struct {
|
||||
DeletedObject
|
||||
Bucket string
|
||||
EventType string
|
||||
OpType replication.Type
|
||||
ResetID string
|
||||
TargetArn string
|
||||
Bucket string
|
||||
EventType string
|
||||
OpType replication.Type
|
||||
ResetID string
|
||||
TargetArn string
|
||||
RetryCount int
|
||||
}
|
||||
|
||||
// isVersionPurge also recognizes the old marker-shaped purge task. Use the
|
||||
// operation's state, rather than one target's possibly missing purge entry.
|
||||
func (di DeletedObjectReplicationInfo) isVersionPurge() bool {
|
||||
return di.VersionID != "" || di.DeleteMarkerVersionID != "" && !di.VersionPurgeStatus().Empty()
|
||||
}
|
||||
|
||||
// creationState is the outcome of re-reading a queued delete-marker creation
|
||||
// against the source.
|
||||
type creationState int
|
||||
|
||||
const (
|
||||
creationCurrent creationState = iota
|
||||
creationStale
|
||||
creationUnverified
|
||||
)
|
||||
|
||||
// deleteMarkerCreationState re-reads the marker a queued creation task refers
|
||||
// to. The version being absent, no longer a marker, or under a version purge
|
||||
// makes the creation stale. A failed read is not absence: the caller retries
|
||||
// later instead of guessing.
|
||||
func deleteMarkerCreationState(ctx context.Context, objectAPI ObjectLayer, dobj DeletedObjectReplicationInfo) creationState {
|
||||
oi, err := objectAPI.GetObjectInfo(ctx, dobj.Bucket, dobj.ObjectName, ObjectOptions{
|
||||
VersionID: dobj.DeleteMarkerVersionID,
|
||||
Versioned: globalBucketVersioningSys.PrefixEnabled(dobj.Bucket, dobj.ObjectName),
|
||||
VersionSuspended: globalBucketVersioningSys.Suspended(dobj.Bucket),
|
||||
})
|
||||
switch {
|
||||
case isErrObjectNotFound(err), isErrVersionNotFound(err):
|
||||
return creationStale
|
||||
case err != nil && !isErrMethodNotAllowed(err):
|
||||
return creationUnverified
|
||||
}
|
||||
if !oi.DeleteMarker || oi.VersionID != dobj.DeleteMarkerVersionID {
|
||||
return creationStale
|
||||
}
|
||||
if !oi.VersionPurgeStatus.Empty() || oi.VersionPurgeStatusInternal != "" {
|
||||
return creationStale
|
||||
}
|
||||
return creationCurrent
|
||||
}
|
||||
|
||||
// Purge metadata uses COMPLETE; operation statistics and audit use COMPLETED.
|
||||
func purgeReplicationStatus(status VersionPurgeStatusType) replication.StatusType {
|
||||
if replication.StatusType(status) == replication.CompletedLegacy {
|
||||
return replication.Completed
|
||||
}
|
||||
return replication.StatusType(status)
|
||||
}
|
||||
|
||||
// ToMRFEntry returns the relevant info needed by MRF
|
||||
@@ -1930,9 +2034,10 @@ func (di DeletedObjectReplicationInfo) ToMRFEntry() MRFReplicateEntry {
|
||||
versionID = di.VersionID
|
||||
}
|
||||
return MRFReplicateEntry{
|
||||
Bucket: di.Bucket,
|
||||
Object: di.ObjectName,
|
||||
versionID: versionID,
|
||||
Bucket: di.Bucket,
|
||||
Object: di.ObjectName,
|
||||
versionID: versionID,
|
||||
RetryCount: di.RetryCount,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2424,6 +2529,7 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
|
||||
case <-p.ctx.Done():
|
||||
case ch <- doi:
|
||||
default:
|
||||
doi.RetryCount++
|
||||
p.queueMRFSave(doi.ToMRFEntry())
|
||||
p.mu.RLock()
|
||||
prio := p.priority
|
||||
@@ -3787,9 +3893,10 @@ func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcf
|
||||
DeleteMarkerMTime: DeleteMarkerMTime{roi.ModTime},
|
||||
DeleteMarker: roi.DeleteMarker,
|
||||
},
|
||||
Bucket: roi.Bucket,
|
||||
OpType: replication.HealReplicationType,
|
||||
EventType: ReplicateHealDelete,
|
||||
Bucket: roi.Bucket,
|
||||
OpType: replication.HealReplicationType,
|
||||
EventType: ReplicateHealDelete,
|
||||
RetryCount: retryCount,
|
||||
}
|
||||
// heal delete marker replication failure or versioned delete replication failure
|
||||
if roi.ReplicationStatus == replication.Pending ||
|
||||
@@ -4072,7 +4179,12 @@ func (p *ReplicationPool) queueMRFHeal() error {
|
||||
VersionID: vID,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
// A versioned marker lookup returns its metadata with a 405. Only
|
||||
// accept that error with a real, matching marker identity.
|
||||
validMarker := isErrMethodNotAllowed(err) && oi.DeleteMarker &&
|
||||
vID != "" && oi.VersionID == vID && !oi.ModTime.IsZero() &&
|
||||
oi.Bucket == e.Bucket && oi.Name != "" && oi.Name == decodeDirObject(e.Object)
|
||||
if err != nil && !validMarker || oi.Name == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
@@ -56,6 +56,7 @@ import (
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/color"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/config/api"
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
@@ -445,6 +446,7 @@ func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) {
|
||||
ctxt.SendBufSize = ctx.Int("send-buf-size")
|
||||
ctxt.RecvBufSize = ctx.Int("recv-buf-size")
|
||||
ctxt.IdleTimeout = ctx.Duration("idle-timeout")
|
||||
ctxt.ReadHeaderTimeout = ctx.Duration("read-header-timeout")
|
||||
ctxt.UserTimeout = ctx.Duration("conn-user-timeout")
|
||||
|
||||
if conf := ctx.String("config"); len(conf) > 0 {
|
||||
@@ -880,6 +882,9 @@ func serverHandleEnvVars() {
|
||||
logger.Warning("%s=%s is ignored: Silo in-place updates are permanently disabled", config.EnvUpdate, updateSetting)
|
||||
}
|
||||
globalInplaceUpdateDisabled = true
|
||||
if env.IsSet(api.EnvAPIMultipartListing) {
|
||||
logger.Warning("%s is ignored: multipart listing always uses persistent metadata; uploads without identity fields are omitted", api.EnvAPIMultipartListing)
|
||||
}
|
||||
|
||||
// Check if the supported credential env vars,
|
||||
// "MINIO_ROOT_USER" and "MINIO_ROOT_PASSWORD" are provided
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
// An exact-version DELETE without a replication decision physically purges
|
||||
// the version. Its response must describe the stored version: a data version
|
||||
// is not a delete marker even while pending-purge metadata makes the lookup
|
||||
// expose it as deleted, and a stored marker stays a marker.
|
||||
func TestDeleteMarkerPurgeResponseIdentity(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, endpoints: []string{"DeleteObject"}, objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, creds auth.Credentials, t *testing.T) {
|
||||
defer replicationTestCapacity(obj)()
|
||||
ctx := t.Context()
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Seed the purge state the DELETE handler would record for a target.
|
||||
arn := "arn:minio:replication::" + mustGetUUID() + ":bucket"
|
||||
pendingPurge := ReplicationState{
|
||||
VersionPurgeStatusInternal: arn + "=PENDING;",
|
||||
PurgeTargets: map[string]VersionPurgeStatusType{arn: replication.VersionPurgePending},
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
marker, purged bool
|
||||
}{
|
||||
{name: "data"},
|
||||
{name: "data-pending-purge", purged: true},
|
||||
{name: "marker", marker: true},
|
||||
{name: "marker-pending-purge", marker: true, purged: true},
|
||||
} {
|
||||
name := "purge-response-" + mustGetUUID()
|
||||
oi, err := obj.PutObject(ctx, bucket, name, mustGetPutObjReader(t, bytes.NewReader([]byte("data")), 4, "", ""), ObjectOptions{Versioned: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
version := oi.VersionID
|
||||
if tc.marker {
|
||||
opts := ObjectOptions{Versioned: true, VersionID: mustGetUUID(), DeleteMarker: true, MTime: UTCNow(), ReplicationRequest: true}
|
||||
opts.SetReplicaStatus(replication.Replica)
|
||||
if _, err := obj.DeleteObject(ctx, bucket, name, opts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
version = opts.VersionID
|
||||
}
|
||||
if tc.purged {
|
||||
if _, err := obj.DeleteObject(ctx, bucket, name, ObjectOptions{Versioned: true, VersionID: version, DeleteReplication: pendingPurge}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
before, err := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{Versioned: true, VersionID: version})
|
||||
if tc.marker || tc.purged {
|
||||
if !isErrMethodNotAllowed(err) || !before.DeleteMarker {
|
||||
t.Fatalf("%s: seeded version not exposed as deleted: %+v %v", tc.name, before, err)
|
||||
}
|
||||
} else if err != nil || before.DeleteMarker {
|
||||
t.Fatalf("%s: seeded data version: %+v %v", tc.name, before, err)
|
||||
}
|
||||
if tc.purged && before.VersionPurgeStatus != replication.VersionPurgePending {
|
||||
t.Fatalf("%s: purge state not seeded: %+v", tc.name, before)
|
||||
}
|
||||
req, err := newTestSignedRequestV4(http.MethodDelete, "/"+bucket+"/"+name+"?versionId="+version, 0, nil, creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("%s: status=%d: %s", tc.name, rec.Code, rec.Body.String())
|
||||
}
|
||||
// The handler sets these headers by map key, not canonical name.
|
||||
if got := rec.Header()[xhttp.AmzVersionID]; len(got) != 1 || got[0] != version {
|
||||
t.Fatalf("%s: response version %q, want %q", tc.name, got, version)
|
||||
}
|
||||
if got := len(rec.Header()[xhttp.AmzDeleteMarker]) == 1 && rec.Header()[xhttp.AmzDeleteMarker][0] == "true"; got != tc.marker {
|
||||
t.Fatalf("%s: x-amz-delete-marker=%v for a stored marker=%v", tc.name, got, tc.marker)
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{Versioned: true, VersionID: version}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Fatalf("%s: version not purged: %v", tc.name, err)
|
||||
}
|
||||
t.Logf("%s %s: purged with x-amz-delete-marker=%q", backend, tc.name, rec.Header()[xhttp.AmzDeleteMarker])
|
||||
}
|
||||
}})
|
||||
}
|
||||
@@ -0,0 +1,524 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/once"
|
||||
)
|
||||
|
||||
func TestDeleteMarkerPurgeIntent(t *testing.T) {
|
||||
version := mustGetUUID()
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
edit func(*ObjectOptions)
|
||||
want bool
|
||||
}{
|
||||
{"ordinary", func(*ObjectOptions) {}, true},
|
||||
{"receiver", func(o *ObjectOptions) { o.ReplicationRequest = true; o.SetReplicaStatus(replication.Replica) }, true},
|
||||
{"untrusted-replica", func(o *ObjectOptions) { o.SetReplicaStatus(replication.Replica) }, false},
|
||||
{"complete-purge", func(o *ObjectOptions) {
|
||||
o.DeleteReplication.VersionPurgeStatusInternal = string(replication.VersionPurgeComplete)
|
||||
}, true},
|
||||
{"pending-purge", func(o *ObjectOptions) { o.DeleteReplication.VersionPurgeStatusInternal = "PENDING" }, false},
|
||||
{"failed-purge", func(o *ObjectOptions) { o.DeleteReplication.VersionPurgeStatusInternal = "FAILED" }, false},
|
||||
{"unknown-purge", func(o *ObjectOptions) { o.DeleteReplication.VersionPurgeStatusInternal = "future-state" }, false},
|
||||
{"pending-creation", func(o *ObjectOptions) { o.DeleteReplication.ReplicationStatusInternal = "PENDING" }, false},
|
||||
{"failed-creation", func(o *ObjectOptions) { o.DeleteReplication.ReplicationStatusInternal = "FAILED" }, false},
|
||||
{"completed-creation", func(o *ObjectOptions) { o.DeleteReplication.ReplicationStatusInternal = "COMPLETED" }, false},
|
||||
{"create-marker", func(o *ObjectOptions) { o.DeleteMarker = true }, false},
|
||||
{"empty", func(o *ObjectOptions) { o.VersionID = "" }, false},
|
||||
{"null", func(o *ObjectOptions) { o.VersionID = nullVersionID }, false},
|
||||
{"invalid", func(o *ObjectOptions) { o.VersionID = "invalid" }, false},
|
||||
{"zero-uuid", func(o *ObjectOptions) { o.VersionID = emptyUUID }, false},
|
||||
{"movement", func(o *ObjectOptions) { o.DataMovement = true }, false},
|
||||
{"free-version", func(o *ObjectOptions) { o.InclFreeVersions = true }, false},
|
||||
{"expiration", func(o *ObjectOptions) { o.Expiration.Expire = true }, false},
|
||||
{"transition", func(o *ObjectOptions) { o.Transition.Status = "complete" }, false},
|
||||
{"restored-expiration", func(o *ObjectOptions) { o.Transition.ExpireRestored = true }, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
o := ObjectOptions{VersionID: version, Versioned: true}
|
||||
tc.edit(&o)
|
||||
if got := o.isVersionPurge(); got != tc.want {
|
||||
t.Fatalf("purge=%v want=%v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Every other StorageAPI method panics through the nil embedding: a proof may
|
||||
// read metadata, but must never delete, heal, or ask a different storage API.
|
||||
type absenceProofDisk struct {
|
||||
StorageAPI
|
||||
t *testing.T
|
||||
err error
|
||||
reads *atomic.Int32
|
||||
deletes *atomic.Int32
|
||||
}
|
||||
|
||||
func (d absenceProofDisk) ReadVersion(_ context.Context, _, _, _, _ string, opts ReadOptions) (FileInfo, error) {
|
||||
if opts.ReadData || opts.Healing {
|
||||
d.t.Error("absence proof requested data/healing")
|
||||
}
|
||||
d.reads.Add(1)
|
||||
return FileInfo{}, d.err
|
||||
}
|
||||
|
||||
func (d absenceProofDisk) DeleteVersion(_ context.Context, _, _ string, _ FileInfo, _ bool, _ DeleteOptions) error {
|
||||
if d.deletes == nil {
|
||||
d.t.Error("read-only absence proof attempted a deletion")
|
||||
} else {
|
||||
d.deletes.Add(1)
|
||||
}
|
||||
return d.err
|
||||
}
|
||||
|
||||
func TestVersionPurgeAbsenceProof(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
errs []error
|
||||
quorum, all bool
|
||||
}{
|
||||
{"all-absent", []error{errFileNotFound, errFileVersionNotFound, errFileNotFound, errFileVersionNotFound}, true, true},
|
||||
{"majority-absent", []error{errFileNotFound, errFileVersionNotFound, errFileNotFound, nil}, true, false},
|
||||
{"half-absent", []error{errFileNotFound, errFileVersionNotFound, errDiskNotFound, errDiskNotFound}, false, false},
|
||||
{"corrupt", []error{errFileNotFound, errFileVersionNotFound, errFileCorrupt, errFileCorrupt}, false, false},
|
||||
{"permission", []error{errFileNotFound, errFileVersionNotFound, errDiskAccessDenied, errVolumeAccessDenied}, false, false},
|
||||
{"volume-missing", []error{errFileNotFound, errFileVersionNotFound, errVolumeNotFound, errVolumeNotFound}, false, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var reads atomic.Int32
|
||||
disks := make([]StorageAPI, len(tc.errs))
|
||||
for i, err := range tc.errs {
|
||||
disks[i] = absenceProofDisk{t: t, err: err, reads: &reads}
|
||||
}
|
||||
er := erasureObjects{getDisks: func() []StorageAPI { return disks }}
|
||||
oldQueue := globalMRFState.opCh
|
||||
globalMRFState.opCh = make(chan PartialOperation, 1)
|
||||
defer func() { globalMRFState.opCh = oldQueue }()
|
||||
all, err := er.confirmVersionAbsent(t.Context(), "bucket", "object", mustGetUUID())
|
||||
if (err == nil) != tc.quorum || all != tc.all || reads.Load() != int32(len(disks)) || len(globalMRFState.opCh) != 0 {
|
||||
t.Fatalf("proof all=%v error=%v reads=%d queued=%d", all, err, reads.Load(), len(globalMRFState.opCh))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionPurgeAggregation(t *testing.T) {
|
||||
for _, pure := range []bool{false, true} {
|
||||
for _, fault := range []error{nil, errFileCorrupt, errDiskAccessDenied} {
|
||||
t.Run(fmt.Sprintf("purge_%v_fault_%v", pure, fault), func(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
disks := make([]StorageAPI, 4)
|
||||
for i, err := range []error{nil, errFileNotFound, errFileVersionNotFound, fault} {
|
||||
disks[i] = absenceProofDisk{t: t, err: err, deletes: &calls}
|
||||
}
|
||||
er := erasureObjects{getDisks: func() []StorageAPI { return disks }}
|
||||
err := er.deleteObjectVersion(t.Context(), "bucket", "object", FileInfo{VersionID: mustGetUUID()}, false, pure)
|
||||
if (err == nil) != pure || calls.Load() != 4 {
|
||||
t.Fatalf("aggregation error=%v calls=%d", err, calls.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMarkerPurgeOmittedPool(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
defer replicationTestCapacity(z)()
|
||||
const name = "partially-absent"
|
||||
er := z.serverPools[1].getHashedSet(name)
|
||||
_, opts := seedPurgeMarker(t, er, bucket, name, false)
|
||||
opts.ReplicationRequest = false
|
||||
opts.DeleteReplication = ReplicationState{}
|
||||
// Pool 0 appears absent at read quorum, but half of it cannot be read.
|
||||
missing := z.serverPools[0].getHashedSet(name)
|
||||
original := missing.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
clear(disks[len(disks)/2:])
|
||||
missing.getDisks = func() []StorageAPI { return disks }
|
||||
defer func() { missing.getDisks = original }()
|
||||
_, err := z.DeleteObject(t.Context(), bucket, name, opts)
|
||||
if !isErrWriteQuorum(err) {
|
||||
t.Errorf("omitted pool acknowledged deletion: %T %v", err, err)
|
||||
}
|
||||
if countPurgeMarkers(t, er.getDisks(), bucket, name, opts.VersionID) != 16 {
|
||||
t.Fatal("mutated known copies before checking omitted pool")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMarkerPurgeReceivingPool(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
defer replicationTestCapacity(z)()
|
||||
for _, duplicate := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("duplicate_%v", duplicate), func(t *testing.T) {
|
||||
name := "receiver-" + mustGetUUID()
|
||||
_, opts := seedPurgeMarker(t, z.serverPools[0].getHashedSet(name), bucket, name, false)
|
||||
if duplicate {
|
||||
create := opts
|
||||
create.DeleteMarker = true
|
||||
if _, err := z.serverPools[1].DeleteObject(t.Context(), bucket, name, create); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
// Latest-key routing selects pool 1, but the addressed marker is
|
||||
// in pool 0. A replica purge addresses a version, not the latest.
|
||||
putConsistencyObject(t, z, bucket, name, 1, "newer-data", ObjectOptions{Versioned: true})
|
||||
}
|
||||
if _, err := z.DeleteObject(t.Context(), bucket, name, opts); err != nil {
|
||||
t.Errorf("replica purge did not find its addressed version: %v", err)
|
||||
}
|
||||
for i, pool := range z.serverPools {
|
||||
if got := countPurgeMarkers(t, pool.getHashedSet(name).getDisks(), bucket, name, opts.VersionID); got != 0 {
|
||||
t.Errorf("replica purge left %d marker copies in pool %d", got, i)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMarkerPurgeCallbackIntent(t *testing.T) {
|
||||
for _, loseCopies := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("missing_after_callback_%v", loseCopies), func(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
defer replicationTestCapacity(z)()
|
||||
const name = "callback-pending"
|
||||
er := z.serverPools[0].getHashedSet(name)
|
||||
_, opts := seedPurgeMarker(t, er, bucket, name, false)
|
||||
original := er.getDisks
|
||||
all := append([]StorageAPI(nil), original()...)
|
||||
defer func() { er.getDisks = original }()
|
||||
opts.ReplicationRequest = false
|
||||
opts.DeleteReplication = ReplicationState{}
|
||||
metadata, retention := 0, 0
|
||||
opts.EvalRetentionBypassFn = func(oi ObjectInfo, err error) error {
|
||||
retention++
|
||||
if !oi.DeleteMarker || !isErrMethodNotAllowed(err) {
|
||||
t.Fatalf("retention callback lost marker: %+v %v", oi, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
opts.EvalMetadataFn = func(*ObjectInfo, error) (ReplicateDecision, error) {
|
||||
metadata++
|
||||
if loseCopies {
|
||||
// Inject a disk-state change after preflight, before the set's
|
||||
// metadata-update read. It returns NotFound at read quorum.
|
||||
for _, disk := range all[:8] {
|
||||
if err := disk.DeleteVersion(t.Context(), bucket, name, FileInfo{VersionID: opts.VersionID}, false, DeleteOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
partial := make([]StorageAPI, len(all))
|
||||
copy(partial, all[:8])
|
||||
er.getDisks = func() []StorageAPI { return partial }
|
||||
}
|
||||
decision := ReplicateDecision{}
|
||||
decision.Set(newReplicateTargetDecision("arn1", true, false))
|
||||
return decision, nil
|
||||
}
|
||||
_, err := z.DeleteObject(t.Context(), bucket, name, opts)
|
||||
if loseCopies {
|
||||
if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Errorf("pending update misclassified as physical purge: %T %v", err, err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Errorf("pending update failed: %v", err)
|
||||
}
|
||||
want := 16
|
||||
if loseCopies {
|
||||
want = 8
|
||||
}
|
||||
if got := countPurgeMarkers(t, all, bucket, name, opts.VersionID); got != want || metadata != 1 || retention != 1 {
|
||||
t.Errorf("pending update: copies=%d want=%d metadata=%d retention=%d", got, want, metadata, retention)
|
||||
}
|
||||
if !loseCopies {
|
||||
fi, err := all[0].ReadVersion(t.Context(), "", bucket, name, opts.VersionID, ReadOptions{})
|
||||
if err != nil || fi.VersionPurgeStatus() != replication.VersionPurgePending {
|
||||
t.Errorf("pending state not persisted: %+v %v", fi.ReplicationState, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMarkerReceivingMetadataUpdate(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
defer replicationTestCapacity(z)()
|
||||
for _, state := range []VersionPurgeStatusType{replication.VersionPurgePending, replication.VersionPurgeFailed} {
|
||||
t.Run(string(state), func(t *testing.T) {
|
||||
name := "update-" + mustGetUUID()
|
||||
_, opts := seedPurgeMarker(t, z.serverPools[0].getHashedSet(name), bucket, name, false)
|
||||
create := opts
|
||||
create.DeleteMarker = true
|
||||
if _, err := z.serverPools[1].DeleteObject(t.Context(), bucket, name, create); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.DeleteReplication = ReplicationState{VersionPurgeStatusInternal: string(state)}
|
||||
if _, err := z.DeleteObject(t.Context(), bucket, name, opts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, pool := range z.serverPools {
|
||||
if got := countPurgeMarkers(t, pool.getHashedSet(name).getDisks(), bucket, name, opts.VersionID); got != 16 {
|
||||
t.Errorf("metadata update removed marker copies in pool %d: %d", i, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionPurgeRetentionGate(t *testing.T) {
|
||||
z, er, all, bucket := markerPurgeFixture(t, 4)
|
||||
data, _ := seedPurgeMarker(t, er, bucket, "protected-data", true)
|
||||
denied := errors.New("retention denied")
|
||||
opts := ObjectOptions{Versioned: true, VersionID: data, EvalRetentionBypassFn: func(oi ObjectInfo, err error) error {
|
||||
if err != nil || oi.VersionID != data || oi.DeleteMarker {
|
||||
t.Errorf("wrong retained version: %+v error=%v", oi, err)
|
||||
}
|
||||
return denied
|
||||
}}
|
||||
_, err := z.DeleteObject(t.Context(), bucket, "protected-data", opts)
|
||||
if !errors.Is(err, denied) {
|
||||
t.Fatalf("retention gate: %v", err)
|
||||
}
|
||||
for _, disk := range all {
|
||||
if _, err := disk.ReadVersion(t.Context(), "", bucket, "protected-data", data, ReadOptions{}); err != nil {
|
||||
t.Errorf("protected version changed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func markerPurgeFixture(t *testing.T, n int) (*erasureServerPools, *erasureObjects, []StorageAPI, string) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
obj, dirs, err := prepareErasure(ctx, n)
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatal(err)
|
||||
}
|
||||
restore := replicationTestCapacity(obj)
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
restore()
|
||||
obj.Shutdown(context.Background())
|
||||
removeRoots(dirs)
|
||||
})
|
||||
bucket := getRandomBucketName()
|
||||
if err := obj.MakeBucket(ctx, bucket, MakeBucketOptions{VersioningEnabled: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
er := z.serverPools[0].sets[0]
|
||||
original := er.getDisks
|
||||
t.Cleanup(func() { er.getDisks = original })
|
||||
return z, er, append([]StorageAPI(nil), original()...), bucket
|
||||
}
|
||||
|
||||
func seedPurgeMarker(t *testing.T, er *erasureObjects, bucket, name string, withData bool) (string, ObjectOptions) {
|
||||
t.Helper()
|
||||
dataVersion := ""
|
||||
if withData {
|
||||
oi, err := er.PutObject(t.Context(), bucket, name, mustGetPutObjReader(t, bytes.NewReader([]byte("data")), 4, "", ""), ObjectOptions{Versioned: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dataVersion = oi.VersionID
|
||||
}
|
||||
opts := ObjectOptions{Versioned: true, VersionID: mustGetUUID(), DeleteMarker: true, ReplicationRequest: true, MTime: UTCNow().Add(-time.Hour), NoAuditLog: true}
|
||||
opts.SetReplicaStatus(replication.Replica)
|
||||
if _, err := er.DeleteObject(t.Context(), bucket, name, opts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.DeleteMarker = false
|
||||
return dataVersion, opts
|
||||
}
|
||||
|
||||
func countPurgeMarkers(t *testing.T, disks []StorageAPI, bucket, name, version string) int {
|
||||
t.Helper()
|
||||
n := 0
|
||||
for _, disk := range disks {
|
||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, name, version, ReadOptions{})
|
||||
if err == nil && fi.Deleted {
|
||||
n++
|
||||
} else if err != errFileNotFound && err != errFileVersionNotFound {
|
||||
t.Fatalf("unexpected disk version: deleted=%v error=%v", fi.Deleted, err)
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func TestDeleteMarkerPurgeQuorum(t *testing.T) {
|
||||
for _, tc := range []struct{ disks, online int }{{4, 2}, {16, 7}, {16, 8}, {16, 9}} {
|
||||
t.Run(fmt.Sprintf("%d_disks_%d_online", tc.disks, tc.online), func(t *testing.T) {
|
||||
z, er, all, bucket := markerPurgeFixture(t, tc.disks)
|
||||
const name = "marker"
|
||||
dataVersion, opts := seedPurgeMarker(t, er, bucket, name, true)
|
||||
partial := make([]StorageAPI, len(all))
|
||||
copy(partial, all[:tc.online])
|
||||
er.getDisks = func() []StorageAPI { return partial }
|
||||
_, first := z.DeleteObject(t.Context(), bucket, name, opts)
|
||||
_, retry := z.DeleteObject(t.Context(), bucket, name, opts)
|
||||
if tc.online <= tc.disks/2 {
|
||||
if !isErrWriteQuorum(first) || !isErrWriteQuorum(retry) {
|
||||
t.Errorf("below write quorum: first=%T %v, retry=%T %v", first, first, retry, retry)
|
||||
}
|
||||
} else if first != nil || (retry != nil && !isErrVersionNotFound(retry) && !isErrObjectNotFound(retry)) {
|
||||
t.Errorf("write quorum available: first=%v retry=%v", first, retry)
|
||||
}
|
||||
want := tc.disks - tc.online
|
||||
if tc.online < tc.disks/2 {
|
||||
want = tc.disks
|
||||
}
|
||||
if got := countPurgeMarkers(t, all, bucket, name, opts.VersionID); got != want {
|
||||
t.Errorf("partial purge markers=%d want=%d", got, want)
|
||||
}
|
||||
er.getDisks = func() []StorageAPI { return all }
|
||||
_, err := z.DeleteObject(t.Context(), bucket, name, opts)
|
||||
if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Errorf("full-online retry: %v", err)
|
||||
}
|
||||
if tc.online <= tc.disks/2 {
|
||||
if got := countPurgeMarkers(t, all, bucket, name, opts.VersionID); got != 0 {
|
||||
t.Errorf("full-online retry recreated/retained %d marker copies", got)
|
||||
}
|
||||
}
|
||||
// A quorum-confirmed missing retry can leave minority residue. Exercise
|
||||
// the real dangling-version healer after every disk has returned.
|
||||
_, _ = er.HealObject(t.Context(), bucket, name, opts.VersionID, madmin.HealOpts{Remove: true})
|
||||
if got := countPurgeMarkers(t, all, bucket, name, opts.VersionID); got != 0 {
|
||||
t.Errorf("marker copies after retry and heal=%d", got)
|
||||
}
|
||||
oi, err := z.GetObjectInfo(t.Context(), bucket, name, ObjectOptions{Versioned: true})
|
||||
if err != nil || oi.DeleteMarker || oi.VersionID != dataVersion {
|
||||
t.Errorf("underlying version not visible: %+v error=%v", oi, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMarkerPurgeMissingKey(t *testing.T) {
|
||||
for _, pooled := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("pooled_%v", pooled), func(t *testing.T) {
|
||||
z, er, all, bucket := markerPurgeFixture(t, 4)
|
||||
_, opts := seedPurgeMarker(t, er, bucket, "marker-only", false)
|
||||
er.getDisks = func() []StorageAPI { return []StorageAPI{all[0], all[1], nil, nil} }
|
||||
deleteFn := er.DeleteObject
|
||||
if pooled {
|
||||
deleteFn = z.DeleteObject
|
||||
}
|
||||
_, _ = deleteFn(t.Context(), bucket, "marker-only", opts)
|
||||
_, err := deleteFn(t.Context(), bucket, "marker-only", opts)
|
||||
if !isErrWriteQuorum(err) {
|
||||
t.Errorf("missing retry with only 2/4 absence votes returned %T %v", err, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMarkerHealReplicationIdentity(t *testing.T) {
|
||||
obj, er, all, bucket := markerPurgeFixture(t, 4)
|
||||
const name = "healed-marker"
|
||||
_, opts := seedPurgeMarker(t, er, bucket, name, true)
|
||||
original, err := all[3].ReadVersion(t.Context(), "", bucket, name, opts.VersionID, ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, disk := range all[:2] {
|
||||
if err := disk.DeleteVersion(t.Context(), bucket, name, FileInfo{VersionID: opts.VersionID}, false, DeleteOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := er.HealObject(t.Context(), bucket, name, opts.VersionID, madmin.HealOpts{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, disk := range all {
|
||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, name, opts.VersionID, ReadOptions{})
|
||||
if err != nil || !maps.Equal(fi.Metadata, original.Metadata) {
|
||||
t.Errorf("disk %d lost marker metadata: before=%v after=%v error=%v", i, original.Metadata, fi.Metadata, err)
|
||||
}
|
||||
}
|
||||
// Only the healed half remains readable. Read actual disk state before
|
||||
// invoking scanner replication, rather than constructing an ObjectInfo.
|
||||
er.getDisks = func() []StorageAPI { return []StorageAPI{all[0], all[1], nil, nil} }
|
||||
oi, err := er.GetObjectInfo(t.Context(), bucket, name, ObjectOptions{VersionID: opts.VersionID, Versioned: true})
|
||||
if !oi.DeleteMarker || !isErrMethodNotAllowed(err) {
|
||||
t.Fatalf("healed marker unreadable: %+v %v", oi, err)
|
||||
}
|
||||
er.getDisks = func() []StorageAPI { return all }
|
||||
var outbound atomic.Int32
|
||||
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodDelete {
|
||||
outbound.Add(1)
|
||||
if r.Header.Get(xhttp.MinIOSourceDeleteMarker) != "true" || r.URL.Query().Get("versionId") != opts.VersionID {
|
||||
t.Errorf("unexpected outbound request: %s %s", r.Method, r.URL)
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer remote.Close()
|
||||
client, err := minio.New(strings.TrimPrefix(remote.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
arn := "arn:minio:replication::" + mustGetUUID() + ":bucket"
|
||||
target := &TargetClient{Client: client, ARN: arn, Bucket: "target"}
|
||||
oldTargets := globalBucketTargetSys
|
||||
globalBucketTargetSys = &BucketTargetSys{arnRemotesMap: map[string]arnTarget{arn: {Client: target, lastRefresh: UTCNow()}}, targetsMap: map[string][]madmin.BucketTarget{bucket: {{Arn: arn, TargetBucket: "target"}}}, hc: map[string]epHealth{client.EndpointURL().Host: {Online: true}}}
|
||||
defer func() { globalBucketTargetSys = oldTargets }()
|
||||
cfg := configs[0]
|
||||
cfg.RoleArn = arn
|
||||
meta, err := globalBucketMetadataSys.Get(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta.replicationConfig = &cfg
|
||||
globalBucketMetadataSys.Set(bucket, meta)
|
||||
worker := make(chan ReplicationWorkerOperation, 1)
|
||||
oldPool := globalReplicationPool
|
||||
globalReplicationPool = once.NewSingleton[ReplicationPool]()
|
||||
globalReplicationPool.Set(&ReplicationPool{ctx: t.Context(), objLayer: obj, workers: []chan ReplicationWorkerOperation{worker}, stats: globalReplicationStats.Load(), mrfSaveCh: make(chan MRFReplicateEntry, 1)})
|
||||
defer func() { globalReplicationPool = oldPool }()
|
||||
roi := queueReplicationHeal(t.Context(), bucket, oi, replicationConfig{Config: &cfg, remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{Arn: arn, TargetBucket: "target"}}}}, 0)
|
||||
select {
|
||||
case op := <-worker:
|
||||
d := op.(DeletedObjectReplicationInfo)
|
||||
result := replicateDeleteToTarget(t.Context(), d, target)
|
||||
t.Errorf("healed replica scheduled for creation: version=%s existing=%v result=%+v", d.DeleteMarkerVersionID, roi.ExistingObjResync.mustResync(), result)
|
||||
default:
|
||||
}
|
||||
if outbound.Load() != 0 {
|
||||
t.Errorf("healed replica sent %d new marker creations", outbound.Load())
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"reflect"
|
||||
@@ -110,6 +111,96 @@ func TestReduceErrs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestReduceWriteQuorumErrsCommitMetaMissing verifies that an in-quorum
|
||||
// errCommitMetaMissing reduces to the sentinel, which renameData() converts
|
||||
// to errErasureWriteQuorum, while below-quorum reduces directly to
|
||||
// errErasureWriteQuorum.
|
||||
func TestReduceWriteQuorumErrsCommitMetaMissing(t *testing.T) {
|
||||
errs := make([]error, 16)
|
||||
for i := range 9 {
|
||||
errs[i] = errCommitMetaMissing
|
||||
}
|
||||
if err := reduceWriteQuorumErrs(t.Context(), errs, objectOpIgnoredErrs, 9); !errors.Is(err, errCommitMetaMissing) {
|
||||
t.Errorf("expected %s, got %s", errCommitMetaMissing, err)
|
||||
}
|
||||
// The sentinel must be distinguishable from errFileNotFound so that
|
||||
// commit failures are never mapped to ObjectNotFound (404).
|
||||
if errors.Is(errCommitMetaMissing, errFileNotFound) {
|
||||
t.Error("errCommitMetaMissing must not match errFileNotFound")
|
||||
}
|
||||
|
||||
// Below quorum the reduction must yield a write-quorum failure (503);
|
||||
// eight sentinels against eight committed drives is not a quorum.
|
||||
belowQuorum := make([]error, 16)
|
||||
for i := range 8 {
|
||||
belowQuorum[i] = errCommitMetaMissing
|
||||
}
|
||||
if err := reduceWriteQuorumErrs(t.Context(), belowQuorum, objectOpIgnoredErrs, 9); !errors.Is(err, errErasureWriteQuorum) {
|
||||
t.Errorf("expected %s, got %s", errErasureWriteQuorum, err)
|
||||
}
|
||||
}
|
||||
|
||||
// renameDataFailingDisk is a StorageAPI stub returning a fixed error
|
||||
// from RenameData, enough to drive renameData() in unit tests.
|
||||
type renameDataFailingDisk struct {
|
||||
StorageAPI
|
||||
err error
|
||||
undoWrite bool
|
||||
}
|
||||
|
||||
func (d *renameDataFailingDisk) RenameData(context.Context, string, string, FileInfo, string, string, RenameOptions) (RenameDataResp, error) {
|
||||
return RenameDataResp{}, d.err
|
||||
}
|
||||
|
||||
func (d *renameDataFailingDisk) DeleteVersion(_ context.Context, _, _ string, _ FileInfo, _ bool, opts DeleteOptions) error {
|
||||
d.undoWrite = opts.UndoWrite
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestRenameDataCommitMetaMissing verifies that renameData() converts an
|
||||
// in-quorum errCommitMetaMissing into errErasureWriteQuorum (HTTP 503),
|
||||
// while an in-quorum errFileNotFound keeps its meaning so that
|
||||
// PutObject() still reports IncompleteBody for client disconnects.
|
||||
func TestRenameDataCommitMetaMissing(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
failures int
|
||||
diskErr error
|
||||
wantErr error
|
||||
}{
|
||||
{"all_commit_metadata_missing", 16, errCommitMetaMissing, errErasureWriteQuorum},
|
||||
{"quorum_commit_metadata_missing", 9, errCommitMetaMissing, errErasureWriteQuorum},
|
||||
{"neither_side_has_quorum", 8, errCommitMetaMissing, errErasureWriteQuorum},
|
||||
{"successful_write_quorum", 7, errCommitMetaMissing, nil},
|
||||
{"staging_failure", 16, errFileNotFound, errFileNotFound},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
metadata := make([]FileInfo, 16)
|
||||
disks := make([]StorageAPI, 16)
|
||||
for i := range disks {
|
||||
metadata[i] = FileInfo{Deleted: true} // valid without erasure info
|
||||
disk := &renameDataFailingDisk{}
|
||||
if i < tc.failures {
|
||||
disk.err = tc.diskErr
|
||||
}
|
||||
disks[i] = disk
|
||||
}
|
||||
_, _, _, err := renameData(t.Context(), disks, minioMetaTmpBucket, "tmp", metadata, "bucket", "object", 9)
|
||||
if !errors.Is(err, tc.wantErr) {
|
||||
t.Fatalf("expected %v, got %v", tc.wantErr, err)
|
||||
}
|
||||
for i, disk := range disks {
|
||||
// Failed writes undo only the drives that committed. A
|
||||
// successful quorum must never be rolled back.
|
||||
wantUndo := tc.wantErr != nil && i >= tc.failures
|
||||
if got := disk.(*renameDataFailingDisk).undoWrite; got != wantUndo {
|
||||
t.Errorf("disk %d: UndoWrite = %v, want %v", i, got, wantUndo)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHashOrder - test order of ints in array
|
||||
func TestHashOrder(t *testing.T) {
|
||||
testCases := []struct {
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
// Copyright (c) 2026 clown15
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestExtractModTimeFromUploadIDDir(t *testing.T) {
|
||||
now := time.Unix(1792056747, 123456789).UTC()
|
||||
nano := now.UnixNano()
|
||||
uuid := "d16ad42a-6fe5-4a3b-8092-69983142238f"
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
dirName string
|
||||
expectedMod time.Time
|
||||
wantZero bool
|
||||
}{
|
||||
{
|
||||
name: "Modern Plaintext Format (<UUID>x<UnixNano>)",
|
||||
dirName: fmt.Sprintf("%sx%d", uuid, nano),
|
||||
expectedMod: time.Unix(0, nano),
|
||||
wantZero: false,
|
||||
},
|
||||
{
|
||||
// This valid UUID decodes to bytes containing 'x'. Checking
|
||||
// for that byte before parsing plaintext loses the timestamp.
|
||||
name: "Plaintext Also Decodes To Base64 Containing x",
|
||||
dirName: fmt.Sprintf("4eacd477-e733-4851-b821-6fd42785ff87x%d", nano),
|
||||
expectedMod: now,
|
||||
},
|
||||
{
|
||||
name: "Legacy Format Without Timestamp (<UUID>)",
|
||||
dirName: uuid,
|
||||
expectedMod: time.Time{},
|
||||
wantZero: true,
|
||||
},
|
||||
{
|
||||
name: "Base64 Encoded Format",
|
||||
dirName: base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%sx%d", uuid, nano))),
|
||||
expectedMod: time.Unix(0, nano),
|
||||
wantZero: false,
|
||||
},
|
||||
{
|
||||
name: "Encoded Format With Deployment Prefix",
|
||||
dirName: base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%s.%sx%d", uuid, uuid, nano))),
|
||||
expectedMod: now,
|
||||
},
|
||||
{
|
||||
name: "Malformed Suffix (<UUID>xinvalid)",
|
||||
dirName: fmt.Sprintf("%sxnotanumber", uuid),
|
||||
expectedMod: time.Time{},
|
||||
wantZero: true,
|
||||
},
|
||||
{
|
||||
name: "Timestamp Overflow",
|
||||
dirName: uuid + "x9223372036854775808",
|
||||
wantZero: true,
|
||||
},
|
||||
{
|
||||
name: "Malformed Plaintext UUID",
|
||||
dirName: fmt.Sprintf("not-a-uuidx%d", nano),
|
||||
wantZero: true,
|
||||
},
|
||||
{
|
||||
name: "Empty Directory Name",
|
||||
dirName: "",
|
||||
expectedMod: time.Time{},
|
||||
wantZero: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := extractModTimeFromUploadIDDir(tc.dirName)
|
||||
if tc.wantZero {
|
||||
if !got.IsZero() {
|
||||
t.Fatalf("expected zero time for %q, got %v", tc.dirName, got)
|
||||
}
|
||||
return
|
||||
}
|
||||
if !got.Equal(tc.expectedMod) {
|
||||
t.Fatalf("extractModTimeFromUploadIDDir(%q) = %v; want %v", tc.dirName, got, tc.expectedMod)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type multipartCleanupDisk struct {
|
||||
StorageAPI
|
||||
root string
|
||||
modTime time.Time
|
||||
readErr error
|
||||
reads int
|
||||
}
|
||||
|
||||
func (d *multipartCleanupDisk) Endpoint() Endpoint {
|
||||
return Endpoint{URL: &url.URL{Path: d.root}}
|
||||
}
|
||||
|
||||
func (d *multipartCleanupDisk) ReadVersion(context.Context, string, string, string, string, ReadOptions) (FileInfo, error) {
|
||||
d.reads++
|
||||
return FileInfo{ModTime: d.modTime}, d.readErr
|
||||
}
|
||||
|
||||
func (d *multipartCleanupDisk) StatVol(context.Context, string) (VolInfo, error) {
|
||||
return VolInfo{Created: time.Now()}, nil
|
||||
}
|
||||
|
||||
func TestCleanupStaleUploadsOnDisk(t *testing.T) {
|
||||
const uuid = "d16ad42a-6fe5-4a3b-8092-69983142238f"
|
||||
expiry := globalAPIConfig.getStaleUploadsExpiry()
|
||||
for _, tc := range []struct {
|
||||
name, prefix string
|
||||
age time.Duration
|
||||
legacy bool
|
||||
readErr error
|
||||
wantReads int
|
||||
wantDeleted bool
|
||||
}{
|
||||
{name: "recent", prefix: uuid, age: expiry / 2, readErr: errFileNotFound},
|
||||
{name: "plaintext_base64_collision", prefix: "4eacd477-e733-4851-b821-6fd42785ff87", age: expiry / 2, readErr: errFileNotFound},
|
||||
{name: "expired", prefix: uuid, age: 2 * expiry, readErr: errFileNotFound, wantDeleted: true},
|
||||
{name: "legacy_recent", prefix: uuid, age: expiry / 2, legacy: true, wantReads: 1},
|
||||
{name: "legacy_expired", prefix: uuid, age: 2 * expiry, legacy: true, wantReads: 1, wantDeleted: true},
|
||||
{name: "legacy_metadata_missing", prefix: uuid, age: 2 * expiry, legacy: true, readErr: errFileNotFound, wantReads: 1},
|
||||
{name: "malformed_metadata_missing", prefix: "not-a-uuid", age: 2 * expiry, readErr: errFileNotFound, wantReads: 1},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
modTime := time.Now().Add(-tc.age)
|
||||
id := tc.prefix
|
||||
if !tc.legacy {
|
||||
id = fmt.Sprintf("%sx%d", tc.prefix, modTime.UnixNano())
|
||||
}
|
||||
disk := &multipartCleanupDisk{root: t.TempDir(), modTime: modTime, readErr: tc.readErr}
|
||||
uploadDir := filepath.Join(disk.root, minioMetaMultipartBucket, "sha", id)
|
||||
if err := os.MkdirAll(uploadDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk)
|
||||
if disk.reads != tc.wantReads {
|
||||
t.Errorf("ReadVersion calls = %d, want %d", disk.reads, tc.wantReads)
|
||||
}
|
||||
_, err := os.Stat(uploadDir)
|
||||
if tc.wantDeleted {
|
||||
if !os.IsNotExist(err) {
|
||||
t.Errorf("expired upload must be removed: %v", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Errorf("upload must be preserved: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -253,7 +253,7 @@ func testAPICompleteMultipartFullObjectChecksumMismatch(obj ObjectLayer, instanc
|
||||
if apiErr.Code != "BadDigest" {
|
||||
t.Fatalf("%s: expected BadDigest, got %q", instanceType, apiErr.Code)
|
||||
}
|
||||
if want := "The CRC32 checksum you specified did not match the calculated checksum."; apiErr.Message != want {
|
||||
if want := "The CRC32 you specified did not match the calculated checksum."; apiErr.Message != want {
|
||||
t.Fatalf("%s: expected message %q, got %q", instanceType, want, apiErr.Message)
|
||||
}
|
||||
|
||||
@@ -361,7 +361,7 @@ func testAPICompleteMultipartCompositeChecksumMismatch(obj ObjectLayer, instance
|
||||
if apiErr.Code != "BadDigest" {
|
||||
t.Fatalf("%s: expected BadDigest, got %q", instanceType, apiErr.Code)
|
||||
}
|
||||
if want := "The CRC32 checksum you specified did not match the calculated checksum."; apiErr.Message != want {
|
||||
if want := "The CRC32 you specified did not match the calculated checksum."; apiErr.Message != want {
|
||||
t.Fatalf("%s: expected message %q, got %q", instanceType, want, apiErr.Message)
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err == nil {
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
// Copyright (c) 2026 Ruohang Feng
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Run explicitly with -benchtime=1x. Fixture creation is excluded from timing;
|
||||
// each request still discovers real on-disk metadata. The 16-drive/10k case
|
||||
// deliberately exceeds the existing aggregate directory-entry budget.
|
||||
func BenchmarkMultipartListingCapacity(b *testing.B) {
|
||||
for _, tc := range []struct {
|
||||
drives, uploads int
|
||||
mixed, limited bool
|
||||
}{
|
||||
{4, 1_000, false, false},
|
||||
{4, 10_000, false, false},
|
||||
{4, 10_000, true, false},
|
||||
{16, 1_000, false, false},
|
||||
{16, 10_000, false, true},
|
||||
} {
|
||||
b.Run(fmt.Sprintf("drives=%d/uploads=%d/mixed=%t", tc.drives, tc.uploads, tc.mixed), func(b *testing.B) {
|
||||
obj, dirs, err := prepareErasure(b.Context(), tc.drives)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
b.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
|
||||
const bucket, other = "multipart-capacity", "multipart-unrelated"
|
||||
for _, name := range []string{bucket, other} {
|
||||
if err := z.MakeBucket(b.Context(), name, MakeBucketOptions{}); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
for n := range tc.uploads {
|
||||
target := bucket
|
||||
if tc.mixed && n%2 != 0 {
|
||||
target = other
|
||||
}
|
||||
key := fmt.Sprintf("dir/%05d", n)
|
||||
mp, err := z.NewMultipartUpload(b.Context(), target, key, ObjectOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if tc.mixed && n%10 == 0 {
|
||||
rewriteMultipartIdentity(b, z.serverPools[0].getHashedSet(key), target, key, mp.UploadID, removeMultipartIdentity)
|
||||
}
|
||||
}
|
||||
z.mpCache.Clear()
|
||||
var directoryCalls, metadataCalls atomic.Int64
|
||||
for _, set := range z.serverPools[0].sets {
|
||||
original := set.getDisks
|
||||
set.getDisks = func() []StorageAPI {
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i, disk := range disks {
|
||||
disks[i] = multipartListingCountingDisk{disk, &directoryCalls, &metadataCalls}
|
||||
}
|
||||
return disks
|
||||
}
|
||||
b.Cleanup(func() { set.getDisks = original })
|
||||
}
|
||||
b.ResetTimer()
|
||||
for range b.N {
|
||||
got, err := z.ListMultipartUploads(b.Context(), bucket, "dir/", "", "", "", 1000)
|
||||
if tc.limited {
|
||||
var limited SlowDown
|
||||
if !errors.As(err, &limited) || len(got.Uploads) != 0 {
|
||||
b.Fatalf("overflow must return SlowDown without a partial page: %+v %v", got, err)
|
||||
}
|
||||
} else if err != nil || len(got.Uploads) != 1000 {
|
||||
b.Fatalf("capacity listing: count=%d err=%v", len(got.Uploads), err)
|
||||
}
|
||||
}
|
||||
b.StopTimer()
|
||||
b.ReportMetric(float64(directoryCalls.Load())/float64(b.N), "ListDir/op")
|
||||
b.ReportMetric(float64(metadataCalls.Load())/float64(b.N), "ReadVersion/op")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,404 @@
|
||||
// Copyright (c) 2026 mr javad seydi and Ruohang Feng
|
||||
//
|
||||
// This file is part of Silo Object Storage stack.
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const multipartScanEntryLimit = 100_000
|
||||
|
||||
var (
|
||||
multipartScanSlots = make(chan struct{}, 2)
|
||||
errMultipartListingIdentity = errors.New("multipart upload identity is invalid")
|
||||
)
|
||||
|
||||
// One budget and admission slot cover the entire request, including all pools.
|
||||
// A slot is released only after the scan workers have actually stopped.
|
||||
type multipartScan struct {
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
remaining atomic.Int64
|
||||
metadataSlots chan struct{}
|
||||
preflight bool
|
||||
sets []multipartScanSet
|
||||
}
|
||||
|
||||
type multipartScanSet struct {
|
||||
Pool int `json:"pool"`
|
||||
Set int `json:"set"`
|
||||
Drives int `json:"drives"`
|
||||
ScannedDrives int `json:"scannedDrives"`
|
||||
UncoveredDrives []int `json:"uncoveredDrives,omitempty"`
|
||||
Candidates int `json:"candidates"`
|
||||
LegacyUploads int `json:"legacyUploads"`
|
||||
OldestLegacy time.Time `json:"oldestLegacy,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type multipartPreflightReport struct {
|
||||
Ready bool `json:"ready"`
|
||||
Complete bool `json:"complete"`
|
||||
Mode string `json:"mode"`
|
||||
ScannedEntries int64 `json:"scannedEntries"`
|
||||
LegacyUploads int `json:"legacyUploads"`
|
||||
Sets []multipartScanSet `json:"sets"`
|
||||
}
|
||||
|
||||
func startMultipartScan(ctx context.Context, preflight bool) (*multipartScan, error) {
|
||||
select {
|
||||
case multipartScanSlots <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
return nil, SlowDown{}
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
s := &multipartScan{ctx: ctx, cancel: cancel, preflight: preflight, metadataSlots: make(chan struct{}, multipartMetadataScanConcurrency)}
|
||||
s.remaining.Store(multipartScanEntryLimit)
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *multipartScan) close() {
|
||||
s.cancel()
|
||||
<-multipartScanSlots
|
||||
}
|
||||
|
||||
func (s *multipartScan) listDir(disk StorageAPI, bucket, dir string) ([]string, error) {
|
||||
if err := s.ctx.Err(); err != nil {
|
||||
return nil, SlowDown{}
|
||||
}
|
||||
remaining := s.remaining.Load()
|
||||
if remaining <= 0 {
|
||||
return nil, SlowDown{}
|
||||
}
|
||||
// Request one extra entry to detect overflow, never a silently partial page.
|
||||
entries, err := disk.ListDir(s.ctx, bucket, minioMetaMultipartBucket, dir, int(remaining)+1)
|
||||
if errors.Is(err, errFileNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.remaining.Add(-int64(len(entries))) < 0 {
|
||||
return nil, SlowDown{}
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func (s *multipartScan) listUploadDirs(disk StorageAPI, bucket string) ([]string, error) {
|
||||
hashDirs, err := s.listDir(disk, bucket, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var candidates []string
|
||||
for _, hashDir := range hashDirs {
|
||||
if !strings.HasSuffix(hashDir, SlashSeparator) {
|
||||
continue
|
||||
}
|
||||
hashDir = strings.TrimSuffix(hashDir, SlashSeparator)
|
||||
uploadDirs, err := s.listDir(disk, bucket, hashDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, uploadDir := range uploadDirs {
|
||||
if strings.HasSuffix(uploadDir, SlashSeparator) {
|
||||
candidates = append(candidates, pathJoin(hashDir, strings.TrimSuffix(uploadDir, SlashSeparator)))
|
||||
}
|
||||
}
|
||||
}
|
||||
return candidates, nil
|
||||
}
|
||||
|
||||
// Identity is immutable at hash/uploadUUID. Check the hash before using even
|
||||
// a single source drive to exclude another bucket. Missing/bad identities must
|
||||
// fall back to the full metadata read; they cannot prove absence.
|
||||
func (er erasureObjects) multipartIdentity(fi FileInfo, shaDir string) (string, string, bool) {
|
||||
bucket, object := fi.Metadata[multipartMetaBucket], fi.Metadata[multipartMetaObject]
|
||||
return bucket, object, bucket != "" && object != "" && IsValidBucketName(bucket) &&
|
||||
IsValidObjectPrefix(object) && er.getMultipartSHADir(bucket, object) == shaDir
|
||||
}
|
||||
|
||||
// Only the absence of both fields identifies the historical format. Empty,
|
||||
// partial or invalid identities must not hide a damaged new upload as legacy.
|
||||
func multipartLegacyIdentity(fi FileInfo) bool {
|
||||
_, bucketPresent := fi.Metadata[multipartMetaBucket]
|
||||
_, objectPresent := fi.Metadata[multipartMetaObject]
|
||||
return !bucketPresent && !objectPresent
|
||||
}
|
||||
|
||||
func (er erasureObjects) readMultipartUploadCandidate(s *multipartScan, bucket, candidate string, source StorageAPI) (MultipartInfo, bool, bool, error) {
|
||||
if s.ctx.Err() != nil {
|
||||
return MultipartInfo{}, false, false, SlowDown{}
|
||||
}
|
||||
shaDir, uploadUUID, ok := strings.Cut(candidate, SlashSeparator)
|
||||
if !ok || shaDir == "" || uploadUUID == "" || strings.Contains(uploadUUID, SlashSeparator) {
|
||||
return MultipartInfo{}, false, false, errMultipartListingIdentity
|
||||
}
|
||||
if !s.preflight && bucket != "" && source != nil {
|
||||
fi, err := source.ReadVersion(s.ctx, bucket, minioMetaMultipartBucket, candidate, "", ReadOptions{})
|
||||
if err == nil {
|
||||
storedBucket, _, valid := er.multipartIdentity(fi, shaDir)
|
||||
if valid && storedBucket != bucket {
|
||||
return MultipartInfo{}, false, false, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
if s.ctx.Err() != nil {
|
||||
return MultipartInfo{}, false, false, SlowDown{}
|
||||
}
|
||||
select {
|
||||
case s.metadataSlots <- struct{}{}:
|
||||
defer func() { <-s.metadataSlots }()
|
||||
case <-s.ctx.Done():
|
||||
return MultipartInfo{}, false, false, SlowDown{}
|
||||
}
|
||||
disks := er.getDisks()
|
||||
metadata, errs := readAllFileInfo(s.ctx, disks, bucket, minioMetaMultipartBucket, candidate, "", false, false)
|
||||
if s.preflight {
|
||||
// The diagnostic counts even a single readable legacy copy. It does not
|
||||
// gate listing; unreadable copies still prevent a complete diagnosis.
|
||||
var oldest MultipartInfo
|
||||
legacy := false
|
||||
_, nativeID := multipartUploadTime(uploadUUID)
|
||||
for i, err := range errs {
|
||||
if errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return MultipartInfo{}, false, false, err
|
||||
}
|
||||
fi := metadata[i]
|
||||
storedBucket, storedObject, valid := er.multipartIdentity(fi, shaDir)
|
||||
isLegacy := multipartLegacyIdentity(fi)
|
||||
if !isLegacy && (!valid || !nativeID) {
|
||||
return MultipartInfo{}, false, false, errMultipartListingIdentity
|
||||
}
|
||||
if isLegacy {
|
||||
info := multipartUploadInfo(storedBucket, storedObject, uploadUUID, fi.ModTime)
|
||||
if !legacy || info.Initiated.Before(oldest.Initiated) {
|
||||
oldest = info
|
||||
}
|
||||
legacy = true
|
||||
}
|
||||
}
|
||||
if legacy {
|
||||
return oldest, false, true, nil
|
||||
}
|
||||
}
|
||||
readQuorum, _, err := objectQuorumFromMeta(s.ctx, metadata, errs, er.defaultParityCount)
|
||||
if err != nil {
|
||||
return MultipartInfo{}, false, false, err
|
||||
}
|
||||
_, modTime, etag := listOnlineDisks(disks, metadata, errs, readQuorum)
|
||||
if err := reduceReadQuorumErrs(s.ctx, errs, objectOpIgnoredErrs, readQuorum); err != nil {
|
||||
return MultipartInfo{}, false, false, err
|
||||
}
|
||||
fi, err := pickValidFileInfo(s.ctx, metadata, modTime, etag, readQuorum)
|
||||
if err != nil {
|
||||
return MultipartInfo{}, false, false, err
|
||||
}
|
||||
// The generic metadata quorum does not compare these identity fields.
|
||||
// Contradictory readable copies cannot certify a historical upload: picking
|
||||
// a copy with missing fields must not silently hide a new-format record.
|
||||
legacy := multipartLegacyIdentity(fi)
|
||||
storedBucket, storedObject, valid := er.multipartIdentity(fi, shaDir)
|
||||
for i, other := range metadata {
|
||||
if errs[i] != nil || !other.IsValid() {
|
||||
continue
|
||||
}
|
||||
if multipartLegacyIdentity(other) != legacy ||
|
||||
(!legacy && (other.Metadata[multipartMetaBucket] != storedBucket || other.Metadata[multipartMetaObject] != storedObject)) {
|
||||
return MultipartInfo{}, false, false, fmt.Errorf("%w: inconsistent copies at %s", errMultipartListingIdentity, candidate)
|
||||
}
|
||||
}
|
||||
info := multipartUploadInfo(storedBucket, storedObject, uploadUUID, fi.ModTime)
|
||||
if legacy {
|
||||
return info, false, true, nil
|
||||
}
|
||||
if !valid {
|
||||
return MultipartInfo{}, false, false, fmt.Errorf("%w: %s", errMultipartListingIdentity, candidate)
|
||||
}
|
||||
if _, ok := multipartUploadTime(uploadUUID); !ok {
|
||||
return MultipartInfo{}, false, false, fmt.Errorf("%w: %s", errMultipartListingIdentity, candidate)
|
||||
}
|
||||
if bucket != "" && bucket != storedBucket {
|
||||
return MultipartInfo{}, false, false, nil
|
||||
}
|
||||
return info, true, false, nil
|
||||
}
|
||||
|
||||
func (er erasureObjects) scanMultipartUploads(s *multipartScan, bucket string, poolIdx, setIdx int) ([]MultipartInfo, error) {
|
||||
disks := er.getDisks()
|
||||
report := multipartScanSet{Pool: poolIdx, Set: setIdx, Drives: er.setDriveCount}
|
||||
var resultErr error
|
||||
defer func() {
|
||||
if resultErr != nil {
|
||||
report.Error = resultErr.Error()
|
||||
}
|
||||
s.sets = append(s.sets, report)
|
||||
}()
|
||||
var candidateMu sync.Mutex
|
||||
candidates := make(map[string]StorageAPI)
|
||||
errs := make([]error, len(disks))
|
||||
var wg sync.WaitGroup
|
||||
for i, disk := range disks {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if disk == nil || !disk.IsOnline() {
|
||||
errs[i] = errDiskNotFound
|
||||
return
|
||||
}
|
||||
paths, err := s.listUploadDirs(disk, bucket)
|
||||
errs[i] = err
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
candidateMu.Lock()
|
||||
defer candidateMu.Unlock()
|
||||
for _, p := range paths {
|
||||
candidates[p] = disk
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
for i, err := range errs {
|
||||
if err == nil {
|
||||
report.ScannedDrives++
|
||||
} else {
|
||||
report.UncoveredDrives = append(report.UncoveredDrives, i)
|
||||
if _, limited := err.(SlowDown); limited {
|
||||
resultErr = err
|
||||
}
|
||||
}
|
||||
}
|
||||
report.Candidates = len(candidates)
|
||||
// A successful scan must intersect every metadata read quorum, including
|
||||
// records left with R copies by a partially failed cancellation.
|
||||
if report.ScannedDrives < er.setDriveCount/2+1 && resultErr == nil {
|
||||
resultErr = toObjectErr(errErasureReadQuorum, bucket)
|
||||
}
|
||||
if resultErr != nil {
|
||||
return nil, resultErr
|
||||
}
|
||||
type candidate struct {
|
||||
path string
|
||||
source StorageAPI
|
||||
}
|
||||
jobs := make(chan candidate)
|
||||
var mu sync.Mutex
|
||||
var uploads []MultipartInfo
|
||||
for range min(16, len(candidates)) {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for c := range jobs {
|
||||
mu.Lock()
|
||||
stopped := resultErr != nil
|
||||
mu.Unlock()
|
||||
if stopped || s.ctx.Err() != nil {
|
||||
continue
|
||||
}
|
||||
upload, found, legacy, err := er.readMultipartUploadCandidate(s, bucket, c.path, c.source)
|
||||
if errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound) {
|
||||
continue
|
||||
}
|
||||
mu.Lock()
|
||||
switch {
|
||||
case err != nil && resultErr == nil:
|
||||
resultErr = err
|
||||
case legacy:
|
||||
report.LegacyUploads++
|
||||
if report.OldestLegacy.IsZero() || upload.Initiated.Before(report.OldestLegacy) {
|
||||
report.OldestLegacy = upload.Initiated
|
||||
}
|
||||
case found && !s.preflight:
|
||||
uploads = append(uploads, upload)
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
dispatch:
|
||||
for p, source := range candidates {
|
||||
select {
|
||||
case jobs <- candidate{p, source}:
|
||||
case <-s.ctx.Done():
|
||||
break dispatch
|
||||
}
|
||||
}
|
||||
close(jobs)
|
||||
wg.Wait()
|
||||
if s.ctx.Err() != nil {
|
||||
resultErr = SlowDown{}
|
||||
}
|
||||
return uploads, resultErr
|
||||
}
|
||||
|
||||
// multipartPreflight scans all pools, sets and drives, independent of caches.
|
||||
// A majority suffices for normal listing; a complete diagnosis requires every
|
||||
// drive to have been inspected. Ready describes the absence of legacy copies,
|
||||
// not whether listing is enabled.
|
||||
func (z *erasureServerPools) multipartPreflight(ctx context.Context) (multipartPreflightReport, error) {
|
||||
s, err := startMultipartScan(ctx, true)
|
||||
if err != nil {
|
||||
return multipartPreflightReport{}, err
|
||||
}
|
||||
defer s.close()
|
||||
report := multipartPreflightReport{Complete: true, Mode: "persistent"}
|
||||
for p, pool := range z.serverPools {
|
||||
for i, set := range pool.sets {
|
||||
_, _ = set.scanMultipartUploads(s, "", p, i)
|
||||
}
|
||||
}
|
||||
report.Sets = s.sets
|
||||
for _, set := range s.sets {
|
||||
report.LegacyUploads += set.LegacyUploads
|
||||
if set.Error != "" || set.ScannedDrives != set.Drives {
|
||||
report.Complete = false
|
||||
}
|
||||
}
|
||||
report.ScannedEntries = multipartScanEntryLimit - s.remaining.Load()
|
||||
report.Ready = report.Complete && report.LegacyUploads == 0
|
||||
return report, nil
|
||||
}
|
||||
|
||||
// MultipartPreflightHandler is a read-only storage-admin diagnostic. It never
|
||||
// accepts an arbitrary deletion path or changes upload lifetime settings.
|
||||
func (a adminAPIHandlers) MultipartPreflightHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
obj, _ := validateAdminReq(ctx, w, r, policy.StorageInfoAdminAction)
|
||||
if obj == nil {
|
||||
return
|
||||
}
|
||||
z, ok := obj.(*erasureServerPools)
|
||||
if !ok {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
report, err := z.multipartPreflight(ctx)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
b, err := json.Marshal(report)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
writeSuccessResponseJSON(w, b)
|
||||
}
|
||||
@@ -0,0 +1,781 @@
|
||||
// Copyright (c) 2026 Ruohang Feng
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
)
|
||||
|
||||
// Check the real handler and storage path, including a successful abort between
|
||||
// pages. Choose IDs increasing in both initiation time and lexical order so the
|
||||
// failure does not depend on differing interpretations of S3 marker ordering.
|
||||
func TestMultipartListingAbortBetweenHTTPPages(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads", "AbortMultipart"}, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var firstID, secondID string
|
||||
for attempt := 0; attempt < 32; attempt++ {
|
||||
one, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
two, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if one.UploadID < two.UploadID {
|
||||
firstID, secondID = one.UploadID, two.UploadID
|
||||
break
|
||||
}
|
||||
for _, id := range []string{one.UploadID, two.UploadID} {
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "a", id, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if firstID == "" {
|
||||
t.Fatal("could not construct increasing upload IDs")
|
||||
}
|
||||
if _, err := z.NewMultipartUpload(t.Context(), bucket, "b", ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request := func(method, u string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
req, err := newTestSignedRequestV4(method, u, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
list := func(keyMarker, uploadMarker string, limit int) ListMultipartUploadsResponse {
|
||||
t.Helper()
|
||||
rec := request(http.MethodGet, getListMultipartUploadsURLWithParams("", bucket, "", keyMarker, uploadMarker, "", strconv.Itoa(limit)))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("list HTTP %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var result ListMultipartUploadsResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
first := list("", "", 1)
|
||||
if len(first.Uploads) != 1 || first.Uploads[0].UploadID != firstID || !first.IsTruncated {
|
||||
t.Fatalf("unexpected first page: %+v", first)
|
||||
}
|
||||
rec := request(http.MethodDelete, getAbortMultipartUploadURL("", bucket, "a", firstID))
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("abort HTTP %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
rest := list(first.NextKeyMarker, first.NextUploadIDMarker, 10)
|
||||
var keys []string
|
||||
for _, upload := range rest.Uploads {
|
||||
keys = append(keys, upload.Key)
|
||||
}
|
||||
t.Logf("GET first page=200; DELETE marker=204; GET next page=200 keys=%v truncated=%v", keys, rest.IsTruncated)
|
||||
if _, err := z.GetMultipartInfo(t.Context(), bucket, "a", secondID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("remaining upload is not valid: %v", err)
|
||||
}
|
||||
if len(rest.Uploads) != 2 || rest.Uploads[0].UploadID != secondID {
|
||||
t.Fatalf("valid remaining upload for key a is missing from continuation: %+v", rest.Uploads)
|
||||
}
|
||||
}
|
||||
|
||||
type multipartListingFaultDisk struct {
|
||||
StorageAPI
|
||||
read func(context.Context, string, string, string, string, ReadOptions) (FileInfo, error)
|
||||
delete func(context.Context, string, string, DeleteOptions) error
|
||||
}
|
||||
|
||||
func (d multipartListingFaultDisk) ReadVersion(ctx context.Context, original, volume, object, version string, opts ReadOptions) (FileInfo, error) {
|
||||
if d.read != nil {
|
||||
return d.read(ctx, original, volume, object, version, opts)
|
||||
}
|
||||
return d.StorageAPI.ReadVersion(ctx, original, volume, object, version, opts)
|
||||
}
|
||||
|
||||
func (d multipartListingFaultDisk) Delete(ctx context.Context, volume, object string, opts DeleteOptions) error {
|
||||
if d.delete != nil {
|
||||
return d.delete(ctx, volume, object, opts)
|
||||
}
|
||||
return d.StorageAPI.Delete(ctx, volume, object, opts)
|
||||
}
|
||||
|
||||
func TestMultipartListingLegacyPreflight(t *testing.T) {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
const other = "multipart-legacy-other"
|
||||
if err := z.MakeBucket(t.Context(), other, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old, err := z.NewMultipartUpload(t.Context(), other, "old", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fi, metadata, err := set.checkUploadIDExists(t.Context(), other, "old", old.UploadID, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range metadata {
|
||||
delete(metadata[i].Metadata, multipartMetaBucket)
|
||||
delete(metadata[i].Metadata, multipartMetaObject)
|
||||
}
|
||||
if _, err = writeAllMetadata(t.Context(), set.getDisks(), other, minioMetaMultipartBucket,
|
||||
set.getUploadIDDir(other, "old", old.UploadID), metadata, fi.WriteQuorum(set.defaultWQuorum())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z.mpCache.Clear()
|
||||
for _, target := range []string{bucket, other} {
|
||||
got, err := z.ListMultipartUploads(t.Context(), target, "", "", "", "", 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if target == bucket {
|
||||
requireMultipartUploadKeys(t, got, "a")
|
||||
} else {
|
||||
requireMultipartUploadKeys(t, got)
|
||||
}
|
||||
}
|
||||
// Exact-looking prefixes now have the same durable semantics.
|
||||
exact, err := z.ListMultipartUploads(t.Context(), other, "old", "", "", "", 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, exact)
|
||||
report, err := z.multipartPreflight(t.Context())
|
||||
if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 || report.Mode != "persistent" {
|
||||
t.Fatalf("legacy preflight: %+v %v", report, err)
|
||||
}
|
||||
if err = z.AbortMultipartUpload(t.Context(), other, "old", old.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
report, err = z.multipartPreflight(t.Context())
|
||||
if err != nil || !report.Ready || report.LegacyUploads != 0 {
|
||||
t.Fatalf("drained preflight: %+v %v", report, err)
|
||||
}
|
||||
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, got, "a")
|
||||
}
|
||||
|
||||
func TestMultipartListingIdentityFallback(t *testing.T) {
|
||||
for _, kind := range []string{"old", "partial", "corrupt", "read-failure", "wrong-bucket"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
if _, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
original := set.getDisks
|
||||
t.Cleanup(func() { set.getDisks = original })
|
||||
var first atomic.Bool
|
||||
set.getDisks = func() []StorageAPI {
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i, d := range disks {
|
||||
disks[i] = multipartListingFaultDisk{StorageAPI: d, read: func(ctx context.Context, b, v, p, version string, opts ReadOptions) (FileInfo, error) {
|
||||
fi, err := d.ReadVersion(ctx, b, v, p, version, opts)
|
||||
if v == minioMetaMultipartBucket && first.CompareAndSwap(false, true) {
|
||||
if kind == "read-failure" {
|
||||
return FileInfo{}, errDiskNotFound
|
||||
}
|
||||
if kind == "corrupt" {
|
||||
return FileInfo{}, errFileCorrupt
|
||||
}
|
||||
fi.Metadata = cloneMSS(fi.Metadata)
|
||||
if kind == "old" || kind == "partial" {
|
||||
delete(fi.Metadata, multipartMetaBucket)
|
||||
if kind == "old" {
|
||||
delete(fi.Metadata, multipartMetaObject)
|
||||
}
|
||||
} else {
|
||||
fi.Metadata[multipartMetaBucket] = "another-bucket"
|
||||
}
|
||||
}
|
||||
return fi, err
|
||||
}}
|
||||
}
|
||||
return disks
|
||||
}
|
||||
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, got, "a")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAbortPoolsAndRetry(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
|
||||
mp, err := z.serverPools[1].NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
emptySet := z.serverPools[0].getHashedSet("a")
|
||||
original := emptySet.getDisks
|
||||
t.Cleanup(func() { emptySet.getDisks = original })
|
||||
emptySet.getDisks = func() []StorageAPI {
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i, d := range disks {
|
||||
disks[i] = multipartListingFaultDisk{StorageAPI: d, read: func(context.Context, string, string, string, string, ReadOptions) (FileInfo, error) {
|
||||
return FileInfo{}, errDiskNotFound
|
||||
}}
|
||||
}
|
||||
return disks
|
||||
}
|
||||
err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{})
|
||||
if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 {
|
||||
t.Fatalf("unknown pool must not acknowledge cancellation: %v", err)
|
||||
}
|
||||
if _, err := z.serverPools[1].GetMultipartInfo(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("earlier pool error changed the later pool: %v", err)
|
||||
}
|
||||
emptySet.getDisks = original
|
||||
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("retry after pool recovery: %v", err)
|
||||
}
|
||||
mp, err = z.serverPools[1].NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = z.serverPools[1].GetMultipartInfo(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err == nil {
|
||||
t.Fatal("empty first pool hid the actual upload")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAbortRetryBelowReadQuorum(t *testing.T) {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
original := set.getDisks
|
||||
t.Cleanup(func() { set.getDisks = original })
|
||||
set.getDisks = func() []StorageAPI {
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
disks[3] = nil
|
||||
d := disks[2]
|
||||
disks[2] = multipartListingFaultDisk{StorageAPI: d, delete: func(ctx context.Context, v, p string, opts DeleteOptions) error {
|
||||
if err := d.Delete(ctx, v, p, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
return context.DeadlineExceeded // operation finished, acknowledgement lost
|
||||
}}
|
||||
return disks
|
||||
}
|
||||
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("best-effort cleanup must retain read-quorum availability: %v", err)
|
||||
}
|
||||
set.getDisks = original
|
||||
var invalid InvalidUploadID
|
||||
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); !errors.As(err, &invalid) {
|
||||
t.Fatalf("minority remnant must retain absent-upload result: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartListingMarkerHTTP(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads"}, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
key, marker string
|
||||
status int
|
||||
}{
|
||||
{"", "not-base64=", 200},
|
||||
{"a", "not-base64=", 404},
|
||||
{"a", base64.RawURLEncoding.EncodeToString([]byte("not-native")), 400},
|
||||
{"a", multipartListingTestID(time.Unix(100, 0), 1), 200},
|
||||
} {
|
||||
u := getListMultipartUploadsURLWithParams("", bucket, "", tc.key, tc.marker, "", "10")
|
||||
req, err := newTestSignedRequestV4(http.MethodGet, u, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != tc.status {
|
||||
t.Fatalf("key=%q marker=%q: %d %s", tc.key, tc.marker, rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartPreflightAdminHTTP(t *testing.T) {
|
||||
bed, err := prepareAdminErasureTestBed(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { bed.done(); bed.objLayer.Shutdown(context.Background()); removeRoots(bed.erasureDirs) })
|
||||
const target = "/minio/admin/v3/multipart-preflight"
|
||||
rec := httptest.NewRecorder()
|
||||
bed.router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, target, nil))
|
||||
if rec.Code != 403 {
|
||||
t.Fatalf("anonymous preflight: %d", rec.Code)
|
||||
}
|
||||
req, err := newTestSignedRequestV4(http.MethodGet, target, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
bed.router.ServeHTTP(rec, req)
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("admin preflight: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var report multipartPreflightReport
|
||||
if err = json.Unmarshal(rec.Body.Bytes(), &report); err != nil || !report.Ready || !report.Complete {
|
||||
t.Fatalf("preflight: %+v %v", report, err)
|
||||
}
|
||||
for range cap(multipartScanSlots) {
|
||||
scan, err := startMultipartScan(t.Context(), false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer scan.close()
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
bed.router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("busy admin preflight: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
type multipartLateCreateDisk struct {
|
||||
StorageAPI
|
||||
late chan<- func() error
|
||||
}
|
||||
|
||||
func (d multipartLateCreateDisk) WriteMetadata(_ context.Context, original, volume, object string, fi FileInfo) error {
|
||||
if volume == minioMetaMultipartBucket {
|
||||
d.late <- func() error { return d.StorageAPI.WriteMetadata(context.Background(), original, volume, object, fi) }
|
||||
return context.DeadlineExceeded
|
||||
}
|
||||
return d.StorageAPI.WriteMetadata(context.Background(), original, volume, object, fi)
|
||||
}
|
||||
|
||||
// This characterization is deliberately NOT an assertion of terminal abort
|
||||
// correctness. It preserves an executable example of the separately scoped
|
||||
// late-creation-write limitation. Change the expectation when fencing is added.
|
||||
func TestMultipartAbortLateCreateBoundary(t *testing.T) {
|
||||
obj, dirs, err := prepareErasure(t.Context(), 16)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
|
||||
t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
|
||||
saved := globalStorageClass
|
||||
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 8}})
|
||||
t.Cleanup(func() { globalStorageClass.Update(saved) })
|
||||
const bucket = "multipart-late-create"
|
||||
if err = z.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
set := z.serverPools[0].getHashedSet("a")
|
||||
original := set.getDisks
|
||||
t.Cleanup(func() { set.getDisks = original })
|
||||
late := make(chan func() error, 16)
|
||||
set.getDisks = func() []StorageAPI {
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i := 9; i < 16; i++ {
|
||||
disks[i] = multipartLateCreateDisk{disks[i], late}
|
||||
}
|
||||
return disks
|
||||
}
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(late) != 7 {
|
||||
t.Fatalf("expected seven timed-out creation writes, got %d", len(late))
|
||||
}
|
||||
set.getDisks = func() []StorageAPI {
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
disks[8] = nil
|
||||
return disks
|
||||
}
|
||||
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for range 7 {
|
||||
if err = (<-late)(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
set.getDisks = original
|
||||
_, metadata, err := set.checkUploadIDExists(t.Context(), bucket, "a", mp.UploadID, false)
|
||||
if err != nil {
|
||||
t.Fatalf("late creation boundary changed; review and remove the documented limitation: %v", err)
|
||||
}
|
||||
count := 0
|
||||
for _, fi := range metadata {
|
||||
if fi.IsValid() {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count != 8 {
|
||||
t.Fatalf("expected eight resurrected copies, got %d", count)
|
||||
}
|
||||
t.Log("KNOWN UNRESOLVED BOUNDARY: seven delayed creation writes plus one offline copy restore a readable upload after acknowledged cancellation")
|
||||
}
|
||||
|
||||
type multipartListingCountingDisk struct {
|
||||
StorageAPI
|
||||
directoryCalls *atomic.Int64
|
||||
metadataCalls *atomic.Int64
|
||||
}
|
||||
|
||||
func (d multipartListingCountingDisk) ListDir(ctx context.Context, original, volume, dir string, count int) ([]string, error) {
|
||||
if volume == minioMetaMultipartBucket {
|
||||
d.directoryCalls.Add(1)
|
||||
}
|
||||
return d.StorageAPI.ListDir(ctx, original, volume, dir, count)
|
||||
}
|
||||
|
||||
func (d multipartListingCountingDisk) ReadVersion(ctx context.Context, original, volume, object, version string, opts ReadOptions) (FileInfo, error) {
|
||||
if volume == minioMetaMultipartBucket {
|
||||
d.metadataCalls.Add(1)
|
||||
}
|
||||
return d.StorageAPI.ReadVersion(ctx, original, volume, object, version, opts)
|
||||
}
|
||||
|
||||
// Counts storage API calls; this is not a deployment throughput benchmark.
|
||||
func TestMultipartListingScanCosts(t *testing.T) {
|
||||
obj, dirs, err := prepareErasure(t.Context(), 4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
|
||||
t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
|
||||
const bucket, otherBucket = "r9-scan-target", "r9-scan-unrelated"
|
||||
for _, name := range []string{bucket, otherBucket} {
|
||||
if err := z.MakeBucket(t.Context(), name, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := z.NewMultipartUpload(t.Context(), bucket, "dir/target", ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var directoryCalls, metadataCalls atomic.Int64
|
||||
for _, pool := range z.serverPools {
|
||||
for _, set := range pool.sets {
|
||||
original := set.getDisks
|
||||
set.getDisks = func() []StorageAPI {
|
||||
disks := original()
|
||||
wrapped := make([]StorageAPI, len(disks))
|
||||
for i, disk := range disks {
|
||||
if disk != nil {
|
||||
wrapped[i] = multipartListingCountingDisk{disk, &directoryCalls, &metadataCalls}
|
||||
}
|
||||
}
|
||||
return wrapped
|
||||
}
|
||||
t.Cleanup(func() { set.getDisks = original })
|
||||
}
|
||||
}
|
||||
measure := func(label string) (int64, int64) {
|
||||
t.Helper()
|
||||
directoryCalls.Store(0)
|
||||
metadataCalls.Store(0)
|
||||
result, err := z.ListMultipartUploads(t.Context(), bucket, "dir/", "", "", "", 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, result, "dir/target")
|
||||
d, m := directoryCalls.Load(), metadataCalls.Load()
|
||||
t.Logf("%s: max-uploads=1 returned=%d ListDir=%d ReadVersion=%d", label, len(result.Uploads), d, m)
|
||||
return d, m
|
||||
}
|
||||
_, baseline := measure("no unrelated uploads")
|
||||
for n := 0; n < 32; n++ {
|
||||
if _, err := z.NewMultipartUpload(t.Context(), otherBucket, fmt.Sprintf("other/%03d", n), ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
_, loaded := measure("32 uploads in another bucket")
|
||||
_, repeated := measure("same query repeated")
|
||||
if loaded != baseline+32 || repeated != loaded {
|
||||
t.Fatalf("unexpected scan accounting: baseline=%d loaded=%d repeated=%d", baseline, loaded, repeated)
|
||||
}
|
||||
}
|
||||
|
||||
func multipartListingTestID(created time.Time, n int) string {
|
||||
return base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("00000000-0000-4000-8000-000000000000.00000000-0000-4000-8000-%012xx%d", n, created.UnixNano())))
|
||||
}
|
||||
|
||||
func multipartListingFixture(t *testing.T) (*erasureServerPools, *erasureObjects, string) {
|
||||
t.Helper()
|
||||
obj, dirs, err := prepareErasure(t.Context(), 4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
|
||||
const bucket = "multipart-listing-test"
|
||||
if err := z.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return z, z.serverPools[0].getHashedSet("a"), bucket
|
||||
}
|
||||
|
||||
func TestMultipartListingMissingMarkers(t *testing.T) {
|
||||
base := time.Unix(100, 0)
|
||||
sameTime := []string{multipartListingTestID(base.Add(time.Second), 1), multipartListingTestID(base.Add(time.Second), 2), multipartListingTestID(base.Add(time.Second), 3)}
|
||||
slices.Sort(sameTime)
|
||||
uploads := []MultipartInfo{
|
||||
{Bucket: "bucket", Object: "a", UploadID: multipartListingTestID(base, 1), Initiated: base},
|
||||
{Bucket: "bucket", Object: "a", UploadID: sameTime[1], Initiated: base.Add(time.Second)},
|
||||
{Bucket: "bucket", Object: "b", UploadID: multipartListingTestID(base, 4), Initiated: base},
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
created time.Time
|
||||
id int
|
||||
want []string
|
||||
}{
|
||||
{"before", base.Add(-time.Second), 0, []string{"a", "a", "b"}},
|
||||
{"between", base.Add(time.Second / 2), 2, []string{"a", "b"}},
|
||||
{"same-time-before", base.Add(time.Second), 2, []string{"a", "b"}},
|
||||
{"same-time-after", base.Add(time.Second), 4, []string{"b"}},
|
||||
{"after", base.Add(2 * time.Second), 5, []string{"b"}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
marker := multipartListingTestID(tc.created, tc.id)
|
||||
if tc.name == "same-time-before" {
|
||||
marker = sameTime[0]
|
||||
}
|
||||
if tc.name == "same-time-after" {
|
||||
marker = sameTime[2]
|
||||
}
|
||||
if err := checkListMultipartArgs(t.Context(), "bucket", "", "a", marker, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := paginateMultipartUploads(uploads, "", "a", marker, "", 10)
|
||||
if !slices.Equal(multipartUploadKeys(got.Uploads), tc.want) {
|
||||
t.Fatalf("got %v want %v", multipartUploadKeys(got.Uploads), tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartListingCoverage(t *testing.T) {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
original := set.getDisks
|
||||
t.Cleanup(func() { set.getDisks = original })
|
||||
disks := original()
|
||||
// Leave a readable two-copy record, simulating a partially failed abort.
|
||||
for _, d := range disks[:2] {
|
||||
if err = d.Delete(t.Context(), minioMetaMultipartBucket, set.getUploadIDDir(bucket, "a", mp.UploadID), DeleteOptions{Recursive: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return []StorageAPI{disks[0], disks[1], nil, nil} }
|
||||
_, err = z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
|
||||
if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 {
|
||||
t.Fatalf("incomplete discovery returned success: %v", err)
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return []StorageAPI{nil, disks[1], disks[2], disks[3]} }
|
||||
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, got, "a")
|
||||
report, err := z.multipartPreflight(t.Context())
|
||||
if err != nil || report.Ready || report.Complete || len(report.Sets[0].UncoveredDrives) != 1 {
|
||||
t.Fatalf("offline upgrade preflight: %+v %v", report, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartListingBudgetAndAdmission(t *testing.T) {
|
||||
_, set, bucket := multipartListingFixture(t)
|
||||
if _, err := set.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
scan, err := startMultipartScan(t.Context(), false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer scan.close()
|
||||
scan.remaining.Store(1)
|
||||
_, err = set.scanMultipartUploads(scan, bucket, 0, 0)
|
||||
var limited SlowDown
|
||||
if !errors.As(err, &limited) {
|
||||
t.Fatalf("budget: %v", err)
|
||||
}
|
||||
if apiErr := toAPIError(t.Context(), err); apiErr.HTTPStatusCode != http.StatusServiceUnavailable || apiErr.Code != "SlowDown" {
|
||||
t.Fatalf("budget error mapping: %+v", apiErr)
|
||||
}
|
||||
second, err := startMultipartScan(t.Context(), false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer second.close()
|
||||
if third, err := startMultipartScan(t.Context(), false); err == nil {
|
||||
third.close()
|
||||
t.Fatal("third scan admitted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartListingAdmissionHTTP(t *testing.T) {
|
||||
z, _, _ := multipartListingFixture(t)
|
||||
bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads"}, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for range cap(multipartScanSlots) {
|
||||
scan, err := startMultipartScan(t.Context(), false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer scan.close()
|
||||
}
|
||||
req, err := newTestSignedRequestV4(http.MethodGet,
|
||||
getListMultipartUploadsURLWithParams("", bucket, "", "", "", "", "1"),
|
||||
0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
var response APIErrorResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec.Code != http.StatusServiceUnavailable || response.Code != "SlowDown" {
|
||||
t.Fatalf("admission returned %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartListingPreflightMinorityLegacy(t *testing.T) {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "old", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := set.getUploadIDDir(bucket, "old", mp.UploadID)
|
||||
disks := set.getDisks()
|
||||
fi, err := disks[0].ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, path, "", ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
delete(fi.Metadata, multipartMetaBucket)
|
||||
delete(fi.Metadata, multipartMetaObject)
|
||||
if err = disks[0].WriteMetadata(t.Context(), bucket, minioMetaMultipartBucket, path, fi); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, d := range disks[1:] {
|
||||
if err = d.Delete(t.Context(), minioMetaMultipartBucket, path, DeleteOptions{Recursive: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
report, err := z.multipartPreflight(t.Context())
|
||||
if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 || report.Mode != "persistent" {
|
||||
t.Fatalf("minority legacy copy must prevent readiness: %+v %v", report, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartListingCancellationRetainsAdmission(t *testing.T) {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
for i := range 40 {
|
||||
if _, err := z.NewMultipartUpload(t.Context(), bucket, fmt.Sprintf("key-%02d", i), ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
original := set.getDisks
|
||||
t.Cleanup(func() { set.getDisks = original })
|
||||
entered := make(chan struct{}, 40)
|
||||
release := make(chan struct{})
|
||||
var once sync.Once
|
||||
defer once.Do(func() { close(release) })
|
||||
var reads atomic.Int32
|
||||
set.getDisks = func() []StorageAPI {
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i, d := range disks {
|
||||
disks[i] = multipartListingFaultDisk{StorageAPI: d, read: func(ctx context.Context, b, v, p, version string, opts ReadOptions) (FileInfo, error) {
|
||||
if v != minioMetaMultipartBucket {
|
||||
return d.ReadVersion(ctx, b, v, p, version, opts)
|
||||
}
|
||||
reads.Add(1)
|
||||
entered <- struct{}{}
|
||||
// Model an RPC which does not return immediately on cancellation.
|
||||
<-release
|
||||
return FileInfo{}, ctx.Err()
|
||||
}}
|
||||
}
|
||||
return disks
|
||||
}
|
||||
second, err := startMultipartScan(t.Context(), false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer second.close()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := z.ListMultipartUploads(ctx, bucket, "", "", "", "", 10)
|
||||
done <- err
|
||||
}()
|
||||
for range 16 {
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("identity workers did not start")
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
if extra, err := startMultipartScan(t.Context(), false); err == nil {
|
||||
extra.close()
|
||||
t.Fatal("canceled request released admission while RPCs were still running")
|
||||
}
|
||||
start := time.Now()
|
||||
once.Do(func() { close(release) })
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Fatal("canceled scan returned a successful partial list")
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("scan did not stop after blocked RPCs returned")
|
||||
}
|
||||
if got := reads.Load(); got != 16 {
|
||||
t.Fatalf("scheduled more identity/metadata reads after cancellation: %d", got)
|
||||
}
|
||||
t.Logf("16 identity RPCs bounded; admission held until return; cancellation settled in %s", time.Since(start))
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
// Copyright (c) 2026 Ruohang Feng
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
"github.com/minio/minio/internal/grid"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
)
|
||||
|
||||
func TestMultipartAbortDefaultAvailability(t *testing.T) {
|
||||
for _, drives := range []int{4, 16} {
|
||||
for _, offline := range []int{0, 1, drives / 2} {
|
||||
t.Run(fmt.Sprintf("drives=%d/offline=%d", drives, offline), func(t *testing.T) {
|
||||
obj, dirs, err := prepareErasure(t.Context(), drives)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
|
||||
|
||||
previous := globalStorageClass
|
||||
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: drives / 2}})
|
||||
t.Cleanup(func() { globalStorageClass.Update(previous) })
|
||||
const bucket, key = "multipart-legacy-availability", "keep-available"
|
||||
if err := z.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, key, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
set := z.serverPools[0].getHashedSet(key)
|
||||
original := set.getDisks
|
||||
disks := original()
|
||||
set.getDisks = func() []StorageAPI {
|
||||
visible := append([]StorageAPI(nil), disks...)
|
||||
clear(visible[:offline])
|
||||
return visible
|
||||
}
|
||||
t.Cleanup(func() { set.getDisks = original })
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, key, mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("released read-quorum availability regressed: %v", err)
|
||||
}
|
||||
for _, disk := range disks[offline:] {
|
||||
_, err := disk.ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, set.getUploadIDDir(bucket, key, mp.UploadID), "", ReadOptions{})
|
||||
if !errors.Is(err, errFileNotFound) {
|
||||
t.Fatalf("online replica not cleaned: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAbortDefaultPoolOrder(t *testing.T) {
|
||||
for _, owner := range []int{0, 1} {
|
||||
t.Run(fmt.Sprintf("owner=%d", owner), func(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
|
||||
mp, err := z.serverPools[owner].NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
emptySet := z.serverPools[1-owner].getHashedSet("a")
|
||||
original := emptySet.getDisks
|
||||
t.Cleanup(func() { emptySet.getDisks = original })
|
||||
emptySet.getDisks = func() []StorageAPI { return make([]StorageAPI, emptySet.setDriveCount) }
|
||||
err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{})
|
||||
if owner == 0 && err != nil {
|
||||
t.Fatalf("unrelated later pool blocked cancellation: %v", err)
|
||||
}
|
||||
if owner == 1 {
|
||||
if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 {
|
||||
t.Fatalf("unknown earlier pool must retain released error behavior: %v", err)
|
||||
}
|
||||
if _, err := z.serverPools[owner].GetMultipartInfo(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("later pool visited despite earlier error: %v", err)
|
||||
}
|
||||
emptySet.getDisks = original
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAbortWrongTargetKeepsCache(t *testing.T) {
|
||||
z, _, bucket := multipartListingFixture(t)
|
||||
|
||||
const other = "multipart-other-bucket"
|
||||
if err := z.MakeBucket(t.Context(), other, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "valid-key", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, target := range [][2]string{{bucket, "wrong-key"}, {other, "valid-key"}} {
|
||||
err := z.AbortMultipartUpload(t.Context(), target[0], target[1], mp.UploadID, ObjectOptions{})
|
||||
var invalid InvalidUploadID
|
||||
if !errors.As(err, &invalid) {
|
||||
t.Fatalf("wrong target result: %v", err)
|
||||
}
|
||||
if _, present := z.mpCache.Load(mp.UploadID); !present {
|
||||
t.Fatal("wrong-target abort evicted another upload")
|
||||
}
|
||||
if _, err := z.GetMultipartInfo(t.Context(), bucket, "valid-key", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("wrong-target abort harmed valid upload: %v", err)
|
||||
}
|
||||
}
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "valid-key", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var invalid InvalidUploadID
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "valid-key", mp.UploadID, ObjectOptions{}); !errors.As(err, &invalid) {
|
||||
t.Fatalf("repeat abort: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAbortRejectsUnsafeID(t *testing.T) {
|
||||
z, _, bucket := multipartListingFixture(t)
|
||||
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "keep", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, suffix := range []string{".", "..", "../other", "a/b", "a\\b", ""} {
|
||||
id := base64.RawURLEncoding.EncodeToString([]byte("deployment." + suffix))
|
||||
err := z.AbortMultipartUpload(t.Context(), bucket, "keep", id, ObjectOptions{})
|
||||
var invalid InvalidUploadID
|
||||
if !errors.As(err, &invalid) {
|
||||
t.Fatalf("unsafe ID accepted: suffix=%q err=%v", suffix, err)
|
||||
}
|
||||
if _, err := z.GetMultipartInfo(t.Context(), bucket, "keep", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("valid upload harmed by rejected ID: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAbortPeerNotification(t *testing.T) {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
tg, err := grid.SetupTestGrid(2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(tg.Cleanup)
|
||||
var notifications atomic.Int32
|
||||
if err := cleanupUploadIDCacheMetaRPC.Register(tg.Managers[1], func(*grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
|
||||
notifications.Add(1)
|
||||
return grid.NoPayload{}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
host, err := xnet.ParseHost(strings.TrimPrefix(tg.Hosts[1], "http://"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previous := globalNotificationSys
|
||||
globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{{
|
||||
host: host,
|
||||
gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[1]) },
|
||||
}}}
|
||||
t.Cleanup(func() { globalNotificationSys = previous })
|
||||
// Use the real distributed lock implementation: Unlock cancels its derived
|
||||
// context. Local locks do not, and would hide a broken notification context.
|
||||
oldMutex, oldLockers := set.nsMutex, set.getLockers
|
||||
lockers := []dsync.NetLocker{newLocker(), newLocker()}
|
||||
set.nsMutex = newNSLock(true)
|
||||
set.getLockers = func() ([]dsync.NetLocker, string) { return lockers, "multipart-test" }
|
||||
t.Cleanup(func() { set.nsMutex, set.getLockers = oldMutex, oldLockers })
|
||||
|
||||
for range 4 {
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "valid", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := notifications.Load()
|
||||
var invalid InvalidUploadID
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "wrong", mp.UploadID, ObjectOptions{}); !errors.As(err, &invalid) {
|
||||
t.Fatalf("wrong target: %v", err)
|
||||
}
|
||||
if notifications.Load() != before {
|
||||
t.Fatal("wrong-target abort sent a destructive peer notification")
|
||||
}
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "valid", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if notifications.Load() != before+1 {
|
||||
t.Fatal("successful abort lost peer notification after unlocking")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAbortBestEffortCleanup(t *testing.T) {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
original := set.getDisks
|
||||
disks := original()
|
||||
set.getDisks = func() []StorageAPI {
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
wrapped[0] = multipartListingFaultDisk{StorageAPI: disks[0], delete: func(context.Context, string, string, DeleteOptions) error { return errFaultyDisk }}
|
||||
return wrapped
|
||||
}
|
||||
t.Cleanup(func() { set.getDisks = original })
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("read-quorum cancellation was tightened: %v", err)
|
||||
}
|
||||
// Best-effort deletion may leave a remnant. Retrying after recovery cleans
|
||||
// it, but the read quorum no longer recognizes it as an active upload.
|
||||
if _, err := disks[0].ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, set.getUploadIDDir(bucket, "a", mp.UploadID), "", ReadOptions{}); err != nil {
|
||||
t.Fatalf("fault fixture did not leave a remnant: %v", err)
|
||||
}
|
||||
set.getDisks = original
|
||||
var invalid InvalidUploadID
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); !errors.As(err, &invalid) {
|
||||
t.Fatalf("minority retry must preserve default absent-upload result: %v", err)
|
||||
}
|
||||
if _, err := disks[0].ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, set.getUploadIDDir(bucket, "a", mp.UploadID), "", ReadOptions{}); !errors.Is(err, errFileNotFound) {
|
||||
t.Fatalf("best-effort retry did not clean remnant: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
// Copyright (c) 2026 Ruohang Feng
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func rewriteMultipartIdentity(t testing.TB, set *erasureObjects, bucket, key, id string, change func(map[string]string)) {
|
||||
t.Helper()
|
||||
fi, metadata, err := set.checkUploadIDExists(t.Context(), bucket, key, id, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range metadata {
|
||||
change(metadata[i].Metadata)
|
||||
}
|
||||
if _, err := writeAllMetadata(t.Context(), set.getDisks(), bucket, minioMetaMultipartBucket,
|
||||
set.getUploadIDDir(bucket, key, id), metadata, fi.WriteQuorum(set.defaultWQuorum())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func removeMultipartIdentity(metadata map[string]string) {
|
||||
delete(metadata, multipartMetaBucket)
|
||||
delete(metadata, multipartMetaObject)
|
||||
}
|
||||
|
||||
// The actual S3 handler must use the same semantics for every prefix and page,
|
||||
// even with old records and a stale, misleading node-local cache.
|
||||
func TestMultipartListingMixedHTTP(t *testing.T) {
|
||||
z, set, _ := multipartListingFixture(t)
|
||||
bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads"}, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, key := range []string{"dir/old", "dir/a", "dir/sub/b", "outside"} {
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, key, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if key == "dir/old" {
|
||||
rewriteMultipartIdentity(t, set, bucket, key, mp.UploadID, removeMultipartIdentity)
|
||||
}
|
||||
}
|
||||
z.mpCache.Clear()
|
||||
z.mpCache.Store("stale", MultipartInfo{Bucket: bucket, Object: "ghost", UploadID: "stale"})
|
||||
list := func(prefix, keyMarker, uploadMarker, delimiter string) ListMultipartUploadsResponse {
|
||||
t.Helper()
|
||||
req, err := newTestSignedRequestV4(http.MethodGet,
|
||||
getListMultipartUploadsURLWithParams("", bucket, prefix, keyMarker, uploadMarker, delimiter, "1"),
|
||||
0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
var got ListMultipartUploadsResponse
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("list %q: HTTP %d %s", prefix, rec.Code, rec.Body.String())
|
||||
}
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
var keys []string
|
||||
keyMarker, uploadMarker := "", ""
|
||||
for page := 0; ; page++ {
|
||||
if page >= 4 {
|
||||
t.Fatal("pagination did not terminate")
|
||||
}
|
||||
got := list("", keyMarker, uploadMarker, "")
|
||||
if len(got.Uploads) != 1 {
|
||||
t.Fatalf("page size: %+v", got)
|
||||
}
|
||||
keys = append(keys, got.Uploads[0].Key)
|
||||
if !got.IsTruncated {
|
||||
break
|
||||
}
|
||||
keyMarker, uploadMarker = got.NextKeyMarker, got.NextUploadIDMarker
|
||||
}
|
||||
if !slices.Equal(keys, []string{"dir/a", "dir/sub/b", "outside"}) {
|
||||
t.Fatalf("mixed full listing: %v", keys)
|
||||
}
|
||||
first := list("dir/", "", "", "/")
|
||||
if len(first.Uploads) != 1 || first.Uploads[0].Key != "dir/a" || !first.IsTruncated {
|
||||
t.Fatalf("prefix first page: %+v", first)
|
||||
}
|
||||
rest := list("dir/", first.NextKeyMarker, first.NextUploadIDMarker, "/")
|
||||
if len(rest.Uploads) != 0 || rest.IsTruncated || len(rest.CommonPrefixes) != 1 || rest.CommonPrefixes[0].Prefix != "dir/sub/" {
|
||||
t.Fatalf("prefix continuation: %+v", rest)
|
||||
}
|
||||
if got := list("dir/old", "", "", ""); len(got.Uploads) != 0 || got.IsTruncated {
|
||||
t.Fatalf("historical exact-looking prefix: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartListingInvalidIdentity(t *testing.T) {
|
||||
for _, kind := range []string{"missing-bucket", "missing-object", "empty-bucket", "empty-object", "empty-both", "wrong-hash", "read-failure"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rewriteMultipartIdentity(t, set, bucket, "a", mp.UploadID, func(metadata map[string]string) {
|
||||
switch kind {
|
||||
case "missing-bucket":
|
||||
delete(metadata, multipartMetaBucket)
|
||||
case "missing-object":
|
||||
delete(metadata, multipartMetaObject)
|
||||
case "empty-bucket":
|
||||
metadata[multipartMetaBucket] = ""
|
||||
case "empty-object":
|
||||
metadata[multipartMetaObject] = ""
|
||||
case "empty-both":
|
||||
metadata[multipartMetaBucket], metadata[multipartMetaObject] = "", ""
|
||||
case "wrong-hash":
|
||||
metadata[multipartMetaObject] = "wrong-key"
|
||||
case "read-failure":
|
||||
removeMultipartIdentity(metadata)
|
||||
}
|
||||
})
|
||||
if kind == "read-failure" {
|
||||
original := set.getDisks
|
||||
t.Cleanup(func() { set.getDisks = original })
|
||||
set.getDisks = func() []StorageAPI {
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i, disk := range disks {
|
||||
disks[i] = multipartListingFaultDisk{StorageAPI: disk, read: func(context.Context, string, string, string, string, ReadOptions) (FileInfo, error) {
|
||||
return FileInfo{}, errFaultyDisk
|
||||
}}
|
||||
}
|
||||
return disks
|
||||
}
|
||||
}
|
||||
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
|
||||
if err == nil || len(got.Uploads) != 0 || toAPIError(t.Context(), err).HTTPStatusCode != http.StatusServiceUnavailable {
|
||||
t.Fatalf("invalid/unreadable record returned a successful page: %+v %v", got, err)
|
||||
}
|
||||
if kind != "read-failure" && !errors.Is(err, errMultipartListingIdentity) {
|
||||
t.Fatalf("identity error: %v", err)
|
||||
}
|
||||
report, err := z.multipartPreflight(t.Context())
|
||||
if err != nil || report.Complete || report.Ready || report.LegacyUploads != 0 {
|
||||
t.Fatalf("invalid/unreadable record counted as historical: %+v %v", report, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartListingRestart(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
obj, dirs, err := prepareErasure(ctx, 4)
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cancel(); obj.Shutdown(context.Background()); removeRoots(dirs) })
|
||||
const bucket, key = "multipart-restart", "dir/new"
|
||||
if err := obj.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mp, err := obj.NewMultipartUpload(t.Context(), bucket, key, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cancel()
|
||||
if err := obj.Shutdown(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reopened, _, err := initObjectLayer(t.Context(), mustGetPoolEndpoints(0, dirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj = reopened
|
||||
if _, cached := obj.(*erasureServerPools).mpCache.Load(mp.UploadID); cached {
|
||||
t.Fatal("restart unexpectedly preserved the upload cache")
|
||||
}
|
||||
got, err := obj.ListMultipartUploads(t.Context(), bucket, "dir/", "", "", "", 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, got, key)
|
||||
if got.Uploads[0].UploadID != mp.UploadID {
|
||||
t.Fatal("restart changed upload identity")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartListingConflictingIdentityCopies(t *testing.T) {
|
||||
for _, oldCopies := range []int{1, 3} {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := set.getUploadIDDir(bucket, "a", mp.UploadID)
|
||||
for _, disk := range set.getDisks()[:oldCopies] {
|
||||
fi, err := disk.ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, path, "", ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
removeMultipartIdentity(fi.Metadata)
|
||||
if err := disk.WriteMetadata(t.Context(), bucket, minioMetaMultipartBucket, path, fi); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
|
||||
if !errors.Is(err, errMultipartListingIdentity) || len(got.Uploads) != 0 {
|
||||
t.Fatalf("%d contradictory copies returned a successful page: %+v %v", oldCopies, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartListingUnsupportedStoredID(t *testing.T) {
|
||||
for _, historical := range []bool{false, true} {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldPath := set.getUploadIDDir(bucket, "a", mp.UploadID)
|
||||
unsupportedPath := pathJoin(set.getMultipartSHADir(bucket, "a"), "unsupported-id")
|
||||
for _, disk := range set.getDisks() {
|
||||
fi, err := disk.ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, oldPath, "", ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if historical {
|
||||
removeMultipartIdentity(fi.Metadata)
|
||||
}
|
||||
if err := disk.WriteMetadata(t.Context(), bucket, minioMetaMultipartBucket, unsupportedPath, fi); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := disk.Delete(t.Context(), minioMetaMultipartBucket, oldPath, DeleteOptions{Recursive: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
|
||||
if historical {
|
||||
if err != nil || len(got.Uploads) != 0 {
|
||||
t.Fatalf("unidentified historical ID: %+v %v", got, err)
|
||||
}
|
||||
} else if !errors.Is(err, errMultipartListingIdentity) {
|
||||
t.Fatalf("invalid new ID silently treated as historical: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
+258
-43
@@ -31,6 +31,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/klauspost/readahead"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
@@ -44,6 +45,15 @@ import (
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
)
|
||||
|
||||
const (
|
||||
multipartMetaBucket = ReservedMetadataPrefixLower + "multipart-v1-bucket"
|
||||
multipartMetaObject = ReservedMetadataPrefixLower + "multipart-v1-object"
|
||||
|
||||
// ponytail: keep scan concurrency fixed until the multipart-list benchmark
|
||||
// establishes a better adaptive limit.
|
||||
multipartMetadataScanConcurrency = 4
|
||||
)
|
||||
|
||||
func (er erasureObjects) getUploadIDDir(bucket, object, uploadID string) string {
|
||||
uploadUUID := uploadID
|
||||
uploadBytes, err := base64.RawURLEncoding.DecodeString(uploadID)
|
||||
@@ -164,6 +174,24 @@ func (er erasureObjects) deleteAll(ctx context.Context, bucket, prefix string) {
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// extractModTimeFromUploadIDDir extracts creation time from the upload ID directory on disk.
|
||||
// On-disk directory names are of the form <UUID>x<UnixNano>.
|
||||
// Parse the native format first: plaintext UUIDs can also be valid Base64.
|
||||
// Preserve the previous encoded-format parser for compatibility.
|
||||
func extractModTimeFromUploadIDDir(uploadIDDir string) time.Time {
|
||||
if modTime, ok := multipartUploadTime(uploadIDDir); ok {
|
||||
return modTime
|
||||
}
|
||||
if b64, err := base64.RawURLEncoding.DecodeString(uploadIDDir); err == nil {
|
||||
if split := strings.Split(string(b64), "x"); len(split) == 2 {
|
||||
if t, err := strconv.ParseInt(split[1], 10, 64); err == nil {
|
||||
return time.Unix(0, t)
|
||||
}
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
// Remove the old multipart uploads on the given disk.
|
||||
func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk StorageAPI) {
|
||||
drivePath := disk.Endpoint().Path
|
||||
@@ -171,16 +199,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
|
||||
readDirFn(pathJoin(drivePath, minioMetaMultipartBucket), func(shaDir string, typ os.FileMode) error {
|
||||
readDirFn(pathJoin(drivePath, minioMetaMultipartBucket, shaDir), func(uploadIDDir string, typ os.FileMode) error {
|
||||
uploadIDPath := pathJoin(shaDir, uploadIDDir)
|
||||
var modTime time.Time
|
||||
// Upload IDs are of the form base64_url(<UUID>x<UnixNano>), we can extract the time from the UUID.
|
||||
if b64, err := base64.RawURLEncoding.DecodeString(uploadIDDir); err == nil {
|
||||
if split := strings.Split(string(b64), "x"); len(split) == 2 {
|
||||
t, err := strconv.ParseInt(split[1], 10, 64)
|
||||
if err == nil {
|
||||
modTime = time.Unix(0, t)
|
||||
}
|
||||
}
|
||||
}
|
||||
modTime := extractModTimeFromUploadIDDir(uploadIDDir)
|
||||
// Fallback for older uploads without time in the ID.
|
||||
if modTime.IsZero() {
|
||||
wait := deleteMultipartCleanupSleeper.Timer(ctx)
|
||||
@@ -251,14 +270,152 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
|
||||
})
|
||||
}
|
||||
|
||||
// ListMultipartUploads - lists all the pending multipart
|
||||
// uploads for a particular object in a bucket.
|
||||
//
|
||||
// Implements minimal S3 compatible ListMultipartUploads API. We do
|
||||
// not support prefix based listing, this is a deliberate attempt
|
||||
// towards simplification of multipart APIs.
|
||||
// The resulting ListMultipartsInfo structure is unmarshalled directly as XML.
|
||||
func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
|
||||
func multipartUploadInfo(bucket, object, uploadUUID string, fallback time.Time) MultipartInfo {
|
||||
initiated := fallback
|
||||
if parsed, ok := multipartUploadTime(uploadUUID); ok {
|
||||
initiated = parsed
|
||||
}
|
||||
return MultipartInfo{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
UploadID: base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID)),
|
||||
Initiated: initiated,
|
||||
}
|
||||
}
|
||||
|
||||
// multipartUploadTime is shared by stored records and continuation markers.
|
||||
// The time is part of the immutable upload ID, so a removed marker still
|
||||
// identifies the same ordering boundary.
|
||||
func multipartUploadTime(uploadUUID string) (time.Time, bool) {
|
||||
if len(uploadUUID) < 38 || uploadUUID[36] != 'x' {
|
||||
return time.Time{}, false
|
||||
}
|
||||
if _, err := uuid.Parse(uploadUUID[:36]); err != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
ns, err := strconv.ParseInt(uploadUUID[37:], 10, 64)
|
||||
if err != nil || ns <= 0 || strconv.FormatInt(ns, 10) != uploadUUID[37:] {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return time.Unix(0, ns), true
|
||||
}
|
||||
|
||||
func multipartMarkerTime(uploadID string) (time.Time, bool) {
|
||||
b, err := base64.RawURLEncoding.DecodeString(uploadID)
|
||||
if err != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
_, uploadUUID, ok := strings.Cut(string(b), ".")
|
||||
if !ok {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return multipartUploadTime(uploadUUID)
|
||||
}
|
||||
|
||||
type multipartListEntry struct {
|
||||
upload *MultipartInfo
|
||||
commonPrefix string
|
||||
}
|
||||
|
||||
// paginateMultipartUploads applies the S3 ordering, prefix, delimiter, marker,
|
||||
// and page rules exactly once after all pools and sets have been merged.
|
||||
func paginateMultipartUploads(uploads []MultipartInfo, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) ListMultipartsInfo {
|
||||
if maxUploads > maxUploadsList {
|
||||
maxUploads = maxUploadsList
|
||||
}
|
||||
result := ListMultipartsInfo{
|
||||
MaxUploads: maxUploads,
|
||||
KeyMarker: keyMarker,
|
||||
UploadIDMarker: uploadIDMarker,
|
||||
Prefix: prefix,
|
||||
Delimiter: delimiter,
|
||||
}
|
||||
|
||||
deduplicated := make([]MultipartInfo, 0, len(uploads))
|
||||
seenUploads := make(map[string]struct{}, len(uploads))
|
||||
for _, upload := range uploads {
|
||||
identity := upload.Bucket + "\x00" + upload.Object + "\x00" + upload.UploadID
|
||||
if _, ok := seenUploads[identity]; ok {
|
||||
continue
|
||||
}
|
||||
seenUploads[identity] = struct{}{}
|
||||
deduplicated = append(deduplicated, upload)
|
||||
}
|
||||
sort.Slice(deduplicated, func(i, j int) bool {
|
||||
if deduplicated[i].Object != deduplicated[j].Object {
|
||||
return deduplicated[i].Object < deduplicated[j].Object
|
||||
}
|
||||
if !deduplicated[i].Initiated.Equal(deduplicated[j].Initiated) {
|
||||
return deduplicated[i].Initiated.Before(deduplicated[j].Initiated)
|
||||
}
|
||||
return deduplicated[i].UploadID < deduplicated[j].UploadID
|
||||
})
|
||||
|
||||
markerTime, _ := multipartMarkerTime(uploadIDMarker)
|
||||
seenPrefixes := make(map[string]struct{})
|
||||
entries := make([]multipartListEntry, 0, len(deduplicated))
|
||||
for i := range deduplicated {
|
||||
upload := &deduplicated[i]
|
||||
if !strings.HasPrefix(upload.Object, prefix) {
|
||||
continue
|
||||
}
|
||||
if keyMarker != "" {
|
||||
switch strings.Compare(upload.Object, keyMarker) {
|
||||
case -1:
|
||||
continue
|
||||
case 0:
|
||||
if uploadIDMarker == "" || upload.Initiated.Before(markerTime) ||
|
||||
(upload.Initiated.Equal(markerTime) && upload.UploadID <= uploadIDMarker) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if delimiter != "" {
|
||||
remainder := strings.TrimPrefix(upload.Object, prefix)
|
||||
if i := strings.Index(remainder, delimiter); i >= 0 {
|
||||
commonPrefix := prefix + remainder[:i+len(delimiter)]
|
||||
if keyMarker != "" && commonPrefix <= keyMarker {
|
||||
continue
|
||||
}
|
||||
if _, ok := seenPrefixes[commonPrefix]; ok {
|
||||
continue
|
||||
}
|
||||
seenPrefixes[commonPrefix] = struct{}{}
|
||||
entries = append(entries, multipartListEntry{commonPrefix: commonPrefix})
|
||||
continue
|
||||
}
|
||||
}
|
||||
entries = append(entries, multipartListEntry{upload: upload})
|
||||
}
|
||||
|
||||
if maxUploads <= 0 {
|
||||
return result
|
||||
}
|
||||
pageSize := min(maxUploads, len(entries))
|
||||
for _, entry := range entries[:pageSize] {
|
||||
if entry.upload != nil {
|
||||
result.Uploads = append(result.Uploads, *entry.upload)
|
||||
continue
|
||||
}
|
||||
result.CommonPrefixes = append(result.CommonPrefixes, entry.commonPrefix)
|
||||
}
|
||||
result.IsTruncated = pageSize < len(entries)
|
||||
if result.IsTruncated && pageSize > 0 {
|
||||
last := entries[pageSize-1]
|
||||
if last.upload != nil {
|
||||
result.NextKeyMarker = last.upload.Object
|
||||
result.NextUploadIDMarker = last.upload.UploadID
|
||||
} else {
|
||||
result.NextKeyMarker = last.commonPrefix
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// listMultipartUploadsExact preserves the hashed exact-object lookup used by
|
||||
// multipart write placement. Public listing always uses the durable scan.
|
||||
func (er erasureObjects) listMultipartUploadsExact(ctx context.Context, bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
|
||||
auditObjectErasureSet(ctx, "ListMultipartUploads", object, &er)
|
||||
|
||||
result.MaxUploads = maxUploads
|
||||
@@ -311,20 +468,16 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
|
||||
if populatedUploadIDs.Contains(uploadID) {
|
||||
continue
|
||||
}
|
||||
// If present, use time stored in ID.
|
||||
startTime := time.Now()
|
||||
if split := strings.Split(uploadID, "x"); len(split) == 2 {
|
||||
t, err := strconv.ParseInt(split[1], 10, 64)
|
||||
if err == nil {
|
||||
startTime = time.Unix(0, t)
|
||||
var fallback time.Time
|
||||
if _, ok := multipartUploadTime(uploadID); !ok {
|
||||
fi, err := disk.ReadVersion(ctx, bucket, minioMetaMultipartBucket,
|
||||
pathJoin(er.getMultipartSHADir(bucket, object), uploadID), "", ReadOptions{})
|
||||
if err != nil {
|
||||
return result, toObjectErr(err, bucket, object)
|
||||
}
|
||||
fallback = fi.ModTime
|
||||
}
|
||||
uploads = append(uploads, MultipartInfo{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
UploadID: base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadID)),
|
||||
Initiated: startTime,
|
||||
})
|
||||
uploads = append(uploads, multipartUploadInfo(bucket, object, uploadID, fallback))
|
||||
populatedUploadIDs.Add(uploadID)
|
||||
}
|
||||
|
||||
@@ -365,6 +518,22 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
|
||||
if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil {
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
scan, err := startMultipartScan(ctx, false)
|
||||
if err != nil {
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
defer scan.close()
|
||||
uploads, err := er.scanMultipartUploads(scan, bucket, 0, 0)
|
||||
if err != nil {
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
|
||||
}
|
||||
|
||||
// newMultipartUpload - wrapper for initializing a new multipart
|
||||
// request; returns a unique upload id.
|
||||
//
|
||||
@@ -402,6 +571,8 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
|
||||
}
|
||||
|
||||
userDefined := cloneMSS(opts.UserDefined)
|
||||
userDefined[multipartMetaBucket] = bucket
|
||||
userDefined[multipartMetaObject] = object
|
||||
if opts.PreserveETag != "" {
|
||||
userDefined["etag"] = opts.PreserveETag
|
||||
}
|
||||
@@ -1459,6 +1630,8 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
// Remove superfluous internal headers.
|
||||
delete(fi.Metadata, hash.MinIOMultipartChecksum)
|
||||
delete(fi.Metadata, hash.MinIOMultipartChecksumType)
|
||||
delete(fi.Metadata, multipartMetaBucket)
|
||||
delete(fi.Metadata, multipartMetaObject)
|
||||
|
||||
// Save the final object size and modtime.
|
||||
fi.Size = objectSize
|
||||
@@ -1529,6 +1702,32 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
defer lk.Unlock(lkctx)
|
||||
}
|
||||
|
||||
if !opts.ReplicationRequest && !opts.ReplicaLockReconcile && !opts.DataMovement {
|
||||
getObjectInfo := er.getObjectInfo
|
||||
if opts.replicaObjectInfo != nil {
|
||||
getObjectInfo = opts.replicaObjectInfo
|
||||
}
|
||||
if fi.VersionID == "" || fi.VersionID == nullVersionID {
|
||||
if err := checkObjectLockedForOverwrite(ctx, bucket, object, getObjectInfo); err != nil {
|
||||
return oi, toObjectErr(err, bucket, object)
|
||||
}
|
||||
} else {
|
||||
_, gerr := getObjectInfo(ctx, bucket, object, ObjectOptions{
|
||||
VersionID: fi.VersionID,
|
||||
Versioned: opts.Versioned,
|
||||
VersionSuspended: opts.VersionSuspended,
|
||||
NoLock: true,
|
||||
})
|
||||
switch {
|
||||
case gerr == nil, errors.As(gerr, &MethodNotAllowed{}):
|
||||
return oi, toObjectErr(errAuthentication, bucket, object)
|
||||
case isErrVersionNotFound(gerr) || isErrObjectNotFound(gerr):
|
||||
default:
|
||||
return oi, toObjectErr(gerr, bucket, object)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
er.cleanupMultipartPath(ctx, paths...) // cleanup all part.N.meta, and skipped part.N's before final rename().
|
||||
|
||||
defer func() {
|
||||
@@ -1586,22 +1785,38 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
|
||||
}
|
||||
|
||||
// AbortMultipartUpload - aborts an ongoing multipart operation
|
||||
// signified by the input uploadID. This is an atomic operation
|
||||
// doesn't require clients to initiate multiple such requests.
|
||||
//
|
||||
// All parts are purged from all disks and reference to the uploadID
|
||||
// would be removed from the system, rollback is not possible on this
|
||||
// operation.
|
||||
func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (err error) {
|
||||
// abortMultipartUpload retains read-quorum validation and best-effort cleanup.
|
||||
// It does not fence creation writes still executing after a storage timeout.
|
||||
func (er erasureObjects) abortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (bool, error) {
|
||||
if !opts.NoAuditLog {
|
||||
auditObjectErasureSet(ctx, "AbortMultipartUpload", object, &er)
|
||||
}
|
||||
|
||||
// Cleanup all uploaded parts.
|
||||
b, err := base64.RawURLEncoding.DecodeString(uploadID)
|
||||
if err != nil {
|
||||
return false, MalformedUploadID{UploadID: uploadID}
|
||||
}
|
||||
_, internalID, ok := strings.Cut(string(b), ".")
|
||||
if !ok || internalID == "" || internalID == "." || internalID == ".." || strings.ContainsAny(internalID, "/\\") {
|
||||
return false, InvalidUploadID{Bucket: bucket, Object: object, UploadID: uploadID}
|
||||
}
|
||||
defer er.deleteAll(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID))
|
||||
|
||||
// Validates if upload ID exists.
|
||||
_, _, err = er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
|
||||
return toObjectErr(err, bucket, object, uploadID)
|
||||
err = toObjectErr(err, bucket, object, uploadID)
|
||||
if _, absent := err.(InvalidUploadID); absent {
|
||||
return false, nil
|
||||
}
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
// AbortMultipartUpload cancels an upload using read-quorum validation.
|
||||
// Offline part data may still need stale-upload cleanup after its drives return.
|
||||
func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (err error) {
|
||||
found, err := er.abortMultipartUpload(ctx, bucket, object, uploadID, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
return InvalidUploadID{Bucket: bucket, Object: object, UploadID: uploadID}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
+115
-6
@@ -150,6 +150,17 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
|
||||
}
|
||||
}
|
||||
|
||||
if !dstOpts.ReplicationRequest && !dstOpts.ReplicaLockReconcile && !dstOpts.DataMovement &&
|
||||
(versionID == "" || versionID == nullVersionID) {
|
||||
getObjectInfo := er.getObjectInfo
|
||||
if dstOpts.replicaObjectInfo != nil {
|
||||
getObjectInfo = dstOpts.replicaObjectInfo
|
||||
}
|
||||
if err := checkObjectLockedForOverwrite(ctx, dstBucket, dstObject, getObjectInfo); err != nil {
|
||||
return oi, toObjectErr(err, dstBucket, dstObject)
|
||||
}
|
||||
}
|
||||
|
||||
modTime = UTCNow() // We only preserve modTime if dstOpts.MTime is true.
|
||||
// in all other cases mtime is latest.
|
||||
|
||||
@@ -718,6 +729,57 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
|
||||
return pickLatestQuorumFilesInfo(ctx, rawFileInfos, errs, bucket, object, readData, inclFreeVers)
|
||||
}
|
||||
|
||||
// readListingFileInfo is the read-only, latest-version metadata path for listing
|
||||
// recovery. Ordinary GET keeps its fast-read, healing and dangling semantics.
|
||||
// Absence is proved before decoding can replace a native storage error: every
|
||||
// configured slot must have returned FileNotFound in this read.
|
||||
func (er erasureObjects) readListingFileInfo(ctx context.Context, bucket, object string) (FileInfo, bool, error) {
|
||||
disks := er.getDisks()
|
||||
if len(disks) == 0 || len(disks) != er.setDriveCount {
|
||||
return FileInfo{}, false, listingReadQuorumError(bucket, object)
|
||||
}
|
||||
// An offline slot is not evidence of absence, even if a test or transport
|
||||
// implementation could return a cached not-found response for it.
|
||||
online := append([]StorageAPI(nil), disks...)
|
||||
for i, disk := range online {
|
||||
if disk != nil && !disk.IsOnline() {
|
||||
online[i] = nil
|
||||
}
|
||||
}
|
||||
raw, errs := readAllRawFileInfo(ctx, online, bucket, object, false)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return FileInfo{}, false, err
|
||||
}
|
||||
absent := true
|
||||
for _, err := range errs {
|
||||
absent = absent && err == errFileNotFound
|
||||
}
|
||||
if absent {
|
||||
return FileInfo{}, true, nil
|
||||
}
|
||||
|
||||
// Keep the same latest-version and actual object parity quorum as GET,
|
||||
// without its opportunistic dangling deletion or generic NotFound mapping.
|
||||
meta, errs := pickLatestQuorumFilesInfo(ctx, raw, errs, bucket, object, false, false)
|
||||
quorum, _, err := objectQuorumFromMeta(ctx, meta, errs, er.defaultParityCount)
|
||||
if err == nil {
|
||||
err = reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, quorum)
|
||||
}
|
||||
if err != nil {
|
||||
return FileInfo{}, false, listingReadQuorumError(bucket, object)
|
||||
}
|
||||
_, modTime, etag := listOnlineDisks(disks, meta, errs, quorum)
|
||||
fi, err := pickValidFileInfo(ctx, meta, modTime, etag, quorum)
|
||||
if err != nil || (!fi.Deleted && len(fi.Erasure.Distribution) != len(disks)) {
|
||||
return FileInfo{}, false, listingReadQuorumError(bucket, object)
|
||||
}
|
||||
return fi, false, nil
|
||||
}
|
||||
|
||||
func listingReadQuorumError(bucket, object string) error {
|
||||
return InsufficientReadQuorum{Bucket: bucket, Object: decodeDirObject(object), Err: errErasureReadQuorum, Type: RQInconsistentMeta}
|
||||
}
|
||||
|
||||
func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object string, opts ObjectOptions, readData bool) (FileInfo, []FileInfo, []StorageAPI, error) {
|
||||
rawArr := make([]RawFileInfo, er.setDriveCount)
|
||||
metaArr := make([]FileInfo, er.setDriveCount)
|
||||
@@ -1072,6 +1134,12 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry str
|
||||
errs := g.Wait()
|
||||
|
||||
err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
|
||||
if errors.Is(err, errCommitMetaMissing) {
|
||||
// Commit metadata went missing on a quorum of drives; the write
|
||||
// did not commit. Return a 503 class error so clients retry
|
||||
// instead of treating the object as absent (404).
|
||||
err = errErasureWriteQuorum
|
||||
}
|
||||
if err != nil {
|
||||
dg := errgroup.WithNErrs(len(disks))
|
||||
for index, nerr := range errs {
|
||||
@@ -1594,6 +1662,17 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
defer lk.Unlock(lkctx)
|
||||
}
|
||||
|
||||
if !opts.ReplicationRequest && !opts.ReplicaLockReconcile && !opts.DataMovement &&
|
||||
(fi.VersionID == "" || fi.VersionID == nullVersionID) {
|
||||
getObjectInfo := er.getObjectInfo
|
||||
if opts.replicaObjectInfo != nil {
|
||||
getObjectInfo = opts.replicaObjectInfo
|
||||
}
|
||||
if err := checkObjectLockedForOverwrite(ctx, bucket, object, getObjectInfo); err != nil {
|
||||
return ObjectInfo{}, toObjectErr(err, bucket, object)
|
||||
}
|
||||
}
|
||||
|
||||
// Rename the successfully written temporary object to final location.
|
||||
onlineDisks, versions, oldDataDir, err := renameData(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, bucket, object, writeQuorum)
|
||||
if err != nil {
|
||||
@@ -1657,7 +1736,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
|
||||
}
|
||||
|
||||
func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object string, fi FileInfo, forceDelMarker bool) error {
|
||||
func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object string, fi FileInfo, forceDelMarker, purge bool) error {
|
||||
disks := er.getDisks()
|
||||
// Assume (N/2 + 1) quorum for Delete()
|
||||
// this is a theoretical assumption such that
|
||||
@@ -1673,7 +1752,13 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
|
||||
if disks[index] == nil {
|
||||
return errDiskNotFound
|
||||
}
|
||||
return disks[index].DeleteVersion(ctx, bucket, object, fi, forceDelMarker, DeleteOptions{})
|
||||
err := disks[index].DeleteVersion(ctx, bucket, object, fi, forceDelMarker, DeleteOptions{})
|
||||
// Physical removal and reliable already-absent replies are the same
|
||||
// outcome. Creation and metadata updates must not use this quorum.
|
||||
if purge && (err == errFileNotFound || err == errFileVersionNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}, index)
|
||||
}
|
||||
// return errors if any during deletion
|
||||
@@ -2023,6 +2108,11 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
|
||||
if opts.DeleteMarker {
|
||||
versionFound = false
|
||||
} else if !tryDel {
|
||||
if opts.isVersionPurge() && (isErrObjectNotFound(gerr) || isErrVersionNotFound(gerr)) {
|
||||
if err := er.checkPurgeAbsent(ctx, bucket, object, opts.VersionID); err != nil {
|
||||
return objInfo, err
|
||||
}
|
||||
}
|
||||
return objInfo, gerr
|
||||
}
|
||||
}
|
||||
@@ -2105,6 +2195,13 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
|
||||
}
|
||||
}
|
||||
|
||||
purge := opts.isVersionPurge()
|
||||
if purge {
|
||||
// A delete marker describes the version being removed; it is not an
|
||||
// instruction to create that marker on disks which already lack it.
|
||||
markDelete, deleteMarker = false, false
|
||||
}
|
||||
|
||||
modTime := opts.MTime
|
||||
if opts.MTime.IsZero() {
|
||||
modTime = UTCNow()
|
||||
@@ -2151,7 +2248,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
|
||||
// delete marker. Add delete marker, since we don't have
|
||||
// any version specified explicitly. Or if a particular
|
||||
// version id needs to be replicated.
|
||||
if err = er.deleteObjectVersion(ctx, bucket, object, fi, opts.DeleteMarker); err != nil {
|
||||
if err = er.deleteObjectVersion(ctx, bucket, object, fi, opts.DeleteMarker, false); err != nil {
|
||||
return objInfo, toObjectErr(err, bucket, object)
|
||||
}
|
||||
oi := fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
|
||||
@@ -2174,11 +2271,19 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
|
||||
if opts.SkipFreeVersion {
|
||||
dfi.SetSkipTierFreeVersion()
|
||||
}
|
||||
if err = er.deleteObjectVersion(ctx, bucket, object, dfi, opts.DeleteMarker); err != nil {
|
||||
if err = er.deleteObjectVersion(ctx, bucket, object, dfi, opts.DeleteMarker, purge); err != nil {
|
||||
return objInfo, toObjectErr(err, bucket, object)
|
||||
}
|
||||
|
||||
return dfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
|
||||
oi := dfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
|
||||
if purge {
|
||||
// Preserve the DELETE response's identity without reusing it as a disk
|
||||
// instruction to create a marker. The lookup also exposes a data
|
||||
// version pending purge as deleted for visibility; only a stored
|
||||
// marker, which carries no erasure layout, is reported as one.
|
||||
oi.DeleteMarker = goi.DeleteMarker && goi.DataBlocks == 0
|
||||
}
|
||||
return oi, nil
|
||||
}
|
||||
|
||||
// Send the successful but partial upload/delete, however ignore
|
||||
@@ -2327,7 +2432,11 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
|
||||
|
||||
fi.Metadata[xhttp.AmzObjectTagging] = tags
|
||||
fi.ReplicationState = opts.PutReplicationState()
|
||||
stamp := monotonicTaggingTimestamp(opts.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp], fi.Metadata[ReservedMetadataPrefixLower+TaggingTimestamp])
|
||||
maps.Copy(fi.Metadata, opts.UserDefined)
|
||||
if stamp != "" {
|
||||
fi.Metadata[ReservedMetadataPrefixLower+TaggingTimestamp] = stamp
|
||||
}
|
||||
|
||||
if err = er.updateObjectMeta(ctx, bucket, object, fi, onlineDisks); err != nil {
|
||||
return ObjectInfo{}, toObjectErr(err, bucket, object)
|
||||
@@ -2463,7 +2572,7 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
|
||||
|
||||
storageDisks := er.getDisks()
|
||||
|
||||
if err = er.deleteObjectVersion(ctx, bucket, object, fi, false); err != nil {
|
||||
if err = er.deleteObjectVersion(ctx, bucket, object, fi, false, false); err != nil {
|
||||
eventName = event.ObjectTransitionFailed
|
||||
}
|
||||
|
||||
|
||||
@@ -242,6 +242,21 @@ func reconcileStoredObjectTags(metadata map[string]string, storedTags, storedTim
|
||||
}
|
||||
}
|
||||
|
||||
// Local tagging mutations must advance the revision they overwrite, even when
|
||||
// a request's clock or lock acquisition order is behind the stored revision.
|
||||
// Replica writes use reconcileStoredObjectTags instead of minting a revision.
|
||||
func monotonicTaggingTimestamp(incoming, stored string) string {
|
||||
requested, err := time.Parse(time.RFC3339Nano, incoming)
|
||||
if err != nil {
|
||||
return incoming
|
||||
}
|
||||
current, err := time.Parse(time.RFC3339Nano, stored)
|
||||
if err != nil || requested.After(current) {
|
||||
return incoming
|
||||
}
|
||||
return current.Add(time.Nanosecond).UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
// A restored version still owns its tier reference even while IsRemote is
|
||||
// false. Only the last copy of a reference may schedule its contents for GC.
|
||||
func sharesTierObject(oi ObjectInfo, copies []PoolObjInfo) bool {
|
||||
@@ -301,6 +316,11 @@ func (z *erasureServerPools) retireReplicaCopies(ctx context.Context, bucket, ob
|
||||
func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
|
||||
copies, err := z.objectPoolInfos(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
if opts.isVersionPurge() && (isErrObjectNotFound(err) || isErrVersionNotFound(err)) {
|
||||
if quorumErr := z.checkPurgeAbsent(ctx, bucket, object, opts.VersionID); quorumErr != nil {
|
||||
return ObjectInfo{}, quorumErr
|
||||
}
|
||||
}
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
primary := copies[0]
|
||||
@@ -350,6 +370,21 @@ func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket,
|
||||
opts.EvalMetadataFn = nil
|
||||
}
|
||||
}
|
||||
if opts.isVersionPurge() {
|
||||
// A missing pool may have only read-quorum absence. Verify every pool
|
||||
// omitted by lookup before mutating the known copies.
|
||||
for i, pool := range z.serverPools {
|
||||
found := false
|
||||
for _, copy := range copies {
|
||||
found = found || copy.Index == i
|
||||
}
|
||||
if !found {
|
||||
if err := pool.getHashedSet(object).checkPurgeAbsent(ctx, bucket, object, opts.VersionID); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if opts.VersionID == "" && (opts.Versioned || opts.VersionSuspended) {
|
||||
// A single new delete marker hides the current version. Older versions
|
||||
// remain history and must not be removed by an unqualified DELETE.
|
||||
|
||||
@@ -618,7 +618,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, idx int, bu
|
||||
if objInfo.isMultipart() {
|
||||
res, err := z.NewMultipartUpload(ctx, bucket, objInfo.Name, ObjectOptions{
|
||||
VersionID: objInfo.VersionID,
|
||||
UserDefined: objInfo.UserDefined,
|
||||
UserDefined: migrationObjectMetadata(objInfo),
|
||||
NoAuditLog: true,
|
||||
SrcPoolIdx: idx,
|
||||
DataMovement: true,
|
||||
@@ -681,7 +681,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, idx int, bu
|
||||
SrcPoolIdx: idx,
|
||||
VersionID: objInfo.VersionID,
|
||||
MTime: objInfo.ModTime,
|
||||
UserDefined: objInfo.UserDefined,
|
||||
UserDefined: migrationObjectMetadata(objInfo),
|
||||
PreserveETag: objInfo.ETag, // Preserve original ETag to ensure same metadata.
|
||||
IndexCB: func() []byte {
|
||||
return objInfo.Parts[0].Index // Preserve part Index to ensure decompression works.
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import xhttp "github.com/minio/minio/internal/http"
|
||||
|
||||
// migrationObjectMetadata restores tags removed from the read representation.
|
||||
// Keep the original revision, including ordered empty states, for the existing
|
||||
// locked reconciliation at PUT or multipart completion. Moving is not a new
|
||||
// tagging mutation, and the write must not modify the reader's metadata map.
|
||||
func migrationObjectMetadata(oi ObjectInfo) map[string]string {
|
||||
metadata := cloneMSS(oi.UserDefined)
|
||||
delete(metadata, xhttp.AmzObjectTagging)
|
||||
if oi.UserTags != "" {
|
||||
metadata[xhttp.AmzObjectTagging] = oi.UserTags
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
@@ -0,0 +1,592 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
const migrationTagRevision = ReservedMetadataPrefixLower + TaggingTimestamp
|
||||
|
||||
var migrationTagStates = []struct {
|
||||
name, tags, revision string
|
||||
hasRevision bool
|
||||
}{
|
||||
{name: "initial", tags: "team=storage&path=a%2Fb"},
|
||||
{name: "empty-revision", tags: "team=storage", hasRevision: true},
|
||||
{name: "invalid-revision", tags: "team=storage", revision: "invalid", hasRevision: true},
|
||||
{name: "ordered", tags: "team=storage", revision: "2026-09-16T09:00:00Z", hasRevision: true},
|
||||
{name: "cleared", revision: "2026-09-16T09:00:00Z", hasRevision: true},
|
||||
{name: "never-tagged"},
|
||||
{name: "empty-empty-revision", hasRevision: true},
|
||||
{name: "empty-invalid-revision", revision: "invalid", hasRevision: true},
|
||||
}
|
||||
|
||||
func TestMigrationObjectMetadata(t *testing.T) {
|
||||
for _, state := range migrationTagStates {
|
||||
for _, residual := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/residual=%t", state.name, residual), func(t *testing.T) {
|
||||
raw := map[string]string{
|
||||
xhttp.AmzObjectTagging: state.tags,
|
||||
"x-amz-meta-team": "storage",
|
||||
"x-amz-meta-empty": "",
|
||||
ReservedMetadataPrefix + "compression": "opaque-compression-state",
|
||||
ReservedMetadataPrefix + "sealed-key": "opaque-key-state",
|
||||
"x-amz-object-lock-retain-until-date": "2030-01-01T00:00:00Z",
|
||||
"x-amz-object-lock-legal-hold": "ON",
|
||||
"x-amz-tagging": "keep-noncanonical-metadata",
|
||||
ReservedMetadataPrefix + "actual-size": "42",
|
||||
ReservedMetadataPrefix + "replica-status": "opaque-replication-state",
|
||||
}
|
||||
if state.hasRevision {
|
||||
raw[migrationTagRevision] = state.revision
|
||||
}
|
||||
oi := (FileInfo{Metadata: raw}).ToObjectInfo("bucket", "object", false)
|
||||
if residual {
|
||||
oi.UserDefined[xhttp.AmzObjectTagging] = "stale=raw"
|
||||
}
|
||||
before := maps.Clone(oi.UserDefined)
|
||||
got := migrationObjectMetadata(oi)
|
||||
if got == nil || !maps.Equal(before, oi.UserDefined) {
|
||||
t.Fatal("helper must return a new non-nil map without changing the read snapshot")
|
||||
}
|
||||
value, exists := got[xhttp.AmzObjectTagging]
|
||||
if value != state.tags || exists != (state.tags != "") {
|
||||
t.Fatalf("raw tags=(%q,%t), want (%q,%t)", value, exists, state.tags, state.tags != "")
|
||||
}
|
||||
stamp, present := got[migrationTagRevision]
|
||||
if stamp != state.revision || present != state.hasRevision {
|
||||
t.Fatalf("revision=(%q,%t), want (%q,%t)", stamp, present, state.revision, state.hasRevision)
|
||||
}
|
||||
for key, value := range before {
|
||||
if stored, exists := got[key]; key != xhttp.AmzObjectTagging && (!exists || stored != value) {
|
||||
t.Errorf("lost metadata %q", key)
|
||||
}
|
||||
}
|
||||
got["x-amz-meta-team"] = "changed"
|
||||
if !maps.Equal(before, oi.UserDefined) {
|
||||
t.Fatal("output aliases the read snapshot")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, metadata := range []map[string]string{nil, {}} {
|
||||
got := migrationObjectMetadata(ObjectInfo{UserDefined: metadata})
|
||||
if got == nil || len(got) != 0 {
|
||||
t.Fatalf("empty input must yield a writable empty map: %#v", got)
|
||||
}
|
||||
got["new"] = "value"
|
||||
if len(metadata) != 0 {
|
||||
t.Fatal("empty map input was aliased")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Allocation ignores the host's used-space percentage, as in the existing tag
|
||||
// fixtures. All object data and metadata still use real erasure-storage disks.
|
||||
func migrationTestPools(t *testing.T) (*erasureServerPools, string) {
|
||||
t.Helper()
|
||||
z, bucket := consistencyPools(t)
|
||||
for _, pool := range z.serverPools {
|
||||
for _, set := range pool.sets {
|
||||
original := set.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i := range disks {
|
||||
disks[i] = tagTestCapacityDisk{StorageAPI: disks[i]}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
t.Cleanup(func() { set.getDisks = original })
|
||||
}
|
||||
}
|
||||
return z, bucket
|
||||
}
|
||||
|
||||
func migrationTestMover(t *testing.T, z *erasureServerPools, source int, kind string) func(context.Context, int, string, *GetObjectReader) error {
|
||||
t.Helper()
|
||||
if kind == "rebalance" {
|
||||
z.rebalMu.Lock()
|
||||
z.rebalMeta = &rebalanceMeta{PoolStats: []*rebalanceStats{{}, {}}}
|
||||
z.rebalMeta.PoolStats[source] = &rebalanceStats{Participating: true, Info: rebalanceInfo{Status: rebalStarted}}
|
||||
z.rebalMu.Unlock()
|
||||
t.Cleanup(func() { z.rebalMu.Lock(); z.rebalMeta = nil; z.rebalMu.Unlock() })
|
||||
return z.rebalanceObject
|
||||
}
|
||||
z.poolMetaMutex.Lock()
|
||||
z.poolMeta.Pools[source].Decommission = &PoolDecommissionInfo{}
|
||||
z.poolMetaMutex.Unlock()
|
||||
t.Cleanup(func() { z.poolMetaMutex.Lock(); z.poolMeta.Pools[source].Decommission = nil; z.poolMetaMutex.Unlock() })
|
||||
return z.decommissionObject
|
||||
}
|
||||
|
||||
func migrationTestObject(t *testing.T, z *erasureServerPools, bucket, object string, source int, multipart bool, parts []string, opts ObjectOptions) ObjectInfo {
|
||||
t.Helper()
|
||||
if !multipart {
|
||||
oi := putConsistencyObject(t, z, bucket, object, source, strings.Join(parts, ""), opts)
|
||||
return migrationTestPersistedInfo(t, z, source, oi)
|
||||
}
|
||||
mp, err := z.serverPools[source].NewMultipartUpload(t.Context(), bucket, object, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
completed := make([]CompletePart, len(parts))
|
||||
for i, body := range parts {
|
||||
part, err := z.serverPools[source].PutObjectPart(t.Context(), bucket, object, mp.UploadID, i+1,
|
||||
mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
completed[i] = CompletePart{PartNumber: i + 1, ETag: part.ETag}
|
||||
}
|
||||
oi, err := z.serverPools[source].CompleteMultipartUpload(t.Context(), bucket, object, mp.UploadID, completed, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !oi.isMultipart() {
|
||||
t.Fatal("fixture did not create a multipart object")
|
||||
}
|
||||
return migrationTestPersistedInfo(t, z, source, oi)
|
||||
}
|
||||
|
||||
func migrationTestPersistedInfo(t *testing.T, z *erasureServerPools, pool int, oi ObjectInfo) ObjectInfo {
|
||||
t.Helper()
|
||||
// PUT can return transient fields such as tier-free-versionID that are not
|
||||
// stored. Take the same persisted read representation used by the movers.
|
||||
got, err := z.serverPools[pool].GetObjectInfo(t.Context(), oi.Bucket, oi.Name, ObjectOptions{VersionID: migrationTestVersion(oi)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func migrationTestVersion(oi ObjectInfo) string {
|
||||
if oi.VersionID == "" {
|
||||
return nullVersionID
|
||||
}
|
||||
return oi.VersionID
|
||||
}
|
||||
|
||||
func migrationTestReader(t *testing.T, z *erasureServerPools, source int, oi ObjectInfo) *GetObjectReader {
|
||||
t.Helper()
|
||||
gr, err := z.serverPools[source].GetObjectNInfo(t.Context(), oi.Bucket, oi.Name, nil, nil,
|
||||
ObjectOptions{VersionID: migrationTestVersion(oi), NoLock: true, NoDecryption: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { gr.Close() })
|
||||
return gr
|
||||
}
|
||||
|
||||
func migrationTestStored(t *testing.T, z *erasureServerPools, pool int, original ObjectInfo, tags, revision string, hasRevision bool, body string) {
|
||||
t.Helper()
|
||||
version := migrationTestVersion(original)
|
||||
got, err := z.serverPools[pool].GetObjectInfo(t.Context(), original.Bucket, original.Name, ObjectOptions{VersionID: version})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.UserTags != tags || got.ETag != original.ETag || !got.ModTime.Equal(original.ModTime) || got.VersionID != original.VersionID || got.Size != original.Size {
|
||||
t.Errorf("pool %d changed tags/object identity: tags=%q want=%q version=%q/%q etag=%q/%q size=%d/%d mtime=%v/%v", pool, got.UserTags, tags, got.VersionID, original.VersionID, got.ETag, original.ETag, got.Size, original.Size, got.ModTime, original.ModTime)
|
||||
}
|
||||
infos, errs := readAllFileInfo(t.Context(), z.serverPools[pool].getHashedSet(original.Name).getDisks(), "", original.Bucket, original.Name, version, false, false)
|
||||
for disk, fi := range infos {
|
||||
if errs[disk] != nil {
|
||||
t.Fatalf("pool %d disk %d: %v", pool, disk, errs[disk])
|
||||
}
|
||||
stamp, exists := fi.Metadata[migrationTagRevision]
|
||||
if fi.Metadata[xhttp.AmzObjectTagging] != tags || stamp != revision || exists != hasRevision {
|
||||
t.Errorf("pool %d disk %d: tags=%q revision=(%q,%t), want %q (%q,%t)", pool, disk, fi.Metadata[xhttp.AmzObjectTagging], stamp, exists, tags, revision, hasRevision)
|
||||
}
|
||||
for key, value := range original.UserDefined {
|
||||
if key != migrationTagRevision && fi.Metadata[key] != value {
|
||||
t.Errorf("pool %d disk %d lost metadata %q", pool, disk, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
gr, err := z.serverPools[pool].GetObjectNInfo(t.Context(), original.Bucket, original.Name, nil, nil, ObjectOptions{VersionID: version})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := io.ReadAll(gr)
|
||||
gr.Close()
|
||||
if err != nil || !bytes.Equal(data, []byte(body)) {
|
||||
t.Fatalf("pool %d content changed: bytes=%d err=%v", pool, len(data), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMigrationPreservesTagState(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for _, method := range []string{"put", "multipart"} {
|
||||
for source := range 2 {
|
||||
for _, state := range migrationTagStates {
|
||||
t.Run(fmt.Sprintf("%s/%s/source=%d/%s", kind, method, source, state.name), func(t *testing.T) {
|
||||
move := migrationTestMover(t, z, source, kind)
|
||||
meta := map[string]string{xhttp.AmzObjectTagging: state.tags, "x-amz-meta-owner": "retained"}
|
||||
if state.hasRevision {
|
||||
meta[migrationTagRevision] = state.revision
|
||||
}
|
||||
oi := migrationTestObject(t, z, bucket, t.Name(), source, method == "multipart", []string{"payload"}, ObjectOptions{Versioned: true, UserDefined: meta})
|
||||
migrationTestStored(t, z, source, oi, state.tags, state.revision, state.hasRevision, "payload")
|
||||
gr := migrationTestReader(t, z, source, oi)
|
||||
before := maps.Clone(gr.ObjInfo.UserDefined)
|
||||
if err := move(t.Context(), source, bucket, gr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !maps.Equal(before, gr.ObjInfo.UserDefined) {
|
||||
t.Error("migration modified its source snapshot")
|
||||
}
|
||||
migrationTestStored(t, z, 1-source, oi, state.tags, state.revision, state.hasRevision, "payload")
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type migrationTestGate struct {
|
||||
entered, resume chan struct{}
|
||||
once, release sync.Once
|
||||
}
|
||||
|
||||
func newMigrationTestGate() *migrationTestGate {
|
||||
return &migrationTestGate{entered: make(chan struct{}), resume: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (g *migrationTestGate) wait(ctx context.Context) error {
|
||||
g.once.Do(func() { close(g.entered) })
|
||||
select {
|
||||
case <-g.resume:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (g *migrationTestGate) unblock() { g.release.Do(func() { close(g.resume) }) }
|
||||
|
||||
type migrationTestGateReader struct {
|
||||
io.Reader
|
||||
ctx context.Context
|
||||
gate *migrationTestGate
|
||||
}
|
||||
|
||||
func (r migrationTestGateReader) Read(p []byte) (int, error) {
|
||||
if err := r.gate.wait(r.ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return r.Reader.Read(p)
|
||||
}
|
||||
|
||||
func TestPoolsMigrationRechecksTags(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
const recent = "2026-09-16T10:00:00Z"
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for _, method := range []string{"put", "multipart"} {
|
||||
for source := range 2 {
|
||||
for _, tags := range []string{"state=after", ""} {
|
||||
t.Run(fmt.Sprintf("%s/%s/source=%d/clear=%t", kind, method, source, tags == ""), func(t *testing.T) {
|
||||
move := migrationTestMover(t, z, source, kind)
|
||||
oi := migrationTestObject(t, z, bucket, t.Name(), source, method == "multipart", []string{"payload"}, ObjectOptions{
|
||||
Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "state=before"},
|
||||
})
|
||||
gr := migrationTestReader(t, z, source, oi)
|
||||
before := maps.Clone(gr.ObjInfo.UserDefined)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
gate := newMigrationTestGate()
|
||||
done, finished := make(chan error, 1), make(chan struct{})
|
||||
if method == "multipart" {
|
||||
// The first part read is after persisted upload initialization,
|
||||
// but before completion takes the object lock and reconciles.
|
||||
gr.Reader = migrationTestGateReader{Reader: gr.Reader, ctx: ctx, gate: gate}
|
||||
go func() { defer close(finished); done <- move(ctx, source, bucket, gr) }()
|
||||
defer func() { gate.unblock(); cancel(); <-finished }()
|
||||
select {
|
||||
case <-gate.entered:
|
||||
case err := <-done:
|
||||
t.Fatalf("migration finished before the part barrier: %v", err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
uploads, err := z.serverPools[1-source].listMultipartUploadsExact(ctx, bucket, oi.Name)
|
||||
if err != nil || len(uploads.Uploads) != 1 {
|
||||
t.Fatalf("upload not persisted at barrier: %+v, %v", uploads, err)
|
||||
}
|
||||
info, err := z.serverPools[1-source].GetMultipartInfo(ctx, bucket, oi.Name, uploads.Uploads[0].UploadID, ObjectOptions{})
|
||||
if err != nil || info.UserDefined[xhttp.AmzObjectTagging] != "state=before" {
|
||||
t.Fatalf("upload lost snapshot tags: %v, %v", info.UserDefined, err)
|
||||
}
|
||||
}
|
||||
// For ordinary PUT this must finish before calling the mover:
|
||||
// blocking its data Read would already hold the object lock.
|
||||
opts := ObjectOptions{VersionID: migrationTestVersion(oi), UserDefined: map[string]string{migrationTagRevision: recent}}
|
||||
var err error
|
||||
if tags == "" {
|
||||
_, err = z.DeleteObjectTags(ctx, bucket, oi.Name, opts)
|
||||
} else {
|
||||
_, err = z.PutObjectTags(ctx, bucket, oi.Name, tags, opts)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if gr.ObjInfo.UserTags != "state=before" || !maps.Equal(before, gr.ObjInfo.UserDefined) {
|
||||
t.Fatal("test must retain the old source snapshot")
|
||||
}
|
||||
migrationTestStored(t, z, source, oi, tags, recent, true, "payload")
|
||||
if method == "multipart" {
|
||||
gate.unblock()
|
||||
err = <-done
|
||||
} else {
|
||||
err = move(ctx, source, bucket, gr)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !maps.Equal(before, gr.ObjInfo.UserDefined) {
|
||||
t.Error("reconciliation changed the reader's map")
|
||||
}
|
||||
migrationTestStored(t, z, 1-source, oi, tags, recent, true, "payload")
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type migrationTestMetadataGateDisk struct {
|
||||
StorageAPI
|
||||
bucket, object string
|
||||
gate *migrationTestGate
|
||||
}
|
||||
|
||||
func (d migrationTestMetadataGateDisk) UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo, opts UpdateMetadataOpts) error {
|
||||
if volume == d.bucket && path == d.object {
|
||||
if err := d.gate.wait(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return d.StorageAPI.UpdateMetadata(ctx, volume, path, fi, opts)
|
||||
}
|
||||
|
||||
func TestPoolsMigrationTagsDuringCleanup(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
const recent = "2026-09-16T10:00:00Z"
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for source := range 2 {
|
||||
for _, tags := range []string{"state=after", ""} {
|
||||
for _, interrupt := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/source=%d/clear=%t/interrupt=%t", kind, source, tags == "", interrupt), func(t *testing.T) {
|
||||
move := migrationTestMover(t, z, source, kind)
|
||||
oi := migrationTestObject(t, z, bucket, t.Name(), source, false, []string{"payload"}, ObjectOptions{
|
||||
Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "state=before"},
|
||||
})
|
||||
if err := move(t.Context(), source, bucket, migrationTestReader(t, z, source, oi)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
migrationTestStored(t, z, 1-source, oi, "state=before", "", false, "payload")
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
mutate := func() (ObjectInfo, error) {
|
||||
opts := ObjectOptions{VersionID: migrationTestVersion(oi), UserDefined: map[string]string{migrationTagRevision: recent}}
|
||||
if tags == "" {
|
||||
return z.DeleteObjectTags(ctx, bucket, oi.Name, opts)
|
||||
}
|
||||
return z.PutObjectTags(ctx, bucket, oi.Name, tags, opts)
|
||||
}
|
||||
set := z.serverPools[source].getHashedSet(oi.Name)
|
||||
cleanup := func() {
|
||||
// Use the same source prefix cleanup as both outer movers.
|
||||
if _, err := set.DeleteObject(ctx, bucket, oi.Name, ObjectOptions{DeletePrefix: true, DeletePrefixObject: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var updated ObjectInfo
|
||||
if !interrupt {
|
||||
var err error
|
||||
updated, err = mutate()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cleanup()
|
||||
} else {
|
||||
gate := newMigrationTestGate()
|
||||
original := set.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i := range disks {
|
||||
disks[i] = migrationTestMetadataGateDisk{StorageAPI: disks[i], bucket: bucket, object: oi.Name, gate: gate}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
defer func() { set.getDisks = original }()
|
||||
done, finished := make(chan error, 1), make(chan struct{})
|
||||
go func() { defer close(finished); _, err := mutate(); done <- err }()
|
||||
defer func() { gate.unblock(); cancel(); <-finished }()
|
||||
select {
|
||||
case <-gate.entered:
|
||||
case err := <-done:
|
||||
t.Fatalf("metadata update missed the barrier: %v", err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
cleanup()
|
||||
gate.unblock()
|
||||
if err := <-done; err == nil {
|
||||
t.Fatal("update reported success after its source copy was removed")
|
||||
}
|
||||
<-finished
|
||||
set.getDisks = original
|
||||
var err error
|
||||
updated, err = mutate()
|
||||
if err != nil {
|
||||
t.Fatalf("metadata retry did not converge: %v", err)
|
||||
}
|
||||
}
|
||||
if _, err := z.serverPools[source].GetObjectInfo(ctx, bucket, oi.Name, ObjectOptions{VersionID: oi.VersionID}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Fatalf("source cleanup left a copy: %v", err)
|
||||
}
|
||||
migrationTestStored(t, z, 1-source, oi, tags, updated.UserDefined[migrationTagRevision], true, "payload")
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMigrationVersionHistory(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for _, method := range []string{"put", "multipart"} {
|
||||
for _, versioning := range []string{"unversioned", "null", "history"} {
|
||||
t.Run(kind+"/"+method+"/"+versioning, func(t *testing.T) {
|
||||
move := migrationTestMover(t, z, 0, kind)
|
||||
opts := ObjectOptions{
|
||||
Versioned: versioning == "history", VersionSuspended: versioning == "null",
|
||||
MTime: time.Date(2026, 9, 16, 8, 0, 0, 0, time.UTC),
|
||||
UserDefined: map[string]string{xhttp.AmzObjectTagging: "version=old"},
|
||||
}
|
||||
versions := []ObjectInfo{migrationTestObject(t, z, bucket, t.Name(), 0, method == "multipart", []string{"old"}, opts)}
|
||||
if versioning == "history" {
|
||||
opts.MTime = opts.MTime.Add(time.Minute)
|
||||
versions = append(versions, migrationTestObject(t, z, bucket, t.Name(), 0, method == "multipart", []string{"old"}, opts))
|
||||
}
|
||||
var newest ObjectInfo
|
||||
if versioning != "unversioned" {
|
||||
newest = migrationTestObject(t, z, bucket, t.Name(), 0, false, []string{"latest"}, ObjectOptions{
|
||||
Versioned: true, MTime: opts.MTime.Add(time.Minute),
|
||||
UserDefined: map[string]string{xhttp.AmzObjectTagging: "version=new"},
|
||||
})
|
||||
}
|
||||
for _, oi := range versions {
|
||||
if err := move(t.Context(), 0, bucket, migrationTestReader(t, z, 0, oi)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
migrationTestStored(t, z, 1, oi, "version=old", "", false, "old")
|
||||
}
|
||||
if versioning != "unversioned" {
|
||||
migrationTestStored(t, z, 0, newest, "version=new", "", false, "latest")
|
||||
if _, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, t.Name(), ObjectOptions{VersionID: newest.VersionID}); !isErrVersionNotFound(err) {
|
||||
t.Fatalf("moving an addressed version affected the newer version: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMigrationMultipartParts(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
parts := []string{strings.Repeat("a", 5<<20), "tail-with-tags"}
|
||||
body := strings.Join(parts, "")
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for source := range 2 {
|
||||
t.Run(fmt.Sprintf("%s/source=%d", kind, source), func(t *testing.T) {
|
||||
move := migrationTestMover(t, z, source, kind)
|
||||
oi := migrationTestObject(t, z, bucket, t.Name(), source, true, parts, ObjectOptions{
|
||||
Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "multipart=kept"},
|
||||
})
|
||||
if err := move(t.Context(), source, bucket, migrationTestReader(t, z, source, oi)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
migrationTestStored(t, z, 1-source, oi, "multipart=kept", "", false, body)
|
||||
got, err := z.serverPools[1-source].GetObjectInfo(t.Context(), bucket, oi.Name, ObjectOptions{VersionID: oi.VersionID})
|
||||
if err != nil || len(got.Parts) != len(oi.Parts) {
|
||||
t.Fatalf("parts changed: %+v, %v", got.Parts, err)
|
||||
}
|
||||
for i, part := range got.Parts {
|
||||
old := oi.Parts[i]
|
||||
if part.Number != old.Number || part.Size != old.Size || part.ActualSize != old.ActualSize || part.ETag != old.ETag || !bytes.Equal(part.Index, old.Index) {
|
||||
t.Errorf("part %d changed: %+v / %+v", i, part, old)
|
||||
}
|
||||
}
|
||||
start := int64(len(parts[0]) - 3)
|
||||
gr, err := z.serverPools[1-source].GetObjectNInfo(t.Context(), bucket, oi.Name, &HTTPRangeSpec{Start: start, End: start + 7}, nil, ObjectOptions{VersionID: oi.VersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := io.ReadAll(gr)
|
||||
gr.Close()
|
||||
if err != nil || string(data) != body[start:start+8] {
|
||||
t.Fatalf("cross-part range changed: %q, %v", data, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMigrationUnreadablePool(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for _, method := range []string{"put", "multipart"} {
|
||||
t.Run(kind+"/"+method, func(t *testing.T) {
|
||||
move := migrationTestMover(t, z, 0, kind)
|
||||
oi := migrationTestObject(t, z, bucket, t.Name(), 0, method == "multipart", []string{"payload"}, ObjectOptions{
|
||||
Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "keep=source"},
|
||||
})
|
||||
gr := migrationTestReader(t, z, 0, oi)
|
||||
set := z.serverPools[0].getHashedSet(oi.Name)
|
||||
original := set.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i := range disks {
|
||||
disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: oi.Name}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
defer func() { set.getDisks = original }()
|
||||
err := move(t.Context(), 0, bucket, gr)
|
||||
var quorum InsufficientReadQuorum
|
||||
if !errors.As(err, &quorum) {
|
||||
t.Fatalf("unreadable pool must fail migration with read quorum error: %v", err)
|
||||
}
|
||||
set.getDisks = original
|
||||
migrationTestStored(t, z, 0, oi, "keep=source", "", false, "payload")
|
||||
if _, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, oi.Name, ObjectOptions{VersionID: oi.VersionID}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Fatalf("failed migration committed a target version: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -265,6 +265,7 @@ func TestPoolsMultipartConditionalLatestVersionAndCallbackOnce(t *testing.T) {
|
||||
if explicit {
|
||||
current = putConsistencyObject(t, z, bucket, object, 1, "latest-other-version", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Minute)})
|
||||
}
|
||||
opts.VersionID = ""
|
||||
id, parts := multipartConditionUpload(t, z, bucket, object, 1, opts)
|
||||
called := 0
|
||||
opts.CheckPrecondFn = func(oi ObjectInfo) bool { called++; return oi.ETag != current.ETag }
|
||||
|
||||
@@ -0,0 +1,716 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
)
|
||||
|
||||
// Change allocation capacity only; metadata and data use real fixture disks.
|
||||
// Restoring the adapters also lets encrypted fixtures move the write target
|
||||
// between requests without changing the production allocation policy.
|
||||
type conditionalPutCapacityDisk struct {
|
||||
StorageAPI
|
||||
full bool
|
||||
}
|
||||
|
||||
func (d conditionalPutCapacityDisk) DiskInfo(ctx context.Context, opts DiskInfoOptions) (DiskInfo, error) {
|
||||
info, err := d.StorageAPI.DiskInfo(ctx, opts)
|
||||
info.Total, info.Used = 1<<40, 0
|
||||
if d.full {
|
||||
info.Used = info.Total - (1 << 20)
|
||||
}
|
||||
info.Free = info.Total - info.Used
|
||||
return info, err
|
||||
}
|
||||
|
||||
// Keep getDisks immutable while background IAM and storage readers use it.
|
||||
// GetDisks takes this same mutex when it copies the backing disk list.
|
||||
func conditionalPutSwapDisks(pool *erasureSets, object string, wrap func(StorageAPI) StorageAPI) func() {
|
||||
setIndex := pool.getHashedSet(object).setIndex
|
||||
pool.erasureDisksMu.Lock()
|
||||
previous := pool.erasureDisks[setIndex]
|
||||
disks := append([]StorageAPI(nil), previous...)
|
||||
for i, disk := range disks {
|
||||
disks[i] = wrap(disk)
|
||||
}
|
||||
pool.erasureDisks[setIndex] = disks
|
||||
pool.erasureDisksMu.Unlock()
|
||||
return func() {
|
||||
pool.erasureDisksMu.Lock()
|
||||
pool.erasureDisks[setIndex] = previous
|
||||
pool.erasureDisksMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func conditionalPutPool(t *testing.T, z *erasureServerPools, object string, target int) func() {
|
||||
t.Helper()
|
||||
var restore []func()
|
||||
for i, pool := range z.serverPools {
|
||||
restore = append(restore, conditionalPutSwapDisks(pool, object, func(disk StorageAPI) StorageAPI {
|
||||
return conditionalPutCapacityDisk{StorageAPI: disk, full: i != target}
|
||||
}))
|
||||
}
|
||||
return func() {
|
||||
for _, fn := range restore {
|
||||
fn()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func conditionalPutBucket(t *testing.T, z *erasureServerPools, mode string) (string, http.Handler) {
|
||||
t.Helper()
|
||||
bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{VersioningEnabled: mode != "unversioned"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mode == "suspended" {
|
||||
if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig,
|
||||
[]byte(`<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return bucket, router
|
||||
}
|
||||
|
||||
func TestPoolsConditionalPutHTTP(t *testing.T) {
|
||||
for _, mode := range []string{"unversioned", "versioned", "suspended"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
bucket, router := conditionalPutBucket(t, z, mode)
|
||||
for target := range 2 {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
oldCopy, localCurrent bool
|
||||
condition string
|
||||
status int
|
||||
}{
|
||||
{"stale-etag-accepted", true, false, "old", http.StatusPreconditionFailed},
|
||||
{"current-etag-rejected", true, false, "current", http.StatusOK},
|
||||
{"create-only-overwrites-other-pool", false, false, "none", http.StatusPreconditionFailed},
|
||||
{"current-etag-missing-in-write-pool", false, false, "current", http.StatusOK},
|
||||
{"control-current-in-write-pool", true, true, "current", http.StatusOK},
|
||||
{"control-stale-etag-rejected", true, true, "old", http.StatusPreconditionFailed},
|
||||
{"none-match-current-etag", true, false, "none-current", http.StatusPreconditionFailed},
|
||||
{"none-match-old-etag", true, false, "none-old", http.StatusOK},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("target=%d/%s", target, tc.name), func(t *testing.T) {
|
||||
object := fmt.Sprintf("%d-%s", target, tc.name)
|
||||
currentPool := 1 - target
|
||||
if tc.localCurrent {
|
||||
currentPool = target
|
||||
}
|
||||
opts := ObjectOptions{Versioned: mode == "versioned", VersionSuspended: mode == "suspended", MTime: UTCNow().Add(-time.Hour)}
|
||||
oldETag := "absent-old"
|
||||
if tc.oldCopy {
|
||||
oldETag = putConsistencyObject(t, z, bucket, object, 1-currentPool, "old", opts).ETag
|
||||
}
|
||||
opts.MTime = UTCNow().Add(-time.Minute)
|
||||
current := putConsistencyObject(t, z, bucket, object, currentPool, "current", opts)
|
||||
defer conditionalPutPool(t, z, object, target)()
|
||||
idx, err := z.getWritePoolIdx(t.Context(), bucket, object, 11, false)
|
||||
if err != nil || idx != target {
|
||||
t.Fatalf("allocation target=%d: idx=%d err=%v", target, idx, err)
|
||||
}
|
||||
headers := map[string]string{xhttp.IfMatch: fmt.Sprintf("%q", current.ETag)}
|
||||
if tc.condition == "old" {
|
||||
headers[xhttp.IfMatch] = fmt.Sprintf("%q", oldETag)
|
||||
}
|
||||
if tc.condition == "none" {
|
||||
headers = map[string]string{xhttp.IfNoneMatch: "*"}
|
||||
}
|
||||
if tc.condition == "none-current" {
|
||||
headers = map[string]string{xhttp.IfNoneMatch: fmt.Sprintf("%q", current.ETag)}
|
||||
}
|
||||
if tc.condition == "none-old" {
|
||||
headers = map[string]string{xhttp.IfNoneMatch: fmt.Sprintf("%q", oldETag)}
|
||||
}
|
||||
url := getPutObjectURL("", bucket, object)
|
||||
before := multipartConditionRequest(t, router, http.MethodGet, url, "", nil)
|
||||
if before.Code != http.StatusOK || before.Body.String() != "current" || multipartConditionResponseETag(before) != current.ETag {
|
||||
t.Fatalf("invalid current object: %d %q %v", before.Code, before.Body.String(), before.Header())
|
||||
}
|
||||
put := multipartConditionRequest(t, router, http.MethodPut, url, "replacement", headers)
|
||||
if put.Code != tc.status {
|
||||
t.Errorf("PUT status=%d want=%d: %s", put.Code, tc.status, put.Body.String())
|
||||
}
|
||||
if put.Code == http.StatusPreconditionFailed {
|
||||
multipartConditionError(t, put, "PreconditionFailed")
|
||||
if multipartConditionResponseETag(put) != current.ETag || put.Header().Get(xhttp.LastModified) != before.Header().Get(xhttp.LastModified) {
|
||||
t.Errorf("412 headers do not describe current object: %v", put.Header())
|
||||
}
|
||||
}
|
||||
get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil)
|
||||
wantBody, wantETag := "current", current.ETag
|
||||
if tc.status == http.StatusOK {
|
||||
wantBody, wantETag = "replacement", fmt.Sprintf("%x", md5.Sum([]byte("replacement")))
|
||||
}
|
||||
t.Logf("PUT %d; GET %d bytes=%q ETag=%s", put.Code, get.Code, get.Body.String(), multipartConditionResponseETag(get))
|
||||
if get.Code != http.StatusOK || get.Body.String() != wantBody || multipartConditionResponseETag(get) != wantETag {
|
||||
t.Errorf("GET=%d bytes=%q ETag=%s; want %q %s", get.Code, get.Body.String(), multipartConditionResponseETag(get), wantBody, wantETag)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsConditionalPutHTTPAbsence(t *testing.T) {
|
||||
for _, state := range []string{"missing", "uuid-marker", "null-marker"} {
|
||||
for _, match := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/match=%t", state, match), func(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
mode := "versioned"
|
||||
if state == "null-marker" {
|
||||
mode = "suspended"
|
||||
}
|
||||
bucket, router := conditionalPutBucket(t, z, mode)
|
||||
object := "absent-key"
|
||||
if state != "missing" {
|
||||
putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
|
||||
vid := mustGetUUID()
|
||||
if state == "null-marker" {
|
||||
vid = nullVersionID
|
||||
}
|
||||
if _, err := z.serverPools[1].DeleteObject(t.Context(), bucket, object, ObjectOptions{Versioned: true, VersionID: vid, DeleteMarker: true, MTime: UTCNow().Add(-time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
defer conditionalPutPool(t, z, object, 0)()
|
||||
headers := map[string]string{xhttp.IfNoneMatch: "*"}
|
||||
want := http.StatusOK
|
||||
if match {
|
||||
headers = map[string]string{xhttp.IfMatch: "*"}
|
||||
want = http.StatusNotFound
|
||||
}
|
||||
url := getPutObjectURL("", bucket, object)
|
||||
put := multipartConditionRequest(t, router, http.MethodPut, url, "new", headers)
|
||||
if put.Code != want {
|
||||
t.Fatalf("PUT %d want %d: %s", put.Code, want, put.Body.String())
|
||||
}
|
||||
get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil)
|
||||
if match {
|
||||
multipartConditionError(t, put, "NoSuchKey")
|
||||
if get.Code != http.StatusNotFound {
|
||||
t.Fatalf("failed PUT exposed data: %d %s", get.Code, get.Body.String())
|
||||
}
|
||||
} else if get.Code != http.StatusOK || get.Body.String() != "new" || multipartConditionResponseETag(get) != multipartConditionResponseETag(put) {
|
||||
t.Fatalf("successful create GET: %d %q %v", get.Code, get.Body.String(), get.Header())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsConditionalPutUnreadable(t *testing.T) {
|
||||
for faultPool := range 2 {
|
||||
for _, present := range []bool{false, true} {
|
||||
for _, match := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("fault-pool=%d/present=%t/match=%t", faultPool, present, match), func(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
bucket, router := conditionalPutBucket(t, z, "unversioned")
|
||||
object := "unreadable-key"
|
||||
if present {
|
||||
putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{MTime: UTCNow().Add(-time.Hour)})
|
||||
putConsistencyObject(t, z, bucket, object, 1, "current", ObjectOptions{MTime: UTCNow().Add(-time.Minute)})
|
||||
}
|
||||
defer conditionalPutPool(t, z, object, 0)()
|
||||
restoreFault := conditionalPutSwapDisks(z.serverPools[faultPool], object, func(disk StorageAPI) StorageAPI {
|
||||
return consistencyReadFaultDisk{StorageAPI: disk, bucket: bucket, object: object}
|
||||
})
|
||||
defer restoreFault()
|
||||
headers := map[string]string{xhttp.IfNoneMatch: "*"}
|
||||
if match {
|
||||
headers = map[string]string{xhttp.IfMatch: "*"}
|
||||
}
|
||||
url := getPutObjectURL("", bucket, object)
|
||||
put := multipartConditionRequest(t, router, http.MethodPut, url, "replacement", headers)
|
||||
if put.Code != http.StatusServiceUnavailable {
|
||||
t.Errorf("unverified PUT must fail: %d %s", put.Code, put.Body.String())
|
||||
}
|
||||
called := 0
|
||||
_, err := z.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader("replacement"), 11, "", ""), ObjectOptions{HasIfMatch: match, CheckPrecondFn: func(ObjectInfo) bool { called++; return false }})
|
||||
if !isErrReadQuorum(err) || called != 0 {
|
||||
t.Errorf("lookup error=%v callback calls=%d", err, called)
|
||||
}
|
||||
restoreFault()
|
||||
get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil)
|
||||
if present {
|
||||
if get.Code != http.StatusOK || get.Body.String() != "current" || multipartConditionResponseETag(get) != fmt.Sprintf("%x", md5.Sum([]byte("current"))) {
|
||||
t.Fatalf("failed PUT changed object: %d %q", get.Code, get.Body.String())
|
||||
}
|
||||
} else if get.Code != http.StatusNotFound {
|
||||
t.Fatalf("failed PUT created object: %d %q", get.Code, get.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsConditionalPutEncryptedETag(t *testing.T) {
|
||||
for _, kind := range []string{"SSE-C", "SSE-S3", "SSE-KMS"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
bucket, router := conditionalPutBucket(t, z, "unversioned")
|
||||
oldKMS, oldTLS := GlobalKMS, globalIsTLS
|
||||
GlobalKMS, globalIsTLS = kms.NewStub("conditional-put-key"), true
|
||||
defer func() { GlobalKMS, globalIsTLS = oldKMS, oldTLS }()
|
||||
headers := map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES}
|
||||
readHeaders := map[string]string{}
|
||||
if kind == "SSE-C" {
|
||||
key := bytes.Repeat([]byte{0x42}, 32)
|
||||
digest := md5.Sum(key)
|
||||
headers = map[string]string{
|
||||
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key),
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(digest[:]),
|
||||
}
|
||||
readHeaders = maps.Clone(headers)
|
||||
}
|
||||
if kind == "SSE-KMS" {
|
||||
headers[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
|
||||
headers[xhttp.AmzServerSideEncryptionKmsID] = "conditional-put-key"
|
||||
}
|
||||
object := "encrypted-key"
|
||||
url := getPutObjectURL("", bucket, object)
|
||||
restore := conditionalPutPool(t, z, object, 0)
|
||||
old := multipartConditionRequest(t, router, http.MethodPut, url, "old", headers)
|
||||
restore()
|
||||
restore = conditionalPutPool(t, z, object, 1)
|
||||
current := multipartConditionRequest(t, router, http.MethodPut, url, "current", headers)
|
||||
restore()
|
||||
if old.Code != http.StatusOK || current.Code != http.StatusOK {
|
||||
t.Fatalf("encrypted setup: %d %s / %d %s", old.Code, old.Body.String(), current.Code, current.Body.String())
|
||||
}
|
||||
defer conditionalPutPool(t, z, object, 0)()
|
||||
for _, stale := range []bool{true, false} {
|
||||
h := maps.Clone(headers)
|
||||
h[xhttp.IfMatch] = fmt.Sprintf("%q", multipartConditionResponseETag(current))
|
||||
wantStatus, wantBody, wantETag := http.StatusOK, "replacement", ""
|
||||
if stale {
|
||||
h[xhttp.IfMatch] = fmt.Sprintf("%q", multipartConditionResponseETag(old))
|
||||
wantStatus, wantBody, wantETag = http.StatusPreconditionFailed, "current", multipartConditionResponseETag(current)
|
||||
}
|
||||
put := multipartConditionRequest(t, router, http.MethodPut, url, "replacement", h)
|
||||
if put.Code != wantStatus {
|
||||
t.Fatalf("encrypted condition: %d want %d: %s", put.Code, wantStatus, put.Body.String())
|
||||
}
|
||||
if !stale {
|
||||
wantETag = multipartConditionResponseETag(put)
|
||||
}
|
||||
get := multipartConditionRequest(t, router, http.MethodGet, url, "", readHeaders)
|
||||
if get.Code != http.StatusOK || get.Body.String() != wantBody || multipartConditionResponseETag(get) != wantETag {
|
||||
t.Fatalf("encrypted GET: %d %q %v", get.Code, get.Body.String(), get.Header())
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsConditionalPutVersionSelection(t *testing.T) {
|
||||
for _, kind := range []string{"public-version", "replica", "replica-preserve-etag", "movement", "no-lock", "tie", "draining"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
object := "version-selection"
|
||||
addressed := putConsistencyObject(t, z, bucket, object, 1, "addressed", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
|
||||
current := putConsistencyObject(t, z, bucket, object, 1, "current", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Minute)})
|
||||
opts := ObjectOptions{Versioned: true, VersionID: addressed.VersionID, HasIfMatch: true}
|
||||
want := current
|
||||
switch kind {
|
||||
case "replica":
|
||||
opts.ReplicaLockReconcile, opts.ReplicationRequest = true, true
|
||||
want = addressed
|
||||
case "replica-preserve-etag":
|
||||
opts.PreserveETag = addressed.ETag
|
||||
opts.ReplicaLockReconcile, opts.ReplicationRequest = true, true
|
||||
want = addressed
|
||||
case "movement":
|
||||
opts.DataMovement, opts.SrcPoolIdx = true, 1
|
||||
want = addressed
|
||||
case "tie":
|
||||
want = putConsistencyObject(t, z, bucket, object, 0, "tie-winner", ObjectOptions{Versioned: true, MTime: current.ModTime})
|
||||
case "draining":
|
||||
z.poolMetaMutex.Lock()
|
||||
z.poolMeta.Pools[1].Decommission = &PoolDecommissionInfo{}
|
||||
z.poolMetaMutex.Unlock()
|
||||
}
|
||||
defer conditionalPutPool(t, z, object, 0)()
|
||||
ctx := t.Context()
|
||||
if kind == "no-lock" {
|
||||
lk := z.NewNSLock(bucket, object)
|
||||
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer lk.Unlock(lkctx)
|
||||
ctx, opts.NoLock = lkctx.Context(), true
|
||||
}
|
||||
called := 0
|
||||
opts.UserDefined = make(map[string]string)
|
||||
opts.CheckPrecondFn = func(oi ObjectInfo) bool {
|
||||
called++
|
||||
if oi.ETag != want.ETag || oi.VersionID != want.VersionID {
|
||||
t.Errorf("comparison ETag/version=%s/%s want %s/%s", oi.ETag, oi.VersionID, want.ETag, want.VersionID)
|
||||
}
|
||||
return oi.ETag != want.ETag
|
||||
}
|
||||
oi, err := z.PutObject(ctx, bucket, object, mustGetPutObjReader(t, strings.NewReader("replacement"), 11, "", ""), opts)
|
||||
if err != nil || called != 1 {
|
||||
t.Fatalf("PUT err=%v callback calls=%d", err, called)
|
||||
}
|
||||
if oi.VersionID != addressed.VersionID {
|
||||
t.Fatalf("destination version changed: %s", oi.VersionID)
|
||||
}
|
||||
if opts.PreserveETag != "" && oi.ETag != opts.PreserveETag {
|
||||
t.Fatalf("PreserveETag changed: %s", oi.ETag)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsConditionalPutReplicaDuplicateHTTP(t *testing.T) {
|
||||
for _, null := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("null=%t", null), func(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
bucket, router := conditionalPutBucket(t, z, "versioned")
|
||||
object := "replica-duplicate"
|
||||
vid := mustGetUUID()
|
||||
if null {
|
||||
vid = nullVersionID
|
||||
}
|
||||
addressed := putConsistencyObject(t, z, bucket, object, 1, "addressed", ObjectOptions{Versioned: true, VersionID: vid, MTime: UTCNow().Add(-time.Hour)})
|
||||
current := putConsistencyObject(t, z, bucket, object, 1, "current", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Minute)})
|
||||
defer conditionalPutPool(t, z, object, 0)()
|
||||
headers := map[string]string{
|
||||
xhttp.MinIOSourceReplicationRequest: "true",
|
||||
xhttp.AmzBucketReplicationStatus: "REPLICA",
|
||||
xhttp.MinIOSourceETag: addressed.ETag,
|
||||
xhttp.MinIOSourceMTime: addressed.ModTime.Format(time.RFC3339Nano),
|
||||
}
|
||||
url := getPutObjectURL("", bucket, object)
|
||||
put := multipartConditionRequest(t, router, http.MethodPut, url+"?versionId="+vid, "addressed", headers)
|
||||
if put.Code != http.StatusPreconditionFailed {
|
||||
t.Fatalf("replica duplicate: %d %s", put.Code, put.Body.String())
|
||||
}
|
||||
multipartConditionError(t, put, "PreconditionFailed")
|
||||
get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil)
|
||||
if get.Code != http.StatusOK || get.Body.String() != "current" || multipartConditionResponseETag(get) != current.ETag {
|
||||
t.Fatalf("duplicate changed current object: %d %q", get.Code, get.Body.String())
|
||||
}
|
||||
get = multipartConditionRequest(t, router, http.MethodGet, url+"?versionId="+vid, "", nil)
|
||||
if get.Code != http.StatusOK || get.Body.String() != "addressed" || multipartConditionResponseETag(get) != addressed.ETag {
|
||||
t.Fatalf("duplicate changed addressed version: %d %q", get.Code, get.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsConditionalPutConcurrentHTTP(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
bucket, router := conditionalPutBucket(t, z, "unversioned")
|
||||
for _, match := range []bool{false, true} {
|
||||
for iteration := range 5 {
|
||||
t.Run(fmt.Sprintf("match=%t/iteration=%d", match, iteration), func(t *testing.T) {
|
||||
object := fmt.Sprintf("concurrent-%t-%d", match, iteration)
|
||||
headers := map[string]string{xhttp.IfNoneMatch: "*"}
|
||||
if match {
|
||||
oi := putConsistencyObject(t, z, bucket, object, 1, "old", ObjectOptions{MTime: UTCNow().Add(-time.Minute)})
|
||||
headers = map[string]string{xhttp.IfMatch: fmt.Sprintf("%q", oi.ETag)}
|
||||
}
|
||||
defer conditionalPutPool(t, z, object, 0)()
|
||||
start := make(chan struct{})
|
||||
results := make(chan *httptest.ResponseRecorder, 2)
|
||||
url := getPutObjectURL("", bucket, object)
|
||||
for i := range 2 {
|
||||
body := fmt.Sprintf("writer-%d", i)
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, url, int64(len(body)), strings.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, headers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go func() {
|
||||
<-start
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
results <- rec
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
success, failed, winnerETag := 0, 0, ""
|
||||
for range 2 {
|
||||
result := <-results
|
||||
switch result.Code {
|
||||
case http.StatusOK:
|
||||
success++
|
||||
winnerETag = multipartConditionResponseETag(result)
|
||||
case http.StatusPreconditionFailed:
|
||||
failed++
|
||||
multipartConditionError(t, result, "PreconditionFailed")
|
||||
default:
|
||||
t.Errorf("unexpected PUT %d: %s", result.Code, result.Body.String())
|
||||
}
|
||||
}
|
||||
if success != 1 || failed != 1 {
|
||||
t.Fatalf("success=%d precondition failures=%d", success, failed)
|
||||
}
|
||||
get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil)
|
||||
if get.Code != http.StatusOK || multipartConditionResponseETag(get) != winnerETag || fmt.Sprintf("%x", md5.Sum(get.Body.Bytes())) != winnerETag {
|
||||
t.Fatalf("winner lost: %d %q %v", get.Code, get.Body.String(), get.Header())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsConditionalPutSerializesMutation(t *testing.T) {
|
||||
for _, deletion := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("delete=%t", deletion), func(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
object := "conditional-mutation"
|
||||
current := putConsistencyObject(t, z, bucket, object, 1, "current", ObjectOptions{MTime: UTCNow().Add(-time.Minute)})
|
||||
defer conditionalPutPool(t, z, object, 0)()
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
gate := &consistencyGateReader{Reader: strings.NewReader("replacement"), entered: make(chan struct{}), resume: make(chan struct{})}
|
||||
release := func() { gate.release.Do(func() { close(gate.resume) }) }
|
||||
defer release()
|
||||
reader := mustGetPutObjReader(t, gate, 11, "", "")
|
||||
written := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := z.PutObject(ctx, bucket, object, reader, ObjectOptions{HasIfMatch: true, CheckPrecondFn: func(oi ObjectInfo) bool { return oi.ETag != current.ETag }})
|
||||
written <- err
|
||||
}()
|
||||
select {
|
||||
case <-gate.entered:
|
||||
case err := <-written:
|
||||
t.Fatalf("PUT failed before body read: %v", err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
mutated := make(chan error, 1)
|
||||
go func() {
|
||||
var err error
|
||||
if deletion {
|
||||
_, err = z.DeleteObject(ctx, bucket, object, ObjectOptions{})
|
||||
} else {
|
||||
_, err = z.PutObjectMetadata(ctx, bucket, object, ObjectOptions{EvalMetadataFn: func(oi *ObjectInfo, _ error) (ReplicateDecision, error) {
|
||||
oi.UserDefined["x-amz-meta-after-put"] = "present"
|
||||
return ReplicateDecision{}, nil
|
||||
}})
|
||||
}
|
||||
mutated <- err
|
||||
}()
|
||||
select {
|
||||
case err := <-mutated:
|
||||
t.Fatalf("mutation escaped PUT lock: %v", err)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
release()
|
||||
if err := <-written; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := <-mutated; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oi, err := z.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
|
||||
if deletion {
|
||||
if !isErrObjectNotFound(err) {
|
||||
t.Fatalf("delete lost: %v", err)
|
||||
}
|
||||
} else if err != nil || oi.UserDefined["x-amz-meta-after-put"] != "present" || oi.ETag != fmt.Sprintf("%x", md5.Sum([]byte("replacement"))) {
|
||||
t.Fatalf("metadata/PUT lost: %+v %v", oi, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSinglePoolConditionalPutHTTP(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
obj, dirs, err := prepareErasure16(ctx)
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
t.Cleanup(func() { cancel(); z.Shutdown(context.Background()); removeRoots(dirs) })
|
||||
if !z.SinglePool() {
|
||||
t.Fatal("fixture is not a single pool")
|
||||
}
|
||||
bucket, router := conditionalPutBucket(t, z, "unversioned")
|
||||
object := "single-pool-condition"
|
||||
defer conditionalPutPool(t, z, object, 0)()
|
||||
url := getPutObjectURL("", bucket, object)
|
||||
for _, tc := range []struct {
|
||||
body, match, none string
|
||||
status int
|
||||
}{
|
||||
{"missing", "*", "", http.StatusNotFound},
|
||||
{"first", "", "*", http.StatusOK},
|
||||
{"blocked", "", "*", http.StatusPreconditionFailed},
|
||||
{"blocked", "stale", "", http.StatusPreconditionFailed},
|
||||
{"second", fmt.Sprintf("%x", md5.Sum([]byte("first"))), "", http.StatusOK},
|
||||
} {
|
||||
rec := multipartConditionRequest(t, router, http.MethodPut, url, tc.body, map[string]string{xhttp.IfMatch: tc.match, xhttp.IfNoneMatch: tc.none})
|
||||
if rec.Code != tc.status {
|
||||
t.Fatalf("PUT %d want %d: %s", rec.Code, tc.status, rec.Body.String())
|
||||
}
|
||||
}
|
||||
get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil)
|
||||
if get.Code != http.StatusOK || get.Body.String() != "second" {
|
||||
t.Fatalf("single-pool GET: %d %q", get.Code, get.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// An internal replica callback without an addressed version is not a public
|
||||
// condition. Preserve its availability when another pool is unreadable. An
|
||||
// addressed replica already requires all pools for lock/tag reconciliation.
|
||||
func TestPoolsConditionalPutReplicaAvailability(t *testing.T) {
|
||||
for _, addressed := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("addressed=%t", addressed), func(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
mode := "unversioned"
|
||||
if addressed {
|
||||
mode = "versioned"
|
||||
}
|
||||
bucket, router := conditionalPutBucket(t, z, mode)
|
||||
object := "replica-availability"
|
||||
oi := putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{Versioned: addressed, MTime: UTCNow().Add(-time.Minute)})
|
||||
defer conditionalPutPool(t, z, object, 0)()
|
||||
restoreFault := conditionalPutSwapDisks(z.serverPools[1], object, func(disk StorageAPI) StorageAPI {
|
||||
return consistencyReadFaultDisk{StorageAPI: disk, bucket: bucket, object: object}
|
||||
})
|
||||
defer restoreFault()
|
||||
headers := map[string]string{
|
||||
xhttp.MinIOSourceReplicationRequest: "true",
|
||||
xhttp.AmzBucketReplicationStatus: "REPLICA",
|
||||
xhttp.MinIOSourceETag: fmt.Sprintf("%x", md5.Sum([]byte("replacement"))),
|
||||
}
|
||||
url := getPutObjectURL("", bucket, object)
|
||||
wantStatus, wantBody := http.StatusOK, "replacement"
|
||||
if addressed {
|
||||
url += "?versionId=" + oi.VersionID
|
||||
wantStatus, wantBody = http.StatusServiceUnavailable, "old"
|
||||
}
|
||||
put := multipartConditionRequest(t, router, http.MethodPut, url, "replacement", headers)
|
||||
if put.Code != wantStatus {
|
||||
t.Fatalf("replica PUT: %d want %d: %s", put.Code, wantStatus, put.Body.String())
|
||||
}
|
||||
restoreFault()
|
||||
get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil)
|
||||
if get.Code != http.StatusOK || get.Body.String() != wantBody {
|
||||
t.Fatalf("replica GET: %d %q", get.Code, get.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A public client cannot address a destination version, with or without a
|
||||
// precondition; only trusted replication may (SN-2026-016).
|
||||
func TestPoolsConditionalPutDestinationVersionHTTP(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
bucket, router := conditionalPutBucket(t, z, "versioned")
|
||||
object := "client-version"
|
||||
old := putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
|
||||
current := putConsistencyObject(t, z, bucket, object, 1, "current", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Minute)})
|
||||
defer conditionalPutPool(t, z, object, 0)()
|
||||
url := getPutObjectURL("", bucket, object) + "?versionId=" + old.VersionID
|
||||
for _, tag := range []string{old.ETag, current.ETag} {
|
||||
put := multipartConditionRequest(t, router, http.MethodPut, url, "replacement", map[string]string{xhttp.IfMatch: fmt.Sprintf("%q", tag)})
|
||||
if put.Code != http.StatusForbidden {
|
||||
t.Fatalf("version-addressed public PUT: %d want 403: %s", put.Code, put.Body.String())
|
||||
}
|
||||
}
|
||||
get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil)
|
||||
if get.Code != http.StatusOK || get.Body.String() != "old" {
|
||||
t.Fatalf("addressed GET: %d %q", get.Code, get.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsConditionalPutDeleteMarkerTie(t *testing.T) {
|
||||
for markerPool := range 2 {
|
||||
t.Run(fmt.Sprintf("marker-pool=%d", markerPool), func(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
bucket, router := conditionalPutBucket(t, z, "versioned")
|
||||
object := "marker-tie"
|
||||
mtime := UTCNow().Add(-time.Minute)
|
||||
putConsistencyObject(t, z, bucket, object, 1-markerPool, "live", ObjectOptions{Versioned: true, MTime: mtime})
|
||||
if _, err := z.serverPools[markerPool].DeleteObject(t.Context(), bucket, object, ObjectOptions{Versioned: true, VersionID: mustGetUUID(), DeleteMarker: true, MTime: mtime}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conditionalPutPool(t, z, object, 0)()
|
||||
url := getPutObjectURL("", bucket, object)
|
||||
before := multipartConditionRequest(t, router, http.MethodGet, url, "", nil)
|
||||
want := http.StatusPreconditionFailed
|
||||
if markerPool == 0 {
|
||||
want = http.StatusOK
|
||||
if before.Code != http.StatusNotFound {
|
||||
t.Fatalf("GET tie: %d", before.Code)
|
||||
}
|
||||
} else if before.Code != http.StatusOK {
|
||||
t.Fatalf("GET tie: %d", before.Code)
|
||||
}
|
||||
put := multipartConditionRequest(t, router, http.MethodPut, url, "replacement", map[string]string{xhttp.IfNoneMatch: "*"})
|
||||
if put.Code != want {
|
||||
t.Fatalf("PUT tie: %d want %d: %s", put.Code, want, put.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Overlap fixture changes with the real IAM Walk reader instead of relying on
|
||||
// its periodic refresh timer to expose an unsynchronized disk-adapter swap.
|
||||
func TestPoolsConditionalPutFixtureConcurrentIAM(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
conditionalPutBucket(t, z, "unversioned")
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
started, finished := make(chan struct{}), make(chan error, 1)
|
||||
iam := globalIAMSys
|
||||
go func() {
|
||||
close(started)
|
||||
for range 20 {
|
||||
if err := iam.Load(ctx, false); err != nil {
|
||||
finished <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
finished <- nil
|
||||
}()
|
||||
<-started
|
||||
for i := range 5000 {
|
||||
restore := conditionalPutPool(t, z, "fixture-concurrent-iam", i%2)
|
||||
restore()
|
||||
}
|
||||
if err := <-finished; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -865,7 +865,7 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, poolIdx int, b
|
||||
if oi.isMultipart() {
|
||||
res, err := z.NewMultipartUpload(ctx, bucket, oi.Name, ObjectOptions{
|
||||
VersionID: oi.VersionID,
|
||||
UserDefined: oi.UserDefined,
|
||||
UserDefined: migrationObjectMetadata(oi),
|
||||
NoAuditLog: true,
|
||||
DataMovement: true,
|
||||
SrcPoolIdx: poolIdx,
|
||||
@@ -924,7 +924,7 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, poolIdx int, b
|
||||
DataMovement: true,
|
||||
VersionID: oi.VersionID,
|
||||
MTime: oi.ModTime,
|
||||
UserDefined: oi.UserDefined,
|
||||
UserDefined: migrationObjectMetadata(oi),
|
||||
PreserveETag: oi.ETag, // Preserve original ETag to ensure same metadata.
|
||||
IndexCB: func() []byte {
|
||||
return oi.Parts[0].Index // Preserve part Index to ensure decompression works.
|
||||
|
||||
+104
-54
@@ -1149,6 +1149,40 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
|
||||
}
|
||||
opts.NoLock = true
|
||||
|
||||
// Public write conditions compare the logical current object while the
|
||||
// pools-layer write lock is held. The destination selected by capacity may
|
||||
// be empty or stale, and draining pools can still hold the current object.
|
||||
// Replica callbacks retain their existing addressed-version semantics and
|
||||
// metadata reconciliation at the set layer.
|
||||
if opts.CheckPrecondFn != nil && !opts.ReplicationRequest &&
|
||||
!opts.ReplicaLockReconcile && !opts.DataMovement {
|
||||
copies, lerr := z.objectPoolInfos(ctx, bucket, object, ObjectOptions{
|
||||
VersionID: "", // Compare the current object, not the write's version.
|
||||
Versioned: opts.Versioned,
|
||||
VersionSuspended: opts.VersionSuspended,
|
||||
NoAuditLog: true,
|
||||
})
|
||||
var latest ObjectInfo
|
||||
if lerr == nil {
|
||||
latest = copies[0].ObjInfo
|
||||
if latest.DeleteMarker {
|
||||
lerr = toObjectErr(errFileNotFound, bucket, object)
|
||||
}
|
||||
}
|
||||
// An unreadable pool may hold the newest object; it is not absence.
|
||||
if lerr != nil && !isErrObjectNotFound(lerr) && !isErrVersionNotFound(lerr) {
|
||||
return ObjectInfo{}, lerr
|
||||
}
|
||||
if lerr == nil && opts.CheckPrecondFn(latest) {
|
||||
return ObjectInfo{}, PreConditionFailed{}
|
||||
}
|
||||
if lerr != nil && opts.HasIfMatch {
|
||||
return ObjectInfo{}, lerr
|
||||
}
|
||||
// Do not repeat an accepted condition against the destination's copy.
|
||||
opts.CheckPrecondFn = nil
|
||||
}
|
||||
|
||||
idx, err := z.getWritePoolIdx(ctx, bucket, object, data.Size(), true)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
@@ -1165,6 +1199,10 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
|
||||
|
||||
if opts.ReplicaLockReconcile || opts.DataMovement {
|
||||
opts.ReplicaLockReconcile = true
|
||||
}
|
||||
// Public overwrites need all pools for Object Lock. Unaddressed replica
|
||||
// callbacks retain their set-local reads unless reconciliation is needed.
|
||||
if opts.ReplicaLockReconcile || !opts.ReplicationRequest {
|
||||
opts.replicaObjectInfo = z.replicaObjectInfo
|
||||
}
|
||||
oi, err := z.serverPools[idx].PutObject(ctx, bucket, object, data, opts)
|
||||
@@ -1205,9 +1243,12 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
||||
return ObjectInfo{}, z.deletePrefix(ctx, bucket, object)
|
||||
}
|
||||
|
||||
// Reconcile ordinary addressed-version deletes independently of pool movement.
|
||||
// Resolve a physical purge by its addressed version even on a replica
|
||||
// receiver: latest-key routing can select a different pool or miss copies.
|
||||
// Marker creation and specialized movement/scanner operations retain their
|
||||
// existing routing.
|
||||
reconcileVersion := opts.VersionID != "" && !opts.DataMovement &&
|
||||
!opts.ReplicationRequest && !opts.Expiration.Expire && !opts.InclFreeVersions
|
||||
(!opts.ReplicationRequest || opts.isVersionPurge()) && !opts.Expiration.Expire && !opts.InclFreeVersions
|
||||
if !z.SinglePool() && (opts.CheckPrecondFn != nil || reconcileVersion) {
|
||||
return z.deleteObjectReconciled(ctx, bucket, object, opts)
|
||||
}
|
||||
@@ -1220,6 +1261,13 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
||||
if _, ok := err.(InsufficientReadQuorum); ok {
|
||||
return objInfo, InsufficientWriteQuorum{}
|
||||
}
|
||||
// Lookup can return before the set's purge confirmation. Check here,
|
||||
// before any callback can change the request into a metadata update.
|
||||
if opts.isVersionPurge() && (isErrObjectNotFound(err) || isErrVersionNotFound(err)) {
|
||||
if quorumErr := z.checkPurgeAbsent(ctx, bucket, object, opts.VersionID); quorumErr != nil {
|
||||
return objInfo, quorumErr
|
||||
}
|
||||
}
|
||||
// A conditional (If-Match) delete addressing a specific version treats an
|
||||
// absent key as an absent version. getPoolInfoExistingWithOpts strips
|
||||
// VersionID, so a missing key surfaces ObjectNotFound here even for a
|
||||
@@ -1250,6 +1298,11 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
||||
if verr != nil && (!isErrMethodNotAllowed(verr) || !vi.DeleteMarker) {
|
||||
// Genuine read failure for the addressed version: a missing
|
||||
// version -> VersionNotFound (NoSuchVersion), read-quorum loss, etc.
|
||||
if opts.isVersionPurge() && (isErrObjectNotFound(verr) || isErrVersionNotFound(verr)) {
|
||||
if quorumErr := z.checkPurgeAbsent(ctx, bucket, object, opts.VersionID); quorumErr != nil {
|
||||
return objInfo, quorumErr
|
||||
}
|
||||
}
|
||||
return objInfo, verr
|
||||
}
|
||||
// verr is nil for a live version, or MethodNotAllowed with a populated
|
||||
@@ -1461,9 +1514,9 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
|
||||
return objInfo, err
|
||||
}
|
||||
|
||||
if !z.SinglePool() && dstOpts.ReplicaLockReconcile {
|
||||
if !z.SinglePool() && (dstOpts.ReplicaLockReconcile || !dstOpts.ReplicationRequest) {
|
||||
dstOpts.replicaObjectInfo = z.replicaObjectInfo
|
||||
if !dstOpts.DataMovement {
|
||||
if dstOpts.ReplicaLockReconcile && !dstOpts.DataMovement {
|
||||
defer func() {
|
||||
if err == nil {
|
||||
err = z.retireReplicaCopies(ctx, dstBucket, dstObject, poolIdx, objInfo)
|
||||
@@ -1500,13 +1553,16 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
|
||||
ServerSideEncryption: dstOpts.ServerSideEncryption,
|
||||
UserDefined: srcInfo.UserDefined,
|
||||
Versioned: dstOpts.Versioned,
|
||||
VersionSuspended: dstOpts.VersionSuspended,
|
||||
VersionID: dstOpts.VersionID,
|
||||
MTime: dstOpts.MTime,
|
||||
NoLock: true,
|
||||
EncryptFn: dstOpts.EncryptFn,
|
||||
WantChecksum: dstOpts.WantChecksum,
|
||||
WantServerSideChecksumType: dstOpts.WantServerSideChecksumType,
|
||||
ReplicationRequest: dstOpts.ReplicationRequest,
|
||||
ReplicaLockReconcile: dstOpts.ReplicaLockReconcile,
|
||||
DataMovement: dstOpts.DataMovement,
|
||||
replicaObjectInfo: dstOpts.replicaObjectInfo,
|
||||
}
|
||||
|
||||
@@ -1857,48 +1913,30 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p
|
||||
if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil {
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
|
||||
poolResult := ListMultipartsInfo{}
|
||||
poolResult.MaxUploads = maxUploads
|
||||
poolResult.KeyMarker = keyMarker
|
||||
poolResult.Prefix = prefix
|
||||
poolResult.Delimiter = delimiter
|
||||
|
||||
// if no prefix provided, return the list from cache
|
||||
if prefix == "" {
|
||||
if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
return ListMultipartsInfo{}, toObjectErr(err, bucket)
|
||||
}
|
||||
|
||||
z.mpCache.Range(func(_ string, mp MultipartInfo) bool {
|
||||
if mp.Bucket == bucket {
|
||||
poolResult.Uploads = append(poolResult.Uploads, mp)
|
||||
}
|
||||
return true
|
||||
})
|
||||
sort.Slice(poolResult.Uploads, func(i int, j int) bool {
|
||||
return poolResult.Uploads[i].Initiated.Before(poolResult.Uploads[j].Initiated)
|
||||
})
|
||||
return poolResult, nil
|
||||
if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
return ListMultipartsInfo{}, toObjectErr(err, bucket)
|
||||
}
|
||||
|
||||
if z.SinglePool() {
|
||||
return z.serverPools[0].ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
|
||||
scan, err := startMultipartScan(ctx, false)
|
||||
if err != nil {
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
defer scan.close()
|
||||
|
||||
var uploads []MultipartInfo
|
||||
for idx, pool := range z.serverPools {
|
||||
if z.IsSuspended(idx) {
|
||||
continue
|
||||
}
|
||||
result, err := pool.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker,
|
||||
delimiter, maxUploads)
|
||||
poolUploads, err := pool.scanMultipartUploads(scan, bucket, idx)
|
||||
if err != nil {
|
||||
return result, err
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
poolResult.Uploads = append(poolResult.Uploads, result.Uploads...)
|
||||
uploads = append(uploads, poolUploads...)
|
||||
}
|
||||
|
||||
return poolResult, nil
|
||||
// Confirmed historical records lack a recoverable bucket/key and are
|
||||
// omitted by the scan. They do not block listing identifiable uploads.
|
||||
return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
|
||||
}
|
||||
|
||||
// Initiate a new multipart upload on a hashedSet based on object name.
|
||||
@@ -1927,7 +1965,7 @@ func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, obj
|
||||
continue
|
||||
}
|
||||
|
||||
result, err := pool.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
|
||||
result, err := pool.listMultipartUploadsExact(ctx, bucket, object)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -2089,13 +2127,18 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
|
||||
if err := checkAbortMultipartArgs(ctx, bucket, object, uploadID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
return toObjectErr(err, bucket)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
// Unlock cancels the derived lock context before this notification runs.
|
||||
// Keep the request context so successful cancellation reaches peer caches.
|
||||
defer func(ctx context.Context) {
|
||||
if err == nil {
|
||||
z.mpCache.Delete(uploadID)
|
||||
globalNotificationSys.DeleteUploadID(ctx, uploadID)
|
||||
}
|
||||
}()
|
||||
}(ctx)
|
||||
|
||||
lk := z.NewNSLock(bucket, pathJoin(object, uploadID))
|
||||
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||
@@ -2105,23 +2148,15 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
|
||||
if z.SinglePool() {
|
||||
return z.serverPools[0].AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
|
||||
}
|
||||
|
||||
for idx, pool := range z.serverPools {
|
||||
if z.IsSuspended(idx) {
|
||||
continue
|
||||
}
|
||||
err := pool.AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
|
||||
if err == nil {
|
||||
return nil
|
||||
found, err := pool.getHashedSet(object).abortMultipartUpload(ctx, bucket, object, uploadID, opts)
|
||||
if found || err != nil {
|
||||
// Preserve the default first-matching-pool behavior.
|
||||
return err
|
||||
}
|
||||
if _, ok := err.(InvalidUploadID); ok {
|
||||
// upload id not found move to next pool
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
return InvalidUploadID{
|
||||
Bucket: bucket,
|
||||
@@ -2213,6 +2248,8 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
|
||||
|
||||
if opts.ReplicaLockReconcile || opts.DataMovement {
|
||||
opts.ReplicaLockReconcile = true
|
||||
}
|
||||
if opts.ReplicaLockReconcile || !opts.ReplicationRequest {
|
||||
opts.replicaObjectInfo = z.replicaObjectInfo
|
||||
}
|
||||
|
||||
@@ -2550,9 +2587,9 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
|
||||
minDisks: listingQuorum,
|
||||
reportNotFound: false,
|
||||
agreed: send,
|
||||
partial: func(entries metaCacheEntries, _ []error) {
|
||||
entry, ok := entries.resolve(&resolver)
|
||||
if ok {
|
||||
partial: func(entries metaCacheEntries, errs []error) {
|
||||
entry := set.resolveListEntry(entries, errs, &resolver)
|
||||
if entry != nil {
|
||||
send(*entry)
|
||||
}
|
||||
},
|
||||
@@ -2654,7 +2691,11 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
|
||||
// Merge all entries from all disks.
|
||||
// We leave quorum at 1, since entries are already resolved to have the desired quorum.
|
||||
// mergeEntryChannels will close 'merged' channel upon completion or cancellation.
|
||||
errCh <- mergeEntryChannels(ctx, entries, merged, 1)
|
||||
errCh <- mergeEntryChannelsWithResolver(ctx, entries, merged, 1, func(int) error {
|
||||
return context.Cause(ctx)
|
||||
}, func(group []*metaCacheEntry) (*metaCacheEntry, error) {
|
||||
return z.resolveListingGroup(ctx, bucket, group)
|
||||
})
|
||||
}()
|
||||
|
||||
return nil
|
||||
@@ -3051,6 +3092,15 @@ func (z *erasureServerPools) PutObjectTags(ctx context.Context, bucket, object s
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
// Ordinary reads and replication can return any owning pool. Persist one
|
||||
// revision beyond all copies, so the returned value and every pool agree.
|
||||
if stamp := opts.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]; stamp != "" {
|
||||
for _, copy := range copies {
|
||||
stamp = monotonicTaggingTimestamp(stamp, copy.ObjInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp])
|
||||
}
|
||||
opts.UserDefined = cloneMSS(opts.UserDefined)
|
||||
opts.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = stamp
|
||||
}
|
||||
opts.NoLock = true
|
||||
opts.VersionID = copies[0].ObjInfo.VersionID
|
||||
if opts.VersionID == "" {
|
||||
|
||||
+35
-4
@@ -869,21 +869,52 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
|
||||
ServerSideEncryption: dstOpts.ServerSideEncryption,
|
||||
UserDefined: srcInfo.UserDefined,
|
||||
Versioned: dstOpts.Versioned,
|
||||
VersionSuspended: dstOpts.VersionSuspended,
|
||||
VersionID: dstOpts.VersionID,
|
||||
MTime: dstOpts.MTime,
|
||||
NoLock: dstOpts.NoLock,
|
||||
EncryptFn: dstOpts.EncryptFn,
|
||||
WantChecksum: dstOpts.WantChecksum,
|
||||
WantServerSideChecksumType: dstOpts.WantServerSideChecksumType,
|
||||
ReplicationRequest: dstOpts.ReplicationRequest,
|
||||
ReplicaLockReconcile: dstOpts.ReplicaLockReconcile,
|
||||
DataMovement: dstOpts.DataMovement,
|
||||
replicaObjectInfo: dstOpts.replicaObjectInfo,
|
||||
}
|
||||
|
||||
return dstSet.putObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
|
||||
}
|
||||
|
||||
func (s *erasureSets) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
|
||||
// In list multipart uploads we are going to treat input prefix as the object,
|
||||
// this means that we are not supporting directory navigation.
|
||||
set := s.getHashedSet(prefix)
|
||||
return set.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
|
||||
if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil {
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
scan, err := startMultipartScan(ctx, false)
|
||||
if err != nil {
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
defer scan.close()
|
||||
uploads, err := s.scanMultipartUploads(scan, bucket, 0)
|
||||
if err != nil {
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
|
||||
}
|
||||
|
||||
func (s *erasureSets) scanMultipartUploads(scan *multipartScan, bucket string, poolIdx int) ([]MultipartInfo, error) {
|
||||
var uploads []MultipartInfo
|
||||
for i, set := range s.sets {
|
||||
setUploads, err := set.scanMultipartUploads(scan, bucket, poolIdx, i)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
uploads = append(uploads, setUploads...)
|
||||
}
|
||||
return uploads, nil
|
||||
}
|
||||
|
||||
func (s *erasureSets) listMultipartUploadsExact(ctx context.Context, bucket, object string) (ListMultipartsInfo, error) {
|
||||
return s.getHashedSet(object).listMultipartUploadsExact(ctx, bucket, object, "", "", "", maxUploadsList)
|
||||
}
|
||||
|
||||
// Initiate a new multipart upload on a hashedSet based on object name.
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
)
|
||||
|
||||
// isVersionPurge distinguishes physical removal from marker creation and
|
||||
// replication-state updates. Call it again after metadata callbacks, which
|
||||
// can change an ordinary deletion into a pending purge update.
|
||||
func (o ObjectOptions) isVersionPurge() bool {
|
||||
if o.VersionID == "" || o.VersionID == nullVersionID || o.DeleteMarker ||
|
||||
o.DeletePrefix || o.DataMovement || o.InclFreeVersions || o.Expiration.Expire ||
|
||||
o.Transition != (TransitionOptions{}) {
|
||||
return false
|
||||
}
|
||||
id, err := uuid.Parse(o.VersionID)
|
||||
if err != nil || id == uuid.Nil {
|
||||
return false
|
||||
}
|
||||
purge := o.VersionPurgeStatus()
|
||||
if purge == replication.VersionPurgeComplete {
|
||||
return true
|
||||
}
|
||||
if !purge.Empty() || o.DeleteReplication.VersionPurgeStatusInternal != "" {
|
||||
return false
|
||||
}
|
||||
status := o.DeleteMarkerReplicationStatus()
|
||||
return (status.Empty() && o.DeleteReplication.ReplicationStatusInternal == "") ||
|
||||
(status == replication.Replica && o.ReplicationRequest)
|
||||
}
|
||||
|
||||
// confirmVersionAbsent is a read-only proof for an already-missing purge.
|
||||
// Read quorum (or a synthesized NotFound) cannot acknowledge a write. Do not
|
||||
// delete here: unreadable minority copies may carry retention we cannot check.
|
||||
// The caller holds the object lock. This function never heals or enqueues work.
|
||||
func (er erasureObjects) confirmVersionAbsent(ctx context.Context, bucket, object, versionID string) (allAbsent bool, err error) {
|
||||
disks := er.getDisks()
|
||||
_, errs := readAllFileInfo(ctx, disks, "", bucket, object, versionID, false, false)
|
||||
absent := 0
|
||||
for _, err := range errs {
|
||||
if err == errFileNotFound || err == errFileVersionNotFound {
|
||||
absent++
|
||||
}
|
||||
}
|
||||
if absent < len(disks)/2+1 {
|
||||
return false, InsufficientWriteQuorum{}
|
||||
}
|
||||
return absent == len(disks), nil
|
||||
}
|
||||
|
||||
// checkPurgeAbsent schedules recovery explicitly, outside the read-only proof.
|
||||
func (er erasureObjects) checkPurgeAbsent(ctx context.Context, bucket, object, versionID string) error {
|
||||
allAbsent, err := er.confirmVersionAbsent(ctx, bucket, object, versionID)
|
||||
if !allAbsent {
|
||||
er.addPartial(bucket, object, versionID)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (z *erasureServerPools) checkPurgeAbsent(ctx context.Context, bucket, object, versionID string) error {
|
||||
for _, pool := range z.serverPools {
|
||||
if err := pool.getHashedSet(object).checkPurgeAbsent(ctx, bucket, object, versionID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -319,6 +319,8 @@ var (
|
||||
|
||||
globalOperationTimeout = newDynamicTimeout(10*time.Minute, 5*time.Minute) // default timeout for general ops
|
||||
globalDeleteOperationTimeout = newDynamicTimeout(5*time.Minute, 1*time.Minute) // default time for delete ops
|
||||
// Listing recovery has its own failure history and a five-second caller budget.
|
||||
globalListingOperationTimeout = newDynamicTimeout(5*time.Second, 5*time.Second)
|
||||
|
||||
globalBucketObjectLockSys *BucketObjectLockSys
|
||||
globalBucketQuotaSys *BucketQuotaSys
|
||||
|
||||
+33
-23
@@ -246,16 +246,6 @@ func extractMetadata(ctx context.Context, mimesHeader ...textproto.MIMEHeader) (
|
||||
|
||||
// extractMetadata extracts metadata from map values.
|
||||
func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error {
|
||||
return extractMetadataFromMimeWithReplication(ctx, v, m, false)
|
||||
}
|
||||
|
||||
// extractReplicationMetadataFromMime restores replication-only metadata after the
|
||||
// caller has validated that the request is a trusted replication write.
|
||||
func extractReplicationMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error {
|
||||
return extractMetadataFromMimeWithReplication(ctx, v, m, true)
|
||||
}
|
||||
|
||||
func extractMetadataFromMimeWithReplication(ctx context.Context, v textproto.MIMEHeader, m map[string]string, allowReplication bool) error {
|
||||
if v == nil {
|
||||
bugLogIf(ctx, errInvalidArgument)
|
||||
return errInvalidArgument
|
||||
@@ -267,18 +257,14 @@ func extractMetadataFromMimeWithReplication(ctx context.Context, v textproto.MIM
|
||||
nv[http.CanonicalHeaderKey(k)] = kv
|
||||
}
|
||||
|
||||
// Save all supported headers.
|
||||
// Save ordinary object metadata. Replication-only headers are restored only
|
||||
// after the request has been validated as a trusted replication write.
|
||||
for _, supportedHeader := range supportedHeaders {
|
||||
value, ok := nv[http.CanonicalHeaderKey(supportedHeader)]
|
||||
if ok {
|
||||
if v, ok := replicationToInternalHeaders[supportedHeader]; ok {
|
||||
if !allowReplication {
|
||||
continue
|
||||
}
|
||||
m[v] = strings.Join(value, ",")
|
||||
} else {
|
||||
m[supportedHeader] = strings.Join(value, ",")
|
||||
}
|
||||
if _, ok := replicationToInternalHeaders[supportedHeader]; ok {
|
||||
continue
|
||||
}
|
||||
if value, ok := nv[http.CanonicalHeaderKey(supportedHeader)]; ok {
|
||||
m[supportedHeader] = strings.Join(value, ",")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -287,8 +273,7 @@ func extractMetadataFromMimeWithReplication(ctx context.Context, v textproto.MIM
|
||||
if !stringsHasPrefixFold(key, prefix) {
|
||||
continue
|
||||
}
|
||||
value, ok := nv[http.CanonicalHeaderKey(key)]
|
||||
if ok {
|
||||
if value, ok := nv[http.CanonicalHeaderKey(key)]; ok {
|
||||
m[key] = strings.Join(value, ",")
|
||||
break
|
||||
}
|
||||
@@ -297,6 +282,31 @@ func extractMetadataFromMimeWithReplication(ctx context.Context, v textproto.MIM
|
||||
return nil
|
||||
}
|
||||
|
||||
// extractReplicationMetadataFromMime restores replication-only metadata after the
|
||||
// caller has validated that the request is a trusted replication write.
|
||||
func extractReplicationMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error {
|
||||
if v == nil {
|
||||
bugLogIf(ctx, errInvalidArgument)
|
||||
return errInvalidArgument
|
||||
}
|
||||
|
||||
nv := make(textproto.MIMEHeader, len(v))
|
||||
for k, kv := range v {
|
||||
// Canonicalize all headers, to remove any duplicates.
|
||||
nv[http.CanonicalHeaderKey(k)] = kv
|
||||
}
|
||||
|
||||
// Ordinary object metadata belongs to the caller. Re-extracting it would
|
||||
// undo normalization (such as removing aws-chunked) or copy an outer
|
||||
// Snowball archive's metadata onto its individual entries.
|
||||
for header, internalHeader := range replicationToInternalHeaders {
|
||||
if value, ok := nv[http.CanonicalHeaderKey(header)]; ok {
|
||||
m[internalHeader] = strings.Join(value, ",")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Returns access credentials in the request Authorization header.
|
||||
func getReqAccessCred(r *http.Request, region string) (cred auth.Credentials) {
|
||||
cred, _, _ = getReqAccessKeyV4(r, region, serviceS3)
|
||||
|
||||
@@ -254,6 +254,9 @@ func TestExtractMetadataFromRequestKeepsQueryCompatibility(t *testing.T) {
|
||||
|
||||
func TestExtractReplicationMetadataHeaders(t *testing.T) {
|
||||
header := http.Header{
|
||||
"Content-Type": []string{"application/wasm"},
|
||||
"Content-Encoding": []string{"aws-chunked"},
|
||||
"X-Amz-Meta-Source": []string{"client"},
|
||||
"X-Minio-Replication-Server-Side-Encryption-Sealed-Key": []string{"sealed-key"},
|
||||
"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm": []string{"DAREv2-HMAC-SHA256"},
|
||||
"X-Minio-Replication-Server-Side-Encryption-Iv": []string{"iv"},
|
||||
@@ -262,12 +265,17 @@ func TestExtractReplicationMetadataHeaders(t *testing.T) {
|
||||
ReplicationSsecChecksumHeader: []string{"checksum"},
|
||||
}
|
||||
|
||||
metadata := make(map[string]string)
|
||||
metadata := map[string]string{
|
||||
"content-type": "application/wasm",
|
||||
"x-amz-meta-source": "client",
|
||||
}
|
||||
if err := extractReplicationMetadataFromMime(t.Context(), textproto.MIMEHeader(header), metadata); err != nil {
|
||||
t.Fatalf("failed to extract replication metadata: %v", err)
|
||||
}
|
||||
|
||||
expected := map[string]string{
|
||||
"content-type": "application/wasm",
|
||||
"x-amz-meta-source": "client",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "sealed-key",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Iv": "iv",
|
||||
@@ -279,6 +287,9 @@ func TestExtractReplicationMetadataHeaders(t *testing.T) {
|
||||
if !reflect.DeepEqual(metadata, expected) {
|
||||
t.Fatalf("unexpected replication metadata: expected %#v, got %#v", expected, metadata)
|
||||
}
|
||||
if _, ok := metadata["content-encoding"]; ok {
|
||||
t.Fatalf("replication metadata restored transport content-encoding: %#v", metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetCopyObjectMetadataFromHeaderReplication(t *testing.T) {
|
||||
|
||||
@@ -24,6 +24,8 @@ func TestIAMPeerDeleteNotificationReloadsCommittedState(t *testing.T) {
|
||||
resetTestGlobals()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
// Both caches use object storage, independent of earlier etcd suites.
|
||||
globalEtcdClient = nil
|
||||
obj, disk, err := prepareFS(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -0,0 +1,300 @@
|
||||
// Copyright (c) 2026 mr javad seydi
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func multipartUploadKeys(uploads []MultipartInfo) []string {
|
||||
keys := make([]string, len(uploads))
|
||||
for i := range uploads {
|
||||
keys[i] = uploads[i].Object
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func requireMultipartUploadKeys(t *testing.T, got ListMultipartsInfo, want ...string) {
|
||||
t.Helper()
|
||||
if keys := multipartUploadKeys(got.Uploads); !slices.Equal(keys, want) {
|
||||
t.Fatalf("uploads = %v, want %v", keys, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListMultipartUploadsS3Compatibility(t *testing.T) {
|
||||
obj, dirs, err := prepareErasureSets32(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
|
||||
t.Cleanup(func() {
|
||||
z.Shutdown(t.Context())
|
||||
removeRoots(dirs)
|
||||
})
|
||||
|
||||
const bucket = "multipart-list-compat"
|
||||
if err = z.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
objects := []string{"t/a_b/p1", "t/a_b/p2", "t/c_d/p1", "u/x"}
|
||||
sets := z.serverPools[0]
|
||||
firstSet := sets.getHashedSetIndex(objects[0])
|
||||
if !slices.ContainsFunc(objects[1:], func(object string) bool {
|
||||
return sets.getHashedSetIndex(object) != firstSet
|
||||
}) {
|
||||
for n := 0; ; n++ {
|
||||
object := fmt.Sprintf("v/cross-set-%d", n)
|
||||
if sets.getHashedSetIndex(object) != firstSet {
|
||||
objects = append(objects, object)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
uploadIDs := make(map[string]string, len(objects))
|
||||
for _, object := range objects {
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("NewMultipartUpload(%q): %v", object, err)
|
||||
}
|
||||
uploadIDs[object] = mp.UploadID
|
||||
}
|
||||
|
||||
// Durable multipart metadata, rather than this node-local cache, must be
|
||||
// authoritative after a restart or when another node handles the request.
|
||||
z.mpCache.Range(func(uploadID string, _ MultipartInfo) bool {
|
||||
z.mpCache.Delete(uploadID)
|
||||
return true
|
||||
})
|
||||
|
||||
all, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, all, objects...)
|
||||
if all.IsTruncated || all.NextKeyMarker != "" || all.NextUploadIDMarker != "" {
|
||||
t.Fatalf("complete listing has truncation state: %+v", all)
|
||||
}
|
||||
|
||||
first, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, first, objects[0])
|
||||
if !first.IsTruncated || first.NextKeyMarker != objects[0] || first.NextUploadIDMarker != uploadIDs[objects[0]] {
|
||||
t.Fatalf("first page markers = (%q, %q, %t), want (%q, %q, true)",
|
||||
first.NextKeyMarker, first.NextUploadIDMarker, first.IsTruncated,
|
||||
objects[0], uploadIDs[objects[0]])
|
||||
}
|
||||
|
||||
rest, err := z.ListMultipartUploads(t.Context(), bucket, "", first.NextKeyMarker, first.NextUploadIDMarker, "", 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, rest, objects[1:]...)
|
||||
|
||||
afterKey, err := z.ListMultipartUploads(t.Context(), bucket, "", objects[1], "", "", 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, afterKey, objects[2:]...)
|
||||
|
||||
prefixed, err := z.ListMultipartUploads(t.Context(), bucket, "t/", "", "", "", 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, prefixed, objects[:3]...)
|
||||
|
||||
nested, err := z.ListMultipartUploads(t.Context(), bucket, "t/a_b/", "", "", "", 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, nested, objects[:2]...)
|
||||
|
||||
grouped, err := z.ListMultipartUploads(t.Context(), bucket, "t/", "", "", SlashSeparator, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, grouped)
|
||||
if want := []string{"t/a_b/", "t/c_d/"}; !slices.Equal(grouped.CommonPrefixes, want) {
|
||||
t.Fatalf("common prefixes = %v, want %v", grouped.CommonPrefixes, want)
|
||||
}
|
||||
|
||||
groupPage, err := z.ListMultipartUploads(t.Context(), bucket, "t/", "", "", SlashSeparator, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := []string{"t/a_b/"}; !slices.Equal(groupPage.CommonPrefixes, want) {
|
||||
t.Fatalf("first common-prefix page = %v, want %v", groupPage.CommonPrefixes, want)
|
||||
}
|
||||
if !groupPage.IsTruncated || groupPage.NextKeyMarker != "t/a_b/" || groupPage.NextUploadIDMarker != "" {
|
||||
t.Fatalf("common-prefix page markers = (%q, %q, %t)",
|
||||
groupPage.NextKeyMarker, groupPage.NextUploadIDMarker, groupPage.IsTruncated)
|
||||
}
|
||||
|
||||
groupRest, err := z.ListMultipartUploads(t.Context(), bucket, "t/", groupPage.NextKeyMarker, "", SlashSeparator, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := []string{"t/c_d/"}; !slices.Equal(groupRest.CommonPrefixes, want) {
|
||||
t.Fatalf("second common-prefix page = %v, want %v", groupRest.CommonPrefixes, want)
|
||||
}
|
||||
if groupRest.IsTruncated {
|
||||
t.Fatalf("last common-prefix page is truncated: %+v", groupRest)
|
||||
}
|
||||
|
||||
part, err := z.PutObjectPart(t.Context(), bucket, objects[0], uploadIDs[objects[0]], 1,
|
||||
mustGetPutObjReader(t, bytes.NewBufferString("part"), 4, "", ""), ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
completed, err := z.CompleteMultipartUpload(t.Context(), bucket, objects[0], uploadIDs[objects[0]],
|
||||
[]CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, key := range []string{multipartMetaBucket, multipartMetaObject} {
|
||||
if _, ok := completed.UserDefined[key]; ok {
|
||||
t.Errorf("completed object retained upload-only metadata %q", key)
|
||||
}
|
||||
}
|
||||
|
||||
// Simulate an upload written by a pre-upgrade server. Its key cannot be
|
||||
// recovered by scanning the hashed namespace. Omit only that record, and
|
||||
// preserve normal operations when its bucket, key and upload ID are known.
|
||||
legacyObject := objects[1]
|
||||
er := sets.getHashedSet(legacyObject)
|
||||
fi, metadata, err := er.checkUploadIDExists(t.Context(), bucket, legacyObject, uploadIDs[legacyObject], true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range metadata {
|
||||
delete(metadata[i].Metadata, multipartMetaBucket)
|
||||
delete(metadata[i].Metadata, multipartMetaObject)
|
||||
}
|
||||
if _, err = writeAllMetadata(t.Context(), er.getDisks(), bucket, minioMetaMultipartBucket,
|
||||
er.getUploadIDDir(bucket, legacyObject, uploadIDs[legacyObject]), metadata, fi.WriteQuorum(er.defaultWQuorum())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacy, err := z.ListMultipartUploads(t.Context(), bucket, legacyObject, "", "", "", 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, legacy)
|
||||
remaining, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, remaining, objects[2:]...)
|
||||
part, err = z.PutObjectPart(t.Context(), bucket, legacyObject, uploadIDs[legacyObject], 1,
|
||||
mustGetPutObjReader(t, bytes.NewBufferString("old part"), 8, "", ""), ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("continue historical upload: %v", err)
|
||||
}
|
||||
if _, err = z.CompleteMultipartUpload(t.Context(), bucket, legacyObject, uploadIDs[legacyObject],
|
||||
[]CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("complete historical upload: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPaginateMultipartUploads(t *testing.T) {
|
||||
base := time.Unix(100, 0)
|
||||
id1, id2, id3 := multipartListingTestID(base, 1), multipartListingTestID(base.Add(time.Second), 2), multipartListingTestID(base, 3)
|
||||
uploads := []MultipartInfo{
|
||||
{Bucket: "bucket", Object: "b", UploadID: id3, Initiated: base},
|
||||
{Bucket: "bucket", Object: "a", UploadID: id2, Initiated: base.Add(time.Second)},
|
||||
{Bucket: "bucket", Object: "a", UploadID: id1, Initiated: base},
|
||||
{Bucket: "bucket", Object: "a", UploadID: id1, Initiated: base}, // duplicate discovery
|
||||
}
|
||||
|
||||
first := paginateMultipartUploads(uploads, "", "", "", "", 1)
|
||||
requireMultipartUploadKeys(t, first, "a")
|
||||
if !first.IsTruncated || first.NextKeyMarker != "a" || first.NextUploadIDMarker != id1 {
|
||||
t.Fatalf("first page = %+v", first)
|
||||
}
|
||||
|
||||
second := paginateMultipartUploads(uploads, "", first.NextKeyMarker, first.NextUploadIDMarker, "", 1)
|
||||
if len(second.Uploads) != 1 || second.Uploads[0].Object != "a" || second.Uploads[0].UploadID != id2 {
|
||||
t.Fatalf("second page uploads = %+v", second.Uploads)
|
||||
}
|
||||
if !second.IsTruncated || second.NextKeyMarker != "a" || second.NextUploadIDMarker != id2 {
|
||||
t.Fatalf("second page = %+v", second)
|
||||
}
|
||||
|
||||
last := paginateMultipartUploads(uploads, "", second.NextKeyMarker, second.NextUploadIDMarker, "", 1)
|
||||
requireMultipartUploadKeys(t, last, "b")
|
||||
if last.IsTruncated || last.NextKeyMarker != "" || last.NextUploadIDMarker != "" {
|
||||
t.Fatalf("last page = %+v", last)
|
||||
}
|
||||
|
||||
missingUploadMarker := paginateMultipartUploads(uploads, "", "a", multipartListingTestID(base.Add(2*time.Second), 4), "", 10)
|
||||
requireMultipartUploadKeys(t, missingUploadMarker, "b")
|
||||
|
||||
if err := checkListMultipartArgs(t.Context(), "bucket", "", "", "not-base64=", ""); err != nil {
|
||||
t.Fatalf("upload-id-marker without key-marker must be ignored: %v", err)
|
||||
}
|
||||
|
||||
overLimit := make([]MultipartInfo, maxUploadsList+1)
|
||||
for i := range overLimit {
|
||||
overLimit[i] = MultipartInfo{Bucket: "bucket", Object: fmt.Sprintf("%04d", i), UploadID: fmt.Sprint(i)}
|
||||
}
|
||||
capped := paginateMultipartUploads(overLimit, "", "", "", "", maxUploadsList+1)
|
||||
if capped.MaxUploads != maxUploadsList || len(capped.Uploads) != maxUploadsList || !capped.IsTruncated {
|
||||
t.Fatalf("over-limit page = MaxUploads %d, uploads %d, truncated %t",
|
||||
capped.MaxUploads, len(capped.Uploads), capped.IsTruncated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListMultipartUploadsGlobalPageAcrossPools(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
|
||||
objects := []string{"a/one", "b/two", "c/three", "d/four"}
|
||||
for i, object := range objects {
|
||||
if _, err := z.serverPools[i%len(z.serverPools)].NewMultipartUpload(t.Context(), bucket, object, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("NewMultipartUpload(%q): %v", object, err)
|
||||
}
|
||||
}
|
||||
z.mpCache.Range(func(uploadID string, _ MultipartInfo) bool {
|
||||
z.mpCache.Delete(uploadID)
|
||||
return true
|
||||
})
|
||||
|
||||
page, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, page, objects[:2]...)
|
||||
if !page.IsTruncated || page.NextKeyMarker != objects[1] {
|
||||
t.Fatalf("first global page = %+v", page)
|
||||
}
|
||||
|
||||
rest, err := z.ListMultipartUploads(t.Context(), bucket, "", page.NextKeyMarker, page.NextUploadIDMarker, "", 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, rest, objects[2:]...)
|
||||
if rest.IsTruncated {
|
||||
t.Fatalf("last global page is truncated: %+v", rest)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,412 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
)
|
||||
|
||||
func listingLockerTestWrite(t *testing.T, l *localLocker, args dsync.LockArgs) {
|
||||
t.Helper()
|
||||
if ok, err := l.Lock(t.Context(), args); !ok || err != nil {
|
||||
t.Fatalf("Lock = %v, %v", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func listingLockerTestUnlock(t *testing.T, l *localLocker, args dsync.LockArgs) {
|
||||
t.Helper()
|
||||
if ok, err := l.Unlock(t.Context(), args); !ok || err != nil {
|
||||
t.Fatalf("Unlock = %v, %v", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalLockerListingReverseWaitRechecks(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
change func(*testing.T, *localLocker, dsync.LockArgs)
|
||||
want bool
|
||||
}{
|
||||
{"writer-released", func(t *testing.T, l *localLocker, w dsync.LockArgs) { listingLockerTestUnlock(t, l, w) }, true},
|
||||
{"writer-still-held", func(*testing.T, *localLocker, dsync.LockArgs) {}, false},
|
||||
{"replacement-writer", func(t *testing.T, l *localLocker, w dsync.LockArgs) {
|
||||
listingLockerTestUnlock(t, l, w)
|
||||
listingLockerTestWrite(t, l, listingLockerTestArgs("new-writer", "key", "writer"))
|
||||
}, false},
|
||||
{"ordinary-reader-arrives", func(t *testing.T, l *localLocker, w dsync.LockArgs) {
|
||||
listingLockerTestUnlock(t, l, w)
|
||||
listingLockerTestRead(t, l, listingLockerTestArgs("ordinary-reader", "key", "ordinary-reader"))
|
||||
}, true},
|
||||
{"listing-reader-arrives", func(t *testing.T, l *localLocker, w dsync.LockArgs) {
|
||||
listingLockerTestUnlock(t, l, w)
|
||||
listingLockerTestRead(t, l, listingLockerTestArgs("other-listing", "key", listingLockerTestSource))
|
||||
}, true},
|
||||
{"force-unlock", func(t *testing.T, l *localLocker, w dsync.LockArgs) {
|
||||
if ok, err := l.ForceUnlock(t.Context(), w); !ok || err != nil {
|
||||
t.Fatalf("ForceUnlock = %v, %v", ok, err)
|
||||
}
|
||||
}, true},
|
||||
{"expiry", func(_ *testing.T, l *localLocker, _ dsync.LockArgs) { l.expireOldLocks(-time.Minute) }, true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
writer := listingLockerTestArgs("writer", "key", "writer")
|
||||
reader := listingLockerTestArgs("listing", "key", listingLockerTestSource)
|
||||
listingLockerTestWrite(t, l, writer)
|
||||
var got bool
|
||||
var err error
|
||||
start := time.Now()
|
||||
go func() { got, err = l.RLock(t.Context(), reader) }()
|
||||
synctest.Wait()
|
||||
if state := listingLockerTestSnapshot(l); state.waiting != 1 {
|
||||
t.Fatalf("waiting = %d, want 1", state.waiting)
|
||||
}
|
||||
listingLockerTestWrite(t, l, listingLockerTestArgs("unrelated", "other-key", "writer"))
|
||||
if elapsed := time.Since(start); elapsed != 0 {
|
||||
t.Fatalf("unrelated key waited %v", elapsed)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
tc.change(t, l, writer)
|
||||
time.Sleep(time.Millisecond)
|
||||
synctest.Wait()
|
||||
if state := listingLockerTestSnapshot(l); got != tc.want || err != nil || state.waiting != 0 {
|
||||
t.Fatalf("RLock = %v, %v, waiting = %d; want %v, nil, 0", got, err, state.waiting, tc.want)
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed != 2*time.Millisecond {
|
||||
t.Fatalf("one attempt elapsed %v, want 2ms", elapsed)
|
||||
}
|
||||
found := false
|
||||
for _, holder := range l.DupLockMap()["key"] {
|
||||
if holder.UID == reader.UID {
|
||||
found = true
|
||||
if holder.Writer || holder.Timestamp != UTCNow().UnixNano() || holder.TimeLastRefresh != holder.Timestamp {
|
||||
t.Fatalf("invalid granted reader: %+v", holder)
|
||||
}
|
||||
}
|
||||
}
|
||||
if found != got {
|
||||
t.Fatalf("reader present = %v, granted = %v", found, got)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalLockerListingReverseWaitCancellation(t *testing.T) {
|
||||
for _, when := range []string{"before-call", "during-wait", "deadline"} {
|
||||
t.Run(when, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
writer := listingLockerTestArgs("writer", "key", "writer")
|
||||
listingLockerTestWrite(t, l, writer)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
if when == "deadline" {
|
||||
cancel()
|
||||
ctx, cancel = context.WithTimeout(t.Context(), time.Millisecond)
|
||||
}
|
||||
defer cancel()
|
||||
if when == "before-call" {
|
||||
cancel()
|
||||
}
|
||||
var got bool
|
||||
var err error
|
||||
start := time.Now()
|
||||
go func() { got, err = l.RLock(ctx, listingLockerTestArgs("listing", "key", listingLockerTestSource)) }()
|
||||
synctest.Wait()
|
||||
if when != "before-call" {
|
||||
if state := listingLockerTestSnapshot(l); state.waiting != 1 {
|
||||
t.Fatalf("waiting = %d, want 1", state.waiting)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
// Removal must not turn the canceled request into a grant.
|
||||
listingLockerTestUnlock(t, l, writer)
|
||||
cancel()
|
||||
} else {
|
||||
listingLockerTestUnlock(t, l, writer)
|
||||
}
|
||||
synctest.Wait()
|
||||
state := listingLockerTestSnapshot(l)
|
||||
if got || err == nil || err != ctx.Err() || state != (listingLockerTestState{}) {
|
||||
t.Fatalf("canceled RLock = %v, %v; context %v; state %+v", got, err, ctx.Err(), state)
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed >= 2*time.Millisecond {
|
||||
t.Fatalf("canceled request waited for full timer: %v", elapsed)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalLockerListingReverseWaitExcluded(t *testing.T) {
|
||||
for _, source := range []string{"ordinary-reader", "", "[<unknown>:0:()]", "listingRLock", "[metacache-server-pool.go:0:(*erasureServerPools).listingRLock()]"} {
|
||||
t.Run(source, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
listingLockerTestWrite(t, l, listingLockerTestArgs("writer", "key", "writer"))
|
||||
start := time.Now()
|
||||
if ok, err := l.RLock(t.Context(), listingLockerTestArgs("reader", "key", source)); ok || err != nil {
|
||||
t.Fatalf("excluded RLock = %v, %v", ok, err)
|
||||
}
|
||||
if state := listingLockerTestSnapshot(l); time.Since(start) != 0 || state.waiting != 0 || state.uids != 1 {
|
||||
t.Fatalf("excluded RLock changed timing or state: %+v", state)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalLockerListingReverseWaitFastPaths(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
start := time.Now()
|
||||
for i, source := range []string{listingLockerTestSource, "ordinary-reader", ""} {
|
||||
listingLockerTestRead(t, l, listingLockerTestArgs(fmt.Sprintf("shared-%d", i), "shared", source))
|
||||
listingLockerTestRead(t, l, listingLockerTestArgs(fmt.Sprintf("free-%d", i), fmt.Sprintf("free-%d", i), source))
|
||||
}
|
||||
if state := listingLockerTestSnapshot(l); time.Since(start) != 0 || state.waiting != 0 || state.uids != 6 {
|
||||
t.Fatalf("uncontended readers changed timing or state: %+v", state)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestLocalLockerListingReverseWaitOverloaded(t *testing.T) {
|
||||
for _, when := range []string{"before-call", "during-wait"} {
|
||||
t.Run(when, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
writer := listingLockerTestArgs("writer", "key", "writer")
|
||||
listingLockerTestWrite(t, l, writer)
|
||||
defer l.waitMutex.Store(0)
|
||||
if when == "before-call" {
|
||||
l.waitMutex.Store(lockMutexWaitLimit + 1)
|
||||
}
|
||||
var got bool
|
||||
var err error
|
||||
go func() {
|
||||
got, err = l.RLock(t.Context(), listingLockerTestArgs("listing", "key", listingLockerTestSource))
|
||||
}()
|
||||
synctest.Wait()
|
||||
if when == "during-wait" {
|
||||
if state := listingLockerTestSnapshot(l); state.waiting != 1 {
|
||||
t.Fatalf("waiting = %d, want 1", state.waiting)
|
||||
}
|
||||
listingLockerTestUnlock(t, l, writer)
|
||||
l.waitMutex.Store(lockMutexWaitLimit + 1)
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
synctest.Wait()
|
||||
} else {
|
||||
listingLockerTestUnlock(t, l, writer)
|
||||
}
|
||||
if state := listingLockerTestSnapshot(l); got || err != nil || state != (listingLockerTestState{}) || l.locksOverloaded.Load() != 1 {
|
||||
t.Fatalf("overloaded RLock = %v, %v; state %+v; overloads %d", got, err, state, l.locksOverloaded.Load())
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalLockerListingWaitSharedAdmission(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
listingLockerTestRead(t, l, listingLockerTestArgs("reader", "read-key", listingLockerTestSource))
|
||||
listingLockerTestWrite(t, l, listingLockerTestArgs("writer", "write-key", "writer"))
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
results := make(chan error, listingLockWaitLimit)
|
||||
for i := range listingLockWaitLimit {
|
||||
go func() {
|
||||
var ok bool
|
||||
var err error
|
||||
if i%2 == 0 {
|
||||
ok, err = l.Lock(ctx, listingLockerTestArgs(fmt.Sprintf("writer-%d", i), "read-key", "writer"))
|
||||
} else {
|
||||
ok, err = l.RLock(ctx, listingLockerTestArgs(fmt.Sprintf("listing-%d", i), "write-key", listingLockerTestSource))
|
||||
}
|
||||
if ok {
|
||||
err = fmt.Errorf("granted over a conflicting holder")
|
||||
}
|
||||
results <- err
|
||||
}()
|
||||
}
|
||||
synctest.Wait()
|
||||
if state := listingLockerTestSnapshot(l); state.waiting != listingLockWaitLimit {
|
||||
t.Fatalf("combined waiting = %d, want %d", state.waiting, listingLockWaitLimit)
|
||||
}
|
||||
start := time.Now()
|
||||
if ok, err := l.Lock(ctx, listingLockerTestArgs("excess-writer", "read-key", "writer")); ok || err != nil {
|
||||
t.Fatalf("excess Lock = %v, %v", ok, err)
|
||||
}
|
||||
if ok, err := l.RLock(ctx, listingLockerTestArgs("excess-listing", "write-key", listingLockerTestSource)); ok || err != nil {
|
||||
t.Fatalf("excess RLock = %v, %v", ok, err)
|
||||
}
|
||||
listingLockerTestWrite(t, l, listingLockerTestArgs("unrelated-writer", "unrelated-write", "writer"))
|
||||
listingLockerTestRead(t, l, listingLockerTestArgs("unrelated-listing", "unrelated-read", listingLockerTestSource))
|
||||
if state := listingLockerTestSnapshot(l); time.Since(start) != 0 || state.waiting != listingLockWaitLimit || state.uids != 4 {
|
||||
t.Fatalf("combined cap changed immediate paths: %+v", state)
|
||||
}
|
||||
cancel()
|
||||
synctest.Wait()
|
||||
for range listingLockWaitLimit {
|
||||
if err := <-results; err != context.Canceled {
|
||||
t.Fatalf("canceled waiter: %v", err)
|
||||
}
|
||||
}
|
||||
if state := listingLockerTestSnapshot(l); state.waiting != 0 || state.uids != 4 {
|
||||
t.Fatalf("combined wait leaked: %+v", state)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
type listingReverseCountLocker struct {
|
||||
*localLocker
|
||||
calls atomic.Int32
|
||||
grants atomic.Int32
|
||||
reject bool
|
||||
}
|
||||
|
||||
func (l *listingReverseCountLocker) RLock(ctx context.Context, args dsync.LockArgs) (bool, error) {
|
||||
l.calls.Add(1)
|
||||
if l.reject {
|
||||
return false, nil
|
||||
}
|
||||
ok, err := l.localLocker.RLock(ctx, args)
|
||||
if ok {
|
||||
l.grants.Add(1)
|
||||
}
|
||||
return ok, err
|
||||
}
|
||||
|
||||
func TestDistributedListingReverseWait(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
free, held int
|
||||
immediateRejects int
|
||||
cancel bool
|
||||
want bool
|
||||
}{
|
||||
{name: "all-writers-release", want: true},
|
||||
{name: "two-writers-remain", held: 2, want: true},
|
||||
{name: "three-writers-remain", held: 3},
|
||||
{name: "partial-grant-then-quorum", free: 1, want: true},
|
||||
{name: "partial-grant-without-quorum", free: 1, held: 3},
|
||||
{name: "late-grant-after-rejection", immediateRejects: 3},
|
||||
{name: "partial-and-late-grants-after-cancel", free: 1, cancel: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
clients := make([]dsync.NetLocker, 4)
|
||||
writer := listingLockerTestArgs("writer", "key", "writer")
|
||||
for i := range clients {
|
||||
l := &listingReverseCountLocker{localLocker: newLocker(), reject: i < tc.immediateRejects}
|
||||
clients[i] = l
|
||||
if i >= tc.free {
|
||||
listingLockerTestWrite(t, l.localLocker, writer)
|
||||
}
|
||||
}
|
||||
ds := &dsync.Dsync{GetLockers: func() ([]dsync.NetLocker, string) { return clients, writer.Owner }, Timeouts: dsync.DefaultTimeouts}
|
||||
dm := dsync.NewDRWMutex(ds, "key")
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
result := make(chan bool, 1)
|
||||
go func() {
|
||||
result <- dm.GetRLock(ctx, nil, "listing", listingLockerTestSource, dsync.Options{Timeout: time.Second, RetryInterval: -1})
|
||||
}()
|
||||
synctest.Wait()
|
||||
for i, c := range clients {
|
||||
l := c.(*listingReverseCountLocker)
|
||||
if i < tc.free && l.grants.Load() != 1 {
|
||||
t.Fatal("uncontended partial grant did not arrive immediately")
|
||||
}
|
||||
if i >= tc.free && !l.reject && listingLockerTestSnapshot(l.localLocker).waiting != 1 {
|
||||
t.Fatal("conflicting reader did not wait in its first RPC")
|
||||
}
|
||||
}
|
||||
earlyResult := tc.immediateRejects == 3 || tc.cancel
|
||||
if tc.cancel {
|
||||
cancel()
|
||||
synctest.Wait()
|
||||
}
|
||||
if earlyResult {
|
||||
select {
|
||||
case got := <-result:
|
||||
if got {
|
||||
t.Fatal("attempt succeeded before delayed replies")
|
||||
}
|
||||
default:
|
||||
t.Fatal("attempt did not return before delayed replies")
|
||||
}
|
||||
for i := range tc.free {
|
||||
if state := listingLockerTestSnapshot(clients[i].(*listingReverseCountLocker).localLocker); state.uids != 0 {
|
||||
t.Fatal("early partial grant was not released")
|
||||
}
|
||||
}
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
for i, c := range clients {
|
||||
if i >= tc.free+tc.held {
|
||||
listingLockerTestUnlock(t, c.(*listingReverseCountLocker).localLocker, writer)
|
||||
}
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
synctest.Wait()
|
||||
if !earlyResult {
|
||||
if got := <-result; got != tc.want {
|
||||
t.Fatalf("GetRLock = %v, want %v", got, tc.want)
|
||||
}
|
||||
}
|
||||
for i, c := range clients {
|
||||
l := c.(*listingReverseCountLocker)
|
||||
if l.calls.Load() != 1 || listingLockerTestSnapshot(l.localLocker).waiting != 0 {
|
||||
t.Fatalf("peer %d calls = %d or leaked admission", i, l.calls.Load())
|
||||
}
|
||||
wantGrant := !l.reject && (i < tc.free || i >= tc.free+tc.held)
|
||||
if (l.grants.Load() == 1) != wantGrant {
|
||||
t.Fatalf("peer %d grants = %d, want granted = %v", i, l.grants.Load(), wantGrant)
|
||||
}
|
||||
if !tc.want {
|
||||
for _, holder := range l.DupLockMap()["key"] {
|
||||
if holder.UID == "listing" {
|
||||
t.Fatalf("peer %d leaked partial or late read grant", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if tc.want {
|
||||
dm.RUnlock(t.Context())
|
||||
synctest.Wait()
|
||||
}
|
||||
for i, c := range clients {
|
||||
l := c.(*listingReverseCountLocker)
|
||||
if i >= tc.free && i < tc.free+tc.held {
|
||||
listingLockerTestUnlock(t, l.localLocker, writer)
|
||||
}
|
||||
if state := listingLockerTestSnapshot(l.localLocker); state != (listingLockerTestState{}) {
|
||||
t.Fatalf("peer %d final state is not empty: %+v", i, state)
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,432 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
)
|
||||
|
||||
const listingLockerTestSource = "[metacache-server-pool.go:274:(*erasureServerPools).listingRLock()]"
|
||||
|
||||
func listingLockerTestArgs(uid, resource, source string) dsync.LockArgs {
|
||||
quorum := 3
|
||||
return dsync.LockArgs{UID: uid, Resources: []string{resource}, Source: source, Owner: "listing-wait-test", Quorum: &quorum}
|
||||
}
|
||||
|
||||
func listingLockerTestRead(t *testing.T, l *localLocker, args dsync.LockArgs) {
|
||||
t.Helper()
|
||||
if ok, err := l.RLock(t.Context(), args); !ok || err != nil {
|
||||
t.Fatalf("RLock = %v, %v", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func listingLockerTestRUnlock(t *testing.T, l *localLocker, args dsync.LockArgs) {
|
||||
t.Helper()
|
||||
if ok, err := l.RUnlock(t.Context(), args); !ok || err != nil {
|
||||
t.Fatalf("RUnlock = %v, %v", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
type listingLockerTestState struct {
|
||||
waiting, resources, uids int
|
||||
}
|
||||
|
||||
func listingLockerTestSnapshot(l *localLocker) listingLockerTestState {
|
||||
defer l.getMutex()()
|
||||
return listingLockerTestState{waiting: l.listingWaiters, resources: len(l.lockMap), uids: len(l.lockUID)}
|
||||
}
|
||||
|
||||
func TestLocalLockerListingSource(t *testing.T) {
|
||||
for _, source := range []string{
|
||||
"", "listingRLock", "[<unknown>:0:()]",
|
||||
strings.Replace(listingLockerTestSource, ":274:", ":", 1),
|
||||
strings.Replace(listingLockerTestSource, "274", "", 1),
|
||||
strings.Replace(listingLockerTestSource, "274", "0", 1),
|
||||
strings.Replace(listingLockerTestSource, "274", "1x", 1),
|
||||
strings.Replace(listingLockerTestSource, "274", "-1", 1),
|
||||
strings.Replace(listingLockerTestSource, "listingRLock", "GetObjectNInfo", 1),
|
||||
strings.Replace(listingLockerTestSource, "listingRLock", "listingRLock.func1", 1),
|
||||
strings.Replace(listingLockerTestSource, "metacache-server-pool.go", "other.go", 1),
|
||||
listingLockerTestSource + "trailing",
|
||||
} {
|
||||
if isListingReadLock([]lockRequesterInfo{{Source: source}}) {
|
||||
t.Errorf("unrecognized source accepted: %q", source)
|
||||
}
|
||||
}
|
||||
if isListingReadLock(nil) || isListingReadLock([]lockRequesterInfo{{Source: listingLockerTestSource, Writer: true}}) {
|
||||
t.Fatal("empty or write holders accepted")
|
||||
}
|
||||
if isListingReadLock([]lockRequesterInfo{{Source: listingLockerTestSource}, {Source: "ordinary-reader"}}) {
|
||||
t.Fatal("mixed readers accepted")
|
||||
}
|
||||
|
||||
// Check the real runtime-generated source, not only a hand-written label.
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
lockers := []dsync.NetLocker{newLocker(), newLocker(), newLocker(), newLocker()}
|
||||
set := &erasureObjects{nsMutex: newNSLock(true), getLockers: func() ([]dsync.NetLocker, string) { return lockers, "listing-source-test" }}
|
||||
z := &erasureServerPools{serverPools: []*erasureSets{{sets: []*erasureObjects{set}}, {sets: []*erasureObjects{set}}}}
|
||||
_, unlock, err := z.listingRLock(t.Context(), "bucket", "object", set)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range lockers {
|
||||
l := c.(*localLocker)
|
||||
for _, holders := range l.DupLockMap() {
|
||||
if !isListingReadLock(holders) {
|
||||
t.Errorf("actual listingRLock source was not recognized: %#v", holders)
|
||||
}
|
||||
}
|
||||
}
|
||||
unlock()
|
||||
synctest.Wait()
|
||||
})
|
||||
}
|
||||
|
||||
func FuzzListingLockSource(f *testing.F) {
|
||||
f.Add(listingLockerTestSource)
|
||||
f.Add("[metacache-server-pool.go:(*erasureServerPools).listingRLock()]")
|
||||
f.Add("")
|
||||
f.Fuzz(func(t *testing.T, source string) {
|
||||
// Source is diagnostic text from a peer, including older versions. No
|
||||
// malformed string may panic or make a writer eligible for this wait.
|
||||
isListingReadLock([]lockRequesterInfo{{Source: source}})
|
||||
if isListingReadLock([]lockRequesterInfo{{Source: source, Writer: true}}) {
|
||||
t.Fatal("writer was accepted as a listing reader")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestLocalLockerListingWaitRechecks(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
change func(*testing.T, *localLocker, dsync.LockArgs)
|
||||
want bool
|
||||
}{
|
||||
{"reader-released", func(t *testing.T, l *localLocker, r dsync.LockArgs) { listingLockerTestRUnlock(t, l, r) }, true},
|
||||
{"reader-still-held", func(*testing.T, *localLocker, dsync.LockArgs) {}, false},
|
||||
{"one-of-two-readers-remains", func(t *testing.T, l *localLocker, r dsync.LockArgs) {
|
||||
other := r
|
||||
other.UID = "second-reader"
|
||||
listingLockerTestRead(t, l, other)
|
||||
listingLockerTestRUnlock(t, l, r)
|
||||
}, false},
|
||||
{"ordinary-reader-arrives", func(t *testing.T, l *localLocker, r dsync.LockArgs) {
|
||||
listingLockerTestRUnlock(t, l, r)
|
||||
listingLockerTestRead(t, l, listingLockerTestArgs("new-reader", "key", "ordinary-reader"))
|
||||
}, false},
|
||||
{"other-writer-wins", func(t *testing.T, l *localLocker, r dsync.LockArgs) {
|
||||
listingLockerTestRUnlock(t, l, r)
|
||||
if ok, err := l.Lock(t.Context(), listingLockerTestArgs("other-writer", "key", "writer")); !ok || err != nil {
|
||||
t.Fatalf("competing Lock = %v, %v", ok, err)
|
||||
}
|
||||
}, false},
|
||||
{"force-unlock", func(t *testing.T, l *localLocker, r dsync.LockArgs) {
|
||||
if ok, err := l.ForceUnlock(t.Context(), r); !ok || err != nil {
|
||||
t.Fatalf("ForceUnlock = %v, %v", ok, err)
|
||||
}
|
||||
}, true},
|
||||
{"expiry", func(_ *testing.T, l *localLocker, _ dsync.LockArgs) { l.expireOldLocks(-time.Minute) }, true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
reader := listingLockerTestArgs("reader", "key", listingLockerTestSource)
|
||||
listingLockerTestRead(t, l, reader)
|
||||
var got bool
|
||||
var err error
|
||||
start := time.Now()
|
||||
go func() { got, err = l.Lock(t.Context(), listingLockerTestArgs("writer", "key", "writer")) }()
|
||||
synctest.Wait()
|
||||
if state := listingLockerTestSnapshot(l); state.waiting != 1 {
|
||||
t.Fatalf("waiting = %d, want 1", state.waiting)
|
||||
}
|
||||
// The global mutex must be released throughout the timer wait.
|
||||
if ok, err := l.Lock(t.Context(), listingLockerTestArgs("unrelated", "other-key", "writer")); !ok || err != nil {
|
||||
t.Fatalf("unrelated key blocked: %v, %v", ok, err)
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed != 0 {
|
||||
t.Fatalf("unrelated key waited %v", elapsed)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
tc.change(t, l, reader)
|
||||
time.Sleep(time.Millisecond)
|
||||
synctest.Wait()
|
||||
if state := listingLockerTestSnapshot(l); got != tc.want || err != nil || state.waiting != 0 {
|
||||
t.Fatalf("Lock = %v, %v, waiting = %d; want %v, nil, 0", got, err, state.waiting, tc.want)
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed != 2*time.Millisecond {
|
||||
t.Fatalf("one attempt elapsed %v, want 2ms", elapsed)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalLockerListingWaitCancellation(t *testing.T) {
|
||||
for _, when := range []string{"before-call", "during-wait", "deadline"} {
|
||||
t.Run(when, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
reader := listingLockerTestArgs("reader", "key", listingLockerTestSource)
|
||||
listingLockerTestRead(t, l, reader)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
if when == "deadline" {
|
||||
cancel()
|
||||
ctx, cancel = context.WithTimeout(t.Context(), time.Millisecond)
|
||||
}
|
||||
defer cancel()
|
||||
if when == "before-call" {
|
||||
cancel()
|
||||
}
|
||||
var got bool
|
||||
var err error
|
||||
go func() { got, err = l.Lock(ctx, listingLockerTestArgs("writer", "key", "writer")) }()
|
||||
synctest.Wait()
|
||||
if when != "before-call" {
|
||||
if state := listingLockerTestSnapshot(l); state.waiting != 1 {
|
||||
t.Fatalf("waiting = %d", state.waiting)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
cancel()
|
||||
}
|
||||
synctest.Wait()
|
||||
if state := listingLockerTestSnapshot(l); got || err != ctx.Err() || err == nil || state.waiting != 0 {
|
||||
t.Fatalf("canceled Lock = %v, %v; context %v; waiting = %d", got, err, ctx.Err(), state.waiting)
|
||||
}
|
||||
listingLockerTestRUnlock(t, l, reader)
|
||||
if state := listingLockerTestSnapshot(l); state.resources != 0 || state.uids != 0 {
|
||||
t.Fatal("canceled wait left a lock")
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalLockerListingWaitExcluded(t *testing.T) {
|
||||
for _, kind := range []string{"ordinary-reader", "unknown-reader", "mixed-readers", "writer", "bulk", "overloaded"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
reader := listingLockerTestArgs("holder", "key", listingLockerTestSource)
|
||||
if kind == "ordinary-reader" {
|
||||
reader.Source = "[erasure-object.go:100:(*erasureObjects).GetObjectNInfo()]"
|
||||
}
|
||||
if kind == "unknown-reader" {
|
||||
reader.Source = ""
|
||||
}
|
||||
if kind == "writer" {
|
||||
if ok, err := l.Lock(t.Context(), reader); !ok || err != nil {
|
||||
t.Fatalf("initial write = %v, %v", ok, err)
|
||||
}
|
||||
} else {
|
||||
listingLockerTestRead(t, l, reader)
|
||||
}
|
||||
if kind == "mixed-readers" {
|
||||
listingLockerTestRead(t, l, listingLockerTestArgs("ordinary", "key", "ordinary-reader"))
|
||||
}
|
||||
writer := listingLockerTestArgs("writer", "key", "writer")
|
||||
if kind == "bulk" {
|
||||
writer.Resources = []string{"key", "other-key"}
|
||||
}
|
||||
if kind == "overloaded" {
|
||||
l.waitMutex.Store(lockMutexWaitLimit + 1)
|
||||
}
|
||||
start := time.Now()
|
||||
if ok, err := l.Lock(t.Context(), writer); ok || err != nil {
|
||||
t.Fatalf("conflicting Lock = %v, %v", ok, err)
|
||||
}
|
||||
state := listingLockerTestSnapshot(l)
|
||||
if elapsed := time.Since(start); elapsed != 0 || state.waiting != 0 {
|
||||
t.Fatalf("excluded path waited %v; waiters = %d", elapsed, state.waiting)
|
||||
}
|
||||
if _, ok := l.DupLockMap()["other-key"]; ok {
|
||||
t.Fatal("bulk request partially acquired an unrelated key")
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalLockerListingWaitAdmission(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
reader := listingLockerTestArgs("reader", "key", listingLockerTestSource)
|
||||
listingLockerTestRead(t, l, reader)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
results := make(chan error, listingLockWaitLimit)
|
||||
for i := range listingLockWaitLimit {
|
||||
go func() {
|
||||
ok, err := l.Lock(ctx, listingLockerTestArgs(fmt.Sprintf("writer-%d", i), "key", "writer"))
|
||||
if ok {
|
||||
err = fmt.Errorf("granted over a live reader")
|
||||
}
|
||||
results <- err
|
||||
}()
|
||||
}
|
||||
synctest.Wait()
|
||||
if state := listingLockerTestSnapshot(l); state.waiting != listingLockWaitLimit {
|
||||
t.Fatalf("waiting = %d, want %d", state.waiting, listingLockWaitLimit)
|
||||
}
|
||||
start := time.Now()
|
||||
if ok, err := l.Lock(ctx, listingLockerTestArgs("excess", "key", "writer")); ok || err != nil {
|
||||
t.Fatalf("excess waiter = %v, %v", ok, err)
|
||||
}
|
||||
if state := listingLockerTestSnapshot(l); time.Since(start) != 0 || state.waiting != listingLockWaitLimit {
|
||||
t.Fatal("excess waiter consumed time or admission")
|
||||
}
|
||||
if ok, err := l.Lock(ctx, listingLockerTestArgs("unrelated", "other-key", "writer")); !ok || err != nil {
|
||||
t.Fatalf("admission limit rejected an uncontended key: %v, %v", ok, err)
|
||||
}
|
||||
cancel()
|
||||
synctest.Wait()
|
||||
for range listingLockWaitLimit {
|
||||
if err := <-results; err != context.Canceled {
|
||||
t.Fatalf("canceled waiter: %v", err)
|
||||
}
|
||||
}
|
||||
if state := listingLockerTestSnapshot(l); state.waiting != 0 {
|
||||
t.Fatalf("admission leaked: %d", state.waiting)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestLocalLockerListingWaitContended(t *testing.T) {
|
||||
for _, n := range []int{2, 8, 32} {
|
||||
t.Run(fmt.Sprintf("%d-writers", n), func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
reader := listingLockerTestArgs("reader", "key", listingLockerTestSource)
|
||||
listingLockerTestRead(t, l, reader)
|
||||
results := make(chan bool, n)
|
||||
for i := range n {
|
||||
go func() {
|
||||
ok, err := l.Lock(t.Context(), listingLockerTestArgs(fmt.Sprintf("writer-%d", i), "key", "writer"))
|
||||
if err != nil {
|
||||
t.Errorf("Lock: %v", err)
|
||||
}
|
||||
results <- ok
|
||||
}()
|
||||
}
|
||||
synctest.Wait()
|
||||
if state := listingLockerTestSnapshot(l); state.waiting != n {
|
||||
t.Fatalf("waiting = %d, want %d", state.waiting, n)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
listingLockerTestRUnlock(t, l, reader)
|
||||
time.Sleep(time.Millisecond)
|
||||
synctest.Wait()
|
||||
granted := 0
|
||||
for range n {
|
||||
if <-results {
|
||||
granted++
|
||||
}
|
||||
}
|
||||
if state := listingLockerTestSnapshot(l); granted != 1 || state.uids != 1 || state.waiting != 0 {
|
||||
t.Fatalf("grants = %d, indexed = %d, waiting = %d", granted, state.uids, state.waiting)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type listingCountLocker struct {
|
||||
*localLocker
|
||||
calls atomic.Int32
|
||||
}
|
||||
|
||||
func (l *listingCountLocker) Lock(ctx context.Context, args dsync.LockArgs) (bool, error) {
|
||||
l.calls.Add(1)
|
||||
return l.localLocker.Lock(ctx, args)
|
||||
}
|
||||
|
||||
// Exercise real DRWMutex quorum and late-response cleanup. The mixed case with
|
||||
// two immediate rejections fails before the upgraded lockers finish waiting.
|
||||
func TestDistributedListingWait(t *testing.T) {
|
||||
for _, ordinaryReaders := range []int{0, 1, 2} {
|
||||
t.Run(fmt.Sprintf("%d-immediate-rejections", ordinaryReaders), func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
clients := make([]dsync.NetLocker, 4)
|
||||
reader := listingLockerTestArgs("reader", "key", listingLockerTestSource)
|
||||
for i := range clients {
|
||||
l := &listingCountLocker{localLocker: newLocker()}
|
||||
clients[i] = l
|
||||
r := reader
|
||||
if i < ordinaryReaders {
|
||||
r.Source = "old-node-unknown-source"
|
||||
}
|
||||
listingLockerTestRead(t, l.localLocker, r)
|
||||
}
|
||||
ds := &dsync.Dsync{GetLockers: func() ([]dsync.NetLocker, string) { return clients, reader.Owner }, Timeouts: dsync.DefaultTimeouts}
|
||||
dm := dsync.NewDRWMutex(ds, "key")
|
||||
var got bool
|
||||
go func() {
|
||||
got = dm.GetLock(t.Context(), nil, "writer", "writer", dsync.Options{Timeout: time.Second, RetryInterval: -1})
|
||||
}()
|
||||
synctest.Wait()
|
||||
time.Sleep(time.Millisecond)
|
||||
for _, c := range clients {
|
||||
listingLockerTestRUnlock(t, c.(*listingCountLocker).localLocker, reader)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
synctest.Wait()
|
||||
if got != (ordinaryReaders < 2) {
|
||||
t.Fatalf("GetLock = %v", got)
|
||||
}
|
||||
granted := 0
|
||||
for _, c := range clients {
|
||||
l := c.(*listingCountLocker)
|
||||
if l.calls.Load() != 1 {
|
||||
t.Fatalf("lock calls = %d, want one round", l.calls.Load())
|
||||
}
|
||||
state := listingLockerTestSnapshot(l.localLocker)
|
||||
if state.resources != 0 {
|
||||
granted++
|
||||
}
|
||||
if state.waiting != 0 {
|
||||
t.Fatal("wait admission leaked")
|
||||
}
|
||||
}
|
||||
if got && granted < 3 {
|
||||
t.Fatalf("writer accepted without quorum: %d", granted)
|
||||
}
|
||||
if !got && granted != 0 {
|
||||
t.Fatalf("late grants leaked after rejected attempt: %d", granted)
|
||||
}
|
||||
if got {
|
||||
dm.Unlock(t.Context())
|
||||
synctest.Wait()
|
||||
}
|
||||
for _, c := range clients {
|
||||
l := c.(*listingCountLocker)
|
||||
if state := listingLockerTestSnapshot(l.localLocker); state.resources != 0 || state.uids != 0 {
|
||||
t.Fatal("final lock state not empty")
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
+89
-4
@@ -23,6 +23,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
@@ -38,6 +39,14 @@ import (
|
||||
// Rejected clients are expected to retry.
|
||||
const lockMutexWaitLimit = 1000
|
||||
|
||||
// A short listing recovery read or conflicting writer may finish before the
|
||||
// rejected request retries. Recheck once in the same request, sharing a bound
|
||||
// for timers outside the mutex. This is not a queue and gives neither priority.
|
||||
const (
|
||||
listingLockWait = 2 * time.Millisecond
|
||||
listingLockWaitLimit = 64
|
||||
)
|
||||
|
||||
// lockRequesterInfo stores various info from the client for each lock that is requested.
|
||||
type lockRequesterInfo struct {
|
||||
Name string // name of the resource lock was requested for
|
||||
@@ -65,6 +74,8 @@ type localLocker struct {
|
||||
waitMutex atomic.Int32
|
||||
lockMap map[string][]lockRequesterInfo
|
||||
lockUID map[string]string // UUID -> resource map.
|
||||
// Protected by mutex. These waiters are not included in waitMutex.
|
||||
listingWaiters int
|
||||
|
||||
// the following are updated on every cleanup defined in lockValidityDuration
|
||||
readers atomic.Int32
|
||||
@@ -96,6 +107,32 @@ func (l *localLocker) canTakeLock(resources ...string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// isListingReadLock recognizes only the existing diagnostic source emitted by
|
||||
// listingRLock. Unknown or changed source formats retain immediate rejection;
|
||||
// Source is not a new wire contract or proof that the read will finish quickly.
|
||||
func isListingReadLock(holders []lockRequesterInfo) bool {
|
||||
if len(holders) == 0 {
|
||||
return false
|
||||
}
|
||||
const prefix = "[metacache-server-pool.go:"
|
||||
const suffix = ":(*erasureServerPools).listingRLock()]"
|
||||
for _, holder := range holders {
|
||||
if holder.Writer || len(holder.Source) <= len(prefix)+len(suffix) || !strings.HasPrefix(holder.Source, prefix) || !strings.HasSuffix(holder.Source, suffix) {
|
||||
return false
|
||||
}
|
||||
line := holder.Source[len(prefix) : len(holder.Source)-len(suffix)]
|
||||
if len(line) == 0 || line[0] == '0' {
|
||||
return false
|
||||
}
|
||||
for _, c := range line {
|
||||
if c < '0' || c > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
|
||||
if len(args.Resources) > maxDeleteList {
|
||||
return false, fmt.Errorf("internal error: localLocker.Lock called with more than %d resources", maxDeleteList)
|
||||
@@ -112,9 +149,33 @@ func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool
|
||||
return false, ctx.Err()
|
||||
}
|
||||
if !l.canTakeLock(args.Resources...) {
|
||||
// Not all locks can be taken on resources,
|
||||
// reject it completely.
|
||||
return false, nil
|
||||
// Only a single-resource write blocked exclusively by listing recovery
|
||||
// readers gets a short, bounded opportunity to succeed in this attempt.
|
||||
if len(args.Resources) != 1 || l.listingWaiters >= listingLockWaitLimit || !isListingReadLock(l.lockMap[args.Resources[0]]) {
|
||||
return false, nil
|
||||
}
|
||||
l.listingWaiters++
|
||||
l.mutex.Unlock()
|
||||
timer := time.NewTimer(listingLockWait)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-timer.C:
|
||||
}
|
||||
timer.Stop()
|
||||
// Restore the mutex before returning to the existing deferred unlock.
|
||||
// Readers or another writer may have entered while it was released.
|
||||
l.getMutex()
|
||||
l.listingWaiters--
|
||||
if ctx.Err() != nil {
|
||||
return false, ctx.Err()
|
||||
}
|
||||
if l.waitMutex.Load() > lockMutexWaitLimit {
|
||||
l.locksOverloaded.Add(1)
|
||||
return false, nil
|
||||
}
|
||||
if !l.canTakeLock(args.Resources...) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// No locks held on the all resources, so claim write
|
||||
@@ -206,6 +267,31 @@ func (l *localLocker) RLock(ctx context.Context, args dsync.LockArgs) (reply boo
|
||||
return false, ctx.Err()
|
||||
}
|
||||
resource := args.Resources[0]
|
||||
lri, ok := l.lockMap[resource]
|
||||
if ok && isWriteLock(lri) && l.listingWaiters < listingLockWaitLimit && isListingReadLock([]lockRequesterInfo{{Source: args.Source}}) {
|
||||
// Only a positively identified listing read blocked by a writer may
|
||||
// wait once. A long or replacement writer still rejects normally.
|
||||
l.listingWaiters++
|
||||
l.mutex.Unlock()
|
||||
timer := time.NewTimer(listingLockWait)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-timer.C:
|
||||
}
|
||||
timer.Stop()
|
||||
// Restore the mutex before the existing deferred unlock, then use
|
||||
// the current holders in the normal read-lock grant path below.
|
||||
l.getMutex()
|
||||
l.listingWaiters--
|
||||
if ctx.Err() != nil {
|
||||
return false, ctx.Err()
|
||||
}
|
||||
if l.waitMutex.Load() > lockMutexWaitLimit {
|
||||
l.locksOverloaded.Add(1)
|
||||
return false, nil
|
||||
}
|
||||
lri, ok = l.lockMap[resource]
|
||||
}
|
||||
now := UTCNow()
|
||||
lrInfo := lockRequesterInfo{
|
||||
Name: resource,
|
||||
@@ -217,7 +303,6 @@ func (l *localLocker) RLock(ctx context.Context, args dsync.LockArgs) (reply boo
|
||||
TimeLastRefresh: now.UnixNano(),
|
||||
Quorum: *args.Quorum,
|
||||
}
|
||||
lri, ok := l.lockMap[resource]
|
||||
if ok {
|
||||
if reply = !isWriteLock(lri); reply {
|
||||
// Unless there is a write lock
|
||||
|
||||
+110
-11
@@ -37,6 +37,10 @@ type metaCacheEntry struct {
|
||||
// Entries without metadata will only be present in non-recursive scans.
|
||||
metadata []byte
|
||||
|
||||
// listingSet marks an eligible unresolved walk entry. It is consumed by the
|
||||
// pool merger before any cache serialization or downstream filtering.
|
||||
listingSet *erasureObjects
|
||||
|
||||
// cached contains the metadata if decoded.
|
||||
cached *xlMetaV2
|
||||
|
||||
@@ -46,17 +50,17 @@ type metaCacheEntry struct {
|
||||
|
||||
// isDir returns if the entry is representing a prefix directory.
|
||||
func (e metaCacheEntry) isDir() bool {
|
||||
return len(e.metadata) == 0 && strings.HasSuffix(e.name, slashSeparator)
|
||||
return e.listingSet == nil && len(e.metadata) == 0 && strings.HasSuffix(e.name, slashSeparator)
|
||||
}
|
||||
|
||||
// isObject returns if the entry is representing an object.
|
||||
func (e metaCacheEntry) isObject() bool {
|
||||
return len(e.metadata) > 0
|
||||
return e.listingSet != nil || len(e.metadata) > 0
|
||||
}
|
||||
|
||||
// isObjectDir returns if the entry is representing an object/
|
||||
func (e metaCacheEntry) isObjectDir() bool {
|
||||
return len(e.metadata) > 0 && strings.HasSuffix(e.name, slashSeparator)
|
||||
return (e.listingSet != nil || len(e.metadata) > 0) && strings.HasSuffix(e.name, slashSeparator)
|
||||
}
|
||||
|
||||
// hasPrefix returns whether an entry has a specific prefix
|
||||
@@ -670,26 +674,82 @@ func (m *metaCacheEntriesSorted) forwardPast(s string) {
|
||||
// If the context is canceled the function will return the error,
|
||||
// otherwise the function will return nil.
|
||||
func mergeEntryChannels(ctx context.Context, in []chan metaCacheEntry, out chan<- metaCacheEntry, readQuorum int) error {
|
||||
return mergeEntryChannelsWithErrors(ctx, in, out, readQuorum, nil)
|
||||
}
|
||||
|
||||
// terminalErr is inspected after each input closes and before the merger
|
||||
// advances beyond it. Callers must publish the error before closing the input.
|
||||
func mergeEntryChannelsWithErrors(ctx context.Context, in []chan metaCacheEntry, out chan<- metaCacheEntry, readQuorum int, terminalErr func(int) error) error {
|
||||
return mergeEntryChannelsWithResolver(ctx, in, out, readQuorum, terminalErr, nil)
|
||||
}
|
||||
|
||||
// resolve is called only for groups containing a pending listing entry. All
|
||||
// selected inputs are advanced (and their terminal errors checked) before it is
|
||||
// called, so recovery never holds an object lock while waiting for input.
|
||||
func mergeEntryChannelsWithResolver(ctx context.Context, in []chan metaCacheEntry, out chan<- metaCacheEntry, readQuorum int, terminalErr func(int) error, resolve func([]*metaCacheEntry) (*metaCacheEntry, error)) error {
|
||||
defer xioutil.SafeClose(out)
|
||||
top := make([]*metaCacheEntry, len(in))
|
||||
nDone := 0
|
||||
ctxDone := ctx.Done()
|
||||
|
||||
// Use simpler forwarder.
|
||||
// Keep the single-input normal path allocation-free. Only a pending entry
|
||||
// requires looking ahead to observe its stream's terminal error first.
|
||||
if len(in) == 1 {
|
||||
var buffered metaCacheEntry
|
||||
haveBuffered, exhausted := false, false
|
||||
for {
|
||||
select {
|
||||
case <-ctxDone:
|
||||
return ctx.Err()
|
||||
case v, ok := <-in[0]:
|
||||
if !ok {
|
||||
return nil
|
||||
var v metaCacheEntry
|
||||
if haveBuffered {
|
||||
v, haveBuffered = buffered, false
|
||||
} else {
|
||||
select {
|
||||
case <-ctxDone:
|
||||
return ctx.Err()
|
||||
case entry, ok := <-in[0]:
|
||||
if !ok {
|
||||
if terminalErr != nil {
|
||||
return terminalErr(0)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
v = entry
|
||||
}
|
||||
}
|
||||
if v.listingSet != nil {
|
||||
if resolve == nil {
|
||||
return errInvalidArgument
|
||||
}
|
||||
select {
|
||||
case <-ctxDone:
|
||||
return ctx.Err()
|
||||
case out <- v:
|
||||
case entry, ok := <-in[0]:
|
||||
buffered, haveBuffered, exhausted = entry, ok, !ok
|
||||
if !ok && terminalErr != nil {
|
||||
if err := terminalErr(0); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
pending := v
|
||||
resolved, err := resolve([]*metaCacheEntry{&pending})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resolved == nil {
|
||||
if exhausted {
|
||||
return nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
v = *resolved
|
||||
}
|
||||
select {
|
||||
case <-ctxDone:
|
||||
return ctx.Err()
|
||||
case out <- v:
|
||||
}
|
||||
if exhausted {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -700,6 +760,11 @@ func mergeEntryChannels(ctx context.Context, in []chan metaCacheEntry, out chan<
|
||||
return ctx.Err()
|
||||
case entry, ok := <-in[idx]:
|
||||
if !ok {
|
||||
if terminalErr != nil {
|
||||
if err := terminalErr(idx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
top[idx] = nil
|
||||
nDone++
|
||||
} else {
|
||||
@@ -778,6 +843,40 @@ func mergeEntryChannels(ctx context.Context, in []chan metaCacheEntry, out chan<
|
||||
}
|
||||
}
|
||||
|
||||
pending := best.listingSet != nil
|
||||
for _, idx := range toMerge {
|
||||
pending = pending || top[idx].listingSet != nil
|
||||
}
|
||||
if pending {
|
||||
if resolve == nil {
|
||||
return errInvalidArgument
|
||||
}
|
||||
group := make([]*metaCacheEntry, 1, len(toMerge)+1)
|
||||
group[0] = best
|
||||
for _, idx := range toMerge {
|
||||
group = append(group, top[idx])
|
||||
if err := selectFrom(idx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := selectFrom(bestIdx); err != nil {
|
||||
return err
|
||||
}
|
||||
resolved, err := resolve(group)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resolved != nil && resolved.name > last {
|
||||
select {
|
||||
case <-ctxDone:
|
||||
return ctx.Err()
|
||||
case out <- *resolved:
|
||||
last = resolved.name
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Merge any unmerged
|
||||
if len(toMerge) > 0 {
|
||||
versions := make([][]xlMetaV2ShallowVersion, 0, len(toMerge)+1)
|
||||
|
||||
@@ -0,0 +1,403 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"slices"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/grid"
|
||||
)
|
||||
|
||||
type listingReadFailDisk struct {
|
||||
StorageAPI
|
||||
object string
|
||||
reads *atomic.Int64
|
||||
}
|
||||
|
||||
func (d *listingReadFailDisk) ReadXL(ctx context.Context, bucket, object string, readData bool) (RawFileInfo, error) {
|
||||
if object == d.object {
|
||||
d.reads.Add(1)
|
||||
return RawFileInfo{}, errDiskNotFound
|
||||
}
|
||||
return d.StorageAPI.ReadXL(ctx, bucket, object, readData)
|
||||
}
|
||||
|
||||
// Install before issuing requests: cache writers can outlive their HTTP
|
||||
// response, so disk faults must be switched without replacing getDisks.
|
||||
// Each published slice must remain immutable.
|
||||
func installListingDiskSwitch(set *erasureObjects) ([]StorageAPI, func([]StorageAPI)) {
|
||||
disks := set.getDisks()
|
||||
var current atomic.Value
|
||||
current.Store(disks)
|
||||
set.getDisks = func() []StorageAPI { return current.Load().([]StorageAPI) }
|
||||
return disks, func(disks []StorageAPI) { current.Store(disks) }
|
||||
}
|
||||
|
||||
func TestListingCachePreservesFallbackError(t *testing.T) {
|
||||
for _, continuation := range []bool{false, true} {
|
||||
name := "cold"
|
||||
if continuation {
|
||||
name = "continuation"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testListingCacheFallbackError(t, continuation, false)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestListingCacheCleanupKeepsOriginalID(t *testing.T) {
|
||||
testListingCacheFallbackError(t, true, true)
|
||||
}
|
||||
|
||||
// Use actual peer RPCs to verify the abandoned cache is updated after the
|
||||
// caller clears its own listing ID. The local-only path cannot observe this.
|
||||
func recordListingPeerCleanup(t *testing.T) func() {
|
||||
t.Helper()
|
||||
tg, err := grid.SetupTestGrid(2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previous := globalNotificationSys
|
||||
t.Cleanup(func() {
|
||||
tg.Cleanup()
|
||||
globalNotificationSys = previous
|
||||
})
|
||||
getIDs := make(chan string, 8)
|
||||
type cleanupResult struct {
|
||||
id string
|
||||
err *grid.RemoteErr
|
||||
}
|
||||
updated := make(chan cleanupResult, 1)
|
||||
server := &peerRESTServer{}
|
||||
if err := getMetacacheListingRPC.Register(tg.Managers[1], func(o *listPathOptions) (*metacache, *grid.RemoteErr) {
|
||||
getIDs <- o.ID
|
||||
return server.GetMetacacheListingHandler(o)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := updateMetacacheListingRPC.Register(tg.Managers[1], func(m *metacache) (*metacache, *grid.RemoteErr) {
|
||||
result, err := server.UpdateMetacacheListingHandler(m)
|
||||
if m.error == "no longer used" {
|
||||
updated <- cleanupResult{id: m.id, err: err}
|
||||
}
|
||||
return result, err
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &peerRESTClient{gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[1]) }}
|
||||
globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{client}, allPeerClients: []*peerRESTClient{client}}
|
||||
return func() {
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
var ids []string
|
||||
for range 2 {
|
||||
select {
|
||||
case id := <-getIDs:
|
||||
ids = append(ids, id)
|
||||
case <-ctx.Done():
|
||||
t.Fatal("peer did not receive the failed cache cleanup lookup")
|
||||
}
|
||||
}
|
||||
if ids[0] == "" || ids[1] != ids[0] {
|
||||
t.Fatalf("cleanup lookup lost the original cache ID: %q", ids)
|
||||
}
|
||||
select {
|
||||
case result := <-updated:
|
||||
if result.err != nil || result.id != ids[0] {
|
||||
t.Fatalf("cleanup update: id=%q err=%v, want id=%q", result.id, result.err, ids[0])
|
||||
}
|
||||
t.Logf("peer cleanup Get/Update retained cache ID %q", result.id)
|
||||
case <-ctx.Done():
|
||||
t.Fatal("peer did not finish the failed cache cleanup update")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testListingCacheFallbackError(t *testing.T, continuation, peer bool) {
|
||||
t.Helper()
|
||||
z, bucket, router := nullQuorumBackend(t)
|
||||
var waitPeerCleanup func()
|
||||
if peer {
|
||||
waitPeerCleanup = recordListingPeerCleanup(t)
|
||||
}
|
||||
set := z.serverPools[0].sets[0]
|
||||
disks, switchDisks := installListingDiskSwitch(set)
|
||||
body := bytes.Repeat([]byte{'a'}, 8192)
|
||||
mtime := time.Now().UTC().Add(-time.Hour)
|
||||
for _, object := range []string{"a", "b", "z"} {
|
||||
nullQuorumPut(t, z, bucket, object, body, ObjectOptions{MTime: mtime})
|
||||
}
|
||||
old := make([][]byte, 3)
|
||||
for i := range old {
|
||||
old[i] = mustReadNullQuorumMeta(t, disks[i], bucket, "z")
|
||||
}
|
||||
nullQuorumPut(t, z, bucket, "z", body, ObjectOptions{MTime: mtime.Add(time.Minute)})
|
||||
var token string
|
||||
if continuation {
|
||||
rec := nullQuorumRequest(t, router, http.MethodGet, getListObjectsV2URL("", bucket, "", "1", "", "", ""))
|
||||
var list ListObjectsV2Response
|
||||
if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &list) != nil || !list.IsTruncated || list.NextContinuationToken == "" {
|
||||
t.Fatalf("first page: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
token = list.NextContinuationToken
|
||||
}
|
||||
for i := range old {
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, "z/"+xlStorageFormatFile, old[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var reads atomic.Int64
|
||||
walkers := slices.Clone(disks)
|
||||
for i := range walkers {
|
||||
walkers[i] = &listingReadFailDisk{StorageAPI: disks[i], object: "z", reads: &reads}
|
||||
if i >= 8 {
|
||||
walkers[i] = &failedMetadataWalkDisk{StorageAPI: walkers[i]}
|
||||
}
|
||||
}
|
||||
switchDisks(walkers)
|
||||
target := getListObjectsV2URL("", bucket, "", "1000", "", "", "")
|
||||
if token != "" {
|
||||
target += "&continuation-token=" + url.QueryEscape(token)
|
||||
}
|
||||
rec := nullQuorumRequest(t, router, http.MethodGet, target)
|
||||
t.Logf("LIST=%d failed metadata reads=%d body=%s", rec.Code, reads.Load(), rec.Body.String())
|
||||
if reads.Load() == 0 {
|
||||
t.Fatal("fallback not exercised")
|
||||
}
|
||||
var apiErr APIErrorResponse
|
||||
if rec.Code != http.StatusServiceUnavailable || xml.Unmarshal(rec.Body.Bytes(), &apiErr) != nil || apiErr.Code != "SlowDownRead" {
|
||||
t.Fatalf("fallback failure lost its read-quorum error: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if waitPeerCleanup != nil {
|
||||
waitPeerCleanup()
|
||||
}
|
||||
if continuation {
|
||||
caches, _ := localMetacacheMgr.getBucket(t.Context(), bucket).cloneCaches()
|
||||
if len(caches) == 0 {
|
||||
t.Fatal("continuation did not create a cache")
|
||||
}
|
||||
for _, cache := range caches {
|
||||
if cache.status != scanStateError {
|
||||
t.Fatalf("incomplete cache was not marked failed: %+v", cache)
|
||||
}
|
||||
}
|
||||
}
|
||||
switchDisks(disks)
|
||||
retry := nullQuorumRequest(t, router, http.MethodGet, target)
|
||||
var recovered ListObjectsV2Response
|
||||
if retry.Code != http.StatusOK || xml.Unmarshal(retry.Body.Bytes(), &recovered) != nil {
|
||||
t.Fatalf("retry after metadata recovers: %d %s", retry.Code, retry.Body.String())
|
||||
}
|
||||
var keys []string
|
||||
for _, entry := range recovered.Contents {
|
||||
keys = append(keys, entry.Key)
|
||||
}
|
||||
want := []string{"a", "b", "z"}
|
||||
if continuation {
|
||||
want = []string{"b", "z"}
|
||||
}
|
||||
if !slices.Equal(keys, want) || recovered.IsTruncated {
|
||||
t.Fatalf("retry returned incomplete result: %+v", recovered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListingCachePagination(t *testing.T) {
|
||||
z, bucket, router := nullQuorumBackend(t)
|
||||
for _, key := range []string{"a", "b", "c", "d"} {
|
||||
nullQuorumPut(t, z, bucket, key, []byte(key), ObjectOptions{})
|
||||
}
|
||||
var keys []string
|
||||
var token string
|
||||
var finished bool
|
||||
for page := 0; page < 8; page++ {
|
||||
target := getListObjectsV2URL("", bucket, "", "1", "", "", "")
|
||||
if token != "" {
|
||||
target += "&continuation-token=" + url.QueryEscape(token)
|
||||
}
|
||||
rec := nullQuorumRequest(t, router, http.MethodGet, target)
|
||||
var list ListObjectsV2Response
|
||||
if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &list) != nil {
|
||||
t.Fatalf("page %d: %d %s", page, rec.Code, rec.Body.String())
|
||||
}
|
||||
for _, entry := range list.Contents {
|
||||
keys = append(keys, entry.Key)
|
||||
}
|
||||
if !list.IsTruncated {
|
||||
finished = true
|
||||
break
|
||||
}
|
||||
if list.NextContinuationToken == "" || list.NextContinuationToken == token {
|
||||
t.Fatal("pagination did not advance")
|
||||
}
|
||||
token = list.NextContinuationToken
|
||||
}
|
||||
if !finished || !slices.Equal(keys, []string{"a", "b", "c", "d"}) {
|
||||
t.Fatalf("pagination returned %v, finished=%v", keys, finished)
|
||||
}
|
||||
}
|
||||
|
||||
// A cached full page must remain streamable even if a later key is slow. The
|
||||
// completion barrier belongs at EOF, not before every page response.
|
||||
func TestListingCacheFullPageDoesNotWaitForTail(t *testing.T) {
|
||||
z, bucket, router := nullQuorumBackend(t)
|
||||
disks, switchDisks := installListingDiskSwitch(z.serverPools[0].sets[0])
|
||||
for _, key := range []string{"a", "b", "c", "d", "e", "zz"} {
|
||||
nullQuorumPut(t, z, bucket, key, []byte(key), ObjectOptions{})
|
||||
}
|
||||
first := nullQuorumRequest(t, router, http.MethodGet, getListObjectsV2URL("", bucket, "", "1", "", "", ""))
|
||||
var page ListObjectsV2Response
|
||||
if first.Code != http.StatusOK || xml.Unmarshal(first.Body.Bytes(), &page) != nil || !page.IsTruncated {
|
||||
t.Fatalf("first page: %d %s", first.Code, first.Body.String())
|
||||
}
|
||||
resume := make(chan struct{})
|
||||
release := sync.OnceFunc(func() { close(resume) })
|
||||
defer release()
|
||||
var blocked, active atomic.Int64
|
||||
walkers := slices.Clone(disks)
|
||||
for i, disk := range walkers {
|
||||
walkers[i] = &listingPausedTailDisk{StorageAPI: disk, bucket: bucket, tail: "zz", resume: resume, blocked: &blocked, active: &active}
|
||||
}
|
||||
switchDisks(walkers)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
target := getListObjectsV2URL("", bucket, "", "1", "", "", "") + "&continuation-token=" + url.QueryEscape(page.NextContinuationToken)
|
||||
req, err := newTestSignedRequestV4(http.MethodGet, target, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response := make(chan *httptest.ResponseRecorder, 1)
|
||||
go func() {
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req.WithContext(ctx))
|
||||
response <- rec
|
||||
}()
|
||||
select {
|
||||
case rec := <-response:
|
||||
page = ListObjectsV2Response{}
|
||||
if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &page) != nil || !page.IsTruncated || len(page.Contents) != 1 || page.Contents[0].Key != "b" || page.NextContinuationToken == "" {
|
||||
t.Fatalf("full page before tail: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if blocked.Load() == 0 || active.Load() == 0 {
|
||||
t.Fatal("full page did not overlap the paused tail")
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("full page waited for the paused tail")
|
||||
}
|
||||
release()
|
||||
waitForListingCondition(t, "disk walkers did not finish", func() bool { return active.Load() == 0 })
|
||||
target = getListObjectsV2URL("", bucket, "", "1000", "", "", "") + "&continuation-token=" + url.QueryEscape(page.NextContinuationToken)
|
||||
rec := nullQuorumRequest(t, router, http.MethodGet, target)
|
||||
page = ListObjectsV2Response{}
|
||||
if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &page) != nil || page.IsTruncated {
|
||||
t.Fatalf("page after tail resumes: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var keys []string
|
||||
for _, entry := range page.Contents {
|
||||
keys = append(keys, entry.Key)
|
||||
}
|
||||
if !slices.Equal(keys, []string{"c", "d", "e", "zz"}) {
|
||||
t.Fatalf("cached remainder lost keys: %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the error update pending at the actual peer RPC boundary. Other cache
|
||||
// operations, including readers and abandoned-cache cleanup, remain live.
|
||||
func listingCacheErrorPeer(t *testing.T, fail bool) (*peerRESTClient, <-chan struct{}, func()) {
|
||||
t.Helper()
|
||||
tg, err := grid.SetupTestGrid(2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ready, resume := make(chan struct{}), make(chan struct{})
|
||||
release := sync.OnceFunc(func() { close(resume) })
|
||||
t.Cleanup(func() {
|
||||
release()
|
||||
tg.Cleanup()
|
||||
})
|
||||
var entered sync.Once
|
||||
server := &peerRESTServer{}
|
||||
if err := getMetacacheListingRPC.Register(tg.Managers[1], server.GetMetacacheListingHandler); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := updateMetacacheListingRPC.Register(tg.Managers[1], func(m *metacache) (*metacache, *grid.RemoteErr) {
|
||||
if m.error != "" && m.error != "no longer used" {
|
||||
entered.Do(func() { close(ready) })
|
||||
select {
|
||||
case <-resume:
|
||||
case <-t.Context().Done():
|
||||
return nil, grid.NewRemoteErr(t.Context().Err())
|
||||
}
|
||||
if fail {
|
||||
return nil, grid.NewRemoteErr(errors.New("cache error update failed"))
|
||||
}
|
||||
}
|
||||
return server.UpdateMetacacheListingHandler(m)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &peerRESTClient{gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[1]) }}, ready, release
|
||||
}
|
||||
|
||||
func TestListingCacheRetainsErrorAfterPeerFailure(t *testing.T) {
|
||||
client, ready, release := listingCacheErrorPeer(t, true)
|
||||
o := listPathOptions{ID: mustGetUUID(), Bucket: "bucket"}
|
||||
m := o.newMetacache()
|
||||
mc := metaCacheRPC{meta: &m, rpc: client, o: o}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
mc.setErr("terminal listing failure")
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
release()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Error("setErr worker did not finish before grid cleanup")
|
||||
}
|
||||
})
|
||||
select {
|
||||
case <-ready:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("error update did not reach peer")
|
||||
}
|
||||
release()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("failed peer update did not return")
|
||||
}
|
||||
if m.id != o.ID || m.bucket != o.Bucket || m.status != scanStateError || m.error != "terminal listing failure" || mc.getErr() == nil {
|
||||
t.Fatalf("peer failure erased local failure: %+v", m)
|
||||
}
|
||||
mc.setErr("later failure")
|
||||
if m.error != "terminal listing failure" {
|
||||
t.Fatalf("later failure replaced the terminal error: %+v", m)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestListFallbackPreservesPoolDeleteMarker(t *testing.T) {
|
||||
for _, markerPool := range []int{0, 1} {
|
||||
t.Run(fmt.Sprint(markerPool), func(t *testing.T) {
|
||||
z, bucket, router := listingPoolsBackend(t)
|
||||
ctx := t.Context()
|
||||
live, deleted := z.serverPools[1-markerPool].sets[0], z.serverPools[markerPool].sets[0]
|
||||
disks, switchDisks := installListingDiskSwitch(deleted)
|
||||
const object = "gone"
|
||||
body := bytes.Repeat([]byte{'a'}, 8192)
|
||||
oldTime := time.Now().UTC().Add(-time.Hour)
|
||||
for _, set := range []*erasureObjects{live, deleted} {
|
||||
_, err := set.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{MTime: oldTime, VersionSuspended: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
old := make([][]byte, 3)
|
||||
for i := range old {
|
||||
old[i] = mustReadNullQuorumMeta(t, disks[i], bucket, object)
|
||||
}
|
||||
_, err := deleted.DeleteObject(ctx, bucket, object, ObjectOptions{VersionSuspended: true, DeleteMarker: true, MTime: oldTime.Add(time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
get := nullQuorumRequest(t, router, http.MethodGet, getGetObjectURL("", bucket, object))
|
||||
if get.Code != http.StatusNotFound {
|
||||
t.Fatalf("expected deleted GET, got %d %s", get.Code, get.Body.String())
|
||||
}
|
||||
first := nullQuorumRequest(t, router, http.MethodGet, getListObjectsV2URL("", bucket, "", "1000", "", "", ""))
|
||||
var page ListObjectsV2Response
|
||||
if first.Code != http.StatusOK || xml.Unmarshal(first.Body.Bytes(), &page) != nil || len(page.Contents) != 0 {
|
||||
t.Fatalf("normal LIST marker: %d %s", first.Code, first.Body.String())
|
||||
}
|
||||
for i := range old {
|
||||
if err := disks[i].WriteAll(ctx, bucket, object+"/"+xlStorageFormatFile, old[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
walkers := slices.Clone(disks)
|
||||
for i := 8; i < len(walkers); i++ {
|
||||
walkers[i] = &failedMetadataWalkDisk{StorageAPI: disks[i]}
|
||||
}
|
||||
switchDisks(walkers)
|
||||
fi, _, _, err := deleted.getObjectFileInfo(ctx, bucket, object, ObjectOptions{}, false)
|
||||
if err != nil || !fi.Deleted {
|
||||
t.Fatalf("read fallback marker: %+v %v", fi, err)
|
||||
}
|
||||
rec := nullQuorumRequest(t, router, http.MethodGet, getListObjectsV2URL("", bucket, "", "1000", "", "", ""))
|
||||
page = ListObjectsV2Response{}
|
||||
if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &page) != nil || len(page.Contents) != 0 {
|
||||
t.Fatalf("fallback resurrected old pool copy: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The regular metadata reader can translate a persistent generation split to
|
||||
// ObjectNotFound even though every drive still has valid metadata. LIST must
|
||||
// preserve that uncertainty instead of treating the key as deleted.
|
||||
func TestListFallbackRejectsSplitGenerations(t *testing.T) {
|
||||
z, bucket, _ := nullQuorumBackend(t)
|
||||
set := z.serverPools[0].sets[0]
|
||||
disks := set.getDisks()
|
||||
const object = "object"
|
||||
mtime := time.Now().UTC().Add(-time.Hour)
|
||||
var generations [3][][]byte
|
||||
for generation := range generations {
|
||||
body := bytes.Repeat([]byte{byte('a' + generation)}, 8192)
|
||||
nullQuorumPut(t, z, bucket, object, body, ObjectOptions{MTime: mtime.Add(time.Duration(generation) * time.Minute)})
|
||||
generations[generation] = make([][]byte, len(disks))
|
||||
for i, disk := range disks {
|
||||
generations[generation][i] = mustReadNullQuorumMeta(t, disk, bucket, object)
|
||||
}
|
||||
}
|
||||
// All sixteen drives are online, but the 7/5/4 split leaves no generation
|
||||
// with the eight matching snapshots required by the listing resolver.
|
||||
for i, disk := range disks {
|
||||
generation := 0
|
||||
if i >= 7 {
|
||||
generation = 1
|
||||
}
|
||||
if i >= 12 {
|
||||
generation = 2
|
||||
}
|
||||
if err := disk.WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, generations[generation][i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, _, _, err := set.getObjectFileInfo(t.Context(), bucket, object, ObjectOptions{}, false); !isErrObjectNotFound(err) {
|
||||
t.Fatalf("split metadata did not exercise ambiguous not-found: %v", err)
|
||||
}
|
||||
entries := make(metaCacheEntries, len(disks))
|
||||
for i, disk := range disks {
|
||||
entries[i] = metaCacheEntry{name: object, metadata: mustReadNullQuorumMeta(t, disk, bucket, object)}
|
||||
if _, err := entries[i].xlmeta(); err != nil {
|
||||
t.Fatalf("drive %d no longer has valid metadata: %v", i, err)
|
||||
}
|
||||
}
|
||||
resolver := metadataResolutionParams{dirQuorum: 8, objQuorum: 8, bucket: bucket, requestedVersions: 1}
|
||||
entry, err := z.resolveListingGroup(t.Context(), bucket, []*metaCacheEntry{set.resolveListEntry(entries, nil, &resolver)})
|
||||
if entry != nil || !isErrReadQuorum(err) {
|
||||
t.Fatalf("split live metadata became absence: entry=%v err=%v", entry, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Exercise the actual LIST pipeline with the disputed entry in pool 1.
|
||||
func TestListFallbackMultiPoolHTTPReadLock(t *testing.T) {
|
||||
for _, object := range []string{"object", "dir/"} {
|
||||
t.Run(object, func(t *testing.T) {
|
||||
z, bucket, router := listingPoolsBackend(t)
|
||||
ctx := t.Context()
|
||||
set := z.serverPools[1].sets[0]
|
||||
disks := set.getDisks()
|
||||
encoded := encodeDirObject(object)
|
||||
body := bytes.Repeat([]byte{'a'}, 8192)
|
||||
if object == "dir/" {
|
||||
body = nil
|
||||
}
|
||||
mtime := time.Now().UTC().Add(-time.Hour)
|
||||
put := func(tm time.Time) {
|
||||
_, err := set.PutObject(ctx, bucket, encoded, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{MTime: tm})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
put(mtime)
|
||||
old := make([][]byte, 3)
|
||||
for i := range old {
|
||||
old[i] = mustReadNullQuorumMeta(t, disks[i], bucket, encoded)
|
||||
}
|
||||
put(mtime.Add(time.Minute))
|
||||
for i := range old {
|
||||
if err := disks[i].WriteAll(ctx, bucket, encoded+"/"+xlStorageFormatFile, old[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
walkers := slices.Clone(disks)
|
||||
for i := 8; i < len(walkers); i++ {
|
||||
walkers[i] = &failedMetadataWalkDisk{StorageAPI: disks[i]}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return walkers }
|
||||
lock := z.NewNSLock(bucket, encoded)
|
||||
lkctx, err := lock.GetLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
locked := true
|
||||
defer func() {
|
||||
if locked {
|
||||
lock.Unlock(lkctx)
|
||||
}
|
||||
}()
|
||||
target := getListObjectsV2URL("", bucket, "", "1000", "", "", "")
|
||||
req, err := newTestSignedRequestV4(http.MethodGet, target, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
router.ServeHTTP(rec, req.WithContext(ctx))
|
||||
}()
|
||||
waitForListingReadLock(t, z.serverPools[0].sets[0].nsMutex, bucket, encoded, done)
|
||||
lock.Unlock(lkctx)
|
||||
locked = false
|
||||
<-done
|
||||
var list ListObjectsV2Response
|
||||
if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &list) != nil {
|
||||
t.Fatalf("LIST HTTP=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if len(list.Contents) != 1 || list.Contents[0].Key != object {
|
||||
t.Fatalf("LIST lost key: %+v", list)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
)
|
||||
|
||||
// Use the real distributed-lock algorithm with in-process lock servers.
|
||||
// This is a protocol-level check, not a network or rolling-restart test.
|
||||
func TestListFallbackDistributedLockRoute(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
multiPool, sharedLockers bool
|
||||
}{
|
||||
{"multi-pool-shared-lockers", true, true},
|
||||
{"multi-pool-disjoint-lockers", true, false},
|
||||
{"single-pool-target-set-lockers", false, false},
|
||||
{"single-pool-first-set-lockers", false, false},
|
||||
{"single-pool-shared-ns-distinct-lockers", false, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
a := []dsync.NetLocker{newLocker(), newLocker()}
|
||||
b := []dsync.NetLocker{newLocker(), newLocker()}
|
||||
if tc.sharedLockers {
|
||||
b = a
|
||||
}
|
||||
set0 := &erasureObjects{nsMutex: newNSLock(true), getLockers: func() ([]dsync.NetLocker, string) { return a, "listing-test" }}
|
||||
target := &erasureObjects{nsMutex: newNSLock(true), getLockers: func() ([]dsync.NetLocker, string) { return b, "listing-test" }}
|
||||
if tc.name == "single-pool-shared-ns-distinct-lockers" {
|
||||
target.nsMutex = set0.nsMutex
|
||||
}
|
||||
z := &erasureServerPools{serverPools: []*erasureSets{{sets: []*erasureObjects{set0, target}}}}
|
||||
if tc.multiPool {
|
||||
z.serverPools = []*erasureSets{{sets: []*erasureObjects{set0}}, {sets: []*erasureObjects{target}}}
|
||||
}
|
||||
writer := target.NewNSLock("bucket", "object")
|
||||
if tc.multiPool || tc.name == "single-pool-first-set-lockers" {
|
||||
writer = z.NewNSLock("bucket", "object")
|
||||
}
|
||||
wctx, err := writer.GetLock(t.Context(), globalOperationTimeout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
locked := true
|
||||
defer func() {
|
||||
if locked {
|
||||
writer.Unlock(wctx)
|
||||
}
|
||||
}()
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 80*time.Millisecond)
|
||||
defer cancel()
|
||||
_, unlock, err := z.listingRLock(ctx, "bucket", "object", target)
|
||||
if err == nil {
|
||||
unlock()
|
||||
t.Fatal("fallback read lock acquired while ordinary writer lock is held")
|
||||
}
|
||||
writer.Unlock(wctx)
|
||||
locked = false
|
||||
ctx, cancel = context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
_, unlock, err = z.listingRLock(ctx, "bucket", "object", target)
|
||||
if err != nil {
|
||||
t.Fatalf("fallback read lock failed after writer unlock: %v", err)
|
||||
}
|
||||
unlock()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Cancellation while waiting for the second domain must release the first.
|
||||
func TestListingSecondLockCancellationReleasesFirst(t *testing.T) {
|
||||
first := &erasureObjects{nsMutex: newNSLock(false)}
|
||||
target := &erasureObjects{nsMutex: newNSLock(false)}
|
||||
z := &erasureServerPools{serverPools: []*erasureSets{{sets: []*erasureObjects{first, target}}}}
|
||||
writer := target.NewNSLock("bucket", "object")
|
||||
wctx, err := writer.GetLock(t.Context(), globalOperationTimeout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer writer.Unlock(wctx)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
done := make(chan struct{})
|
||||
var lockErr error
|
||||
go func() {
|
||||
defer close(done)
|
||||
_, unlock, err := z.listingRLock(ctx, "bucket", "object", target)
|
||||
lockErr = err
|
||||
if err == nil {
|
||||
unlock()
|
||||
}
|
||||
}()
|
||||
waitForListingReadLock(t, target.nsMutex, "bucket", "object", done)
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("second lock did not cancel")
|
||||
}
|
||||
if lockErr == nil {
|
||||
t.Fatal("canceled second lock succeeded")
|
||||
}
|
||||
poolWriter := z.NewNSLock("bucket", "object")
|
||||
ctx, cancel = context.WithTimeout(t.Context(), time.Second)
|
||||
defer cancel()
|
||||
pctx, err := poolWriter.GetLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
t.Fatalf("first lock leaked after second failed: %v", err)
|
||||
}
|
||||
poolWriter.Unlock(pctx)
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The fallback on pool 1 must contend with the
|
||||
// pools-layer lock used by ordinary multi-pool writers (anchored at pool 0).
|
||||
func TestListFallbackMultiPoolReadLock(t *testing.T) {
|
||||
z, bucket, _ := listingPoolsBackend(t)
|
||||
ctx := t.Context()
|
||||
set := z.serverPools[1].sets[0]
|
||||
disks := set.getDisks()
|
||||
const object = "object"
|
||||
body := bytes.Repeat([]byte{'a'}, 8192)
|
||||
mtime := time.Now().UTC().Add(-time.Hour)
|
||||
put := func(tm time.Time) {
|
||||
_, err := set.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{MTime: tm})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
put(mtime)
|
||||
old := make([][]byte, len(disks))
|
||||
for i := range old {
|
||||
old[i] = mustReadNullQuorumMeta(t, disks[i], bucket, object)
|
||||
}
|
||||
put(mtime.Add(time.Minute))
|
||||
entries := make(metaCacheEntries, len(disks))
|
||||
errs := make([]error, len(disks))
|
||||
for i := range 3 {
|
||||
entries[i] = metaCacheEntry{name: object, metadata: old[i]}
|
||||
}
|
||||
for i := 3; i < 8; i++ {
|
||||
entries[i] = metaCacheEntry{name: object, metadata: mustReadNullQuorumMeta(t, disks[i], bucket, object)}
|
||||
}
|
||||
for i := 8; i < len(disks); i++ {
|
||||
errs[i] = errDiskNotFound
|
||||
}
|
||||
resolver := metadataResolutionParams{dirQuorum: 8, objQuorum: 8, bucket: bucket, requestedVersions: 1}
|
||||
lock := z.NewNSLock(bucket, object)
|
||||
lkctx, err := lock.GetLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
locked := true
|
||||
defer func() {
|
||||
if locked {
|
||||
lock.Unlock(lkctx)
|
||||
}
|
||||
}()
|
||||
done := make(chan struct{})
|
||||
var entry *metaCacheEntry
|
||||
var readErr error
|
||||
go func() {
|
||||
defer close(done)
|
||||
entry, readErr = z.resolveListingGroup(ctx, bucket, []*metaCacheEntry{set.resolveListEntry(entries, errs, &resolver)})
|
||||
}()
|
||||
waitForListingReadLock(t, z.serverPools[0].sets[0].nsMutex, bucket, object, done)
|
||||
lock.Unlock(lkctx)
|
||||
locked = false
|
||||
<-done
|
||||
if readErr != nil || entry == nil || entry.name != object {
|
||||
t.Fatalf("fallback after writer unlock: entry=%v err=%v", entry, readErr)
|
||||
}
|
||||
}
|
||||
|
||||
// Observe the reader registered on the writer's actual namespace lock before
|
||||
// releasing the writer. This tests mutual exclusion without a timed sleep.
|
||||
func waitForListingReadLock(t *testing.T, mutex *nsLockMap, bucket, object string, done <-chan struct{}) {
|
||||
t.Helper()
|
||||
timer := time.NewTimer(5 * time.Second)
|
||||
defer timer.Stop()
|
||||
tick := time.NewTicker(time.Millisecond)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
mutex.lockMapMutex.Lock()
|
||||
lock := mutex.lockMap[pathJoin(bucket, object)]
|
||||
waiting := lock != nil && lock.ref > 1
|
||||
mutex.lockMapMutex.Unlock()
|
||||
if waiting {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("listing completed while the writer held its namespace lock")
|
||||
case <-timer.C:
|
||||
t.Fatal("listing did not wait on the writer's namespace lock")
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func listingPoolsBackend(t *testing.T) (*erasureServerPools, string, http.Handler) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
obj, dirs, err := prepareErasurePoolsWithContext(ctx)
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
previous := newObjectLayerFn()
|
||||
setObjectLayer(z)
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
z.Shutdown(context.Background())
|
||||
removeRoots(dirs)
|
||||
setObjectLayer(previous)
|
||||
})
|
||||
bucket, router, err := initAPIHandlerTest(ctx, z, nil, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return z, bucket, router
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"slices"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/s2"
|
||||
"github.com/tinylib/msgp/msgp"
|
||||
)
|
||||
|
||||
// Replay real disk-walk metadata in its existing wire format, but pause before
|
||||
// the final key. This keeps the healthy pool's listing alive after the other
|
||||
// pool has already failed, without forging entries or listing errors.
|
||||
type listingPausedTailDisk struct {
|
||||
StorageAPI
|
||||
bucket string
|
||||
tail string
|
||||
resume <-chan struct{}
|
||||
blocked *atomic.Int64
|
||||
active *atomic.Int64
|
||||
}
|
||||
|
||||
func (d *listingPausedTailDisk) WalkDir(ctx context.Context, opts WalkDirOptions, out io.Writer) (result error) {
|
||||
if opts.Bucket != d.bucket {
|
||||
return d.StorageAPI.WalkDir(ctx, opts, out)
|
||||
}
|
||||
if d.active != nil {
|
||||
d.active.Add(1)
|
||||
defer d.active.Add(-1)
|
||||
}
|
||||
var source bytes.Buffer
|
||||
if err := d.StorageAPI.WalkDir(ctx, opts, &source); err != nil {
|
||||
return err
|
||||
}
|
||||
r := newMetacacheReader(bytes.NewReader(source.Bytes()))
|
||||
defer r.Close()
|
||||
compressed := s2.NewWriter(out)
|
||||
defer func() {
|
||||
if err := compressed.Close(); result == nil {
|
||||
result = err
|
||||
}
|
||||
}()
|
||||
w := msgp.NewWriter(compressed)
|
||||
if err := w.WriteByte(metacacheStreamVersion); err != nil {
|
||||
return err
|
||||
}
|
||||
for {
|
||||
entry, err := r.next()
|
||||
if err == io.EOF {
|
||||
if err = w.WriteBool(false); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.name == d.tail {
|
||||
if err = w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = compressed.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
d.blocked.Add(1)
|
||||
select {
|
||||
case <-d.resume:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
if err = w.WriteBool(true); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = w.WriteString(entry.name); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = w.WriteBytes(entry.metadata); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListingMergeDoesNotSkipFailedPool(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second)
|
||||
defer cancel()
|
||||
z, bucket, router := listingPoolsBackend(t)
|
||||
broken, healthy := z.serverPools[0].sets[0], z.serverPools[1].sets[0]
|
||||
brokenDisks, switchBrokenDisks := installListingDiskSwitch(broken)
|
||||
healthyDisks, switchHealthyDisks := installListingDiskSwitch(healthy)
|
||||
body := bytes.Repeat([]byte{'a'}, 8192)
|
||||
mtime := time.Now().UTC().Add(-time.Hour)
|
||||
put := func(set *erasureObjects, name string, tm time.Time) {
|
||||
_, err := set.PutObject(ctx, bucket, name, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{MTime: tm})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
put(broken, "a", mtime)
|
||||
put(broken, "e", mtime)
|
||||
for _, name := range []string{"f", "g", "h", "i", "j", "k", "zz"} {
|
||||
put(healthy, name, mtime)
|
||||
}
|
||||
first := nullQuorumRequest(t, router, http.MethodGet, getListObjectsV2URL("", bucket, "", "1", "", "", ""))
|
||||
var page ListObjectsV2Response
|
||||
if first.Code != http.StatusOK || xml.Unmarshal(first.Body.Bytes(), &page) != nil || !page.IsTruncated || page.NextContinuationToken == "" || len(page.Contents) != 1 || page.Contents[0].Key != "a" {
|
||||
t.Fatalf("first page HTTP=%d body=%s", first.Code, first.Body.String())
|
||||
}
|
||||
firstToken := page.NextContinuationToken
|
||||
old := make([][]byte, 3)
|
||||
for i := range old {
|
||||
old[i] = mustReadNullQuorumMeta(t, brokenDisks[i], bucket, "e")
|
||||
}
|
||||
put(broken, "e", mtime.Add(time.Minute))
|
||||
for i := range old {
|
||||
if err := brokenDisks[i].WriteAll(ctx, bucket, "e/"+xlStorageFormatFile, old[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var reads, blocked, active atomic.Int64
|
||||
brokenWalkers := slices.Clone(brokenDisks)
|
||||
for i := range brokenWalkers {
|
||||
brokenWalkers[i] = &listingReadFailDisk{StorageAPI: brokenDisks[i], object: "e", reads: &reads}
|
||||
if i >= 8 {
|
||||
brokenWalkers[i] = &failedMetadataWalkDisk{StorageAPI: brokenWalkers[i]}
|
||||
}
|
||||
}
|
||||
switchBrokenDisks(brokenWalkers)
|
||||
resume := make(chan struct{})
|
||||
defer close(resume)
|
||||
healthyWalkers := slices.Clone(healthyDisks)
|
||||
for i, disk := range healthyWalkers {
|
||||
healthyWalkers[i] = &listingPausedTailDisk{StorageAPI: disk, bucket: bucket, tail: "zz", resume: resume, blocked: &blocked, active: &active}
|
||||
}
|
||||
switchHealthyDisks(healthyWalkers)
|
||||
|
||||
target := getListObjectsV2URL("", bucket, "", "2", "", "", "") + "&continuation-token=" + url.QueryEscape(firstToken)
|
||||
req, err := newTestSignedRequestV4(http.MethodGet, target, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response := make(chan *httptest.ResponseRecorder, 1)
|
||||
go func() {
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req.WithContext(ctx))
|
||||
response <- rec
|
||||
}()
|
||||
select {
|
||||
case rec := <-response:
|
||||
t.Logf("continuation LIST=%d failed fallback ReadXL=%d paused healthy walkers=%d body=%s", rec.Code, reads.Load(), blocked.Load(), rec.Body.String())
|
||||
if reads.Load() == 0 || blocked.Load() == 0 {
|
||||
t.Fatal("schedule did not exercise fallback failure and unfinished healthy pool")
|
||||
}
|
||||
var apiErr APIErrorResponse
|
||||
if rec.Code != http.StatusServiceUnavailable || xml.Unmarshal(rec.Body.Bytes(), &apiErr) != nil || apiErr.Code != "SlowDownRead" {
|
||||
t.Fatalf("failed pool advanced the page instead of reporting read quorum: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
waitForListingCondition(t, "canceled disk walkers did not exit", func() bool { return active.Load() == 0 })
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("no early response: %v reads=%d blocked=%d", ctx.Err(), reads.Load(), blocked.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestListingMergeCancellation(t *testing.T) {
|
||||
z, bucket, _ := nullQuorumBackend(t)
|
||||
for _, key := range []string{"a", "b", "zz"} {
|
||||
nullQuorumPut(t, z, bucket, key, []byte(key), ObjectOptions{})
|
||||
}
|
||||
set := z.serverPools[0].sets[0]
|
||||
var blocked, active atomic.Int64
|
||||
resume := make(chan struct{})
|
||||
defer close(resume)
|
||||
walkers := slices.Clone(set.getDisks())
|
||||
for i, disk := range walkers {
|
||||
walkers[i] = &listingPausedTailDisk{StorageAPI: disk, bucket: bucket, tail: "zz", resume: resume, blocked: &blocked, active: &active}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return walkers }
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
entries := make(chan metaCacheEntry, 10)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- z.listMerged(ctx, listPathOptions{Bucket: bucket, Recursive: true, Limit: 10}, entries)
|
||||
}()
|
||||
waitForListingCondition(t, "disk walks did not reach the paused tail", func() bool { return blocked.Load() > 0 })
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("canceled merge returned %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("canceled merge did not finish")
|
||||
}
|
||||
// listMerged must close its output and release every paused disk walk.
|
||||
for range entries {
|
||||
}
|
||||
waitForListingCondition(t, "canceled disk walkers did not exit", func() bool { return active.Load() == 0 })
|
||||
}
|
||||
|
||||
func waitForListingCondition(t *testing.T, failure string, ready func() bool) {
|
||||
t.Helper()
|
||||
timer := time.NewTimer(5 * time.Second)
|
||||
defer timer.Stop()
|
||||
tick := time.NewTicker(time.Millisecond)
|
||||
defer tick.Stop()
|
||||
for !ready() {
|
||||
select {
|
||||
case <-timer.C:
|
||||
t.Fatal(failure)
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,391 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"slices"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
type listingRecoveryDisk struct {
|
||||
StorageAPI
|
||||
reads *atomic.Int64
|
||||
err error
|
||||
emptySuccess bool
|
||||
}
|
||||
|
||||
func (d *listingRecoveryDisk) ReadXL(ctx context.Context, bucket, object string, data bool) (RawFileInfo, error) {
|
||||
if d.reads != nil {
|
||||
d.reads.Add(1)
|
||||
}
|
||||
if d.emptySuccess {
|
||||
return RawFileInfo{}, nil
|
||||
}
|
||||
if d.err != nil {
|
||||
return RawFileInfo{}, d.err
|
||||
}
|
||||
return d.StorageAPI.ReadXL(ctx, bucket, object, data)
|
||||
}
|
||||
|
||||
func TestListingRecoveryStrictAbsence(t *testing.T) {
|
||||
z, bucket, _ := nullQuorumBackend(t)
|
||||
set := z.serverPools[0].sets[0]
|
||||
disks, switchDisks := installListingDiskSwitch(set)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
err error
|
||||
offline, empty, absent bool
|
||||
}{
|
||||
{name: "all-native-not-found", absent: true},
|
||||
{name: "offline", offline: true},
|
||||
{name: "volume-not-found", err: errVolumeNotFound},
|
||||
{name: "version-not-found", err: errFileVersionNotFound},
|
||||
{name: "corrupt", err: errFileCorrupt},
|
||||
{name: "decode-generated-not-found", empty: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
next := slices.Clone(disks)
|
||||
next[0] = &listingRecoveryDisk{StorageAPI: disks[0], err: tc.err, emptySuccess: tc.empty}
|
||||
if tc.offline {
|
||||
next[0] = &offlineNullQuorumDisk{StorageAPI: disks[0]}
|
||||
}
|
||||
switchDisks(next)
|
||||
_, absent, err := set.readListingFileInfo(t.Context(), bucket, "missing")
|
||||
if absent != tc.absent || (tc.absent && err != nil) || (!tc.absent && !isErrReadQuorum(err)) {
|
||||
t.Fatalf("absent=%v err=%v, want absent=%v", absent, err, tc.absent)
|
||||
}
|
||||
})
|
||||
}
|
||||
switchDisks(disks)
|
||||
}
|
||||
|
||||
// The target walk can predate the copy. Only current cross-pool reads can
|
||||
// distinguish this copy-before-delete schedule from a genuine object deletion.
|
||||
func TestListingRecoveryCurrentPools(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
target, marker, unknown, stale bool
|
||||
}{
|
||||
{name: "copied-before-source-delete", target: true},
|
||||
{name: "deleted-with-stale-other-walk", stale: true},
|
||||
{name: "current-marker-over-stale-live", target: true, marker: true, stale: true},
|
||||
{name: "missing-source-unknown-target", unknown: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
z, bucket, _ := listingPoolsBackend(t)
|
||||
target, source := z.serverPools[0].sets[0], z.serverPools[1].sets[0]
|
||||
targetDisks, switchTarget := installListingDiskSwitch(target)
|
||||
sourceDisks, switchSource := installListingDiskSwitch(source)
|
||||
const object = "moved"
|
||||
body := bytes.Repeat([]byte{'a'}, 8192)
|
||||
mtime := time.Now().UTC().Add(-time.Hour)
|
||||
put := func(set *erasureObjects) {
|
||||
_, err := set.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{MTime: mtime, VersionSuspended: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
put(source)
|
||||
var stale *metaCacheEntry
|
||||
if tc.stale || tc.target {
|
||||
put(target)
|
||||
stale = &metaCacheEntry{name: object, metadata: mustReadNullQuorumMeta(t, targetDisks[0], bucket, object)}
|
||||
}
|
||||
if tc.stale && !tc.target {
|
||||
if _, err := target.DeleteObject(t.Context(), bucket, object, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if tc.marker {
|
||||
if _, err := target.DeleteObject(t.Context(), bucket, object, ObjectOptions{VersionSuspended: true, DeleteMarker: true, MTime: mtime.Add(time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Actual copy is committed before actual source cleanup.
|
||||
if _, err := source.DeleteObject(t.Context(), bucket, object, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var sourceReads, targetReads atomic.Int64
|
||||
nextSource, nextTarget := slices.Clone(sourceDisks), slices.Clone(targetDisks)
|
||||
for i := range nextSource {
|
||||
nextSource[i] = &listingRecoveryDisk{StorageAPI: sourceDisks[i], reads: &sourceReads}
|
||||
}
|
||||
for i := range nextTarget {
|
||||
nextTarget[i] = &listingRecoveryDisk{StorageAPI: targetDisks[i], reads: &targetReads}
|
||||
}
|
||||
if tc.unknown {
|
||||
nextTarget[0] = &listingRecoveryDisk{StorageAPI: targetDisks[0], reads: &targetReads, err: errDiskNotFound}
|
||||
}
|
||||
switchSource(nextSource)
|
||||
switchTarget(nextTarget)
|
||||
group := []*metaCacheEntry{{name: object, listingSet: source}}
|
||||
if stale != nil && tc.stale {
|
||||
group = append(group, stale)
|
||||
}
|
||||
entry, err := z.resolveListingGroup(t.Context(), bucket, group)
|
||||
if sourceReads.Load() != int64(len(sourceDisks)) || targetReads.Load() != int64(len(targetDisks)) {
|
||||
t.Fatalf("reads source=%d target=%d", sourceReads.Load(), targetReads.Load())
|
||||
}
|
||||
if tc.unknown {
|
||||
if entry != nil || !isErrReadQuorum(err) {
|
||||
t.Fatalf("unknown became success: entry=%v err=%v", entry, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !tc.target {
|
||||
if entry != nil {
|
||||
t.Fatalf("stale walk survived current absence: %+v", entry)
|
||||
}
|
||||
return
|
||||
}
|
||||
if entry == nil || entry.listingSet != nil {
|
||||
t.Fatalf("missing current result: %+v", entry)
|
||||
}
|
||||
fi, err := entry.fileInfo(bucket)
|
||||
if err != nil || fi.Deleted != tc.marker {
|
||||
t.Fatalf("current result deleted=%v err=%v, want marker=%v", fi.Deleted, err, tc.marker)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestListingRecoveryNormalPathDoesNotReadOrLock(t *testing.T) {
|
||||
z, bucket, _ := listingPoolsBackend(t)
|
||||
const object = "normal"
|
||||
set := z.serverPools[1].sets[0]
|
||||
body := []byte("normal data")
|
||||
if _, err := set.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var reads atomic.Int64
|
||||
for _, pool := range z.serverPools {
|
||||
for _, set := range pool.sets {
|
||||
disks, switchDisks := installListingDiskSwitch(set)
|
||||
wrapped := slices.Clone(disks)
|
||||
for i := range wrapped {
|
||||
wrapped[i] = &listingRecoveryDisk{StorageAPI: disks[i], reads: &reads}
|
||||
}
|
||||
switchDisks(wrapped)
|
||||
}
|
||||
}
|
||||
writer := z.NewNSLock(bucket, object)
|
||||
lkctx, err := writer.GetLock(t.Context(), globalOperationTimeout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer writer.Unlock(lkctx)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
out := make(chan metaCacheEntry, 10)
|
||||
err = z.listMerged(ctx, listPathOptions{Bucket: bucket, Recursive: true, Limit: 10}, out)
|
||||
if err != nil && err != io.EOF {
|
||||
t.Fatalf("normal listing waited on writer or failed: %v", err)
|
||||
}
|
||||
var names []string
|
||||
for entry := range out {
|
||||
names = append(names, entry.name)
|
||||
if entry.listingSet != nil {
|
||||
t.Fatal("pending escaped")
|
||||
}
|
||||
}
|
||||
if !slices.Equal(names, []string{object}) || reads.Load() != 0 {
|
||||
t.Fatalf("normal names=%v additional ReadXL=%d", names, reads.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestListingPendingDirectoryCollision(t *testing.T) {
|
||||
for _, objectFirst := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint(objectFirst), func(t *testing.T) {
|
||||
in := []chan metaCacheEntry{make(chan metaCacheEntry, 1), make(chan metaCacheEntry, 1)}
|
||||
objectIndex := 1
|
||||
if objectFirst {
|
||||
objectIndex = 0
|
||||
}
|
||||
in[objectIndex] <- metaCacheEntry{name: "dir/", listingSet: &erasureObjects{}}
|
||||
in[1-objectIndex] <- metaCacheEntry{name: "dir/"}
|
||||
for _, c := range in {
|
||||
close(c)
|
||||
}
|
||||
out := make(chan metaCacheEntry, 2)
|
||||
calls := 0
|
||||
err := mergeEntryChannelsWithResolver(t.Context(), in, out, 1, nil, func(group []*metaCacheEntry) (*metaCacheEntry, error) {
|
||||
calls++
|
||||
if len(group) != 1 || group[0].listingSet == nil {
|
||||
t.Fatalf("wrong pending group: %+v", group)
|
||||
}
|
||||
return &metaCacheEntry{name: "dir/", metadata: []byte{1}}, nil
|
||||
})
|
||||
if err != nil || calls != 1 || len(out) != 1 {
|
||||
t.Fatalf("err=%v recoveries=%d outputs=%d", err, calls, len(out))
|
||||
}
|
||||
if entry := <-out; entry.isDir() || !entry.isObjectDir() {
|
||||
t.Fatalf("object became directory: %+v", entry)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestListingRecoveryRejectsUnexpectedSinglePoolSet(t *testing.T) {
|
||||
sets := []*erasureObjects{{}, {}}
|
||||
pool := &erasureSets{sets: sets, distributionAlgo: formatErasureVersionV3DistributionAlgoV3}
|
||||
z := &erasureServerPools{serverPools: []*erasureSets{pool}}
|
||||
for _, object := range []string{"key", "directory/"} {
|
||||
target := pool.getHashedSet(encodeDirObject(object))
|
||||
wrong := sets[0]
|
||||
if wrong == target {
|
||||
wrong = sets[1]
|
||||
}
|
||||
entry, err := z.resolveListingGroup(t.Context(), "bucket", []*metaCacheEntry{{name: object, listingSet: wrong}})
|
||||
if entry != nil || !isErrReadQuorum(err) {
|
||||
t.Fatalf("unexpected set became success: %v %v", entry, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListingRecoveryReleasesBeforeOutput(t *testing.T) {
|
||||
z, bucket, _ := nullQuorumBackend(t)
|
||||
const object = "object/"
|
||||
nullQuorumPut(t, z, bucket, object, []byte("data"), ObjectOptions{})
|
||||
set := z.serverPools[0].sets[0]
|
||||
input := make(chan metaCacheEntry, 1)
|
||||
input <- metaCacheEntry{name: object, listingSet: set}
|
||||
close(input)
|
||||
out := make(chan metaCacheEntry)
|
||||
resolved := make(chan struct{})
|
||||
done := make(chan error, 1)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
go func() {
|
||||
done <- mergeEntryChannelsWithResolver(ctx, []chan metaCacheEntry{input}, out, 1, nil, func(group []*metaCacheEntry) (*metaCacheEntry, error) {
|
||||
entry, err := z.resolveListingGroup(ctx, bucket, group)
|
||||
close(resolved)
|
||||
return entry, err
|
||||
})
|
||||
}()
|
||||
select {
|
||||
case <-resolved:
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
lock := set.NewNSLock(bucket, encodeDirObject(object))
|
||||
lkctx, err := lock.GetLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
t.Fatalf("writer blocked by output backpressure: %v", err)
|
||||
}
|
||||
lock.Unlock(lkctx)
|
||||
entry := <-out
|
||||
if entry.name != object || !entry.isObjectDir() || entry.isDir() || entry.listingSet != nil {
|
||||
t.Fatalf("directory object identity lost: %+v", entry)
|
||||
}
|
||||
if err := <-done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListingRecoveryTerminalErrorPrecedesLock(t *testing.T) {
|
||||
for _, n := range []int{1, 2} {
|
||||
t.Run(fmt.Sprint(n), func(t *testing.T) {
|
||||
inputs := make([]chan metaCacheEntry, n)
|
||||
for i := range inputs {
|
||||
inputs[i] = make(chan metaCacheEntry, 1)
|
||||
inputs[i] <- metaCacheEntry{name: "key", listingSet: &erasureObjects{}}
|
||||
close(inputs[i])
|
||||
}
|
||||
out := make(chan metaCacheEntry, 1)
|
||||
want := errors.New("walk failed")
|
||||
called := false
|
||||
err := mergeEntryChannelsWithResolver(t.Context(), inputs, out, 1, func(int) error { return want }, func([]*metaCacheEntry) (*metaCacheEntry, error) { called = true; return nil, nil })
|
||||
if !errors.Is(err, want) || called || len(out) != 0 {
|
||||
t.Fatalf("err=%v recovery=%v output=%d", err, called, len(out))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestListingPendingCannotBeSerialized(t *testing.T) {
|
||||
entry := metaCacheEntry{name: "directory/", listingSet: &erasureObjects{}}
|
||||
if entry.isDir() || !entry.isObjectDir() {
|
||||
t.Fatal("pending object became a directory")
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
w := newMetacacheWriter(&buf, 0)
|
||||
if err := w.write(entry); err == nil {
|
||||
t.Fatal("pending entry serialized")
|
||||
}
|
||||
_ = w.Close()
|
||||
buf.Reset()
|
||||
w = newMetacacheWriter(&buf, 0)
|
||||
stream, err := w.stream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stream <- entry
|
||||
close(stream)
|
||||
if err = w.Close(); err == nil {
|
||||
t.Fatal("stream serialized pending entry")
|
||||
}
|
||||
}
|
||||
|
||||
func TestListingEntryRepresentationCost(t *testing.T) {
|
||||
type before struct {
|
||||
_ string // name
|
||||
_ []byte // metadata
|
||||
_ *xlMetaV2 // cached
|
||||
_ bool // reusable
|
||||
}
|
||||
t.Logf("metaCacheEntry before=%d after=%d bytes", unsafe.Sizeof(before{}), unsafe.Sizeof(metaCacheEntry{}))
|
||||
}
|
||||
|
||||
func BenchmarkListingMergeNormal(b *testing.B) {
|
||||
for _, n := range []int{1, 2} {
|
||||
for _, resolver := range []bool{false, true} {
|
||||
b.Run(fmt.Sprintf("inputs=%d/resolver=%v", n, resolver), func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
inputs := make([]chan metaCacheEntry, n)
|
||||
for i := range inputs {
|
||||
inputs[i] = make(chan metaCacheEntry, 100)
|
||||
for j := range 100 {
|
||||
inputs[i] <- metaCacheEntry{name: fmt.Sprintf("key-%03d-%d/", j, i)}
|
||||
}
|
||||
close(inputs[i])
|
||||
}
|
||||
out := make(chan metaCacheEntry, n*100)
|
||||
var resolveGroup func([]*metaCacheEntry) (*metaCacheEntry, error)
|
||||
if resolver {
|
||||
resolveGroup = func([]*metaCacheEntry) (*metaCacheEntry, error) { return nil, io.ErrUnexpectedEOF }
|
||||
}
|
||||
if err := mergeEntryChannelsWithResolver(context.Background(), inputs, out, 1, nil, resolveGroup); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,883 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
madmin "github.com/minio/madmin-go/v3"
|
||||
)
|
||||
|
||||
// Hold nine completed disk walks while a PUT overwrites one name. The other
|
||||
// seven walks start after that PUT completes. Both generations come from real
|
||||
// disk walkers and valid PUT metadata; only the reader schedule is controlled.
|
||||
type nullQuorumWalkSchedule struct {
|
||||
bucket string
|
||||
oldReady chan struct{}
|
||||
resume chan struct{}
|
||||
mu sync.Mutex
|
||||
snapshots [16][]byte
|
||||
}
|
||||
|
||||
type nullQuorumWalkDisk struct {
|
||||
StorageAPI
|
||||
schedule *nullQuorumWalkSchedule
|
||||
index int
|
||||
}
|
||||
|
||||
type failedMetadataWalkDisk struct {
|
||||
StorageAPI
|
||||
}
|
||||
|
||||
func (d *failedMetadataWalkDisk) WalkDir(context.Context, WalkDirOptions, io.Writer) error {
|
||||
return errDiskNotFound
|
||||
}
|
||||
|
||||
// offlineNullQuorumDisk models a drive of a node that is away.
|
||||
type offlineNullQuorumDisk struct {
|
||||
StorageAPI
|
||||
}
|
||||
|
||||
func (d *offlineNullQuorumDisk) IsOnline() bool { return false }
|
||||
|
||||
func (d *offlineNullQuorumDisk) DiskInfo(context.Context, DiskInfoOptions) (DiskInfo, error) {
|
||||
return DiskInfo{}, errDiskNotFound
|
||||
}
|
||||
|
||||
func (d *offlineNullQuorumDisk) WalkDir(context.Context, WalkDirOptions, io.Writer) error {
|
||||
return errDiskNotFound
|
||||
}
|
||||
|
||||
func (d *offlineNullQuorumDisk) ReadXL(context.Context, string, string, bool) (RawFileInfo, error) {
|
||||
return RawFileInfo{}, errDiskNotFound
|
||||
}
|
||||
|
||||
func (d *offlineNullQuorumDisk) ReadVersion(context.Context, string, string, string, string, ReadOptions) (FileInfo, error) {
|
||||
return FileInfo{}, errDiskNotFound
|
||||
}
|
||||
|
||||
func (d *nullQuorumWalkDisk) WalkDir(ctx context.Context, opts WalkDirOptions, out io.Writer) error {
|
||||
if opts.Bucket != d.schedule.bucket {
|
||||
return d.StorageAPI.WalkDir(ctx, opts, out)
|
||||
}
|
||||
var stream bytes.Buffer
|
||||
if d.index < 9 {
|
||||
if err := d.StorageAPI.WalkDir(ctx, opts, &stream); err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case d.schedule.oldReady <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-d.schedule.resume:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
if d.index >= 9 {
|
||||
if err := d.StorageAPI.WalkDir(ctx, opts, &stream); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
d.schedule.mu.Lock()
|
||||
d.schedule.snapshots[d.index] = bytes.Clone(stream.Bytes())
|
||||
d.schedule.mu.Unlock()
|
||||
_, err := out.Write(stream.Bytes())
|
||||
return err
|
||||
}
|
||||
|
||||
func nullQuorumBackend(t *testing.T) (*erasureServerPools, string, http.Handler) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
obj, dirs, err := prepareErasure16(ctx)
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
previous := newObjectLayerFn()
|
||||
setObjectLayer(z)
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
z.Shutdown(context.Background())
|
||||
removeRoots(dirs)
|
||||
setObjectLayer(previous)
|
||||
})
|
||||
bucket, router, err := initAPIHandlerTest(ctx, z, nil, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return z, bucket, router
|
||||
}
|
||||
|
||||
func nullQuorumRequest(t *testing.T, router http.Handler, method, target string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
req, err := newTestSignedRequestV4(method, target, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req.WithContext(t.Context()))
|
||||
return rec
|
||||
}
|
||||
|
||||
func nullQuorumPut(t *testing.T, z *erasureServerPools, bucket, object string, body []byte, opts ObjectOptions) {
|
||||
t.Helper()
|
||||
if _, err := z.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), opts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func nullQuorumListV2Keys(t *testing.T, router http.Handler, bucket string) (int, []string) {
|
||||
t.Helper()
|
||||
rec := nullQuorumRequest(t, router, http.MethodGet, getListObjectsV2URL("", bucket, "", "1000", "", "", ""))
|
||||
if rec.Code != http.StatusOK {
|
||||
return rec.Code, nil
|
||||
}
|
||||
var list ListObjectsV2Response
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &list); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keys := make([]string, 0, len(list.Contents))
|
||||
for _, object := range list.Contents {
|
||||
keys = append(keys, object.Key)
|
||||
}
|
||||
return rec.Code, keys
|
||||
}
|
||||
|
||||
func TestListObjectsSingleNullQuorumHTTP(t *testing.T) {
|
||||
z, bucket, router := nullQuorumBackend(t)
|
||||
oldBody := bytes.Repeat([]byte{'a'}, 8192)
|
||||
newBody := bytes.Repeat([]byte{'b'}, 8192)
|
||||
oldTime := time.Now().UTC().Add(-time.Hour)
|
||||
newTime := oldTime.Add(time.Minute)
|
||||
var want []string
|
||||
for i := range 16 {
|
||||
name := fmt.Sprintf("fixed/%02d", i)
|
||||
want = append(want, name)
|
||||
_, err := z.PutObject(t.Context(), bucket, name, mustGetPutObjReader(t, bytes.NewReader(oldBody), int64(len(oldBody)), "", ""), ObjectOptions{MTime: oldTime})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
const overwritten = "fixed/10"
|
||||
schedule := &nullQuorumWalkSchedule{bucket: bucket, oldReady: make(chan struct{}, 16), resume: make(chan struct{})}
|
||||
set := z.serverPools[0].sets[0]
|
||||
disks := set.getDisks()
|
||||
wrapped := make([]StorageAPI, len(disks))
|
||||
for i, disk := range disks {
|
||||
wrapped[i] = &nullQuorumWalkDisk{StorageAPI: disk, schedule: schedule, index: i}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return wrapped }
|
||||
if globalAPIConfig.getListQuorum() != "strict" {
|
||||
t.Fatal("test requires strict listing across all sixteen disks")
|
||||
}
|
||||
writeDone := make(chan error, 1)
|
||||
putReader := mustGetPutObjReader(t, bytes.NewReader(newBody), int64(len(newBody)), "", "")
|
||||
go func() {
|
||||
defer close(schedule.resume)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
for range 9 {
|
||||
select {
|
||||
case <-schedule.oldReady:
|
||||
case <-ctx.Done():
|
||||
writeDone <- ctx.Err()
|
||||
return
|
||||
}
|
||||
}
|
||||
_, err := z.PutObject(ctx, bucket, overwritten, putReader, ObjectOptions{MTime: newTime})
|
||||
writeDone <- err
|
||||
}()
|
||||
rec := nullQuorumRequest(t, router, http.MethodGet, getListObjectsV2URL("", bucket, "fixed/", "1000", "", "", ""))
|
||||
if err := <-writeDone; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var list ListObjectsV2Response
|
||||
if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &list) != nil {
|
||||
t.Fatalf("LIST: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var got []string
|
||||
for _, object := range list.Contents {
|
||||
got = append(got, object.Key)
|
||||
}
|
||||
t.Logf("LIST HTTP=%d KeyCount=%d IsTruncated=%v keys=%v", rec.Code, list.KeyCount, list.IsTruncated, got)
|
||||
if list.KeyCount != 16 || list.IsTruncated || !slices.Equal(got, want) {
|
||||
t.Errorf("LIST omitted a name during overwrite: got %v, want %v", got, want)
|
||||
}
|
||||
|
||||
// Verify the actual reader inputs, including shape and EC, instead of
|
||||
// assuming the scheduling barrier produced the intended resolver case.
|
||||
schedule.mu.Lock()
|
||||
snapshots := schedule.snapshots
|
||||
schedule.mu.Unlock()
|
||||
for i, snapshot := range snapshots {
|
||||
reader := newMetacacheReader(bytes.NewReader(snapshot))
|
||||
var found bool
|
||||
for {
|
||||
entry, err := reader.next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if entry.name != overwritten {
|
||||
continue
|
||||
}
|
||||
xl, err := entry.xlmeta()
|
||||
if err != nil || len(xl.versions) != 1 {
|
||||
t.Fatalf("disk %d: invalid input shape: %v", i, err)
|
||||
}
|
||||
header := xl.versions[0].header
|
||||
wantTime := oldTime
|
||||
if i >= 9 {
|
||||
wantTime = newTime
|
||||
}
|
||||
if header.ModTime != wantTime.UnixNano() || header.VersionID != [16]byte{} || header.Type != ObjectType || header.FreeVersion() {
|
||||
t.Fatalf("disk %d: unexpected header %v", i, header)
|
||||
}
|
||||
t.Logf("disk=%d versions=1 header=%v", i, header)
|
||||
found = true
|
||||
}
|
||||
reader.Close()
|
||||
if !found {
|
||||
t.Fatalf("disk %d did not emit the overwritten name", i)
|
||||
}
|
||||
}
|
||||
get := nullQuorumRequest(t, router, http.MethodGet, getGetObjectURL("", bucket, overwritten))
|
||||
head := nullQuorumRequest(t, router, http.MethodHead, getGetObjectURL("", bucket, overwritten))
|
||||
t.Logf("completed PUT readback: GET=%d bytes=%d HEAD=%d length=%s", get.Code, get.Body.Len(), head.Code, head.Header().Get("Content-Length"))
|
||||
if get.Code != http.StatusOK || !bytes.Equal(get.Body.Bytes(), newBody) || head.Code != http.StatusOK || head.Header().Get("Content-Length") != "8192" {
|
||||
t.Fatalf("completed PUT was not readable: GET=%d HEAD=%d", get.Code, head.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListObjectsFallsBackToReadableMetadata(t *testing.T) {
|
||||
z, bucket, router := nullQuorumBackend(t)
|
||||
set := z.serverPools[0].sets[0]
|
||||
disks, switchDisks := installListingDiskSwitch(set)
|
||||
const object = "object"
|
||||
oldBody := bytes.Repeat([]byte{'a'}, 8192)
|
||||
newBody := bytes.Repeat([]byte{'b'}, 8192)
|
||||
oldTime := time.Now().UTC().Add(-time.Hour)
|
||||
|
||||
if _, err := z.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(oldBody), int64(len(oldBody)), "", ""), ObjectOptions{MTime: oldTime}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldMeta := make([][]byte, len(disks))
|
||||
for i, disk := range disks {
|
||||
oldMeta[i] = mustReadNullQuorumMeta(t, disk, bucket, object)
|
||||
}
|
||||
if _, err := z.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(newBody), int64(len(newBody)), "", ""), ObjectOptions{MTime: oldTime.Add(time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The eight readable walk streams are split 3/5 across two generations;
|
||||
// the other eight streams fail. Neither generation is proven readable by
|
||||
// the listing inputs, so returning HTTP 200 would silently omit the key.
|
||||
for i := range 3 {
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
wrapped := make([]StorageAPI, len(disks))
|
||||
for i, disk := range disks {
|
||||
wrapped[i] = disk
|
||||
if i >= 8 {
|
||||
wrapped[i] = &failedMetadataWalkDisk{StorageAPI: disk}
|
||||
}
|
||||
}
|
||||
switchDisks(wrapped)
|
||||
|
||||
rec := nullQuorumRequest(t, router, http.MethodGet, getListObjectsV2URL("", bucket, "", "1000", "", "", ""))
|
||||
var list ListObjectsV2Response
|
||||
if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &list) != nil {
|
||||
t.Fatalf("LIST failed to resolve readable metadata: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if list.KeyCount != 1 || len(list.Contents) != 1 || list.Contents[0].Key != object {
|
||||
t.Fatalf("LIST omitted the readable key: %+v", list)
|
||||
}
|
||||
|
||||
// The object itself still has a read quorum outside the failed walk
|
||||
// streams, demonstrating why the LIST must report uncertainty, not absence.
|
||||
get := nullQuorumRequest(t, router, http.MethodGet, getGetObjectURL("", bucket, object))
|
||||
if get.Code != http.StatusOK || !bytes.Equal(get.Body.Bytes(), newBody) {
|
||||
t.Fatalf("GET lost the readable generation: %d %q", get.Code, get.Body.String())
|
||||
}
|
||||
// Walk uses a separate producer but must share the same latest-only pool
|
||||
// recovery. Versioned walks retain their existing no-fallback behavior.
|
||||
for _, latestOnly := range []bool{false, true} {
|
||||
out := make(chan itemOrErr[ObjectInfo], 10)
|
||||
if err := z.Walk(t.Context(), bucket, "", out, WalkOptions{LatestOnly: latestOnly, AskDisks: "strict"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var objects []ObjectInfo
|
||||
for result := range out {
|
||||
if result.Err != nil {
|
||||
t.Fatal(result.Err)
|
||||
}
|
||||
objects = append(objects, result.Item)
|
||||
}
|
||||
if latestOnly {
|
||||
if len(objects) != 1 || objects[0].Name != object || !objects[0].ModTime.Equal(oldTime.Add(time.Minute)) {
|
||||
t.Fatalf("Walk latest recovery: %+v", objects)
|
||||
}
|
||||
} else if len(objects) != 0 {
|
||||
t.Fatalf("versioned Walk unexpectedly enabled recovery: %+v", objects)
|
||||
}
|
||||
}
|
||||
failedDisks := slices.Clone(wrapped)
|
||||
for i, disk := range failedDisks {
|
||||
failedDisks[i] = &listingReadFailDisk{StorageAPI: disk, object: object, reads: &atomic.Int64{}}
|
||||
}
|
||||
switchDisks(failedDisks)
|
||||
out := make(chan itemOrErr[ObjectInfo], 10)
|
||||
if err := z.Walk(t.Context(), bucket, "", out, WalkOptions{LatestOnly: true, AskDisks: "strict"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failed := false
|
||||
for result := range out {
|
||||
if result.Err == nil {
|
||||
t.Fatalf("failed Walk yielded object: %+v", result.Item)
|
||||
}
|
||||
if isErrReadQuorum(result.Err) {
|
||||
failed = true
|
||||
}
|
||||
}
|
||||
if !failed {
|
||||
t.Fatal("failed Walk recovery did not report quorum error")
|
||||
}
|
||||
}
|
||||
|
||||
// The fallback must read under the object read lock. A lock-free read of an
|
||||
// object being overwritten stably reports a live object as absent, which is
|
||||
// the omission this repair exists to remove. Hold the write lock and assert
|
||||
// the listing waits for it rather than resolving straight through.
|
||||
func TestListFallbackTakesObjectReadLock(t *testing.T) {
|
||||
z, bucket, router := nullQuorumBackend(t)
|
||||
set := z.serverPools[0].sets[0]
|
||||
disks := set.getDisks()
|
||||
const object = "object"
|
||||
oldBody, newBody := bytes.Repeat([]byte{'a'}, 8192), bytes.Repeat([]byte{'b'}, 8192)
|
||||
oldTime := time.Now().UTC().Add(-time.Hour)
|
||||
|
||||
if _, err := z.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(oldBody), int64(len(oldBody)), "", ""), ObjectOptions{MTime: oldTime}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldMeta := make([][]byte, len(disks))
|
||||
for i, disk := range disks {
|
||||
oldMeta[i] = mustReadNullQuorumMeta(t, disk, bucket, object)
|
||||
}
|
||||
if _, err := z.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(newBody), int64(len(newBody)), "", ""), ObjectOptions{MTime: oldTime.Add(time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Split the readable walk streams so the entry cannot be resolved from
|
||||
// the snapshots and the fallback has to run.
|
||||
for i := range 3 {
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
wrapped := make([]StorageAPI, len(disks))
|
||||
for i, disk := range disks {
|
||||
wrapped[i] = disk
|
||||
if i >= 8 {
|
||||
wrapped[i] = &failedMetadataWalkDisk{StorageAPI: disk}
|
||||
}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return wrapped }
|
||||
|
||||
lock := set.NewNSLock(bucket, object)
|
||||
lkctx, err := lock.GetLock(t.Context(), globalOperationTimeout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
locked := true
|
||||
defer func() {
|
||||
if locked {
|
||||
lock.Unlock(lkctx)
|
||||
}
|
||||
}()
|
||||
target := getListObjectsV2URL("", bucket, "", "1000", "", "", "")
|
||||
req, err := newTestSignedRequestV4(http.MethodGet, target, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
router.ServeHTTP(rec, req)
|
||||
}()
|
||||
waitForListingReadLock(t, set.nsMutex, bucket, object, done)
|
||||
lock.Unlock(lkctx)
|
||||
locked = false
|
||||
<-done
|
||||
|
||||
var list ListObjectsV2Response
|
||||
if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &list) != nil {
|
||||
t.Fatalf("LIST failed after the lock was released: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if list.KeyCount != 1 || len(list.Contents) != 1 || list.Contents[0].Key != object {
|
||||
t.Fatalf("LIST omitted the readable key: %+v", list)
|
||||
}
|
||||
}
|
||||
|
||||
// A truncated xl.meta is what a concurrent overwrite looks like on one drive,
|
||||
// and the walker skips that entry. Skipping is safe and deliberate: the key is
|
||||
// only omitted when no generation reaches quorum, which listPath now resolves
|
||||
// against the read path. Pin that, so the walker is not "hardened" into
|
||||
// failing a whole listing over a minority of unreadable drives.
|
||||
func TestListObjectsSurvivesTruncatedMetadataMinority(t *testing.T) {
|
||||
z, bucket, router := nullQuorumBackend(t)
|
||||
set := z.serverPools[0].sets[0]
|
||||
disks := set.getDisks()
|
||||
const object = "object"
|
||||
body := bytes.Repeat([]byte{'a'}, 8192)
|
||||
|
||||
if _, err := z.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Truncate xl.meta on a minority of drives, as an overwrite in flight does.
|
||||
for i := range 4 {
|
||||
meta := mustReadNullQuorumMeta(t, disks[i], bucket, object)
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, meta[:len(meta)/2]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
rec := nullQuorumRequest(t, router, http.MethodGet, getListObjectsV2URL("", bucket, "", "1000", "", "", ""))
|
||||
var list ListObjectsV2Response
|
||||
if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &list) != nil {
|
||||
t.Fatalf("a truncated minority failed the listing: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if list.KeyCount != 1 || len(list.Contents) != 1 || list.Contents[0].Key != object {
|
||||
t.Fatalf("a truncated minority omitted the key: %+v", list)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListStaleMinorityIgnored(t *testing.T) {
|
||||
z, bucket, router := nullQuorumBackend(t)
|
||||
disks := z.serverPools[0].sets[0].getDisks()
|
||||
body := bytes.Repeat([]byte{'a'}, 8192)
|
||||
nullQuorumPut(t, z, bucket, "gone", body, ObjectOptions{})
|
||||
nullQuorumPut(t, z, bucket, "keep", body, ObjectOptions{})
|
||||
for _, disk := range disks[4:] {
|
||||
if err := disk.Delete(t.Context(), bucket, "gone", DeleteOptions{Recursive: true, Immediate: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if code, keys := nullQuorumListV2Keys(t, router, bucket); code != http.StatusOK || !slices.Equal(keys, []string{"keep"}) {
|
||||
t.Errorf("ListObjectsV2: got %d %v; want 200 [keep]", code, keys)
|
||||
}
|
||||
results := make(chan itemOrErr[ObjectInfo], 16)
|
||||
if err := z.Walk(t.Context(), bucket, "", results, WalkOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var names []string
|
||||
for result := range results {
|
||||
if result.Err != nil {
|
||||
t.Fatal(result.Err)
|
||||
}
|
||||
names = append(names, result.Item.Name)
|
||||
}
|
||||
if !slices.Equal(names, []string{"keep"}) {
|
||||
t.Errorf("Walk: got %v; want [keep]", names)
|
||||
}
|
||||
|
||||
rec := nullQuorumRequest(t, router, http.MethodGet, getListObjectVersionsURL("", bucket, "", "1000", ""))
|
||||
var list struct {
|
||||
Versions []struct {
|
||||
Key string `xml:"Key"`
|
||||
} `xml:"Version"`
|
||||
}
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Errorf("ListObjectVersions: got %d; want 200", rec.Code)
|
||||
} else if err := xml.Unmarshal(rec.Body.Bytes(), &list); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if len(list.Versions) != 1 || list.Versions[0].Key != "keep" {
|
||||
t.Errorf("ListObjectVersions: got %+v; want [keep]", list.Versions)
|
||||
}
|
||||
|
||||
if _, err := z.DeleteObject(t.Context(), bucket, "keep", ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := z.DeleteBucket(t.Context(), bucket, DeleteBucketOptions{}); err != nil {
|
||||
t.Errorf("DeleteBucket with only a stale minority: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMListingIgnoresStaleMinority(t *testing.T) {
|
||||
z, _, _ := nullQuorumBackend(t)
|
||||
body := []byte(`{"version":1}`)
|
||||
alive := iamConfigUsersPrefix + "alive/identity.json"
|
||||
ghost := iamConfigUsersPrefix + "ghost/identity.json"
|
||||
nullQuorumPut(t, z, minioMetaBucket, alive, body, ObjectOptions{})
|
||||
nullQuorumPut(t, z, minioMetaBucket, ghost, body, ObjectOptions{})
|
||||
for _, disk := range z.serverPools[0].getHashedSet(ghost).getDisks()[4:] {
|
||||
if err := disk.Delete(t.Context(), minioMetaBucket, ghost, DeleteOptions{Recursive: true, Immediate: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
items, err := newIAMObjectStore(z, MinIOUsersSysType).listAllIAMConfigItems(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("IAM listing failed on a stale minority (retriable=%v): %v", configRetriableErrors(err), err)
|
||||
}
|
||||
if got := items["users/"]; !slices.Equal(got, []string{"alive/identity.json"}) {
|
||||
t.Errorf("IAM users: got %v; want [alive/identity.json]", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListSplitDirectoryObject(t *testing.T) {
|
||||
z, bucket, router := nullQuorumBackend(t)
|
||||
set := z.serverPools[0].sets[0]
|
||||
disks := set.getDisks()
|
||||
const object = "dir/"
|
||||
encoded := encodeDirObject(object)
|
||||
mtime := time.Now().UTC().Add(-time.Hour)
|
||||
|
||||
nullQuorumPut(t, z, bucket, object, nil, ObjectOptions{MTime: mtime})
|
||||
old := make([][]byte, 3)
|
||||
for i := range old {
|
||||
old[i] = mustReadNullQuorumMeta(t, disks[i], bucket, encoded)
|
||||
}
|
||||
nullQuorumPut(t, z, bucket, object, nil, ObjectOptions{MTime: mtime.Add(time.Minute)})
|
||||
for i := range old {
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, encoded+"/"+xlStorageFormatFile, old[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
walkers := slices.Clone(disks)
|
||||
for i := 8; i < len(walkers); i++ {
|
||||
walkers[i] = &failedMetadataWalkDisk{StorageAPI: disks[i]}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return walkers }
|
||||
if code, keys := nullQuorumListV2Keys(t, router, bucket); code != http.StatusOK || !slices.Equal(keys, []string{object}) {
|
||||
t.Errorf("ListObjectsV2: got %d %v; want 200 [%s]", code, keys, object)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListFallbackConfirmsCurrentAbsence(t *testing.T) {
|
||||
z, bucket, _ := nullQuorumBackend(t)
|
||||
set := z.serverPools[0].sets[0]
|
||||
disks := set.getDisks()
|
||||
const object = "object"
|
||||
body := bytes.Repeat([]byte{'a'}, 8192)
|
||||
mtime := time.Now().UTC().Add(-time.Hour)
|
||||
|
||||
nullQuorumPut(t, z, bucket, object, body, ObjectOptions{MTime: mtime})
|
||||
old := make([][]byte, len(disks))
|
||||
for i := range disks {
|
||||
old[i] = mustReadNullQuorumMeta(t, disks[i], bucket, object)
|
||||
}
|
||||
nullQuorumPut(t, z, bucket, object, body, ObjectOptions{MTime: mtime.Add(time.Minute)})
|
||||
newer := make([][]byte, len(disks))
|
||||
for i := range disks {
|
||||
newer[i] = mustReadNullQuorumMeta(t, disks[i], bucket, object)
|
||||
if err := disks[i].Delete(t.Context(), bucket, object, DeleteOptions{Recursive: true, Immediate: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
entries := make(metaCacheEntries, len(disks))
|
||||
errs := make([]error, len(disks))
|
||||
for i := range 3 {
|
||||
entries[i] = metaCacheEntry{name: object, metadata: old[i]}
|
||||
}
|
||||
for i := 3; i < 8; i++ {
|
||||
entries[i] = metaCacheEntry{name: object, metadata: newer[i]}
|
||||
}
|
||||
for i := 8; i < len(errs); i++ {
|
||||
errs[i] = errDiskNotFound
|
||||
}
|
||||
resolver := metadataResolutionParams{dirQuorum: 8, objQuorum: 8, bucket: bucket, requestedVersions: 1}
|
||||
entry, err := z.resolveListingGroup(t.Context(), bucket, []*metaCacheEntry{set.resolveListEntry(entries, errs, &resolver)})
|
||||
if entry != nil || err != nil {
|
||||
t.Fatalf("strict current absence: got entry=%v err=%v; want no entry or error", entry, err)
|
||||
}
|
||||
}
|
||||
|
||||
// G0 was written 8+8 while a node was away. G1 was later written 12+4 but
|
||||
// missed drive 0, which still holds G0. Then a node holding four G1 drives
|
||||
// goes away: eleven G1 and one G0 remain online. G1 exists but lacks one of
|
||||
// its twelve data shards until the node returns, so HEAD must not report 404
|
||||
// and LIST must still show the key.
|
||||
func TestListCrossParityMinorityStaysVisible(t *testing.T) {
|
||||
z, bucket, router := nullQuorumBackend(t)
|
||||
set := z.serverPools[0].sets[0]
|
||||
disks, switchDisks := installListingDiskSwitch(set)
|
||||
const object = "object"
|
||||
mtime := time.Now().UTC().Add(-time.Hour)
|
||||
|
||||
nullQuorumPut(t, z, bucket, object, bytes.Repeat([]byte{'a'}, 256<<10), ObjectOptions{MTime: mtime, MaxParity: true})
|
||||
g0 := mustReadNullQuorumMeta(t, disks[0], bucket, object)
|
||||
var x xlMetaV2
|
||||
if err := x.LoadOrConvert(g0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h := x.versions[0].header; h.EcM != 8 || h.EcN != 8 {
|
||||
t.Fatalf("G0 is %d+%d, want 8+8", h.EcM, h.EcN)
|
||||
}
|
||||
g1Body := bytes.Repeat([]byte{'b'}, 256<<10)
|
||||
nullQuorumPut(t, z, bucket, object, g1Body, ObjectOptions{MTime: mtime.Add(time.Minute)})
|
||||
if err := disks[0].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, g0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
online := slices.Clone(disks)
|
||||
for i := 12; i < len(online); i++ {
|
||||
online[i] = &offlineNullQuorumDisk{StorageAPI: disks[i]}
|
||||
}
|
||||
switchDisks(online)
|
||||
if code := nullQuorumRequest(t, router, http.MethodHead, getHeadObjectURL("", bucket, object)).Code; code == http.StatusNotFound {
|
||||
t.Errorf("HEAD with a node away: 404 for an existing object")
|
||||
}
|
||||
if code, keys := nullQuorumListV2Keys(t, router, bucket); code != http.StatusOK || !slices.Equal(keys, []string{object}) {
|
||||
t.Errorf("ListObjectsV2 with a node away: got %d %v; want 200 [%s]", code, keys, object)
|
||||
}
|
||||
|
||||
switchDisks(disks)
|
||||
if get := nullQuorumRequest(t, router, http.MethodGet, getGetObjectURL("", bucket, object)); get.Code != http.StatusOK || !bytes.Equal(get.Body.Bytes(), g1Body) {
|
||||
t.Fatalf("GET after the node returns: %d", get.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingleNullQuorumReadAndConsumers(t *testing.T) {
|
||||
z, bucket, router := nullQuorumBackend(t)
|
||||
set := z.serverPools[0].sets[0]
|
||||
disks := set.getDisks()
|
||||
const object = "object"
|
||||
oldBody, newBody := bytes.Repeat([]byte{'a'}, 8192), bytes.Repeat([]byte{'b'}, 8192)
|
||||
oldTime := time.Now().UTC().Add(-time.Hour)
|
||||
put := func(body []byte, modTime time.Time) {
|
||||
t.Helper()
|
||||
_, err := z.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{MTime: modTime})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
put(oldBody, oldTime)
|
||||
oldMeta := make([][]byte, len(disks))
|
||||
for i, disk := range disks {
|
||||
var err error
|
||||
oldMeta[i], err = disk.ReadAll(t.Context(), bucket, object+"/"+xlStorageFormatFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
put(newBody, oldTime.Add(time.Minute))
|
||||
newMinorityMeta := mustReadNullQuorumMeta(t, disks[14], bucket, object)
|
||||
// Model an incomplete overwrite using real inline shards: fifteen old
|
||||
// copies retain a read quorum; the final disk contains the newer minority.
|
||||
for i := range 15 {
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
checkRead := func(body []byte, modTime time.Time) {
|
||||
t.Helper()
|
||||
get := nullQuorumRequest(t, router, http.MethodGet, getGetObjectURL("", bucket, object))
|
||||
head := nullQuorumRequest(t, router, http.MethodHead, getGetObjectURL("", bucket, object))
|
||||
if get.Code != 200 || !bytes.Equal(get.Body.Bytes(), body) || head.Code != 200 || head.Header().Get("Content-Length") != "8192" {
|
||||
t.Fatalf("GET/HEAD lost readable quorum: GET=%d HEAD=%d body=%q", get.Code, head.Code, get.Body.String())
|
||||
}
|
||||
oi, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||
if err != nil || !oi.ModTime.Equal(modTime) {
|
||||
t.Fatalf("wrong readable generation: %v, %v", oi.ModTime, err)
|
||||
}
|
||||
}
|
||||
checkRead(oldBody, oldTime)
|
||||
// Reading must not reinterpret the minority as absent or delete it.
|
||||
minority, err := disks[15].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||
if err != nil || !minority.ModTime.Equal(oldTime.Add(time.Minute)) {
|
||||
t.Fatalf("read mutated the minority: %+v, %v", minority, err)
|
||||
}
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
var names []string
|
||||
consume := func(entry metaCacheEntry) {
|
||||
versions, err := entry.fileInfoVersions(bucket)
|
||||
if err != nil || len(versions.Versions) != 1 || !versions.Versions[0].ModTime.Equal(oldTime) {
|
||||
t.Errorf("invalid migration metadata: %+v, %v", versions, err)
|
||||
return
|
||||
}
|
||||
// Migration reopens the selected name/version on all drives;
|
||||
// it must not treat the selected listing metadata as disk agreement.
|
||||
reader, err := set.GetObjectNInfo(t.Context(), bucket, entry.name, nil, nil, ObjectOptions{VersionID: nullVersionID, NoLock: true, NoDecryption: true})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
body, err := io.ReadAll(reader)
|
||||
reader.Close()
|
||||
if err != nil || !bytes.Equal(body, oldBody) {
|
||||
t.Errorf("migration could not read selected object data: %v", err)
|
||||
}
|
||||
names = append(names, entry.name)
|
||||
}
|
||||
var err error
|
||||
if kind == "rebalance" {
|
||||
err = set.listObjectsToRebalance(t.Context(), bucket, consume)
|
||||
} else {
|
||||
err = set.listObjectsToDecommission(t.Context(), decomBucketInfo{Name: bucket}, consume)
|
||||
}
|
||||
if err != nil || !slices.Equal(names, []string{object}) {
|
||||
t.Fatalf("migration listing dropped the quorum: %v, %v", names, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, latestOnly := range []bool{false, true} {
|
||||
results := make(chan itemOrErr[ObjectInfo], 16)
|
||||
if err := z.Walk(t.Context(), bucket, "", results, WalkOptions{AskDisks: "strict", LatestOnly: latestOnly}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var names []string
|
||||
for result := range results {
|
||||
if result.Err != nil || !result.Item.ModTime.Equal(oldTime) {
|
||||
t.Fatalf("Walk returned invalid metadata: %+v", result)
|
||||
}
|
||||
names = append(names, result.Item.Name)
|
||||
}
|
||||
if !slices.Equal(names, []string{object}) {
|
||||
t.Fatalf("Walk dropped the quorum: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
// Exercise the scanner's actual abandoned-child path. Make the scan drive
|
||||
// miss the object, while keeping an older quorum and a newer minority on
|
||||
// the remaining disks. Its queued heal must read all disks and repair both.
|
||||
if err := disks[14].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, newMinorityMeta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(disks[15].Endpoint().Path, bucket, object)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
scanNullQuorumAbandoned(t, z, bucket, object, oldTime)
|
||||
for i, disk := range disks {
|
||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{Healing: true})
|
||||
if err != nil || !fi.ModTime.Equal(oldTime) {
|
||||
t.Fatalf("scanner heal did not reconcile disk %d: %v, %v", i, fi.ModTime, err)
|
||||
}
|
||||
}
|
||||
checkRead(oldBody, oldTime)
|
||||
put(newBody, oldTime.Add(2*time.Minute))
|
||||
checkRead(newBody, oldTime.Add(2*time.Minute))
|
||||
}
|
||||
|
||||
func mustReadNullQuorumMeta(t *testing.T, disk StorageAPI, bucket, object string) []byte {
|
||||
t.Helper()
|
||||
data, err := disk.ReadAll(t.Context(), bucket, object+"/"+xlStorageFormatFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func scanNullQuorumAbandoned(t *testing.T, z *erasureServerPools, bucket, object string, oldTime time.Time) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
disks := z.serverPools[0].sets[0].getDisks()
|
||||
seq := newBgHealSequence()
|
||||
defer seq.cancelCtx()
|
||||
previousState, previousRoutine := globalBackgroundHealState, globalBackgroundHealRoutine
|
||||
globalBackgroundHealState = &allHealState{healSeqMap: map[string]*healSequence{"test": seq}}
|
||||
routine := &healRoutine{tasks: make(chan healTask)}
|
||||
globalBackgroundHealRoutine = routine
|
||||
defer func() { globalBackgroundHealState, globalBackgroundHealRoutine = previousState, previousRoutine }()
|
||||
workerDone := make(chan struct{})
|
||||
var healed []string
|
||||
go func() {
|
||||
defer close(workerDone)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case task := <-routine.tasks:
|
||||
var result madmin.HealResultItem
|
||||
var err error
|
||||
if task.object == "" {
|
||||
result, err = z.HealBucket(ctx, task.bucket, task.opts)
|
||||
} else {
|
||||
healed = append(healed, task.object+"/"+task.versionID)
|
||||
result, err = z.HealObject(ctx, task.bucket, task.object, task.versionID, task.opts)
|
||||
}
|
||||
task.respCh <- healResult{result: result, err: err}
|
||||
}
|
||||
}
|
||||
}()
|
||||
cache := dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}
|
||||
cache.replace(bucket, "", dataUsageEntry{})
|
||||
cache.replace(bucket+"/"+object, bucket, dataUsageEntry{Size: 8192, Objects: 1, Versions: 1})
|
||||
scanner := folderScanner{
|
||||
root: disks[15].Endpoint().Path, oldCache: cache,
|
||||
newCache: dataUsageCache{Info: cache.Info}, updateCache: dataUsageCache{Info: cache.Info},
|
||||
disks: disks, disksQuorum: 8, healObjectSelect: 1,
|
||||
weSleep: func() bool { return false }, shouldHeal: func() bool { return true }, updateCurrentPath: func(string) {},
|
||||
getSize: func(item scannerItem) (sizeSummary, error) {
|
||||
if filepath.Base(item.Path) != xlStorageFormatFile {
|
||||
return sizeSummary{}, errSkipFile
|
||||
}
|
||||
data, err := os.ReadFile(item.Path)
|
||||
if err != nil {
|
||||
return sizeSummary{}, err
|
||||
}
|
||||
var xl xlMetaV2
|
||||
if err := xl.Load(data); err != nil {
|
||||
return sizeSummary{}, err
|
||||
}
|
||||
fi, err := xl.ToFileInfo(bucket, object, "", false, false)
|
||||
if err != nil || !fi.ModTime.Equal(oldTime) {
|
||||
return sizeSummary{}, fmt.Errorf("scanner re-read wrong generation: %v, %v", fi.ModTime, err)
|
||||
}
|
||||
return sizeSummary{totalSize: fi.Size, versions: 1}, nil
|
||||
},
|
||||
}
|
||||
var usage dataUsageEntry
|
||||
err := scanner.scanFolder(ctx, cachedFolder{name: bucket, objectHealProbDiv: 1}, &usage)
|
||||
cancel()
|
||||
<-workerDone
|
||||
if err != nil || len(healed) != 1 || healed[0] != object+"/"+nullVersionID && healed[0] != object+"/" {
|
||||
t.Fatalf("scanner did not queue a name-based heal: %v, %v", healed, err)
|
||||
}
|
||||
flat := scanner.newCache.sizeRecursive(bucket)
|
||||
if flat == nil || flat.Objects != 1 || flat.Size != 8192 {
|
||||
t.Fatalf("scanner lost the healed object from usage: %+v", flat)
|
||||
}
|
||||
t.Logf("scanner queued %v; healed old quorum, repaired minority/missing copies, and retained one 8192-byte object", healed)
|
||||
}
|
||||
@@ -0,0 +1,444 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/grid"
|
||||
)
|
||||
|
||||
// Read real cache objects and use an actual peer RPC, counting only the block
|
||||
// under test. Gates control publication, never replace quorum or EOF decisions.
|
||||
type listingCachePartsProbe struct {
|
||||
state atomic.Pointer[metacache]
|
||||
rpcFails atomic.Bool
|
||||
reads atomic.Int64
|
||||
mu sync.Mutex
|
||||
probes []int
|
||||
checks []time.Time
|
||||
onProbe func(int) bool
|
||||
onCheck func(int)
|
||||
target string
|
||||
readErr error
|
||||
}
|
||||
|
||||
func (p *listingCachePartsProbe) setState(o listPathOptions, failed bool) {
|
||||
m := o.newMetacache()
|
||||
m.lastUpdate = time.Now()
|
||||
if failed {
|
||||
m.status, m.error = scanStateError, "cache producer failed"
|
||||
} else {
|
||||
m.status = scanStateStarted
|
||||
}
|
||||
p.state.Store(&m)
|
||||
}
|
||||
|
||||
func (p *listingCachePartsProbe) snapshot() ([]int, []time.Time) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return slices.Clone(p.probes), slices.Clone(p.checks)
|
||||
}
|
||||
|
||||
type listingCachePartsDisk struct {
|
||||
StorageAPI
|
||||
index int
|
||||
probe *listingCachePartsProbe
|
||||
}
|
||||
|
||||
func (d *listingCachePartsDisk) ReadXL(ctx context.Context, bucket, object string, readData bool) (RawFileInfo, error) {
|
||||
if bucket == minioMetaBucket && object == d.probe.target {
|
||||
d.probe.reads.Add(1)
|
||||
if d.probe.readErr != nil {
|
||||
return RawFileInfo{}, d.probe.readErr
|
||||
}
|
||||
}
|
||||
return d.StorageAPI.ReadXL(ctx, bucket, object, readData)
|
||||
}
|
||||
|
||||
func (d *listingCachePartsDisk) ReadVersion(ctx context.Context, origVolume, volume, path, versionID string, opts ReadOptions) (FileInfo, error) {
|
||||
if volume == minioMetaBucket && path == d.probe.target {
|
||||
d.probe.mu.Lock()
|
||||
d.probe.probes = append(d.probe.probes, d.index)
|
||||
d.probe.mu.Unlock()
|
||||
if d.probe.onProbe != nil && d.probe.onProbe(d.index) {
|
||||
return FileInfo{}, errFileNotFound
|
||||
}
|
||||
}
|
||||
return d.StorageAPI.ReadVersion(ctx, origVolume, volume, path, versionID, opts)
|
||||
}
|
||||
|
||||
func listingCachePartsBackend(t *testing.T, part int) (*erasureObjects, listPathOptions, *listingCachePartsProbe, func([]StorageAPI)) {
|
||||
t.Helper()
|
||||
z, bucket, _ := nullQuorumBackend(t)
|
||||
set := z.serverPools[0].sets[0]
|
||||
o := listPathOptions{ID: mustGetUUID(), Bucket: bucket, Limit: 100, Recursive: true, IncludeDirectories: true}
|
||||
p := &listingCachePartsProbe{target: o.objectPath(part)}
|
||||
p.setState(o, false)
|
||||
disks, switchDisks := installListingDiskSwitch(set)
|
||||
wrapped := slices.Clone(disks)
|
||||
for i, disk := range disks {
|
||||
wrapped[i] = &listingCachePartsDisk{StorageAPI: disk, index: i, probe: p}
|
||||
}
|
||||
switchDisks(wrapped)
|
||||
|
||||
tg, err := grid.SetupTestGrid(2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previous := globalNotificationSys
|
||||
t.Cleanup(func() {
|
||||
globalNotificationSys = previous
|
||||
tg.Cleanup()
|
||||
})
|
||||
if err := getMetacacheListingRPC.Register(tg.Managers[1], func(got *listPathOptions) (*metacache, *grid.RemoteErr) {
|
||||
if got.ID != o.ID || got.Create {
|
||||
return nil, grid.NewRemoteErr(errors.New("invalid cache state lookup"))
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.checks = append(p.checks, time.Now())
|
||||
n := len(p.checks)
|
||||
p.mu.Unlock()
|
||||
if p.onCheck != nil {
|
||||
p.onCheck(n)
|
||||
}
|
||||
if p.rpcFails.Load() {
|
||||
return nil, grid.NewRemoteErr(errors.New("cache state RPC failed"))
|
||||
}
|
||||
m := *p.state.Load()
|
||||
return &m, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &peerRESTClient{gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[1]) }}
|
||||
globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{client}, allPeerClients: []*peerRESTClient{client}}
|
||||
return set, o, p, switchDisks
|
||||
}
|
||||
|
||||
func putListingCachePart(t *testing.T, set *erasureObjects, o listPathOptions, part int, name string, eos bool) {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
w := newMetacacheWriter(&buf, 0)
|
||||
if err := w.write(metaCacheEntry{name: name}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
block := metacacheBlock{n: part, First: name, Last: name, EOS: eos}
|
||||
_, err := set.putMetacacheObject(t.Context(), o.objectPath(part), mustGetPutObjReader(t, bytes.NewReader(buf.Bytes()), int64(buf.Len()), "", ""), ObjectOptions{UserDefined: block.headerKV()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamMetadataPartsHealthyNoStateRPC(t *testing.T) {
|
||||
set, o, p, _ := listingCachePartsBackend(t, 1)
|
||||
putListingCachePart(t, set, o, 0, "a/", false)
|
||||
putListingCachePart(t, set, o, 1, "b/", true)
|
||||
entries, err := set.streamMetadataParts(t.Context(), o)
|
||||
if !errors.Is(err, io.EOF) || entries.len() != 2 || entries.o[0].name != "a/" || entries.o[1].name != "b/" {
|
||||
t.Fatalf("healthy two-block read: entries=%v err=%v", entries.o, err)
|
||||
}
|
||||
probes, checks := p.snapshot()
|
||||
if len(probes) != 0 || len(checks) != 0 {
|
||||
t.Fatalf("healthy read added probes=%v state RPCs=%d", probes, len(checks))
|
||||
}
|
||||
t.Logf("healthy persisted blocks: state RPCs=0 retry probes=0 target ReadXL=%d", p.reads.Load())
|
||||
}
|
||||
|
||||
func TestStreamMetadataPartsTerminalError(t *testing.T) {
|
||||
for _, part := range []int{0, 1} {
|
||||
t.Run(fmt.Sprintf("part=%d", part), func(t *testing.T) {
|
||||
set, o, p, _ := listingCachePartsBackend(t, part)
|
||||
if part > 0 {
|
||||
putListingCachePart(t, set, o, 0, "a/", false)
|
||||
}
|
||||
p.setState(o, true)
|
||||
start := time.Now()
|
||||
entries, err := set.streamMetadataParts(t.Context(), o)
|
||||
elapsed := time.Since(start)
|
||||
if err == nil || errors.Is(err, io.EOF) || !strings.Contains(err.Error(), "cache producer failed") || entries.len() != part {
|
||||
t.Fatalf("failed cache became successful: entries=%v err=%v", entries.o, err)
|
||||
}
|
||||
probes, checks := p.snapshot()
|
||||
if len(checks) != 1 || len(probes) != 0 || elapsed > time.Second {
|
||||
t.Fatalf("terminal error waited for disk retries: elapsed=%v state RPCs=%d probes=%v", elapsed, len(checks), probes)
|
||||
}
|
||||
t.Logf("terminal part=%d elapsed=%v state RPCs=%d probes=%d target ReadXL=%d", part, elapsed, len(checks), len(probes), p.reads.Load())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamMetadataPartsLateTerminalError(t *testing.T) {
|
||||
set, o, p, _ := listingCachePartsBackend(t, 1)
|
||||
putListingCachePart(t, set, o, 0, "a/", false)
|
||||
p.onProbe = func(int) bool {
|
||||
p.setState(o, true)
|
||||
return false
|
||||
}
|
||||
start := time.Now()
|
||||
entries, err := set.streamMetadataParts(t.Context(), o)
|
||||
elapsed := time.Since(start)
|
||||
probes, checks := p.snapshot()
|
||||
if err == nil || errors.Is(err, io.EOF) || !strings.Contains(err.Error(), "cache producer failed") || entries.len() != 1 {
|
||||
t.Fatalf("late producer error lost: entries=%v err=%v", entries.o, err)
|
||||
}
|
||||
if elapsed > time.Second || len(probes) != 2 || !slices.Equal(probes, []int{0, 1}) || len(checks) != 2 {
|
||||
t.Fatalf("late error retry work: elapsed=%v probes=%v state RPCs=%d", elapsed, probes, len(checks))
|
||||
}
|
||||
if checks[1].Sub(checks[0]) < 250*time.Millisecond || p.reads.Load() > 16 {
|
||||
t.Fatalf("unbounded retry: state interval=%v target ReadXL=%d", checks[1].Sub(checks[0]), p.reads.Load())
|
||||
}
|
||||
t.Logf("late terminal elapsed=%v state RPCs=%d probes=%v target ReadXL=%d", elapsed, len(checks), probes, p.reads.Load())
|
||||
}
|
||||
|
||||
type listingCachePartsResult struct {
|
||||
entries metaCacheEntriesSorted
|
||||
err error
|
||||
}
|
||||
|
||||
func startListingCacheParts(t *testing.T, set *erasureObjects, o listPathOptions) (<-chan listingCachePartsResult, context.CancelFunc) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
done := make(chan listingCachePartsResult, 1)
|
||||
joined := make(chan struct{})
|
||||
go func() {
|
||||
defer close(joined)
|
||||
entries, err := set.streamMetadataParts(ctx, o)
|
||||
done <- listingCachePartsResult{entries: entries, err: err}
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
select {
|
||||
case <-joined:
|
||||
case <-time.After(8 * time.Second):
|
||||
t.Error("cache continuation worker did not finish")
|
||||
}
|
||||
})
|
||||
return done, cancel
|
||||
}
|
||||
|
||||
func receiveListingCacheParts(t *testing.T, done <-chan listingCachePartsResult) listingCachePartsResult {
|
||||
t.Helper()
|
||||
select {
|
||||
case result := <-done:
|
||||
return result
|
||||
case <-time.After(8 * time.Second):
|
||||
t.Fatal("cache continuation did not finish")
|
||||
return listingCachePartsResult{}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamMetadataPartsGeneratingThenPublished(t *testing.T) {
|
||||
set, o, p, _ := listingCachePartsBackend(t, 1)
|
||||
putListingCachePart(t, set, o, 0, "a/", false)
|
||||
checked := make(chan struct{})
|
||||
p.onCheck = func(n int) {
|
||||
if n == 1 {
|
||||
close(checked)
|
||||
}
|
||||
}
|
||||
// The first selected replica is absent even after the producer publishes
|
||||
// the block elsewhere. Rotation must reach a readable replica.
|
||||
p.onProbe = func(index int) bool { return index == 0 }
|
||||
done, _ := startListingCacheParts(t, set, o)
|
||||
select {
|
||||
case <-checked:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("missing block did not check producer state")
|
||||
}
|
||||
select {
|
||||
case result := <-done:
|
||||
t.Fatalf("still-generating cache finished early: %+v", result)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
putListingCachePart(t, set, o, 1, "b/", true)
|
||||
result := receiveListingCacheParts(t, done)
|
||||
if !errors.Is(result.err, io.EOF) || result.entries.len() != 2 || result.entries.o[0].name != "a/" || result.entries.o[1].name != "b/" {
|
||||
t.Fatalf("published second block lost: entries=%v err=%v", result.entries.o, result.err)
|
||||
}
|
||||
probes, checks := p.snapshot()
|
||||
if len(probes) < 2 || probes[0] != 0 || probes[1] != 1 {
|
||||
t.Fatalf("retry did not rotate: probes=%v checks=%d", probes, len(checks))
|
||||
}
|
||||
t.Logf("later block recovered: state RPCs=%d probes=%v target ReadXL=%d", len(checks), probes, p.reads.Load())
|
||||
}
|
||||
|
||||
func TestStreamMetadataPartsRPCFailure(t *testing.T) {
|
||||
set, o, p, _ := listingCachePartsBackend(t, 1)
|
||||
putListingCachePart(t, set, o, 0, "a/", false)
|
||||
p.rpcFails.Store(true)
|
||||
entries, err := set.streamMetadataParts(t.Context(), o)
|
||||
if err == nil || errors.Is(err, io.EOF) || !strings.Contains(err.Error(), "cache state RPC failed") || entries.len() != 1 {
|
||||
t.Fatalf("RPC failure became successful EOF: entries=%v err=%v", entries.o, err)
|
||||
}
|
||||
probes, checks := p.snapshot()
|
||||
if len(checks) != 1 || len(probes) != 0 {
|
||||
t.Fatalf("state RPC failure continued retrying: probes=%v checks=%d", probes, len(checks))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamMetadataPartsMissingIndex(t *testing.T) {
|
||||
set, o, p, _ := listingCachePartsBackend(t, 0)
|
||||
putListingCachePart(t, set, o, 0, "a/", false)
|
||||
o.Marker = "b/"
|
||||
p.setState(o, true)
|
||||
entries, err := set.streamMetadataParts(t.Context(), o)
|
||||
if err == nil || errors.Is(err, io.EOF) || !strings.Contains(err.Error(), "cache producer failed") || entries.len() != 0 {
|
||||
t.Fatalf("unpublished index became successful EOF: entries=%v err=%v", entries.o, err)
|
||||
}
|
||||
probes, checks := p.snapshot()
|
||||
if len(checks) != 1 || len(probes) != 0 {
|
||||
t.Fatalf("missing index delayed terminal state: probes=%v checks=%d", probes, len(checks))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamMetadataPartsOtherErrorRetriesOnce(t *testing.T) {
|
||||
set, o, p, _ := listingCachePartsBackend(t, 0)
|
||||
p.readErr = errVolumeNotFound
|
||||
entries, err := set.streamMetadataParts(t.Context(), o)
|
||||
if err == nil || errors.Is(err, io.EOF) || !strings.Contains(err.Error(), "reading first part metadata:") || entries.len() != 0 {
|
||||
t.Fatalf("unexpected metadata error result: entries=%v err=%v", entries.o, err)
|
||||
}
|
||||
probes, checks := p.snapshot()
|
||||
if p.reads.Load() != 32 || len(probes) != 0 || len(checks) != 1 {
|
||||
t.Fatalf("other metadata error did not get exactly one full retry: ReadXL=%d probes=%v checks=%d", p.reads.Load(), probes, len(checks))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamMetadataPartsMissingBlockWorkBound(t *testing.T) {
|
||||
set, o, p, _ := listingCachePartsBackend(t, 0)
|
||||
checked := make(chan struct{})
|
||||
p.onCheck = func(n int) {
|
||||
if n == 3 {
|
||||
close(checked)
|
||||
}
|
||||
}
|
||||
start := time.Now()
|
||||
done, cancel := startListingCacheParts(t, set, o)
|
||||
select {
|
||||
case <-checked:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("missing cache stopped checking producer state")
|
||||
}
|
||||
cancel()
|
||||
result := receiveListingCacheParts(t, done)
|
||||
probes, checks := p.snapshot()
|
||||
if !errors.Is(result.err, context.Canceled) || result.entries.len() != 0 {
|
||||
t.Fatalf("generating cache became successful EOF: %+v", result)
|
||||
}
|
||||
if len(probes) != 3 || !slices.Equal(probes, []int{0, 1, 2}) || p.reads.Load() != 16 || len(checks) != 3 {
|
||||
t.Fatalf("missing block exceeded work bound: probes=%v target ReadXL=%d checks=%d", probes, p.reads.Load(), len(checks))
|
||||
}
|
||||
for i := 1; i < len(checks); i++ {
|
||||
// Receive timestamps allow a small amount of peer scheduling jitter.
|
||||
if checks[i].Sub(checks[i-1]) < 240*time.Millisecond {
|
||||
t.Fatalf("cache state queried too often: %v", checks)
|
||||
}
|
||||
}
|
||||
t.Logf("still missing for %v: state RPCs=%d probes=%v target ReadXL=%d", time.Since(start), len(checks), probes, p.reads.Load())
|
||||
}
|
||||
|
||||
func TestStreamMetadataPartsCancelRetryWait(t *testing.T) {
|
||||
set, o, p, _ := listingCachePartsBackend(t, 1)
|
||||
putListingCachePart(t, set, o, 0, "a/", false)
|
||||
probed := make(chan struct{})
|
||||
p.onProbe = func(index int) bool {
|
||||
if index == 1 {
|
||||
close(probed)
|
||||
}
|
||||
return false
|
||||
}
|
||||
done, cancel := startListingCacheParts(t, set, o)
|
||||
select {
|
||||
case <-probed:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("retry probe did not run")
|
||||
}
|
||||
// Let the probe return and enter the regular 250 ms wait before canceling.
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
start := time.Now()
|
||||
cancel()
|
||||
result := receiveListingCacheParts(t, done)
|
||||
elapsed := time.Since(start)
|
||||
if !errors.Is(result.err, context.Canceled) || elapsed > 150*time.Millisecond {
|
||||
t.Fatalf("retry wait ignored cancellation: elapsed=%v err=%v", elapsed, result.err)
|
||||
}
|
||||
t.Logf("cancel during retry wait: elapsed=%v", elapsed)
|
||||
}
|
||||
|
||||
func TestStreamMetadataPartsChangedDisks(t *testing.T) {
|
||||
for _, empty := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("empty=%v", empty), func(t *testing.T) {
|
||||
testStreamMetadataPartsChangedDisks(t, empty)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func testStreamMetadataPartsChangedDisks(t *testing.T, empty bool) {
|
||||
set, o, p, switchDisks := listingCachePartsBackend(t, 1)
|
||||
putListingCachePart(t, set, o, 0, "a/", false)
|
||||
disks := set.getDisks()
|
||||
probeDone, restore := make(chan struct{}), make(chan struct{})
|
||||
p.onProbe = func(index int) bool {
|
||||
if index == 0 {
|
||||
// A later probe must safely skip an empty/offline disk snapshot.
|
||||
if empty {
|
||||
switchDisks([]StorageAPI{})
|
||||
} else {
|
||||
switchDisks([]StorageAPI{nil})
|
||||
}
|
||||
close(probeDone)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
p.onCheck = func(n int) {
|
||||
if n == 2 {
|
||||
close(restore)
|
||||
}
|
||||
}
|
||||
done, _ := startListingCacheParts(t, set, o)
|
||||
select {
|
||||
case <-probeDone:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("initial retry probe did not run")
|
||||
}
|
||||
select {
|
||||
case <-restore:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("empty disk snapshot prevented cache state checks")
|
||||
}
|
||||
switchDisks(disks)
|
||||
putListingCachePart(t, set, o, 1, "b/", true)
|
||||
result := receiveListingCacheParts(t, done)
|
||||
if !errors.Is(result.err, io.EOF) || result.entries.len() != 2 {
|
||||
t.Fatalf("changed disks did not recover: entries=%v err=%v", result.entries.o, result.err)
|
||||
}
|
||||
}
|
||||
+226
-14
@@ -191,19 +191,19 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
|
||||
return entries, err
|
||||
}
|
||||
entries.truncate(0)
|
||||
go func() {
|
||||
rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix))
|
||||
rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix))
|
||||
go func(o listPathOptions) {
|
||||
if rpc != nil {
|
||||
ctx, cancel := context.WithTimeout(GlobalContext, 5*time.Second)
|
||||
defer cancel()
|
||||
c, err := rpc.GetMetacacheListing(ctx, *o)
|
||||
c, err := rpc.GetMetacacheListing(ctx, o)
|
||||
if err == nil {
|
||||
c.error = "no longer used"
|
||||
c.status = scanStateError
|
||||
rpc.UpdateMetacacheListing(ctx, *c)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}(*o)
|
||||
o.ID = ""
|
||||
}
|
||||
|
||||
@@ -264,6 +264,176 @@ func (z *erasureServerPools) listPathShortcutEOF(ctx context.Context, bucket str
|
||||
return io.EOF
|
||||
}
|
||||
|
||||
// listingRLock covers the lock domains used by normal same-key mutations.
|
||||
// Multi-pool writes use the pools lock. In a single pool, DELETE/COPY use the
|
||||
// first set while PUT and metadata updates use the destination set, so recovery
|
||||
// takes both in that order. No writer routing or wire protocol changes here.
|
||||
func (z *erasureServerPools) listingRLock(ctx context.Context, bucket, object string, source *erasureObjects) (LockContext, func(), error) {
|
||||
firstSet := z.serverPools[0].sets[0]
|
||||
first := z.NewNSLock(bucket, object)
|
||||
firstCtx, err := first.GetRLock(ctx, globalListingOperationTimeout)
|
||||
if err != nil {
|
||||
return LockContext{}, nil, err
|
||||
}
|
||||
if !z.SinglePool() || source == firstSet || (!source.nsMutex.isDistErasure && source.nsMutex == firstSet.nsMutex) {
|
||||
return firstCtx, func() { first.RUnlock(firstCtx) }, nil
|
||||
}
|
||||
// Shared nsMutex pointers do not prove shared distributed locker sets.
|
||||
second := source.NewNSLock(bucket, object)
|
||||
secondCtx, err := second.GetRLock(firstCtx.Context(), globalListingOperationTimeout)
|
||||
if err != nil {
|
||||
first.RUnlock(firstCtx)
|
||||
return LockContext{}, nil, err
|
||||
}
|
||||
return secondCtx, func() {
|
||||
second.RUnlock(secondCtx)
|
||||
first.RUnlock(firstCtx)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// resolveListingGroup recovers a complete same-name group, never a private
|
||||
// per-set contribution. A strictly absent source requires current reads from
|
||||
// every pool, since another pool's earlier walk may predate a migration copy.
|
||||
func (z *erasureServerPools) resolveListingGroup(ctx context.Context, bucket string, group []*metaCacheEntry) (*metaCacheEntry, error) {
|
||||
var source *erasureObjects
|
||||
for _, entry := range group {
|
||||
if entry.listingSet != nil {
|
||||
source = entry.listingSet
|
||||
break
|
||||
}
|
||||
}
|
||||
if source == nil {
|
||||
return nil, errInvalidArgument
|
||||
}
|
||||
// Recovery consumes this group. Return reusable walk buffers only after
|
||||
// their contents have been copied into the newly encoded result.
|
||||
defer func() {
|
||||
for _, entry := range group {
|
||||
if entry.reusable {
|
||||
metaDataPoolPut(entry.metadata)
|
||||
}
|
||||
}
|
||||
}()
|
||||
name := group[0].name
|
||||
object := encodeDirObject(name)
|
||||
if z.SinglePool() {
|
||||
target := z.serverPools[0].getHashedSet(object)
|
||||
for _, entry := range group {
|
||||
if entry.listingSet != nil && entry.listingSet != target {
|
||||
// Unexpected placement is not evidence of absence at the
|
||||
// current write target. Do not extend recovery to arbitrary sets.
|
||||
return nil, listingReadQuorumError(bucket, object)
|
||||
}
|
||||
}
|
||||
}
|
||||
type readResult struct {
|
||||
fi FileInfo
|
||||
absent bool
|
||||
}
|
||||
reads := make(map[*erasureObjects]readResult)
|
||||
currentOnly := false
|
||||
var current *FileInfo
|
||||
err := func() error {
|
||||
lockCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
lkctx, unlock, err := z.listingRLock(lockCtx, bucket, object, source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer unlock()
|
||||
read := func(set *erasureObjects) (readResult, error) {
|
||||
if result, ok := reads[set]; ok {
|
||||
return result, nil
|
||||
}
|
||||
fi, absent, err := set.readListingFileInfo(lkctx.Context(), bucket, object)
|
||||
result := readResult{fi: fi, absent: absent}
|
||||
if err == nil {
|
||||
reads[set] = result
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
for _, entry := range group {
|
||||
if entry.listingSet == nil {
|
||||
continue
|
||||
}
|
||||
result, err := read(entry.listingSet)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
currentOnly = currentOnly || result.absent
|
||||
}
|
||||
if !currentOnly {
|
||||
return lkctx.Context().Err()
|
||||
}
|
||||
// Migration publishes the replacement through the pooled write lock
|
||||
// before source cleanup. While this read lock is held, new publication
|
||||
// cannot pass these reads; any concurrent cleanup already has a copy.
|
||||
// Unknown pools still fail rather than being counted as absent.
|
||||
for _, pool := range z.serverPools {
|
||||
result, err := read(pool.getHashedSet(object))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if result.absent {
|
||||
continue
|
||||
}
|
||||
// Match latest GET: newest ModTime, then lowest pool index.
|
||||
if current == nil || result.fi.ModTime.After(current.ModTime) {
|
||||
fi := result.fi
|
||||
current = &fi
|
||||
}
|
||||
}
|
||||
return lkctx.Context().Err()
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// All FileInfo values above are owned decoded results. Encoding and merge
|
||||
// allocation happen after RUnlock, as do all stream sends and cache RPCs.
|
||||
encode := func(fi FileInfo) (*metaCacheEntry, error) {
|
||||
xl := &xlMetaV2{}
|
||||
if err := xl.AddVersion(fi); err != nil {
|
||||
return nil, listingReadQuorumError(bucket, object)
|
||||
}
|
||||
entry := &metaCacheEntry{name: name, cached: xl, reusable: true}
|
||||
var err error
|
||||
entry.metadata, err = xl.AppendTo(metaDataPoolGet())
|
||||
return entry, err
|
||||
}
|
||||
if currentOnly {
|
||||
if current == nil {
|
||||
return nil, nil
|
||||
}
|
||||
// Replace the whole group: no stale live copy or marker can override
|
||||
// this current global decision.
|
||||
return encode(*current)
|
||||
}
|
||||
versions := make([][]xlMetaV2ShallowVersion, 0, len(group))
|
||||
for _, entry := range group {
|
||||
if entry.listingSet != nil {
|
||||
fi := reads[entry.listingSet].fi
|
||||
xl := &xlMetaV2{}
|
||||
if err := xl.AddVersion(fi); err != nil {
|
||||
return nil, listingReadQuorumError(bucket, object)
|
||||
}
|
||||
versions = append(versions, xl.versions)
|
||||
continue
|
||||
}
|
||||
xl, err := entry.xlmeta()
|
||||
if err != nil {
|
||||
return nil, listingReadQuorumError(bucket, object)
|
||||
}
|
||||
versions = append(versions, xl.versions)
|
||||
}
|
||||
// Preserve the existing cross-pool merge rule for recovered live metadata
|
||||
// and delete markers when no source was strictly absent.
|
||||
xl := &xlMetaV2{versions: mergeXLV2Versions(1, true, 0, versions...)}
|
||||
// Do not retain shallow versions referencing reusable input buffers.
|
||||
resolved := &metaCacheEntry{name: name, reusable: true}
|
||||
resolved.metadata, err = xl.AppendTo(metaDataPoolGet())
|
||||
return resolved, err
|
||||
}
|
||||
|
||||
// listMerged will list across all sets and return a merged results stream.
|
||||
// The result channel is closed when no more results are expected.
|
||||
func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions, results chan<- metaCacheEntry) error {
|
||||
@@ -285,11 +455,14 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
|
||||
defer wg.Done()
|
||||
err := set.listPath(listCtx, o, innerResults)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if err == nil {
|
||||
allAtEOF = false
|
||||
}
|
||||
errs[i] = err
|
||||
mu.Unlock()
|
||||
// The merger must observe the terminal error before treating
|
||||
// this stream as exhausted and advancing to another pool.
|
||||
xioutil.SafeClose(innerResults)
|
||||
}(len(errs), set)
|
||||
errs = append(errs, nil)
|
||||
}
|
||||
@@ -298,7 +471,18 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
|
||||
|
||||
// Gather results to a single channel.
|
||||
// Quorum is one since we are merging across sets.
|
||||
err := mergeEntryChannels(ctx, inputs, results, 1)
|
||||
err := mergeEntryChannelsWithResolver(ctx, inputs, results, 1, func(i int) error {
|
||||
mu.Lock()
|
||||
err := errs[i]
|
||||
mu.Unlock()
|
||||
// Keep the existing aggregate handling of absent paths and buckets.
|
||||
if errors.Is(err, io.EOF) || isAllNotFound([]error{err}) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}, func(group []*metaCacheEntry) (*metaCacheEntry, error) {
|
||||
return z.resolveListingGroup(ctx, o.Bucket, group)
|
||||
})
|
||||
|
||||
cancelList()
|
||||
wg.Wait()
|
||||
@@ -415,13 +599,24 @@ func (z *erasureServerPools) listAndSave(ctx context.Context, o *listPathOptions
|
||||
cancel()
|
||||
}()
|
||||
|
||||
// Do listing...
|
||||
// Publish the typed terminal result before either consumer observes EOF.
|
||||
// mergeEntryChannels closes inCh before listMerged has inspected set errors.
|
||||
listingDone := make(chan struct{})
|
||||
var cacheErrorDone chan struct{}
|
||||
var listingErr error
|
||||
go func(o listPathOptions) {
|
||||
err := z.listMerged(listCtx, o, inCh)
|
||||
if err != nil {
|
||||
meta.setErr(err.Error())
|
||||
listingErr = z.listMerged(listCtx, o, inCh)
|
||||
failed := listingErr != nil && !errors.Is(listingErr, io.EOF)
|
||||
if failed {
|
||||
cacheErrorDone = make(chan struct{})
|
||||
}
|
||||
o.debugln("listAndSave: listing", o.ID, "finished with ", err)
|
||||
close(listingDone)
|
||||
if failed {
|
||||
meta.setErr(listingErr.Error())
|
||||
cancel()
|
||||
close(cacheErrorDone)
|
||||
}
|
||||
o.debugln("listAndSave: listing", o.ID, "finished with ", listingErr)
|
||||
}(*o)
|
||||
|
||||
// Keep track of when we return since we no longer have to send entries to output.
|
||||
@@ -451,16 +646,33 @@ func (z *erasureServerPools) listAndSave(ctx context.Context, o *listPathOptions
|
||||
entry.reusable = returned
|
||||
saveCh <- entry
|
||||
}
|
||||
// The typed scan result is enough to finish the HTTP response.
|
||||
<-listingDone
|
||||
if !returned {
|
||||
xioutil.SafeClose(outCh)
|
||||
}
|
||||
// Closing saveCh writes an end-of-stream cache block. On failure,
|
||||
// wait until the error is recorded and the writer is canceled.
|
||||
if cacheErrorDone != nil {
|
||||
<-cacheErrorDone
|
||||
}
|
||||
xioutil.SafeClose(saveCh)
|
||||
}()
|
||||
|
||||
entries, err = filteredResults()
|
||||
if err == nil {
|
||||
// Check if listing recorded an error.
|
||||
err = meta.getErr()
|
||||
if err == nil || errors.Is(err, io.EOF) {
|
||||
// Full pages may return before the scan ends. A final page has
|
||||
// crossed the completion barrier above and must preserve scan errors.
|
||||
select {
|
||||
case <-listingDone:
|
||||
if listingErr != nil && !errors.Is(listingErr, io.EOF) {
|
||||
return entries, listingErr
|
||||
}
|
||||
default:
|
||||
}
|
||||
if cacheErr := meta.getErr(); cacheErr != nil {
|
||||
return entries, cacheErr
|
||||
}
|
||||
}
|
||||
return entries, err
|
||||
}
|
||||
|
||||
+120
-79
@@ -39,7 +39,6 @@ import (
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
"github.com/minio/minio/internal/color"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/pgsty/silo-pkg/v3/console"
|
||||
)
|
||||
|
||||
@@ -423,63 +422,89 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
|
||||
retryDelay = 50 * time.Millisecond
|
||||
retryDelay250 = 250 * time.Millisecond
|
||||
)
|
||||
var lastStateCheck time.Time
|
||||
retryWait := func() error {
|
||||
if contextCanceled(ctx) {
|
||||
return ctx.Err()
|
||||
}
|
||||
// A failed producer cannot publish another block. Check on the first
|
||||
// failure, then at most once per retry interval while it is still running.
|
||||
if time.Since(lastStateCheck) >= retryDelay250 {
|
||||
lastStateCheck = time.Now()
|
||||
if err := o.checkMetacacheState(ctx, rpc); err != nil {
|
||||
return fmt.Errorf("remote listing canceled: %w", err)
|
||||
}
|
||||
}
|
||||
retries++
|
||||
delay := retryDelay250
|
||||
if retries == 1 {
|
||||
delay = retryDelay
|
||||
}
|
||||
timer := time.NewTimer(delay)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
// Probe one online disk per retry, rotating so a missing replica cannot
|
||||
// pin the reader to one disk. A miss only delays the next probe: it never
|
||||
// establishes absence or replaces the quorum read after a probe succeeds.
|
||||
// This bounds missing-block I/O without sleeping once for every disk.
|
||||
nextProbeDisk := 0
|
||||
probePart := func(part int) bool {
|
||||
disks := er.getDisks()
|
||||
for range disks {
|
||||
if contextCanceled(ctx) {
|
||||
return false
|
||||
}
|
||||
i := nextProbeDisk % len(disks)
|
||||
nextProbeDisk = i + 1
|
||||
disk := disks[i]
|
||||
if disk == nil || !disk.IsOnline() {
|
||||
continue
|
||||
}
|
||||
_, err := disk.ReadVersion(ctx, "", minioMetaBucket, o.objectPath(part), "", ReadOptions{})
|
||||
return err == nil
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
retryOtherError := false
|
||||
for {
|
||||
if contextCanceled(ctx) {
|
||||
return entries, ctx.Err()
|
||||
}
|
||||
|
||||
// If many failures, check the cache state.
|
||||
if retries > 10 {
|
||||
err := o.checkMetacacheState(ctx, rpc)
|
||||
if err != nil {
|
||||
return entries, fmt.Errorf("remote listing canceled: %w", err)
|
||||
}
|
||||
retries = 1
|
||||
}
|
||||
|
||||
// All operations are performed without locks, so we must be careful and allow for failures.
|
||||
// Read metadata associated with the object from a disk.
|
||||
if retries > 0 {
|
||||
for _, disk := range er.getDisks() {
|
||||
if disk == nil {
|
||||
continue
|
||||
}
|
||||
if !disk.IsOnline() {
|
||||
continue
|
||||
}
|
||||
_, err := disk.ReadVersion(ctx, "", minioMetaBucket,
|
||||
o.objectPath(0), "", ReadOptions{})
|
||||
if err != nil {
|
||||
time.Sleep(retryDelay250)
|
||||
retries++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
retryWait := func() {
|
||||
retries++
|
||||
if retries == 1 {
|
||||
time.Sleep(retryDelay)
|
||||
} else {
|
||||
time.Sleep(retryDelay250)
|
||||
if retries > 0 && !retryOtherError && !probePart(0) {
|
||||
if err := retryWait(); err != nil {
|
||||
return entries, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Load first part metadata...
|
||||
// Read metadata associated with the object from all disks.
|
||||
retryOtherError = false
|
||||
fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, minioMetaBucket, o.objectPath(0), ObjectOptions{}, true)
|
||||
if err != nil {
|
||||
switch toObjectErr(err, minioMetaBucket, o.objectPath(0)).(type) {
|
||||
case ObjectNotFound, InsufficientReadQuorum:
|
||||
retryWait()
|
||||
if err := retryWait(); err != nil {
|
||||
return entries, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Allow one fast retry for other errors.
|
||||
if retries > 0 {
|
||||
return entries, fmt.Errorf("reading first part metadata: %v", err)
|
||||
}
|
||||
retryWait()
|
||||
retryOtherError = true
|
||||
if err := retryWait(); err != nil {
|
||||
return entries, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -487,14 +512,9 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
|
||||
switch {
|
||||
case err == nil:
|
||||
case errors.Is(err, io.ErrUnexpectedEOF):
|
||||
if retries == 10 {
|
||||
err := o.checkMetacacheState(ctx, rpc)
|
||||
if err != nil {
|
||||
return entries, fmt.Errorf("remote listing canceled: %w", err)
|
||||
}
|
||||
retries = -1
|
||||
if err := retryWait(); err != nil {
|
||||
return entries, err
|
||||
}
|
||||
retryWait()
|
||||
continue
|
||||
case errors.Is(err, io.EOF):
|
||||
return entries, io.EOF
|
||||
@@ -508,39 +528,19 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
|
||||
}
|
||||
|
||||
if partN != loadedPart {
|
||||
if retries > 10 {
|
||||
err := o.checkMetacacheState(ctx, rpc)
|
||||
if err != nil {
|
||||
return entries, fmt.Errorf("waiting for next part %d: %w", partN, err)
|
||||
}
|
||||
retries = 1
|
||||
}
|
||||
|
||||
if retries > 0 {
|
||||
// Load from one disk only
|
||||
for _, disk := range er.getDisks() {
|
||||
if disk == nil {
|
||||
continue
|
||||
}
|
||||
if !disk.IsOnline() {
|
||||
continue
|
||||
}
|
||||
_, err := disk.ReadVersion(ctx, "", minioMetaBucket,
|
||||
o.objectPath(partN), "", ReadOptions{})
|
||||
if err != nil {
|
||||
time.Sleep(retryDelay250)
|
||||
retries++
|
||||
continue
|
||||
}
|
||||
break
|
||||
if retries > 0 && !probePart(partN) {
|
||||
if err := retryWait(); err != nil {
|
||||
return entries, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Load partN metadata...
|
||||
fi, metaArr, onlineDisks, err = er.getObjectFileInfo(ctx, minioMetaBucket, o.objectPath(partN), ObjectOptions{}, true)
|
||||
if err != nil {
|
||||
time.Sleep(retryDelay250)
|
||||
retries++
|
||||
if err := retryWait(); err != nil {
|
||||
return entries, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
loadedPart = partN
|
||||
@@ -576,13 +576,15 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
switch toObjectErr(err, minioMetaBucket, o.objectPath(partN)).(type) {
|
||||
case ObjectNotFound:
|
||||
retries++
|
||||
time.Sleep(retryDelay250)
|
||||
if err := retryWait(); err != nil {
|
||||
return entries, err
|
||||
}
|
||||
continue
|
||||
case InsufficientReadQuorum:
|
||||
retries++
|
||||
loadedPart = -1
|
||||
time.Sleep(retryDelay250)
|
||||
if err := retryWait(); err != nil {
|
||||
return entries, err
|
||||
}
|
||||
continue
|
||||
default:
|
||||
internalLogIf(ctx, err)
|
||||
@@ -700,9 +702,46 @@ func getQuorumDisks(disks []StorageAPI, infos []DiskInfo, readQuorum int) (newDi
|
||||
return newDisks
|
||||
}
|
||||
|
||||
// resolveListEntry keeps the existing walk resolution and fallback eligibility.
|
||||
// Eligible latest-only disagreements are resolved at the pool merger, where a
|
||||
// current absence can replace the entire same-name group, including stale walks.
|
||||
func (er *erasureObjects) resolveListEntry(entries metaCacheEntries, errs []error, resolver *metadataResolutionParams) *metaCacheEntry {
|
||||
entry, ok := entries.resolve(resolver)
|
||||
if ok {
|
||||
return entry
|
||||
}
|
||||
if resolver.requestedVersions != 1 {
|
||||
return nil
|
||||
}
|
||||
valid := 0
|
||||
for i := range entries {
|
||||
if !entries[i].isObject() {
|
||||
continue
|
||||
}
|
||||
if _, err := entries[i].xlmeta(); err != nil {
|
||||
continue
|
||||
}
|
||||
valid++
|
||||
if entry == nil {
|
||||
entry = &entries[i]
|
||||
}
|
||||
}
|
||||
for _, err := range errs {
|
||||
if err != nil {
|
||||
valid++
|
||||
}
|
||||
}
|
||||
if entry == nil || valid < resolver.objQuorum {
|
||||
return nil
|
||||
}
|
||||
// This entry exists only in the internal set-to-pool stream. Do not retain
|
||||
// an arbitrary walk snapshot or encode the pending state as metadata.
|
||||
return &metaCacheEntry{name: entry.name, listingSet: er}
|
||||
}
|
||||
|
||||
// Will return io.EOF if continuing would not yield more results.
|
||||
func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions, results chan<- metaCacheEntry) (err error) {
|
||||
defer xioutil.SafeClose(results)
|
||||
// The caller records the terminal error before closing results.
|
||||
o.debugf(color.Green("listPath:")+" with options: %#v", o)
|
||||
|
||||
// get prioritized non-healing disks for listing
|
||||
@@ -778,8 +817,8 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions, resul
|
||||
},
|
||||
partial: func(entries metaCacheEntries, errs []error) {
|
||||
// Results Disagree :-(
|
||||
entry, ok := entries.resolve(&resolver)
|
||||
if ok {
|
||||
entry := er.resolveListEntry(entries, errs, &resolver)
|
||||
if entry != nil {
|
||||
select {
|
||||
case <-ctxDone:
|
||||
case results <- *entry:
|
||||
@@ -809,7 +848,9 @@ func (m *metaCacheRPC) setErr(err string) {
|
||||
// An error is already set.
|
||||
return
|
||||
}
|
||||
meta, _ = m.o.updateMetacacheListing(meta, m.rpc)
|
||||
if updated, err := m.o.updateMetacacheListing(meta, m.rpc); err == nil {
|
||||
meta = updated
|
||||
}
|
||||
*m.meta = meta
|
||||
}
|
||||
|
||||
|
||||
@@ -126,6 +126,9 @@ func (w *metacacheWriter) write(objs ...metaCacheEntry) error {
|
||||
}
|
||||
}
|
||||
for _, o := range objs {
|
||||
if o.listingSet != nil {
|
||||
return errors.New("metacacheWriter: unresolved listing entry")
|
||||
}
|
||||
if len(o.name) == 0 {
|
||||
return errors.New("metacacheWriter: no name provided")
|
||||
}
|
||||
@@ -170,6 +173,9 @@ func (w *metacacheWriter) stream() (chan<- metaCacheEntry, error) {
|
||||
go func() {
|
||||
defer w.streamWg.Done()
|
||||
for o := range objs {
|
||||
if o.listingSet != nil && w.streamErr == nil {
|
||||
w.streamErr = errors.New("metacacheWriter: unresolved listing entry")
|
||||
}
|
||||
if len(o.name) == 0 || w.streamErr != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -69,6 +69,8 @@ func loadCPUMetrics(ctx context.Context, m MetricValues, c *metricsCache) error
|
||||
|
||||
// metrics-resource.go runs a job to collect resource metrics including their Avg values and
|
||||
// stores them in resourceMetricsMap. We can use it to get the Avg values of CPU idle and IOWait.
|
||||
resourceMetricsMapMu.RLock()
|
||||
defer resourceMetricsMapMu.RUnlock()
|
||||
cpuResourceMetrics, found := resourceMetricsMap[cpuSubsystem]
|
||||
if found {
|
||||
if cpuIdleMetric, ok := cpuResourceMetrics[getResourceKey(cpuIdle, nil)]; ok {
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
// Copyright (c) 2026 Ruohang Feng
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/cachevalue"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
cpustats "github.com/shirou/gopsutil/v3/cpu"
|
||||
"github.com/shirou/gopsutil/v3/load"
|
||||
)
|
||||
|
||||
// These tests replace global resource metrics and must not run in parallel.
|
||||
// Concurrent writers must finish before the fixture is restored.
|
||||
func setCPUResourceMetricsForTest(t *testing.T, value map[MetricSubsystem]ResourceMetrics) {
|
||||
t.Helper()
|
||||
resourceMetricsMapMu.Lock()
|
||||
saved := resourceMetricsMap
|
||||
resourceMetricsMap = value
|
||||
resourceMetricsMapMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
resourceMetricsMapMu.Lock()
|
||||
resourceMetricsMap = saved
|
||||
resourceMetricsMapMu.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
func newCPUMetricsTestRegistry() *prometheus.Registry {
|
||||
c := &metricsCache{cpuMetrics: cachevalue.NewFromFunc(time.Hour, cachevalue.Opts{},
|
||||
func(context.Context) (madmin.CPUMetrics, error) {
|
||||
return madmin.CPUMetrics{
|
||||
CPUCount: 4,
|
||||
LoadStat: &load.AvgStat{Load1: 2},
|
||||
TimesStat: &cpustats.TimesStat{
|
||||
User: 10, System: 20, Idle: 60, Iowait: 5, Nice: 3, Steal: 2,
|
||||
},
|
||||
}, nil
|
||||
})}
|
||||
_, _ = c.cpuMetrics.Get() // Warm the cache: the cache does not protect the resource map.
|
||||
g := NewMetricsGroup(systemCPUCollectorPath, []MetricDescriptor{
|
||||
sysCPUAvgIdleMD, sysCPUAvgIOWaitMD, sysCPULoadMD, sysCPULoadPercMD,
|
||||
sysCPUNiceMD, sysCPUStealMD, sysCPUSystemMD, sysCPUUserMD,
|
||||
}, loadCPUMetrics)
|
||||
g.SetCache(c)
|
||||
r := prometheus.NewPedanticRegistry()
|
||||
r.MustRegister(g)
|
||||
return r
|
||||
}
|
||||
|
||||
func TestLoadCPUMetricsValues(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
data map[MetricSubsystem]ResourceMetrics
|
||||
want map[string]float64
|
||||
}{
|
||||
{name: "nil-map"},
|
||||
{name: "empty-map", data: map[MetricSubsystem]ResourceMetrics{}},
|
||||
{name: "no-cpu", data: map[MetricSubsystem]ResourceMetrics{memSubsystem: {}}},
|
||||
{name: "nil-cpu", data: map[MetricSubsystem]ResourceMetrics{cpuSubsystem: nil}},
|
||||
{name: "empty-cpu", data: map[MetricSubsystem]ResourceMetrics{cpuSubsystem: {}}},
|
||||
{name: "idle-only", data: map[MetricSubsystem]ResourceMetrics{cpuSubsystem: {
|
||||
getResourceKey(cpuIdle, nil): {Avg: 87.654},
|
||||
}}, want: map[string]float64{"minio_system_cpu_avg_idle": 87.65}},
|
||||
{name: "iowait-only", data: map[MetricSubsystem]ResourceMetrics{cpuSubsystem: {
|
||||
getResourceKey(cpuIOWait, nil): {Avg: 1.236},
|
||||
}}, want: map[string]float64{"minio_system_cpu_avg_iowait": 1.24}},
|
||||
{name: "both-rounded", data: map[MetricSubsystem]ResourceMetrics{cpuSubsystem: {
|
||||
getResourceKey(cpuIdle, nil): {Avg: 87.654},
|
||||
getResourceKey(cpuIOWait, nil): {Avg: 1.236},
|
||||
}}, want: map[string]float64{"minio_system_cpu_avg_idle": 87.65, "minio_system_cpu_avg_iowait": 1.24}},
|
||||
{name: "zero-keeps-existing-omission", data: map[MetricSubsystem]ResourceMetrics{cpuSubsystem: {
|
||||
getResourceKey(cpuIdle, nil): {Avg: 0},
|
||||
getResourceKey(cpuIOWait, nil): {Avg: 0},
|
||||
}}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
setCPUResourceMetricsForTest(t, tc.data)
|
||||
families, err := newCPUMetricsTestRegistry().Gather()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]float64{
|
||||
"minio_system_cpu_load": 2, "minio_system_cpu_load_perc": 50,
|
||||
"minio_system_cpu_nice": 3, "minio_system_cpu_steal": 2,
|
||||
"minio_system_cpu_system": 20, "minio_system_cpu_user": 10,
|
||||
}
|
||||
for k, v := range tc.want {
|
||||
want[k] = v
|
||||
}
|
||||
if len(families) != len(want) {
|
||||
t.Fatalf("got %d families, want %d", len(families), len(want))
|
||||
}
|
||||
for _, family := range families {
|
||||
v, ok := want[family.GetName()]
|
||||
if !ok || family.GetType() != dto.MetricType_GAUGE || len(family.Metric) != 1 || family.Metric[0].GetGauge().GetValue() != v {
|
||||
t.Errorf("unexpected family: %v (want %v)", family, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadCPUMetricsConcurrentUpdate(t *testing.T) {
|
||||
for _, subsystem := range []MetricSubsystem{cpuSubsystem, memSubsystem} {
|
||||
for _, readers := range []int{1, 4} {
|
||||
t.Run(fmt.Sprintf("writer-%s/readers-%d", subsystem, readers), func(t *testing.T) {
|
||||
setCPUResourceMetricsForTest(t, map[MetricSubsystem]ResourceMetrics{})
|
||||
updateResourceMetrics(cpuSubsystem, cpuIdle, 80, nil, false)
|
||||
updateResourceMetrics(cpuSubsystem, cpuIOWait, 5, nil, false)
|
||||
r := newCPUMetricsTestRegistry()
|
||||
stop := make(chan struct{})
|
||||
done := make(chan struct{})
|
||||
ready := make(chan struct{})
|
||||
var updates atomic.Uint64
|
||||
go func() {
|
||||
defer close(done)
|
||||
if subsystem == cpuSubsystem {
|
||||
updateResourceMetrics(subsystem, cpuIdle, 80, nil, false)
|
||||
} else {
|
||||
updateResourceMetrics(subsystem, memUsed, 1024, nil, false)
|
||||
}
|
||||
close(ready)
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
default:
|
||||
}
|
||||
if subsystem == cpuSubsystem {
|
||||
updateResourceMetrics(subsystem, cpuIdle, 80, nil, false)
|
||||
updateResourceMetrics(subsystem, cpuIOWait, 5, nil, false)
|
||||
} else {
|
||||
updateResourceMetrics(subsystem, memUsed, 1024, nil, false)
|
||||
}
|
||||
updates.Add(1)
|
||||
runtime.Gosched()
|
||||
}
|
||||
}()
|
||||
<-ready
|
||||
defer func() { close(stop); <-done }()
|
||||
var wg sync.WaitGroup
|
||||
for range readers {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for range 1000 {
|
||||
families, err := r.Gather()
|
||||
if err != nil || len(families) != 8 {
|
||||
t.Errorf("Gather: families=%d, error=%v", len(families), err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if updates.Load() == 0 {
|
||||
t.Fatal("writer made no progress")
|
||||
}
|
||||
t.Logf("completed %d gathers with %d concurrent update iterations", readers*1000, updates.Load())
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -162,7 +162,7 @@ func toObjectErr(oerr error, params ...string) error {
|
||||
apiErr.Type = v.Type
|
||||
}
|
||||
return apiErr
|
||||
case errErasureWriteQuorum.Error():
|
||||
case errErasureWriteQuorum.Error(), errCommitMetaMissing.Error():
|
||||
apiErr := InsufficientWriteQuorum{}
|
||||
if len(params) >= 1 {
|
||||
apiErr.Bucket = params[0]
|
||||
@@ -668,7 +668,7 @@ var (
|
||||
func completeMultipartChecksumMismatch(algorithm string) error {
|
||||
description := "The checksum you specified did not match the calculated checksum."
|
||||
if algorithm != "" {
|
||||
description = fmt.Sprintf("The %s checksum you specified did not match the calculated checksum.", algorithm)
|
||||
description = fmt.Sprintf("The %s you specified did not match the calculated checksum.", algorithm)
|
||||
}
|
||||
return fmt.Errorf("%w: %s", errCompleteMultipartChecksumMismatch, description)
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
@@ -83,7 +84,8 @@ func checkListMultipartArgs(ctx context.Context, bucket, prefix, keyMarker, uplo
|
||||
if err := checkListObjsArgs(ctx, bucket, prefix, keyMarker); err != nil {
|
||||
return err
|
||||
}
|
||||
if uploadIDMarker != "" {
|
||||
// S3 ignores upload-id-marker when key-marker is absent.
|
||||
if uploadIDMarker != "" && keyMarker != "" {
|
||||
if HasSuffix(keyMarker, SlashSeparator) {
|
||||
return InvalidUploadIDKeyCombination{
|
||||
UploadIDMarker: uploadIDMarker,
|
||||
@@ -96,6 +98,9 @@ func checkListMultipartArgs(ctx context.Context, bucket, prefix, keyMarker, uplo
|
||||
UploadID: uploadIDMarker,
|
||||
}
|
||||
}
|
||||
if _, ok := multipartMarkerTime(uploadIDMarker); !ok {
|
||||
return InvalidArgument{Bucket: bucket, Object: keyMarker, Err: errors.New("upload-id-marker must contain a native multipart upload ID")}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -347,6 +347,11 @@ func putOpts(ctx context.Context, bucket, object, vid string, hdrs http.Header,
|
||||
versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)
|
||||
|
||||
vid = strings.TrimSpace(vid)
|
||||
// A caller-chosen version ID writes that version in place. Only trusted
|
||||
// replication may recreate a source version under its original ID.
|
||||
if vid != "" && !trustedReplication {
|
||||
return opts, errAuthentication
|
||||
}
|
||||
if vid != "" && vid != nullVersionID {
|
||||
_, err := uuid.Parse(vid)
|
||||
if err != nil {
|
||||
|
||||
@@ -62,12 +62,13 @@ func testAPIPutObjectChunkedChecksum(obj ObjectLayer, instanceType, bucketName s
|
||||
// A well-formed 4-byte value that does not match the content.
|
||||
wrongCRC := base64.StdEncoding.EncodeToString([]byte{0x01, 0x02, 0x03, 0x04})
|
||||
|
||||
apiCode := func(rec *httptest.ResponseRecorder) string {
|
||||
apiErrOf := func(rec *httptest.ResponseRecorder) APIErrorResponse {
|
||||
var apiErr APIErrorResponse
|
||||
b, _ := io.ReadAll(rec.Body)
|
||||
_ = xml.Unmarshal(b, &apiErr)
|
||||
return apiErr.Code
|
||||
return apiErr
|
||||
}
|
||||
apiCode := func(rec *httptest.ResponseRecorder) string { return apiErrOf(rec).Code }
|
||||
|
||||
// newChunkedJavaForm builds a non-trailer signed chunked PutObject request that
|
||||
// mirrors the AWS Java SDK v2 wire form: the checksum value sits in the header
|
||||
@@ -131,8 +132,12 @@ func testAPIPutObjectChunkedChecksum(obj ObjectLayer, instanceType, bucketName s
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("%s: chunked+wrong CRC32: expected 400, got %d", instanceType, rec.Code)
|
||||
}
|
||||
if code := apiCode(rec); code != "XAmzContentChecksumMismatch" {
|
||||
t.Fatalf("%s: chunked+wrong CRC32: want XAmzContentChecksumMismatch, got %q", instanceType, code)
|
||||
gotErr := apiErrOf(rec)
|
||||
if gotErr.Code != "XAmzContentChecksumMismatch" {
|
||||
t.Fatalf("%s: chunked+wrong CRC32: want XAmzContentChecksumMismatch, got %q", instanceType, gotErr.Code)
|
||||
}
|
||||
if gotErr.Message != "The CRC32 you specified did not match the calculated checksum." {
|
||||
t.Fatalf("%s: chunked+wrong CRC32: message = %q, want S3 algorithm-specific wording", instanceType, gotErr.Message)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -240,7 +240,10 @@ func checkPreconditionsPUT(ctx context.Context, w http.ResponseWriter, r *http.R
|
||||
// updated. The predicate is the incoming request's restored SSE-C metadata,
|
||||
// not what the destination happens to hold.
|
||||
ssecReplica := isReplicaTrusted(r.Context()) && crypto.SSEC.IsEncrypted(opts.UserDefined)
|
||||
if etagMatch && vidMatch && !ssecReplica {
|
||||
// Matching content does not imply that its tag revision was delivered.
|
||||
// Keep client preconditions above; relax only the internal duplicate check.
|
||||
newerTags := isReplicaTrusted(r.Context()) && olderThan(objInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp], opts.ReplicationSourceTaggingTimestamp)
|
||||
if etagMatch && vidMatch && !ssecReplica && !newerTags {
|
||||
writeHeaders()
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL)
|
||||
return true
|
||||
|
||||
+48
-30
@@ -1797,6 +1797,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
// source timestamp is newer than the stored one, and a stale update must
|
||||
// leave the stored state in place instead of erasing it.
|
||||
storedLock := storedObjectLockState(srcInfo.UserDefined)
|
||||
storedTagTimestamp := srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]
|
||||
|
||||
srcInfo.UserDefined, err = getCpObjMetadataFromHeader(ctx, r, srcInfo.UserDefined, allowReplicationMetadata)
|
||||
if err != nil {
|
||||
@@ -1814,23 +1815,29 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
}
|
||||
|
||||
if objTags != "" {
|
||||
lastTaggingTimestamp := srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]
|
||||
if dstOpts.ReplicationRequest {
|
||||
srcTimestamp := dstOpts.ReplicationSourceTaggingTimestamp
|
||||
if !srcTimestamp.IsZero() {
|
||||
ondiskTimestamp, err := time.Parse(time.RFC3339Nano, lastTaggingTimestamp)
|
||||
// update tagging metadata only if replica timestamp is newer than what's on disk
|
||||
if err != nil || (err == nil && !ondiskTimestamp.After(srcTimestamp)) {
|
||||
srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = srcTimestamp.UTC().Format(time.RFC3339Nano)
|
||||
srcInfo.UserDefined[xhttp.AmzObjectTagging] = objTags
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if dstOpts.ReplicationRequest {
|
||||
srcTimestamp := dstOpts.ReplicationSourceTaggingTimestamp
|
||||
if !srcTimestamp.IsZero() {
|
||||
// An empty value with a timestamp is an ordered deletion. Recheck
|
||||
// the captured state even if metadata REPLACE rebuilt the map.
|
||||
srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = srcTimestamp.UTC().Format(time.RFC3339Nano)
|
||||
srcInfo.UserDefined[xhttp.AmzObjectTagging] = objTags
|
||||
srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
reconcileStoredObjectTags(srcInfo.UserDefined, srcInfo.UserTags, storedTagTimestamp)
|
||||
} else {
|
||||
srcInfo.UserDefined[xhttp.AmzObjectTagging] = srcInfo.UserTags
|
||||
if storedTagTimestamp != "" {
|
||||
srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = storedTagTimestamp
|
||||
} else {
|
||||
delete(srcInfo.UserDefined, ReservedMetadataPrefixLower+TaggingTimestamp)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
srcInfo.UserDefined[xhttp.AmzObjectTagging] = objTags
|
||||
srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
}
|
||||
// SSE-C rotation snapshots reserved metadata before the tag decision. Its
|
||||
// later merge must not put the old timestamp back over the accepted state.
|
||||
delete(encMetadata, ReservedMetadataPrefixLower+TaggingTimestamp)
|
||||
|
||||
srcInfo.UserDefined = filterReplicationStatusMetadata(srcInfo.UserDefined)
|
||||
srcInfo.UserDefined = objectlock.FilterObjectLockMetadata(srcInfo.UserDefined, true, true)
|
||||
@@ -2313,6 +2320,9 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if opts.ReplicationRequest && !opts.ReplicationSourceTaggingTimestamp.IsZero() {
|
||||
metadata[ReservedMetadataPrefixLower+TaggingTimestamp] = opts.ReplicationSourceTaggingTimestamp.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
actualSize := size
|
||||
var idxCb func() []byte
|
||||
@@ -3425,27 +3435,16 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
// Check permissions to perform this object retention operation
|
||||
if s3Error := authenticateRequest(ctx, r, policy.PutObjectRetentionAction); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if !validateLengthAndChecksum(r) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if rcfg, _ := globalBucketObjectLockSys.Get(bucket); !rcfg.LockEnabled {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketObjectLockConfiguration), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
objRetention, err := objectlock.ParseObjectRetention(r.Body)
|
||||
if err != nil {
|
||||
apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
|
||||
@@ -3457,6 +3456,24 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
reqInfo.SetTags("retention", objRetention.String())
|
||||
|
||||
// Check permissions to perform this object retention operation. The
|
||||
// object-lock condition keys come from the request body, so this is
|
||||
// decided before any bucket or object state is read.
|
||||
if !isRetentionActionAllowed(policy.PutObjectRetentionAction, bucket, object, objRetention, r, reqInfo.Cred, reqInfo.Owner) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if rcfg, _ := globalBucketObjectLockSys.Get(bucket); !rcfg.LockEnabled {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketObjectLockConfiguration), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
opts, err := getOpts(ctx, r, bucket, object)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
@@ -3760,11 +3777,11 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
|
||||
}
|
||||
|
||||
dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(objInfo.UserDefined, tagsStr, objInfo.ReplicationStatus, replication.MetadataReplicationType, opts))
|
||||
stamp := UTCNow().Format(time.RFC3339Nano)
|
||||
opts.UserDefined = map[string]string{ReservedMetadataPrefixLower + TaggingTimestamp: stamp}
|
||||
if dsc.ReplicateAny() {
|
||||
opts.UserDefined = make(map[string]string)
|
||||
opts.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
opts.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = stamp
|
||||
opts.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus()
|
||||
opts.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
// Put object tags
|
||||
@@ -3863,9 +3880,10 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
|
||||
}
|
||||
|
||||
dsc := mustReplicate(ctx, bucket, object, oi.getMustReplicateOptions(replication.MetadataReplicationType, opts))
|
||||
stamp := UTCNow().Format(time.RFC3339Nano)
|
||||
opts.UserDefined = map[string]string{ReservedMetadataPrefixLower + TaggingTimestamp: stamp}
|
||||
if dsc.ReplicateAny() {
|
||||
opts.UserDefined = make(map[string]string)
|
||||
opts.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
opts.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = stamp
|
||||
opts.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus()
|
||||
}
|
||||
|
||||
|
||||
@@ -312,6 +312,10 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
// Completion orders the upload's persisted tag state under the object lock.
|
||||
if opts.ReplicationRequest && !opts.ReplicationSourceTaggingTimestamp.IsZero() {
|
||||
metadata[ReservedMetadataPrefixLower+TaggingTimestamp] = opts.ReplicationSourceTaggingTimestamp.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
if r.Header.Get(xhttp.IfMatch) != "" {
|
||||
opts.HasIfMatch = true
|
||||
@@ -1162,7 +1166,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
|
||||
return
|
||||
}
|
||||
|
||||
if _, _, _, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, objectAPI.GetObjectInfo, ErrNone, ErrNone, false); s3Err != ErrNone {
|
||||
if _, _, _, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, objectAPI.GetObjectInfo, ErrNone, ErrNone, trustedReplication); s3Err != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,679 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
// Copyright (c) 2026 PGSTY
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/amztime"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
func retentionAuthzRequest(t *testing.T, router http.Handler, creds auth.Credentials, method, target string, body []byte, headers map[string]string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
var rd io.ReadSeeker
|
||||
if body != nil {
|
||||
rd = bytes.NewReader(body)
|
||||
}
|
||||
req, err := newTestSignedRequestV4(method, target, int64(len(body)), rd, creds.AccessKey, creds.SecretKey, headers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
func retentionBody(mode string, until time.Time) []byte {
|
||||
if mode == "" {
|
||||
return []byte(`<Retention xmlns="http://s3.amazonaws.com/doc/2006-03-01/"></Retention>`)
|
||||
}
|
||||
return fmt.Appendf(nil, `<Retention xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Mode>%s</Mode><RetainUntilDate>%s</RetainUntilDate></Retention>`,
|
||||
mode, until.UTC().Format(time.RFC3339))
|
||||
}
|
||||
|
||||
func putRetention(t *testing.T, router http.Handler, cred auth.Credentials, bucket, key, vid, mode string, until time.Time, bypass bool) int {
|
||||
t.Helper()
|
||||
body := retentionBody(mode, until)
|
||||
sum := md5.Sum(body)
|
||||
headers := map[string]string{xhttp.ContentMD5: base64.StdEncoding.EncodeToString(sum[:])}
|
||||
if bypass {
|
||||
headers[objectlock.AmzObjectLockBypassRetGovernance] = "true"
|
||||
}
|
||||
rec := retentionAuthzRequest(t, router, cred, http.MethodPut,
|
||||
makeTestTargetURL("", bucket, key, url.Values{"retention": {""}, "versionId": {vid}}), body, headers)
|
||||
return rec.Code
|
||||
}
|
||||
|
||||
func newRetentionPolicyUser(t *testing.T, instanceType, policyJSON string) auth.Credentials {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
accessKey, secretKey, err := auth.GenerateCredentials()
|
||||
if err != nil {
|
||||
t.Fatalf("%s: generate credentials: %v", instanceType, err)
|
||||
}
|
||||
creds := auth.Credentials{AccessKey: accessKey, SecretKey: secretKey}
|
||||
if _, err = globalIAMSys.CreateUser(ctx, creds.AccessKey, madmin.AddOrUpdateUserReq{
|
||||
SecretKey: creds.SecretKey,
|
||||
Status: madmin.AccountEnabled,
|
||||
}); err != nil {
|
||||
t.Fatalf("%s: create user: %v", instanceType, err)
|
||||
}
|
||||
parsed, err := policy.ParseConfig(strings.NewReader(policyJSON))
|
||||
if err != nil {
|
||||
t.Fatalf("%s: parse policy: %v", instanceType, err)
|
||||
}
|
||||
policyName := "retention-authz-" + mustGetUUID()
|
||||
if _, err = globalIAMSys.SetPolicy(ctx, policyName, *parsed); err != nil {
|
||||
t.Fatalf("%s: install policy: %v", instanceType, err)
|
||||
}
|
||||
if _, err = globalIAMSys.PolicyDBSet(ctx, creds.AccessKey, policyName, regUser, false); err != nil {
|
||||
t.Fatalf("%s: attach policy: %v", instanceType, err)
|
||||
}
|
||||
return creds
|
||||
}
|
||||
|
||||
// TestAPIPutObjectRetentionAuthorization covers SN-2026-015: the governance
|
||||
// bypass header is intent only; s3:PutObjectRetention is always required and
|
||||
// weakening unexpired GOVERNANCE also requires s3:BypassGovernanceRetention.
|
||||
func TestAPIPutObjectRetentionAuthorization(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPIPutObjectRetentionAuthorization,
|
||||
})
|
||||
}
|
||||
|
||||
func testAPIPutObjectRetentionAuthorization(obj ObjectLayer, instanceType, bucket string, router http.Handler, root auth.Credentials, t *testing.T) {
|
||||
enableBucketObjectLock(t, bucket)
|
||||
|
||||
noRet := newObjectAttributesAuthzUser(t, instanceType, bucket, `"s3:GetObject"`)
|
||||
retOnly := newObjectAttributesAuthzUser(t, instanceType, bucket, `"s3:PutObjectRetention"`)
|
||||
retBypass := newObjectAttributesAuthzUser(t, instanceType, bucket, `"s3:PutObjectRetention","s3:BypassGovernanceRetention"`)
|
||||
bypassOnly := newObjectAttributesAuthzUser(t, instanceType, bucket, `"s3:BypassGovernanceRetention"`)
|
||||
condDenyShort := newRetentionPolicyUser(t, instanceType, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:PutObjectRetention", "s3:BypassGovernanceRetention"],
|
||||
"Resource": ["arn:aws:s3:::`+bucket+`/*"]
|
||||
},
|
||||
{
|
||||
"Effect": "Deny",
|
||||
"Action": ["s3:PutObjectRetention"],
|
||||
"Resource": ["arn:aws:s3:::`+bucket+`/*"],
|
||||
"Condition": {"NumericLessThan": {"s3:object-lock-remaining-retention-days": "60"}}
|
||||
}
|
||||
]
|
||||
}`)
|
||||
explicitDenyBypass := newRetentionPolicyUser(t, instanceType, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:*"],
|
||||
"Resource": ["arn:aws:s3:::`+bucket+`/*"]
|
||||
},
|
||||
{
|
||||
"Effect": "Deny",
|
||||
"Action": ["s3:BypassGovernanceRetention"],
|
||||
"Resource": ["arn:aws:s3:::`+bucket+`/*"]
|
||||
}
|
||||
]
|
||||
}`)
|
||||
|
||||
far := UTCNow().Add(365 * 24 * time.Hour).Truncate(time.Second)
|
||||
nearer := UTCNow().Add(30 * 24 * time.Hour).Truncate(time.Second)
|
||||
farther := far.Add(24 * time.Hour)
|
||||
|
||||
put := func(t *testing.T, object, mode string) string {
|
||||
t.Helper()
|
||||
payload := []byte("retention authorization")
|
||||
opts := ObjectOptions{Versioned: true, UserDefined: map[string]string{}}
|
||||
if mode != "" {
|
||||
opts.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = mode
|
||||
opts.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(far)
|
||||
}
|
||||
info, err := obj.PutObject(t.Context(), bucket, object,
|
||||
mustGetPutObjReader(t, bytes.NewReader(payload), int64(len(payload)), "", ""), opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return info.VersionID
|
||||
}
|
||||
stored := func(t *testing.T, object, vid string) objectlock.ObjectRetention {
|
||||
t.Helper()
|
||||
info, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: vid})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return objectlock.GetObjectRetentionMeta(info.UserDefined)
|
||||
}
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
creds auth.Credentials
|
||||
existing string // stored retention mode on the version, until far
|
||||
mode string // requested mode, "" removes retention
|
||||
until time.Time
|
||||
bypass bool
|
||||
want int
|
||||
changed bool
|
||||
}{
|
||||
{name: "no-permission removal", creds: noRet, existing: "GOVERNANCE", bypass: true, want: http.StatusForbidden},
|
||||
{name: "no-permission mode change", creds: noRet, existing: "GOVERNANCE", mode: "COMPLIANCE", until: nearer, bypass: true, want: http.StatusForbidden},
|
||||
{name: "no-permission new compliance", creds: noRet, mode: "COMPLIANCE", until: nearer, bypass: true, want: http.StatusForbidden},
|
||||
{name: "retention-only shorten with header", creds: retOnly, existing: "GOVERNANCE", mode: "GOVERNANCE", until: nearer, bypass: true, want: http.StatusForbidden},
|
||||
{name: "retention-only removal with header", creds: retOnly, existing: "GOVERNANCE", bypass: true, want: http.StatusForbidden},
|
||||
{name: "retention-only mode change with header", creds: retOnly, existing: "GOVERNANCE", mode: "COMPLIANCE", until: nearer, bypass: true, want: http.StatusForbidden},
|
||||
{name: "retention-only shorten without header", creds: retOnly, existing: "GOVERNANCE", mode: "GOVERNANCE", until: nearer, want: http.StatusBadRequest},
|
||||
{name: "retention-only extend with header", creds: retOnly, existing: "GOVERNANCE", mode: "GOVERNANCE", until: farther, bypass: true, want: http.StatusOK, changed: true},
|
||||
{name: "retention-only new retention", creds: retOnly, mode: "GOVERNANCE", until: nearer, want: http.StatusOK, changed: true},
|
||||
{name: "bypass shorten with header", creds: retBypass, existing: "GOVERNANCE", mode: "GOVERNANCE", until: nearer, bypass: true, want: http.StatusOK, changed: true},
|
||||
{name: "bypass removal with header", creds: retBypass, existing: "GOVERNANCE", bypass: true, want: http.StatusOK, changed: true},
|
||||
{name: "bypass shorten without header", creds: retBypass, existing: "GOVERNANCE", mode: "GOVERNANCE", until: nearer, want: http.StatusBadRequest},
|
||||
{name: "bypass shorten compliance", creds: retBypass, existing: "COMPLIANCE", mode: "COMPLIANCE", until: nearer, bypass: true, want: http.StatusBadRequest},
|
||||
{name: "bypass-only extend", creds: bypassOnly, existing: "GOVERNANCE", mode: "GOVERNANCE", until: farther, bypass: true, want: http.StatusForbidden},
|
||||
{name: "conditional-deny-put-retention with bypass header", creds: condDenyShort, existing: "GOVERNANCE", mode: "GOVERNANCE", until: nearer, bypass: true, want: http.StatusForbidden},
|
||||
{name: "conditional-allow-put-retention satisfied", creds: condDenyShort, existing: "GOVERNANCE", mode: "GOVERNANCE", until: farther, bypass: true, want: http.StatusOK, changed: true},
|
||||
{name: "explicit-deny-bypass shorten with header", creds: explicitDenyBypass, existing: "GOVERNANCE", mode: "GOVERNANCE", until: nearer, bypass: true, want: http.StatusForbidden},
|
||||
{name: "owner shorten with header", creds: root, existing: "GOVERNANCE", mode: "GOVERNANCE", until: nearer, bypass: true, want: http.StatusOK, changed: true},
|
||||
} {
|
||||
t.Run(instanceType+"/"+test.name, func(t *testing.T) {
|
||||
object := "retention-authz/" + strings.ReplaceAll(test.name, " ", "-")
|
||||
vid := put(t, object, test.existing)
|
||||
before := stored(t, object, vid)
|
||||
|
||||
body := retentionBody(test.mode, test.until)
|
||||
sum := md5.Sum(body)
|
||||
headers := map[string]string{xhttp.ContentMD5: base64.StdEncoding.EncodeToString(sum[:])}
|
||||
if test.bypass {
|
||||
headers[objectlock.AmzObjectLockBypassRetGovernance] = "true"
|
||||
}
|
||||
target := makeTestTargetURL("", bucket, object, url.Values{"retention": {""}, "versionId": {vid}})
|
||||
rec := retentionAuthzRequest(t, router, test.creds, http.MethodPut, target, body, headers)
|
||||
if rec.Code != test.want {
|
||||
t.Fatalf("status = %d, want %d: %s", rec.Code, test.want, rec.Body.String())
|
||||
}
|
||||
after := stored(t, object, vid)
|
||||
if changed := after.Mode != before.Mode || !after.RetainUntilDate.Equal(before.RetainUntilDate.Time); changed != test.changed {
|
||||
t.Fatalf("retention changed = %v, want %v (before %s, after %s)", changed, test.changed, before.String(), after.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run(instanceType+"/no-permission missing bucket", func(t *testing.T) {
|
||||
body := retentionBody("", time.Time{})
|
||||
sum := md5.Sum(body)
|
||||
target := makeTestTargetURL("", "retention-authz-missing", "object", url.Values{"retention": {""}})
|
||||
rec := retentionAuthzRequest(t, router, noRet, http.MethodPut, target, body,
|
||||
map[string]string{xhttp.ContentMD5: base64.StdEncoding.EncodeToString(sum[:])})
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status = %d, want 403 before bucket lookup: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestAPIVersionAddressedWriteRequiresReplicationTrust covers SN-2026-016: a
|
||||
// caller-chosen destination versionId is honored only for trusted replication,
|
||||
// and write paths cannot overwrite locked versions via directory keys or
|
||||
// version-addressed multipart completions.
|
||||
func TestAPIVersionAddressedWriteRequiresReplicationTrust(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPIVersionAddressedWriteRequiresReplicationTrust,
|
||||
makeBucketOptions: MakeBucketOptions{VersioningEnabled: true, LockEnabled: true},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPIVersionAddressedWriteRequiresReplicationTrust(obj ObjectLayer, instanceType, bucket string, router http.Handler, root auth.Credentials, t *testing.T) {
|
||||
writer := newObjectAttributesAuthzUser(t, instanceType, bucket, `"s3:GetObject","s3:PutObject"`)
|
||||
original := []byte("original version content")
|
||||
info, err := obj.PutObject(t.Context(), bucket, "addressed",
|
||||
mustGetPutObjReader(t, bytes.NewReader(original), int64(len(original)), "", ""), ObjectOptions{Versioned: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vid := info.VersionID
|
||||
|
||||
unchanged := func(t *testing.T) {
|
||||
t.Helper()
|
||||
got, err := obj.GetObjectInfo(t.Context(), bucket, "addressed", ObjectOptions{VersionID: vid})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Size != int64(len(original)) || got.ETag != info.ETag {
|
||||
t.Fatalf("version %s was rewritten: size %d etag %s", vid, got.Size, got.ETag)
|
||||
}
|
||||
}
|
||||
|
||||
for _, creds := range []struct {
|
||||
name string
|
||||
creds auth.Credentials
|
||||
}{{"writer", writer}, {"owner", root}} {
|
||||
t.Run(instanceType+"/"+creds.name+"/put", func(t *testing.T) {
|
||||
body := []byte("replacement")
|
||||
target := makeTestTargetURL("", bucket, "addressed", url.Values{"versionId": {vid}})
|
||||
rec := retentionAuthzRequest(t, router, creds.creds, http.MethodPut, target, body, nil)
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status = %d, want 403: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
unchanged(t)
|
||||
})
|
||||
t.Run(instanceType+"/"+creds.name+"/null-version", func(t *testing.T) {
|
||||
body := []byte("replacement")
|
||||
target := makeTestTargetURL("", bucket, "addressed", url.Values{"versionId": {nullVersionID}})
|
||||
rec := retentionAuthzRequest(t, router, creds.creds, http.MethodPut, target, body, nil)
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status = %d, want 403: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
t.Run(instanceType+"/"+creds.name+"/copy", func(t *testing.T) {
|
||||
target := makeTestTargetURL("", bucket, "addressed", url.Values{"versionId": {vid}})
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, target, 0, nil, creds.creds.AccessKey, creds.creds.SecretKey,
|
||||
map[string]string{xhttp.AmzCopySource: "/" + bucket + "/addressed?versionId=" + vid, xhttp.AmzMetadataDirective: "REPLACE"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status = %d, want 403: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
unchanged(t)
|
||||
})
|
||||
t.Run(instanceType+"/"+creds.name+"/multipart", func(t *testing.T) {
|
||||
target := makeTestTargetURL("", bucket, "addressed", url.Values{"uploads": {""}, "versionId": {vid}})
|
||||
rec := retentionAuthzRequest(t, router, creds.creds, http.MethodPost, target, nil, nil)
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status = %d, want 403: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
t.Run(instanceType+"/"+creds.name+"/complete-multipart-existing-version", func(t *testing.T) {
|
||||
// Simulate a pending upload persisted with an existing version ID
|
||||
// (e.g. initiated pre-upgrade or by a replication peer).
|
||||
up, err := obj.NewMultipartUpload(t.Context(), bucket, "addressed", ObjectOptions{
|
||||
Versioned: true,
|
||||
VersionID: vid,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer obj.AbortMultipartUpload(t.Context(), bucket, "addressed", up.UploadID, ObjectOptions{})
|
||||
|
||||
partPayload := []byte("replacement-part")
|
||||
pi, err := obj.PutObjectPart(t.Context(), bucket, "addressed", up.UploadID, 1,
|
||||
mustGetPutObjReader(t, bytes.NewReader(partPayload), int64(len(partPayload)), "", ""), ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
completeBody := fmt.Appendf(nil, `<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>"%s"</ETag></Part></CompleteMultipartUpload>`, pi.ETag)
|
||||
target := makeTestTargetURL("", bucket, "addressed", url.Values{"uploadId": {up.UploadID}})
|
||||
rec := retentionAuthzRequest(t, router, creds.creds, http.MethodPost, target, completeBody, nil)
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status = %d, want 403: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
unchanged(t)
|
||||
})
|
||||
}
|
||||
|
||||
t.Run(instanceType+"/locked-directory-object", func(t *testing.T) {
|
||||
far := UTCNow().Add(365 * 24 * time.Hour).Truncate(time.Second)
|
||||
for _, lockCase := range []struct {
|
||||
name string
|
||||
meta map[string]string
|
||||
}{
|
||||
{
|
||||
name: "governance",
|
||||
meta: map[string]string{
|
||||
strings.ToLower(xhttp.AmzObjectLockMode): "GOVERNANCE",
|
||||
strings.ToLower(xhttp.AmzObjectLockRetainUntilDate): amztime.ISO8601Format(far),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "compliance",
|
||||
meta: map[string]string{
|
||||
strings.ToLower(xhttp.AmzObjectLockMode): "COMPLIANCE",
|
||||
strings.ToLower(xhttp.AmzObjectLockRetainUntilDate): amztime.ISO8601Format(far),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "legal-hold",
|
||||
meta: map[string]string{
|
||||
strings.ToLower(xhttp.AmzObjectLockLegalHold): "ON",
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(lockCase.name, func(t *testing.T) {
|
||||
dirKey := "locked-dir-" + lockCase.name + "/"
|
||||
if _, err := obj.PutObject(t.Context(), bucket, dirKey,
|
||||
mustGetPutObjReader(t, bytes.NewReader(nil), 0, "", ""),
|
||||
ObjectOptions{Versioned: true, VersionID: nullVersionID, UserDefined: lockCase.meta}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target := makeTestTargetURL("", bucket, dirKey, nil)
|
||||
rec := retentionAuthzRequest(t, router, writer, http.MethodPut, target, nil, nil)
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("PUT locked dir status = %d, want 400: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
got, err := obj.GetObjectInfo(t.Context(), bucket, dirKey, ObjectOptions{VersionID: nullVersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for k, wantVal := range lockCase.meta {
|
||||
if got.UserDefined[k] != wantVal {
|
||||
t.Fatalf("metadata %s = %q, want %q", k, got.UserDefined[k], wantVal)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run(instanceType+"/snowball-archive-version-id", func(t *testing.T) {
|
||||
var tarBuf bytes.Buffer
|
||||
tw := tar.NewWriter(&tarBuf)
|
||||
entryData := []byte("snowball-overwrite")
|
||||
if err := tw.WriteHeader(&tar.Header{
|
||||
Typeflag: tar.TypeReg,
|
||||
Name: "addressed",
|
||||
Size: int64(len(entryData)),
|
||||
Format: tar.FormatPAX,
|
||||
PAXRecords: map[string]string{"minio.versionId": vid},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tw.Write(entryData); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Untrusted writer cannot overwrite an existing version via PAX minio.versionId.
|
||||
target := makeTestTargetURL("", bucket, "archive.tar", nil)
|
||||
rec := retentionAuthzRequest(t, router, writer, http.MethodPut, target, tarBuf.Bytes(),
|
||||
map[string]string{"x-amz-meta-snowball-auto-extract": "true"})
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("untrusted snowball status = %d, want 403: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
unchanged(t)
|
||||
|
||||
// Trusted batch replication (with outer MinIOSourceReplicationRequest) preserves PAX minio.versionId.
|
||||
replicaVID := mustGetUUID()
|
||||
var repBuf bytes.Buffer
|
||||
repTW := tar.NewWriter(&repBuf)
|
||||
repData := []byte("replicated-via-snowball")
|
||||
if err := repTW.WriteHeader(&tar.Header{
|
||||
Typeflag: tar.TypeReg,
|
||||
Name: "snowball-replicated",
|
||||
Size: int64(len(repData)),
|
||||
Format: tar.FormatPAX,
|
||||
PAXRecords: map[string]string{"minio.versionId": replicaVID},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := repTW.Write(repData); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := repTW.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repRec := retentionAuthzRequest(t, router, root, http.MethodPut, target, repBuf.Bytes(),
|
||||
map[string]string{
|
||||
"x-amz-meta-snowball-auto-extract": "true",
|
||||
xhttp.MinIOSourceReplicationRequest: "true",
|
||||
})
|
||||
if repRec.Code != http.StatusOK {
|
||||
t.Fatalf("trusted snowball status = %d, want 200: %s", repRec.Code, repRec.Body.String())
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(t.Context(), bucket, "snowball-replicated", ObjectOptions{VersionID: replicaVID}); err != nil {
|
||||
t.Fatalf("trusted snowball replication did not preserve version ID: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run(instanceType+"/trusted replication", func(t *testing.T) {
|
||||
replicaVID := mustGetUUID()
|
||||
body := []byte("replicated version")
|
||||
target := makeTestTargetURL("", bucket, "replicated", url.Values{"versionId": {replicaVID}})
|
||||
rec := retentionAuthzRequest(t, router, root, http.MethodPut, target, body,
|
||||
map[string]string{xhttp.MinIOSourceReplicationRequest: "true"})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(t.Context(), bucket, "replicated", ObjectOptions{VersionID: replicaVID}); err != nil {
|
||||
t.Fatalf("trusted replication did not keep its version ID: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPutObjectRetentionClearMinimumDaysPolicy(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
makeBucketOptions: MakeBucketOptions{VersioningEnabled: true, LockEnabled: true},
|
||||
objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, root auth.Credentials, t *testing.T) {
|
||||
for _, conditionalAllow := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/conditional-allow-%v", instance, conditionalAllow), func(t *testing.T) {
|
||||
statement := `{"Effect":"Allow","Action":["s3:PutObjectRetention","s3:BypassGovernanceRetention"],"Resource":["arn:aws:s3:::` + bucket + `/*"]}, {"Effect":"Deny","Action":["s3:PutObjectRetention"],"Resource":["arn:aws:s3:::` + bucket + `/*"],"Condition":{"NumericLessThan":{"s3:object-lock-remaining-retention-days":"60"}}}`
|
||||
if conditionalAllow {
|
||||
statement = `{"Effect":"Allow","Action":["s3:BypassGovernanceRetention"],"Resource":["arn:aws:s3:::` + bucket + `/*"]}, {"Effect":"Allow","Action":["s3:PutObjectRetention"],"Resource":["arn:aws:s3:::` + bucket + `/*"],"Condition":{"NumericGreaterThanEquals":{"s3:object-lock-remaining-retention-days":"60"}}}`
|
||||
}
|
||||
cred := newRetentionPolicyUser(t, instance, `{"Version":"2012-10-17","Statement":[`+statement+`]}`)
|
||||
key := fmt.Sprintf("review-minimum-%v", conditionalAllow)
|
||||
far := UTCNow().Add(365 * 24 * time.Hour).Truncate(time.Second)
|
||||
info, err := obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, bytes.NewReader(nil), 0, "", ""), ObjectOptions{
|
||||
Versioned: true,
|
||||
UserDefined: map[string]string{
|
||||
strings.ToLower(xhttp.AmzObjectLockMode): "GOVERNANCE",
|
||||
strings.ToLower(xhttp.AmzObjectLockRetainUntilDate): amztime.ISO8601Format(far),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status := putRetention(t, router, cred, bucket, key, info.VersionID, "GOVERNANCE", UTCNow().Add(30*24*time.Hour), true); status != http.StatusForbidden {
|
||||
t.Fatalf("shortening to 30 days status = %d, want 403", status)
|
||||
}
|
||||
status := putRetention(t, router, cred, bucket, key, info.VersionID, "", time.Time{}, true)
|
||||
after, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: info.VersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ret := objectlock.GetObjectRetentionMeta(after.UserDefined)
|
||||
if status != http.StatusForbidden || ret.Mode != objectlock.RetGovernance {
|
||||
t.Fatalf("clearing retention status = %d, stored retention = %s; want 403 and original GOVERNANCE", status, ret.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
type hookBeforeWrite struct {
|
||||
ObjectLayer
|
||||
key string
|
||||
before func()
|
||||
}
|
||||
|
||||
func (p *hookBeforeWrite) fire(key string) {
|
||||
if key == p.key && p.before != nil {
|
||||
f := p.before
|
||||
p.before = nil
|
||||
f()
|
||||
}
|
||||
}
|
||||
|
||||
func (p *hookBeforeWrite) PutObject(ctx context.Context, bucket, key string, r *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
|
||||
p.fire(key)
|
||||
return p.ObjectLayer.PutObject(ctx, bucket, key, r, opts)
|
||||
}
|
||||
|
||||
func (p *hookBeforeWrite) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (ObjectInfo, error) {
|
||||
p.fire(dstObject)
|
||||
return p.ObjectLayer.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
|
||||
}
|
||||
|
||||
func (p *hookBeforeWrite) CompleteMultipartUpload(ctx context.Context, bucket, key, uploadID string, parts []CompletePart, opts ObjectOptions) (ObjectInfo, error) {
|
||||
p.fire(key)
|
||||
return p.ObjectLayer.CompleteMultipartUpload(ctx, bucket, key, uploadID, parts, opts)
|
||||
}
|
||||
|
||||
func TestDirectoryObjectLockAfterPreflight(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
makeBucketOptions: MakeBucketOptions{VersioningEnabled: true, LockEnabled: true},
|
||||
objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, root auth.Credentials, t *testing.T) {
|
||||
writer := newObjectAttributesAuthzUser(t, instance, bucket, `"s3:GetObject","s3:PutObject"`)
|
||||
srcDir := "other-dir/"
|
||||
if rec := retentionAuthzRequest(t, router, writer, http.MethodPut, makeTestTargetURL("", bucket, srcDir, nil), nil, nil); rec.Code != http.StatusOK {
|
||||
t.Fatalf("srcDir PUT: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
for _, mode := range []string{"GOVERNANCE", "COMPLIANCE", "LEGAL-HOLD"} {
|
||||
for _, op := range []string{"PutObject", "CopyObject", "SelfCopy"} {
|
||||
t.Run(instance+"/"+mode+"/"+op, func(t *testing.T) {
|
||||
key := fmt.Sprintf("review-race-%s-%s/", mode, op)
|
||||
if rec := retentionAuthzRequest(t, router, writer, http.MethodPut, makeTestTargetURL("", bucket, key, nil), nil, nil); rec.Code != http.StatusOK {
|
||||
t.Fatalf("initial PUT: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
var locked ObjectInfo
|
||||
hook := &hookBeforeWrite{ObjectLayer: obj, key: key}
|
||||
hook.before = func() {
|
||||
if mode == "LEGAL-HOLD" {
|
||||
body := []byte(`<LegalHold><Status>ON</Status></LegalHold>`)
|
||||
sum := md5.Sum(body)
|
||||
rec := retentionAuthzRequest(t, router, root, http.MethodPut, makeTestTargetURL("", bucket, key, url.Values{"legal-hold": {""}, "versionId": {nullVersionID}}), body, map[string]string{xhttp.ContentMD5: base64.StdEncoding.EncodeToString(sum[:])})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("set legal hold: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
} else if status := putRetention(t, router, root, bucket, key, nullVersionID, mode, UTCNow().Add(365*24*time.Hour), false); status != http.StatusOK {
|
||||
t.Fatalf("set retention: %d", status)
|
||||
}
|
||||
var err error
|
||||
locked, err = obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: nullVersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
setObjectLayer(hook)
|
||||
defer setObjectLayer(obj)
|
||||
|
||||
var rec *httptest.ResponseRecorder
|
||||
switch op {
|
||||
case "PutObject":
|
||||
rec = retentionAuthzRequest(t, router, writer, http.MethodPut, makeTestTargetURL("", bucket, key, nil), nil, nil)
|
||||
case "CopyObject":
|
||||
rec = retentionAuthzRequest(t, router, writer, http.MethodPut, makeTestTargetURL("", bucket, key, nil), nil, map[string]string{
|
||||
xhttp.AmzCopySource: pathJoin(bucket, srcDir),
|
||||
})
|
||||
case "SelfCopy":
|
||||
rec = retentionAuthzRequest(t, router, writer, http.MethodPut, makeTestTargetURL("", bucket, key, nil), nil, map[string]string{
|
||||
xhttp.AmzCopySource: pathJoin(bucket, key), xhttp.AmzMetadataDirective: "REPLACE", "x-amz-meta-updated": "true",
|
||||
})
|
||||
}
|
||||
if hook.before != nil {
|
||||
t.Fatalf("%s never reached backend: %d %s", op, rec.Code, rec.Body.String())
|
||||
}
|
||||
after, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: nullVersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("locked null write via %s: %d %s", op, rec.Code, rec.Body.String())
|
||||
}
|
||||
multipartConditionError(t, rec, "InvalidRequest")
|
||||
retentionOverwriteUnchanged(t, locked, after)
|
||||
})
|
||||
}
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func TestMultipartVersioningTransitions(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, root auth.Credentials, t *testing.T) {
|
||||
t.Run(instance+"/unversioned-to-enabled", func(t *testing.T) {
|
||||
key := "review-upload-before-versioning"
|
||||
initRec := retentionAuthzRequest(t, router, root, http.MethodPost, makeTestTargetURL("", bucket, key, url.Values{"uploads": {""}}), nil, nil)
|
||||
if initRec.Code != http.StatusOK {
|
||||
t.Fatalf("init: %d %s", initRec.Code, initRec.Body.String())
|
||||
}
|
||||
var mp InitiateMultipartUploadResponse
|
||||
if err := xml.Unmarshal(initRec.Body.Bytes(), &mp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
part := retentionAuthzRequest(t, router, root, http.MethodPut, getPutObjectPartURL("", bucket, key, mp.UploadID, "1"), []byte("ordinary pending upload"), nil)
|
||||
if part.Code != http.StatusOK {
|
||||
t.Fatalf("part: %d %s", part.Code, part.Body.String())
|
||||
}
|
||||
if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, []byte(`<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := fmt.Appendf(nil, `<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>%s</ETag></Part></CompleteMultipartUpload>`, multipartConditionResponseETag(part))
|
||||
rec := retentionAuthzRequest(t, router, root, http.MethodPost, makeTestTargetURL("", bucket, key, url.Values{"uploadId": {mp.UploadID}}), body, nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("ordinary upload initiated before versioning fails completion: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
t.Run(instance+"/addressed-upload-after-suspension", func(t *testing.T) {
|
||||
key := "review-addressed-before-suspension"
|
||||
original := []byte("original")
|
||||
info, err := obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, bytes.NewReader(original), int64(len(original)), "", ""), ObjectOptions{Versioned: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
up, err := obj.NewMultipartUpload(t.Context(), bucket, key, ObjectOptions{Versioned: true, VersionID: info.VersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
part := []byte("replaced existing version")
|
||||
pi, err := obj.PutObjectPart(t.Context(), bucket, key, up.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(part), int64(len(part)), "", ""), ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, []byte(`<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writer := newObjectAttributesAuthzUser(t, instance, bucket, `"s3:PutObject"`)
|
||||
body := fmt.Appendf(nil, `<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>"%s"</ETag></Part></CompleteMultipartUpload>`, pi.ETag)
|
||||
rec := retentionAuthzRequest(t, router, writer, http.MethodPost, makeTestTargetURL("", bucket, key, url.Values{"uploadId": {up.UploadID}}), body, nil)
|
||||
after, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: info.VersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec.Code != http.StatusForbidden || after.ETag != info.ETag {
|
||||
t.Fatalf("suspended-bucket existing UUID overwritten: status=%d old-etag=%s new-etag=%s body=%s", rec.Code, info.ETag, after.ETag, rec.Body.String())
|
||||
}
|
||||
})
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,546 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This file is part of Silo Object Storage stack.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/amztime"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
func retentionOverwriteUnchanged(t *testing.T, before, after ObjectInfo) {
|
||||
t.Helper()
|
||||
if after.ETag != before.ETag || after.Size != before.Size || after.VersionID != before.VersionID || !after.ModTime.Equal(before.ModTime) {
|
||||
t.Error("rejected write changed the stored object")
|
||||
}
|
||||
if objectlock.GetObjectRetentionMeta(after.UserDefined).String() != objectlock.GetObjectRetentionMeta(before.UserDefined).String() || objectlock.GetObjectLegalHoldMeta(after.UserDefined).Status != objectlock.GetObjectLegalHoldMeta(before.UserDefined).Status {
|
||||
t.Error("rejected write changed the stored lock")
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectLockOverwriteConfigUnavailable(t *testing.T) {
|
||||
previous := newObjectLayerFn()
|
||||
setObjectLayer(nil)
|
||||
defer setObjectLayer(previous)
|
||||
err := checkObjectLockedForOverwrite(t.Context(), "lock-config-unavailable", "object", func(context.Context, string, string, ObjectOptions) (ObjectInfo, error) {
|
||||
t.Fatal("object must not be read without a bucket configuration")
|
||||
return ObjectInfo{}, nil
|
||||
})
|
||||
if !errors.Is(err, errServerNotInitialized) {
|
||||
t.Fatalf("unavailable lock configuration must fail closed, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutObjectRetentionMaximumDaysPolicy(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
makeBucketOptions: MakeBucketOptions{VersioningEnabled: true, LockEnabled: true},
|
||||
objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
for _, conditionalAllow := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/conditional-allow=%t", instance, conditionalAllow), func(t *testing.T) {
|
||||
statement := fmt.Sprintf(`{"Effect":"Allow","Action":["s3:PutObjectRetention"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"NumericLessThanEquals":{"s3:object-lock-remaining-retention-days":"200000"}}}`, bucket)
|
||||
if !conditionalAllow {
|
||||
statement = fmt.Sprintf(`{"Effect":"Allow","Action":["s3:PutObjectRetention"],"Resource":["arn:aws:s3:::%s/*"]},{"Effect":"Deny","Action":["s3:PutObjectRetention"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"NumericGreaterThan":{"s3:object-lock-remaining-retention-days":"200000"}}}`, bucket, bucket)
|
||||
}
|
||||
cred := newRetentionPolicyUser(t, instance, `{"Version":"2012-10-17","Statement":[`+statement+`]}`)
|
||||
key := fmt.Sprintf("maximum-retention-%t", conditionalAllow)
|
||||
initial, err := obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, bytes.NewReader(nil), 0, "", ""), ObjectOptions{Versioned: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Both dates exceed time.Duration's range (~292 years). A saturated
|
||||
// Time.Sub would report 106752 days and incorrectly allow 1000 years.
|
||||
if status := putRetention(t, router, cred, bucket, key, initial.VersionID, "GOVERNANCE", UTCNow().AddDate(1000, 0, 0), false); status != http.StatusForbidden {
|
||||
t.Fatalf("1000-year retention: want 403, got %d", status)
|
||||
}
|
||||
after, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: initial.VersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retentionOverwriteUnchanged(t, initial, after)
|
||||
if status := putRetention(t, router, cred, bucket, key, initial.VersionID, "GOVERNANCE", UTCNow().AddDate(500, 0, 0), false); status != http.StatusOK {
|
||||
t.Fatalf("500-year retention: want 200, got %d", status)
|
||||
}
|
||||
})
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func TestAPICopyObjectPreservesVersionAfterSuspension(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
makeBucketOptions: MakeBucketOptions{VersioningEnabled: true},
|
||||
objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, root auth.Credentials, t *testing.T) {
|
||||
t.Run(instance, func(t *testing.T) {
|
||||
key := "self-copy-after-suspension"
|
||||
original, err := obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, bytes.NewBufferString("payload"), 7, "", ""), ObjectOptions{
|
||||
Versioned: true, UserDefined: map[string]string{"x-amz-meta-state": "original"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig,
|
||||
[]byte(`<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := retentionAuthzRequest(t, router, root, http.MethodPut, makeTestTargetURL("", bucket, key, nil), nil, map[string]string{
|
||||
xhttp.AmzCopySource: "/" + bucket + "/" + key, xhttp.AmzMetadataDirective: "REPLACE", "x-amz-meta-state": "replacement",
|
||||
})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("copy: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
old, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: original.VersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if old.UserDefined["x-amz-meta-state"] != "original" {
|
||||
t.Errorf("copy changed metadata of existing UUID %s: %v", original.VersionID, old.UserDefined)
|
||||
}
|
||||
current, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: nullVersionID})
|
||||
if err != nil {
|
||||
t.Fatalf("copy must create the null version: %v", err)
|
||||
}
|
||||
if current.UserDefined["X-Amz-Meta-State"] != "replacement" {
|
||||
t.Errorf("new null version metadata: %v", current.UserDefined)
|
||||
}
|
||||
})
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// A persisted null upload can survive an upgrade or a versioning transition.
|
||||
// The completion must see locks committed after the handler's preflight.
|
||||
func TestMultipartNullObjectLockAfterPreflight(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
makeBucketOptions: MakeBucketOptions{VersioningEnabled: true, LockEnabled: true},
|
||||
objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, root auth.Credentials, t *testing.T) {
|
||||
writer := newObjectAttributesAuthzUser(t, instance, bucket, `"s3:PutObject"`)
|
||||
for _, mode := range []string{"GOVERNANCE", "COMPLIANCE", "LEGAL-HOLD"} {
|
||||
t.Run(instance+"/"+mode, func(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
key := "multipart-null-" + mode
|
||||
opts := ObjectOptions{Versioned: true, VersionID: nullVersionID, ReplicationRequest: true}
|
||||
if _, err := obj.PutObject(ctx, bucket, key, mustGetPutObjReader(t, bytes.NewBufferString("original"), 8, "", ""), opts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mp, err := obj.NewMultipartUpload(ctx, bucket, key, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
part, err := obj.PutObjectPart(ctx, bucket, key, mp.UploadID, 1, mustGetPutObjReader(t, bytes.NewBufferString("replacement"), 11, "", ""), ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var locked ObjectInfo
|
||||
hook := &hookBeforeWrite{ObjectLayer: obj, key: key, before: func() {
|
||||
if mode == "LEGAL-HOLD" {
|
||||
body := []byte(`<LegalHold><Status>ON</Status></LegalHold>`)
|
||||
sum := md5.Sum(body)
|
||||
rec := retentionAuthzRequest(t, router, root, http.MethodPut, makeTestTargetURL("", bucket, key, url.Values{"legal-hold": {""}, "versionId": {nullVersionID}}), body, map[string]string{xhttp.ContentMD5: base64.StdEncoding.EncodeToString(sum[:])})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("set legal hold: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
} else if status := putRetention(t, router, root, bucket, key, nullVersionID, mode, UTCNow().Add(365*24*time.Hour), false); status != http.StatusOK {
|
||||
t.Fatalf("set retention: %d", status)
|
||||
}
|
||||
var err error
|
||||
locked, err = obj.GetObjectInfo(ctx, bucket, key, ObjectOptions{VersionID: nullVersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}}
|
||||
setObjectLayer(hook)
|
||||
defer setObjectLayer(obj)
|
||||
body := []byte(multipartConditionCompleteBody([]CompletePart{{PartNumber: 1, ETag: part.ETag}}))
|
||||
rec := retentionAuthzRequest(t, router, writer, http.MethodPost, makeTestTargetURL("", bucket, key, url.Values{"uploadId": {mp.UploadID}}), body, nil)
|
||||
if hook.before != nil || rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("completion after locking: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
multipartConditionError(t, rec, "InvalidRequest")
|
||||
after, err := obj.GetObjectInfo(ctx, bucket, key, ObjectOptions{VersionID: nullVersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retentionOverwriteUnchanged(t, locked, after)
|
||||
parts, err := obj.ListObjectParts(ctx, bucket, key, mp.UploadID, 0, 100, ObjectOptions{})
|
||||
if err != nil || len(parts.Parts) != 1 || parts.Parts[0].ETag != part.ETag {
|
||||
t.Fatalf("rejected completion removed upload parts: %v %+v", err, parts.Parts)
|
||||
}
|
||||
})
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// The newer, unlocked copy is in the write pool; the lock exists only in the
|
||||
// other pool. A set-local check would miss it even while holding a write lock.
|
||||
func TestPoolsObjectLockOverwrite(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
bucket, _, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{VersioningEnabled: true, LockEnabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, state := range []string{"GOVERNANCE", "COMPLIANCE", "LEGAL-HOLD", "expired", "unlocked", "unreadable"} {
|
||||
for _, op := range []string{"put", "copy", "self-copy", "multipart"} {
|
||||
t.Run(state+"/"+op, func(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
key := encodeDirObject("null-" + state + "-" + op + "/")
|
||||
meta := map[string]string{}
|
||||
switch state {
|
||||
case "GOVERNANCE", "COMPLIANCE":
|
||||
meta[strings.ToLower(xhttp.AmzObjectLockMode)] = state
|
||||
meta[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(UTCNow().Add(365 * 24 * time.Hour))
|
||||
case "LEGAL-HOLD":
|
||||
meta[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = "ON"
|
||||
case "expired":
|
||||
meta[strings.ToLower(xhttp.AmzObjectLockMode)] = "GOVERNANCE"
|
||||
meta[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(UTCNow().Add(-24 * time.Hour))
|
||||
}
|
||||
seed := ObjectOptions{Versioned: true, VersionID: nullVersionID, ReplicationRequest: true, MTime: UTCNow().Add(-time.Hour), UserDefined: meta}
|
||||
putConsistencyObject(t, z, bucket, key, 1, "original", seed)
|
||||
seed.UserDefined = map[string]string{}
|
||||
seed.MTime = UTCNow()
|
||||
putConsistencyObject(t, z, bucket, key, 0, "original", seed)
|
||||
before := make([]ObjectInfo, len(z.serverPools))
|
||||
for i, pool := range z.serverPools {
|
||||
var err error
|
||||
before[i], err = pool.GetObjectInfo(ctx, bucket, key, ObjectOptions{VersionID: nullVersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
defer conditionalPutPool(t, z, key, 0)()
|
||||
var restore func()
|
||||
if state == "unreadable" {
|
||||
restore = conditionalPutSwapDisks(z.serverPools[1], key, func(d StorageAPI) StorageAPI {
|
||||
return consistencyReadFaultDisk{StorageAPI: d, bucket: bucket, object: key}
|
||||
})
|
||||
defer restore()
|
||||
}
|
||||
opts := ObjectOptions{Versioned: true, VersionID: nullVersionID, UserDefined: map[string]string{"x-amz-meta-updated": "true"}}
|
||||
var writeErr error
|
||||
switch op {
|
||||
case "put":
|
||||
_, writeErr = z.PutObject(ctx, bucket, key, mustGetPutObjReader(t, bytes.NewBufferString("replacement"), 11, "", ""), opts)
|
||||
case "copy", "self-copy":
|
||||
source := key
|
||||
if op == "copy" {
|
||||
source = "source-" + state
|
||||
putConsistencyObject(t, z, bucket, source, 0, "replacement", ObjectOptions{Versioned: true})
|
||||
}
|
||||
gr, gerr := z.GetObjectNInfo(ctx, bucket, source, nil, nil, ObjectOptions{})
|
||||
if gerr != nil {
|
||||
t.Fatal(gerr)
|
||||
}
|
||||
defer gr.Close()
|
||||
info := gr.ObjInfo
|
||||
info.UserDefined = opts.UserDefined
|
||||
info.metadataOnly = op == "self-copy"
|
||||
info.PutObjReader = mustGetPutObjReader(t, gr, info.Size, "", "")
|
||||
info.Reader = info.PutObjReader.Reader
|
||||
_, writeErr = z.CopyObject(ctx, bucket, source, bucket, key, info, ObjectOptions{VersionID: nullVersionID}, opts)
|
||||
case "multipart":
|
||||
id, parts := multipartConditionUpload(t, z, bucket, key, 0, opts)
|
||||
_, writeErr = z.CompleteMultipartUpload(ctx, bucket, key, id, parts, ObjectOptions{Versioned: true})
|
||||
}
|
||||
allowed := state == "expired" || state == "unlocked"
|
||||
if allowed {
|
||||
if writeErr != nil {
|
||||
t.Fatalf("unlocked write: %v", writeErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
if state == "unreadable" {
|
||||
if !isErrReadQuorum(writeErr) {
|
||||
t.Fatalf("unreadable pool: want quorum error, got %v", writeErr)
|
||||
}
|
||||
restore()
|
||||
} else if !errors.As(writeErr, &ObjectLocked{}) {
|
||||
t.Fatalf("lock in other pool: want ObjectLocked, got %v", writeErr)
|
||||
}
|
||||
for i, original := range before {
|
||||
after, gerr := z.serverPools[i].GetObjectInfo(ctx, bucket, key, ObjectOptions{VersionID: nullVersionID})
|
||||
if gerr != nil {
|
||||
t.Fatal(gerr)
|
||||
}
|
||||
retentionOverwriteUnchanged(t, original, after)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type planReviewGatedBody struct {
|
||||
io.ReadCloser
|
||||
first sync.Once
|
||||
ready chan struct{}
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (g *planReviewGatedBody) Read(p []byte) (int, error) {
|
||||
g.first.Do(func() { close(g.ready); <-g.release })
|
||||
return g.ReadCloser.Read(p)
|
||||
}
|
||||
|
||||
func TestPlanReviewSlowDirectoryPut(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
makeBucketOptions: MakeBucketOptions{VersioningEnabled: true, LockEnabled: true},
|
||||
objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, root auth.Credentials, t *testing.T) {
|
||||
writer := newObjectAttributesAuthzUser(t, instance, bucket, `"s3:PutObject"`)
|
||||
t.Run(instance, func(t *testing.T) {
|
||||
key := "slow-body-directory/"
|
||||
before, err := obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, bytes.NewBufferString("original"), 8, "", ""), ObjectOptions{Versioned: true, VersionID: nullVersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
payload := []byte("replacement-body")
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, makeTestTargetURL("", bucket, key, nil), int64(len(payload)), bytes.NewReader(payload), writer.AccessKey, writer.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gate := &planReviewGatedBody{ReadCloser: req.Body, ready: make(chan struct{}), release: make(chan struct{})}
|
||||
req.Body = gate
|
||||
var once sync.Once
|
||||
release := func() { once.Do(func() { close(gate.release) }) }
|
||||
defer release()
|
||||
rec := httptest.NewRecorder()
|
||||
done := make(chan struct{})
|
||||
go func() { router.ServeHTTP(rec, req); close(done) }()
|
||||
select {
|
||||
case <-gate.ready:
|
||||
case <-done:
|
||||
t.Fatalf("PUT finished before body: %d %s", rec.Code, rec.Body.String())
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("PUT did not reach body read")
|
||||
}
|
||||
locked := make(chan int, 1)
|
||||
go func() {
|
||||
locked <- putRetention(t, router, root, bucket, key, nullVersionID, "COMPLIANCE", UTCNow().Add(365*24*time.Hour), false)
|
||||
}()
|
||||
select {
|
||||
case status := <-locked:
|
||||
if status != http.StatusOK {
|
||||
t.Fatalf("set lock while body paused: %d", status)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("retention update blocked behind paused body")
|
||||
}
|
||||
release()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("PUT did not finish")
|
||||
}
|
||||
after, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: nullVersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec.Code != http.StatusBadRequest || after.ETag != before.ETag || objectlock.GetObjectRetentionMeta(after.UserDefined).Mode != objectlock.RetCompliance {
|
||||
t.Fatal("paused ordinary PUT overwrote the COMPLIANCE null version after the lock was acknowledged")
|
||||
}
|
||||
})
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
type planReviewReadFault struct {
|
||||
ObjectLayer
|
||||
key string
|
||||
fired bool
|
||||
putCalled bool
|
||||
}
|
||||
|
||||
func (p *planReviewReadFault) GetObjectInfo(ctx context.Context, bucket, key string, opts ObjectOptions) (ObjectInfo, error) {
|
||||
if key == p.key && opts.VersionID == nullVersionID && !p.fired {
|
||||
p.fired = true
|
||||
return ObjectInfo{}, InsufficientReadQuorum{}
|
||||
}
|
||||
return p.ObjectLayer.GetObjectInfo(ctx, bucket, key, opts)
|
||||
}
|
||||
|
||||
func (p *planReviewReadFault) PutObject(ctx context.Context, bucket, key string, data *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
|
||||
if key == p.key {
|
||||
p.putCalled = true
|
||||
}
|
||||
return p.ObjectLayer.PutObject(ctx, bucket, key, data, opts)
|
||||
}
|
||||
|
||||
func TestPlanReviewPreflightReadFailure(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
makeBucketOptions: MakeBucketOptions{VersioningEnabled: true, LockEnabled: true},
|
||||
objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
writer := newObjectAttributesAuthzUser(t, instance, bucket, `"s3:PutObject"`)
|
||||
t.Run(instance, func(t *testing.T) {
|
||||
key := "locked-directory-read-fault/"
|
||||
_, err := obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, bytes.NewBufferString("original"), 8, "", ""), ObjectOptions{
|
||||
Versioned: true, VersionID: nullVersionID, ReplicationRequest: true,
|
||||
UserDefined: map[string]string{
|
||||
strings.ToLower(xhttp.AmzObjectLockMode): "COMPLIANCE",
|
||||
strings.ToLower(xhttp.AmzObjectLockRetainUntilDate): amztime.ISO8601Format(UTCNow().Add(365 * 24 * time.Hour)),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
initial, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: nullVersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fault := &planReviewReadFault{ObjectLayer: obj, key: key}
|
||||
setObjectLayer(fault)
|
||||
defer setObjectLayer(obj)
|
||||
rec := retentionAuthzRequest(t, router, writer, http.MethodPut, makeTestTargetURL("", bucket, key, nil), []byte("replacement"), nil)
|
||||
if !fault.fired {
|
||||
t.Fatal("preflight fault was not injected")
|
||||
}
|
||||
if fault.putCalled {
|
||||
t.Fatal("preflight read error allowed PutObject to proceed to storage layer")
|
||||
}
|
||||
after, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: nullVersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec.Code < 400 {
|
||||
t.Errorf("metadata read quorum error was treated as absence: status=%d", rec.Code)
|
||||
}
|
||||
retentionOverwriteUnchanged(t, initial, after)
|
||||
})
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func TestPlanReviewLegacyMPUCompliance(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
makeBucketOptions: MakeBucketOptions{VersioningEnabled: true, LockEnabled: true},
|
||||
objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, root auth.Credentials, t *testing.T) {
|
||||
writer := newObjectAttributesAuthzUser(t, instance, bucket, `"s3:PutObject"`)
|
||||
t.Run(instance, func(t *testing.T) {
|
||||
key := "legacy-mpu-compliance"
|
||||
initial, err := obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, bytes.NewBufferString("original"), 8, "", ""), ObjectOptions{Versioned: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
up, err := obj.NewMultipartUpload(t.Context(), bucket, key, ObjectOptions{Versioned: true, VersionID: initial.VersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
payload := []byte("replacement-part")
|
||||
part, err := obj.PutObjectPart(t.Context(), bucket, key, up.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(payload), int64(len(payload)), "", ""), ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status := putRetention(t, router, root, bucket, key, initial.VersionID, "COMPLIANCE", UTCNow().Add(365*24*time.Hour), false); status != http.StatusOK {
|
||||
t.Fatalf("set compliance: %d", status)
|
||||
}
|
||||
before, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: initial.VersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := fmt.Appendf(nil, `<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>"%s"</ETag></Part></CompleteMultipartUpload>`, part.ETag)
|
||||
rec := retentionAuthzRequest(t, router, writer, http.MethodPost, makeTestTargetURL("", bucket, key, url.Values{"uploadId": {up.UploadID}}), body, nil)
|
||||
after, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: initial.VersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Errorf("legacy upload replaced locked UUID: status=%d", rec.Code)
|
||||
}
|
||||
retentionOverwriteUnchanged(t, before, after)
|
||||
})
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// A recovered upload retains an authentic server-generated version ID while a
|
||||
// surviving replica reintroduces its already-completed version and lock.
|
||||
// Only the fixture uses replication privileges; completion uses PutObject only.
|
||||
func TestRecoveredMultipartUploadAfterReplica(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
makeBucketOptions: MakeBucketOptions{VersioningEnabled: true, LockEnabled: true},
|
||||
objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, root auth.Credentials, t *testing.T) {
|
||||
writer := newObjectAttributesAuthzUser(t, instance, bucket, `"s3:PutObject"`)
|
||||
t.Run(instance, func(t *testing.T) {
|
||||
z := obj.(*erasureServerPools)
|
||||
key := "recovered-generated-upload"
|
||||
up, err := obj.NewMultipartUpload(t.Context(), bucket, key, ObjectOptions{Versioned: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
er := z.serverPools[0].getHashedSet(key)
|
||||
fi, _, err := er.checkUploadIDExists(t.Context(), bucket, key, up.UploadID, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = obj.PutObject(t.Context(), bucket, key,
|
||||
mustGetPutObjReader(t, bytes.NewBufferString("protected-original"), 18, "", ""),
|
||||
ObjectOptions{Versioned: true, VersionID: fi.VersionID, ReplicationRequest: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status := putRetention(t, router, root, bucket, key, fi.VersionID, "COMPLIANCE", UTCNow().Add(365*24*time.Hour), false); status != http.StatusOK {
|
||||
t.Fatalf("setting compliance lock: %d", status)
|
||||
}
|
||||
before, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: fi.VersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
partRec := retentionAuthzRequest(t, router, writer, http.MethodPut,
|
||||
makeTestTargetURL("", bucket, key, url.Values{"uploadId": {up.UploadID}, "partNumber": {"1"}}),
|
||||
[]byte("attacker-replacement"), nil)
|
||||
if partRec.Code != http.StatusOK {
|
||||
t.Fatalf("upload part: %d %s", partRec.Code, partRec.Body.String())
|
||||
}
|
||||
etag := multipartConditionResponseETag(partRec)
|
||||
body := []byte(multipartConditionCompleteBody([]CompletePart{{PartNumber: 1, ETag: etag}}))
|
||||
rec := retentionAuthzRequest(t, router, writer, http.MethodPost,
|
||||
makeTestTargetURL("", bucket, key, url.Values{"uploadId": {up.UploadID}}), body, nil)
|
||||
after, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: fi.VersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Errorf("recovered upload bypassed existing UUID guard: status=%d", rec.Code)
|
||||
}
|
||||
retentionOverwriteUnchanged(t, before, after)
|
||||
})
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/xml"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
// Exercise authenticated handlers and actual disk metadata, including the
|
||||
// response headers consumers see after replication has completed.
|
||||
func TestAPIReplicaContentEncoding(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPIReplicaContentEncoding})
|
||||
}
|
||||
|
||||
func testAPIReplicaContentEncoding(obj ObjectLayer, instance, bucket string, router http.Handler, owner auth.Credentials, t *testing.T) {
|
||||
ordinary := newObjectAttributesAuthzUser(t, instance, bucket, `"s3:PutObject","s3:GetObject"`)
|
||||
replicator := newObjectAttributesAuthzUser(t, instance, bucket, `"s3:PutObject","s3:GetObject","s3:ReplicateObject"`)
|
||||
for _, mode := range []string{"ordinary", "untrusted-marker", "replica"} {
|
||||
for _, tc := range []struct{ name, wire, want string }{
|
||||
{"bare", "aws-chunked", ""}, {"mixed", "aws-chunked,gzip", "gzip"}, {"gzip", "gzip", "gzip"},
|
||||
} {
|
||||
for _, operation := range []string{"put", "copy-replace", "multipart"} {
|
||||
t.Run(instance+"/"+mode+"/"+tc.name+"/"+operation, func(t *testing.T) {
|
||||
object := mode + "/" + tc.name + "/" + operation
|
||||
payload := replicaEncodingPayload(t, tc.want)
|
||||
creds := ordinary
|
||||
headers := map[string]string{xhttp.ContentEncoding: tc.wire, xhttp.ContentType: "application/octet-stream", "X-Amz-Meta-Source": "encoding-test"}
|
||||
if mode != "ordinary" {
|
||||
headers[xhttp.MinIOSourceReplicationRequest] = "true"
|
||||
}
|
||||
if mode == "replica" {
|
||||
creds = replicator
|
||||
headers[xhttp.AmzBucketReplicationStatus] = "REPLICA"
|
||||
}
|
||||
send := func(method, target string, data []byte, hdrs map[string]string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
req, err := newTestSignedRequestV4(method, target, int64(len(data)), bytes.NewReader(data), creds.AccessKey, creds.SecretKey, hdrs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return replicaEncodingServe(t, router, req, http.StatusOK)
|
||||
}
|
||||
switch operation {
|
||||
case "put":
|
||||
if strings.Contains(tc.wire, "aws-chunked") {
|
||||
req := replicaEncodingStream(t, getPutObjectURL("", bucket, object), payload, creds, headers)
|
||||
replicaEncodingServe(t, router, req, http.StatusOK)
|
||||
} else {
|
||||
send(http.MethodPut, getPutObjectURL("", bucket, object), payload, headers)
|
||||
}
|
||||
case "copy-replace":
|
||||
source := object + "-source"
|
||||
if _, err := obj.PutObject(t.Context(), bucket, source, mustGetPutObjReader(t, bytes.NewReader(payload), int64(len(payload)), "", ""), ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
headers[xhttp.AmzCopySource] = url.QueryEscape("/" + bucket + "/" + source)
|
||||
headers[xhttp.AmzMetadataDirective] = replaceDirective
|
||||
send(http.MethodPut, getCopyObjectURL("", bucket, object), nil, headers)
|
||||
case "multipart":
|
||||
rec := send(http.MethodPost, getNewMultipartURL("", bucket, object), nil, headers)
|
||||
var init InitiateMultipartUploadResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &init); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Part/completion metadata must not replace the encoding saved at initiation.
|
||||
partHeaders := map[string]string{xhttp.ContentEncoding: "br"}
|
||||
if mode == "replica" {
|
||||
partHeaders[xhttp.MinIOSourceReplicationRequest] = "true"
|
||||
partHeaders[xhttp.AmzBucketReplicationStatus] = "REPLICA"
|
||||
}
|
||||
part := send(http.MethodPut, getPutObjectPartURL("", bucket, object, init.UploadID, "1"), payload, partHeaders)
|
||||
partETags := part.Header()[xhttp.ETag]
|
||||
if len(partETags) != 1 {
|
||||
t.Fatalf("missing part ETag: %#v", part.Header())
|
||||
}
|
||||
complete, err := xml.Marshal(CompleteMultipartUpload{Parts: []CompletePart{{PartNumber: 1, ETag: canonicalizeETag(partETags[0])}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
send(http.MethodPost, getCompleteMultipartUploadURL("", bucket, object, init.UploadID), complete, partHeaders)
|
||||
}
|
||||
assertReplicaEncodingObject(t, obj, router, owner, bucket, object, tc.want, payload)
|
||||
info, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := info.UserDefined[xhttp.AmzBucketReplicationStatus]; (got == "REPLICA") != (mode == "replica") {
|
||||
t.Errorf("replica status %q for mode %s", got, mode)
|
||||
}
|
||||
if info.ContentType != "application/octet-stream" {
|
||||
t.Errorf("content-type=%q", info.ContentType)
|
||||
}
|
||||
if value, ok := caseInsensitiveMap(info.UserDefined).Lookup("x-amz-meta-source"); !ok || value != "encoding-test" {
|
||||
t.Errorf("user metadata lost: %#v", info.UserDefined)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Run(instance+"/unauthorized-replica", func(t *testing.T) {
|
||||
object := "denied-replica"
|
||||
req := replicaEncodingStream(t, getPutObjectURL("", bucket, object), []byte("denied"), ordinary, map[string]string{xhttp.ContentEncoding: "aws-chunked", xhttp.MinIOSourceReplicationRequest: "true", xhttp.AmzBucketReplicationStatus: "REPLICA"})
|
||||
rec := replicaEncodingServe(t, router, req, http.StatusForbidden)
|
||||
var response APIErrorResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.Code != "AccessDenied" {
|
||||
t.Fatalf("expected permission denial, got %s", response.Code)
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); err == nil {
|
||||
t.Error("denied replica created an object")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func replicaEncodingPayload(t *testing.T, encoding string) []byte {
|
||||
t.Helper()
|
||||
data := bytes.Repeat([]byte("replica encoding payload\n"), 128)
|
||||
if encoding != "gzip" {
|
||||
return data
|
||||
}
|
||||
var b bytes.Buffer
|
||||
w := gzip.NewWriter(&b)
|
||||
if _, err := w.Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func replicaEncodingStream(t *testing.T, target string, data []byte, creds auth.Credentials, headers map[string]string) *http.Request {
|
||||
t.Helper()
|
||||
const chunkSize = 64
|
||||
body := bytes.NewReader(data)
|
||||
req, err := newTestStreamingRequest(http.MethodPut, target, int64(len(data)), chunkSize, body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
now := UTCNow()
|
||||
signature, err := signStreamingRequest(req, creds.AccessKey, creds.SecretKey, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req, err = assembleStreamingChunks(req, body, chunkSize, creds.SecretKey, signature, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
func replicaEncodingServe(t *testing.T, router http.Handler, req *http.Request, want int) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != want {
|
||||
t.Fatalf("%s %s: status=%d want=%d body=%s", req.Method, req.URL, rec.Code, want, rec.Body.String())
|
||||
}
|
||||
return rec
|
||||
}
|
||||
|
||||
func assertReplicaEncodingObject(t *testing.T, obj ObjectLayer, router http.Handler, creds auth.Credentials, bucket, object, encoding string, data []byte) {
|
||||
t.Helper()
|
||||
info, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.ContentEncoding != encoding {
|
||||
t.Errorf("persisted content-encoding=%q want=%q", info.ContentEncoding, encoding)
|
||||
}
|
||||
if encoding == "" {
|
||||
if _, present := info.UserDefined["content-encoding"]; present {
|
||||
t.Error("transport-only content-encoding key persisted")
|
||||
}
|
||||
}
|
||||
for _, method := range []string{http.MethodGet, http.MethodHead} {
|
||||
req, err := newTestSignedRequestV4(method, getPutObjectURL("", bucket, object), 0, nil, creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := replicaEncodingServe(t, router, req, http.StatusOK)
|
||||
if got := rec.Header().Get(xhttp.ContentEncoding); got != encoding {
|
||||
t.Errorf("%s content-encoding=%q want=%q", method, got, encoding)
|
||||
}
|
||||
if encoding == "" {
|
||||
if _, present := rec.Header()[xhttp.ContentEncoding]; present {
|
||||
t.Errorf("%s sent an empty/transport encoding header", method)
|
||||
}
|
||||
}
|
||||
if method == http.MethodGet && !bytes.Equal(rec.Body.Bytes(), data) {
|
||||
t.Errorf("GET body differs: got %d bytes want %d", rec.Body.Len(), len(data))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPISnowballReplicaContentEncoding(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, creds auth.Credentials, t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
pax map[string]string
|
||||
want string
|
||||
}{
|
||||
{name: "no-pax"},
|
||||
{name: "pax-without-encoding", pax: map[string]string{"minio.metadata.Content-Type": "application/octet-stream"}},
|
||||
{name: "pax-bare", pax: map[string]string{"minio.metadata.Content-Encoding": "aws-chunked"}},
|
||||
{name: "pax-mixed", pax: map[string]string{"minio.metadata.Content-Encoding": "aws-chunked,gzip"}, want: "gzip"},
|
||||
} {
|
||||
t.Run(instance+"/"+tc.name, func(t *testing.T) {
|
||||
object := "snowball/" + tc.name
|
||||
data := replicaEncodingPayload(t, tc.want)
|
||||
var archive bytes.Buffer
|
||||
tw := tar.NewWriter(&archive)
|
||||
if err := tw.WriteHeader(&tar.Header{Name: object, Mode: 0o600, Size: int64(len(data)), PAXRecords: tc.pax}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tw.Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ordinaryMetadata map[string]string
|
||||
// An unauthorized entry in a REPLICA request is rejected. Compare the
|
||||
// same archive across ordinary and authorized replica requests instead.
|
||||
for _, replica := range []bool{false, true} {
|
||||
headers := map[string]string{
|
||||
xhttp.ContentEncoding: "aws-chunked", xhttp.AmzSnowballExtract: "true",
|
||||
xhttp.ContentType: "application/x-tar", xhttp.CacheControl: "max-age=123",
|
||||
"X-Amz-Meta-Archive": "outer-request",
|
||||
}
|
||||
if replica {
|
||||
headers[xhttp.MinIOSourceReplicationRequest] = "true"
|
||||
headers[xhttp.AmzBucketReplicationStatus] = "REPLICA"
|
||||
}
|
||||
req := replicaEncodingStream(t, getPutObjectURL("", bucket, "archive.tar"), archive.Bytes(), creds, headers)
|
||||
replicaEncodingServe(t, router, req, http.StatusOK)
|
||||
assertReplicaEncodingObject(t, obj, router, creds, bucket, object, tc.want, data)
|
||||
info, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
metadata := maps.Clone(info.UserDefined)
|
||||
for _, key := range []string{xhttp.AmzBucketReplicationStatus, ReservedMetadataPrefixLower + ReplicaStatus, ReservedMetadataPrefixLower + ReplicaTimestamp, "etag"} {
|
||||
delete(metadata, key)
|
||||
}
|
||||
if !replica {
|
||||
ordinaryMetadata = metadata
|
||||
} else if !reflect.DeepEqual(metadata, ordinaryMetadata) {
|
||||
t.Errorf("replica inherited ordinary archive metadata: got %#v want %#v", metadata, ordinaryMetadata)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}})
|
||||
}
|
||||
@@ -106,6 +106,7 @@ func TestReplicateDeleteMarkerTargetSemantics(t *testing.T) {
|
||||
|
||||
func testReplicateDeleteMarkerPurge(obj ObjectLayer, instanceType, bucket string, router http.Handler, creds auth.Credentials, t *testing.T, legacy bool) {
|
||||
ctx := t.Context()
|
||||
defer replicationTestCapacity(obj)()
|
||||
const arn = "arn:minio:replication::af470089-d354-4473-934c-9e1f52f6da89:bucket"
|
||||
const name = "marker"
|
||||
version := mustGetUUID()
|
||||
@@ -208,27 +209,12 @@ func testReplicateDeleteMarkerPurge(obj ObjectLayer, instanceType, bucket string
|
||||
t.Errorf("purge scheduled as marker creation: version=%q marker=%q", deletion.VersionID, deletion.DeleteMarkerVersionID)
|
||||
}
|
||||
if legacy {
|
||||
// Reproduce the old producer's state and let the existing scanner/heal
|
||||
// path recover it. Upgrades must also finish purges already left pending.
|
||||
// Old task shapes must complete directly; scanner/MRF recovery is
|
||||
// covered separately by TestReplicationMRFMarkerRecovery.
|
||||
deletion.VersionID, deletion.DeleteMarkerVersionID = "", version
|
||||
replicateDelete(ctx, deletion, obj)
|
||||
oi, _ := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: version, Versioned: true})
|
||||
if oi.VersionPurgeStatus != replication.VersionPurgePending {
|
||||
t.Fatalf("legacy source purge = %s, want PENDING", oi.VersionPurgeStatus)
|
||||
}
|
||||
targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queueReplicationHeal(ctx, bucket, oi, replicationConfig{Config: &cfg, remotes: targets}, 0)
|
||||
select {
|
||||
case op := <-worker:
|
||||
deletion = op.(DeletedObjectReplicationInfo)
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("legacy pending purge was not scheduled for healing")
|
||||
}
|
||||
}
|
||||
result := replicateDelete(context.Background(), deletion, obj)
|
||||
|
||||
result := replicateDelete(context.Background(), deletion, markerPurgeUpdateLayer{ObjectLayer: obj, t: t})
|
||||
if result.VersionPurgeStatus() != replication.VersionPurgeComplete {
|
||||
t.Errorf("remote purge result = %s, want COMPLETE", result.VersionPurgeStatus())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,665 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
loghttp "github.com/minio/minio/internal/logger/target/http"
|
||||
"github.com/minio/minio/internal/once"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
)
|
||||
|
||||
type markerRecoveryCase struct {
|
||||
name string
|
||||
legacy, creation, lostReply, partial, exhaust, lockFirst, invalidMRF, replicaSource, offline, unrecordedPurge bool
|
||||
targets int
|
||||
}
|
||||
|
||||
func TestReplicationMRFMarkerRecovery(t *testing.T) {
|
||||
for _, tc := range []markerRecoveryCase{
|
||||
{name: "canonical", targets: 1},
|
||||
{name: "lock-failure", lockFirst: true, targets: 1},
|
||||
{name: "invalid-MRF-metadata", invalidMRF: true, targets: 1},
|
||||
{name: "legacy", legacy: true, targets: 1},
|
||||
{name: "unrecorded-purge", unrecordedPurge: true, targets: 2},
|
||||
{name: "two-targets", legacy: true, targets: 2},
|
||||
{name: "two-targets-offline", offline: true, targets: 2},
|
||||
{name: "replica-source", replicaSource: true, targets: 1},
|
||||
{name: "lost-reply-canonical", lostReply: true, targets: 1},
|
||||
{name: "lost-reply-legacy", legacy: true, lostReply: true, targets: 1},
|
||||
{name: "creation", creation: true, targets: 1},
|
||||
{name: "partial-creation-block", partial: true, targets: 2},
|
||||
{name: "retry-budget-and-scanner", exhaust: true, targets: 1},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, endpoints: []string{"DeleteObject"}, objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, creds auth.Credentials, t *testing.T) {
|
||||
testReplicationMRFMarkerRecovery(t, obj, backend, bucket, router, creds, tc)
|
||||
}})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type markerRecoveryTarget struct {
|
||||
arn, bucket string
|
||||
client *minio.Client
|
||||
reject, loseReply atomic.Bool
|
||||
deletes atomic.Int32
|
||||
}
|
||||
|
||||
// Only adapt host capacity accounting, using the existing real-disk adapter.
|
||||
// Reads, object metadata, MRF files and writes all still use the fixture disks.
|
||||
func replicationTestCapacity(obj ObjectLayer) func() {
|
||||
var restore []func()
|
||||
for _, pool := range obj.(*erasureServerPools).serverPools {
|
||||
pool.erasureDisksMu.Lock()
|
||||
for _, set := range pool.sets {
|
||||
original := pool.erasureDisks[set.setIndex]
|
||||
disks := append([]StorageAPI(nil), original...)
|
||||
for i, disk := range disks {
|
||||
if disk != nil {
|
||||
disks[i] = tagTestCapacityDisk{StorageAPI: disk}
|
||||
}
|
||||
}
|
||||
// GetDisks copies this list under the same mutex. Keep its function
|
||||
// stable while background IAM scans use the fixture, both when
|
||||
// installing the adapter and when restoring the original disks.
|
||||
pool.erasureDisks[set.setIndex] = disks
|
||||
restore = append(restore, func() {
|
||||
pool.erasureDisksMu.Lock()
|
||||
pool.erasureDisks[set.setIndex] = original
|
||||
pool.erasureDisksMu.Unlock()
|
||||
})
|
||||
}
|
||||
pool.erasureDisksMu.Unlock()
|
||||
}
|
||||
return func() {
|
||||
for _, fn := range restore {
|
||||
fn()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplicationCapacityConcurrentIAM(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
if _, _, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
iam := globalIAMSys
|
||||
started, finished := make(chan struct{}), make(chan error, 1)
|
||||
go func() {
|
||||
close(started)
|
||||
for range 20 {
|
||||
if err := iam.Load(ctx, false); err != nil {
|
||||
finished <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
finished <- nil
|
||||
}()
|
||||
<-started
|
||||
for range 5000 {
|
||||
replicationTestCapacity(z)()
|
||||
}
|
||||
if err := <-finished; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func testReplicationMRFMarkerRecovery(t *testing.T, obj ObjectLayer, backend, bucket string, router http.Handler, creds auth.Credentials, tc markerRecoveryCase) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
defer replicationTestCapacity(obj)()
|
||||
stats := NewReplicationStats(ctx, nil)
|
||||
oldStats := globalReplicationStats.Swap(stats)
|
||||
defer globalReplicationStats.Store(oldStats)
|
||||
oldPool := globalReplicationPool
|
||||
defer func() { globalReplicationPool = oldPool }()
|
||||
const name = "marker"
|
||||
version := mustGetUUID()
|
||||
creationTime := UTCNow().Add(-time.Hour).Truncate(time.Second)
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := obj.PutObject(ctx, bucket, name, mustGetPutObjReader(t, bytes.NewReader([]byte("data")), 4, "", ""), ObjectOptions{Versioned: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var targets []*markerRecoveryTarget
|
||||
cfg := replication.Config{}
|
||||
creationStates := make(map[string]replication.StatusType)
|
||||
var creationInternal string
|
||||
for i := 0; i < tc.targets; i++ {
|
||||
target := &markerRecoveryTarget{arn: "arn:minio:replication::" + mustGetUUID() + ":bucket", bucket: getRandomBucketName()}
|
||||
if err := obj.MakeBucket(ctx, target.bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := obj.PutObject(ctx, target.bucket, name, mustGetPutObjReader(t, bytes.NewReader([]byte("data")), 4, "", ""), ObjectOptions{Versioned: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !tc.creation {
|
||||
opts := ObjectOptions{VersionID: version, Versioned: true, DeleteMarker: true, ReplicationRequest: true, MTime: creationTime}
|
||||
opts.SetReplicaStatus(replication.Replica)
|
||||
if _, err := obj.DeleteObject(ctx, target.bucket, name, opts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
opts := ObjectOptions{VersionID: r.URL.Query().Get("versionId"), Versioned: true}
|
||||
switch r.Method {
|
||||
case http.MethodHead:
|
||||
oi, err := obj.GetObjectInfo(r.Context(), target.bucket, name, opts)
|
||||
if oi.DeleteMarker {
|
||||
w.Header().Set(xhttp.AmzDeleteMarker, "true")
|
||||
w.Header().Set(xhttp.AmzVersionID, oi.VersionID)
|
||||
}
|
||||
if err != nil {
|
||||
writeErrorResponseHeadersOnly(w, toAPIError(r.Context(), err))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case http.MethodDelete:
|
||||
target.deletes.Add(1)
|
||||
if got := r.Header.Get(xhttp.MinIOSourceDeleteMarker) == "true"; got != tc.creation {
|
||||
t.Errorf("purge/creation wire flag=%v creation=%v", got, tc.creation)
|
||||
}
|
||||
if opts.VersionID == "" {
|
||||
t.Error("missing remote versionId")
|
||||
}
|
||||
if target.reject.Load() {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
fmt.Fprint(w, `<Error><Code>AccessDenied</Code></Error>`)
|
||||
return
|
||||
}
|
||||
opts.DeleteMarker = r.Header.Get(xhttp.MinIOSourceDeleteMarker) == "true"
|
||||
opts.ReplicationRequest = true
|
||||
opts.SetReplicaStatus(replication.Replica)
|
||||
_, err := obj.DeleteObject(r.Context(), target.bucket, name, opts)
|
||||
if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL)
|
||||
return
|
||||
}
|
||||
if target.loseReply.Swap(false) {
|
||||
conn, _, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
conn.Close()
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
default:
|
||||
t.Errorf("unexpected remote method %s", r.Method)
|
||||
}
|
||||
}))
|
||||
defer remote.Close()
|
||||
client, err := minio.New(strings.TrimPrefix(remote.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target.client = client
|
||||
globalBucketTargetSys.Lock()
|
||||
globalBucketTargetSys.arnRemotesMap[target.arn] = arnTarget{Client: &TargetClient{Client: client, ARN: target.arn, Bucket: target.bucket}, lastRefresh: UTCNow()}
|
||||
globalBucketTargetSys.targetsMap[bucket] = append(globalBucketTargetSys.targetsMap[bucket], madmin.BucketTarget{Arn: target.arn, TargetBucket: target.bucket})
|
||||
globalBucketTargetSys.Unlock()
|
||||
globalBucketTargetSys.hMutex.Lock()
|
||||
globalBucketTargetSys.hc[client.EndpointURL().Host] = epHealth{Online: true}
|
||||
globalBucketTargetSys.hMutex.Unlock()
|
||||
rule := configs[0].Rules[0]
|
||||
rule.Priority = i + 1
|
||||
rule.Destination = replication.Destination{ARN: target.arn, Bucket: target.bucket}
|
||||
cfg.Rules = append(cfg.Rules, rule)
|
||||
creationStates[target.arn] = replication.Completed
|
||||
creationInternal += target.arn + "=COMPLETED;"
|
||||
targets = append(targets, target)
|
||||
}
|
||||
if tc.targets == 1 {
|
||||
cfg.RoleArn = targets[0].arn
|
||||
}
|
||||
if !tc.creation {
|
||||
opts := ObjectOptions{VersionID: version, Versioned: true, DeleteMarker: true, MTime: creationTime, DeleteReplication: ReplicationState{Targets: creationStates, ReplicationStatusInternal: creationInternal, ReplicationTimeStamp: creationTime}}
|
||||
if tc.replicaSource {
|
||||
opts.DeleteReplication = ReplicationState{}
|
||||
opts.SetReplicaStatus(replication.Replica)
|
||||
}
|
||||
if _, err := obj.DeleteObject(ctx, bucket, name, opts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
meta, err := globalBucketMetadataSys.Get(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta.replicationConfig = &cfg
|
||||
globalBucketMetadataSys.Set(bucket, meta)
|
||||
newPool := func() *ReplicationPool {
|
||||
p := &ReplicationPool{ctx: ctx, objLayer: obj, workers: []chan ReplicationWorkerOperation{make(chan ReplicationWorkerOperation, 8)}, stats: stats, mrfSaveCh: make(chan MRFReplicateEntry, 8)}
|
||||
globalReplicationPool = once.NewSingleton[ReplicationPool]()
|
||||
globalReplicationPool.Set(p)
|
||||
return p
|
||||
}
|
||||
p := newPool()
|
||||
receive := func() DeletedObjectReplicationInfo {
|
||||
select {
|
||||
case op := <-p.workers[0]:
|
||||
d, ok := op.(DeletedObjectReplicationInfo)
|
||||
if !ok {
|
||||
t.Fatalf("wrong queued operation %T", op)
|
||||
}
|
||||
return d
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("no marker task from actual MRF/handler/scanner queue")
|
||||
return DeletedObjectReplicationInfo{}
|
||||
}
|
||||
}
|
||||
uri := "/" + bucket + "/" + name
|
||||
if !tc.creation {
|
||||
uri += "?versionId=" + version
|
||||
}
|
||||
req, err := newTestSignedRequestV4(http.MethodDelete, uri, 0, nil, creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusNoContent {
|
||||
t.Fatalf("source DELETE status %d: %s", w.Code, w.Body)
|
||||
}
|
||||
deletion := receive()
|
||||
if tc.creation {
|
||||
version = deletion.DeleteMarkerVersionID
|
||||
} else if deletion.VersionID != version || deletion.DeleteMarkerVersionID != "" {
|
||||
t.Fatalf("current producer emitted noncanonical purge: %+v", deletion)
|
||||
}
|
||||
if tc.unrecordedPurge {
|
||||
// Exercise a task carrying purge state while the disk marker still has
|
||||
// only creation metadata. Recreate only this isolated fixture marker.
|
||||
if _, err := obj.DeleteObject(ctx, bucket, name, ObjectOptions{VersionID: version, Versioned: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts := ObjectOptions{VersionID: version, Versioned: true, DeleteMarker: true, MTime: creationTime, DeleteReplication: ReplicationState{Targets: creationStates, ReplicationStatusInternal: creationInternal, ReplicationTimeStamp: creationTime}}
|
||||
if _, err := obj.DeleteObject(ctx, bucket, name, opts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if tc.legacy {
|
||||
deletion.VersionID, deletion.DeleteMarkerVersionID = "", version
|
||||
}
|
||||
if tc.partial {
|
||||
deletion.TargetArn = targets[len(targets)-1].arn
|
||||
}
|
||||
if tc.invalidMRF {
|
||||
testReplicationMRFInvalidLookups(ctx, t, obj, p, newPool, deletion)
|
||||
return
|
||||
}
|
||||
var auditStatuses <-chan string
|
||||
if tc.name == "canonical" {
|
||||
var stopAudit func()
|
||||
auditStatuses, stopAudit = replicationTestAudit(ctx, t, bucket)
|
||||
defer stopAudit()
|
||||
}
|
||||
assertAudit := func(want string) {
|
||||
if auditStatuses == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case got := <-auditStatuses:
|
||||
if got != want {
|
||||
t.Fatalf("replication audit status=%q want %q", got, want)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("missing replication audit event")
|
||||
}
|
||||
}
|
||||
deletion.OpType = replication.HealReplicationType // Exercise operation-status statistics too.
|
||||
failing := targets[len(targets)-1]
|
||||
failing.reject.Store(!tc.lostReply)
|
||||
failing.loseReply.Store(tc.lostReply)
|
||||
if tc.offline {
|
||||
globalBucketTargetSys.hMutex.Lock()
|
||||
globalBucketTargetSys.hc[failing.client.EndpointURL().Host] = epHealth{Online: false}
|
||||
globalBucketTargetSys.hMutex.Unlock()
|
||||
}
|
||||
before, _ := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: version})
|
||||
beforeCreation := before.ReplicationStatusInternal
|
||||
beforeStamp := before.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp]
|
||||
beforeReplica := before.UserDefined[ReservedMetadataPrefixLower+ReplicaStatus]
|
||||
beforeReplicaStamp := before.UserDefined[ReservedMetadataPrefixLower+ReplicaTimestamp]
|
||||
if !tc.creation && !tc.replicaSource && (len(replicationStatusesMap(beforeCreation)) != tc.targets || beforeStamp == "") {
|
||||
t.Fatalf("missing seeded creation block: %+v", before)
|
||||
}
|
||||
if tc.replicaSource && (beforeReplica != "REPLICA" || beforeReplicaStamp == "") {
|
||||
t.Fatalf("missing replica block: %+v", before)
|
||||
}
|
||||
updateObj := obj
|
||||
if !tc.creation {
|
||||
updateObj = markerPurgeUpdateLayer{ObjectLayer: obj, t: t}
|
||||
}
|
||||
rounds := 2
|
||||
if tc.lostReply {
|
||||
rounds = 1
|
||||
}
|
||||
if tc.exhaust {
|
||||
rounds = mrfRetryLimit + 1
|
||||
}
|
||||
for round := 1; round <= rounds; round++ {
|
||||
callObj := updateObj
|
||||
if tc.lockFirst && round == 1 {
|
||||
callObj = markerLockFailureLayer{ObjectLayer: updateObj}
|
||||
}
|
||||
result := replicateDelete(ctx, deletion, callObj)
|
||||
switch {
|
||||
case tc.lockFirst && round == 1:
|
||||
if len(result.Targets) != 0 {
|
||||
t.Fatal("lock failure attempted a target")
|
||||
}
|
||||
case tc.creation:
|
||||
if result.ReplicationStatus() != replication.Failed {
|
||||
t.Fatalf("creation result: %+v", result)
|
||||
}
|
||||
case result.VersionPurgeStatus() != replication.VersionPurgeFailed:
|
||||
t.Fatalf("purge failure result: %+v", result)
|
||||
}
|
||||
assertAudit("FAILED")
|
||||
if round == 1 && !tc.lockFirst && !tc.creation {
|
||||
stats.RLock()
|
||||
failed := stats.Cache[bucket].Stats[failing.arn].FailStats.SinceUptime
|
||||
stats.RUnlock()
|
||||
if failed.Count != 1 || failed.Bytes != 0 {
|
||||
t.Fatalf("purge failure stats=%+v, want count 1 and zero bytes", failed)
|
||||
}
|
||||
}
|
||||
oi, err := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: version})
|
||||
if !isErrMethodNotAllowed(err) || !oi.DeleteMarker {
|
||||
t.Fatalf("source marker metadata/405 missing: %+v %v", oi, err)
|
||||
}
|
||||
if !tc.creation && (oi.ReplicationStatusInternal != beforeCreation || oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] != beforeStamp) {
|
||||
t.Fatalf("purge rewrote creation block: before=%q/%q after=%q/%q", beforeCreation, beforeStamp, oi.ReplicationStatusInternal, oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp])
|
||||
}
|
||||
if !tc.creation && (oi.UserDefined[ReservedMetadataPrefixLower+ReplicaStatus] != beforeReplica || oi.UserDefined[ReservedMetadataPrefixLower+ReplicaTimestamp] != beforeReplicaStamp) {
|
||||
t.Fatal("purge rewrote replica block")
|
||||
}
|
||||
if tc.targets == 2 && !tc.partial {
|
||||
purges := versionPurgeStatusesMap(oi.VersionPurgeStatusInternal)
|
||||
if purges[targets[0].arn] != replication.VersionPurgeComplete || purges[failing.arn] != replication.VersionPurgeFailed {
|
||||
t.Fatalf("incorrect persisted target states: %v", purges)
|
||||
}
|
||||
if targets[0].deletes.Load() != 1 {
|
||||
t.Fatalf("successful target resent %d times", targets[0].deletes.Load())
|
||||
}
|
||||
}
|
||||
if tc.partial {
|
||||
select {
|
||||
case <-p.mrfSaveCh:
|
||||
default:
|
||||
t.Fatal("partial failure missing MRF")
|
||||
}
|
||||
dsc := deletion.ReplicationState.ReplicateDecisionStr
|
||||
deletion.ReplicationState = oi.ReplicationState()
|
||||
deletion.ReplicationState.ReplicateDecisionStr = dsc
|
||||
continue
|
||||
}
|
||||
if tc.exhaust && round > mrfRetryLimit {
|
||||
if len(p.mrfSaveCh) != 0 || atomic.LoadUint64(&stats.mrfStats.TotalDroppedCount) != 1 {
|
||||
t.Fatalf("retry budget not applied: queued=%d drops=%d", len(p.mrfSaveCh), stats.mrfStats.TotalDroppedCount)
|
||||
}
|
||||
break
|
||||
}
|
||||
var entry MRFReplicateEntry
|
||||
select {
|
||||
case entry = <-p.mrfSaveCh:
|
||||
default:
|
||||
t.Fatal("failure did not enter MRF")
|
||||
}
|
||||
if entry.RetryCount != round || entry.versionID != version {
|
||||
t.Fatalf("MRF entry lost identity/budget: %+v, round=%d", entry, round)
|
||||
}
|
||||
p.saveMRFEntries(ctx, map[string]MRFReplicateEntry{entry.versionID: entry})
|
||||
record, err := p.loadMRF()
|
||||
if err != nil || len(record.Entries) != 1 {
|
||||
t.Fatalf("disk MRF missing: %+v %v", record, err)
|
||||
}
|
||||
if got := record.Entries[version]; got.RetryCount != round || got.Object != name || got.Bucket != bucket {
|
||||
t.Fatalf("disk MRF mismatch: %+v", got)
|
||||
}
|
||||
p.saveMRFEntries(ctx, record.Entries) // loadMRF consumes the file; each replay uses a fresh disk record.
|
||||
p = newPool() // no in-memory entries carried to the replacement pool.
|
||||
if err := p.queueMRFHeal(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deletion = receive()
|
||||
if deletion.RetryCount != round {
|
||||
t.Fatalf("MRF retry count=%d want %d", deletion.RetryCount, round)
|
||||
}
|
||||
if !tc.creation && (deletion.VersionID != version || deletion.DeleteMarkerVersionID != "") {
|
||||
t.Fatalf("MRF emitted wrong purge: %+v", deletion)
|
||||
}
|
||||
}
|
||||
if tc.partial {
|
||||
return
|
||||
}
|
||||
failing.reject.Store(false)
|
||||
globalBucketTargetSys.hMutex.Lock()
|
||||
globalBucketTargetSys.hc[failing.client.EndpointURL().Host] = epHealth{Online: true}
|
||||
globalBucketTargetSys.hMutex.Unlock()
|
||||
if tc.exhaust {
|
||||
oi, _ := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: version})
|
||||
QueueReplicationHeal(ctx, bucket, oi, 0) // Separately prove the existing scanner fallback after MRF exhaustion.
|
||||
deletion = receive()
|
||||
if deletion.RetryCount != 0 {
|
||||
t.Fatal("scanner did not start a fresh retry budget")
|
||||
}
|
||||
}
|
||||
result := replicateDelete(ctx, deletion, updateObj)
|
||||
assertAudit("COMPLETED")
|
||||
if tc.creation {
|
||||
if result.ReplicationStatus() != replication.Completed {
|
||||
t.Fatalf("creation recovery failed: %+v", result)
|
||||
}
|
||||
} else if result.VersionPurgeStatus() != replication.VersionPurgeComplete {
|
||||
t.Fatalf("purge recovery failed: %+v", result)
|
||||
}
|
||||
for _, b := range append([]string{bucket}, func() []string {
|
||||
var b []string
|
||||
for _, target := range targets {
|
||||
b = append(b, target.bucket)
|
||||
}
|
||||
return b
|
||||
}()...) {
|
||||
oi, err := obj.GetObjectInfo(ctx, b, name, ObjectOptions{VersionID: version})
|
||||
if tc.creation {
|
||||
if !oi.DeleteMarker || !isErrMethodNotAllowed(err) {
|
||||
t.Fatalf("creation missing in %s: %+v %v", b, oi, err)
|
||||
}
|
||||
} else if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Fatalf("purge left marker in %s: %+v %v", b, oi, err)
|
||||
}
|
||||
}
|
||||
if !tc.creation {
|
||||
// Re-deliver an ambiguous purge directly. An absent marker must stay absent.
|
||||
duplicate := deletion
|
||||
duplicate.VersionID, duplicate.DeleteMarkerVersionID = "", version
|
||||
duplicate.ReplicationState.PurgeTargets = map[string]VersionPurgeStatusType{failing.arn: replication.VersionPurgePending}
|
||||
duplicate.ReplicationState.VersionPurgeStatusInternal = ""
|
||||
got := replicateDeleteToTarget(ctx, duplicate, &TargetClient{Client: failing.client, ARN: failing.arn, Bucket: failing.bucket})
|
||||
if got.VersionPurgeStatus != replication.VersionPurgeComplete {
|
||||
t.Fatalf("duplicate purge: %+v", got)
|
||||
}
|
||||
if oi, err := obj.GetObjectInfo(ctx, failing.bucket, name, ObjectOptions{VersionID: version}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Fatalf("duplicate recreated marker: %+v %v", oi, err)
|
||||
}
|
||||
}
|
||||
if len(p.mrfSaveCh) != 0 || len(p.workers[0]) != 0 {
|
||||
t.Fatalf("completion left queued work: mrf=%d worker=%d", len(p.mrfSaveCh), len(p.workers[0]))
|
||||
}
|
||||
if !tc.creation && !tc.exhaust && !tc.lostReply {
|
||||
stats.RLock()
|
||||
got := stats.Cache[bucket].Stats[failing.arn].ReplicatedCount
|
||||
size := stats.Cache[bucket].Stats[failing.arn].ReplicatedSize
|
||||
stats.RUnlock()
|
||||
if got != 1 || size != 0 {
|
||||
t.Fatalf("purge completion statistic=%d want 1", got)
|
||||
}
|
||||
}
|
||||
t.Logf("%s: %s recovered; %d target(s), persisted MRF, source/target metadata checked", backend, tc.name, tc.targets)
|
||||
}
|
||||
|
||||
func TestReplicationDeleteQueueFullRetryBudget(t *testing.T) {
|
||||
stats := NewReplicationStats(t.Context(), nil)
|
||||
p := &ReplicationPool{ctx: t.Context(), objLayer: &replicationMRFTestObjectLayer{}, workers: []chan ReplicationWorkerOperation{make(chan ReplicationWorkerOperation)}, stats: stats, mrfSaveCh: make(chan MRFReplicateEntry, 1), priority: "slow"}
|
||||
d := DeletedObjectReplicationInfo{Bucket: "bucket", DeletedObject: DeletedObject{ObjectName: "marker", VersionID: mustGetUUID()}, RetryCount: 2}
|
||||
p.queueReplicaDeleteTask(d)
|
||||
if got := <-p.mrfSaveCh; got.RetryCount != 3 {
|
||||
t.Fatalf("queue-full retry count=%d", got.RetryCount)
|
||||
}
|
||||
d.RetryCount = mrfRetryLimit
|
||||
p.queueReplicaDeleteTask(d)
|
||||
if len(p.mrfSaveCh) != 0 || atomic.LoadUint64(&stats.mrfStats.TotalDroppedCount) != 1 {
|
||||
t.Fatal("queue-full retry exceeded budget without a visible drop")
|
||||
}
|
||||
}
|
||||
|
||||
type markerLockFailureLayer struct{ ObjectLayer }
|
||||
|
||||
func (markerLockFailureLayer) NewNSLock(string, ...string) RWLocker { return markerFailedLock{} }
|
||||
|
||||
type markerFailedLock struct{ RWLocker }
|
||||
|
||||
func (markerFailedLock) GetLock(context.Context, *dynamicTimeout) (LockContext, error) {
|
||||
return LockContext{}, context.DeadlineExceeded
|
||||
}
|
||||
|
||||
type markerLookupLayer struct {
|
||||
ObjectLayer
|
||||
mutate func(ObjectInfo, error) (ObjectInfo, error)
|
||||
lookedUp chan struct{}
|
||||
}
|
||||
|
||||
func (l markerLookupLayer) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
|
||||
oi, err := l.ObjectLayer.GetObjectInfo(ctx, bucket, object, opts)
|
||||
defer close(l.lookedUp)
|
||||
return l.mutate(oi, err)
|
||||
}
|
||||
|
||||
func testReplicationMRFInvalidLookups(ctx context.Context, t *testing.T, obj ObjectLayer, p *ReplicationPool, newPool func() *ReplicationPool, d DeletedObjectReplicationInfo) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
mutate func(ObjectInfo, error) (ObjectInfo, error)
|
||||
}{
|
||||
{"empty-info", func(_ ObjectInfo, e error) (ObjectInfo, error) { return ObjectInfo{}, e }},
|
||||
{"not-a-marker", func(o ObjectInfo, e error) (ObjectInfo, error) { o.DeleteMarker = false; return o, e }},
|
||||
{"wrong-version", func(o ObjectInfo, e error) (ObjectInfo, error) { o.VersionID = mustGetUUID(); return o, e }},
|
||||
{"wrong-bucket", func(o ObjectInfo, e error) (ObjectInfo, error) { o.Bucket = "different-bucket"; return o, e }},
|
||||
{"wrong-object", func(o ObjectInfo, e error) (ObjectInfo, error) { o.Name = "different-object"; return o, e }},
|
||||
{"zero-modtime", func(o ObjectInfo, e error) (ObjectInfo, error) { o.ModTime = time.Time{}; return o, e }},
|
||||
{"missing", func(o ObjectInfo, _ error) (ObjectInfo, error) {
|
||||
return o, ObjectNotFound{Bucket: o.Bucket, Object: o.Name}
|
||||
}},
|
||||
{"read-error", func(o ObjectInfo, _ error) (ObjectInfo, error) { return o, InsufficientReadQuorum{} }},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
entry := d.ToMRFEntry()
|
||||
p.saveMRFEntries(ctx, map[string]MRFReplicateEntry{entry.versionID: entry})
|
||||
fresh := newPool()
|
||||
looked := make(chan struct{})
|
||||
fresh.objLayer = markerLookupLayer{ObjectLayer: obj, mutate: tc.mutate, lookedUp: looked}
|
||||
if err := fresh.queueMRFHeal(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-looked:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("disk MRF entry was not read")
|
||||
}
|
||||
select {
|
||||
case op := <-fresh.workers[0]:
|
||||
t.Fatalf("invalid lookup scheduled %T", op)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Capture the actual internal audit event through the supported webhook sink.
|
||||
func replicationTestAudit(ctx context.Context, t *testing.T, bucket string) (<-chan string, func()) {
|
||||
t.Helper()
|
||||
if len(logger.AuditTargets()) != 0 {
|
||||
t.Fatal("unexpected pre-existing test audit targets")
|
||||
}
|
||||
statuses := make(chan string, 16)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var entry struct {
|
||||
API struct{ Name, Bucket, Status string }
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&entry); err != nil {
|
||||
t.Error(err)
|
||||
} else if entry.API.Name == ReplicateDeleteAPI && entry.API.Bucket == bucket {
|
||||
statuses <- entry.API.Status
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
endpoint, err := xnet.ParseHTTPURL(server.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if errs := logger.UpdateAuditWebhooks(ctx, map[string]loghttp.Config{"r6": {Enabled: true, Name: "r6", Endpoint: endpoint, BatchSize: 1, QueueSize: 128, MaxRetry: 1, RetryIntvl: time.Millisecond, HTTPTimeout: time.Second}}); len(errs) > 0 {
|
||||
t.Fatal(errs)
|
||||
}
|
||||
targets := logger.AuditTargets()
|
||||
return statuses, func() {
|
||||
logger.UpdateAuditWebhooks(ctx, nil)
|
||||
for _, target := range targets {
|
||||
target.Cancel()
|
||||
}
|
||||
server.Close()
|
||||
}
|
||||
}
|
||||
|
||||
type markerPurgeUpdateLayer struct {
|
||||
ObjectLayer
|
||||
t *testing.T
|
||||
}
|
||||
|
||||
func (l markerPurgeUpdateLayer) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
|
||||
l.t.Helper()
|
||||
rs := opts.DeleteReplication
|
||||
if rs.ReplicationStatusInternal != "" || rs.Targets != nil || rs.ReplicaStatus != "" || !rs.CompositeReplicationStatus().Empty() {
|
||||
l.t.Fatalf("purge has a nonempty creation update: %+v", rs)
|
||||
}
|
||||
if rs.CompositeVersionPurgeStatus().Empty() {
|
||||
l.t.Fatal("purge update lost its purge status")
|
||||
}
|
||||
return l.ObjectLayer.DeleteObject(ctx, bucket, object, opts)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user