mirror of
https://github.com/pgsty/minio.git
synced 2026-10-28 19:47:23 +03:00
Compare commits
154 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 39278e068e | |||
| 263ee0510d | |||
| 641eb51a40 | |||
| 45d6e5d065 | |||
| dabc894132 | |||
| fb08ca7b6a | |||
| 41822710e4 | |||
| 034f7bbf73 | |||
| 79eca5e920 | |||
| aec7ddac46 | |||
| 475640e7e1 | |||
| 62124c779c | |||
| 030c6d740e | |||
| 5f1f4178ae | |||
| 0828861bd0 | |||
| 6921f8c186 | |||
| 25d928c9df | |||
| ee3e7ea716 | |||
| 2a04d5f485 | |||
| 9449e6a4a8 | |||
| 9c4bb39571 | |||
| 86ce8d6012 | |||
| b15aa97c94 | |||
| f54c04846e | |||
| 6fdea3b8d9 | |||
| 380fcbfe47 | |||
| a02fa9644c | |||
| 652a1c09d3 | |||
| c86df14f4d | |||
| 03875d61e3 | |||
| 1865985aa1 | |||
| 076a13f9d2 | |||
| 9a4e089f56 | |||
| 9255f443a6 | |||
| ea7c794a94 | |||
| 16c21428d9 | |||
| 7e83f9950a | |||
| 1e3ab56b06 | |||
| 5ad1f344b7 | |||
| 8a4097309f | |||
| 0501a198e3 | |||
| d2eb0ade91 | |||
| 5c074efd1d | |||
| 03fe5e9c21 | |||
| 0b1c21cb4b | |||
| b55495aef0 | |||
| d8b77e5f3b | |||
| cd1f6cec3c | |||
| 4f17a23e33 | |||
| be44ff2669 | |||
| d16f5ed9c8 | |||
| 6d890fc860 | |||
| aafe4f8886 | |||
| a2073d2514 | |||
| 3c82748e14 | |||
| 1e94c5a30d | |||
| 000e653038 | |||
| 945dc16060 | |||
| a30270d9ba | |||
| 1841fce4c4 | |||
| f2b8d4b9cb | |||
| 8ed5c90dcd | |||
| 789a2327ba | |||
| e3e0c6215d | |||
| b91e1c6f2a | |||
| 969a6a42d3 | |||
| 1b5477f3df | |||
| a07d655dc6 | |||
| d43adc0f2b | |||
| 3df51549bb | |||
| 324bb308dd | |||
| 625c11e7e1 | |||
| 4059c25e22 | |||
| 011b0df5f5 | |||
| 9faeb7a114 | |||
| 856f8f4c55 | |||
| 0cd388c666 | |||
| 1748f62c7e | |||
| 60c7179ddb | |||
| 7007c164a5 | |||
| 51b7de6a0d | |||
| 4733e1423c | |||
| 4189869e43 | |||
| 0f9199e0c3 | |||
| bb60871f57 | |||
| 2c820895d7 | |||
| 2080b3d031 | |||
| 9954392605 | |||
| 3dd53eb81f | |||
| 86fb2b833f | |||
| fbcb511978 | |||
| 87c1c6b3dc | |||
| 1e0e5ac53c | |||
| 7e9454456e | |||
| 4ed86dfe62 | |||
| 4e55fc7db4 | |||
| 2616d43fbc | |||
| 2a4db6a4f9 | |||
| e90cb2067d | |||
| 00a375c4e2 | |||
| ab4c97e607 | |||
| 721dbfd1b7 | |||
| 2926474688 | |||
| a7370aa9ac | |||
| a592d69f45 | |||
| bfc7906064 | |||
| eeff112491 | |||
| a70156e45b | |||
| 99245e3ef8 | |||
| 7a3ca0f821 | |||
| d94b3b3c92 | |||
| 86c5a92eef | |||
| cdc3b8cb10 | |||
| 0d3ba5d4cb | |||
| 71a1631f13 | |||
| 2d57b975d1 | |||
| 14042b5aab | |||
| 38a5ab549c | |||
| fa565ec9fc | |||
| 3196480da9 | |||
| cb85461c42 | |||
| ba6e8c8d69 | |||
| a54d0c4354 | |||
| 00d4a206f7 | |||
| f480086b6b | |||
| 968c30238c | |||
| fec1e0804a | |||
| bcb5fc9566 | |||
| 394d9db212 | |||
| 8c784f6133 | |||
| 258b167b1c | |||
| 69d5e92791 | |||
| efd5698597 | |||
| 2fde3cf535 | |||
| 2a4d51406b | |||
| 0596685ae7 | |||
| 358ab38fb0 | |||
| 254b19ac07 | |||
| eb4f5e5b31 | |||
| 8d06424b12 | |||
| fced863036 | |||
| 5af0865aab | |||
| 83821f0f1f | |||
| bf053386a4 | |||
| e7963381ba | |||
| 37c0edc7ca | |||
| a2fe70424e | |||
| 027d43b4eb | |||
| f99ed829b5 | |||
| 956a1a8e33 | |||
| 584b5a3a8f | |||
| 39f49d548f | |||
| e85c4c8dfb | |||
| 15090dc4fd |
@@ -27,6 +27,10 @@ archives:
|
||||
ids:
|
||||
- silo
|
||||
name_template: "silo_{{ .Env.PKG_VERSION }}_{{ .Os }}_{{ .Arch }}"
|
||||
formats: ["tar.gz"]
|
||||
format_overrides:
|
||||
- goos: windows
|
||||
formats: ["zip"]
|
||||
# Explicit so the license materials cannot silently drop out of the
|
||||
# binary archives: GoReleaser's default file globs would miss NOTICE.
|
||||
# CREDITS stays out deliberately -- at 1.8MB it would dominate the
|
||||
|
||||
@@ -93,8 +93,35 @@ jobs:
|
||||
- name: Run conditional PUT tests under race detector
|
||||
run: go test -race ./cmd -run '^Test(PoolsConditionalPut|SinglePoolConditionalPutHTTP)' -count=1 -timeout=5m
|
||||
|
||||
- name: Run multipart listing and cancellation tests under race detector
|
||||
run: go test -race ./cmd -run '^Test(MultipartListing|MultipartAbort|PaginateMultipartUploads|ListMultipartUploads)' -count=1 -timeout=5m
|
||||
- name: Run multi-pool delete consistency tests under race detector
|
||||
run: go test -race ./cmd -run '^Test(Pools(Delete|BulkDelete|RetiringCopyPreservesSharedTierObject)|BulkDeleteQuorumEvidence)' -count=1 -timeout=5m
|
||||
|
||||
- name: Run multipart listing, cancellation and cleanup tests under race detector
|
||||
env:
|
||||
MINIO_API_REQUESTS_MAX: "10000"
|
||||
run: go test -race ./cmd -run '^Test(Multipart|PaginateMultipartUploads|ListMultipartUploads|CleanupStaleUploads|ExtractModTime)' -count=1 -timeout=5m
|
||||
|
||||
- name: Run listing metadata recovery, pagination and cancellation tests under race detector
|
||||
run: go test -race ./cmd -run '^Test(ListFallback|ListingCache|ListingMerge|ListingRecovery|ListingPending|ListingEntry|ListingSecondLock|StreamMetadataParts|LocalLockerListing|DistributedListing|MergeXLV2LatestNullCrossParity|CountXLV2MixedECGroups|ListCrossParity)' -count=1 -timeout=5m
|
||||
|
||||
- name: Run lock cancellation and late-grant cleanup tests under race detector
|
||||
run: go test -race ./internal/dsync ./internal/lsync -run '^Test(DRWMutex|LRWMutex)' -count=1 -timeout=5m
|
||||
|
||||
- name: Run CPU metrics tests under race detector
|
||||
run: go test -race ./cmd -run '^TestLoadCPUMetrics' -count=1 -timeout=5m
|
||||
|
||||
- name: Run tag replication tests under race detector
|
||||
run: go test -race ./cmd -run '^TestAPITagging' -count=1 -timeout=5m
|
||||
|
||||
- name: Run Object Lock authorization and overwrite tests under race detector
|
||||
env:
|
||||
MINIO_API_REQUESTS_MAX: "10000"
|
||||
run: go test -race -tags kqueue,dev ./cmd -run '^Test(APIPutObjectRetention|APIVersionAddressedWrite|PutObjectRetention|DirectoryObjectLock|ObjectLockOverwrite|APICopyObjectPreservesVersionAfterSuspension|MultipartNullObjectLock|PoolsObjectLockOverwrite|MultipartVersioningTransitions|PlanReview|RecoveredMultipartUpload)' -count=1 -timeout=5m
|
||||
|
||||
- name: Run maintenance write, rollback and cleanup tests under race detector
|
||||
env:
|
||||
MINIO_API_REQUESTS_MAX: "10000"
|
||||
run: go test -race -tags kqueue,dev ./cmd ./internal/grid ./internal/ioutil -run '^Test(Mutation|RenameCancellation|PutObjectGenerationFollowsCommitLock|MaintenancePartialFailedOverwriteAvailability|SingleMutationWaitsBeyondDefaultTimeout|WithoutDefaultTimeoutStillEndsOnDisconnectOrCancellation|WithDeadlineAndWait)' -count=1 -timeout=5m
|
||||
|
||||
crosscompile:
|
||||
name: Cross Compile
|
||||
|
||||
@@ -70,6 +70,12 @@ jobs:
|
||||
buildscripts/verify-rebrand.sh
|
||||
dockerscripts/docker-entrypoint_test.sh
|
||||
|
||||
# Keep this gate aligned with the Object Lock race regression in go.yml.
|
||||
- name: Gate release on Object Lock authorization and overwrite tests under race detector
|
||||
env:
|
||||
MINIO_API_REQUESTS_MAX: "10000"
|
||||
run: go test -race -tags kqueue,dev ./cmd -run '^Test(APIPutObjectRetention|APIVersionAddressedWrite|PutObjectRetention|DirectoryObjectLock|ObjectLockOverwrite|APICopyObjectPreservesVersionAfterSuspension|MultipartNullObjectLock|PoolsObjectLockOverwrite|MultipartVersioningTransitions|PlanReview|RecoveredMultipartUpload)' -count=1 -timeout=5m
|
||||
|
||||
- name: Compute release variables
|
||||
env:
|
||||
# Passed through the environment, never interpolated into the script
|
||||
@@ -132,9 +138,9 @@ jobs:
|
||||
cosign-release: v3.1.2
|
||||
|
||||
- name: Build Draft release with GoReleaser
|
||||
uses: goreleaser/goreleaser-action@v7
|
||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
|
||||
with:
|
||||
version: "~> v2"
|
||||
version: v2.18.3
|
||||
args: release --clean --skip=validate --config .github/goreleaser.yml
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -150,7 +156,7 @@ jobs:
|
||||
- name: Install nFPM
|
||||
run: |
|
||||
set -euo pipefail
|
||||
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.47.0
|
||||
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.48.0
|
||||
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Build nFPM packages
|
||||
|
||||
@@ -34,10 +34,11 @@ jobs:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
|
||||
with:
|
||||
python-version: "3.13"
|
||||
- run: python -m pip install -r buildscripts/repository-cards/requirements.txt
|
||||
- name: Use the runner's Python in an isolated environment
|
||||
run: |
|
||||
python3 -m venv "$RUNNER_TEMP/repository-cards-venv"
|
||||
"$RUNNER_TEMP/repository-cards-venv/bin/python" -m pip install -r buildscripts/repository-cards/requirements.txt
|
||||
echo "$RUNNER_TEMP/repository-cards-venv/bin" >> "$GITHUB_PATH"
|
||||
- run: python -m unittest discover -s buildscripts/repository-cards -p 'test_*.py' -v
|
||||
|
||||
publish:
|
||||
@@ -54,10 +55,11 @@ jobs:
|
||||
OUTPUT_BRANCH: codex/repository-cards
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
|
||||
with:
|
||||
python-version: "3.13"
|
||||
- run: python -m pip install -r buildscripts/repository-cards/requirements.txt
|
||||
- name: Use the runner's Python in an isolated environment
|
||||
run: |
|
||||
python3 -m venv "$RUNNER_TEMP/repository-cards-venv"
|
||||
"$RUNNER_TEMP/repository-cards-venv/bin/python" -m pip install -r buildscripts/repository-cards/requirements.txt
|
||||
echo "$RUNNER_TEMP/repository-cards-venv/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Load the generated-assets branch
|
||||
shell: bash
|
||||
|
||||
@@ -4,6 +4,8 @@ on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
paths:
|
||||
- "go.mod"
|
||||
- "go.sum"
|
||||
- ".github/goreleaser.yml"
|
||||
- ".github/nfpm.yml"
|
||||
- "Dockerfile.goreleaser"
|
||||
@@ -58,7 +60,7 @@ jobs:
|
||||
docker build --build-arg TARGETARCH=${{ matrix.arch }} \
|
||||
--target curl-runtime -f Dockerfile.goreleaser -t silo-curl-test .
|
||||
docker run --rm silo-curl-test --version | tee curl-version.txt
|
||||
grep -F 'curl 8.22.0 ' curl-version.txt
|
||||
grep -F 'curl 8.23.0 ' curl-version.txt
|
||||
grep -F 'HTTP2' curl-version.txt
|
||||
docker run --rm silo-curl-test --fail --silent --show-error \
|
||||
--connect-timeout 15 --max-time 60 https://curl.se/robots.txt
|
||||
@@ -93,9 +95,9 @@ jobs:
|
||||
echo "LDFLAGS: ${LDFLAGS}"
|
||||
|
||||
- name: GoReleaser config check
|
||||
uses: goreleaser/goreleaser-action@v7
|
||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
|
||||
with:
|
||||
version: "~> v2"
|
||||
version: v2.18.3
|
||||
args: check --config .github/goreleaser.yml
|
||||
|
||||
- name: Validate Helm chart and legacy upgrade identity
|
||||
@@ -107,9 +109,9 @@ jobs:
|
||||
syft-version: v1.50.0
|
||||
|
||||
- name: Build snapshot artifacts
|
||||
uses: goreleaser/goreleaser-action@v7
|
||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
|
||||
with:
|
||||
version: "~> v2"
|
||||
version: v2.18.3
|
||||
# A pull-request snapshot has no trusted release identity. Exercise
|
||||
# the SBOM/checksum pipeline here, and reserve keyless signing for
|
||||
# the tag-triggered release workflow with GitHub OIDC.
|
||||
@@ -118,13 +120,34 @@ jobs:
|
||||
LDFLAGS: ${{ env.LDFLAGS }}
|
||||
PKG_VERSION: ${{ env.PKG_VERSION }}
|
||||
|
||||
- name: Verify archive SBOM and checksum coverage
|
||||
- name: Verify archive formats, documentation, SBOMs, and checksums
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mapfile -t archives < <(find dist -maxdepth 1 -type f -name 'silo_*.tar.gz' | sort)
|
||||
mapfile -t sboms < <(find dist -maxdepth 1 -type f -name 'silo_*.tar.gz.sbom.json' | sort)
|
||||
mapfile -t archives < <(find dist -maxdepth 1 -type f \( -name 'silo_*.tar.gz' -o -name 'silo_*.zip' \) | sort)
|
||||
mapfile -t sboms < <(find dist -maxdepth 1 -type f \( -name 'silo_*.tar.gz.sbom.json' -o -name 'silo_*.zip.sbom.json' \) | sort)
|
||||
test "${#archives[@]}" -eq 6
|
||||
test "${#sboms[@]}" -eq 6
|
||||
for os in linux darwin windows; do
|
||||
format=tar.gz
|
||||
binary=silo
|
||||
if [ "${os}" = windows ]; then
|
||||
format=zip
|
||||
binary=silo.exe
|
||||
fi
|
||||
for arch in amd64 arm64; do
|
||||
archive="dist/silo_${PKG_VERSION}_${os}_${arch}.${format}"
|
||||
test -s "${archive}"
|
||||
jq -e '.spdxVersion | startswith("SPDX-")' "${archive}.sbom.json" >/dev/null
|
||||
if [ "${format}" = zip ]; then
|
||||
listing="$(unzip -Z1 "${archive}")"
|
||||
else
|
||||
listing="$(tar -tzf "${archive}")"
|
||||
fi
|
||||
for member in "${binary}" LICENSE NOTICE README.md; do
|
||||
grep -Fx "${member}" <<< "${listing}"
|
||||
done
|
||||
done
|
||||
done
|
||||
manifest="dist/silo_${PKG_VERSION}_checksums.txt"
|
||||
test -s "${manifest}"
|
||||
(
|
||||
@@ -141,7 +164,7 @@ jobs:
|
||||
- name: Install package validation tools
|
||||
run: |
|
||||
set -euo pipefail
|
||||
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.47.0
|
||||
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.48.0
|
||||
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
|
||||
sudo apt-get update
|
||||
sudo apt-get install --yes rpm binutils
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
|
||||
- name: Install govulncheck
|
||||
run: |
|
||||
go install golang.org/x/vuln/cmd/govulncheck@v1.8.0
|
||||
go install golang.org/x/vuln/cmd/govulncheck@v1.9.0
|
||||
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Run govulncheck
|
||||
|
||||
+2
-2
@@ -62,10 +62,10 @@ dist/
|
||||
|
||||
.claude/
|
||||
.codex/
|
||||
AGENTS.md
|
||||
CLAUDE.md
|
||||
_bmad/
|
||||
_bmad-output/
|
||||
# Investigation and editorial working files belong outside this repository.
|
||||
docs/investigations/
|
||||
docs/security/
|
||||
docs/rebranding.md
|
||||
.release-sign/
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
# SILO Repository Guide
|
||||
|
||||
## Project map
|
||||
|
||||
- Server: `pgsty/silo` (this checkout, default branch `main`)
|
||||
- Console: `pgsty/silo-console` (usual sibling checkout `../silo-console`)
|
||||
- Client: `pgsty/mc`, shipped as `mcli` (usual sibling checkout `../mc`)
|
||||
- Shared packages: `pgsty/silo-pkg` (usual sibling checkout `../silo-pkg`)
|
||||
- Documentation: `pgsty/silo.pgsty.com` (usual sibling checkout `../silo.pgsty.com`),
|
||||
published at <https://silo.pgsty.com/>
|
||||
|
||||
The server executable, package, systemd service, and container image use `silo`;
|
||||
the public Docker image is `docker.io/pgsty/silo`. The server module remains
|
||||
`github.com/minio/minio`. Consult the current `go.mod` and release configuration
|
||||
for selected component versions and compatibility replacements.
|
||||
|
||||
## Supported stack and compatibility policy
|
||||
|
||||
The maintained, release-gating product graph is the coordinated PGSTY stack:
|
||||
`silo` + `silo-console` + `mc` + `silo-pkg`.
|
||||
|
||||
Keep compatibility with upstream MinIO/MC on a best-effort basis. Preserve
|
||||
inexpensive wire, configuration, CLI, migration, and import compatibility when
|
||||
it helps users, and document known differences. Do not infer from source
|
||||
lineage, MinIO-compatible protocols, retained `MINIO_*`/`MC_*` names, or an
|
||||
inherited upstream test that unmodified upstream MinIO is a supported release
|
||||
target.
|
||||
|
||||
Upstream-only compatibility checks may remain as advisory evidence, but they
|
||||
must not force a downgrade of a maintained SILO component, a fork-only API
|
||||
workaround, or a release block. Make upstream compatibility a hard gate only
|
||||
when the user explicitly requests that scope.
|
||||
|
||||
## Dependency selection
|
||||
|
||||
- When PGSTY maintains a component, import and require it directly under its own
|
||||
module path. In particular, prefer `github.com/pgsty/silo-pkg/v3` over
|
||||
`github.com/minio/pkg/v3` in maintained SILO source.
|
||||
- Do not use an upstream package merely to make unmodified upstream MinIO or MC
|
||||
compile. Unavoidable transitive upstream modules should be documented and
|
||||
kept separate from the maintained product dependency.
|
||||
- `github.com/minio/minio-go/v7` is the explicit exception: use the verified
|
||||
upstream module/commit while it contains the required fixes; do not recreate
|
||||
a SILO fork without a concrete functional divergence.
|
||||
- Check the current `go.mod` before changing versions. Coordinate breaking
|
||||
import-path changes across package, client, Console, and server releases.
|
||||
|
||||
## Documentation and delivery
|
||||
|
||||
The companion site owns product, operations, migration, security-advisory,
|
||||
design, and release documentation. Maintain English and Chinese content in
|
||||
that repository and link to its canonical URLs from this one. The site's
|
||||
`content/docs/` is the documentation entry point; detailed pages live under
|
||||
`content/operations/`, `administration/`, `reference/`, `compatibility/`,
|
||||
`about/`, and `blog/`, following the existing structure.
|
||||
|
||||
Keep repository entry points and contributor instructions here. Retained
|
||||
upstream documents, examples, and test fixtures under `docs/` may be updated
|
||||
when a code or tooling change requires it; do not build a second documentation
|
||||
site in this tree.
|
||||
|
||||
Investigation plans, prompts, AI session transcripts, execution logs, temporary
|
||||
reports, and private security material belong outside the checkout, for example
|
||||
in a task-specific directory under `~/tmp/`. Do not recreate
|
||||
`docs/investigations/`, `docs/security/`, or `docs/rebranding.md`, or move their
|
||||
working material to another repository directory. Extract reusable, verified
|
||||
knowledge into the companion site; keep private evidence outside public Git.
|
||||
|
||||
Compatibility pages may continue to describe similarities with upstream, but
|
||||
must label that compatibility as best effort. The supported and tested server
|
||||
for Console and mcli administration is `pgsty/silo`.
|
||||
|
||||
Before removing or moving documentation, check repository links and scripts,
|
||||
the companion site's links and anchors, and references from this guide. Use
|
||||
fixed commit URLs for historical evidence that should survive deletion from
|
||||
the current tree. Run the site's `make check` for site changes and this
|
||||
repository's `make rebrand-guard` for documentation cleanup that changes the
|
||||
identifier inventory. When new public URLs are involved, publish the site
|
||||
before publishing source changes that depend on those URLs.
|
||||
|
||||
Keep changes in the repository that owns the affected surface. Treat every
|
||||
repository commit, tag, release, image, documentation update, and deployment as
|
||||
a separate deliverable. Follow `CONTRIBUTING.md`, including DCO sign-off
|
||||
(`git commit -s`). `CLAUDE.md` imports this guide so both agents use one policy.
|
||||
+144
-172
@@ -1,189 +1,161 @@
|
||||
# Changelog
|
||||
|
||||
## Unreleased
|
||||
## RELEASE.2026-10-25T00-00-00Z
|
||||
|
||||
The entries below describe source changes on main since the latest published Server.
|
||||
**The latest published Server remains 20260903.** These changes are not in its
|
||||
binaries, packages or images. See the [component matrix](https://silo.pgsty.com/compatibility/versions/)
|
||||
and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-09-03T13-18-01Z...main).
|
||||
Package version: `20261025000000.0.0`. This source is prepared for a Draft
|
||||
Release; public Server artifacts and images remain at 20261018 until promotion.
|
||||
|
||||
### Authorization and security
|
||||
- Fix LDAP login for long normalized DNs without hiding unreadable IAM
|
||||
revocation records. Only length errors for valid keys with an ASCII path
|
||||
component over 255 bytes are treated as absent; long-DN writes still fail.
|
||||
Thanks to [@910patin](https://github.com/910patin) for the original fix and
|
||||
reproduction in [#250](https://github.com/pgsty/silo/pull/250).
|
||||
- Build the server and standalone debugging tools with Go 1.27.3 and refresh
|
||||
Go dependencies, including silo-pkg v3.15.1, the published mcli 20261025
|
||||
source and upstream minio-go `ab93db7ba435` for multipart, read/seek and
|
||||
batch-delete fixes. Retain output-format and NetBSD compatibility selections.
|
||||
- Embed the published Console v2.5.0 source `989fa7150792`, including its
|
||||
GUI update, diagnostic WebSocket closing handshake and complete health-report
|
||||
decoding. Bundle published mcli 20261025 in container builds and select it
|
||||
for the verified client installer and Helm client jobs.
|
||||
- Read current bucket inventory in account-info responses so Console refreshes
|
||||
immediately reflect successful creation and deletion. This management endpoint
|
||||
now performs a live list instead of returning a cached inventory.
|
||||
- Restore the native built-in policy immediately after resetting an override.
|
||||
Peer and etcd notifications reload committed state, so delayed delete events
|
||||
preserve subsequent edits. Keep deletion revisions, policy-in-use checks and
|
||||
custom-policy deletion behavior; storage failures retain the previous policy.
|
||||
- Reload policies without holding the authentication-cache lock, retain IAM
|
||||
write ordering and publish replacements only after a complete successful load.
|
||||
Propagate cancellation through IAM/KMS and SSE-S3 decryption without shortening
|
||||
the caller's policy-query deadline.
|
||||
- Preserve authoritative copies when a multi-pool delete or bulk cleanup fails,
|
||||
choose an available pool for a new delete marker, and keep tier data referenced
|
||||
by surviving copies. This does not introduce strict cross-pool read recovery.
|
||||
- Preserve remote tier data when lifecycle expiry only adds a delete marker,
|
||||
including non-null versions in suspended buckets. Forward `FULL_OBJECT`
|
||||
checksum type for multipart replicas. Thanks to
|
||||
[@botdeniska](https://github.com/botdeniska) for these lifecycle, replication
|
||||
and multi-pool deletion contributions.
|
||||
- Preserve Kafka's default broker protocol 2.1 and terminal-width behavior;
|
||||
fix legacy DARE v1 key unsealing and verify DARE v1/v2 and MinKMS TLS compatibility.
|
||||
- Wait for admitted storage mutations to finish before releasing local guards,
|
||||
deciding rollback or cleaning staging data. Rollback verifies ownership and
|
||||
restores only its overwritten version, preserving later writes and metadata
|
||||
edits. Abandoned-data cleanup retains backups still needed by commit/undo.
|
||||
Generate PUT timestamps under the commit lock. Keep the prior read-quorum
|
||||
and healing policy, including readability after a failed partial overwrite.
|
||||
- Update bundled curl to 8.23.0 and require patched static libraries. Apply
|
||||
[Red Hat's compromised-CA mitigation](https://access.redhat.com/security/vulnerabilities/RHSB-2026-012);
|
||||
the root blocklist also rejects legitimate chains under the affected roots.
|
||||
- Ship Windows platform archives as ZIP; Linux and macOS retain tar.gz.
|
||||
|
||||
- Restrict embedded Console's anonymous sharing proxy to object-content GETs
|
||||
at the configured S3 origin, and reject every redirect. Internal metrics,
|
||||
system paths and non-download S3 operations cannot be reached through it.
|
||||
Normal public, presigned and versioned downloads remain available without a
|
||||
new setting; a full sharing-disable switch is not introduced. See
|
||||
[Console #56](https://github.com/pgsty/silo-console/pull/56) and the
|
||||
[design record](https://github.com/pgsty/silo-console/issues/52).
|
||||
Thanks to Jiri Pejchal (@jiri-pejchal) for the report.
|
||||
- Persist IAM deletion revisions and parent revocation boundaries so stale site
|
||||
events cannot restore deleted identities, policies or their older grants
|
||||
(#191, #192). Peer deletion notifications reload committed storage; deliberate
|
||||
recreation requires a newer revision, and credentials issued before the
|
||||
parent's revocation remain invalid.
|
||||
**Coordinated upgrade required:** upgrade every participating node and site.
|
||||
Mixed old/new nodes sharing an IAM backend and rolling downgrade are
|
||||
unsupported. Back up complete IAM storage and encryption material; an admin
|
||||
export of live records omits deletion history. Reissue credentials for
|
||||
recreated parents and explicitly reconcile pre-upgrade revocations whose
|
||||
history is already lost. Restoring an older backup can lose later revocations;
|
||||
keep affected sites isolated until reconciliation/rekeying is complete. See
|
||||
[the operator runbook](https://github.com/pgsty/silo.pgsty.com/blob/7bd2d57c2ce5aaa804d0b1a2fe0e5eed69d15235/content/operations/replication/iam-upgrade.md).
|
||||
- Enforce an absolute HTTP/1 request-header deadline through the connection
|
||||
wrapper (#196). Repeated small reads no longer extend that deadline, and
|
||||
`--read-header-timeout` / `MINIO_READ_HEADER_TIMEOUT` now reaches the HTTP
|
||||
server. HTTP/1 request bodies retain the rolling idle timeout; this does not
|
||||
impose a total upload/download duration. A shorter setting also constrains
|
||||
TLS handshake reads. The wrapper's strict header mode is not applied to HTTP/2.
|
||||
- Reject unsigned `x-amz-*` request headers that could turn a signed PUT into a
|
||||
copy of another object accessible to the signer (SN-2026-011). The latest
|
||||
public Server is affected; the fix is on main. See [the advisory ledger](docs/security/advisories.md).
|
||||
- Align signed request fields with policy conditions and enforce header-only
|
||||
presigned payload checksums. See [the signed-header review](https://silo.pgsty.com/blog/design/signed-header-coverage/).
|
||||
- **Breaking policy semantics:** separate self-service `admin:ChangeMyPassword`
|
||||
from `admin:CreateUser`. Built-in read-only policies follow the split. Preserve
|
||||
both denies if the previous combined restriction must survive upgrades or
|
||||
rollback. Saved policies are not rewritten. Deploy with the matching Console
|
||||
and pkg; see [the migration guide](docs/iam/password-permissions.md).
|
||||
Storage limits remain: this does not provide distributed fencing after lease
|
||||
loss or network partition, eliminate stale reads after partial metadata writes,
|
||||
or repair previously inconsistent objects. Stricter read refusal and additional
|
||||
cross-pool recovery remain deferred; their diagnostic tests are retained under
|
||||
`consistency_strict`. Waiting for admitted I/O can extend same-object waits on
|
||||
a stuck drive, and interrupted cleanup can retain rollback backups. Upgrade all
|
||||
writers before relying on the new rollback ownership checks.
|
||||
|
||||
### Object storage and replication
|
||||
These source changes are not included in the published 20261018 artifacts.
|
||||
|
||||
- Make `ListMultipartUploads` discover quorum-valid uploads from durable state
|
||||
across pools, erasure sets and drives, then apply S3 prefix, delimiter,
|
||||
marker, ordering and 1,000-entry pagination semantics globally (#198). New uploads
|
||||
store their canonical bucket and key as reserved fields in the existing
|
||||
quorum-written `xl.meta`; completion removes those upload-only fields. Native
|
||||
markers remain usable after their upload is completed or canceled. Strict
|
||||
listing returns a diagnostic 503 for legacy uploads or uncertain coverage;
|
||||
the default remains the released exact-key/cache-based `legacy` behavior.
|
||||
Opt into strict mode only through `MINIO_API_MULTIPART_LISTING=strict`, after
|
||||
upgrading every writer, draining old uploads, checking the read-only admin
|
||||
`multipart-preflight` report and validating scan capacity. The process-only
|
||||
setting is not persisted into shared API configuration. Historical
|
||||
`multipart_listing` keys are ignored and can be removed with a targeted
|
||||
`mcli admin config reset ALIAS api multipart_listing` before rollback. Per-process
|
||||
admission, directory-entry, worker and time budgets bound scan scheduling;
|
||||
each page still scans durable state. See [issue #79](https://github.com/pgsty/silo/issues/79)
|
||||
and its [design record](https://silo.pgsty.com/blog/design/list-multipart-uploads/).
|
||||
Thanks to mr javad seydi (@mrjavadseydi) for the original implementation.
|
||||
- Retain released read-quorum and best-effort multipart cancellation in default
|
||||
legacy mode. Strict mode requires majority deletion acknowledgements and
|
||||
permits retries below read quorum. When most drives were already empty,
|
||||
failed deletion of an observed remnant now returns 503 instead of being
|
||||
masked by empty-drive successes. Wrong-key or wrong-bucket cancellation
|
||||
preserves the valid upload's cache entry; successful cancellation notifies
|
||||
peers with the request context after the distributed lock is released.
|
||||
The HTTP response for an absent
|
||||
upload remains 204; this is not proof of physical cleanup. **Known boundary:**
|
||||
delayed creation writes can still restore an upload after cancellation;
|
||||
this change does not add a durable creation fence.
|
||||
- Preserve object tags during multi-pool metadata reconciliation by reading the
|
||||
resolved tag field together with its revision (#189). Previously, reconciliation
|
||||
could replace existing tags with an empty value.
|
||||
- Preserve the tag revision on SSE-KMS metadata replication (#193), and advance
|
||||
tag revisions monotonically on local PUT/DELETE tagging (#196). Empty tags
|
||||
participate in reconciliation as an ordered deletion, preventing older
|
||||
events from restoring removed tags. SSE-C key rotation also retains the tag
|
||||
revision. Malformed historical revisions can fail and retry; their missing
|
||||
history is not reconstructed by the upgrade.
|
||||
- Complete delete-marker version purges and preserve their identity and retry
|
||||
state through MRF recovery (#196). Recovery accepts a 405 marker response only
|
||||
when its version, bucket, object name and modification time match the task.
|
||||
Purge audit status is normalized from `COMPLETE` to `COMPLETED`.
|
||||
Thanks to Julien Laurenceau (@julienlau) for the investigation and proposed
|
||||
fix in #184 that helped shape this follow-up.
|
||||
- Restore only the six replication-specific metadata fields after ordinary
|
||||
request metadata extraction (#194). This prevents transport-only `aws-chunked`
|
||||
from being stored as Content-Encoding while preserving the signed-header
|
||||
protections. Trusted Snowball entries no longer inherit the outer archive's
|
||||
ordinary metadata. Thanks to Mikhail Khadarenka (@chodorenko) for the fix in #187.
|
||||
**Existing data:** these repairs prevent new errors; they do not scan or rewrite
|
||||
historical object metadata, recover lost tags or prove that old purge work has
|
||||
converged. Follow the [read-only audit procedure](https://github.com/pgsty/silo.pgsty.com/blob/7bd2d57c2ce5aaa804d0b1a2fe0e5eed69d15235/content/operations/replication/replica-metadata-audit.md)
|
||||
before planning any repair of stored state.
|
||||
## RELEASE.2026-10-18T00-00-00Z
|
||||
|
||||
- Evaluate conditional multipart completion against the logical current object
|
||||
across all pools while holding the existing object lock. A stale `If-Match`
|
||||
can no longer replace newer data in another pool, and the current ETag is no
|
||||
longer rejected because the upload resides next to an older copy. Conditions
|
||||
are evaluated once; a current delete marker counts as an absent object.
|
||||
**Availability change:** if any pool's metadata cannot be read, conditional
|
||||
completion fails even when another pool can still serve GET/HEAD. This also
|
||||
applies when the unreadable pool may not hold the object: absence cannot be
|
||||
verified. Retry after the pool recovers. Unconditional completion and the
|
||||
single-pool path retain their existing behavior.
|
||||
Package version: `20261018000000.0.0`; published on 2026-10-18 from source
|
||||
`0cd388c666f8211f552604ae73448a066e9c25ce`. This section covers the changes
|
||||
after the public 20260916 source, `2a4d51406b7ed87af5fe6fe0f801f3290f96eb3c`.
|
||||
|
||||
- Evaluate ordinary multi-pool conditional PUT against the logical current
|
||||
object across all pools, including draining pools, under the existing object
|
||||
lock (#207). A stale destination copy no longer accepts a stale ETag or rejects
|
||||
the current one; a current delete marker is treated as absence.
|
||||
**Availability change:** if any pool's object metadata cannot be verified,
|
||||
the condition fails even when GET can use another pool; read-quorum failures
|
||||
return 503. Restore readability or heal before retrying. Unconditional PUT,
|
||||
single-pool conditions and internal replication retain their existing behavior.
|
||||
A public condition with a destination `versionId` compares the current object
|
||||
while preserving the requested write version. This change does not retire
|
||||
stale copies in other pools, undo historical accepted overwrites or provide
|
||||
a new global clock-ordering guarantee. The multipart-completion repair in #190
|
||||
neither introduced nor repaired this separate PUT defect.
|
||||
Full [English release notes](https://silo.pgsty.com/blog/release/silo-20261018/)
|
||||
and [Chinese release notes](https://silo.pgsty.com/zh/blog/release/silo-20261018/)
|
||||
cover the upgrade contract, evidence and remaining limits.
|
||||
|
||||
- Reconcile ordinary single-object version DELETE across all pools, including
|
||||
null versions, delete markers and unqualified directory-marker DELETE. This
|
||||
applies the deletion to every resolved pool copy under existing quorum
|
||||
rules. Pending outbound delete replication retains versions until the
|
||||
existing replication worker completes their purge; a successful response
|
||||
does not imply immediate physical removal from every drive. Unreadable
|
||||
pools now consistently return 503 instead of depending on pool traversal
|
||||
order; insufficient read quorum returns `SlowDownRead`. This extends the
|
||||
existing failure surface. Retry after recovery.
|
||||
Cleanup failures also return an error. Batch deletion already fans out across
|
||||
pools; replication and scanner cleanup keep their existing contracts. See
|
||||
[scope and limitations](docs/bucket/lifecycle/access-tiering-removal.md#version-deletion-scope).
|
||||
- **Object Lock (#234, #235):** require `s3:PutObjectRetention` with request-body
|
||||
conditions before object lookup; weakening active GOVERNANCE additionally
|
||||
requires bypass intent and permission. Protect caller-addressed write versions,
|
||||
directory null versions and persisted multipart targets under the write lock;
|
||||
fail closed on unreadable lock state. See SN-2026-015/016 in the
|
||||
[advisory ledger](https://silo.pgsty.com/about/security-advisories/).
|
||||
- **IAM Admin authorization (#242, `995439260`):** enforce explicit self denies,
|
||||
session restrictions and plugin decisions; prevent restricted delegation and
|
||||
non-root mutation of root-derived credentials under built-in authorization.
|
||||
Preserve empty-policy inheritance and damaged-key cleanup. Normalize bulk
|
||||
self-list aliases and allow ordinary policyless self inspection. See
|
||||
[SN-2026-017](https://silo.pgsty.com/blog/security/20261018-release-hardening/#sn-2026-017).
|
||||
- **Scoped STS revocation (`0cd388c66`):** enumerate durable parent/type-selected
|
||||
tokens, propagate each committed tombstone to sibling nodes and sites, and
|
||||
recover missed notifications through existing refresh/healing. Tokens issued
|
||||
only at an isolated site are outside the requesting cluster's snapshot;
|
||||
propagation is eventual and newly issued keys remain valid.
|
||||
- **Pool migration (#248, `60c7179dd`, `1748f62c7`):** prevent stale moved
|
||||
payloads from overwriting newer copies, preserve null/UUID delete markers,
|
||||
recheck source versions and clean only confirmed moved versions under the
|
||||
object lock. Added per-pool reads and per-version cleanup have maintenance
|
||||
costs; earlier data loss is not reconstructed.
|
||||
|
||||
- Remove the opt-in GET-frequency pool-tiering feature from PR #60, including
|
||||
its tracker, mover, scanner hooks, configuration, XML actions and metrics.
|
||||
Accept and ignore retired configuration/XML and preserve ordinary statistics
|
||||
when reading v9 caches. See [migration notes](docs/bucket/lifecycle/access-tiering-removal.md).
|
||||
The [decision record](docs/investigations/access-tiering-revert.md) preserves
|
||||
the feature's introduction, subsequent fixes, rollback scope and review history.
|
||||
- Preserve the independent multi-pool write, metadata, healing and conditional
|
||||
deletion fixes from PR #178, including shared remote-tier reference protection.
|
||||
- Enforce `If-Match` on DELETE, preserve retention and independently ordered
|
||||
Object Lock/tag updates, and correctly retransmit encrypted replicas.
|
||||
- Preserve plaintext part sizes and raw SSE-C replicas; prevent SSE-C
|
||||
compression, honor key-rotation checksums, and complete attributes pagination.
|
||||
- Repair federated CopyObject checksums, destination timestamps, reserved
|
||||
metadata, encrypted-object forwarding, legal hold and KMS context.
|
||||
- Make resync counters, target selection, cancellation and worker lifetimes
|
||||
reflect actual work, and report bounded MRF drops.
|
||||
- Converge bucket metadata with deterministic source state, deletion tombstones,
|
||||
creation time recovery and diagnostics. The mixed-version export gate requires
|
||||
coordinated upgrades before tombstones are exported. See [the #77 record](docs/investigations/issue-77-current.md).
|
||||
- Include per-bucket CORS in metadata export/import, close metadata publication
|
||||
and logger races, and report effective bucket quotas in metrics.
|
||||
- **LIST recovery (#239, `3dd53eb81`, `2080b3d03`):** confirm eligible uncertain
|
||||
latest-object groups under the appropriate read locks, distinguish absence
|
||||
from failure, preserve directory/common-prefix entries and terminal pagination
|
||||
errors, and make lock backoff cancelable. Resolved entries keep their fast path;
|
||||
version history, quorum and disk formats retain their contracts. The strict
|
||||
sub-5% performance target remains unconfirmed in the recorded comparison.
|
||||
- **Multipart listing (`4733e1423`):** withdraw #243's default persistent scan.
|
||||
Public listing uses node-local cache or exact-key lookup; restart omissions,
|
||||
prefix and cache-pagination limits remain. The retired strict switch is ignored
|
||||
with a startup warning. **Former strict users lose durable listing and strict
|
||||
cancellation.** Keep identity metadata, path/wrong-target safety, default
|
||||
best-effort Abort and peer notification. Diagnostic `multipart-preflight`
|
||||
reports `mode=legacy`; its budgets do not gate public operations. #79 remains
|
||||
open, including delayed creation after cancellation.
|
||||
- **Multipart cleanup (#236, `0f9199e0c`):** parse native timestamps before the
|
||||
Base64 fallback and remove empty hash parents without a second expiry wait.
|
||||
Keep active/unreadable uploads; default expiry/check intervals stay 24h/6h.
|
||||
- **Commit errors (#232):** missing staged `xl.meta` at final commit becomes
|
||||
HTTP 503 `SlowDownWrite`; preserve this classification across storage RPC and
|
||||
healing. Real incomplete bodies retain their existing errors.
|
||||
- **Reused metadata (#233):** clear absent legacy `EcM`/`Signature` and locally
|
||||
reset mutually exclusive scanner response fields without changing disk/RPC formats.
|
||||
- **Checksum and Console errors (#225, #229, #238):** align Content-MD5 and
|
||||
algorithm-specific messages, retaining the generic unknown-algorithm message;
|
||||
embed Console's wrapped-error and `InvalidBucketState` handling.
|
||||
- **Lambda and erasure documentation (#220, `2fde3cf53`):** fix inverted
|
||||
`TargetIDSet.IsEmpty`; document parity/quorum per object and erasure set.
|
||||
- **Coordinated dependencies:** Console v2.4.2 (`ef2dd2c41d21`), mcli 20261018
|
||||
(`fa3b8bf72458`), silo-pkg v3.14.3, upstream minio-go v7.4.1 and Go 1.27.2.
|
||||
Standalone Console/mcli still declare pkg v3.14.2. The Server package update
|
||||
speeds admin-action classification without changing policy behavior. Refresh
|
||||
Go x/*, go-jose v4.1.6, LDAP v3.4.15, compress v1.20.3, msgp v1.6.5, etcd v3.7.2,
|
||||
fwd v1.2.1 and cpuid v2.4.1. OIDC RSA JWKs above 4096 bits are rejected.
|
||||
Keep the NetBSD go-systemd v22.6.0 replacement; parallel SDK GET/Compose remain opt-in.
|
||||
- **Build and delivery:** gate releases on Object Lock regressions; use
|
||||
Git for rebrand scanning without a ripgrep dependency (`7007c164a`). Refresh UBI 9 pins,
|
||||
GoReleaser v2.18.3, nFPM v2.48.0, golangci-lint v2.15.0 and govulncheck v1.9.0.
|
||||
Update bundled client hashes/helper and the Helm client image to mcli 20261018.
|
||||
The chart's Server default stays at the public 20260916 image until publication.
|
||||
|
||||
### Console, dependencies and delivery
|
||||
Upgrade every writer and replication site; follow the
|
||||
[IAM recovery guide](https://silo.pgsty.com/operations/replication/iam-upgrade/).
|
||||
Old strict multipart users must review the capability loss before upgrading.
|
||||
#217's multi-site delete-marker late creation and crash residue remain unresolved.
|
||||
Source tests, draft artifacts, public images and production rollout are separate
|
||||
acceptance boundaries; historical 20261015 test counts do not certify this source.
|
||||
|
||||
- Restore embedded Console login over loopback TLS, trusted-proxy handling and
|
||||
all four WebSocket connection limits. Preserve Go TLS defaults across transports.
|
||||
- Directly require `github.com/pgsty/silo-pkg/v3` v3.14.0; select Console
|
||||
`v0.0.0-20260916034812-56dfe455ac2f` and MC
|
||||
`v0.0.0-20260913012246-4f609a4da3bb` with explicit PGSTY replacements.
|
||||
- Pin upstream minio-go `v7.3.1-0.20260910142817-60bd07042d49`; refresh Go x/*
|
||||
modules and security fixes including bounded AMQP frame handling. Keep Go
|
||||
1.27.1 and go-systemd v22.6.0's NetBSD compatibility replacement.
|
||||
- Refresh container base digests and build static curl 8.22.0 from verified
|
||||
source for both Linux architectures. Pin the actual mcli 20260913 archives and
|
||||
hashes. Helm's client image follows that release; its Server image still names
|
||||
the latest published Server 20260903.
|
||||
## RELEASE.2026-09-16T00-00-00Z
|
||||
|
||||
The dependency update passed the final candidate's Go, vulnerability and Test
|
||||
Release workflows; native curl builds passed on both architectures. A local
|
||||
ARM64 image passed startup, health, S3 transfer and embedded Console checks.
|
||||
These checks do not publish a Server tag or production image and do not replace
|
||||
cluster upgrade/rollback acceptance for the next release. Dated investigations
|
||||
retain the exact source and runtime boundaries they tested.
|
||||
Package version: `20260916000000.0.0`; published source:
|
||||
`2a4d51406b7ed87af5fe6fe0f801f3290f96eb3c`.
|
||||
|
||||
This release includes signed-header and payload-policy hardening (SN-2026-011/012),
|
||||
durable IAM revocations (SN-2026-013), the bounded Console sharing proxy
|
||||
(SN-2026-014), password-permission separation, multi-pool conditions/deletion/tag
|
||||
repairs, encrypted/federated copy repairs, delete-marker retry fixes, opt-in strict
|
||||
multipart discovery, Go-default TLS key exchange, absolute HTTP/1 header deadlines
|
||||
and synchronized CPU metrics. It embeds Console v2.4.1, mcli 20260916 and pkg v3.14.1.
|
||||
|
||||
The complete release, contributor credits, historical artifact acceptance and
|
||||
upgrade requirements are preserved in the canonical
|
||||
[English](https://silo.pgsty.com/blog/release/silo-20260916/) and
|
||||
[Chinese](https://silo.pgsty.com/zh/blog/release/silo-20260916/) release records.
|
||||
|
||||
## RELEASE.2026-09-03T13-18-01Z
|
||||
|
||||
|
||||
@@ -53,6 +53,16 @@ Before opening a pull request:
|
||||
- Explain any preserved `MINIO_*`, `minio_*`, `x-minio-*`, `/minio/*`,
|
||||
`.minio.sys`, ARN, module/import-path, or serialized compatibility name.
|
||||
|
||||
The maintenance branch retains the existing read-quorum and healing policy.
|
||||
Deferred strict-read and cross-pool recovery tests are preserved behind the
|
||||
`consistency_strict` build tag. Run them explicitly with
|
||||
`go test -tags kqueue,dev,consistency_strict ./cmd`; known failures must be
|
||||
reported separately, not described as fixed or included in a passing result.
|
||||
They cover late commits after lease loss, stale metadata reads, and stricter
|
||||
read refusal. The normal suite covers the retained write/undo/cleanup fixes
|
||||
and continued readability after a failed partial overwrite. This scope does
|
||||
not provide distributed fencing or repair previously inconsistent objects.
|
||||
|
||||
### Commit changes
|
||||
|
||||
After verification, commit your changes with a concise message and a DCO
|
||||
@@ -77,6 +87,16 @@ test evidence, compatibility notes, and documentation impact. Public product
|
||||
documentation is owned by the separate
|
||||
[`pgsty/silo.pgsty.com`](https://github.com/pgsty/silo.pgsty.com) repository.
|
||||
|
||||
## Contributor recognition
|
||||
|
||||
Every human issue or pull-request author is recognized in [CONTRIBUTORS.md](CONTRIBUTORS.md),
|
||||
including open issues, draft PRs, and PRs closed without merging. Merged fixes,
|
||||
adopted proposals, and reports that lead to fixes receive greater prominence;
|
||||
first participation guides the remaining order. Work incorporated through a
|
||||
later PR retains credit without changing the original PR's recorded status.
|
||||
Security disclosures are credited with the reporter's agreement. DCO sign-off
|
||||
applies to code commits, not to opening an issue.
|
||||
|
||||
## Licensing of Contributions
|
||||
|
||||
Code contributions to PGSTY SILO (`pgsty/silo`) are accepted under the
|
||||
|
||||
+153
-129
File diff suppressed because one or more lines are too long
+19
-7
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.27.1-alpine@sha256:cf6fca6641884b8433441b2b0652976f975e1d0fdd26d177eaaf8596087f3125 AS curl-build
|
||||
FROM golang:1.27.3-alpine@sha256:d530cf60723abf1ffae0cd16bb546b379ec0550d2f3815e5b16ed69dbc00fba4 AS curl-build
|
||||
ARG TARGETARCH
|
||||
COPY dockerscripts/build-static-curl.sh /build/build-static-curl
|
||||
RUN /bin/sh /build/build-static-curl
|
||||
@@ -9,7 +9,7 @@ COPY --from=curl-build /go/bin/curl /curl
|
||||
COPY --from=curl-build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
ENTRYPOINT ["/curl"]
|
||||
|
||||
FROM golang:1.27.1-alpine@sha256:cf6fca6641884b8433441b2b0652976f975e1d0fdd26d177eaaf8596087f3125 AS build
|
||||
FROM golang:1.27.3-alpine@sha256:d530cf60723abf1ffae0cd16bb546b379ec0550d2f3815e5b16ed69dbc00fba4 AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
|
||||
@@ -17,9 +17,9 @@ ENV GOPATH=/go
|
||||
ENV CGO_ENABLED=0
|
||||
|
||||
ARG MC_REPO=pgsty/mc
|
||||
ARG MC_VERSION=RELEASE.2026-09-13T00-00-00Z
|
||||
ARG MC_AMD64_SHA256=9d2a92de9c7b887d9b944fe9ddce68d23f1b6df3415092e737594e56593f5e2b
|
||||
ARG MC_ARM64_SHA256=3d82e9ea6c601c4cb44fe5dd5f2ad1b7d7d64369378110f9ada9c524688a452a
|
||||
ARG MC_VERSION=RELEASE.2026-10-25T00-00-00Z
|
||||
ARG MC_AMD64_SHA256=54bd131039dad6677f083a75207b28f82818c067c8e65fae1973f51510d268e1
|
||||
ARG MC_ARM64_SHA256=9bb294f1912f8c6b9ac08c9a0b8d2964b86306a7c440b987ee5f0570d18de3c5
|
||||
|
||||
RUN apk add -U --no-cache \
|
||||
ca-certificates \
|
||||
@@ -67,14 +67,26 @@ RUN apk add -U --no-cache \
|
||||
chmod +x /go/bin/mcli && \
|
||||
ln -sf mcli /go/bin/mc
|
||||
|
||||
FROM registry.access.redhat.com/ubi9/ubi:latest@sha256:206b65b8ee0f04b992818c9a51b29081b14974630d4850bc358097d0c44ea156 AS certs
|
||||
FROM registry.access.redhat.com/ubi9/ubi:latest@sha256:16e9e43d99e181463513cc84a514859b1abb9460f1c4d9e208da4b4e51cd5cc3 AS certs
|
||||
# Native root blocklist mitigation: https://access.redhat.com/security/vulnerabilities/RHSB-2026-012
|
||||
RUN dnf -y install ca-certificates && \
|
||||
if [ -d /etc/pki/ca-trust/source/blocklist ]; then \
|
||||
blocklist=/etc/pki/ca-trust/source/blocklist; \
|
||||
else \
|
||||
blocklist=/etc/pki/ca-trust/source/blacklist; \
|
||||
fi && \
|
||||
trust extract --overwrite --format=pem-bundle \
|
||||
--filter='pkcs11:object=Comodo%20AAA%20Services%20root;type=cert' \
|
||||
"$blocklist/Comodo_AAA_Services_root.pem" && \
|
||||
trust extract --overwrite --format=pem-bundle \
|
||||
--filter='pkcs11:object=emSign%20ECC%20Root%20CA%20-%20G3;type=cert' \
|
||||
"$blocklist/emSign_ECC_Root_CA_-_G3.pem" && \
|
||||
update-ca-trust && \
|
||||
cp /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /tmp/ca-certificates.crt && \
|
||||
dnf clean all && \
|
||||
rm -rf /var/cache/dnf
|
||||
|
||||
FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:f332c99eb8f798a8486821c91937f10ad64ee83d7e739303be2df051040918f6
|
||||
FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:5be877a2a280ae36eb9ab80ca4efb28bbf77c57abbba38137cdc37a03b8465e6
|
||||
|
||||
LABEL org.opencontainers.image.title="Silo" \
|
||||
org.opencontainers.image.description="S3-Interface Libre Object Storage" \
|
||||
|
||||
@@ -4,7 +4,7 @@ LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
|
||||
|
||||
GOOS ?= $(shell go env GOOS)
|
||||
GOARCH ?= $(shell go env GOARCH)
|
||||
GOLANGCI_VERSION ?= v2.13.1
|
||||
GOLANGCI_VERSION ?= v2.15.0
|
||||
|
||||
VERSION ?= $(shell git describe --tags)
|
||||
REPO ?= docker.io/pgsty
|
||||
|
||||
@@ -35,13 +35,20 @@
|
||||
> [!NOTE]
|
||||
> Renamed from `pgsty/minio` to `pgsty/silo`, default branch `master` → `main`, on 2026-08-06. Artifacts under the original MinIO identity stay published on the archived [`minio`](https://github.com/pgsty/silo/tree/minio) branch and in releases up to [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z).
|
||||
|
||||
## Current release and main branch
|
||||
## SILO 20261018000000
|
||||
|
||||
The latest published Server is [20260903](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z).
|
||||
As of 2026-09-13, the main branch has newer security, storage, Console and
|
||||
shared-package changes that have not shipped in a Server release. See
|
||||
[CHANGELOG.md](CHANGELOG.md) and the [component version matrix](https://silo.pgsty.com/compatibility/versions/)
|
||||
for the exact release/source boundary, including SN-2026-011 and password-policy migration.
|
||||
[SILO 20261018000000](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-18T00-00-00Z) is available as platform archives, Linux packages, and standard or distroless Docker images for amd64 and arm64. The release tag is `RELEASE.2026-10-18T00-00-00Z`; the package version is `20261018000000.0.0`.
|
||||
|
||||
This release strengthens Object Lock and IAM authorization, makes scoped STS revocation durable across processes and sites, protects concurrent updates during pool migration, and improves LIST recovery, multipart cleanup, and S3 error handling. It includes Console v2.4.2 and silo-pkg v3.14.3, and is built with Go 1.27.2. Use mcli 20261018 for administration.
|
||||
|
||||
Read the [release notes](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-18T00-00-00Z) before upgrading, especially the removal of strict multipart listing, coordinated server upgrades, and measured performance costs. See [CONTRIBUTORS.md](CONTRIBUTORS.md) for the people behind the code, reports, and documentation.
|
||||
|
||||
The 20261025 source is prepared for a Draft Release with Go 1.27.3,
|
||||
silo-pkg v3.15.1, released mcli 20261025, Console v2.5.0 (`989fa7150792`),
|
||||
and upstream minio-go `ab93db7ba435`. It includes the Server IAM cache repair
|
||||
needed by Console's **Reset to default** action and bounded write/rollback/cleanup
|
||||
repairs. Read the [changes and remaining storage limits](CHANGELOG.md#release2026-10-25t00-00-00z).
|
||||
The published artifacts above retain their original component versions.
|
||||
|
||||
## Overview
|
||||
|
||||
@@ -62,7 +69,7 @@ docker run -d --name silo -p 9000:9000 -p 9001:9001 \
|
||||
-e MINIO_ROOT_USER=minioadmin \
|
||||
-e MINIO_ROOT_PASSWORD=change-me-long-password \
|
||||
-v "$PWD/data:/data" \
|
||||
docker.io/pgsty/silo:latest server /data --console-address ":9001"
|
||||
docker.io/pgsty/silo:RELEASE.2026-10-18T00-00-00Z server /data --console-address ":9001"
|
||||
```
|
||||
|
||||
<p align="center">
|
||||
@@ -93,12 +100,15 @@ Every release ships checksums, SPDX SBOMs, Sigstore-signed manifests, and GitHub
|
||||
|
||||
## Compatibility
|
||||
|
||||
The S3 API, `MINIO_*` variables, `minio_*` metrics, `x-minio-*` headers, `/minio/*` routes, the `github.com/minio/*` import paths, and the on-disk format (including `.minio.sys`) are preserved and held in place by a CI compatibility check. Only Silo-owned delivery surfaces change: the `silo` executable, package, service, Helm chart, and container image — no `minio` binary alias is installed.
|
||||
Silo preserves S3 and storage-format compatibility, including existing `MINIO_*` variables, `minio_*` metrics, `x-minio-*` headers, `/minio/*` routes, and `.minio.sys` data. CI guards selected compatibility identifiers; release notes document intentional security and behavior changes. Silo-owned delivery surfaces use the `silo` executable, package, service, Helm chart, and container image; no `minio` server binary alias is installed.
|
||||
|
||||
The supported release stack is `pgsty/silo` + `pgsty/silo-console` + `pgsty/mc` + `pgsty/silo-pkg`; compatibility with unmodified upstream MinIO/MC is best effort. The Server, Console, and client retain their historical module paths where needed, while maintained code imports `github.com/pgsty/silo-pkg/v3` directly. The SDK `github.com/minio/minio-go/v7` is an explicit upstream dependency. See the current [go.mod](go.mod) for versions and replacements.
|
||||
|
||||
Every divergence from upstream is listed in the code-verified [compatibility audit](https://silo.pgsty.com/compatibility/server/). Treat each release as a downstream upgrade: pin versions, read the [release notes](https://silo.pgsty.com/tags/silo/), and keep a rollback path.
|
||||
|
||||
### TLS and Go upgrades
|
||||
|
||||
Server 20260916 restored Go's default key exchange; Server 20261018 retains it with Go 1.27.2.
|
||||
TLS key exchange follows Go's defaults across the S3 listener, node links,
|
||||
replication, identity providers, etcd, and external HTTP services. If an endpoint
|
||||
cannot accept ML-KEM, `GODEBUG=tlsmlkem=0` disables the default hybrid exchanges
|
||||
@@ -115,7 +125,16 @@ values to restore Keychain trust. Explicit certificates in the configured `CAs`
|
||||
directory remain additive to the selected root pool.
|
||||
Go 1.27 binaries require macOS 13 or later. See the
|
||||
[Go release notes](https://go.dev/doc/go1.27) and the
|
||||
[SILO stack investigation](docs/investigations/go127-stack.md).
|
||||
[Go 1.27 TLS and OIDC discovery guide](https://silo.pgsty.com/blog/design/go127-tls-oidc-discovery/).
|
||||
|
||||
## Documentation ownership
|
||||
|
||||
User documentation is maintained at [silo.pgsty.com](https://silo.pgsty.com/docs/),
|
||||
with source in [pgsty/silo.pgsty.com](https://github.com/pgsty/silo.pgsty.com).
|
||||
The remaining `docs/` tree contains inherited references, examples, and tooling
|
||||
fixtures. Investigation logs, AI work records, and temporary reports are kept
|
||||
outside this repository; reusable findings belong in the companion site.
|
||||
See [AGENTS.md](AGENTS.md) for repository ownership and maintenance rules.
|
||||
|
||||
## Security & Contributing
|
||||
|
||||
@@ -125,7 +144,7 @@ Report vulnerabilities privately as described in [`SECURITY.md`](SECURITY.md); e
|
||||
|
||||
<!-- Generated by silo.pgsty.com/bin/contributors.py. -->
|
||||
|
||||
Every human issue or pull-request author is part of the SILO community, including open and unmerged work. Merged fixes, adopted proposals, and actionable reports receive priority, with first participation guiding the remaining order. Gold rings highlight reviewed significant contributions.
|
||||
Every human issue or pull-request author is part of the SILO community, including open and unmerged work. We also recognize security reports, including reports sent by email, with the reporter's permission. Merged fixes, adopted proposals, and actionable reports receive priority, with first participation guiding the remaining order. Gold rings highlight reviewed significant contributions.
|
||||
|
||||
<a href="CONTRIBUTORS.md">
|
||||
<picture>
|
||||
|
||||
+31
-8
@@ -35,12 +35,20 @@
|
||||
> [!NOTE]
|
||||
> 2026-08-06,本仓库由 `pgsty/minio` 更名为 `pgsty/silo`,默认分支由 `master` 更名为 `main`。以原 MinIO 形态维持的归档构件仍位于归档的 [`minio`](https://github.com/pgsty/silo/tree/minio) 分支,以及截止 [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z) 的历次发布中。
|
||||
|
||||
## 当前发行版与主分支
|
||||
## SILO 20261018000000
|
||||
|
||||
最新已发布的 Server 仍为 [20260903](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z)。
|
||||
截至 2026-09-13,主分支已合入更新的安全、存储、Console 与共享包改动,但尚未发布新 Server。
|
||||
准确的已发布/源码边界见 [CHANGELOG.md](CHANGELOG.md) 与[组件版本矩阵](https://silo.pgsty.com/zh/compatibility/versions/),
|
||||
其中包括 SN-2026-011 修复状态与密码权限迁移要求。
|
||||
[SILO 20261018000000](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-18T00-00-00Z) 已正式发布,提供各平台归档包、Linux 软件包,以及 amd64/arm64 标准版和 distroless Docker 镜像。版本标签为 `RELEASE.2026-10-18T00-00-00Z`,包版本为 `20261018000000.0.0`。
|
||||
|
||||
本版加强 Object Lock 与 IAM 鉴权,实现跨进程和站点的持久 STS 撤销,保护多池迁移中的并发更新,并改进 LIST 恢复、分片清理与 S3 错误处理。内置 Console v2.4.2、silo-pkg v3.14.3,使用 Go 1.27.2 构建,配套管理客户端为 mcli 20261018。
|
||||
|
||||
升级前请阅读[发行说明](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-18T00-00-00Z),特别是 strict 分片列表的移除、服务端协调升级要求与实测性能成本。代码、问题报告和文档贡献者见 [CONTRIBUTORS.md](CONTRIBUTORS.md)。
|
||||
|
||||
20261025 源码正在准备 Draft Release:使用 Go 1.27.3、silo-pkg v3.15.1、
|
||||
已发布的 mcli 20261025、Console v2.5.0(`989fa7150792`)与上游 minio-go
|
||||
`ab93db7ba435`。其中包含 Console **Reset to default(恢复默认)** 所需的
|
||||
Server IAM 缓存修复,以及限定范围的写入、回滚与清理修复。
|
||||
请阅读[变更与仍存的存储限制](CHANGELOG.md#release2026-10-25t00-00-00z);
|
||||
上述已发布制品仍保留原有组件版本。
|
||||
|
||||
## 概述
|
||||
|
||||
@@ -61,7 +69,7 @@ docker run -d --name silo -p 9000:9000 -p 9001:9001 \
|
||||
-e MINIO_ROOT_USER=minioadmin \
|
||||
-e MINIO_ROOT_PASSWORD=change-me-long-password \
|
||||
-v "$PWD/data:/data" \
|
||||
docker.io/pgsty/silo:latest server /data --console-address ":9001"
|
||||
docker.io/pgsty/silo:RELEASE.2026-10-18T00-00-00Z server /data --console-address ":9001"
|
||||
```
|
||||
|
||||
<p align="center">
|
||||
@@ -92,10 +100,25 @@ docker exec silo mcli mb local/demo && docker exec silo mcli ls local
|
||||
|
||||
## 兼容性
|
||||
|
||||
S3 API、`MINIO_*` 环境变量、`minio_*` 指标、`x-minio-*` 头、`/minio/*` 路由、`github.com/minio/*` 导入路径与磁盘格式(含 `.minio.sys`)原样保留,并由 CI 兼容性门禁冻结。只有 Silo 自有交付面改名:`silo` 可执行文件、软件包、服务、Helm Chart 与容器镜像 —— 原生交付物不会安装 `minio` 二进制别名。
|
||||
Silo 保留 S3 与存储格式兼容性,包括既有 `MINIO_*` 环境变量、`minio_*` 指标、`x-minio-*` 头、`/minio/*` 路由与 `.minio.sys` 数据。CI 守卫检查选定的兼容性标识,有意的安全与行为变化在发布说明中记录。Silo 自有交付面使用 `silo` 可执行文件、软件包、服务、Helm Chart 与容器镜像;原生交付物不会安装 `minio` 服务端二进制别名。
|
||||
|
||||
正式支持和发布验收的组合为 `pgsty/silo` + `pgsty/silo-console` + `pgsty/mc` + `pgsty/silo-pkg`,对未修改的上游 MinIO/MC 尽最大努力保持兼容。Server、Console 与客户端按需保留历史模块路径,维护源码直接导入 `github.com/pgsty/silo-pkg/v3`;SDK `github.com/minio/minio-go/v7` 是明确保留的上游依赖。具体版本与 replace 以当前 [go.mod](go.mod) 为准。
|
||||
|
||||
与上游的全部分歧,以逐项核验代码的[兼容性审计](https://silo.pgsty.com/zh/compatibility/server/)形式维护。每个版本仍应视为下游升级:锁定版本,阅读[版本说明](https://silo.pgsty.com/zh/tags/silo/),并保留回滚路径。
|
||||
|
||||
### TLS 与 Go 升级
|
||||
|
||||
Server 20260916 已恢复 Go 默认密钥交换策略,20261018 在 Go 1.27.2 下保留该行为。
|
||||
`GODEBUG=tlsmlkem=0`、`tlssecpmlkem=0` 的适用范围、macOS 根证书来源变化,
|
||||
以及 OIDC discovery 的诊断方法见 [Go 1.27 TLS 与 OIDC 指南](https://silo.pgsty.com/zh/blog/design/go127-tls-oidc-discovery/)。
|
||||
|
||||
## 文档归属
|
||||
|
||||
用户文档统一维护在 [silo.pgsty.com](https://silo.pgsty.com/zh/docs/),源码位于
|
||||
[pgsty/silo.pgsty.com](https://github.com/pgsty/silo.pgsty.com)。本仓库保留的 `docs/`
|
||||
主要是继承的参考材料、示例和工具测试夹具。调查日志、AI 工作记录与临时报告放在仓库外;
|
||||
可复用的结论应整理进伴生文档站。仓库职责与维护规则见 [AGENTS.md](AGENTS.md)。
|
||||
|
||||
## 安全与贡献
|
||||
|
||||
请按照 [`SECURITY.md`](SECURITY.md) 私密报告漏洞;每项修复都会发布公开[安全公告](https://silo.pgsty.com/zh/blog/security/)。本项目不要求签署 CLA:贡献按 AGPL-3.0-or-later(inbound=outbound)接收,只需 DCO 签署(`git commit -s`),详见 [`CONTRIBUTING.md`](CONTRIBUTING.md)。
|
||||
@@ -104,7 +127,7 @@ S3 API、`MINIO_*` 环境变量、`minio_*` 指标、`x-minio-*` 头、`/minio/*
|
||||
|
||||
<!-- Generated by silo.pgsty.com/bin/contributors.py. -->
|
||||
|
||||
每位 issue 或 PR 的作者都是 SILO 社区的一员,包括尚未合并的工作。已合并的修复、被采纳的方案和有效报告优先展示,其余参考首次参与时间;金色圆环突出经过审核的显著贡献。
|
||||
每位 issue 或 PR 的作者,以及同意公开署名的安全报告者,都是 SILO 社区的一员,包括尚未合并的工作。已合并的修复、被采纳的方案和有效报告优先展示,其余参考首次参与时间;金色圆环突出经过审核的显著贡献。
|
||||
|
||||
<a href="CONTRIBUTORS.md">
|
||||
<picture>
|
||||
|
||||
+3
-3
@@ -7,7 +7,7 @@ Silo-specific fixes or release notes.
|
||||
## Supported Versions
|
||||
|
||||
Security fixes are tracked on the active development branch and summarized in
|
||||
[docs/security/advisories.md](docs/security/advisories.md). Only the current
|
||||
[the security advisory ledger](https://silo.pgsty.com/about/security-advisories/). Only the current
|
||||
Silo release line is supported unless an advisory says otherwise.
|
||||
|
||||
## Inherited Fix Evidence
|
||||
@@ -26,7 +26,7 @@ separately even when the fork preserves the original commit object and SHA.
|
||||
The inherited [service-account](https://github.com/pgsty/silo/blob/c1a49490c78e9c3ebcad86ba0662319138ace190/cmd/admin-handlers-users_test.go#L211-L212)
|
||||
and [STS](https://github.com/pgsty/silo/blob/c1a49490c78e9c3ebcad86ba0662319138ace190/cmd/sts-handlers_test.go#L45-L46)
|
||||
regression groups remain part of `go test ./cmd`; see the
|
||||
[canonical ledger](docs/security/advisories.md#inherited-upstream-advisory-baseline)
|
||||
[canonical ledger](https://silo.pgsty.com/about/security-advisories/#inherited)
|
||||
for the operator-facing record.
|
||||
|
||||
## Reporting a Vulnerability
|
||||
@@ -42,4 +42,4 @@ For vulnerabilities in this fork:
|
||||
|
||||
## Disclosure Process
|
||||
|
||||
Fork-specific fixes and user-visible upgrade notes are published in [docs/security/advisories.md](docs/security/advisories.md). The fork-specific triage and remediation process is described in [VULNERABILITY_REPORT.md](VULNERABILITY_REPORT.md).
|
||||
Fork-specific fixes and user-visible upgrade notes are published in [the security advisory ledger](https://silo.pgsty.com/about/security-advisories/). The fork-specific triage and remediation process is described in [VULNERABILITY_REPORT.md](VULNERABILITY_REPORT.md).
|
||||
|
||||
@@ -34,4 +34,4 @@ Based on the report, the Silo maintainers investigate:
|
||||
If the vulnerability exists in this fork itself, the maintainers will, when
|
||||
feasible, fix the issue or implement reasonable countermeasures such that the
|
||||
vulnerability can no longer be exploited. Fork-specific upgrade notes and
|
||||
security advisories are published in `docs/security/advisories.md`.
|
||||
security advisories are published in [the security advisory ledger](https://silo.pgsty.com/about/security-advisories/).
|
||||
|
||||
@@ -7,7 +7,7 @@ _init() {
|
||||
|
||||
## Minimum required versions for build dependencies
|
||||
GIT_VERSION="1.0"
|
||||
GO_VERSION="1.27.1"
|
||||
GO_VERSION="1.27.3"
|
||||
OSX_VERSION="10.8"
|
||||
KNAME=$(uname -s)
|
||||
ARCH=$(uname -m)
|
||||
|
||||
@@ -40,7 +40,7 @@ if [ -n "${MCLI_BIN:-}" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
release=${MCLI_RELEASE:-RELEASE.2026-09-13T00-00-00Z}
|
||||
release=${MCLI_RELEASE:-RELEASE.2026-10-25T00-00-00Z}
|
||||
version_hyphen=${release#RELEASE.}
|
||||
package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')
|
||||
if [ "${package_version}" = "${version_hyphen}" ]; then
|
||||
|
||||
@@ -132,7 +132,6 @@
|
||||
"MINIO_API_DELETE_CLEANUP_INTERVAL",
|
||||
"MINIO_API_DISABLE_ODIRECT",
|
||||
"MINIO_API_GZIP_OBJECTS",
|
||||
"MINIO_API_LEGACY_BUCKET_RESOURCE_MATCH",
|
||||
"MINIO_API_LIST_QUORUM",
|
||||
"MINIO_API_MULTIPART_LISTING",
|
||||
"MINIO_API_OBJECT_MAX_VERSIONS",
|
||||
@@ -572,6 +571,14 @@
|
||||
"minio_resource_stats",
|
||||
"minio_s3",
|
||||
"minio_stats",
|
||||
"minio_system_cpu_avg_idle",
|
||||
"minio_system_cpu_avg_iowait",
|
||||
"minio_system_cpu_load",
|
||||
"minio_system_cpu_load_perc",
|
||||
"minio_system_cpu_nice",
|
||||
"minio_system_cpu_steal",
|
||||
"minio_system_cpu_system",
|
||||
"minio_system_cpu_user",
|
||||
"minio_test"
|
||||
],
|
||||
"headers": [
|
||||
@@ -628,7 +635,6 @@
|
||||
"x-minio-origin-endpoint",
|
||||
"x-minio-prefixes-total",
|
||||
"x-minio-read-quorum",
|
||||
"x-minio-replication",
|
||||
"x-minio-replication-actual-object-size",
|
||||
"x-minio-replication-delete-status",
|
||||
"x-minio-replication-deletemarker-status",
|
||||
|
||||
@@ -115,9 +115,8 @@ func collect(repo string) (manifest, error) {
|
||||
fset := token.NewFileSet()
|
||||
|
||||
for _, rel := range files {
|
||||
// Investigation artifacts contain synthetic routes and archived configurations.
|
||||
// Migration notes and guard fixtures contain archived identifiers.
|
||||
if rel == "SILO_REBRANDING_MIGRATION.md" ||
|
||||
strings.HasPrefix(rel, "docs/investigations/") ||
|
||||
strings.HasPrefix(rel, "buildscripts/rebrand-guard/") ||
|
||||
strings.HasPrefix(rel, "buildscripts/helm-migration-guard/") {
|
||||
continue
|
||||
|
||||
@@ -113,7 +113,7 @@ helm_run template my-release "${new_chart}" \
|
||||
go run ./buildscripts/helm-migration-guard "${old_render}" "${new_render}"
|
||||
|
||||
helm_run package "${new_chart}" --destination "${output_dir}" >/dev/null
|
||||
test -s "${work_dir}/silo-7.0.2.tgz"
|
||||
test -s "${work_dir}/silo-7.0.3.tgz"
|
||||
if find "${work_dir}" -maxdepth 1 -type f -name 'minio-*.tgz' | grep -q .; then
|
||||
echo "Helm packaging emitted a legacy MinIO chart name" >&2
|
||||
exit 1
|
||||
|
||||
@@ -11,6 +11,33 @@ fail() {
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Git is already a build prerequisite. --no-index also scans untracked files
|
||||
# and exported source trees. As before, honor .gitignore only in a worktree.
|
||||
search_ignore=--no-exclude-standard
|
||||
if git rev-parse --show-toplevel >/dev/null 2>&1; then
|
||||
search_ignore=--exclude-standard
|
||||
fi
|
||||
search_errors="$(mktemp)"
|
||||
trap 'rm -f "${search_errors}"' EXIT
|
||||
|
||||
search_text() {
|
||||
local status=0
|
||||
git --no-pager grep --no-index "${search_ignore}" --no-color -nIHE "$@" 2>"${search_errors}" || status=$?
|
||||
# Git can return 1 for both no matches and a file read error. Check stderr
|
||||
# as well, before callers filter matches through their allowlists.
|
||||
if [ "${status}" -gt 1 ] || [ -s "${search_errors}" ]; then
|
||||
cat "${search_errors}" >&2
|
||||
fail "could not scan source files"
|
||||
fi
|
||||
}
|
||||
|
||||
# Keep hidden files and build output out of the repository-wide scans.
|
||||
source_excludes=(
|
||||
':(exclude,glob)**/.*'
|
||||
':(exclude,glob)**/.*/**'
|
||||
':(exclude)dist/**'
|
||||
)
|
||||
|
||||
require_file() {
|
||||
[ -f "$1" ] || fail "missing required file: $1"
|
||||
}
|
||||
@@ -152,7 +179,11 @@ if grep -Ev '^[[:space:]]*(#|$)' silo.env | grep -q '='; then
|
||||
fail "silo.env must not contain active assignments that shadow /etc/default/minio"
|
||||
fi
|
||||
|
||||
if rg -n 'pgsty/minio:' .github/workflows Dockerfile.goreleaser helm/silo; then
|
||||
delivery_hits="$(search_text 'pgsty/minio:' -- .github/workflows Dockerfile.goreleaser helm/silo \
|
||||
':(exclude,glob).github/workflows/**/.*' ':(exclude,glob).github/workflows/**/.*/**' \
|
||||
':(exclude,glob)helm/silo/**/.*' ':(exclude,glob)helm/silo/**/.*/**')"
|
||||
if [ -n "${delivery_hits}" ]; then
|
||||
printf '%s\n' "${delivery_hits}" >&2
|
||||
fail "an active delivery surface still publishes the frozen pgsty/minio image"
|
||||
fi
|
||||
|
||||
@@ -165,10 +196,10 @@ fi
|
||||
# So two rules. First, no live URL may resolve to the old repository anywhere,
|
||||
# READMEs and CONTRIBUTORS.md included.
|
||||
old_repo_pattern='pgsty/minio(\.git)?([^[:alnum:]_.-]|$)'
|
||||
stale_repo_url="$(rg -n -e "github\.com/${old_repo_pattern}" -e "hub\.docker\.com/r/${old_repo_pattern}" \
|
||||
--glob '!.git/**' --glob '!dist/**' \
|
||||
--glob '!SILO_REBRANDING_MIGRATION.md' \
|
||||
--glob '!buildscripts/rebrand-guard/compat-baseline.json' . |
|
||||
stale_repo_url="$(search_text -e "github\.com/${old_repo_pattern}" -e "hub\.docker\.com/r/${old_repo_pattern}" -- . \
|
||||
"${source_excludes[@]}" ':(exclude)SILO_REBRANDING_MIGRATION.md' \
|
||||
':(exclude)buildscripts/rebrand-guard/compat-baseline.json')"
|
||||
stale_repo_url="$(printf '%s\n' "${stale_repo_url}" |
|
||||
sed 's#^\./##' | grep -v '^buildscripts/verify-rebrand\.sh:' || true)"
|
||||
if [ -n "${stale_repo_url}" ]; then
|
||||
printf '%s\n' "${stale_repo_url}" >&2
|
||||
@@ -180,32 +211,32 @@ fi
|
||||
# legacy image, the two READMEs that document the rename and the archived
|
||||
# minio branch, and historical issue titles in CONTRIBUTORS.md.
|
||||
repo_guard_allowlist='^(buildscripts/minio-upgrade\.sh|buildscripts/verify-rebrand\.sh|buildscripts/helm-migration-guard/main\.go|README\.md|README_ZH\.md|CONTRIBUTORS\.md):'
|
||||
stale_repo="$(rg -n "${old_repo_pattern}" --glob '!.git/**' --glob '!dist/**' \
|
||||
--glob '!SILO_REBRANDING_MIGRATION.md' \
|
||||
--glob '!buildscripts/rebrand-guard/compat-baseline.json' . |
|
||||
stale_repo="$(search_text "${old_repo_pattern}" -- . "${source_excludes[@]}" \
|
||||
':(exclude)SILO_REBRANDING_MIGRATION.md' ':(exclude)buildscripts/rebrand-guard/compat-baseline.json')"
|
||||
stale_repo="$(printf '%s\n' "${stale_repo}" |
|
||||
sed 's#^\./##' | grep -Ev "${repo_guard_allowlist}" || true)"
|
||||
if [ -n "${stale_repo}" ]; then
|
||||
printf '%s\n' "${stale_repo}" >&2
|
||||
fail "a source reference still names the pre-rename pgsty/minio repository"
|
||||
fi
|
||||
|
||||
stale_branch="$(rg -n 'pgsty/silo/(blob/|tree/|raw/)?master' \
|
||||
--glob '!.git/**' --glob '!dist/**' . || true)"
|
||||
stale_branch="$(search_text 'pgsty/silo/(blob/|tree/|raw/)?master' -- . "${source_excludes[@]}")"
|
||||
if [ -n "${stale_branch}" ]; then
|
||||
printf '%s\n' "${stale_branch}" >&2
|
||||
fail "a link still targets the retired master branch; raw and Actions URLs do not follow a branch rename"
|
||||
fi
|
||||
|
||||
for workflow in .github/workflows/go.yml .github/workflows/vulncheck.yml; do
|
||||
if rg -q '^\s+- master$' "${workflow}"; then
|
||||
stale_filter="$(search_text '^[[:space:]]+- master$' -- "${workflow}")"
|
||||
if [ -n "${stale_filter}" ]; then
|
||||
fail "${workflow} still filters on master and would go silently dormant on main"
|
||||
fi
|
||||
require_text "${workflow}" " - main"
|
||||
done
|
||||
|
||||
network_hits="$(rg -n --glob '*.go' --glob '!**/*_test.go' \
|
||||
'https?://[^"`[:space:]]*(dl\.min\.io|subnet\.min\.io|api\.min\.io|slack\.min\.io|play\.min\.io)' \
|
||||
cmd internal || true)"
|
||||
network_hits="$(search_text \
|
||||
'https?://[^"`[:space:]]*(dl\.min\.io|subnet\.min\.io|api\.min\.io|slack\.min\.io|play\.min\.io)' -- \
|
||||
':(glob)cmd/**/*.go' ':(glob)internal/**/*.go' ':(exclude,glob)**/*_test.go' "${source_excludes[@]}")"
|
||||
network_hits="$(printf '%s\n' "${network_hits}" | grep -Ev '^[^:]+:[0-9]+:[[:space:]]*//' || true)"
|
||||
if [ -n "${network_hits}" ]; then
|
||||
printf '%s\n' "${network_hits}" >&2
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type abandonedCleanupCommitDisk struct {
|
||||
StorageAPI
|
||||
entered chan<- struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (d abandonedCleanupCommitDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, o RenameOptions) (RenameDataResp, error) {
|
||||
var success int64
|
||||
gate := &scopeStagedContext{
|
||||
Context: ctx, meta: filepath.Join(d.Endpoint().Path, dv, dp, fi.DataDir, "part.1"),
|
||||
entered: d.entered, release: d.release,
|
||||
tracker: healthDiskCtxValue{lastSuccess: &success},
|
||||
}
|
||||
return d.StorageAPI.RenameData(gate, sv, sp, fi, dv, dp, o)
|
||||
}
|
||||
|
||||
// A lost namespace lease can admit cleanup while a disk transaction is between
|
||||
// moving its external shards and replacing xl.meta. Those shards are live work,
|
||||
// even though the old metadata does not refer to them yet. Cleanup must wait.
|
||||
func TestMutationAbandonedCleanupWaitsForCommit(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "cleanup-during-commit"
|
||||
body := strings.Repeat("new payload", 200000)
|
||||
mutationPut(t, z, bucket, object, 0, strings.Repeat("old payload", 200000), time.Now().Add(-time.Minute))
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
entered, release := make(chan struct{}, len(disks)), make(chan struct{})
|
||||
unblock := sync.OnceFunc(func() { close(release) })
|
||||
defer unblock()
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
wrapped[i] = abandonedCleanupCommitDisk{d, entered, release}
|
||||
}
|
||||
swap(wrapped)
|
||||
committed := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := z.serverPools[0].PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{NoLock: true})
|
||||
committed <- err
|
||||
}()
|
||||
for range disks {
|
||||
select {
|
||||
case <-entered:
|
||||
case err := <-committed:
|
||||
t.Fatalf("PUT returned before shard barrier: %v", err)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("PUT did not move all external shards")
|
||||
}
|
||||
}
|
||||
t.Log("All eight new external shard directories exist; old xl.meta is still active")
|
||||
cleaned := make(chan error, len(disks))
|
||||
for _, d := range disks {
|
||||
go func(d StorageAPI) { cleaned <- d.CleanAbandonedData(context.Background(), bucket, object) }(d)
|
||||
}
|
||||
early := 0
|
||||
timer := time.NewTimer(150 * time.Millisecond)
|
||||
wait:
|
||||
for {
|
||||
select {
|
||||
case err := <-cleaned:
|
||||
early++
|
||||
t.Errorf("abandoned-data cleanup overtook unfinished commit: %v", err)
|
||||
if early == len(disks) {
|
||||
break wait
|
||||
}
|
||||
case <-timer.C:
|
||||
break wait
|
||||
}
|
||||
}
|
||||
timer.Stop()
|
||||
unblock()
|
||||
select {
|
||||
case err := <-committed:
|
||||
if err != nil {
|
||||
t.Fatalf("PUT failed: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("PUT did not finish")
|
||||
}
|
||||
for i := early; i < len(disks); i++ {
|
||||
select {
|
||||
case err := <-cleaned:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("cleanup did not finish")
|
||||
}
|
||||
}
|
||||
swap(disks)
|
||||
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("acknowledged PUT unreadable after abandoned-data cleanup: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil || string(data) != body {
|
||||
t.Fatalf("acknowledged external shards lost: bytes=%d err=%v", len(data), err)
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup can also arrive after a disk has committed but before its delayed
|
||||
// reply lets the coordinator decide to undo a failed quorum. The previous
|
||||
// generation's backup is still owned by that unfinished transaction.
|
||||
func TestMutationAbandonedCleanupPreservesUndoBackup(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "cleanup-before-undo"
|
||||
body := strings.Repeat("last acknowledged", 150000)
|
||||
base := time.Now().Add(-time.Minute)
|
||||
mutationPut(t, z, bucket, object, 0, body, base)
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
entered, release := make(chan struct{}, 4), make(chan struct{})
|
||||
unblock := sync.OnceFunc(func() { close(release) })
|
||||
defer unblock()
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
wrapped[i] = delayedRenameResult{d, i >= 4, entered, release}
|
||||
}
|
||||
swap(wrapped)
|
||||
committed := make(chan error, 1)
|
||||
go func() {
|
||||
next := strings.Repeat("failed update", 180000)
|
||||
_, err := z.serverPools[0].PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(next), int64(len(next)), "", ""), ObjectOptions{NoLock: true, MTime: base.Add(time.Second)})
|
||||
committed <- err
|
||||
}()
|
||||
for range 4 {
|
||||
select {
|
||||
case <-entered:
|
||||
case err := <-committed:
|
||||
t.Fatalf("PUT returned before reply barrier: %v", err)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("disk commit did not reach reply barrier")
|
||||
}
|
||||
}
|
||||
for _, d := range disks[:4] {
|
||||
if err := d.CleanAbandonedData(t.Context(), bucket, object); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
unblock()
|
||||
select {
|
||||
case err := <-committed:
|
||||
if err == nil {
|
||||
t.Fatal("four-of-eight commit unexpectedly reached write quorum")
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("failed PUT did not finish undo")
|
||||
}
|
||||
swap(disks)
|
||||
for i, d := range disks {
|
||||
fi, err := d.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||
if err != nil || !fi.ModTime.Equal(base) {
|
||||
t.Errorf("disk%d lost last acknowledged generation after failed PUT: mtime=%v err=%v", i, fi.ModTime, err)
|
||||
}
|
||||
}
|
||||
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("last acknowledged value unreadable after cleanup and undo: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil || string(data) != body {
|
||||
t.Fatalf("last acknowledged value lost after cleanup and undo: bytes=%d err=%v", len(data), err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/cachevalue"
|
||||
)
|
||||
|
||||
func TestAccountInfoReflectsBucketMutations(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
bed, err := prepareAdminErasureTestBed(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer bed.TearDown()
|
||||
previousCache := listBucketsCache
|
||||
defer func() { listBucketsCache = previousCache }()
|
||||
|
||||
// Hold a real pre-mutation snapshot past the assertions so the regression
|
||||
// cannot disappear merely because a slow test outlasted the cache TTL.
|
||||
freezeListing := func() {
|
||||
t.Helper()
|
||||
buckets, err := bed.objLayer.ListBuckets(ctx, BucketOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
listBucketsCache = cachevalue.NewFromFunc(time.Hour, cachevalue.Opts{}, func(context.Context) ([]BucketInfo, error) {
|
||||
return buckets, nil
|
||||
})
|
||||
if _, err := listBucketsCache.GetWithCtx(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
const bucket = "account-info-freshness"
|
||||
assertPresent := func(want bool) {
|
||||
t.Helper()
|
||||
req, err := buildAdminRequest(nil, http.MethodGet, "/accountinfo", 0, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
bed.router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("account info: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var info madmin.AccountInfo
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &info); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
for _, item := range info.Buckets {
|
||||
if item.Name == bucket {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if found != want {
|
||||
t.Errorf("account info bucket present = %v, want %v", found, want)
|
||||
}
|
||||
}
|
||||
|
||||
freezeListing()
|
||||
if err := bed.objLayer.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertPresent(true)
|
||||
freezeListing()
|
||||
if err := bed.objLayer.DeleteBucket(ctx, bucket, DeleteBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertPresent(false)
|
||||
}
|
||||
@@ -249,8 +249,10 @@ func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.R
|
||||
|
||||
// In case of LDAP/OIDC we need to set `opts.claims` to ensure
|
||||
// it is associated with the LDAP/OIDC user properly.
|
||||
// The new credential's policy comes from the request, not the caller.
|
||||
for k, v := range cred.Claims {
|
||||
if k == expClaim {
|
||||
switch k {
|
||||
case expClaim, policy.SessionPolicyName, sessionPolicyNameExtracted, iamPolicyClaimNameSA():
|
||||
continue
|
||||
}
|
||||
opts.claims[k] = v
|
||||
|
||||
@@ -0,0 +1,556 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
polplugin "github.com/minio/minio/internal/config/policy/plugin"
|
||||
"github.com/minio/mux"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
func iamAdminFixture(t *testing.T) (context.Context, *IAMSys) {
|
||||
t.Helper()
|
||||
ctx, sys, obj := prepareIAMRevisionFixture(t)
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = obj
|
||||
globalObjLayerMutex.Unlock()
|
||||
return ctx, sys
|
||||
}
|
||||
|
||||
func iamAdminUser(ctx context.Context, t *testing.T, sys *IAMSys, name, policyName string) auth.Credentials {
|
||||
t.Helper()
|
||||
cred := auth.Credentials{AccessKey: name, SecretKey: "review-user-valid-secret"}
|
||||
_, err := sys.CreateUser(ctx, name, madmin.AddOrUpdateUserReq{SecretKey: cred.SecretKey, Status: madmin.AccountEnabled})
|
||||
mustIAM(t, err)
|
||||
if policyName != "" {
|
||||
_, err = sys.PolicyDBSet(ctx, name, policyName, regUser, false)
|
||||
mustIAM(t, err)
|
||||
}
|
||||
return cred
|
||||
}
|
||||
|
||||
func iamAdminServiceAccount(ctx context.Context, t *testing.T, sys *IAMSys, parent, key, document string) auth.Credentials {
|
||||
t.Helper()
|
||||
var sp *policy.Policy
|
||||
var err error
|
||||
if document != "" {
|
||||
sp, err = policy.ParseConfig(strings.NewReader(document))
|
||||
mustIAM(t, err)
|
||||
}
|
||||
cred, _, err := sys.NewServiceAccount(ctx, parent, nil, newServiceAccountOpts{accessKey: key, secretKey: "review-service-valid-secret", sessionPolicy: sp})
|
||||
mustIAM(t, err)
|
||||
return cred
|
||||
}
|
||||
|
||||
func iamAdminRequest(t *testing.T, cred auth.Credentials, method, target string, payload any) *http.Request {
|
||||
t.Helper()
|
||||
var body []byte
|
||||
if payload != nil {
|
||||
raw, err := json.Marshal(payload)
|
||||
mustIAM(t, err)
|
||||
body, err = madmin.EncryptData(cred.SecretKey, raw)
|
||||
mustIAM(t, err)
|
||||
}
|
||||
headers := map[string]string{}
|
||||
if cred.IsTemp() && !cred.IsServiceAccount() {
|
||||
headers["X-Amz-Security-Token"] = cred.SessionToken
|
||||
}
|
||||
req, err := newTestSignedRequestV4(method, "http://localhost"+target, int64(len(body)), bytes.NewReader(body), cred.AccessKey, cred.SecretKey, headers)
|
||||
mustIAM(t, err)
|
||||
mustIAM(t, req.ParseForm())
|
||||
return req
|
||||
}
|
||||
|
||||
func iamAdminCall(t *testing.T, cred auth.Credentials, method, target string, payload any, vars map[string]string, handler http.HandlerFunc) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
req := iamAdminRequest(t, cred, method, target, payload)
|
||||
req = mux.SetURLVars(req, vars)
|
||||
rec := httptest.NewRecorder()
|
||||
req = req.WithContext(newContext(req, rec, "IAMAdminAuthorization"))
|
||||
handler(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
func assertIAMAdminStatus(t *testing.T, operation string, rec *httptest.ResponseRecorder, want int) {
|
||||
t.Helper()
|
||||
if rec.Code != want {
|
||||
t.Errorf("%s status=%d want=%d body=%s", operation, rec.Code, want, rec.Body.String())
|
||||
return
|
||||
}
|
||||
if want == http.StatusForbidden || want == http.StatusNotFound {
|
||||
var response APIErrorResponse
|
||||
mustIAM(t, json.Unmarshal(rec.Body.Bytes(), &response))
|
||||
wantCode := errorCodes.ToAPIErr(ErrAccessDenied).Code
|
||||
if want == http.StatusNotFound {
|
||||
wantCode = errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound).Code
|
||||
}
|
||||
if response.Code != wantCode {
|
||||
t.Errorf("%s error code=%s want=%s", operation, response.Code, wantCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func iamAdminPlugin(t *testing.T, allow bool) {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]bool{"result": allow})
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
u, err := xnet.ParseHTTPURL(server.URL)
|
||||
mustIAM(t, err)
|
||||
previous := newGlobalAuthZPluginFn()
|
||||
setGlobalAuthZPlugin(polplugin.New(polplugin.Args{URL: u, CloseRespFn: func(r io.ReadCloser) { r.Close() }}))
|
||||
t.Cleanup(func() { setGlobalAuthZPlugin(previous) })
|
||||
}
|
||||
|
||||
func TestIAMAdminRootEmptyPolicy(t *testing.T) {
|
||||
for _, doc := range []string{"", "null", "{}", `{"Statement":null}`} {
|
||||
t.Run("policy="+doc, func(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
caller := iamAdminServiceAccount(ctx, t, sys, globalActiveCred.AccessKey, "review-root-empty", doc)
|
||||
sibling := iamAdminServiceAccount(ctx, t, sys, globalActiveCred.AccessKey, "review-root-sibling", "")
|
||||
req := iamAdminRequest(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil)
|
||||
checked, owner, code := checkKeyValid(req, caller.AccessKey)
|
||||
if code != ErrNone {
|
||||
t.Fatalf("checkKeyValid: %v", code)
|
||||
}
|
||||
active, _ := isAllowedBySessionPolicyForServiceAccount(policy.Args{Claims: checked.Claims})
|
||||
t.Logf("owner=%v parsed_active_session_policy=%v", owner, active)
|
||||
api := adminAPIHandlers{}
|
||||
assertIAMAdminStatus(t, "create", iamAdminCall(t, caller, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{AccessKey: "review-root-new", SecretKey: "review-service-valid-secret"}, nil, api.AddServiceAccount), 200)
|
||||
assertIAMAdminStatus(t, "update", iamAdminCall(t, caller, "POST", "/minio/admin/v3/update-service-account?accessKey="+sibling.AccessKey, madmin.UpdateServiceAccountReq{NewName: "review"}, map[string]string{"accessKey": sibling.AccessKey}, api.UpdateServiceAccount), 204)
|
||||
assertIAMAdminStatus(t, "delete", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+sibling.AccessKey, nil, map[string]string{"accessKey": sibling.AccessKey}, api.DeleteServiceAccount), 204)
|
||||
assertIAMAdminStatus(t, "revoke", iamAdminCall(t, caller, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true", nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 204)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminEmptyPolicyDelegation(t *testing.T) {
|
||||
for _, parentType := range []string{"root", "user"} {
|
||||
for _, document := range []string{"null", "{}", `{"Statement":null}`} {
|
||||
t.Run(parentType+"/"+document, func(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
parent := globalActiveCred
|
||||
if parentType == "user" {
|
||||
parent = iamAdminUser(ctx, t, sys, "empty-policy-parent", "readwrite")
|
||||
}
|
||||
sp, err := policy.ParseConfig(strings.NewReader(document))
|
||||
mustIAM(t, err)
|
||||
caller, _, err := sys.NewServiceAccount(ctx, parent.AccessKey, nil, newServiceAccountOpts{
|
||||
accessKey: "empty-policy-caller", secretKey: "empty-policy-caller-secret", sessionPolicy: sp,
|
||||
claims: map[string]any{"identity-attribute": "preserved"},
|
||||
})
|
||||
mustIAM(t, err)
|
||||
const limitedPolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:ListServiceAccounts"]}]}`
|
||||
for _, tc := range []struct {
|
||||
policy string
|
||||
listStatus int
|
||||
allowObject bool
|
||||
}{
|
||||
{"", http.StatusOK, true},
|
||||
{"null", http.StatusOK, true},
|
||||
{"{}", http.StatusOK, true},
|
||||
{`{"Statement":null}`, http.StatusOK, true},
|
||||
{limitedPolicy, http.StatusOK, false},
|
||||
{`{"Version":"2012-10-17"}`, http.StatusForbidden, false},
|
||||
{`{"Statement":[]}`, http.StatusForbidden, false},
|
||||
} {
|
||||
expires := UTCNow().Add(time.Hour)
|
||||
req := madmin.AddServiceAccountReq{Expiration: &expires}
|
||||
if tc.policy != "" {
|
||||
req.Policy = json.RawMessage(tc.policy)
|
||||
}
|
||||
api := adminAPIHandlers{}
|
||||
rec := iamAdminCall(t, caller, "PUT", "/minio/admin/v3/add-service-account", req, nil, api.AddServiceAccount)
|
||||
assertIAMAdminStatus(t, "create expiring child", rec, http.StatusOK)
|
||||
data, err := madmin.DecryptData(caller.SecretKey, rec.Body)
|
||||
mustIAM(t, err)
|
||||
var response madmin.AddServiceAccountResp
|
||||
mustIAM(t, json.Unmarshal(data, &response))
|
||||
child, ok := sys.store.GetUser(response.Credentials.AccessKey)
|
||||
if !ok {
|
||||
t.Fatal("new service account missing")
|
||||
}
|
||||
// Exercise the issued key through authentication and authorization.
|
||||
assertIAMAdminStatus(t, "use expiring child policy="+tc.policy, iamAdminCall(t, child.Credentials, "GET", "/minio/admin/v3/list-service-accounts", nil, nil, api.ListServiceAccounts), tc.listStatus)
|
||||
signed := iamAdminRequest(t, child.Credentials, "GET", "/example/object", nil)
|
||||
checked, owner, code := checkKeyValid(signed, child.Credentials.AccessKey)
|
||||
if code != ErrNone {
|
||||
t.Fatalf("child authentication: %v", code)
|
||||
}
|
||||
if checked.Claims["identity-attribute"] != "preserved" {
|
||||
t.Fatal("creation lost the caller's identity context")
|
||||
}
|
||||
allowed := sys.IsAllowed(policy.Args{AccountName: checked.AccessKey, Claims: checked.Claims, IsOwner: owner, Action: policy.GetObjectAction, BucketName: "example", ObjectName: "object"})
|
||||
if allowed != tc.allowObject {
|
||||
t.Errorf("GetObject allowed=%v with requested policy=%s", allowed, tc.policy)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminCorruptServiceCleanup(t *testing.T) {
|
||||
for _, status := range []string{auth.AccountOn, auth.AccountOff, "expired"} {
|
||||
t.Run(status, func(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
caller := iamAdminUser(ctx, t, sys, "review-admin", "consoleAdmin")
|
||||
parent := iamAdminUser(ctx, t, sys, "review-parent", "readwrite")
|
||||
svc := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "rv-corrupt", "")
|
||||
if status == auth.AccountOff {
|
||||
_, err := sys.UpdateServiceAccount(ctx, svc.AccessKey, updateServiceAccountOpts{status: status})
|
||||
mustIAM(t, err)
|
||||
}
|
||||
cache := sys.store.lock()
|
||||
u := cache.iamUsersMap[svc.AccessKey]
|
||||
u.Credentials.SessionToken = "broken-session-token"
|
||||
if status == "expired" {
|
||||
u.Credentials.Expiration = UTCNow().Add(-time.Minute)
|
||||
}
|
||||
cache.iamUsersMap[svc.AccessKey] = u
|
||||
sys.store.unlock()
|
||||
_, _, err := sys.GetServiceAccount(ctx, svc.AccessKey)
|
||||
if err == nil {
|
||||
t.Fatal("fault injection failed")
|
||||
}
|
||||
raw, exists := sys.store.GetUser(svc.AccessKey)
|
||||
_, loginOK := sys.GetUser(ctx, svc.AccessKey)
|
||||
t.Logf("metadata_exists=%v parent_known=%v GetUser_ok=%v", exists, raw.Credentials.ParentUser == parent.AccessKey, loginOK)
|
||||
api := adminAPIHandlers{}
|
||||
outsider := iamAdminUser(ctx, t, sys, "cleanup-outsider", "readwrite")
|
||||
assertIAMAdminStatus(t, "unauthorized cleanup", iamAdminCall(t, outsider, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.DeleteServiceAccount), 404)
|
||||
assertIAMAdminStatus(t, "admin cleanup", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.DeleteServiceAccount), 204)
|
||||
if _, ok := sys.store.GetUser(svc.AccessKey); ok {
|
||||
t.Fatal("deleted service account remains in the IAM cache")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminPolicylessSelf(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
caller := iamAdminUser(ctx, t, sys, "review-policyless", "")
|
||||
svc := iamAdminServiceAccount(ctx, t, sys, caller.AccessKey, "rv-policyless-child", "")
|
||||
other := iamAdminUser(ctx, t, sys, "bulk-other", "readwrite")
|
||||
_ = iamAdminServiceAccount(ctx, t, sys, other.AccessKey, "bulk-other-child", "")
|
||||
api := adminAPIHandlers{}
|
||||
assertIAMAdminStatus(t, "list", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil, nil, api.ListServiceAccounts), 200)
|
||||
rec := iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-access-keys-bulk?listType=all", nil, nil, api.ListAccessKeysBulk)
|
||||
assertIAMAdminStatus(t, "list bulk self", rec, 200)
|
||||
data, err := madmin.DecryptData(caller.SecretKey, rec.Body)
|
||||
mustIAM(t, err)
|
||||
var listed map[string]madmin.ListAccessKeysResp
|
||||
mustIAM(t, json.Unmarshal(data, &listed))
|
||||
entry, ok := listed[caller.AccessKey]
|
||||
if len(listed) != 1 || !ok || len(entry.ServiceAccounts) != 1 || entry.ServiceAccounts[0].AccessKey != svc.AccessKey || len(entry.STSKeys) != 0 {
|
||||
t.Fatalf("policyless bulk self returned unexpected identities: %+v", listed)
|
||||
}
|
||||
assertIAMAdminStatus(t, "info service", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.InfoServiceAccount), 200)
|
||||
assertIAMAdminStatus(t, "info access key", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-access-key?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.InfoAccessKey), 200)
|
||||
assertIAMAdminStatus(t, "delete", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.DeleteServiceAccount), 204)
|
||||
assertIAMAdminStatus(t, "revoke", iamAdminCall(t, caller, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true", nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 204)
|
||||
assertIAMAdminStatus(t, "list another user", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts?user="+globalActiveCred.AccessKey, nil, nil, api.ListServiceAccounts), 403)
|
||||
assertIAMAdminStatus(t, "revoke another user", iamAdminCall(t, caller, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true&user="+globalActiveCred.AccessKey, nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 403)
|
||||
}
|
||||
|
||||
func TestIAMAdminExpiringInheritedPolicy(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
parent := iamAdminUser(ctx, t, sys, "review-legacy-parent", "consoleAdmin")
|
||||
document := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:CreateServiceAccount","admin:UpdateServiceAccount"]}]}`
|
||||
// Model a credential persisted by the old handler: it copied its caller's
|
||||
// policy claims but set the newly created service's type to inherited.
|
||||
expires := UTCNow().Add(time.Hour)
|
||||
caller, _, err := sys.NewServiceAccount(ctx, parent.AccessKey, nil, newServiceAccountOpts{
|
||||
accessKey: "rv-exp-inherited", secretKey: "review-service-valid-secret", expiration: &expires,
|
||||
claims: map[string]any{policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(document))},
|
||||
})
|
||||
mustIAM(t, err)
|
||||
req := iamAdminRequest(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil)
|
||||
checked, owner, code := checkKeyValid(req, caller.AccessKey)
|
||||
if code != ErrNone {
|
||||
t.Fatalf("checkKeyValid: %v", code)
|
||||
}
|
||||
args := policy.Args{AccountName: caller.AccessKey, Claims: checked.Claims, IsOwner: owner, Action: policy.CreateServiceAccountAdminAction, DenyOnly: true}
|
||||
if !sys.IsAllowed(args) {
|
||||
t.Fatal("fixture cannot create")
|
||||
}
|
||||
args.Action = policy.CreateUserAdminAction
|
||||
args.DenyOnly = false
|
||||
if sys.IsAllowed(args) {
|
||||
t.Fatal("fixture not constrained")
|
||||
}
|
||||
t.Logf("service=%v temp=%v inherited=%v; create allowed, create-user denied", checked.IsServiceAccount(), checked.IsTemp(), checked.Claims[iamPolicyClaimNameSA()] == inheritedPolicyType)
|
||||
api := adminAPIHandlers{}
|
||||
rec := iamAdminCall(t, caller, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{AccessKey: "review-escaped", SecretKey: "review-service-valid-secret"}, nil, api.AddServiceAccount)
|
||||
if rec.Code == 200 {
|
||||
child, ok := sys.GetUser(ctx, "review-escaped")
|
||||
if !ok {
|
||||
t.Fatal("new key missing")
|
||||
}
|
||||
childReq := iamAdminRequest(t, child.Credentials, "GET", "/minio/admin/v3/list-service-accounts", nil)
|
||||
childChecked, childOwner, childCode := checkKeyValid(childReq, child.Credentials.AccessKey)
|
||||
if childCode != ErrNone {
|
||||
t.Fatalf("child auth=%v", childCode)
|
||||
}
|
||||
allowed := sys.IsAllowed(policy.Args{AccountName: childChecked.AccessKey, Claims: childChecked.Claims, IsOwner: childOwner, Action: policy.CreateUserAdminAction})
|
||||
t.Errorf("constrained inherited service created a new key; new key CreateUser allowed=%v", allowed)
|
||||
} else {
|
||||
assertIAMAdminStatus(t, "reject derived creation", rec, 403)
|
||||
}
|
||||
updated := iamAdminCall(t, caller, "POST", "/minio/admin/v3/update-service-account?accessKey="+caller.AccessKey, madmin.UpdateServiceAccountReq{NewName: "metadata-only"}, map[string]string{"accessKey": caller.AccessKey}, api.UpdateServiceAccount)
|
||||
if updated.Code == 204 {
|
||||
after, ok := sys.GetUser(ctx, caller.AccessKey)
|
||||
if !ok {
|
||||
t.Fatal("updated key missing")
|
||||
}
|
||||
afterReq := iamAdminRequest(t, after.Credentials, "GET", "/minio/admin/v3/list-service-accounts", nil)
|
||||
afterChecked, afterOwner, afterCode := checkKeyValid(afterReq, caller.AccessKey)
|
||||
if afterCode != ErrNone {
|
||||
t.Fatalf("updated auth=%v", afterCode)
|
||||
}
|
||||
allowed := sys.IsAllowed(policy.Args{AccountName: caller.AccessKey, Claims: afterChecked.Claims, IsOwner: afterOwner, Action: policy.CreateUserAdminAction})
|
||||
t.Errorf("constrained inherited service updated itself; CreateUser allowed after update=%v", allowed)
|
||||
} else {
|
||||
assertIAMAdminStatus(t, "reject inherited self update", updated, 403)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminSessionPolicyCannotDelegate(t *testing.T) {
|
||||
for _, temp := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "service", true: "sts"}[temp], func(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
parent := iamAdminUser(ctx, t, sys, "rv-parent", "consoleAdmin")
|
||||
doc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:CreateServiceAccount","admin:UpdateServiceAccount"]}]}`
|
||||
var caller auth.Credentials
|
||||
if !temp {
|
||||
caller = iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "review-restricted", doc)
|
||||
} else {
|
||||
var err error
|
||||
caller, err = auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(), policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(doc))}, globalActiveCred.SecretKey)
|
||||
mustIAM(t, err)
|
||||
caller.ParentUser = parent.AccessKey
|
||||
_, err = sys.SetTempUser(ctx, caller.AccessKey, caller, "")
|
||||
mustIAM(t, err)
|
||||
}
|
||||
sibling := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "rv-sibling", "")
|
||||
api := adminAPIHandlers{}
|
||||
assertIAMAdminStatus(t, "restrict create", iamAdminCall(t, caller, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{AccessKey: "review-bad-new", SecretKey: "review-service-valid-secret"}, nil, api.AddServiceAccount), 403)
|
||||
assertIAMAdminStatus(t, "restrict LDAP create", iamAdminCall(t, caller, "PUT", "/minio/admin/v3/idp/ldap/add-service-account", madmin.AddServiceAccountReq{AccessKey: "review-bad-new", SecretKey: "review-service-valid-secret"}, nil, api.AddServiceAccountLDAP), 403)
|
||||
assertIAMAdminStatus(t, "restrict update", iamAdminCall(t, caller, "POST", "/minio/admin/v3/update-service-account?accessKey="+sibling.AccessKey, madmin.UpdateServiceAccountReq{NewSecretKey: "review-known-new-secret"}, map[string]string{"accessKey": sibling.AccessKey}, api.UpdateServiceAccount), 403)
|
||||
if _, ok := sys.store.GetUser("review-bad-new"); ok {
|
||||
t.Fatal("denied creation persisted a credential")
|
||||
}
|
||||
u, ok := sys.store.GetUser(sibling.AccessKey)
|
||||
if !ok || u.Credentials.SecretKey != sibling.SecretKey {
|
||||
t.Fatal("denied update changed the target credential")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminSelfDenials(t *testing.T) {
|
||||
for _, mode := range []string{"parent_deny", "group_deny", "session_implicit_deny", "session_explicit_deny", "plugin_deny"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
parent := iamAdminUser(ctx, t, sys, "rv-self-parent", "readwrite")
|
||||
target := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "rv-self-target", "")
|
||||
caller := parent
|
||||
doc := `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":["admin:ListServiceAccounts","admin:RemoveServiceAccount"]}]}`
|
||||
switch mode {
|
||||
case "parent_deny", "group_deny":
|
||||
p, err := policy.ParseConfig(strings.NewReader(doc))
|
||||
mustIAM(t, err)
|
||||
_, err = sys.SetPolicy(ctx, "review-self-deny", *p)
|
||||
mustIAM(t, err)
|
||||
if mode == "group_deny" {
|
||||
_, err = sys.AddUsersToGroup(ctx, "self-deny-group", []string{parent.AccessKey})
|
||||
mustIAM(t, err)
|
||||
_, err = sys.PolicyDBSet(ctx, "self-deny-group", "review-self-deny", regUser, true)
|
||||
} else {
|
||||
_, err = sys.PolicyDBSet(ctx, parent.AccessKey, "readwrite,review-self-deny", regUser, false)
|
||||
}
|
||||
mustIAM(t, err)
|
||||
case "session_implicit_deny":
|
||||
caller = iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "rv-self-caller", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::example/*"]}]}`)
|
||||
case "session_explicit_deny":
|
||||
caller = iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "rv-self-caller", doc)
|
||||
case "plugin_deny":
|
||||
iamAdminPlugin(t, false)
|
||||
}
|
||||
api := adminAPIHandlers{}
|
||||
assertIAMAdminStatus(t, "list", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil, nil, api.ListServiceAccounts), 403)
|
||||
assertIAMAdminStatus(t, "list bulk self", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-access-keys-bulk?listType=all", nil, nil, api.ListAccessKeysBulk), 403)
|
||||
assertIAMAdminStatus(t, "info service", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.InfoServiceAccount), 403)
|
||||
assertIAMAdminStatus(t, "info access key", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-access-key?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.InfoAccessKey), 403)
|
||||
assertIAMAdminStatus(t, "delete", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.DeleteServiceAccount), 403)
|
||||
assertIAMAdminStatus(t, "revoke", iamAdminCall(t, caller, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true", nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 403)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminNonRootCannotManageRoot(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
caller := iamAdminUser(ctx, t, sys, "rv-console-admin", "consoleAdmin")
|
||||
target := iamAdminServiceAccount(ctx, t, sys, globalActiveCred.AccessKey, "rv-root-target", "")
|
||||
api := adminAPIHandlers{}
|
||||
assertIAMAdminStatus(t, "create root", iamAdminCall(t, caller, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{AccessKey: "rv-new-root", SecretKey: "review-service-valid-secret", TargetUser: globalActiveCred.AccessKey}, nil, api.AddServiceAccount), 403)
|
||||
assertIAMAdminStatus(t, "update root", iamAdminCall(t, caller, "POST", "/minio/admin/v3/update-service-account?accessKey="+target.AccessKey, madmin.UpdateServiceAccountReq{NewSecretKey: "review-known-new-secret"}, map[string]string{"accessKey": target.AccessKey}, api.UpdateServiceAccount), 403)
|
||||
assertIAMAdminStatus(t, "delete root", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.DeleteServiceAccount), 404)
|
||||
assertIAMAdminStatus(t, "revoke root", iamAdminCall(t, caller, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true&user="+globalActiveCred.AccessKey, nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 403)
|
||||
// Restricted root-derived credentials and the site replicator are not root
|
||||
// administrators, even when their effective policy allows these actions.
|
||||
limited := iamAdminServiceAccount(ctx, t, sys, globalActiveCred.AccessKey, "limited-root", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:*"]}]}`)
|
||||
assertIAMAdminStatus(t, "limited root delete", iamAdminCall(t, limited, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.DeleteServiceAccount), 404)
|
||||
assertIAMAdminStatus(t, "limited root revoke", iamAdminCall(t, limited, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true", nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 403)
|
||||
replicator, _, err := sys.NewServiceAccount(ctx, globalActiveCred.AccessKey, nil, newServiceAccountOpts{accessKey: siteReplicatorSvcAcc, secretKey: "replicator-valid-secret", allowSiteReplicatorAccount: true})
|
||||
mustIAM(t, err)
|
||||
assertIAMAdminStatus(t, "replicator create root", iamAdminCall(t, replicator, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{}, nil, api.AddServiceAccount), 403)
|
||||
|
||||
// An external plugin remains authoritative, including for root targets.
|
||||
iamAdminPlugin(t, true)
|
||||
assertIAMAdminStatus(t, "plugin update root", iamAdminCall(t, limited, "POST", "/minio/admin/v3/update-service-account?accessKey="+target.AccessKey, madmin.UpdateServiceAccountReq{NewName: "plugin-authorized"}, map[string]string{"accessKey": target.AccessKey}, api.UpdateServiceAccount), 204)
|
||||
assertIAMAdminStatus(t, "plugin create root", iamAdminCall(t, limited, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{}, nil, api.AddServiceAccount), 200)
|
||||
}
|
||||
|
||||
func TestIAMAdminDeleteCorruptServiceReplicates(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
caller := iamAdminUser(ctx, t, sys, "cleanup-admin", "consoleAdmin")
|
||||
parent := iamAdminUser(ctx, t, sys, "cleanup-parent", "readwrite")
|
||||
target := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "cleanup-target", "")
|
||||
cache := sys.store.lock()
|
||||
u := cache.iamUsersMap[target.AccessKey]
|
||||
u.Credentials.SessionToken = "broken-session-token"
|
||||
u.Credentials.Status = auth.AccountOff
|
||||
cache.iamUsersMap[target.AccessKey] = u
|
||||
sys.store.unlock()
|
||||
|
||||
var deletions atomic.Int32
|
||||
peer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/minio/health/live" {
|
||||
return
|
||||
}
|
||||
var batch iamRevisionBatch
|
||||
if err := json.NewDecoder(r.Body).Decode(&batch); err != nil {
|
||||
t.Error(err)
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if len(batch.Items) != 1 || batch.Items[0].SvcAccChange == nil || batch.Items[0].SvcAccChange.Delete == nil || batch.Items[0].SvcAccChange.Delete.AccessKey != target.AccessKey {
|
||||
t.Error("replication did not carry the service account deletion")
|
||||
} else {
|
||||
deletions.Add(1)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: "peer", Instance: "peer-boot", Digest: "deleted"}})
|
||||
}))
|
||||
t.Cleanup(peer.Close)
|
||||
globalSiteReplicatorCred.RLock()
|
||||
previousSecret := globalSiteReplicatorCred.secretKey
|
||||
globalSiteReplicatorCred.RUnlock()
|
||||
globalSiteReplicatorCred.Set(caller.SecretKey)
|
||||
t.Cleanup(func() { globalSiteReplicatorCred.Set(previousSecret) })
|
||||
globalSiteReplicationSys.Lock()
|
||||
globalSiteReplicationSys.enabled = true
|
||||
globalSiteReplicationSys.state = srState{ServiceAccountAccessKey: caller.AccessKey, Peers: map[string]madmin.PeerInfo{
|
||||
globalDeploymentID(): {DeploymentID: globalDeploymentID(), Name: "local"},
|
||||
"peer": {DeploymentID: "peer", Name: "peer", Endpoint: peer.URL},
|
||||
}}
|
||||
globalSiteReplicationSys.Unlock()
|
||||
t.Cleanup(func() {
|
||||
globalSiteReplicationSys.Lock()
|
||||
globalSiteReplicationSys.enabled = false
|
||||
globalSiteReplicationSys.Unlock()
|
||||
})
|
||||
api := adminAPIHandlers{}
|
||||
assertIAMAdminStatus(t, "replicated cleanup", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.DeleteServiceAccount), 204)
|
||||
if deletions.Load() != 1 {
|
||||
t.Fatalf("expected one immediate deletion notification, got %d", deletions.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminListSelfAliases(t *testing.T) {
|
||||
for _, kind := range []string{"service", "sts"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
parent := iamAdminUser(ctx, t, sys, "alias-parent", "readwrite")
|
||||
target := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "alias-target", "")
|
||||
serviceCaller := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "alias-caller", "")
|
||||
caller := serviceCaller
|
||||
other := iamAdminUser(ctx, t, sys, "alias-other", "readwrite")
|
||||
_ = iamAdminServiceAccount(ctx, t, sys, other.AccessKey, "alias-other-child", "")
|
||||
if kind == "sts" {
|
||||
var err error
|
||||
caller, err = auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix()}, globalActiveCred.SecretKey)
|
||||
mustIAM(t, err)
|
||||
caller.ParentUser = parent.AccessKey
|
||||
_, err = sys.SetTempUser(ctx, caller.AccessKey, caller, "")
|
||||
mustIAM(t, err)
|
||||
}
|
||||
api := adminAPIHandlers{}
|
||||
for _, alias := range []string{"", caller.AccessKey, parent.AccessKey} {
|
||||
rec := iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts?user="+alias, nil, nil, api.ListServiceAccounts)
|
||||
assertIAMAdminStatus(t, "list self alias", rec, 200)
|
||||
data, err := madmin.DecryptData(caller.SecretKey, rec.Body)
|
||||
mustIAM(t, err)
|
||||
var response madmin.ListServiceAccountsResp
|
||||
mustIAM(t, json.Unmarshal(data, &response))
|
||||
found := false
|
||||
for _, account := range response.Accounts {
|
||||
found = found || account.AccessKey == target.AccessKey
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("self alias %q did not list the parent's service accounts", alias)
|
||||
}
|
||||
|
||||
query := url.Values{"listType": {madmin.AccessKeyListAll}}
|
||||
if alias != "" {
|
||||
query.Set("users", alias)
|
||||
}
|
||||
bulk := iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-access-keys-bulk?"+query.Encode(), nil, nil, api.ListAccessKeysBulk)
|
||||
assertIAMAdminStatus(t, "list bulk self alias", bulk, 200)
|
||||
data, err = madmin.DecryptData(caller.SecretKey, bulk.Body)
|
||||
mustIAM(t, err)
|
||||
var listed map[string]madmin.ListAccessKeysResp
|
||||
mustIAM(t, json.Unmarshal(data, &listed))
|
||||
entry, ok := listed[parent.AccessKey]
|
||||
if len(listed) != 1 || !ok || len(entry.ServiceAccounts) != 2 {
|
||||
t.Fatalf("bulk self alias %q returned unexpected identities: %+v", alias, listed)
|
||||
}
|
||||
want := map[string]bool{target.AccessKey: true, serviceCaller.AccessKey: true}
|
||||
for _, account := range entry.ServiceAccounts {
|
||||
if !want[account.AccessKey] {
|
||||
t.Fatalf("bulk self alias %q returned unexpected or duplicate service account %s", alias, account.AccessKey)
|
||||
}
|
||||
delete(want, account.AccessKey)
|
||||
}
|
||||
if kind == "sts" {
|
||||
if len(entry.STSKeys) != 1 || entry.STSKeys[0].AccessKey != caller.AccessKey {
|
||||
t.Fatalf("bulk self alias %q returned unexpected STS accounts: %+v", alias, entry.STSKeys)
|
||||
}
|
||||
} else if len(entry.STSKeys) != 0 {
|
||||
t.Fatalf("service caller returned unexpected STS accounts: %+v", entry.STSKeys)
|
||||
}
|
||||
if bytes.Contains(data, []byte(serviceCaller.SecretKey)) || bytes.Contains(data, []byte(`"secretKey"`)) || bytes.Contains(data, []byte(`"sessionToken"`)) {
|
||||
t.Fatal("bulk listing returned credential secrets")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+146
-86
@@ -714,8 +714,10 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
// In case of LDAP/OIDC we need to set `opts.claims` to ensure
|
||||
// it is associated with the LDAP/OIDC user properly.
|
||||
// The new credential's policy comes from the request, not the caller.
|
||||
for k, v := range cred.Claims {
|
||||
if k == expClaim {
|
||||
switch k {
|
||||
case expClaim, policy.SessionPolicyName, sessionPolicyNameExtracted, iamPolicyClaimNameSA():
|
||||
continue
|
||||
}
|
||||
opts.claims[k] = v
|
||||
@@ -855,10 +857,9 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
|
||||
// Permission checks:
|
||||
//
|
||||
// 1. Any type of account (i.e. access keys (previously/still called service
|
||||
// accounts), STS accounts, internal IDP accounts, etc) with the
|
||||
// policy.UpdateServiceAccountAdminAction permission can update any service
|
||||
// account.
|
||||
// 1. Updating requires policy.UpdateServiceAccountAdminAction. Under built-in
|
||||
// IAM, session-limited callers cannot update credentials, and only unrestricted
|
||||
// root credentials can manage root's service accounts.
|
||||
//
|
||||
// 2. We would like to let a user update their own access keys, however it
|
||||
// is currently blocked pending a re-design. Users are still able to delete
|
||||
@@ -874,6 +875,11 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
if newGlobalAuthZPluginFn() == nil && (hasActiveSessionPolicy(cred) ||
|
||||
(svcAccount.ParentUser == globalActiveCred.AccessKey && !canManageRootCredentials(cred))) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var sp *policy.Policy
|
||||
if len(updateReq.NewPolicy) > 0 {
|
||||
@@ -951,23 +957,21 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
requestUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
requestUser = cred.ParentUser
|
||||
}
|
||||
if !isAllowedServiceAccountAction(cred, policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: requestUser == svcAccount.ParentUser,
|
||||
}) {
|
||||
requestUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
requestUser = cred.ParentUser
|
||||
}
|
||||
|
||||
if requestUser != svcAccount.ParentUser {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// if session policy is nil or empty, then it is implied policy
|
||||
@@ -1039,29 +1043,27 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
|
||||
return
|
||||
}
|
||||
|
||||
var targetAccount string
|
||||
|
||||
// If listing is requested for a specific user (who is not the request
|
||||
// sender), check that the user has permissions.
|
||||
selfUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
selfUser = cred.ParentUser
|
||||
}
|
||||
user := r.Form.Get("user")
|
||||
if user != "" && user != cred.AccessKey {
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
targetAccount = user
|
||||
} else {
|
||||
targetAccount = cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
targetAccount = cred.ParentUser
|
||||
}
|
||||
selfOnly := user == "" || user == cred.AccessKey || user == selfUser
|
||||
if !isAllowedServiceAccountAction(cred, policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: selfOnly,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
targetAccount := user
|
||||
if selfOnly {
|
||||
targetAccount = selfUser
|
||||
}
|
||||
|
||||
serviceAccounts, err := globalIAMSys.ListServiceAccounts(ctx, targetAccount)
|
||||
@@ -1110,7 +1112,7 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil || globalNotificationSys == nil {
|
||||
if objectAPI == nil || globalNotificationSys == nil || !globalIAMSys.Initialized() {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -1131,35 +1133,41 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidArgument), r.URL)
|
||||
return
|
||||
}
|
||||
// We do not care if service account is readable or not at this point,
|
||||
// since this is a delete call we shall allow it to be deleted if possible.
|
||||
svcAccount, _, err := globalIAMSys.GetServiceAccount(ctx, serviceAccount)
|
||||
if errors.Is(err, errNoSuchServiceAccount) {
|
||||
// Deletion needs the stored identity, not a valid token or session policy.
|
||||
// Keep disabled, expired and damaged service accounts removable, and retain
|
||||
// their parent for authorization and the site replication hook below.
|
||||
u, ok := globalIAMSys.store.GetUser(serviceAccount)
|
||||
if !ok || !u.Credentials.IsServiceAccount() {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound), r.URL)
|
||||
return
|
||||
}
|
||||
svcAccount := u.Credentials
|
||||
|
||||
adminPrivilege := globalIAMSys.IsAllowed(policy.Args{
|
||||
parentUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
parentUser = cred.ParentUser
|
||||
}
|
||||
isSelf := parentUser == svcAccount.ParentUser
|
||||
if !isAllowedServiceAccountAction(cred, policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.RemoveServiceAccountAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
})
|
||||
|
||||
if !adminPrivilege {
|
||||
parentUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
parentUser = cred.ParentUser
|
||||
}
|
||||
if svcAccount.ParentUser != "" && parentUser != svcAccount.ParentUser {
|
||||
// The service account belongs to another user but return not
|
||||
// found error to mitigate brute force attacks. or the
|
||||
// serviceAccount doesn't exist.
|
||||
DenyOnly: isSelf,
|
||||
}) {
|
||||
if !isSelf {
|
||||
// Hide the existence of another user's service account.
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound), r.URL)
|
||||
return
|
||||
}
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
if newGlobalAuthZPluginFn() == nil && svcAccount.ParentUser == globalActiveCred.AccessKey && !canManageRootCredentials(cred) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err := globalIAMSys.DeleteServiceAccount(ctx, serviceAccount, true); err != nil {
|
||||
@@ -1229,7 +1237,7 @@ func (a adminAPIHandlers) ListAccessKeysBulk(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
}
|
||||
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
if !isAllowedServiceAccountAction(cred, policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
@@ -1242,12 +1250,12 @@ func (a adminAPIHandlers) ListAccessKeysBulk(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
if selfOnly && len(users) == 0 {
|
||||
if selfOnly {
|
||||
selfUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
selfUser = cred.ParentUser
|
||||
}
|
||||
users = append(users, selfUser)
|
||||
users = []string{selfUser}
|
||||
}
|
||||
|
||||
var checkedUserList []string
|
||||
@@ -1439,7 +1447,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
return buckets[i].Name < buckets[j].Name
|
||||
})
|
||||
} else {
|
||||
buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{Cached: true})
|
||||
// Console refreshes this inventory immediately after bucket mutations.
|
||||
// A stale cached list can otherwise hide a creation or retain a deletion
|
||||
// indefinitely in the client's snapshot.
|
||||
buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -2046,26 +2057,30 @@ func (a adminAPIHandlers) RevokeTokens(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
adminPrivilege := globalIAMSys.IsAllowed(policy.Args{
|
||||
parentUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
parentUser = cred.ParentUser
|
||||
}
|
||||
isSelfTarget := isTokenSelfRevoke || user == parentUser
|
||||
if !isAllowedServiceAccountAction(cred, policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.RemoveServiceAccountAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
})
|
||||
|
||||
if !adminPrivilege || isTokenSelfRevoke {
|
||||
parentUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
parentUser = cred.ParentUser
|
||||
}
|
||||
if !isTokenSelfRevoke && user != parentUser {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
DenyOnly: isSelfTarget,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
if isSelfTarget {
|
||||
user = parentUser
|
||||
}
|
||||
if newGlobalAuthZPluginFn() == nil && user == globalActiveCred.AccessKey && !canManageRootCredentials(cred) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Infer token revoke type from the request if requestor is STS.
|
||||
if isTokenSelfRevoke && tokenRevokeType == "" && !fullRevoke {
|
||||
@@ -2112,29 +2127,22 @@ func (a adminAPIHandlers) InfoAccessKey(w http.ResponseWriter, r *http.Request)
|
||||
u, ok := globalIAMSys.GetUser(ctx, accessKey)
|
||||
targetCred := u.Credentials
|
||||
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
requestUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
requestUser = cred.ParentUser
|
||||
}
|
||||
isSelf := ok && targetCred.ParentUser != "" && requestUser == targetCred.ParentUser
|
||||
if !isAllowedServiceAccountAction(cred, policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: isSelf,
|
||||
}) {
|
||||
// If requested user does not exist and requestor is not allowed to list service accounts, return access denied.
|
||||
if !ok {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
requestUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
requestUser = cred.ParentUser
|
||||
}
|
||||
|
||||
if requestUser != targetCred.ParentUser {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if !ok {
|
||||
@@ -2908,6 +2916,44 @@ func addExpirationToCondValues(exp *time.Time, condValues map[string][]string) e
|
||||
return nil
|
||||
}
|
||||
|
||||
// hasActiveSessionPolicy follows the same credential dispatch as IAMSys.IsAllowed.
|
||||
// Expiring service accounts also satisfy IsTemp and are evaluated as STS there.
|
||||
// Callers use this only after authorization under built-in IAM.
|
||||
func hasActiveSessionPolicy(cred auth.Credentials) bool {
|
||||
args := policy.Args{Claims: cred.Claims}
|
||||
if cred.IsTemp() {
|
||||
hasSP, _ := isAllowedBySessionPolicy(args)
|
||||
return hasSP
|
||||
}
|
||||
if cred.IsServiceAccount() {
|
||||
if cred.Claims[iamPolicyClaimNameSA()] == inheritedPolicyType {
|
||||
return false
|
||||
}
|
||||
hasSP, _ := isAllowedBySessionPolicyForServiceAccount(args)
|
||||
return hasSP
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// canManageRootCredentials is an additional guard after built-in IAM authorization.
|
||||
// The signature's owner flag is insufficient: historical empty session policies
|
||||
// clear it even though the service account still inherits root's permissions.
|
||||
func canManageRootCredentials(cred auth.Credentials) bool {
|
||||
return cred.AccessKey == globalActiveCred.AccessKey ||
|
||||
(cred.ParentUser == globalActiveCred.AccessKey && cred.AccessKey != siteReplicatorSvcAcc && !hasActiveSessionPolicy(cred))
|
||||
}
|
||||
|
||||
// isAllowedServiceAccountAction preserves regular users' self inspection and
|
||||
// cleanup even without a policy attachment, while enforcing explicit denies.
|
||||
// Derived credentials and external authorization plugins use full IAM evaluation.
|
||||
func isAllowedServiceAccountAction(cred auth.Credentials, args policy.Args) bool {
|
||||
if args.DenyOnly && !args.IsOwner && !cred.IsTemp() && !cred.IsServiceAccount() && newGlobalAuthZPluginFn() == nil {
|
||||
policies, err := globalIAMSys.PolicyDBGet(args.AccountName, args.Groups...)
|
||||
return err == nil && globalIAMSys.GetCombinedPolicy(policies...).IsAllowed(args)
|
||||
}
|
||||
return globalIAMSys.IsAllowed(args)
|
||||
}
|
||||
|
||||
func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.Credentials, newServiceAccountOpts, madmin.AddServiceAccountReq, string, APIError) {
|
||||
ctx := r.Context()
|
||||
|
||||
@@ -2996,6 +3042,14 @@ func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.
|
||||
}) {
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAccessDenied)
|
||||
}
|
||||
effectiveTarget := targetUser
|
||||
if (targetUser == cred.AccessKey || targetUser == cred.ParentUser) && cred.ParentUser != "" {
|
||||
effectiveTarget = cred.ParentUser
|
||||
}
|
||||
if newGlobalAuthZPluginFn() == nil && (hasActiveSessionPolicy(cred) ||
|
||||
(effectiveTarget == globalActiveCred.AccessKey && !canManageRootCredentials(cred))) {
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAccessDenied)
|
||||
}
|
||||
|
||||
var sp *policy.Policy
|
||||
if len(createReq.Policy) > 0 {
|
||||
@@ -3003,6 +3057,12 @@ func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.
|
||||
if err != nil {
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err)
|
||||
}
|
||||
// Empty service policies mean inheritance, including for expiring keys.
|
||||
// Normalize the replication request as well as the local credential.
|
||||
if sp.Version == "" && sp.Statements == nil && sp.ID == "" {
|
||||
sp = nil
|
||||
createReq.Policy = nil
|
||||
}
|
||||
}
|
||||
|
||||
opts.sessionPolicy = sp
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
//
|
||||
// This file is part of SILO Object Storage.
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/hash"
|
||||
)
|
||||
|
||||
func TestChecksumMismatchAPIMessage(t *testing.T) {
|
||||
for _, algorithm := range []string{"CRC32", "CRC32C", "SHA1", "SHA256", "CRC64NVME", ""} {
|
||||
t.Run(algorithm, func(t *testing.T) {
|
||||
mismatch := hash.ChecksumMismatch{Algorithm: algorithm, Want: "expected", Got: "actual"}
|
||||
want := errorCodes.ToAPIErr(ErrContentChecksumMismatch).Description
|
||||
if algorithm != "" {
|
||||
want = fmt.Sprintf("The %s you specified did not match the calculated checksum.", algorithm)
|
||||
}
|
||||
for _, err := range []error{mismatch, fmt.Errorf("upload: %w", mismatch)} {
|
||||
got := toAPIError(t.Context(), err)
|
||||
if got.HTTPStatusCode != http.StatusBadRequest || got.Code != "XAmzContentChecksumMismatch" || got.Description != want {
|
||||
t.Fatalf("toAPIError(%v) = %+v, want 400 XAmzContentChecksumMismatch: %s", err, got, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+10
-4
@@ -450,7 +450,7 @@ const (
|
||||
ErrAdminNoSecretKey
|
||||
|
||||
ErrIAMNotInitialized
|
||||
ErrMultipartListingLegacy
|
||||
ErrMultipartListingLegacy // Reserved: legacy uploads no longer block listing.
|
||||
ErrMultipartListingIdentity
|
||||
ErrSlowDown
|
||||
|
||||
@@ -550,7 +550,7 @@ var errorCodes = errorCodeMap{
|
||||
},
|
||||
ErrBadDigest: {
|
||||
Code: "BadDigest",
|
||||
Description: "The Content-Md5 you specified did not match what we received.",
|
||||
Description: "The Content-MD5 you specified did not match what we received.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrEntityTooSmall: {
|
||||
@@ -2191,8 +2191,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
err = unwrapAll(err)
|
||||
|
||||
switch err {
|
||||
case errMultipartListingLegacy:
|
||||
apiErr = ErrMultipartListingLegacy
|
||||
case errMultipartListingIdentity:
|
||||
apiErr = ErrMultipartListingIdentity
|
||||
case errCompleteMultipartChecksumMismatch, errCompleteMultipartChecksumTypeMismatch:
|
||||
@@ -2505,6 +2503,14 @@ func toAPIError(ctx context.Context, err error) APIError {
|
||||
case errors.Is(err, errMissingPartChecksum):
|
||||
apiErr.Description = strings.TrimPrefix(err.Error(), errMissingPartChecksum.Error()+": ")
|
||||
}
|
||||
// PutObject/UploadPart checksum mismatches name the algorithm in the
|
||||
// message, matching S3 (e.g. "The CRC32 you specified did not match the
|
||||
// calculated checksum."). Falls through to the generic table description
|
||||
// when the algorithm is unknown.
|
||||
var checksumMismatch hash.ChecksumMismatch
|
||||
if errors.As(err, &checksumMismatch) && checksumMismatch.Algorithm != "" {
|
||||
apiErr.Description = fmt.Sprintf("The %s you specified did not match the calculated checksum.", checksumMismatch.Algorithm)
|
||||
}
|
||||
switch apiErr.Code {
|
||||
case "NotImplemented":
|
||||
apiErr = APIError{
|
||||
|
||||
@@ -28,13 +28,10 @@ import (
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
"github.com/minio/minio/internal/etag"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
@@ -733,48 +730,6 @@ func setAuthMiddleware(h http.Handler) http.Handler {
|
||||
})
|
||||
}
|
||||
|
||||
func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate time.Time, retMode objectlock.RetMode, byPassSet bool, r *http.Request, cred auth.Credentials, owner bool) (s3Err APIErrorCode) {
|
||||
var retSet bool
|
||||
if cred.AccessKey == "" {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
conditions := getConditionValues(r, "", cred)
|
||||
conditions["object-lock-mode"] = []string{string(retMode)}
|
||||
conditions["object-lock-retain-until-date"] = []string{retDate.UTC().Format(time.RFC3339)}
|
||||
if retDays > 0 {
|
||||
conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(retDays)}
|
||||
}
|
||||
if retMode == objectlock.RetGovernance && byPassSet {
|
||||
byPassSet = globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.BypassGovernanceRetentionAction,
|
||||
BucketName: bucketName,
|
||||
ObjectName: objectName,
|
||||
ConditionValues: conditions,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
})
|
||||
}
|
||||
if globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.PutObjectRetentionAction,
|
||||
BucketName: bucketName,
|
||||
ConditionValues: conditions,
|
||||
ObjectName: objectName,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
retSet = true
|
||||
}
|
||||
if byPassSet || retSet {
|
||||
return ErrNone
|
||||
}
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
// isPutActionAllowed - check if PUT operation is allowed on the resource, this
|
||||
// call verifies bucket policies and IAM policies, supports multi user
|
||||
// checks etc.
|
||||
|
||||
@@ -583,7 +583,11 @@ func toObjectInfo(bucket, object string, objInfo minio.ObjectInfo) ObjectInfo {
|
||||
func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLayer, remoteClnt *minio.Client, entries []ObjectInfo, prefix string) error {
|
||||
input := make(chan minio.SnowballObject, 1)
|
||||
opts := minio.SnowballOptions{
|
||||
Opts: minio.PutObjectOptions{},
|
||||
Opts: minio.PutObjectOptions{
|
||||
Internal: minio.AdvancedPutOptions{
|
||||
ReplicationRequest: true,
|
||||
},
|
||||
},
|
||||
InMemory: *r.Source.Snowball.InMemory,
|
||||
Compress: *r.Source.Snowball.Compress,
|
||||
SkipErrs: *r.Source.Snowball.SkipErrs,
|
||||
|
||||
@@ -511,14 +511,13 @@ func testSiteReplicationStatusDetectsCorsTimestampMismatch(obj ObjectLayer, _ st
|
||||
}))
|
||||
defer remote.Close()
|
||||
|
||||
serviceCred, err := auth.CreateCredentials("cors-status-service", "cors-status-service-secret-key")
|
||||
serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, credentials.AccessKey, nil, newServiceAccountOpts{
|
||||
accessKey: "cors-status-service",
|
||||
secretKey: "cors-status-service-secret-key",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serviceCred.ParentUser = credentials.AccessKey
|
||||
if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
|
||||
|
||||
globalSiteReplicationSys.Lock()
|
||||
@@ -585,14 +584,13 @@ func testSiteReplicationStatusCountsCorsPerSite(obj ObjectLayer, _ string, bucke
|
||||
}))
|
||||
defer remote.Close()
|
||||
|
||||
serviceCred, err := auth.CreateCredentials("cors-count-service", "cors-count-service-secret-key")
|
||||
serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, credentials.AccessKey, nil, newServiceAccountOpts{
|
||||
accessKey: "cors-count-service",
|
||||
secretKey: "cors-count-service-secret-key",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serviceCred.ParentUser = credentials.AccessKey
|
||||
if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
|
||||
|
||||
globalSiteReplicationSys.Lock()
|
||||
@@ -1190,14 +1188,13 @@ func testCorsReplicationDispatchStatusHealReload(obj ObjectLayer, _ string, buck
|
||||
}))
|
||||
defer remote.Close()
|
||||
|
||||
serviceCred, err := auth.CreateCredentials("cors-integration-svc", "cors-integration-service-secret")
|
||||
serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, credentials.AccessKey, nil, newServiceAccountOpts{
|
||||
accessKey: "cors-integration-svc",
|
||||
secretKey: "cors-integration-service-secret",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serviceCred.ParentUser = credentials.AccessKey
|
||||
if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
|
||||
|
||||
globalSiteReplicationSys.Lock()
|
||||
|
||||
@@ -1273,6 +1273,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
|
||||
return
|
||||
}
|
||||
if isDirObject(object) {
|
||||
if _, _, _, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, objectAPI.GetObjectInfo, ErrNone, ErrNone, false); s3Err != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
opts.WantChecksum = checksum
|
||||
|
||||
fanOutOpts := fanOutOptions{Checksum: checksum}
|
||||
|
||||
+19
-7
@@ -632,13 +632,25 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
|
||||
}
|
||||
|
||||
// Delete remote object from warm-tier
|
||||
err := deleteObjectFromRemoteTier(ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier)
|
||||
if err == nil {
|
||||
// Skip adding free version since we successfully deleted the
|
||||
// remote object
|
||||
opts.SkipFreeVersion = true
|
||||
} else {
|
||||
transitionLogIf(ctx, err)
|
||||
//
|
||||
// Unless the expiry only puts a delete marker on top of the version, as
|
||||
// for an object that is not transitioned: with versioning enabled, or
|
||||
// suspended and the version is not the null version, the version stays
|
||||
// as a noncurrent version and must remain readable. Its remote object is
|
||||
// removed later, through the free version added when the version itself
|
||||
// is deleted.
|
||||
opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name)
|
||||
keepRemote := opts.VersionID == "" &&
|
||||
(opts.Versioned || (opts.VersionSuspended && oi.VersionID != "" && oi.VersionID != nullVersionID))
|
||||
if !keepRemote {
|
||||
err := deleteObjectFromRemoteTier(ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier)
|
||||
if err == nil {
|
||||
// Skip adding free version since we successfully deleted the
|
||||
// remote object
|
||||
opts.SkipFreeVersion = true
|
||||
} else {
|
||||
transitionLogIf(ctx, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Now, delete object from hot-tier namespace
|
||||
|
||||
@@ -19,11 +19,17 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
@@ -245,3 +251,180 @@ func TestValidateTransitionTier(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tierExpiryTestBackend is a remote tier held in memory.
|
||||
type tierExpiryTestBackend struct {
|
||||
mu sync.Mutex
|
||||
objects map[string][]byte
|
||||
removed []string
|
||||
}
|
||||
|
||||
func (b *tierExpiryTestBackend) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) {
|
||||
return b.PutWithMeta(ctx, object, r, length, nil)
|
||||
}
|
||||
|
||||
func (b *tierExpiryTestBackend) PutWithMeta(_ context.Context, object string, r io.Reader, _ int64, _ map[string]string) (remoteVersionID, error) {
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.objects[object] = data
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (b *tierExpiryTestBackend) Get(_ context.Context, object string, _ remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
data, ok := b.objects[object]
|
||||
if !ok {
|
||||
return nil, ObjectNotFound{Bucket: "tier", Object: object}
|
||||
}
|
||||
if opts.length > 0 {
|
||||
data = data[opts.startOffset : opts.startOffset+opts.length]
|
||||
}
|
||||
return io.NopCloser(bytes.NewReader(data)), nil
|
||||
}
|
||||
|
||||
func (b *tierExpiryTestBackend) Remove(_ context.Context, object string, _ remoteVersionID) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
delete(b.objects, object)
|
||||
b.removed = append(b.removed, object)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *tierExpiryTestBackend) InUse(context.Context) (bool, error) { return false, nil }
|
||||
|
||||
// The expiry of a transitioned latest version that only puts a delete marker
|
||||
// on top of it must keep its remote object, so the version stays readable as
|
||||
// a noncurrent version. The remote object goes when the version is deleted.
|
||||
func TestExpireTransitionedObjectKeepsNoncurrentData(t *testing.T) {
|
||||
z, _ := migrationTestPools(t)
|
||||
ctx := t.Context()
|
||||
tier := &tierExpiryTestBackend{objects: map[string][]byte{}}
|
||||
prevTier := globalTierConfigMgr
|
||||
globalTierConfigMgr = NewTierConfigMgr()
|
||||
globalTierConfigMgr.drivercache["WARM"] = tier
|
||||
t.Cleanup(func() { globalTierConfigMgr = prevTier })
|
||||
|
||||
const expire = `<LifecycleConfiguration><Rule><ID>exp</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter>` +
|
||||
`<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration></Rule></LifecycleConfiguration>`
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
versioning string // "", Enabled or Suspended
|
||||
nullLatest bool // the latest version is the null version
|
||||
keep bool // the latest version survives as noncurrent
|
||||
want []string // versions after the expiry, newest first
|
||||
}{
|
||||
{name: "versioned", versioning: "Enabled", keep: true, want: []string{"DM", "v2", "v1"}},
|
||||
{name: "suspended", versioning: "Suspended", keep: true, want: []string{"DM", "v2", "v1"}},
|
||||
{name: "suspended-null", versioning: "Suspended", nullLatest: true, want: []string{"DM", "v1"}},
|
||||
{name: "unversioned", nullLatest: true, want: nil},
|
||||
}
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
bucket := "tier-expiry-" + tc.name
|
||||
if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{VersioningEnabled: tc.versioning != ""}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta, err := globalBucketMetadataSys.Get(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tc.versioning == "Suspended" {
|
||||
meta.VersioningConfigXML = []byte("<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>")
|
||||
}
|
||||
meta.LifecycleConfigXML = []byte(expire)
|
||||
if err := meta.Save(ctx, z); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
globalBucketMetadataSys.Set(bucket, meta)
|
||||
|
||||
// tiered writes a version whose data is in the remote tier.
|
||||
labels := map[string]string{}
|
||||
tiered := func(label string, opts ObjectOptions) ObjectInfo {
|
||||
t.Helper()
|
||||
oi := putConsistencyObject(t, z, bucket, "obj", 0, "obj "+label, opts)
|
||||
vid := migrationTestVersion(oi)
|
||||
infos, errs := readAllFileInfo(ctx, z.serverPools[0].getHashedSet("obj").getDisks(), "", bucket, "obj", vid, false, false)
|
||||
if errs[0] != nil {
|
||||
t.Fatal(errs[0])
|
||||
}
|
||||
fi := infos[0]
|
||||
fi.TransitionStatus, fi.TransitionTier, fi.TransitionedObjName = lifecycle.TransitionComplete, "WARM", bucket+"/"+label
|
||||
if _, err := tier.Put(ctx, fi.TransitionedObjName, strings.NewReader("obj "+label), -1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := z.serverPools[0].DecomTieredObject(ctx, bucket, "obj", fi, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
labels[vid] = label
|
||||
return oi
|
||||
}
|
||||
var latest ObjectInfo
|
||||
if tc.versioning != "" {
|
||||
tiered("v1", ObjectOptions{Versioned: true})
|
||||
}
|
||||
if tc.nullLatest {
|
||||
latest = tiered("v2", ObjectOptions{})
|
||||
} else {
|
||||
latest = tiered("v2", ObjectOptions{Versioned: true})
|
||||
}
|
||||
|
||||
// Expire the latest version as the expiry worker does for the scanner.
|
||||
getOpts := ObjectOptions{Versioned: tc.versioning == "Enabled", VersionSuspended: tc.versioning == "Suspended"}
|
||||
oi, err := z.GetObjectInfo(ctx, bucket, "obj", getOpts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lc, err := globalLifecycleSys.Get(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
evt := evalActionFromLifecycle(ctx, *lc, objectlock.Retention{}, nil, oi)
|
||||
if evt.Action != lifecycle.DeleteAction || oi.TransitionedObject.Status != lifecycle.TransitionComplete {
|
||||
t.Fatalf("fixture: action %v, transition status %q", evt.Action, oi.TransitionedObject.Status)
|
||||
}
|
||||
if err := expireTransitionedObject(ctx, z, &oi, evt, lcEventSrc_Scanner); err != nil {
|
||||
t.Errorf("expiry: %v", err)
|
||||
}
|
||||
|
||||
var got []string
|
||||
for _, set := range z.serverPools[0].sets {
|
||||
err := set.listObjectsToDecommission(ctx, decomBucketInfo{Name: bucket}, func(entry metaCacheEntry) {
|
||||
fivs, err := entry.fileInfoVersions(bucket)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
for _, fi := range fivs.Versions {
|
||||
if fi.Deleted {
|
||||
got = append(got, "DM")
|
||||
} else {
|
||||
got = append(got, labels[migrationTestVersion(fi.ToObjectInfo(bucket, "obj", true))])
|
||||
}
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(got, tc.want) {
|
||||
t.Errorf("versions after expiry: got %v, want %v", got, tc.want)
|
||||
}
|
||||
remote := bucket + "/v2"
|
||||
if removed := slices.Contains(tier.removed, remote); removed == tc.keep {
|
||||
t.Errorf("remote object of the expired version removed: %v, want %v", removed, !tc.keep)
|
||||
}
|
||||
if tc.keep {
|
||||
body := migrationTestBody(t, z.GetObjectNInfo, bucket, "obj", migrationTestVersion(latest))
|
||||
if body != "obj v2" {
|
||||
t.Errorf("v2 reads %q after the expiry, want %q", body, "obj v2")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+122
-61
@@ -20,8 +20,8 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -164,76 +164,129 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
|
||||
return nil
|
||||
}
|
||||
|
||||
// enforceRetentionBypassForPut enforces whether an existing object under governance can be overwritten
|
||||
// with governance bypass headers set in the request.
|
||||
// Objects under site wide WORM cannot be overwritten.
|
||||
// For objects in "Governance" mode, overwrite is allowed if a) object retention date is past OR
|
||||
// governance bypass headers are set and user has governance bypass permissions.
|
||||
// Objects in compliance mode can be overwritten only if retention date is being extended. No mode change is permitted.
|
||||
// isRetentionActionAllowed evaluates action for a PutObjectRetention request
|
||||
// with the object-lock condition keys derived from the requested retention.
|
||||
// The keys come from the request, not from the stored version, so the
|
||||
// handler can decide s3:PutObjectRetention before reading any object state.
|
||||
func isRetentionActionAllowed(action policy.Action, bucket, object string, objRetention *objectlock.ObjectRetention, r *http.Request, cred auth.Credentials, owner bool) bool {
|
||||
if cred.AccessKey == "" {
|
||||
return false
|
||||
}
|
||||
t, err := objectlock.UTCNowNTP()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
conditions := getConditionValues(r, "", cred)
|
||||
conditions["object-lock-mode"] = []string{string(objRetention.Mode)}
|
||||
conditions["object-lock-retain-until-date"] = []string{objRetention.RetainUntilDate.UTC().Format(time.RFC3339)}
|
||||
// Relative days from now, for "object-lock-remaining-retention-days".
|
||||
// Clearing retention (<Retention/>) has a zero RetainUntilDate, which must
|
||||
// evaluate as 0 remaining days rather than a large positive math.Abs delta
|
||||
// from year 0001.
|
||||
days := 0
|
||||
if !objRetention.RetainUntilDate.IsZero() && objRetention.RetainUntilDate.After(t) {
|
||||
// Time.Sub saturates after about 292 years. Count seconds instead so
|
||||
// maximum-retention policies also apply to distant RFC3339 dates.
|
||||
seconds := objRetention.RetainUntilDate.Unix() - t.Unix()
|
||||
days = int(seconds / 86400)
|
||||
if seconds%86400 != 0 || objRetention.RetainUntilDate.Nanosecond() > t.Nanosecond() {
|
||||
days++
|
||||
}
|
||||
}
|
||||
conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(days)}
|
||||
return globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: action,
|
||||
BucketName: bucket,
|
||||
ConditionValues: conditions,
|
||||
ObjectName: object,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
})
|
||||
}
|
||||
|
||||
// enforceRetentionBypassForPut enforces the stored retention of oi against a
|
||||
// PutObjectRetention request. The caller must already hold
|
||||
// s3:PutObjectRetention; this only decides changes that depend on the stored
|
||||
// state. Expired or absent retention may be changed freely. Unexpired
|
||||
// GOVERNANCE retention may be shortened, removed or have its mode changed only
|
||||
// with the bypass header and s3:BypassGovernanceRetention. Unexpired
|
||||
// COMPLIANCE retention may only be extended.
|
||||
func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi ObjectInfo, objRetention *objectlock.ObjectRetention, cred auth.Credentials, owner bool) error {
|
||||
byPassSet := objectlock.IsObjectLockGovernanceBypassSet(r.Header)
|
||||
ret := objectlock.GetObjectRetentionMeta(oi.UserDefined)
|
||||
if !ret.Mode.Valid() {
|
||||
return nil
|
||||
}
|
||||
|
||||
t, err := objectlock.UTCNowNTP()
|
||||
if err != nil {
|
||||
internalLogIf(ctx, err, logger.WarningKind)
|
||||
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
||||
}
|
||||
if ret.RetainUntilDate.Before(t) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Pass in relative days from current time, to additionally
|
||||
// to verify "object-lock-remaining-retention-days" policy if any.
|
||||
days := int(math.Ceil(math.Abs(objRetention.RetainUntilDate.Sub(t).Hours()) / 24))
|
||||
|
||||
ret := objectlock.GetObjectRetentionMeta(oi.UserDefined)
|
||||
if ret.Mode.Valid() {
|
||||
// Retention has expired you may change whatever you like.
|
||||
if ret.RetainUntilDate.Before(t) {
|
||||
apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
|
||||
days, objRetention.RetainUntilDate.Time,
|
||||
objRetention.Mode, byPassSet, r, cred,
|
||||
owner)
|
||||
if apiErr == ErrAccessDenied {
|
||||
return errAuthentication
|
||||
}
|
||||
return nil
|
||||
weakens := objRetention.Mode != ret.Mode || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time)
|
||||
if !weakens {
|
||||
return nil
|
||||
}
|
||||
switch ret.Mode {
|
||||
case objectlock.RetGovernance:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html#object-lock-bypass-governance
|
||||
if !objectlock.IsObjectLockGovernanceBypassSet(r.Header) {
|
||||
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
||||
}
|
||||
|
||||
switch ret.Mode {
|
||||
case objectlock.RetGovernance:
|
||||
govPerm := isPutRetentionAllowed(oi.Bucket, oi.Name, days,
|
||||
objRetention.RetainUntilDate.Time, objRetention.Mode,
|
||||
byPassSet, r, cred, owner)
|
||||
// Governance mode retention period cannot be shortened, if x-amz-bypass-governance is not set.
|
||||
if !byPassSet {
|
||||
if objRetention.Mode != objectlock.RetGovernance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) {
|
||||
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
||||
}
|
||||
}
|
||||
if govPerm == ErrAccessDenied {
|
||||
return errAuthentication
|
||||
}
|
||||
return nil
|
||||
case objectlock.RetCompliance:
|
||||
// Compliance retention mode cannot be changed or shortened.
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
|
||||
if objRetention.Mode != objectlock.RetCompliance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) {
|
||||
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
||||
}
|
||||
apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
|
||||
days, objRetention.RetainUntilDate.Time, objRetention.Mode,
|
||||
false, r, cred, owner)
|
||||
if apiErr == ErrAccessDenied {
|
||||
return errAuthentication
|
||||
}
|
||||
return nil
|
||||
if !isRetentionActionAllowed(policy.BypassGovernanceRetentionAction, oi.Bucket, oi.Name, objRetention, r, cred, owner) {
|
||||
return errAuthentication
|
||||
}
|
||||
return nil
|
||||
} // No pre-existing retention metadata present.
|
||||
default:
|
||||
// Compliance retention mode cannot be changed or shortened.
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
|
||||
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
||||
}
|
||||
}
|
||||
|
||||
apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
|
||||
days, objRetention.RetainUntilDate.Time,
|
||||
objRetention.Mode, byPassSet, r, cred, owner)
|
||||
if apiErr == ErrAccessDenied {
|
||||
return errAuthentication
|
||||
// checkObjectLockedForOverwrite re-checks under the object write lock whether an
|
||||
// in-place write to the null version (such as a directory object) would
|
||||
// overwrite a locked version in an Object Lock bucket.
|
||||
func checkObjectLockedForOverwrite(ctx context.Context, bucket, object string, getObjectInfoFn GetObjectInfoFn) error {
|
||||
if isMinioMetaBucketName(bucket) {
|
||||
return nil
|
||||
}
|
||||
retentionCfg, err := globalBucketObjectLockSys.Get(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !retentionCfg.LockEnabled {
|
||||
return nil
|
||||
}
|
||||
objInfo, err := getObjectInfoFn(ctx, bucket, object, ObjectOptions{
|
||||
VersionID: nullVersionID,
|
||||
NoLock: true,
|
||||
NoAuditLog: true,
|
||||
})
|
||||
if err != nil {
|
||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) || errors.As(err, &MethodNotAllowed{}) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined).Status == objectlock.LegalHoldOn {
|
||||
return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: nullVersionID}
|
||||
}
|
||||
r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
|
||||
if r.Mode == objectlock.RetCompliance || r.Mode == objectlock.RetGovernance {
|
||||
t, err := objectlock.UTCNowNTP()
|
||||
if err != nil {
|
||||
internalLogIf(ctx, err, logger.WarningKind)
|
||||
return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: nullVersionID}
|
||||
}
|
||||
if r.RetainUntilDate.After(t) {
|
||||
return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: nullVersionID}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -274,16 +327,24 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
||||
if err != nil {
|
||||
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
|
||||
}
|
||||
if isDirObject(object) && opts.VersionID == "" {
|
||||
opts.VersionID = nullVersionID
|
||||
}
|
||||
|
||||
if opts.VersionID != "" && !replicaTrusted {
|
||||
if objInfo, err := getObjectInfoFn(ctx, bucket, object, opts); err == nil {
|
||||
objInfo, err := getObjectInfoFn(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) && !errors.As(err, &MethodNotAllowed{}) {
|
||||
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
|
||||
}
|
||||
} else {
|
||||
r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
|
||||
t, err := objectlock.UTCNowNTP()
|
||||
if err != nil {
|
||||
internalLogIf(ctx, err, logger.WarningKind)
|
||||
return mode, retainDate, legalHold, ErrObjectLocked
|
||||
}
|
||||
if r.Mode == objectlock.RetCompliance && r.RetainUntilDate.After(t) {
|
||||
if (r.Mode == objectlock.RetCompliance || r.Mode == objectlock.RetGovernance) && r.RetainUntilDate.After(t) {
|
||||
return mode, retainDate, legalHold, ErrObjectLocked
|
||||
}
|
||||
mode = r.Mode
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
// Copyright (c) 2026 Denis Babchuk
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"maps"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
// checksumTestObject builds the xl.meta checksum of an object the way
|
||||
// PutObject and CompleteMultipartUpload store it.
|
||||
func checksumTestObject(t *testing.T, alg hash.ChecksumType, full bool, parts ...[]byte) FileInfo {
|
||||
t.Helper()
|
||||
fi := FileInfo{Volume: "bucket", Name: "object"}
|
||||
if len(parts) == 1 {
|
||||
fi.Size = int64(len(parts[0]))
|
||||
fi.Checksum = hash.NewChecksumFromData(alg, parts[0]).AppendTo(nil, nil)
|
||||
return fi
|
||||
}
|
||||
var combined, all []byte
|
||||
for i, p := range parts {
|
||||
combined = append(combined, hash.NewChecksumFromData(alg, p).Raw...)
|
||||
all = append(all, p...)
|
||||
fi.Parts = append(fi.Parts, ObjectPartInfo{Number: i + 1, Size: int64(len(p)), ActualSize: int64(len(p))})
|
||||
fi.Size += int64(len(p))
|
||||
}
|
||||
typ := alg | hash.ChecksumMultipart | hash.ChecksumIncludesMultipart
|
||||
if full {
|
||||
typ |= hash.ChecksumFullObject
|
||||
}
|
||||
cs := hash.NewChecksumFromData(typ, combined)
|
||||
if typ.FullObjectRequested() {
|
||||
// Full object CRC, as merged from the parts on completion.
|
||||
whole := hash.NewChecksumFromData(alg, all)
|
||||
cs = &hash.Checksum{Type: typ, Raw: whole.Raw, Encoded: whole.Encoded}
|
||||
}
|
||||
fi.Checksum = cs.AppendTo(nil, combined)
|
||||
return fi
|
||||
}
|
||||
|
||||
// TestGetCRCMetaChecksumType checks the checksum headers that replication
|
||||
// sends for an object: a FULL_OBJECT checksum type only for multipart full
|
||||
// object CRCs, where the target needs it to create a matching upload.
|
||||
func TestGetCRCMetaChecksumType(t *testing.T) {
|
||||
p1, p2 := []byte("first part"), []byte("second part")
|
||||
enc := func(alg hash.ChecksumType, b []byte) string { return hash.NewChecksumFromData(alg, b).Encoded }
|
||||
whole := append(append([]byte{}, p1...), p2...)
|
||||
composite := func(alg hash.ChecksumType) string {
|
||||
var combined []byte
|
||||
combined = append(combined, hash.NewChecksumFromData(alg, p1).Raw...)
|
||||
combined = append(combined, hash.NewChecksumFromData(alg, p2).Raw...)
|
||||
return enc(alg, combined) + "-2"
|
||||
}
|
||||
full := map[string]string{xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject}
|
||||
tests := []struct {
|
||||
name string
|
||||
fi FileInfo
|
||||
alg hash.ChecksumType
|
||||
isMP bool
|
||||
value string
|
||||
extra map[string]string
|
||||
}{
|
||||
{"single-part SHA256", checksumTestObject(t, hash.ChecksumSHA256, false, p1), hash.ChecksumSHA256, false, enc(hash.ChecksumSHA256, p1), nil},
|
||||
{"single-part CRC32", checksumTestObject(t, hash.ChecksumCRC32, false, p1), hash.ChecksumCRC32, false, enc(hash.ChecksumCRC32, p1), nil},
|
||||
{"single-part CRC64NVME", checksumTestObject(t, hash.ChecksumCRC64NVME, false, p1), hash.ChecksumCRC64NVME, false, enc(hash.ChecksumCRC64NVME, p1), nil},
|
||||
{"multipart COMPOSITE CRC32C", checksumTestObject(t, hash.ChecksumCRC32C, false, p1, p2), hash.ChecksumCRC32C, true, composite(hash.ChecksumCRC32C), nil},
|
||||
{"multipart COMPOSITE SHA256", checksumTestObject(t, hash.ChecksumSHA256, false, p1, p2), hash.ChecksumSHA256, true, composite(hash.ChecksumSHA256), nil},
|
||||
{"multipart FULL_OBJECT CRC32", checksumTestObject(t, hash.ChecksumCRC32, true, p1, p2), hash.ChecksumCRC32, true, enc(hash.ChecksumCRC32, whole), full},
|
||||
{"multipart FULL_OBJECT CRC32C", checksumTestObject(t, hash.ChecksumCRC32C, true, p1, p2), hash.ChecksumCRC32C, true, enc(hash.ChecksumCRC32C, whole), full},
|
||||
{"multipart CRC64NVME", checksumTestObject(t, hash.ChecksumCRC64NVME, false, p1, p2), hash.ChecksumCRC64NVME, true, enc(hash.ChecksumCRC64NVME, whole), full},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
oi := tt.fi.ToObjectInfo("bucket", "object", true)
|
||||
want := map[string]string{tt.alg.Key(): tt.value, xhttp.AmzChecksumAlgo: tt.alg.String()}
|
||||
maps.Copy(want, tt.extra)
|
||||
got, isMP := getCRCMeta(oi, 0, nil)
|
||||
if isMP != tt.isMP {
|
||||
t.Errorf("isMP: got %t, want %t", isMP, tt.isMP)
|
||||
}
|
||||
if !maps.Equal(got, want) {
|
||||
t.Errorf("object headers: got %v, want %v", got, want)
|
||||
}
|
||||
if !tt.isMP {
|
||||
return
|
||||
}
|
||||
// Part headers carry the part checksum and never a type.
|
||||
want = map[string]string{tt.alg.Key(): enc(tt.alg, p2), xhttp.AmzChecksumAlgo: tt.alg.String()}
|
||||
if got, _ := getCRCMeta(oi, 2, nil); !maps.Equal(got, want) {
|
||||
t.Errorf("part 2 headers: got %v, want %v", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestReplicateObjectChecksumTypes replicates objects with checksums through
|
||||
// the S3 API of a test server, and checks that the replica gets the same
|
||||
// checksum as the source.
|
||||
func TestReplicateObjectChecksumTypes(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
// Leave no object layer behind: unit tests that run later in the package
|
||||
// rely on newObjectLayerFn() returning nil.
|
||||
defer resetTestGlobals()
|
||||
ts := StartTestServer(t, ErasureSDStr)
|
||||
defer ts.Stop()
|
||||
|
||||
clnt, err := minio.New(strings.TrimPrefix(ts.Server.URL, "http://"), &minio.Options{
|
||||
Creds: credentials.NewStaticV4(ts.AccessKey, ts.SecretKey, ""),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
core := &minio.Core{Client: clnt}
|
||||
for _, bucket := range []string{"src", "dst"} {
|
||||
if err := clnt.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := clnt.EnableVersioning(ctx, bucket); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
small := []byte("single part object")
|
||||
p1 := bytes.Repeat([]byte("a"), 5<<20)
|
||||
p2 := []byte("last part")
|
||||
tests := []struct {
|
||||
name string
|
||||
alg hash.ChecksumType
|
||||
multipart bool
|
||||
full bool
|
||||
copyOf string // server-side copy of an object created above
|
||||
}{
|
||||
{"put-SHA256", hash.ChecksumSHA256, false, false, ""},
|
||||
{"put-CRC32", hash.ChecksumCRC32, false, false, ""},
|
||||
{"mp-COMPOSITE-CRC32C", hash.ChecksumCRC32C, true, false, ""},
|
||||
{"mp-COMPOSITE-SHA256", hash.ChecksumSHA256, true, false, ""},
|
||||
{"mp-FULL_OBJECT-CRC32", hash.ChecksumCRC32, true, true, ""},
|
||||
{"mp-FULL_OBJECT-CRC32C", hash.ChecksumCRC32C, true, true, ""},
|
||||
{"mp-CRC64NVME", hash.ChecksumCRC64NVME, true, false, ""},
|
||||
// The copy keeps the multipart FULL_OBJECT checksum of its source
|
||||
// but is not multipart, so it replicates with a single PUT.
|
||||
{"copy-FULL_OBJECT-CRC32", hash.ChecksumCRC32, false, true, "mp-FULL_OBJECT-CRC32"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
object := tt.name
|
||||
// Upload the source object like an S3 client does.
|
||||
switch {
|
||||
case tt.copyOf != "":
|
||||
_, err := core.CopyObject(ctx, "src", tt.copyOf, "src", object, nil, minio.CopySrcOptions{}, minio.PutObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case !tt.multipart:
|
||||
_, err := core.PutObject(ctx, "src", object, bytes.NewReader(small), int64(len(small)), "", "", minio.PutObjectOptions{
|
||||
UserMetadata: map[string]string{tt.alg.Key(): hash.NewChecksumFromData(tt.alg, small).Encoded},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
default:
|
||||
meta := map[string]string{xhttp.AmzChecksumAlgo: tt.alg.String()}
|
||||
if tt.full {
|
||||
meta[xhttp.AmzChecksumType] = xhttp.AmzChecksumTypeFullObject
|
||||
}
|
||||
uploadID, err := core.NewMultipartUpload(ctx, "src", object, minio.PutObjectOptions{UserMetadata: meta})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var parts []minio.CompletePart
|
||||
for i, p := range [][]byte{p1, p2} {
|
||||
h := http.Header{}
|
||||
h.Set(tt.alg.Key(), hash.NewChecksumFromData(tt.alg, p).Encoded)
|
||||
pi, err := core.PutObjectPart(ctx, "src", object, uploadID, i+1, bytes.NewReader(p), int64(len(p)), minio.PutObjectPartOptions{CustomHeader: h})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parts = append(parts, minio.CompletePart{
|
||||
PartNumber: pi.PartNumber, ETag: pi.ETag,
|
||||
ChecksumCRC32: pi.ChecksumCRC32, ChecksumCRC32C: pi.ChecksumCRC32C,
|
||||
ChecksumSHA1: pi.ChecksumSHA1, ChecksumSHA256: pi.ChecksumSHA256,
|
||||
ChecksumCRC64NVME: pi.ChecksumCRC64NVME,
|
||||
})
|
||||
}
|
||||
if _, err := core.CompleteMultipartUpload(ctx, "src", object, uploadID, parts, minio.PutObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Replicate it the way replicateObject does.
|
||||
gr, err := ts.Obj.GetObjectNInfo(ctx, "src", object, nil, http.Header{}, ObjectOptions{Versioned: true, ReplicationRequest: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer gr.Close()
|
||||
objInfo := gr.ObjInfo
|
||||
putOpts, isMP, err := putReplicationOpts(ctx, "", objInfo)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if isMP != tt.multipart {
|
||||
t.Fatalf("multipart replication: got %t, want %t", isMP, tt.multipart)
|
||||
}
|
||||
if isMP {
|
||||
err = replicateObjectWithMultipart(ctx, core, "dst", object, gr, objInfo, putOpts)
|
||||
} else {
|
||||
size, serr := objInfo.GetActualSize()
|
||||
if serr != nil {
|
||||
t.Fatal(serr)
|
||||
}
|
||||
_, err = core.PutObject(ctx, "dst", object, gr, size, "", "", putOpts)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("replication failed: %v", err)
|
||||
}
|
||||
|
||||
src, err := clnt.StatObject(ctx, "src", object, minio.StatObjectOptions{Checksum: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dst, err := clnt.StatObject(ctx, "dst", object, minio.StatObjectOptions{Checksum: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checksums := func(oi minio.ObjectInfo) string {
|
||||
return strings.Join([]string{oi.ChecksumMode, oi.ChecksumCRC32, oi.ChecksumCRC32C, oi.ChecksumSHA1, oi.ChecksumSHA256, oi.ChecksumCRC64NVME}, "|")
|
||||
}
|
||||
if checksums(src) == "|||||" || checksums(dst) != checksums(src) {
|
||||
t.Errorf("replica checksum %q, source %q", checksums(dst), checksums(src))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -504,6 +504,35 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
|
||||
if !isPurge && dobj.DeleteMarkerVersionID != "" {
|
||||
// A creation task carries the marker as it looked when it was queued
|
||||
// by the DELETE handler, a GET/HEAD/LIST heal, the scanner or MRF.
|
||||
// While it waited for this lock, another frontend may have purged the
|
||||
// marker and replicated that purge. The targets no longer hold the
|
||||
// marker, so the queued creation would recreate it from the stale
|
||||
// snapshot. Confirm the source version under the lock first.
|
||||
switch deleteMarkerCreationState(ctx, objectAPI, dobj) {
|
||||
case creationStale:
|
||||
return replicatedInfos{}
|
||||
case creationUnverified:
|
||||
dobj.RetryCount++
|
||||
globalReplicationPool.Get().queueMRFSave(dobj.ToMRFEntry())
|
||||
sendEvent(eventArgs{
|
||||
BucketName: bucket,
|
||||
Object: ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: dobj.ObjectName,
|
||||
VersionID: versionID,
|
||||
DeleteMarker: dobj.DeleteMarker,
|
||||
},
|
||||
UserAgent: "Internal: [Replication]",
|
||||
Host: globalLocalNodeName,
|
||||
EventName: event.ObjectReplicationNotTracked,
|
||||
})
|
||||
return replicatedInfos{}
|
||||
}
|
||||
}
|
||||
|
||||
rinfos := replicatedInfos{Targets: make([]replicatedTargetInfo, 0, len(dsc.targetsMap))}
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
@@ -1955,6 +1984,41 @@ func (di DeletedObjectReplicationInfo) isVersionPurge() bool {
|
||||
return di.VersionID != "" || di.DeleteMarkerVersionID != "" && !di.VersionPurgeStatus().Empty()
|
||||
}
|
||||
|
||||
// creationState is the outcome of re-reading a queued delete-marker creation
|
||||
// against the source.
|
||||
type creationState int
|
||||
|
||||
const (
|
||||
creationCurrent creationState = iota
|
||||
creationStale
|
||||
creationUnverified
|
||||
)
|
||||
|
||||
// deleteMarkerCreationState re-reads the marker a queued creation task refers
|
||||
// to. The version being absent, no longer a marker, or under a version purge
|
||||
// makes the creation stale. A failed read is not absence: the caller retries
|
||||
// later instead of guessing.
|
||||
func deleteMarkerCreationState(ctx context.Context, objectAPI ObjectLayer, dobj DeletedObjectReplicationInfo) creationState {
|
||||
oi, err := objectAPI.GetObjectInfo(ctx, dobj.Bucket, dobj.ObjectName, ObjectOptions{
|
||||
VersionID: dobj.DeleteMarkerVersionID,
|
||||
Versioned: globalBucketVersioningSys.PrefixEnabled(dobj.Bucket, dobj.ObjectName),
|
||||
VersionSuspended: globalBucketVersioningSys.Suspended(dobj.Bucket),
|
||||
})
|
||||
switch {
|
||||
case isErrObjectNotFound(err), isErrVersionNotFound(err):
|
||||
return creationStale
|
||||
case err != nil && !isErrMethodNotAllowed(err):
|
||||
return creationUnverified
|
||||
}
|
||||
if !oi.DeleteMarker || oi.VersionID != dobj.DeleteMarkerVersionID {
|
||||
return creationStale
|
||||
}
|
||||
if !oi.VersionPurgeStatus.Empty() || oi.VersionPurgeStatusInternal != "" {
|
||||
return creationStale
|
||||
}
|
||||
return creationCurrent
|
||||
}
|
||||
|
||||
// Purge metadata uses COMPLETE; operation statistics and audit use COMPLETED.
|
||||
func purgeReplicationStatus(status VersionPurgeStatusType) replication.StatusType {
|
||||
if replication.StatusType(status) == replication.CompletedLegacy {
|
||||
@@ -4186,6 +4250,11 @@ func getCRCMeta(oi ObjectInfo, partNum int, h http.Header) (cs map[string]string
|
||||
if cktype.IsSet() {
|
||||
meta[cktype.Key()] = v
|
||||
meta[xhttp.AmzChecksumAlgo] = cktype.String()
|
||||
// Without the type the target creates a COMPOSITE multipart
|
||||
// upload and rejects the full object CRC on completion.
|
||||
if isMP && cktype.CanMerge() && cs[xhttp.AmzChecksumType] == xhttp.AmzChecksumTypeFullObject {
|
||||
meta[xhttp.AmzChecksumType] = xhttp.AmzChecksumTypeFullObject
|
||||
}
|
||||
}
|
||||
}
|
||||
return meta, isMP
|
||||
|
||||
@@ -56,6 +56,7 @@ import (
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/color"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/config/api"
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
@@ -881,6 +882,9 @@ func serverHandleEnvVars() {
|
||||
logger.Warning("%s=%s is ignored: Silo in-place updates are permanently disabled", config.EnvUpdate, updateSetting)
|
||||
}
|
||||
globalInplaceUpdateDisabled = true
|
||||
if env.IsSet(api.EnvAPIMultipartListing) {
|
||||
logger.Warning("%s is ignored: multipart listing uses the legacy cache/exact-key behavior; persistent listing is deferred", api.EnvAPIMultipartListing)
|
||||
}
|
||||
|
||||
// Check if the supported credential env vars,
|
||||
// "MINIO_ROOT_USER" and "MINIO_ROOT_PASSWORD" are provided
|
||||
|
||||
@@ -91,7 +91,7 @@ func readConfigWithoutMigrate(ctx context.Context, objAPI ObjectLayer) (config.C
|
||||
return nil, err
|
||||
}
|
||||
|
||||
data, err = decryptData(data, configFile)
|
||||
data, err = decryptData(ctx, data, configFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+3
-3
@@ -68,7 +68,7 @@ func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData b
|
||||
continue
|
||||
}
|
||||
|
||||
data, err = decryptData(data, obj.Name)
|
||||
data, err = decryptData(ctx, data, obj.Name)
|
||||
if err != nil {
|
||||
// ignore history file that cannot be loaded.
|
||||
continue
|
||||
@@ -110,7 +110,7 @@ func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV str
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return decryptData(data, historyFile)
|
||||
return decryptData(ctx, data, historyFile)
|
||||
}
|
||||
|
||||
func saveServerConfigHistory(ctx context.Context, objAPI ObjectLayer, kv []byte) error {
|
||||
@@ -162,7 +162,7 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer, data []byte) (con
|
||||
return nil, err
|
||||
}
|
||||
|
||||
data, err = decryptData(data, configFile)
|
||||
data, err = decryptData(ctx, data, configFile)
|
||||
if err != nil {
|
||||
lookupConfigs(srvCfg, objAPI)
|
||||
return nil, err
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
|
||||
// Tests create endpoints with ellipses and without.
|
||||
func TestCreateServerEndpoints(t *testing.T) {
|
||||
useStaticTestEndpoints(t)
|
||||
testCases := []struct {
|
||||
serverAddr string
|
||||
args []string
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
@@ -25,9 +26,42 @@ import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/dnscache"
|
||||
)
|
||||
|
||||
type endpointTestResolver struct {
|
||||
dnscache.DNSResolver
|
||||
}
|
||||
|
||||
func (r endpointTestResolver) LookupHost(ctx context.Context, host string) ([]string, error) {
|
||||
// These endpoint fixtures test locality and parsing, not public DNS.
|
||||
for i, name := range []string{"example.org", "example.com", "example.net", "example.edu"} {
|
||||
if host == name {
|
||||
return []string{fmt.Sprintf("203.0.113.%d", i+1)}, nil
|
||||
}
|
||||
}
|
||||
return r.DNSResolver.LookupHost(ctx, host)
|
||||
}
|
||||
|
||||
func useTestEndpointDNS(t *testing.T) {
|
||||
t.Helper()
|
||||
previous := globalDNSCache
|
||||
globalDNSCache = &dnscache.Resolver{Resolver: endpointTestResolver{net.DefaultResolver}}
|
||||
t.Cleanup(func() { globalDNSCache = previous })
|
||||
}
|
||||
|
||||
func useStaticTestEndpoints(t *testing.T) {
|
||||
t.Helper()
|
||||
useTestEndpointDNS(t)
|
||||
// These fixtures deliberately use localhost, including inside containers.
|
||||
previous := orchestrated
|
||||
orchestrated = false
|
||||
t.Cleanup(func() { orchestrated = previous })
|
||||
}
|
||||
|
||||
func TestNewEndpoint(t *testing.T) {
|
||||
useTestEndpointDNS(t)
|
||||
u2, _ := url.Parse("https://example.org/path")
|
||||
u4, _ := url.Parse("http://192.168.253.200/path")
|
||||
rootSlashFoo, _ := filepath.Abs("/foo")
|
||||
@@ -130,6 +164,7 @@ func TestNewEndpoints(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCreateEndpoints(t *testing.T) {
|
||||
useStaticTestEndpoints(t)
|
||||
tempGlobalMinioPort := globalMinioPort
|
||||
defer func() {
|
||||
globalMinioPort = tempGlobalMinioPort
|
||||
@@ -353,6 +388,7 @@ func TestCreateEndpoints(t *testing.T) {
|
||||
// So it means that if you have say localhost:9000 and localhost:9001 as endpointArgs then localhost:9001
|
||||
// is considered a remote service from localhost:9000 perspective.
|
||||
func TestGetLocalPeer(t *testing.T) {
|
||||
useStaticTestEndpoints(t)
|
||||
tempGlobalMinioPort := globalMinioPort
|
||||
defer func() {
|
||||
globalMinioPort = tempGlobalMinioPort
|
||||
@@ -393,6 +429,7 @@ func TestGetLocalPeer(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGetRemotePeers(t *testing.T) {
|
||||
useStaticTestEndpoints(t)
|
||||
tempGlobalMinioPort := globalMinioPort
|
||||
defer func() {
|
||||
globalMinioPort = tempGlobalMinioPort
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
// An exact-version DELETE without a replication decision physically purges
|
||||
// the version. Its response must describe the stored version: a data version
|
||||
// is not a delete marker even while pending-purge metadata makes the lookup
|
||||
// expose it as deleted, and a stored marker stays a marker.
|
||||
func TestDeleteMarkerPurgeResponseIdentity(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, endpoints: []string{"DeleteObject"}, objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, creds auth.Credentials, t *testing.T) {
|
||||
defer replicationTestCapacity(obj)()
|
||||
ctx := t.Context()
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Seed the purge state the DELETE handler would record for a target.
|
||||
arn := "arn:minio:replication::" + mustGetUUID() + ":bucket"
|
||||
pendingPurge := ReplicationState{
|
||||
VersionPurgeStatusInternal: arn + "=PENDING;",
|
||||
PurgeTargets: map[string]VersionPurgeStatusType{arn: replication.VersionPurgePending},
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
marker, purged bool
|
||||
}{
|
||||
{name: "data"},
|
||||
{name: "data-pending-purge", purged: true},
|
||||
{name: "marker", marker: true},
|
||||
{name: "marker-pending-purge", marker: true, purged: true},
|
||||
} {
|
||||
name := "purge-response-" + mustGetUUID()
|
||||
oi, err := obj.PutObject(ctx, bucket, name, mustGetPutObjReader(t, bytes.NewReader([]byte("data")), 4, "", ""), ObjectOptions{Versioned: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
version := oi.VersionID
|
||||
if tc.marker {
|
||||
opts := ObjectOptions{Versioned: true, VersionID: mustGetUUID(), DeleteMarker: true, MTime: UTCNow(), ReplicationRequest: true}
|
||||
opts.SetReplicaStatus(replication.Replica)
|
||||
if _, err := obj.DeleteObject(ctx, bucket, name, opts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
version = opts.VersionID
|
||||
}
|
||||
if tc.purged {
|
||||
if _, err := obj.DeleteObject(ctx, bucket, name, ObjectOptions{Versioned: true, VersionID: version, DeleteReplication: pendingPurge}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
before, err := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{Versioned: true, VersionID: version})
|
||||
if tc.marker || tc.purged {
|
||||
if !isErrMethodNotAllowed(err) || !before.DeleteMarker {
|
||||
t.Fatalf("%s: seeded version not exposed as deleted: %+v %v", tc.name, before, err)
|
||||
}
|
||||
} else if err != nil || before.DeleteMarker {
|
||||
t.Fatalf("%s: seeded data version: %+v %v", tc.name, before, err)
|
||||
}
|
||||
if tc.purged && before.VersionPurgeStatus != replication.VersionPurgePending {
|
||||
t.Fatalf("%s: purge state not seeded: %+v", tc.name, before)
|
||||
}
|
||||
req, err := newTestSignedRequestV4(http.MethodDelete, "/"+bucket+"/"+name+"?versionId="+version, 0, nil, creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("%s: status=%d: %s", tc.name, rec.Code, rec.Body.String())
|
||||
}
|
||||
// The handler sets these headers by map key, not canonical name.
|
||||
if got := rec.Header()[xhttp.AmzVersionID]; len(got) != 1 || got[0] != version {
|
||||
t.Fatalf("%s: response version %q, want %q", tc.name, got, version)
|
||||
}
|
||||
if got := len(rec.Header()[xhttp.AmzDeleteMarker]) == 1 && rec.Header()[xhttp.AmzDeleteMarker][0] == "true"; got != tc.marker {
|
||||
t.Fatalf("%s: x-amz-delete-marker=%v for a stored marker=%v", tc.name, got, tc.marker)
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{Versioned: true, VersionID: version}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Fatalf("%s: version not purged: %v", tc.name, err)
|
||||
}
|
||||
t.Logf("%s %s: purged with x-amz-delete-marker=%q", backend, tc.name, rec.Header()[xhttp.AmzDeleteMarker])
|
||||
}
|
||||
}})
|
||||
}
|
||||
@@ -0,0 +1,524 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/once"
|
||||
)
|
||||
|
||||
func TestDeleteMarkerPurgeIntent(t *testing.T) {
|
||||
version := mustGetUUID()
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
edit func(*ObjectOptions)
|
||||
want bool
|
||||
}{
|
||||
{"ordinary", func(*ObjectOptions) {}, true},
|
||||
{"receiver", func(o *ObjectOptions) { o.ReplicationRequest = true; o.SetReplicaStatus(replication.Replica) }, true},
|
||||
{"untrusted-replica", func(o *ObjectOptions) { o.SetReplicaStatus(replication.Replica) }, false},
|
||||
{"complete-purge", func(o *ObjectOptions) {
|
||||
o.DeleteReplication.VersionPurgeStatusInternal = string(replication.VersionPurgeComplete)
|
||||
}, true},
|
||||
{"pending-purge", func(o *ObjectOptions) { o.DeleteReplication.VersionPurgeStatusInternal = "PENDING" }, false},
|
||||
{"failed-purge", func(o *ObjectOptions) { o.DeleteReplication.VersionPurgeStatusInternal = "FAILED" }, false},
|
||||
{"unknown-purge", func(o *ObjectOptions) { o.DeleteReplication.VersionPurgeStatusInternal = "future-state" }, false},
|
||||
{"pending-creation", func(o *ObjectOptions) { o.DeleteReplication.ReplicationStatusInternal = "PENDING" }, false},
|
||||
{"failed-creation", func(o *ObjectOptions) { o.DeleteReplication.ReplicationStatusInternal = "FAILED" }, false},
|
||||
{"completed-creation", func(o *ObjectOptions) { o.DeleteReplication.ReplicationStatusInternal = "COMPLETED" }, false},
|
||||
{"create-marker", func(o *ObjectOptions) { o.DeleteMarker = true }, false},
|
||||
{"empty", func(o *ObjectOptions) { o.VersionID = "" }, false},
|
||||
{"null", func(o *ObjectOptions) { o.VersionID = nullVersionID }, false},
|
||||
{"invalid", func(o *ObjectOptions) { o.VersionID = "invalid" }, false},
|
||||
{"zero-uuid", func(o *ObjectOptions) { o.VersionID = emptyUUID }, false},
|
||||
{"movement", func(o *ObjectOptions) { o.DataMovement = true }, false},
|
||||
{"free-version", func(o *ObjectOptions) { o.InclFreeVersions = true }, false},
|
||||
{"expiration", func(o *ObjectOptions) { o.Expiration.Expire = true }, false},
|
||||
{"transition", func(o *ObjectOptions) { o.Transition.Status = "complete" }, false},
|
||||
{"restored-expiration", func(o *ObjectOptions) { o.Transition.ExpireRestored = true }, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
o := ObjectOptions{VersionID: version, Versioned: true}
|
||||
tc.edit(&o)
|
||||
if got := o.isVersionPurge(); got != tc.want {
|
||||
t.Fatalf("purge=%v want=%v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Every other StorageAPI method panics through the nil embedding: a proof may
|
||||
// read metadata, but must never delete, heal, or ask a different storage API.
|
||||
type absenceProofDisk struct {
|
||||
StorageAPI
|
||||
t *testing.T
|
||||
err error
|
||||
reads *atomic.Int32
|
||||
deletes *atomic.Int32
|
||||
}
|
||||
|
||||
func (d absenceProofDisk) ReadVersion(_ context.Context, _, _, _, _ string, opts ReadOptions) (FileInfo, error) {
|
||||
if opts.ReadData || opts.Healing {
|
||||
d.t.Error("absence proof requested data/healing")
|
||||
}
|
||||
d.reads.Add(1)
|
||||
return FileInfo{}, d.err
|
||||
}
|
||||
|
||||
func (d absenceProofDisk) DeleteVersion(_ context.Context, _, _ string, _ FileInfo, _ bool, _ DeleteOptions) error {
|
||||
if d.deletes == nil {
|
||||
d.t.Error("read-only absence proof attempted a deletion")
|
||||
} else {
|
||||
d.deletes.Add(1)
|
||||
}
|
||||
return d.err
|
||||
}
|
||||
|
||||
func TestVersionPurgeAbsenceProof(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
errs []error
|
||||
quorum, all bool
|
||||
}{
|
||||
{"all-absent", []error{errFileNotFound, errFileVersionNotFound, errFileNotFound, errFileVersionNotFound}, true, true},
|
||||
{"majority-absent", []error{errFileNotFound, errFileVersionNotFound, errFileNotFound, nil}, true, false},
|
||||
{"half-absent", []error{errFileNotFound, errFileVersionNotFound, errDiskNotFound, errDiskNotFound}, false, false},
|
||||
{"corrupt", []error{errFileNotFound, errFileVersionNotFound, errFileCorrupt, errFileCorrupt}, false, false},
|
||||
{"permission", []error{errFileNotFound, errFileVersionNotFound, errDiskAccessDenied, errVolumeAccessDenied}, false, false},
|
||||
{"volume-missing", []error{errFileNotFound, errFileVersionNotFound, errVolumeNotFound, errVolumeNotFound}, false, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var reads atomic.Int32
|
||||
disks := make([]StorageAPI, len(tc.errs))
|
||||
for i, err := range tc.errs {
|
||||
disks[i] = absenceProofDisk{t: t, err: err, reads: &reads}
|
||||
}
|
||||
er := erasureObjects{getDisks: func() []StorageAPI { return disks }}
|
||||
oldQueue := globalMRFState.opCh
|
||||
globalMRFState.opCh = make(chan PartialOperation, 1)
|
||||
defer func() { globalMRFState.opCh = oldQueue }()
|
||||
all, err := er.confirmVersionAbsent(t.Context(), "bucket", "object", mustGetUUID())
|
||||
if (err == nil) != tc.quorum || all != tc.all || reads.Load() != int32(len(disks)) || len(globalMRFState.opCh) != 0 {
|
||||
t.Fatalf("proof all=%v error=%v reads=%d queued=%d", all, err, reads.Load(), len(globalMRFState.opCh))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionPurgeAggregation(t *testing.T) {
|
||||
for _, pure := range []bool{false, true} {
|
||||
for _, fault := range []error{nil, errFileCorrupt, errDiskAccessDenied} {
|
||||
t.Run(fmt.Sprintf("purge_%v_fault_%v", pure, fault), func(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
disks := make([]StorageAPI, 4)
|
||||
for i, err := range []error{nil, errFileNotFound, errFileVersionNotFound, fault} {
|
||||
disks[i] = absenceProofDisk{t: t, err: err, deletes: &calls}
|
||||
}
|
||||
er := erasureObjects{getDisks: func() []StorageAPI { return disks }}
|
||||
err := er.deleteObjectVersion(t.Context(), "bucket", "object", FileInfo{VersionID: mustGetUUID()}, false, pure)
|
||||
if (err == nil) != pure || calls.Load() != 4 {
|
||||
t.Fatalf("aggregation error=%v calls=%d", err, calls.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMarkerPurgeOmittedPool(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
defer replicationTestCapacity(z)()
|
||||
const name = "partially-absent"
|
||||
er := z.serverPools[1].getHashedSet(name)
|
||||
_, opts := seedPurgeMarker(t, er, bucket, name, false)
|
||||
opts.ReplicationRequest = false
|
||||
opts.DeleteReplication = ReplicationState{}
|
||||
// Pool 0 appears absent at read quorum, but half of it cannot be read.
|
||||
missing := z.serverPools[0].getHashedSet(name)
|
||||
original := missing.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
clear(disks[len(disks)/2:])
|
||||
missing.getDisks = func() []StorageAPI { return disks }
|
||||
defer func() { missing.getDisks = original }()
|
||||
_, err := z.DeleteObject(t.Context(), bucket, name, opts)
|
||||
if !isErrWriteQuorum(err) {
|
||||
t.Errorf("omitted pool acknowledged deletion: %T %v", err, err)
|
||||
}
|
||||
if countPurgeMarkers(t, er.getDisks(), bucket, name, opts.VersionID) != 16 {
|
||||
t.Fatal("mutated known copies before checking omitted pool")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMarkerPurgeReceivingPool(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
defer replicationTestCapacity(z)()
|
||||
for _, duplicate := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("duplicate_%v", duplicate), func(t *testing.T) {
|
||||
name := "receiver-" + mustGetUUID()
|
||||
_, opts := seedPurgeMarker(t, z.serverPools[0].getHashedSet(name), bucket, name, false)
|
||||
if duplicate {
|
||||
create := opts
|
||||
create.DeleteMarker = true
|
||||
if _, err := z.serverPools[1].DeleteObject(t.Context(), bucket, name, create); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
// Latest-key routing selects pool 1, but the addressed marker is
|
||||
// in pool 0. A replica purge addresses a version, not the latest.
|
||||
putConsistencyObject(t, z, bucket, name, 1, "newer-data", ObjectOptions{Versioned: true})
|
||||
}
|
||||
if _, err := z.DeleteObject(t.Context(), bucket, name, opts); err != nil {
|
||||
t.Errorf("replica purge did not find its addressed version: %v", err)
|
||||
}
|
||||
for i, pool := range z.serverPools {
|
||||
if got := countPurgeMarkers(t, pool.getHashedSet(name).getDisks(), bucket, name, opts.VersionID); got != 0 {
|
||||
t.Errorf("replica purge left %d marker copies in pool %d", got, i)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMarkerPurgeCallbackIntent(t *testing.T) {
|
||||
for _, loseCopies := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("missing_after_callback_%v", loseCopies), func(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
defer replicationTestCapacity(z)()
|
||||
const name = "callback-pending"
|
||||
er := z.serverPools[0].getHashedSet(name)
|
||||
_, opts := seedPurgeMarker(t, er, bucket, name, false)
|
||||
original := er.getDisks
|
||||
all := append([]StorageAPI(nil), original()...)
|
||||
defer func() { er.getDisks = original }()
|
||||
opts.ReplicationRequest = false
|
||||
opts.DeleteReplication = ReplicationState{}
|
||||
metadata, retention := 0, 0
|
||||
opts.EvalRetentionBypassFn = func(oi ObjectInfo, err error) error {
|
||||
retention++
|
||||
if !oi.DeleteMarker || !isErrMethodNotAllowed(err) {
|
||||
t.Fatalf("retention callback lost marker: %+v %v", oi, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
opts.EvalMetadataFn = func(*ObjectInfo, error) (ReplicateDecision, error) {
|
||||
metadata++
|
||||
if loseCopies {
|
||||
// Inject a disk-state change after preflight, before the set's
|
||||
// metadata-update read. It returns NotFound at read quorum.
|
||||
for _, disk := range all[:8] {
|
||||
if err := disk.DeleteVersion(t.Context(), bucket, name, FileInfo{VersionID: opts.VersionID}, false, DeleteOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
partial := make([]StorageAPI, len(all))
|
||||
copy(partial, all[:8])
|
||||
er.getDisks = func() []StorageAPI { return partial }
|
||||
}
|
||||
decision := ReplicateDecision{}
|
||||
decision.Set(newReplicateTargetDecision("arn1", true, false))
|
||||
return decision, nil
|
||||
}
|
||||
_, err := z.DeleteObject(t.Context(), bucket, name, opts)
|
||||
if loseCopies {
|
||||
if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Errorf("pending update misclassified as physical purge: %T %v", err, err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Errorf("pending update failed: %v", err)
|
||||
}
|
||||
want := 16
|
||||
if loseCopies {
|
||||
want = 8
|
||||
}
|
||||
if got := countPurgeMarkers(t, all, bucket, name, opts.VersionID); got != want || metadata != 1 || retention != 1 {
|
||||
t.Errorf("pending update: copies=%d want=%d metadata=%d retention=%d", got, want, metadata, retention)
|
||||
}
|
||||
if !loseCopies {
|
||||
fi, err := all[0].ReadVersion(t.Context(), "", bucket, name, opts.VersionID, ReadOptions{})
|
||||
if err != nil || fi.VersionPurgeStatus() != replication.VersionPurgePending {
|
||||
t.Errorf("pending state not persisted: %+v %v", fi.ReplicationState, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMarkerReceivingMetadataUpdate(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
defer replicationTestCapacity(z)()
|
||||
for _, state := range []VersionPurgeStatusType{replication.VersionPurgePending, replication.VersionPurgeFailed} {
|
||||
t.Run(string(state), func(t *testing.T) {
|
||||
name := "update-" + mustGetUUID()
|
||||
_, opts := seedPurgeMarker(t, z.serverPools[0].getHashedSet(name), bucket, name, false)
|
||||
create := opts
|
||||
create.DeleteMarker = true
|
||||
if _, err := z.serverPools[1].DeleteObject(t.Context(), bucket, name, create); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.DeleteReplication = ReplicationState{VersionPurgeStatusInternal: string(state)}
|
||||
if _, err := z.DeleteObject(t.Context(), bucket, name, opts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, pool := range z.serverPools {
|
||||
if got := countPurgeMarkers(t, pool.getHashedSet(name).getDisks(), bucket, name, opts.VersionID); got != 16 {
|
||||
t.Errorf("metadata update removed marker copies in pool %d: %d", i, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionPurgeRetentionGate(t *testing.T) {
|
||||
z, er, all, bucket := markerPurgeFixture(t, 4)
|
||||
data, _ := seedPurgeMarker(t, er, bucket, "protected-data", true)
|
||||
denied := errors.New("retention denied")
|
||||
opts := ObjectOptions{Versioned: true, VersionID: data, EvalRetentionBypassFn: func(oi ObjectInfo, err error) error {
|
||||
if err != nil || oi.VersionID != data || oi.DeleteMarker {
|
||||
t.Errorf("wrong retained version: %+v error=%v", oi, err)
|
||||
}
|
||||
return denied
|
||||
}}
|
||||
_, err := z.DeleteObject(t.Context(), bucket, "protected-data", opts)
|
||||
if !errors.Is(err, denied) {
|
||||
t.Fatalf("retention gate: %v", err)
|
||||
}
|
||||
for _, disk := range all {
|
||||
if _, err := disk.ReadVersion(t.Context(), "", bucket, "protected-data", data, ReadOptions{}); err != nil {
|
||||
t.Errorf("protected version changed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func markerPurgeFixture(t *testing.T, n int) (*erasureServerPools, *erasureObjects, []StorageAPI, string) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
obj, dirs, err := prepareErasure(ctx, n)
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatal(err)
|
||||
}
|
||||
restore := replicationTestCapacity(obj)
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
restore()
|
||||
obj.Shutdown(context.Background())
|
||||
removeRoots(dirs)
|
||||
})
|
||||
bucket := getRandomBucketName()
|
||||
if err := obj.MakeBucket(ctx, bucket, MakeBucketOptions{VersioningEnabled: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
er := z.serverPools[0].sets[0]
|
||||
original := er.getDisks
|
||||
t.Cleanup(func() { er.getDisks = original })
|
||||
return z, er, append([]StorageAPI(nil), original()...), bucket
|
||||
}
|
||||
|
||||
func seedPurgeMarker(t *testing.T, er *erasureObjects, bucket, name string, withData bool) (string, ObjectOptions) {
|
||||
t.Helper()
|
||||
dataVersion := ""
|
||||
if withData {
|
||||
oi, err := er.PutObject(t.Context(), bucket, name, mustGetPutObjReader(t, bytes.NewReader([]byte("data")), 4, "", ""), ObjectOptions{Versioned: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dataVersion = oi.VersionID
|
||||
}
|
||||
opts := ObjectOptions{Versioned: true, VersionID: mustGetUUID(), DeleteMarker: true, ReplicationRequest: true, MTime: UTCNow().Add(-time.Hour), NoAuditLog: true}
|
||||
opts.SetReplicaStatus(replication.Replica)
|
||||
if _, err := er.DeleteObject(t.Context(), bucket, name, opts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.DeleteMarker = false
|
||||
return dataVersion, opts
|
||||
}
|
||||
|
||||
func countPurgeMarkers(t *testing.T, disks []StorageAPI, bucket, name, version string) int {
|
||||
t.Helper()
|
||||
n := 0
|
||||
for _, disk := range disks {
|
||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, name, version, ReadOptions{})
|
||||
if err == nil && fi.Deleted {
|
||||
n++
|
||||
} else if err != errFileNotFound && err != errFileVersionNotFound {
|
||||
t.Fatalf("unexpected disk version: deleted=%v error=%v", fi.Deleted, err)
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func TestDeleteMarkerPurgeQuorum(t *testing.T) {
|
||||
for _, tc := range []struct{ disks, online int }{{4, 2}, {16, 7}, {16, 8}, {16, 9}} {
|
||||
t.Run(fmt.Sprintf("%d_disks_%d_online", tc.disks, tc.online), func(t *testing.T) {
|
||||
z, er, all, bucket := markerPurgeFixture(t, tc.disks)
|
||||
const name = "marker"
|
||||
dataVersion, opts := seedPurgeMarker(t, er, bucket, name, true)
|
||||
partial := make([]StorageAPI, len(all))
|
||||
copy(partial, all[:tc.online])
|
||||
er.getDisks = func() []StorageAPI { return partial }
|
||||
_, first := z.DeleteObject(t.Context(), bucket, name, opts)
|
||||
_, retry := z.DeleteObject(t.Context(), bucket, name, opts)
|
||||
if tc.online <= tc.disks/2 {
|
||||
if !isErrWriteQuorum(first) || !isErrWriteQuorum(retry) {
|
||||
t.Errorf("below write quorum: first=%T %v, retry=%T %v", first, first, retry, retry)
|
||||
}
|
||||
} else if first != nil || (retry != nil && !isErrVersionNotFound(retry) && !isErrObjectNotFound(retry)) {
|
||||
t.Errorf("write quorum available: first=%v retry=%v", first, retry)
|
||||
}
|
||||
want := tc.disks - tc.online
|
||||
if tc.online < tc.disks/2 {
|
||||
want = tc.disks
|
||||
}
|
||||
if got := countPurgeMarkers(t, all, bucket, name, opts.VersionID); got != want {
|
||||
t.Errorf("partial purge markers=%d want=%d", got, want)
|
||||
}
|
||||
er.getDisks = func() []StorageAPI { return all }
|
||||
_, err := z.DeleteObject(t.Context(), bucket, name, opts)
|
||||
if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Errorf("full-online retry: %v", err)
|
||||
}
|
||||
if tc.online <= tc.disks/2 {
|
||||
if got := countPurgeMarkers(t, all, bucket, name, opts.VersionID); got != 0 {
|
||||
t.Errorf("full-online retry recreated/retained %d marker copies", got)
|
||||
}
|
||||
}
|
||||
// A quorum-confirmed missing retry can leave minority residue. Exercise
|
||||
// the real dangling-version healer after every disk has returned.
|
||||
_, _ = er.HealObject(t.Context(), bucket, name, opts.VersionID, madmin.HealOpts{Remove: true})
|
||||
if got := countPurgeMarkers(t, all, bucket, name, opts.VersionID); got != 0 {
|
||||
t.Errorf("marker copies after retry and heal=%d", got)
|
||||
}
|
||||
oi, err := z.GetObjectInfo(t.Context(), bucket, name, ObjectOptions{Versioned: true})
|
||||
if err != nil || oi.DeleteMarker || oi.VersionID != dataVersion {
|
||||
t.Errorf("underlying version not visible: %+v error=%v", oi, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMarkerPurgeMissingKey(t *testing.T) {
|
||||
for _, pooled := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("pooled_%v", pooled), func(t *testing.T) {
|
||||
z, er, all, bucket := markerPurgeFixture(t, 4)
|
||||
_, opts := seedPurgeMarker(t, er, bucket, "marker-only", false)
|
||||
er.getDisks = func() []StorageAPI { return []StorageAPI{all[0], all[1], nil, nil} }
|
||||
deleteFn := er.DeleteObject
|
||||
if pooled {
|
||||
deleteFn = z.DeleteObject
|
||||
}
|
||||
_, _ = deleteFn(t.Context(), bucket, "marker-only", opts)
|
||||
_, err := deleteFn(t.Context(), bucket, "marker-only", opts)
|
||||
if !isErrWriteQuorum(err) {
|
||||
t.Errorf("missing retry with only 2/4 absence votes returned %T %v", err, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteMarkerHealReplicationIdentity(t *testing.T) {
|
||||
obj, er, all, bucket := markerPurgeFixture(t, 4)
|
||||
const name = "healed-marker"
|
||||
_, opts := seedPurgeMarker(t, er, bucket, name, true)
|
||||
original, err := all[3].ReadVersion(t.Context(), "", bucket, name, opts.VersionID, ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, disk := range all[:2] {
|
||||
if err := disk.DeleteVersion(t.Context(), bucket, name, FileInfo{VersionID: opts.VersionID}, false, DeleteOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := er.HealObject(t.Context(), bucket, name, opts.VersionID, madmin.HealOpts{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, disk := range all {
|
||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, name, opts.VersionID, ReadOptions{})
|
||||
if err != nil || !maps.Equal(fi.Metadata, original.Metadata) {
|
||||
t.Errorf("disk %d lost marker metadata: before=%v after=%v error=%v", i, original.Metadata, fi.Metadata, err)
|
||||
}
|
||||
}
|
||||
// Only the healed half remains readable. Read actual disk state before
|
||||
// invoking scanner replication, rather than constructing an ObjectInfo.
|
||||
er.getDisks = func() []StorageAPI { return []StorageAPI{all[0], all[1], nil, nil} }
|
||||
oi, err := er.GetObjectInfo(t.Context(), bucket, name, ObjectOptions{VersionID: opts.VersionID, Versioned: true})
|
||||
if !oi.DeleteMarker || !isErrMethodNotAllowed(err) {
|
||||
t.Fatalf("healed marker unreadable: %+v %v", oi, err)
|
||||
}
|
||||
er.getDisks = func() []StorageAPI { return all }
|
||||
var outbound atomic.Int32
|
||||
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodDelete {
|
||||
outbound.Add(1)
|
||||
if r.Header.Get(xhttp.MinIOSourceDeleteMarker) != "true" || r.URL.Query().Get("versionId") != opts.VersionID {
|
||||
t.Errorf("unexpected outbound request: %s %s", r.Method, r.URL)
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer remote.Close()
|
||||
client, err := minio.New(strings.TrimPrefix(remote.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
arn := "arn:minio:replication::" + mustGetUUID() + ":bucket"
|
||||
target := &TargetClient{Client: client, ARN: arn, Bucket: "target"}
|
||||
oldTargets := globalBucketTargetSys
|
||||
globalBucketTargetSys = &BucketTargetSys{arnRemotesMap: map[string]arnTarget{arn: {Client: target, lastRefresh: UTCNow()}}, targetsMap: map[string][]madmin.BucketTarget{bucket: {{Arn: arn, TargetBucket: "target"}}}, hc: map[string]epHealth{client.EndpointURL().Host: {Online: true}}}
|
||||
defer func() { globalBucketTargetSys = oldTargets }()
|
||||
cfg := configs[0]
|
||||
cfg.RoleArn = arn
|
||||
meta, err := globalBucketMetadataSys.Get(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta.replicationConfig = &cfg
|
||||
globalBucketMetadataSys.Set(bucket, meta)
|
||||
worker := make(chan ReplicationWorkerOperation, 1)
|
||||
oldPool := globalReplicationPool
|
||||
globalReplicationPool = once.NewSingleton[ReplicationPool]()
|
||||
globalReplicationPool.Set(&ReplicationPool{ctx: t.Context(), objLayer: obj, workers: []chan ReplicationWorkerOperation{worker}, stats: globalReplicationStats.Load(), mrfSaveCh: make(chan MRFReplicateEntry, 1)})
|
||||
defer func() { globalReplicationPool = oldPool }()
|
||||
roi := queueReplicationHeal(t.Context(), bucket, oi, replicationConfig{Config: &cfg, remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{Arn: arn, TargetBucket: "target"}}}}, 0)
|
||||
select {
|
||||
case op := <-worker:
|
||||
d := op.(DeletedObjectReplicationInfo)
|
||||
result := replicateDeleteToTarget(t.Context(), d, target)
|
||||
t.Errorf("healed replica scheduled for creation: version=%s existing=%v result=%+v", d.DeleteMarkerVersionID, roi.ExistingObjResync.mustResync(), result)
|
||||
default:
|
||||
}
|
||||
if outbound.Load() != 0 {
|
||||
t.Errorf("healed replica sent %d new marker creations", outbound.Load())
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"reflect"
|
||||
@@ -110,6 +111,96 @@ func TestReduceErrs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestReduceWriteQuorumErrsCommitMetaMissing verifies that an in-quorum
|
||||
// errCommitMetaMissing reduces to the sentinel, which renameData() converts
|
||||
// to errErasureWriteQuorum, while below-quorum reduces directly to
|
||||
// errErasureWriteQuorum.
|
||||
func TestReduceWriteQuorumErrsCommitMetaMissing(t *testing.T) {
|
||||
errs := make([]error, 16)
|
||||
for i := range 9 {
|
||||
errs[i] = errCommitMetaMissing
|
||||
}
|
||||
if err := reduceWriteQuorumErrs(t.Context(), errs, objectOpIgnoredErrs, 9); !errors.Is(err, errCommitMetaMissing) {
|
||||
t.Errorf("expected %s, got %s", errCommitMetaMissing, err)
|
||||
}
|
||||
// The sentinel must be distinguishable from errFileNotFound so that
|
||||
// commit failures are never mapped to ObjectNotFound (404).
|
||||
if errors.Is(errCommitMetaMissing, errFileNotFound) {
|
||||
t.Error("errCommitMetaMissing must not match errFileNotFound")
|
||||
}
|
||||
|
||||
// Below quorum the reduction must yield a write-quorum failure (503);
|
||||
// eight sentinels against eight committed drives is not a quorum.
|
||||
belowQuorum := make([]error, 16)
|
||||
for i := range 8 {
|
||||
belowQuorum[i] = errCommitMetaMissing
|
||||
}
|
||||
if err := reduceWriteQuorumErrs(t.Context(), belowQuorum, objectOpIgnoredErrs, 9); !errors.Is(err, errErasureWriteQuorum) {
|
||||
t.Errorf("expected %s, got %s", errErasureWriteQuorum, err)
|
||||
}
|
||||
}
|
||||
|
||||
// renameDataFailingDisk is a StorageAPI stub returning a fixed error
|
||||
// from RenameData, enough to drive renameData() in unit tests.
|
||||
type renameDataFailingDisk struct {
|
||||
StorageAPI
|
||||
err error
|
||||
undoWrite bool
|
||||
}
|
||||
|
||||
func (d *renameDataFailingDisk) RenameData(context.Context, string, string, FileInfo, string, string, RenameOptions) (RenameDataResp, error) {
|
||||
return RenameDataResp{}, d.err
|
||||
}
|
||||
|
||||
func (d *renameDataFailingDisk) DeleteVersion(_ context.Context, _, _ string, _ FileInfo, _ bool, opts DeleteOptions) error {
|
||||
d.undoWrite = opts.UndoWrite
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestRenameDataCommitMetaMissing verifies that renameData() converts an
|
||||
// in-quorum errCommitMetaMissing into errErasureWriteQuorum (HTTP 503),
|
||||
// while an in-quorum errFileNotFound keeps its meaning so that
|
||||
// PutObject() still reports IncompleteBody for client disconnects.
|
||||
func TestRenameDataCommitMetaMissing(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
failures int
|
||||
diskErr error
|
||||
wantErr error
|
||||
}{
|
||||
{"all_commit_metadata_missing", 16, errCommitMetaMissing, errErasureWriteQuorum},
|
||||
{"quorum_commit_metadata_missing", 9, errCommitMetaMissing, errErasureWriteQuorum},
|
||||
{"neither_side_has_quorum", 8, errCommitMetaMissing, errErasureWriteQuorum},
|
||||
{"successful_write_quorum", 7, errCommitMetaMissing, nil},
|
||||
{"staging_failure", 16, errFileNotFound, errFileNotFound},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
metadata := make([]FileInfo, 16)
|
||||
disks := make([]StorageAPI, 16)
|
||||
for i := range disks {
|
||||
metadata[i] = FileInfo{Deleted: true} // valid without erasure info
|
||||
disk := &renameDataFailingDisk{}
|
||||
if i < tc.failures {
|
||||
disk.err = tc.diskErr
|
||||
}
|
||||
disks[i] = disk
|
||||
}
|
||||
_, _, _, err := renameData(t.Context(), disks, minioMetaTmpBucket, "tmp", metadata, "bucket", "object", 9)
|
||||
if !errors.Is(err, tc.wantErr) {
|
||||
t.Fatalf("expected %v, got %v", tc.wantErr, err)
|
||||
}
|
||||
for i, disk := range disks {
|
||||
// Failed writes undo only the drives that committed. A
|
||||
// successful quorum must never be rolled back.
|
||||
wantUndo := tc.wantErr != nil && i >= tc.failures
|
||||
if got := disk.(*renameDataFailingDisk).undoWrite; got != wantUndo {
|
||||
t.Errorf("disk %d: UndoWrite = %v, want %v", i, got, wantUndo)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHashOrder - test order of ints in array
|
||||
func TestHashOrder(t *testing.T) {
|
||||
testCases := []struct {
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
// Copyright (c) 2026 clown15
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestExtractModTimeFromUploadIDDir(t *testing.T) {
|
||||
now := time.Unix(1792056747, 123456789).UTC()
|
||||
nano := now.UnixNano()
|
||||
uuid := "d16ad42a-6fe5-4a3b-8092-69983142238f"
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
dirName string
|
||||
expectedMod time.Time
|
||||
wantZero bool
|
||||
}{
|
||||
{
|
||||
name: "Modern Plaintext Format (<UUID>x<UnixNano>)",
|
||||
dirName: fmt.Sprintf("%sx%d", uuid, nano),
|
||||
expectedMod: time.Unix(0, nano),
|
||||
wantZero: false,
|
||||
},
|
||||
{
|
||||
// This valid UUID decodes to bytes containing 'x'. Checking
|
||||
// for that byte before parsing plaintext loses the timestamp.
|
||||
name: "Plaintext Also Decodes To Base64 Containing x",
|
||||
dirName: fmt.Sprintf("4eacd477-e733-4851-b821-6fd42785ff87x%d", nano),
|
||||
expectedMod: now,
|
||||
},
|
||||
{
|
||||
name: "Legacy Format Without Timestamp (<UUID>)",
|
||||
dirName: uuid,
|
||||
expectedMod: time.Time{},
|
||||
wantZero: true,
|
||||
},
|
||||
{
|
||||
name: "Base64 Encoded Format",
|
||||
dirName: base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%sx%d", uuid, nano))),
|
||||
expectedMod: time.Unix(0, nano),
|
||||
wantZero: false,
|
||||
},
|
||||
{
|
||||
name: "Encoded Format With Deployment Prefix",
|
||||
dirName: base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%s.%sx%d", uuid, uuid, nano))),
|
||||
expectedMod: now,
|
||||
},
|
||||
{
|
||||
name: "Malformed Suffix (<UUID>xinvalid)",
|
||||
dirName: fmt.Sprintf("%sxnotanumber", uuid),
|
||||
expectedMod: time.Time{},
|
||||
wantZero: true,
|
||||
},
|
||||
{
|
||||
name: "Timestamp Overflow",
|
||||
dirName: uuid + "x9223372036854775808",
|
||||
wantZero: true,
|
||||
},
|
||||
{
|
||||
name: "Malformed Plaintext UUID",
|
||||
dirName: fmt.Sprintf("not-a-uuidx%d", nano),
|
||||
wantZero: true,
|
||||
},
|
||||
{
|
||||
name: "Empty Directory Name",
|
||||
dirName: "",
|
||||
expectedMod: time.Time{},
|
||||
wantZero: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := extractModTimeFromUploadIDDir(tc.dirName)
|
||||
if tc.wantZero {
|
||||
if !got.IsZero() {
|
||||
t.Fatalf("expected zero time for %q, got %v", tc.dirName, got)
|
||||
}
|
||||
return
|
||||
}
|
||||
if !got.Equal(tc.expectedMod) {
|
||||
t.Fatalf("extractModTimeFromUploadIDDir(%q) = %v; want %v", tc.dirName, got, tc.expectedMod)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type multipartCleanupDisk struct {
|
||||
StorageAPI
|
||||
root string
|
||||
modTime time.Time
|
||||
readErr error
|
||||
reads int
|
||||
}
|
||||
|
||||
func (d *multipartCleanupDisk) Endpoint() Endpoint {
|
||||
return Endpoint{URL: &url.URL{Path: d.root}}
|
||||
}
|
||||
|
||||
func (d *multipartCleanupDisk) ReadVersion(context.Context, string, string, string, string, ReadOptions) (FileInfo, error) {
|
||||
d.reads++
|
||||
return FileInfo{ModTime: d.modTime}, d.readErr
|
||||
}
|
||||
|
||||
func (d *multipartCleanupDisk) StatVol(ctx context.Context, volume string) (VolInfo, error) {
|
||||
return (&xlStorage{drivePath: d.root}).StatVol(ctx, volume)
|
||||
}
|
||||
|
||||
func TestCleanupStaleUploadsOnDisk(t *testing.T) {
|
||||
const uuid = "d16ad42a-6fe5-4a3b-8092-69983142238f"
|
||||
expiry := globalAPIConfig.getStaleUploadsExpiry()
|
||||
for _, tc := range []struct {
|
||||
name, prefix string
|
||||
age time.Duration
|
||||
legacy bool
|
||||
readErr error
|
||||
wantReads int
|
||||
wantDeleted bool
|
||||
}{
|
||||
{name: "recent", prefix: uuid, age: expiry / 2, readErr: errFileNotFound},
|
||||
{name: "plaintext_base64_collision", prefix: "4eacd477-e733-4851-b821-6fd42785ff87", age: expiry / 2, readErr: errFileNotFound},
|
||||
{name: "expired", prefix: uuid, age: 2 * expiry, readErr: errFileNotFound, wantDeleted: true},
|
||||
{name: "legacy_recent", prefix: uuid, age: expiry / 2, legacy: true, wantReads: 1},
|
||||
{name: "legacy_expired", prefix: uuid, age: 2 * expiry, legacy: true, wantReads: 1, wantDeleted: true},
|
||||
{name: "legacy_metadata_missing", prefix: uuid, age: 2 * expiry, legacy: true, readErr: errFileNotFound, wantReads: 1},
|
||||
{name: "malformed_metadata_missing", prefix: "not-a-uuid", age: 2 * expiry, readErr: errFileNotFound, wantReads: 1},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
modTime := time.Now().Add(-tc.age)
|
||||
id := tc.prefix
|
||||
if !tc.legacy {
|
||||
id = fmt.Sprintf("%sx%d", tc.prefix, modTime.UnixNano())
|
||||
}
|
||||
disk := &multipartCleanupDisk{root: t.TempDir(), modTime: modTime, readErr: tc.readErr}
|
||||
uploadDir := filepath.Join(disk.root, minioMetaMultipartBucket, "sha", id)
|
||||
for _, dir := range []string{uploadDir, filepath.Join(disk.root, minioMetaTmpDeletedBucket)} {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// An old parent must not bypass a skipped upload's age or read error.
|
||||
if err := os.Chtimes(filepath.Dir(uploadDir), modTime, modTime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk)
|
||||
if disk.reads != tc.wantReads {
|
||||
t.Errorf("ReadVersion calls = %d, want %d", disk.reads, tc.wantReads)
|
||||
}
|
||||
_, err := os.Stat(uploadDir)
|
||||
if tc.wantDeleted {
|
||||
if !os.IsNotExist(err) {
|
||||
t.Errorf("expired upload must be removed: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Dir(uploadDir)); !os.IsNotExist(err) {
|
||||
t.Errorf("empty hash directory must be removed in the same pass: %v", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Errorf("upload must be preserved: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupStaleUploadsEmptyHashDirectory(t *testing.T) {
|
||||
for _, age := range []time.Duration{0, 2 * globalAPIConfig.getStaleUploadsExpiry()} {
|
||||
t.Run(age.String(), func(t *testing.T) {
|
||||
disk := &multipartCleanupDisk{root: t.TempDir()}
|
||||
shaDir := filepath.Join(disk.root, minioMetaMultipartBucket, "sha")
|
||||
for _, dir := range []string{shaDir, filepath.Join(disk.root, minioMetaTmpDeletedBucket)} {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
modTime := time.Now().Add(-age)
|
||||
if err := os.Chtimes(shaDir, modTime, modTime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk)
|
||||
if _, err := os.Stat(shaDir); !os.IsNotExist(err) {
|
||||
t.Fatalf("empty hash directory must be removed regardless of age: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupStaleUploadsSharedHashDirectory(t *testing.T) {
|
||||
disk := &multipartCleanupDisk{root: t.TempDir(), readErr: errFileNotFound}
|
||||
shaDir := filepath.Join(disk.root, minioMetaMultipartBucket, "sha")
|
||||
now := time.Now()
|
||||
expiredID := fmt.Sprintf("d16ad42a-6fe5-4a3b-8092-69983142238fx%d", now.Add(-2*globalAPIConfig.getStaleUploadsExpiry()).UnixNano())
|
||||
recentID := fmt.Sprintf("4eacd477-e733-4851-b821-6fd42785ff87x%d", now.UnixNano())
|
||||
for _, id := range []string{expiredID, recentID} {
|
||||
if err := os.MkdirAll(filepath.Join(shaDir, id), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
partPath := filepath.Join(shaDir, recentID, "part.1")
|
||||
const part = "recent upload part"
|
||||
if err := os.WriteFile(partPath, []byte(part), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk)
|
||||
if _, err := os.Stat(filepath.Join(shaDir, expiredID)); !os.IsNotExist(err) {
|
||||
t.Errorf("expired upload must be removed: %v", err)
|
||||
}
|
||||
if got, err := os.ReadFile(partPath); err != nil || string(got) != part {
|
||||
t.Errorf("recent upload must remain intact: got %q, err %v", got, err)
|
||||
}
|
||||
}
|
||||
@@ -253,7 +253,7 @@ func testAPICompleteMultipartFullObjectChecksumMismatch(obj ObjectLayer, instanc
|
||||
if apiErr.Code != "BadDigest" {
|
||||
t.Fatalf("%s: expected BadDigest, got %q", instanceType, apiErr.Code)
|
||||
}
|
||||
if want := "The CRC32 checksum you specified did not match the calculated checksum."; apiErr.Message != want {
|
||||
if want := "The CRC32 you specified did not match the calculated checksum."; apiErr.Message != want {
|
||||
t.Fatalf("%s: expected message %q, got %q", instanceType, want, apiErr.Message)
|
||||
}
|
||||
|
||||
@@ -361,7 +361,7 @@ func testAPICompleteMultipartCompositeChecksumMismatch(obj ObjectLayer, instance
|
||||
if apiErr.Code != "BadDigest" {
|
||||
t.Fatalf("%s: expected BadDigest, got %q", instanceType, apiErr.Code)
|
||||
}
|
||||
if want := "The CRC32 checksum you specified did not match the calculated checksum."; apiErr.Message != want {
|
||||
if want := "The CRC32 you specified did not match the calculated checksum."; apiErr.Message != want {
|
||||
t.Fatalf("%s: expected message %q, got %q", instanceType, want, apiErr.Message)
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err == nil {
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
// Copyright (c) 2026 Ruohang Feng
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Run explicitly with -benchtime=1x. Fixture creation is excluded from timing;
|
||||
// each request still discovers real on-disk metadata. The 16-drive/10k case
|
||||
// deliberately exceeds the existing aggregate directory-entry budget.
|
||||
func BenchmarkMultipartDiagnosticScanCapacity(b *testing.B) {
|
||||
for _, tc := range []struct {
|
||||
drives, uploads int
|
||||
mixed, limited bool
|
||||
}{
|
||||
{4, 1_000, false, false},
|
||||
{4, 10_000, false, false},
|
||||
{4, 10_000, true, false},
|
||||
{16, 1_000, false, false},
|
||||
{16, 10_000, false, true},
|
||||
} {
|
||||
b.Run(fmt.Sprintf("drives=%d/uploads=%d/mixed=%t", tc.drives, tc.uploads, tc.mixed), func(b *testing.B) {
|
||||
obj, dirs, err := prepareErasure(b.Context(), tc.drives)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
b.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
|
||||
const bucket, other = "multipart-capacity", "multipart-unrelated"
|
||||
for _, name := range []string{bucket, other} {
|
||||
if err := z.MakeBucket(b.Context(), name, MakeBucketOptions{}); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
for n := range tc.uploads {
|
||||
target := bucket
|
||||
if tc.mixed && n%2 != 0 {
|
||||
target = other
|
||||
}
|
||||
key := fmt.Sprintf("dir/%05d", n)
|
||||
mp, err := z.NewMultipartUpload(b.Context(), target, key, ObjectOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if tc.mixed && n%10 == 0 {
|
||||
rewriteMultipartIdentity(b, z.serverPools[0].getHashedSet(key), target, key, mp.UploadID, removeMultipartIdentity)
|
||||
}
|
||||
}
|
||||
z.mpCache.Clear()
|
||||
var directoryCalls, metadataCalls atomic.Int64
|
||||
for _, set := range z.serverPools[0].sets {
|
||||
original := set.getDisks
|
||||
set.getDisks = func() []StorageAPI {
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i, disk := range disks {
|
||||
disks[i] = multipartListingCountingDisk{disk, &directoryCalls, &metadataCalls}
|
||||
}
|
||||
return disks
|
||||
}
|
||||
b.Cleanup(func() { set.getDisks = original })
|
||||
}
|
||||
b.ResetTimer()
|
||||
for range b.N {
|
||||
got, err := z.scanMultipartUploadsForTest(b.Context(), bucket, "dir/", "", "", "", 1000)
|
||||
if tc.limited {
|
||||
var limited SlowDown
|
||||
if !errors.As(err, &limited) || len(got.Uploads) != 0 {
|
||||
b.Fatalf("overflow must return SlowDown without a partial page: %+v %v", got, err)
|
||||
}
|
||||
} else if err != nil || len(got.Uploads) != 1000 {
|
||||
b.Fatalf("capacity listing: count=%d err=%v", len(got.Uploads), err)
|
||||
}
|
||||
}
|
||||
b.StopTimer()
|
||||
b.ReportMetric(float64(directoryCalls.Load())/float64(b.N), "ListDir/op")
|
||||
b.ReportMetric(float64(metadataCalls.Load())/float64(b.N), "ReadVersion/op")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The public path is measured with a populated cache as well as after cache
|
||||
// loss. An empty post-restart response alone is not capacity acceptance.
|
||||
func BenchmarkMultipartPublicListingCapacity(b *testing.B) {
|
||||
for _, tc := range []struct {
|
||||
drives, uploads int
|
||||
sameKey bool
|
||||
}{
|
||||
{4, 10_000, false},
|
||||
{16, 3_126, false},
|
||||
{16, 10_000, true},
|
||||
} {
|
||||
b.Run(fmt.Sprintf("drives=%d/uploads=%d/same-key=%t", tc.drives, tc.uploads, tc.sameKey), func(b *testing.B) {
|
||||
obj, dirs, err := prepareErasure(b.Context(), tc.drives)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
b.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
|
||||
const bucket, other = "multipart-public-capacity", "multipart-public-other"
|
||||
for _, name := range []string{bucket, other} {
|
||||
if err := z.MakeBucket(b.Context(), name, MakeBucketOptions{}); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
wantCached, wantExact := 0, 1
|
||||
for n := range tc.uploads {
|
||||
target, key := bucket, fmt.Sprintf("dir/%05d", n)
|
||||
if tc.sameKey {
|
||||
key = "dir/00000"
|
||||
} else if n%2 != 0 {
|
||||
target = other
|
||||
}
|
||||
if target == bucket {
|
||||
wantCached++
|
||||
}
|
||||
if _, err := z.NewMultipartUpload(b.Context(), target, key, ObjectOptions{}); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
if tc.sameKey {
|
||||
wantExact = 1000
|
||||
}
|
||||
if tc.drives == 16 && !tc.sameKey {
|
||||
_, err := z.scanMultipartUploadsForTest(b.Context(), bucket, "", "", "", "", 1000)
|
||||
var limited SlowDown
|
||||
if !errors.As(err, &limited) {
|
||||
b.Fatalf("fixture did not reproduce the previous global budget failure: %v", err)
|
||||
}
|
||||
}
|
||||
var directoryCalls, metadataCalls atomic.Int64
|
||||
for _, set := range z.serverPools[0].sets {
|
||||
original := set.getDisks
|
||||
set.getDisks = func() []StorageAPI {
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i, disk := range disks {
|
||||
disks[i] = multipartListingCountingDisk{disk, &directoryCalls, &metadataCalls}
|
||||
}
|
||||
return disks
|
||||
}
|
||||
b.Cleanup(func() { set.getDisks = original })
|
||||
}
|
||||
measure := func(name, prefix string, want int) {
|
||||
b.Run(name, func(b *testing.B) {
|
||||
directoryCalls.Store(0)
|
||||
metadataCalls.Store(0)
|
||||
b.ResetTimer()
|
||||
for range b.N {
|
||||
got, err := z.ListMultipartUploads(b.Context(), bucket, prefix, "", "", "", 1000)
|
||||
if err != nil || len(got.Uploads) != want {
|
||||
b.Fatalf("public listing: count=%d want=%d err=%v", len(got.Uploads), want, err)
|
||||
}
|
||||
}
|
||||
b.StopTimer()
|
||||
b.ReportMetric(float64(directoryCalls.Load())/float64(b.N), "ListDir/op")
|
||||
b.ReportMetric(float64(metadataCalls.Load())/float64(b.N), "ReadVersion/op")
|
||||
})
|
||||
}
|
||||
measure("populated-cache", "", wantCached)
|
||||
z.mpCache.Clear()
|
||||
measure("lost-cache", "", 0)
|
||||
measure("known-key", "dir/00000", wantExact)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -23,7 +23,6 @@ const multipartScanEntryLimit = 100_000
|
||||
|
||||
var (
|
||||
multipartScanSlots = make(chan struct{}, 2)
|
||||
errMultipartListingLegacy = errors.New("legacy multipart uploads require a coordinated upgrade and drain")
|
||||
errMultipartListingIdentity = errors.New("multipart upload identity is invalid")
|
||||
)
|
||||
|
||||
@@ -133,6 +132,14 @@ func (er erasureObjects) multipartIdentity(fi FileInfo, shaDir string) (string,
|
||||
IsValidObjectPrefix(object) && er.getMultipartSHADir(bucket, object) == shaDir
|
||||
}
|
||||
|
||||
// Only the absence of both fields identifies the historical format. Empty,
|
||||
// partial or invalid identities must not hide a damaged new upload as legacy.
|
||||
func multipartLegacyIdentity(fi FileInfo) bool {
|
||||
_, bucketPresent := fi.Metadata[multipartMetaBucket]
|
||||
_, objectPresent := fi.Metadata[multipartMetaObject]
|
||||
return !bucketPresent && !objectPresent
|
||||
}
|
||||
|
||||
func (er erasureObjects) readMultipartUploadCandidate(s *multipartScan, bucket, candidate string, source StorageAPI) (MultipartInfo, bool, bool, error) {
|
||||
if s.ctx.Err() != nil {
|
||||
return MultipartInfo{}, false, false, SlowDown{}
|
||||
@@ -162,8 +169,8 @@ func (er erasureObjects) readMultipartUploadCandidate(s *multipartScan, bucket,
|
||||
disks := er.getDisks()
|
||||
metadata, errs := readAllFileInfo(s.ctx, disks, bucket, minioMetaMultipartBucket, candidate, "", false, false)
|
||||
if s.preflight {
|
||||
// Readiness is stronger than listing liveness: even one readable legacy
|
||||
// copy must be drained, and an unreadable copy cannot certify readiness.
|
||||
// The diagnostic counts even a single readable legacy copy. It does not
|
||||
// gate listing; unreadable copies still prevent a complete diagnosis.
|
||||
var oldest MultipartInfo
|
||||
legacy := false
|
||||
_, nativeID := multipartUploadTime(uploadUUID)
|
||||
@@ -176,10 +183,11 @@ func (er erasureObjects) readMultipartUploadCandidate(s *multipartScan, bucket,
|
||||
}
|
||||
fi := metadata[i]
|
||||
storedBucket, storedObject, valid := er.multipartIdentity(fi, shaDir)
|
||||
if storedBucket != "" && storedObject != "" && !valid {
|
||||
isLegacy := multipartLegacyIdentity(fi)
|
||||
if !isLegacy && (!valid || !nativeID) {
|
||||
return MultipartInfo{}, false, false, errMultipartListingIdentity
|
||||
}
|
||||
if !valid || !nativeID {
|
||||
if isLegacy {
|
||||
info := multipartUploadInfo(storedBucket, storedObject, uploadUUID, fi.ModTime)
|
||||
if !legacy || info.Initiated.Before(oldest.Initiated) {
|
||||
oldest = info
|
||||
@@ -203,16 +211,29 @@ func (er erasureObjects) readMultipartUploadCandidate(s *multipartScan, bucket,
|
||||
if err != nil {
|
||||
return MultipartInfo{}, false, false, err
|
||||
}
|
||||
// The generic metadata quorum does not compare these identity fields.
|
||||
// Contradictory readable copies cannot certify a historical upload: picking
|
||||
// a copy with missing fields must not silently hide a new-format record.
|
||||
legacy := multipartLegacyIdentity(fi)
|
||||
storedBucket, storedObject, valid := er.multipartIdentity(fi, shaDir)
|
||||
for i, other := range metadata {
|
||||
if errs[i] != nil || !other.IsValid() {
|
||||
continue
|
||||
}
|
||||
if multipartLegacyIdentity(other) != legacy ||
|
||||
(!legacy && (other.Metadata[multipartMetaBucket] != storedBucket || other.Metadata[multipartMetaObject] != storedObject)) {
|
||||
return MultipartInfo{}, false, false, fmt.Errorf("%w: inconsistent copies at %s", errMultipartListingIdentity, candidate)
|
||||
}
|
||||
}
|
||||
info := multipartUploadInfo(storedBucket, storedObject, uploadUUID, fi.ModTime)
|
||||
if storedBucket == "" || storedObject == "" {
|
||||
if legacy {
|
||||
return info, false, true, nil
|
||||
}
|
||||
if !valid {
|
||||
return MultipartInfo{}, false, false, fmt.Errorf("%w: %s", errMultipartListingIdentity, candidate)
|
||||
}
|
||||
if _, ok := multipartUploadTime(uploadUUID); !ok {
|
||||
return info, false, true, nil
|
||||
return MultipartInfo{}, false, false, fmt.Errorf("%w: %s", errMultipartListingIdentity, candidate)
|
||||
}
|
||||
if bucket != "" && bucket != storedBucket {
|
||||
return MultipartInfo{}, false, false, nil
|
||||
@@ -220,7 +241,7 @@ func (er erasureObjects) readMultipartUploadCandidate(s *multipartScan, bucket,
|
||||
return info, true, false, nil
|
||||
}
|
||||
|
||||
func (er erasureObjects) scanMultipartUploads(s *multipartScan, bucket string, poolIdx, setIdx int) ([]MultipartInfo, bool, error) {
|
||||
func (er erasureObjects) scanMultipartUploads(s *multipartScan, bucket string, poolIdx, setIdx int) ([]MultipartInfo, error) {
|
||||
disks := er.getDisks()
|
||||
report := multipartScanSet{Pool: poolIdx, Set: setIdx, Drives: er.setDriveCount}
|
||||
var resultErr error
|
||||
@@ -272,7 +293,7 @@ func (er erasureObjects) scanMultipartUploads(s *multipartScan, bucket string, p
|
||||
resultErr = toObjectErr(errErasureReadQuorum, bucket)
|
||||
}
|
||||
if resultErr != nil {
|
||||
return nil, false, resultErr
|
||||
return nil, resultErr
|
||||
}
|
||||
type candidate struct {
|
||||
path string
|
||||
@@ -325,25 +346,23 @@ dispatch:
|
||||
if s.ctx.Err() != nil {
|
||||
resultErr = SlowDown{}
|
||||
}
|
||||
return uploads, report.LegacyUploads != 0, resultErr
|
||||
return uploads, resultErr
|
||||
}
|
||||
|
||||
// multipartPreflight scans all pools, sets and drives, independent of caches.
|
||||
// A majority suffices for normal listing; upgrade readiness requires every
|
||||
// drive to have been inspected. The operator must also upgrade all writers.
|
||||
// A majority suffices for normal listing; a complete diagnosis requires every
|
||||
// drive to have been inspected. Ready describes the absence of legacy copies,
|
||||
// not whether listing is enabled.
|
||||
func (z *erasureServerPools) multipartPreflight(ctx context.Context) (multipartPreflightReport, error) {
|
||||
s, err := startMultipartScan(ctx, true)
|
||||
if err != nil {
|
||||
return multipartPreflightReport{}, err
|
||||
}
|
||||
defer s.close()
|
||||
report := multipartPreflightReport{Complete: true, Mode: "strict"}
|
||||
if globalAPIConfig.getMultipartListingLegacy() {
|
||||
report.Mode = "legacy"
|
||||
}
|
||||
report := multipartPreflightReport{Complete: true, Mode: "legacy"}
|
||||
for p, pool := range z.serverPools {
|
||||
for i, set := range pool.sets {
|
||||
_, _, _ = set.scanMultipartUploads(s, "", p, i)
|
||||
_, _ = set.scanMultipartUploads(s, "", p, i)
|
||||
}
|
||||
}
|
||||
report.Sets = s.sets
|
||||
|
||||
@@ -22,16 +22,59 @@ import (
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
)
|
||||
|
||||
// Check the real handler and storage path, including a successful abort between
|
||||
// pages. Choose IDs increasing in both initiation time and lexical order so the
|
||||
// failure does not depend on differing interpretations of S3 marker ordering.
|
||||
func TestMultipartListingAbortBetweenHTTPPages(t *testing.T) {
|
||||
// The persistent scanner is retained for diagnostics. This test-only wrapper
|
||||
// preserves its pagination coverage without exposing it through public listing.
|
||||
func (z *erasureServerPools) scanMultipartUploadsForTest(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
|
||||
if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil {
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
return ListMultipartsInfo{}, toObjectErr(err, bucket)
|
||||
}
|
||||
scan, err := startMultipartScan(ctx, false)
|
||||
if err != nil {
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
defer scan.close()
|
||||
|
||||
var uploads []MultipartInfo
|
||||
for idx, pool := range z.serverPools {
|
||||
if z.IsSuspended(idx) {
|
||||
continue
|
||||
}
|
||||
poolUploads, err := pool.scanMultipartUploadsForTest(scan, bucket, idx)
|
||||
if err != nil {
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
uploads = append(uploads, poolUploads...)
|
||||
}
|
||||
|
||||
// Confirmed historical records lack a recoverable bucket/key and are
|
||||
// omitted by the scan. They do not block listing identifiable uploads.
|
||||
return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
|
||||
}
|
||||
|
||||
func (s *erasureSets) scanMultipartUploadsForTest(scan *multipartScan, bucket string, poolIdx int) ([]MultipartInfo, error) {
|
||||
var uploads []MultipartInfo
|
||||
for i, set := range s.sets {
|
||||
setUploads, err := set.scanMultipartUploads(scan, bucket, poolIdx, i)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
uploads = append(uploads, setUploads...)
|
||||
}
|
||||
return uploads, nil
|
||||
}
|
||||
|
||||
// Check that a successful HTTP abort removes the cached upload. The restored
|
||||
// cache path still ignores page limits; this is a documented legacy limitation.
|
||||
func TestMultipartListingAbortHTTPUpdatesCache(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads", "AbortMultipart"}, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
setMultipartListingTestMode(t, false)
|
||||
|
||||
var firstID, secondID string
|
||||
for attempt := 0; attempt < 32; attempt++ {
|
||||
one, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
@@ -81,8 +124,8 @@ func TestMultipartListingAbortBetweenHTTPPages(t *testing.T) {
|
||||
return result
|
||||
}
|
||||
first := list("", "", 1)
|
||||
if len(first.Uploads) != 1 || first.Uploads[0].UploadID != firstID || !first.IsTruncated {
|
||||
t.Fatalf("unexpected first page: %+v", first)
|
||||
if len(first.Uploads) != 3 || first.Uploads[0].UploadID != firstID || first.IsTruncated {
|
||||
t.Fatalf("unexpected legacy cache view: %+v", first)
|
||||
}
|
||||
rec := request(http.MethodDelete, getAbortMultipartUploadURL("", bucket, "a", firstID))
|
||||
if rec.Code != http.StatusNoContent {
|
||||
@@ -93,12 +136,12 @@ func TestMultipartListingAbortBetweenHTTPPages(t *testing.T) {
|
||||
for _, upload := range rest.Uploads {
|
||||
keys = append(keys, upload.Key)
|
||||
}
|
||||
t.Logf("GET first page=200; DELETE marker=204; GET next page=200 keys=%v truncated=%v", keys, rest.IsTruncated)
|
||||
t.Logf("GET cache=200; DELETE upload=204; GET cache=200 keys=%v truncated=%v", keys, rest.IsTruncated)
|
||||
if _, err := z.GetMultipartInfo(t.Context(), bucket, "a", secondID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("remaining upload is not valid: %v", err)
|
||||
}
|
||||
if len(rest.Uploads) != 2 || rest.Uploads[0].UploadID != secondID {
|
||||
t.Fatalf("valid remaining upload for key a is missing from continuation: %+v", rest.Uploads)
|
||||
t.Fatalf("valid remaining upload for key a is missing from cache: %+v", rest.Uploads)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,28 +191,25 @@ func TestMultipartListingLegacyPreflight(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z.mpCache.Clear()
|
||||
t.Run("legacy-upgrade", func(t *testing.T) {
|
||||
setMultipartListingTestMode(t, true)
|
||||
exact, err := z.ListMultipartUploads(t.Context(), other, "old", "", "", "", 10)
|
||||
for _, target := range []string{bucket, other} {
|
||||
got, err := z.scanMultipartUploadsForTest(t.Context(), target, "", "", "", "", 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, exact, "old")
|
||||
const empty = "multipart-legacy-empty"
|
||||
if err := z.MakeBucket(t.Context(), empty, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
if target == bucket {
|
||||
requireMultipartUploadKeys(t, got, "a")
|
||||
} else {
|
||||
requireMultipartUploadKeys(t, got)
|
||||
}
|
||||
got, err := z.ListMultipartUploads(t.Context(), empty, "", "", "", "", 10)
|
||||
if err != nil || len(got.Uploads) != 0 {
|
||||
t.Fatalf("old upload in another bucket broke legacy listing: %+v %v", got, err)
|
||||
}
|
||||
})
|
||||
_, err = z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
|
||||
if !errors.Is(err, errMultipartListingLegacy) {
|
||||
t.Fatalf("old upload in another bucket: %v", err)
|
||||
}
|
||||
// Diagnostic discovery cannot recover an unidentified historical key.
|
||||
exact, err := z.scanMultipartUploadsForTest(t.Context(), other, "old", "", "", "", 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, exact)
|
||||
report, err := z.multipartPreflight(t.Context())
|
||||
if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 {
|
||||
if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 || report.Mode != "legacy" {
|
||||
t.Fatalf("legacy preflight: %+v %v", report, err)
|
||||
}
|
||||
if err = z.AbortMultipartUpload(t.Context(), other, "old", old.UploadID, ObjectOptions{}); err != nil {
|
||||
@@ -179,15 +219,15 @@ func TestMultipartListingLegacyPreflight(t *testing.T) {
|
||||
if err != nil || !report.Ready || report.LegacyUploads != 0 {
|
||||
t.Fatalf("drained preflight: %+v %v", report, err)
|
||||
}
|
||||
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
|
||||
got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, got, "a")
|
||||
}
|
||||
|
||||
func TestMultipartListingIdentityFallback(t *testing.T) {
|
||||
for _, kind := range []string{"old", "corrupt", "read-failure", "wrong-bucket"} {
|
||||
func TestMultipartDiagnosticIdentityFallback(t *testing.T) {
|
||||
for _, kind := range []string{"old", "partial", "corrupt", "read-failure", "wrong-bucket"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
if _, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}); err != nil {
|
||||
@@ -209,8 +249,11 @@ func TestMultipartListingIdentityFallback(t *testing.T) {
|
||||
return FileInfo{}, errFileCorrupt
|
||||
}
|
||||
fi.Metadata = cloneMSS(fi.Metadata)
|
||||
if kind == "old" {
|
||||
if kind == "old" || kind == "partial" {
|
||||
delete(fi.Metadata, multipartMetaBucket)
|
||||
if kind == "old" {
|
||||
delete(fi.Metadata, multipartMetaObject)
|
||||
}
|
||||
} else {
|
||||
fi.Metadata[multipartMetaBucket] = "another-bucket"
|
||||
}
|
||||
@@ -220,7 +263,7 @@ func TestMultipartListingIdentityFallback(t *testing.T) {
|
||||
}
|
||||
return disks
|
||||
}
|
||||
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
|
||||
got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -231,7 +274,7 @@ func TestMultipartListingIdentityFallback(t *testing.T) {
|
||||
|
||||
func TestMultipartAbortPoolsAndRetry(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
setMultipartListingTestMode(t, false)
|
||||
|
||||
mp, err := z.serverPools[1].NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -252,10 +295,12 @@ func TestMultipartAbortPoolsAndRetry(t *testing.T) {
|
||||
if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 {
|
||||
t.Fatalf("unknown pool must not acknowledge cancellation: %v", err)
|
||||
}
|
||||
if _, err := z.serverPools[1].GetMultipartInfo(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("earlier pool error changed the later pool: %v", err)
|
||||
}
|
||||
emptySet.getDisks = original
|
||||
err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{})
|
||||
if _, ok := err.(InvalidUploadID); !ok {
|
||||
t.Fatalf("retry after all pools confirm absence: %v", err)
|
||||
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("retry after pool recovery: %v", err)
|
||||
}
|
||||
mp, err = z.serverPools[1].NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
@@ -289,12 +334,13 @@ func TestMultipartAbortRetryBelowReadQuorum(t *testing.T) {
|
||||
}}
|
||||
return disks
|
||||
}
|
||||
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err == nil {
|
||||
t.Fatal("lost acknowledgement must fail")
|
||||
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("best-effort cleanup must retain read-quorum availability: %v", err)
|
||||
}
|
||||
set.getDisks = original
|
||||
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("one remaining metadata copy prevented retry: %v", err)
|
||||
var invalid InvalidUploadID
|
||||
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); !errors.As(err, &invalid) {
|
||||
t.Fatalf("minority remnant must retain absent-upload result: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -304,7 +350,7 @@ func TestMultipartListingMarkerHTTP(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
setMultipartListingTestMode(t, false)
|
||||
|
||||
for _, tc := range []struct {
|
||||
key, marker string
|
||||
status int
|
||||
@@ -388,7 +434,7 @@ func TestMultipartAbortLateCreateBoundary(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
setMultipartListingTestMode(t, false)
|
||||
|
||||
t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
|
||||
saved := globalStorageClass
|
||||
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 8}})
|
||||
@@ -417,9 +463,7 @@ func TestMultipartAbortLateCreateBoundary(t *testing.T) {
|
||||
}
|
||||
set.getDisks = func() []StorageAPI {
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i := 2; i < 9; i++ {
|
||||
disks[i] = nil
|
||||
}
|
||||
disks[8] = nil
|
||||
return disks
|
||||
}
|
||||
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
@@ -431,7 +475,7 @@ func TestMultipartAbortLateCreateBoundary(t *testing.T) {
|
||||
}
|
||||
}
|
||||
set.getDisks = original
|
||||
_, metadata, err := set.checkUploadIDExists(t.Context(), bucket, "a", mp.UploadID, true)
|
||||
_, metadata, err := set.checkUploadIDExists(t.Context(), bucket, "a", mp.UploadID, false)
|
||||
if err != nil {
|
||||
t.Fatalf("late creation boundary changed; review and remove the documented limitation: %v", err)
|
||||
}
|
||||
@@ -441,10 +485,10 @@ func TestMultipartAbortLateCreateBoundary(t *testing.T) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count != 14 {
|
||||
t.Fatalf("expected fourteen resurrected copies, got %d", count)
|
||||
if count != 8 {
|
||||
t.Fatalf("expected eight resurrected copies, got %d", count)
|
||||
}
|
||||
t.Log("KNOWN UNRESOLVED BOUNDARY: seven delayed creation writes plus seven offline copies restore a writable upload after acknowledged cancellation")
|
||||
t.Log("KNOWN UNRESOLVED BOUNDARY: seven delayed creation writes plus one offline copy restore a readable upload after acknowledged cancellation")
|
||||
}
|
||||
|
||||
type multipartListingCountingDisk struct {
|
||||
@@ -468,13 +512,13 @@ func (d multipartListingCountingDisk) ReadVersion(ctx context.Context, original,
|
||||
}
|
||||
|
||||
// Counts storage API calls; this is not a deployment throughput benchmark.
|
||||
func TestMultipartListingScanCosts(t *testing.T) {
|
||||
func TestMultipartDiagnosticScanCosts(t *testing.T) {
|
||||
obj, dirs, err := prepareErasure(t.Context(), 4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
setMultipartListingTestMode(t, false)
|
||||
|
||||
t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
|
||||
const bucket, otherBucket = "r9-scan-target", "r9-scan-unrelated"
|
||||
for _, name := range []string{bucket, otherBucket} {
|
||||
@@ -506,7 +550,7 @@ func TestMultipartListingScanCosts(t *testing.T) {
|
||||
t.Helper()
|
||||
directoryCalls.Store(0)
|
||||
metadataCalls.Store(0)
|
||||
result, err := z.ListMultipartUploads(t.Context(), bucket, "dir/", "", "", "", 1)
|
||||
result, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "dir/", "", "", "", 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -544,7 +588,7 @@ func multipartListingFixture(t *testing.T) (*erasureServerPools, *erasureObjects
|
||||
if err := z.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
setMultipartListingTestMode(t, false)
|
||||
|
||||
return z, z.serverPools[0].getHashedSet("a"), bucket
|
||||
}
|
||||
|
||||
@@ -588,46 +632,7 @@ func TestMultipartListingMissingMarkers(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartListingAbortRecovery(t *testing.T) {
|
||||
for _, offline := range []int{1, 2} {
|
||||
t.Run(fmt.Sprint(offline), func(t *testing.T) {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
original := set.getDisks
|
||||
t.Cleanup(func() { set.getDisks = original })
|
||||
set.getDisks = func() []StorageAPI {
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i := 0; i < offline; i++ {
|
||||
disks[i] = nil
|
||||
}
|
||||
return disks
|
||||
}
|
||||
err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{})
|
||||
if offline == 1 && err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if offline == 2 && (err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503) {
|
||||
t.Fatalf("two offline drives must not acknowledge cancellation: %v", err)
|
||||
}
|
||||
set.getDisks = original
|
||||
if offline == 2 {
|
||||
// Retry a partially completed deletion after the original disks return.
|
||||
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 100)
|
||||
if err != nil || len(got.Uploads) != 0 {
|
||||
t.Fatalf("canceled upload reappeared: %+v %v", got, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartListingCoverage(t *testing.T) {
|
||||
func TestMultipartDiagnosticCoverage(t *testing.T) {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
@@ -643,12 +648,12 @@ func TestMultipartListingCoverage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return []StorageAPI{disks[0], disks[1], nil, nil} }
|
||||
_, err = z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
|
||||
_, err = z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
|
||||
if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 {
|
||||
t.Fatalf("incomplete discovery returned success: %v", err)
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return []StorageAPI{nil, disks[1], disks[2], disks[3]} }
|
||||
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
|
||||
got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -659,7 +664,7 @@ func TestMultipartListingCoverage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartListingBudgetAndAdmission(t *testing.T) {
|
||||
func TestMultipartDiagnosticBudgetAndAdmission(t *testing.T) {
|
||||
_, set, bucket := multipartListingFixture(t)
|
||||
if _, err := set.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -670,7 +675,7 @@ func TestMultipartListingBudgetAndAdmission(t *testing.T) {
|
||||
}
|
||||
defer scan.close()
|
||||
scan.remaining.Store(1)
|
||||
_, _, err = set.scanMultipartUploads(scan, bucket, 0, 0)
|
||||
_, err = set.scanMultipartUploads(scan, bucket, 0, 0)
|
||||
var limited SlowDown
|
||||
if !errors.As(err, &limited) {
|
||||
t.Fatalf("budget: %v", err)
|
||||
@@ -689,13 +694,13 @@ func TestMultipartListingBudgetAndAdmission(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartListingAdmissionHTTP(t *testing.T) {
|
||||
func TestMultipartPublicListingAdmissionHTTP(t *testing.T) {
|
||||
z, _, _ := multipartListingFixture(t)
|
||||
bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads"}, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
setMultipartListingTestMode(t, false)
|
||||
|
||||
for range cap(multipartScanSlots) {
|
||||
scan, err := startMultipartScan(t.Context(), false)
|
||||
if err != nil {
|
||||
@@ -711,12 +716,12 @@ func TestMultipartListingAdmissionHTTP(t *testing.T) {
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
var response APIErrorResponse
|
||||
var response ListMultipartUploadsResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec.Code != http.StatusServiceUnavailable || response.Code != "SlowDown" {
|
||||
t.Fatalf("admission returned %d %s", rec.Code, rec.Body.String())
|
||||
if rec.Code != http.StatusOK || len(response.Uploads) != 0 {
|
||||
t.Fatalf("diagnostic admission blocked public listing: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -743,12 +748,12 @@ func TestMultipartListingPreflightMinorityLegacy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
report, err := z.multipartPreflight(t.Context())
|
||||
if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 {
|
||||
if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 || report.Mode != "legacy" {
|
||||
t.Fatalf("minority legacy copy must prevent readiness: %+v %v", report, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartListingCancellationRetainsAdmission(t *testing.T) {
|
||||
func TestMultipartDiagnosticCancellationRetainsAdmission(t *testing.T) {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
for i := range 40 {
|
||||
if _, err := z.NewMultipartUpload(t.Context(), bucket, fmt.Sprintf("key-%02d", i), ObjectOptions{}); err != nil {
|
||||
@@ -787,7 +792,7 @@ func TestMultipartListingCancellationRetainsAdmission(t *testing.T) {
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := z.ListMultipartUploads(ctx, bucket, "", "", "", "", 10)
|
||||
_, err := z.scanMultipartUploadsForTest(ctx, bucket, "", "", "", "", 10)
|
||||
done <- err
|
||||
}()
|
||||
for range 16 {
|
||||
|
||||
@@ -12,41 +12,13 @@ import (
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/config/api"
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
"github.com/minio/minio/internal/grid"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
)
|
||||
|
||||
func setMultipartListingTestMode(t *testing.T, legacy bool) {
|
||||
t.Helper()
|
||||
globalAPIConfig.mu.Lock()
|
||||
previous := globalAPIConfig.multipartListingStrict
|
||||
globalAPIConfig.multipartListingStrict = !legacy
|
||||
globalAPIConfig.mu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
globalAPIConfig.mu.Lock()
|
||||
globalAPIConfig.multipartListingStrict = previous
|
||||
globalAPIConfig.mu.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
func TestMultipartListingDefaultMode(t *testing.T) {
|
||||
var uninitialized apiConfig
|
||||
if !uninitialized.getMultipartListingLegacy() {
|
||||
t.Fatal("runtime zero value enabled strict mode before config initialization")
|
||||
}
|
||||
for _, mode := range []string{"", "legacy", "strict"} {
|
||||
var local apiConfig
|
||||
local.init(api.Config{RequestsMax: 1, MultipartListing: mode}, []int{4}, false)
|
||||
if local.getMultipartListingLegacy() != (mode != "strict") {
|
||||
t.Fatalf("unexpected mode for %q", mode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAbortLegacyAvailability(t *testing.T) {
|
||||
func TestMultipartAbortDefaultAvailability(t *testing.T) {
|
||||
for _, drives := range []int{4, 16} {
|
||||
for _, offline := range []int{0, 1, drives / 2} {
|
||||
t.Run(fmt.Sprintf("drives=%d/offline=%d", drives, offline), func(t *testing.T) {
|
||||
@@ -56,7 +28,7 @@ func TestMultipartAbortLegacyAvailability(t *testing.T) {
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
|
||||
setMultipartListingTestMode(t, true)
|
||||
|
||||
previous := globalStorageClass
|
||||
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: drives / 2}})
|
||||
t.Cleanup(func() { globalStorageClass.Update(previous) })
|
||||
@@ -91,11 +63,11 @@ func TestMultipartAbortLegacyAvailability(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAbortLegacyPoolOrder(t *testing.T) {
|
||||
func TestMultipartAbortDefaultPoolOrder(t *testing.T) {
|
||||
for _, owner := range []int{0, 1} {
|
||||
t.Run(fmt.Sprintf("owner=%d", owner), func(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
setMultipartListingTestMode(t, true)
|
||||
|
||||
mp, err := z.serverPools[owner].NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -106,7 +78,7 @@ func TestMultipartAbortLegacyPoolOrder(t *testing.T) {
|
||||
emptySet.getDisks = func() []StorageAPI { return make([]StorageAPI, emptySet.setDriveCount) }
|
||||
err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{})
|
||||
if owner == 0 && err != nil {
|
||||
t.Fatalf("unrelated later pool blocked legacy cancellation: %v", err)
|
||||
t.Fatalf("unrelated later pool blocked cancellation: %v", err)
|
||||
}
|
||||
if owner == 1 {
|
||||
if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 {
|
||||
@@ -124,123 +96,56 @@ func TestMultipartAbortLegacyPoolOrder(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAbortMinorityLegacyCleanup(t *testing.T) {
|
||||
for _, failDelete := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("delete-fails=%v", failDelete), func(t *testing.T) {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "old", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := set.getUploadIDDir(bucket, "old", mp.UploadID)
|
||||
original := set.getDisks
|
||||
disks := original()
|
||||
fi, err := disks[0].ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, path, "", ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
delete(fi.Metadata, multipartMetaBucket)
|
||||
delete(fi.Metadata, multipartMetaObject)
|
||||
if err := disks[0].WriteMetadata(t.Context(), bucket, minioMetaMultipartBucket, path, fi); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, d := range disks[1:] {
|
||||
if err := d.Delete(t.Context(), minioMetaMultipartBucket, path, DeleteOptions{Recursive: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
report, err := z.multipartPreflight(t.Context())
|
||||
if err != nil || report.Ready || report.LegacyUploads != 1 {
|
||||
t.Fatalf("invalid legacy fixture: %+v %v", report, err)
|
||||
}
|
||||
if failDelete {
|
||||
set.getDisks = func() []StorageAPI {
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
wrapped[0] = multipartListingFaultDisk{StorageAPI: disks[0], delete: func(context.Context, string, string, DeleteOptions) error { return errFaultyDisk }}
|
||||
return wrapped
|
||||
}
|
||||
t.Cleanup(func() { set.getDisks = original })
|
||||
err := z.AbortMultipartUpload(t.Context(), bucket, "old", mp.UploadID, ObjectOptions{})
|
||||
if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 {
|
||||
t.Fatalf("empty drives masked failed deletion of the observed replica: %v", err)
|
||||
}
|
||||
if _, present := z.mpCache.Load(mp.UploadID); !present {
|
||||
t.Fatal("failed cancellation evicted upload from cache")
|
||||
}
|
||||
set.getDisks = original
|
||||
}
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "old", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
report, err = z.multipartPreflight(t.Context())
|
||||
if err != nil || !report.Ready || report.LegacyUploads != 0 {
|
||||
t.Fatalf("acknowledged cleanup left online legacy remnants: %+v %v", report, err)
|
||||
}
|
||||
if _, present := z.mpCache.Load(mp.UploadID); present {
|
||||
t.Fatal("successful cancellation retained cache entry")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAbortWrongTargetKeepsCache(t *testing.T) {
|
||||
for _, legacy := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("legacy=%v", legacy), func(t *testing.T) {
|
||||
z, _, bucket := multipartListingFixture(t)
|
||||
setMultipartListingTestMode(t, legacy)
|
||||
const other = "multipart-other-bucket"
|
||||
if err := z.MakeBucket(t.Context(), other, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "valid-key", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, target := range [][2]string{{bucket, "wrong-key"}, {other, "valid-key"}} {
|
||||
err := z.AbortMultipartUpload(t.Context(), target[0], target[1], mp.UploadID, ObjectOptions{})
|
||||
var invalid InvalidUploadID
|
||||
if !errors.As(err, &invalid) {
|
||||
t.Fatalf("wrong target result: %v", err)
|
||||
}
|
||||
if _, present := z.mpCache.Load(mp.UploadID); !present {
|
||||
t.Fatal("wrong-target abort evicted another upload")
|
||||
}
|
||||
if _, err := z.GetMultipartInfo(t.Context(), bucket, "valid-key", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("wrong-target abort harmed valid upload: %v", err)
|
||||
}
|
||||
}
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "valid-key", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var invalid InvalidUploadID
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "valid-key", mp.UploadID, ObjectOptions{}); !errors.As(err, &invalid) {
|
||||
t.Fatalf("repeat abort: %v", err)
|
||||
}
|
||||
})
|
||||
z, _, bucket := multipartListingFixture(t)
|
||||
|
||||
const other = "multipart-other-bucket"
|
||||
if err := z.MakeBucket(t.Context(), other, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "valid-key", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, target := range [][2]string{{bucket, "wrong-key"}, {other, "valid-key"}} {
|
||||
err := z.AbortMultipartUpload(t.Context(), target[0], target[1], mp.UploadID, ObjectOptions{})
|
||||
var invalid InvalidUploadID
|
||||
if !errors.As(err, &invalid) {
|
||||
t.Fatalf("wrong target result: %v", err)
|
||||
}
|
||||
if _, present := z.mpCache.Load(mp.UploadID); !present {
|
||||
t.Fatal("wrong-target abort evicted another upload")
|
||||
}
|
||||
if _, err := z.GetMultipartInfo(t.Context(), bucket, "valid-key", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("wrong-target abort harmed valid upload: %v", err)
|
||||
}
|
||||
}
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "valid-key", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var invalid InvalidUploadID
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "valid-key", mp.UploadID, ObjectOptions{}); !errors.As(err, &invalid) {
|
||||
t.Fatalf("repeat abort: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAbortRejectsUnsafeID(t *testing.T) {
|
||||
for _, legacy := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("legacy=%v", legacy), func(t *testing.T) {
|
||||
z, _, bucket := multipartListingFixture(t)
|
||||
setMultipartListingTestMode(t, legacy)
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "keep", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, suffix := range []string{".", "..", "../other", "a/b", "a\\b", ""} {
|
||||
id := base64.RawURLEncoding.EncodeToString([]byte("deployment." + suffix))
|
||||
err := z.AbortMultipartUpload(t.Context(), bucket, "keep", id, ObjectOptions{})
|
||||
var invalid InvalidUploadID
|
||||
if !errors.As(err, &invalid) {
|
||||
t.Fatalf("unsafe ID accepted: suffix=%q err=%v", suffix, err)
|
||||
}
|
||||
if _, err := z.GetMultipartInfo(t.Context(), bucket, "keep", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("valid upload harmed by rejected ID: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
z, _, bucket := multipartListingFixture(t)
|
||||
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "keep", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, suffix := range []string{".", "..", "../other", "a/b", "a\\b", ""} {
|
||||
id := base64.RawURLEncoding.EncodeToString([]byte("deployment." + suffix))
|
||||
err := z.AbortMultipartUpload(t.Context(), bucket, "keep", id, ObjectOptions{})
|
||||
var invalid InvalidUploadID
|
||||
if !errors.As(err, &invalid) {
|
||||
t.Fatalf("unsafe ID accepted: suffix=%q err=%v", suffix, err)
|
||||
}
|
||||
if _, err := z.GetMultipartInfo(t.Context(), bucket, "keep", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("valid upload harmed by rejected ID: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -275,34 +180,30 @@ func TestMultipartAbortPeerNotification(t *testing.T) {
|
||||
set.nsMutex = newNSLock(true)
|
||||
set.getLockers = func() ([]dsync.NetLocker, string) { return lockers, "multipart-test" }
|
||||
t.Cleanup(func() { set.nsMutex, set.getLockers = oldMutex, oldLockers })
|
||||
for _, legacy := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("legacy=%v", legacy), func(t *testing.T) {
|
||||
setMultipartListingTestMode(t, legacy)
|
||||
for range 4 {
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "valid", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := notifications.Load()
|
||||
var invalid InvalidUploadID
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "wrong", mp.UploadID, ObjectOptions{}); !errors.As(err, &invalid) {
|
||||
t.Fatalf("wrong target: %v", err)
|
||||
}
|
||||
if notifications.Load() != before {
|
||||
t.Fatal("wrong-target abort sent a destructive peer notification")
|
||||
}
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "valid", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if notifications.Load() != before+1 {
|
||||
t.Fatal("successful abort lost peer notification after unlocking")
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
for range 4 {
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "valid", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := notifications.Load()
|
||||
var invalid InvalidUploadID
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "wrong", mp.UploadID, ObjectOptions{}); !errors.As(err, &invalid) {
|
||||
t.Fatalf("wrong target: %v", err)
|
||||
}
|
||||
if notifications.Load() != before {
|
||||
t.Fatal("wrong-target abort sent a destructive peer notification")
|
||||
}
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "valid", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if notifications.Load() != before+1 {
|
||||
t.Fatal("successful abort lost peer notification after unlocking")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAbortStrictMajority(t *testing.T) {
|
||||
func TestMultipartAbortBestEffortCleanup(t *testing.T) {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
@@ -317,15 +218,19 @@ func TestMultipartAbortStrictMajority(t *testing.T) {
|
||||
}
|
||||
t.Cleanup(func() { set.getDisks = original })
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("ordinary majority cancellation was tightened: %v", err)
|
||||
t.Fatalf("read-quorum cancellation was tightened: %v", err)
|
||||
}
|
||||
// One failed deletion is tolerated in the ordinary majority path. Retrying
|
||||
// after recovery must also clean the now-minority remnant.
|
||||
// Best-effort deletion may leave a remnant. Retrying after recovery cleans
|
||||
// it, but the read quorum no longer recognizes it as an active upload.
|
||||
if _, err := disks[0].ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, set.getUploadIDDir(bucket, "a", mp.UploadID), "", ReadOptions{}); err != nil {
|
||||
t.Fatalf("fault fixture did not leave a remnant: %v", err)
|
||||
}
|
||||
set.getDisks = original
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
var invalid InvalidUploadID
|
||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); !errors.As(err, &invalid) {
|
||||
t.Fatalf("minority retry must preserve default absent-upload result: %v", err)
|
||||
}
|
||||
if _, err := disks[0].ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, set.getUploadIDDir(bucket, "a", mp.UploadID), "", ReadOptions{}); !errors.Is(err, errFileNotFound) {
|
||||
t.Fatalf("best-effort retry did not clean remnant: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
// Copyright (c) 2026 Ruohang Feng
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type multipartRootScanGuard struct {
|
||||
StorageAPI
|
||||
roots *atomic.Int64
|
||||
}
|
||||
|
||||
func (d multipartRootScanGuard) ListDir(ctx context.Context, original, volume, dir string, count int) ([]string, error) {
|
||||
if volume == minioMetaMultipartBucket && dir == "" {
|
||||
d.roots.Add(1)
|
||||
return nil, errFaultyDisk
|
||||
}
|
||||
return d.StorageAPI.ListDir(ctx, original, volume, dir, count)
|
||||
}
|
||||
|
||||
// A busy or unreadable diagnostic namespace must not become a prerequisite for
|
||||
// ordinary listing, including the lower-level exact-object entry points.
|
||||
func TestMultipartPublicListingIndependentOfDiagnostics(t *testing.T) {
|
||||
for _, busy := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "unreadable-root", true: "busy-diagnostics"}[busy], func(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads"}, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const key = "dir/known"
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, key, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var roots atomic.Int64
|
||||
for _, pool := range z.serverPools {
|
||||
for _, set := range pool.sets {
|
||||
original := set.getDisks
|
||||
set.getDisks = func() []StorageAPI {
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i, disk := range disks {
|
||||
if disk != nil {
|
||||
disks[i] = multipartRootScanGuard{disk, &roots}
|
||||
}
|
||||
}
|
||||
return disks
|
||||
}
|
||||
t.Cleanup(func() { set.getDisks = original })
|
||||
}
|
||||
}
|
||||
if busy {
|
||||
for range cap(multipartScanSlots) {
|
||||
scan, err := startMultipartScan(t.Context(), true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer scan.close()
|
||||
}
|
||||
}
|
||||
for _, prefix := range []string{"", key, "dir/"} {
|
||||
req, err := newTestSignedRequestV4(http.MethodGet,
|
||||
getListMultipartUploadsURLWithParams("", bucket, prefix, "", "", "", "1"),
|
||||
0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("prefix %q: HTTP %d %s", prefix, rec.Code, rec.Body.String())
|
||||
}
|
||||
var got ListMultipartUploadsResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if prefix == "dir/" {
|
||||
if len(got.Uploads) != 0 {
|
||||
t.Fatalf("partial prefix must retain the documented exact-key limitation: %+v", got)
|
||||
}
|
||||
} else if len(got.Uploads) != 1 || got.Uploads[0].UploadID != mp.UploadID {
|
||||
t.Fatalf("known upload missing: %+v", got)
|
||||
}
|
||||
}
|
||||
for _, pool := range z.serverPools {
|
||||
for _, list := range []func(context.Context, string, string, string, string, string, int) (ListMultipartsInfo, error){
|
||||
pool.ListMultipartUploads, pool.getHashedSet(key).ListMultipartUploads,
|
||||
} {
|
||||
if _, err := list(t.Context(), bucket, key, "", "", "", 10); err != nil {
|
||||
t.Fatalf("lower-level exact listing: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if roots.Load() != 0 {
|
||||
t.Fatalf("public listing attempted %d global multipart scans", roots.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
// Copyright (c) 2026 Ruohang Feng
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func rewriteMultipartIdentity(t testing.TB, set *erasureObjects, bucket, key, id string, change func(map[string]string)) {
|
||||
t.Helper()
|
||||
fi, metadata, err := set.checkUploadIDExists(t.Context(), bucket, key, id, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range metadata {
|
||||
change(metadata[i].Metadata)
|
||||
}
|
||||
if _, err := writeAllMetadata(t.Context(), set.getDisks(), bucket, minioMetaMultipartBucket,
|
||||
set.getUploadIDDir(bucket, key, id), metadata, fi.WriteQuorum(set.defaultWQuorum())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func removeMultipartIdentity(metadata map[string]string) {
|
||||
delete(metadata, multipartMetaBucket)
|
||||
delete(metadata, multipartMetaObject)
|
||||
}
|
||||
|
||||
// The retained diagnostic scanner handles mixed records independently of a
|
||||
// stale cache. Public HTTP behavior is tested separately.
|
||||
func TestMultipartDiagnosticMixed(t *testing.T) {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
for _, key := range []string{"dir/old", "dir/a", "dir/sub/b", "outside"} {
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, key, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if key == "dir/old" {
|
||||
rewriteMultipartIdentity(t, set, bucket, key, mp.UploadID, removeMultipartIdentity)
|
||||
}
|
||||
}
|
||||
z.mpCache.Clear()
|
||||
z.mpCache.Store("stale", MultipartInfo{Bucket: bucket, Object: "ghost", UploadID: "stale"})
|
||||
list := func(prefix, keyMarker, uploadMarker, delimiter string) ListMultipartUploadsResponse {
|
||||
t.Helper()
|
||||
got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, prefix, keyMarker, uploadMarker, delimiter, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return generateListMultipartUploadsResponse(bucket, got, "")
|
||||
}
|
||||
var keys []string
|
||||
keyMarker, uploadMarker := "", ""
|
||||
for page := 0; ; page++ {
|
||||
if page >= 4 {
|
||||
t.Fatal("pagination did not terminate")
|
||||
}
|
||||
got := list("", keyMarker, uploadMarker, "")
|
||||
if len(got.Uploads) != 1 {
|
||||
t.Fatalf("page size: %+v", got)
|
||||
}
|
||||
keys = append(keys, got.Uploads[0].Key)
|
||||
if !got.IsTruncated {
|
||||
break
|
||||
}
|
||||
keyMarker, uploadMarker = got.NextKeyMarker, got.NextUploadIDMarker
|
||||
}
|
||||
if !slices.Equal(keys, []string{"dir/a", "dir/sub/b", "outside"}) {
|
||||
t.Fatalf("mixed full listing: %v", keys)
|
||||
}
|
||||
first := list("dir/", "", "", "/")
|
||||
if len(first.Uploads) != 1 || first.Uploads[0].Key != "dir/a" || !first.IsTruncated {
|
||||
t.Fatalf("prefix first page: %+v", first)
|
||||
}
|
||||
rest := list("dir/", first.NextKeyMarker, first.NextUploadIDMarker, "/")
|
||||
if len(rest.Uploads) != 0 || rest.IsTruncated || len(rest.CommonPrefixes) != 1 || rest.CommonPrefixes[0].Prefix != "dir/sub/" {
|
||||
t.Fatalf("prefix continuation: %+v", rest)
|
||||
}
|
||||
if got := list("dir/old", "", "", ""); len(got.Uploads) != 0 || got.IsTruncated {
|
||||
t.Fatalf("historical exact-looking prefix: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartDiagnosticInvalidIdentity(t *testing.T) {
|
||||
for _, kind := range []string{"missing-bucket", "missing-object", "empty-bucket", "empty-object", "empty-both", "wrong-hash", "read-failure"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rewriteMultipartIdentity(t, set, bucket, "a", mp.UploadID, func(metadata map[string]string) {
|
||||
switch kind {
|
||||
case "missing-bucket":
|
||||
delete(metadata, multipartMetaBucket)
|
||||
case "missing-object":
|
||||
delete(metadata, multipartMetaObject)
|
||||
case "empty-bucket":
|
||||
metadata[multipartMetaBucket] = ""
|
||||
case "empty-object":
|
||||
metadata[multipartMetaObject] = ""
|
||||
case "empty-both":
|
||||
metadata[multipartMetaBucket], metadata[multipartMetaObject] = "", ""
|
||||
case "wrong-hash":
|
||||
metadata[multipartMetaObject] = "wrong-key"
|
||||
case "read-failure":
|
||||
removeMultipartIdentity(metadata)
|
||||
}
|
||||
})
|
||||
if kind == "read-failure" {
|
||||
original := set.getDisks
|
||||
t.Cleanup(func() { set.getDisks = original })
|
||||
set.getDisks = func() []StorageAPI {
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i, disk := range disks {
|
||||
disks[i] = multipartListingFaultDisk{StorageAPI: disk, read: func(context.Context, string, string, string, string, ReadOptions) (FileInfo, error) {
|
||||
return FileInfo{}, errFaultyDisk
|
||||
}}
|
||||
}
|
||||
return disks
|
||||
}
|
||||
}
|
||||
got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
|
||||
if err == nil || len(got.Uploads) != 0 || toAPIError(t.Context(), err).HTTPStatusCode != http.StatusServiceUnavailable {
|
||||
t.Fatalf("invalid/unreadable record returned a successful page: %+v %v", got, err)
|
||||
}
|
||||
if kind != "read-failure" && !errors.Is(err, errMultipartListingIdentity) {
|
||||
t.Fatalf("identity error: %v", err)
|
||||
}
|
||||
report, err := z.multipartPreflight(t.Context())
|
||||
if err != nil || report.Complete || report.Ready || report.LegacyUploads != 0 {
|
||||
t.Fatalf("invalid/unreadable record counted as historical: %+v %v", report, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartListingRestart(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
obj, dirs, err := prepareErasure(ctx, 4)
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cancel(); obj.Shutdown(context.Background()); removeRoots(dirs) })
|
||||
const bucket, key = "multipart-restart", "dir/new"
|
||||
if err := obj.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mp, err := obj.NewMultipartUpload(t.Context(), bucket, key, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cancel()
|
||||
if err := obj.Shutdown(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reopened, _, err := initObjectLayer(t.Context(), mustGetPoolEndpoints(0, dirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj = reopened
|
||||
if _, cached := obj.(*erasureServerPools).mpCache.Load(mp.UploadID); cached {
|
||||
t.Fatal("restart unexpectedly preserved the upload cache")
|
||||
}
|
||||
for _, prefix := range []string{"", "dir/"} {
|
||||
got, err := obj.ListMultipartUploads(t.Context(), bucket, prefix, "", "", "", 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, got)
|
||||
}
|
||||
got, err := obj.ListMultipartUploads(t.Context(), bucket, key, "", "", "", 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, got, key)
|
||||
if got.Uploads[0].UploadID != mp.UploadID {
|
||||
t.Fatal("restart changed upload identity")
|
||||
}
|
||||
part, err := obj.PutObjectPart(t.Context(), bucket, key, mp.UploadID, 1,
|
||||
mustGetPutObjReader(t, bytes.NewBufferString("part"), 4, "", ""), ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("continue known upload after restart: %v", err)
|
||||
}
|
||||
if _, err := obj.CompleteMultipartUpload(t.Context(), bucket, key, mp.UploadID,
|
||||
[]CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("complete known upload after restart: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartDiagnosticConflictingIdentityCopies(t *testing.T) {
|
||||
for _, oldCopies := range []int{1, 3} {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := set.getUploadIDDir(bucket, "a", mp.UploadID)
|
||||
for _, disk := range set.getDisks()[:oldCopies] {
|
||||
fi, err := disk.ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, path, "", ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
removeMultipartIdentity(fi.Metadata)
|
||||
if err := disk.WriteMetadata(t.Context(), bucket, minioMetaMultipartBucket, path, fi); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
|
||||
if !errors.Is(err, errMultipartListingIdentity) || len(got.Uploads) != 0 {
|
||||
t.Fatalf("%d contradictory copies returned a successful page: %+v %v", oldCopies, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartDiagnosticUnsupportedStoredID(t *testing.T) {
|
||||
for _, historical := range []bool{false, true} {
|
||||
z, set, bucket := multipartListingFixture(t)
|
||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldPath := set.getUploadIDDir(bucket, "a", mp.UploadID)
|
||||
unsupportedPath := pathJoin(set.getMultipartSHADir(bucket, "a"), "unsupported-id")
|
||||
for _, disk := range set.getDisks() {
|
||||
fi, err := disk.ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, oldPath, "", ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if historical {
|
||||
removeMultipartIdentity(fi.Metadata)
|
||||
}
|
||||
if err := disk.WriteMetadata(t.Context(), bucket, minioMetaMultipartBucket, unsupportedPath, fi); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := disk.Delete(t.Context(), minioMetaMultipartBucket, oldPath, DeleteOptions{Recursive: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
|
||||
if historical {
|
||||
if err != nil || len(got.Uploads) != 0 {
|
||||
t.Fatalf("unidentified historical ID: %+v %v", got, err)
|
||||
}
|
||||
} else if !errors.Is(err, errMultipartListingIdentity) {
|
||||
t.Fatalf("invalid new ID silently treated as historical: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
+65
-101
@@ -174,34 +174,46 @@ func (er erasureObjects) deleteAll(ctx context.Context, bucket, prefix string) {
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// extractModTimeFromUploadIDDir extracts creation time from the upload ID directory on disk.
|
||||
// On-disk directory names are of the form <UUID>x<UnixNano>.
|
||||
// Parse the native format first: plaintext UUIDs can also be valid Base64.
|
||||
// Preserve the previous encoded-format parser for compatibility.
|
||||
func extractModTimeFromUploadIDDir(uploadIDDir string) time.Time {
|
||||
if modTime, ok := multipartUploadTime(uploadIDDir); ok {
|
||||
return modTime
|
||||
}
|
||||
if b64, err := base64.RawURLEncoding.DecodeString(uploadIDDir); err == nil {
|
||||
if split := strings.Split(string(b64), "x"); len(split) == 2 {
|
||||
if t, err := strconv.ParseInt(split[1], 10, 64); err == nil {
|
||||
return time.Unix(0, t)
|
||||
}
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
// Remove the old multipart uploads on the given disk.
|
||||
func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk StorageAPI) {
|
||||
drivePath := disk.Endpoint().Path
|
||||
|
||||
readDirFn(pathJoin(drivePath, minioMetaMultipartBucket), func(shaDir string, typ os.FileMode) error {
|
||||
hasSkippedUploads := false
|
||||
readDirFn(pathJoin(drivePath, minioMetaMultipartBucket, shaDir), func(uploadIDDir string, typ os.FileMode) error {
|
||||
uploadIDPath := pathJoin(shaDir, uploadIDDir)
|
||||
var modTime time.Time
|
||||
// Upload IDs are of the form base64_url(<UUID>x<UnixNano>), we can extract the time from the UUID.
|
||||
if b64, err := base64.RawURLEncoding.DecodeString(uploadIDDir); err == nil {
|
||||
if split := strings.Split(string(b64), "x"); len(split) == 2 {
|
||||
t, err := strconv.ParseInt(split[1], 10, 64)
|
||||
if err == nil {
|
||||
modTime = time.Unix(0, t)
|
||||
}
|
||||
}
|
||||
}
|
||||
modTime := extractModTimeFromUploadIDDir(uploadIDDir)
|
||||
// Fallback for older uploads without time in the ID.
|
||||
if modTime.IsZero() {
|
||||
wait := deleteMultipartCleanupSleeper.Timer(ctx)
|
||||
fi, err := disk.ReadVersion(ctx, "", minioMetaMultipartBucket, uploadIDPath, "", ReadOptions{})
|
||||
if err != nil {
|
||||
hasSkippedUploads = true
|
||||
return nil
|
||||
}
|
||||
modTime = fi.ModTime
|
||||
wait()
|
||||
}
|
||||
if time.Since(modTime) < globalAPIConfig.getStaleUploadsExpiry() {
|
||||
hasSkippedUploads = true
|
||||
return nil
|
||||
}
|
||||
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
|
||||
@@ -214,24 +226,15 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
|
||||
return nil
|
||||
})
|
||||
})
|
||||
// Get the modtime of the shaDir.
|
||||
vi, err := disk.StatVol(ctx, pathJoin(minioMetaMultipartBucket, shaDir))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
// Modtime is returned in the Created field. See (*xlStorage).StatVol
|
||||
if time.Since(vi.Created) < globalAPIConfig.getStaleUploadsExpiry() {
|
||||
if hasSkippedUploads {
|
||||
return nil
|
||||
}
|
||||
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
|
||||
return w.Run(func() error {
|
||||
wait := deleteMultipartCleanupSleeper.Timer(ctx)
|
||||
pathUUID := mustGetUUID()
|
||||
targetPath := pathJoin(drivePath, minioMetaTmpDeletedBucket, pathUUID)
|
||||
|
||||
// We are not deleting shaDir recursively here, if shaDir is empty
|
||||
// and its older then we can happily delete it.
|
||||
Rename(pathJoin(drivePath, minioMetaMultipartBucket, shaDir), targetPath)
|
||||
// Removing an upload updates the parent's mtime. Remove the parent
|
||||
// immediately if it is empty; other uploads keep it in place.
|
||||
_ = Remove(pathJoin(drivePath, minioMetaMultipartBucket, shaDir))
|
||||
wait()
|
||||
return nil
|
||||
})
|
||||
@@ -405,7 +408,7 @@ func paginateMultipartUploads(uploads []MultipartInfo, prefix, keyMarker, upload
|
||||
}
|
||||
|
||||
// listMultipartUploadsExact preserves the hashed exact-object lookup used by
|
||||
// multipart write placement and by rolling-upgrade legacy mode.
|
||||
// multipart write placement and nonempty public listing prefixes.
|
||||
func (er erasureObjects) listMultipartUploadsExact(ctx context.Context, bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
|
||||
auditObjectErasureSet(ctx, "ListMultipartUploads", object, &er)
|
||||
|
||||
@@ -513,19 +516,7 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, prefi
|
||||
if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil {
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
scan, err := startMultipartScan(ctx, false)
|
||||
if err != nil {
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
defer scan.close()
|
||||
uploads, legacy, err := er.scanMultipartUploads(scan, bucket, 0, 0)
|
||||
if err != nil {
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
if legacy {
|
||||
return ListMultipartsInfo{}, errMultipartListingLegacy
|
||||
}
|
||||
return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
|
||||
return er.listMultipartUploadsExact(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
|
||||
}
|
||||
|
||||
// newMultipartUpload - wrapper for initializing a new multipart
|
||||
@@ -1696,6 +1687,32 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
defer lk.Unlock(lkctx)
|
||||
}
|
||||
|
||||
if !opts.ReplicationRequest && !opts.ReplicaLockReconcile && !opts.DataMovement {
|
||||
getObjectInfo := er.getObjectInfo
|
||||
if opts.replicaObjectInfo != nil {
|
||||
getObjectInfo = opts.replicaObjectInfo
|
||||
}
|
||||
if fi.VersionID == "" || fi.VersionID == nullVersionID {
|
||||
if err := checkObjectLockedForOverwrite(ctx, bucket, object, getObjectInfo); err != nil {
|
||||
return oi, toObjectErr(err, bucket, object)
|
||||
}
|
||||
} else {
|
||||
_, gerr := getObjectInfo(ctx, bucket, object, ObjectOptions{
|
||||
VersionID: fi.VersionID,
|
||||
Versioned: opts.Versioned,
|
||||
VersionSuspended: opts.VersionSuspended,
|
||||
NoLock: true,
|
||||
})
|
||||
switch {
|
||||
case gerr == nil, errors.As(gerr, &MethodNotAllowed{}):
|
||||
return oi, toObjectErr(errAuthentication, bucket, object)
|
||||
case isErrVersionNotFound(gerr) || isErrObjectNotFound(gerr):
|
||||
default:
|
||||
return oi, toObjectErr(gerr, bucket, object)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
er.cleanupMultipartPath(ctx, paths...) // cleanup all part.N.meta, and skipped part.N's before final rename().
|
||||
|
||||
defer func() {
|
||||
@@ -1753,11 +1770,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
|
||||
}
|
||||
|
||||
// abortMultipartUpload retains read-quorum validation and best-effort cleanup
|
||||
// in legacy mode. Strict mode requires majority deletion acknowledgements and
|
||||
// permits retrying remnants below read quorum. Neither mode fences creation
|
||||
// writes still executing after a storage timeout.
|
||||
func (er erasureObjects) abortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions, legacy bool) (bool, error) {
|
||||
// abortMultipartUpload retains read-quorum validation and best-effort cleanup.
|
||||
// It does not fence creation writes still executing after a storage timeout.
|
||||
func (er erasureObjects) abortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (bool, error) {
|
||||
if !opts.NoAuditLog {
|
||||
auditObjectErasureSet(ctx, "AbortMultipartUpload", object, &er)
|
||||
}
|
||||
@@ -1769,70 +1784,19 @@ func (er erasureObjects) abortMultipartUpload(ctx context.Context, bucket, objec
|
||||
if !ok || internalID == "" || internalID == "." || internalID == ".." || strings.ContainsAny(internalID, "/\\") {
|
||||
return false, InvalidUploadID{Bucket: bucket, Object: object, UploadID: uploadID}
|
||||
}
|
||||
if legacy {
|
||||
// Keep the released read-quorum and best-effort cleanup behavior.
|
||||
// The upload ID safety check above applies to both modes.
|
||||
defer er.deleteAll(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID))
|
||||
_, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
|
||||
err = toObjectErr(err, bucket, object, uploadID)
|
||||
if _, absent := err.(InvalidUploadID); absent {
|
||||
return false, nil
|
||||
}
|
||||
return err == nil, err
|
||||
}
|
||||
disks := er.getDisks()
|
||||
uploadPath := er.getUploadIDDir(bucket, object, uploadID)
|
||||
_, errs := readAllFileInfo(ctx, disks, bucket, minioMetaMultipartBucket, uploadPath, "", false, false)
|
||||
quorum := er.setDriveCount/2 + 1
|
||||
found, absent := false, 0
|
||||
observed := make([]bool, len(disks))
|
||||
for i, err := range errs {
|
||||
switch {
|
||||
case err == nil, errors.Is(err, errFileCorrupt):
|
||||
found = true
|
||||
observed[i] = true
|
||||
case errors.Is(err, errFileNotFound), errors.Is(err, errFileVersionNotFound):
|
||||
absent++
|
||||
}
|
||||
}
|
||||
if absent >= quorum && !found {
|
||||
defer er.deleteAll(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID))
|
||||
_, _, err = er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
|
||||
err = toObjectErr(err, bucket, object, uploadID)
|
||||
if _, absent := err.(InvalidUploadID); absent {
|
||||
return false, nil
|
||||
}
|
||||
if !found {
|
||||
return false, toObjectErr(errErasureReadQuorum, bucket, object, uploadID)
|
||||
}
|
||||
g := errgroup.WithNErrs(len(disks))
|
||||
for i, disk := range disks {
|
||||
g.Go(func() error {
|
||||
if disk == nil {
|
||||
return errDiskNotFound
|
||||
}
|
||||
err := disk.Delete(ctx, minioMetaMultipartBucket, uploadPath, DeleteOptions{Recursive: true, Immediate: false})
|
||||
if errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}, i)
|
||||
}
|
||||
deleteErrs := g.Wait()
|
||||
if err := reduceWriteQuorumErrs(ctx, deleteErrs, nil, quorum); err != nil {
|
||||
return true, toObjectErr(err, bucket, object, uploadID)
|
||||
}
|
||||
if absent >= quorum {
|
||||
// Missing disks must not mask a failed cleanup of known remnants.
|
||||
for i, found := range observed {
|
||||
if found && deleteErrs[i] != nil {
|
||||
return true, toObjectErr(errErasureWriteQuorum, bucket, object, uploadID)
|
||||
}
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
// AbortMultipartUpload cancels an upload using the configured mode. Offline
|
||||
// part data may still need stale-upload cleanup after its drives return.
|
||||
// AbortMultipartUpload cancels an upload using read-quorum validation.
|
||||
// Offline part data may still need stale-upload cleanup after its drives return.
|
||||
func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (err error) {
|
||||
found, err := er.abortMultipartUpload(ctx, bucket, object, uploadID, opts, globalAPIConfig.getMultipartListingLegacy())
|
||||
found, err := er.abortMultipartUpload(ctx, bucket, object, uploadID, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+152
-13
@@ -150,6 +150,17 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
|
||||
}
|
||||
}
|
||||
|
||||
if !dstOpts.ReplicationRequest && !dstOpts.ReplicaLockReconcile && !dstOpts.DataMovement &&
|
||||
(versionID == "" || versionID == nullVersionID) {
|
||||
getObjectInfo := er.getObjectInfo
|
||||
if dstOpts.replicaObjectInfo != nil {
|
||||
getObjectInfo = dstOpts.replicaObjectInfo
|
||||
}
|
||||
if err := checkObjectLockedForOverwrite(ctx, dstBucket, dstObject, getObjectInfo); err != nil {
|
||||
return oi, toObjectErr(err, dstBucket, dstObject)
|
||||
}
|
||||
}
|
||||
|
||||
modTime = UTCNow() // We only preserve modTime if dstOpts.MTime is true.
|
||||
// in all other cases mtime is latest.
|
||||
|
||||
@@ -718,6 +729,57 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
|
||||
return pickLatestQuorumFilesInfo(ctx, rawFileInfos, errs, bucket, object, readData, inclFreeVers)
|
||||
}
|
||||
|
||||
// readListingFileInfo is the read-only, latest-version metadata path for listing
|
||||
// recovery. Ordinary GET keeps its fast-read, healing and dangling semantics.
|
||||
// Absence is proved before decoding can replace a native storage error: every
|
||||
// configured slot must have returned FileNotFound in this read.
|
||||
func (er erasureObjects) readListingFileInfo(ctx context.Context, bucket, object string) (FileInfo, bool, error) {
|
||||
disks := er.getDisks()
|
||||
if len(disks) == 0 || len(disks) != er.setDriveCount {
|
||||
return FileInfo{}, false, listingReadQuorumError(bucket, object)
|
||||
}
|
||||
// An offline slot is not evidence of absence, even if a test or transport
|
||||
// implementation could return a cached not-found response for it.
|
||||
online := append([]StorageAPI(nil), disks...)
|
||||
for i, disk := range online {
|
||||
if disk != nil && !disk.IsOnline() {
|
||||
online[i] = nil
|
||||
}
|
||||
}
|
||||
raw, errs := readAllRawFileInfo(ctx, online, bucket, object, false)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return FileInfo{}, false, err
|
||||
}
|
||||
absent := true
|
||||
for _, err := range errs {
|
||||
absent = absent && err == errFileNotFound
|
||||
}
|
||||
if absent {
|
||||
return FileInfo{}, true, nil
|
||||
}
|
||||
|
||||
// Keep the same latest-version and actual object parity quorum as GET,
|
||||
// without its opportunistic dangling deletion or generic NotFound mapping.
|
||||
meta, errs := pickLatestQuorumFilesInfo(ctx, raw, errs, bucket, object, false, false)
|
||||
quorum, _, err := objectQuorumFromMeta(ctx, meta, errs, er.defaultParityCount)
|
||||
if err == nil {
|
||||
err = reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, quorum)
|
||||
}
|
||||
if err != nil {
|
||||
return FileInfo{}, false, listingReadQuorumError(bucket, object)
|
||||
}
|
||||
_, modTime, etag := listOnlineDisks(disks, meta, errs, quorum)
|
||||
fi, err := pickValidFileInfo(ctx, meta, modTime, etag, quorum)
|
||||
if err != nil || (!fi.Deleted && len(fi.Erasure.Distribution) != len(disks)) {
|
||||
return FileInfo{}, false, listingReadQuorumError(bucket, object)
|
||||
}
|
||||
return fi, false, nil
|
||||
}
|
||||
|
||||
func listingReadQuorumError(bucket, object string) error {
|
||||
return InsufficientReadQuorum{Bucket: bucket, Object: decodeDirObject(object), Err: errErasureReadQuorum, Type: RQInconsistentMeta}
|
||||
}
|
||||
|
||||
func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object string, opts ObjectOptions, readData bool) (FileInfo, []FileInfo, []StorageAPI, error) {
|
||||
rawArr := make([]RawFileInfo, er.setDriveCount)
|
||||
metaArr := make([]FileInfo, er.setDriveCount)
|
||||
@@ -1032,11 +1094,19 @@ func (er erasureObjects) getObjectInfoAndQuorum(ctx context.Context, bucket, obj
|
||||
|
||||
// Similar to rename but renames data from srcEntry to dstEntry at dataDir
|
||||
func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry string, metadata []FileInfo, dstBucket, dstEntry string, writeQuorum int) ([]StorageAPI, []byte, string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, nil, "", err
|
||||
}
|
||||
// Admission is cancelable. After admission use actual disk results for
|
||||
// commit/rollback, even if the client or namespace lease is canceled.
|
||||
ctx = context.WithoutCancel(ctx)
|
||||
g := errgroup.WithNErrs(len(disks))
|
||||
|
||||
fvID := mustGetUUID()
|
||||
for index := range disks {
|
||||
metadata[index].Metadata = maps.Clone(metadata[index].Metadata)
|
||||
metadata[index].SetTierFreeVersionID(fvID)
|
||||
metadata[index].Metadata[rollbackIDKey] = fvID
|
||||
}
|
||||
|
||||
diskVersions := make([][]byte, len(disks))
|
||||
@@ -1072,6 +1142,12 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry str
|
||||
errs := g.Wait()
|
||||
|
||||
err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
|
||||
if errors.Is(err, errCommitMetaMissing) {
|
||||
// Commit metadata went missing on a quorum of drives; the write
|
||||
// did not commit. Return a 503 class error so clients retry
|
||||
// instead of treating the object as absent (404).
|
||||
err = errErasureWriteQuorum
|
||||
}
|
||||
if err != nil {
|
||||
dg := errgroup.WithNErrs(len(disks))
|
||||
for index, nerr := range errs {
|
||||
@@ -1488,6 +1564,18 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
compIndex = opts.IndexCB()
|
||||
}
|
||||
|
||||
if !opts.NoLock {
|
||||
lk := er.NewNSLock(bucket, object)
|
||||
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
}
|
||||
|
||||
// Mint the generation while holding the commit lock. Staged uploads may
|
||||
// acquire the lock in a different order from their encoding completion.
|
||||
modTime := opts.MTime
|
||||
if opts.MTime.IsZero() {
|
||||
modTime = UTCNow()
|
||||
@@ -1584,14 +1672,15 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
}
|
||||
}
|
||||
|
||||
if !opts.NoLock {
|
||||
lk := er.NewNSLock(bucket, object)
|
||||
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
if !opts.ReplicationRequest && !opts.ReplicaLockReconcile && !opts.DataMovement &&
|
||||
(fi.VersionID == "" || fi.VersionID == nullVersionID) {
|
||||
getObjectInfo := er.getObjectInfo
|
||||
if opts.replicaObjectInfo != nil {
|
||||
getObjectInfo = opts.replicaObjectInfo
|
||||
}
|
||||
if err := checkObjectLockedForOverwrite(ctx, bucket, object, getObjectInfo); err != nil {
|
||||
return ObjectInfo{}, toObjectErr(err, bucket, object)
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
}
|
||||
|
||||
// Rename the successfully written temporary object to final location.
|
||||
@@ -1657,7 +1746,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
|
||||
}
|
||||
|
||||
func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object string, fi FileInfo, forceDelMarker bool) error {
|
||||
func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object string, fi FileInfo, forceDelMarker, purge bool) error {
|
||||
disks := er.getDisks()
|
||||
// Assume (N/2 + 1) quorum for Delete()
|
||||
// this is a theoretical assumption such that
|
||||
@@ -1673,7 +1762,13 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
|
||||
if disks[index] == nil {
|
||||
return errDiskNotFound
|
||||
}
|
||||
return disks[index].DeleteVersion(ctx, bucket, object, fi, forceDelMarker, DeleteOptions{})
|
||||
err := disks[index].DeleteVersion(ctx, bucket, object, fi, forceDelMarker, DeleteOptions{})
|
||||
// Physical removal and reliable already-absent replies are the same
|
||||
// outcome. Creation and metadata updates must not use this quorum.
|
||||
if purge && (err == errFileNotFound || err == errFileVersionNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}, index)
|
||||
}
|
||||
// return errors if any during deletion
|
||||
@@ -1806,6 +1901,23 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
if err == errFileNotFound || err == errFileVersionNotFound {
|
||||
// A retry after partial physical deletion may mix removals
|
||||
// with already-absent replies. Both prove the desired outcome.
|
||||
// Marker creation and replication-state updates still require
|
||||
// successful writes; absence cannot acknowledge those operations.
|
||||
physical := true
|
||||
for _, v := range dedupVersions[i].Versions {
|
||||
if v.Deleted || v.MarkDeleted || v.ReplicationState.ReplicationStatusInternal != "" ||
|
||||
v.ReplicationState.VersionPurgeStatusInternal != "" || v.ReplicationState.ReplicaStatus != "" {
|
||||
physical = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if physical {
|
||||
continue
|
||||
}
|
||||
}
|
||||
for _, v := range dedupVersions[i].Versions {
|
||||
delObjErrs[index][v.Idx] = err
|
||||
}
|
||||
@@ -2023,6 +2135,11 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
|
||||
if opts.DeleteMarker {
|
||||
versionFound = false
|
||||
} else if !tryDel {
|
||||
if opts.isVersionPurge() && (isErrObjectNotFound(gerr) || isErrVersionNotFound(gerr)) {
|
||||
if err := er.checkPurgeAbsent(ctx, bucket, object, opts.VersionID); err != nil {
|
||||
return objInfo, err
|
||||
}
|
||||
}
|
||||
return objInfo, gerr
|
||||
}
|
||||
}
|
||||
@@ -2105,6 +2222,20 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
|
||||
}
|
||||
}
|
||||
|
||||
if opts.DataMovement && opts.DeleteMarker {
|
||||
// Moving a null marker replaces any older null data/marker. Moving
|
||||
// a UUID marker preserves it on retries instead of purging it.
|
||||
deleteMarker = true
|
||||
markDelete = opts.VersionID != nullVersionID
|
||||
}
|
||||
|
||||
purge := opts.isVersionPurge()
|
||||
if purge {
|
||||
// A delete marker describes the version being removed; it is not an
|
||||
// instruction to create that marker on disks which already lack it.
|
||||
markDelete, deleteMarker = false, false
|
||||
}
|
||||
|
||||
modTime := opts.MTime
|
||||
if opts.MTime.IsZero() {
|
||||
modTime = UTCNow()
|
||||
@@ -2151,7 +2282,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
|
||||
// delete marker. Add delete marker, since we don't have
|
||||
// any version specified explicitly. Or if a particular
|
||||
// version id needs to be replicated.
|
||||
if err = er.deleteObjectVersion(ctx, bucket, object, fi, opts.DeleteMarker); err != nil {
|
||||
if err = er.deleteObjectVersion(ctx, bucket, object, fi, opts.DeleteMarker, false); err != nil {
|
||||
return objInfo, toObjectErr(err, bucket, object)
|
||||
}
|
||||
oi := fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
|
||||
@@ -2174,11 +2305,19 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
|
||||
if opts.SkipFreeVersion {
|
||||
dfi.SetSkipTierFreeVersion()
|
||||
}
|
||||
if err = er.deleteObjectVersion(ctx, bucket, object, dfi, opts.DeleteMarker); err != nil {
|
||||
if err = er.deleteObjectVersion(ctx, bucket, object, dfi, opts.DeleteMarker, purge); err != nil {
|
||||
return objInfo, toObjectErr(err, bucket, object)
|
||||
}
|
||||
|
||||
return dfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
|
||||
oi := dfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
|
||||
if purge {
|
||||
// Preserve the DELETE response's identity without reusing it as a disk
|
||||
// instruction to create a marker. The lookup also exposes a data
|
||||
// version pending purge as deleted for visibility; only a stored
|
||||
// marker, which carries no erasure layout, is reported as one.
|
||||
oi.DeleteMarker = goi.DeleteMarker && goi.DataBlocks == 0
|
||||
}
|
||||
return oi, nil
|
||||
}
|
||||
|
||||
// Send the successful but partial upload/delete, however ignore
|
||||
@@ -2467,7 +2606,7 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
|
||||
|
||||
storageDisks := er.getDisks()
|
||||
|
||||
if err = er.deleteObjectVersion(ctx, bucket, object, fi, false); err != nil {
|
||||
if err = er.deleteObjectVersion(ctx, bucket, object, fi, false, false); err != nil {
|
||||
eventName = event.ObjectTransitionFailed
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,309 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||
// maintenance release gate; failures record intentionally unresolved risks.
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
madmin "github.com/minio/madmin-go/v3"
|
||||
)
|
||||
|
||||
// All metadata is valid and readable. Only the completed generations differ.
|
||||
// No timing, missing files, synthetic read error or malformed metadata is needed.
|
||||
func TestReadGenerationConflictIsNotAbsence(t *testing.T) {
|
||||
raw := make([]RawFileInfo, 8)
|
||||
for i := range raw {
|
||||
fi := newFileInfo("object", 4, 4)
|
||||
fi.Erasure.Index = i + 1
|
||||
fi.ModTime = time.Unix(100+int64(i%4), 0)
|
||||
fi.DataDir = mustGetUUID()
|
||||
fi.Metadata = map[string]string{"etag": fmt.Sprint(i % 4)}
|
||||
var meta xlMetaV2
|
||||
if err := meta.AddVersion(fi); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
buf, err := meta.AppendTo(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw[i] = RawFileInfo{Buf: buf}
|
||||
}
|
||||
_, errs := pickLatestQuorumFilesInfo(t.Context(), raw, make([]error, len(raw)), "bucket", "object", true, false)
|
||||
err := reduceReadQuorumErrs(t.Context(), errs, objectOpIgnoredErrs, 4)
|
||||
if api := toAPIError(t.Context(), toObjectErr(err, "bucket", "object")); api.HTTPStatusCode != http.StatusServiceUnavailable {
|
||||
t.Fatalf("readable conflicting generations reported as absence: %v (%+v)", err, api)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsReadGenerationConflictDoesNotExposeOldValue(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
router := initTestAPIEndPoints(z, []string{"GetObject", "HeadObject"})
|
||||
for _, tc := range []struct {
|
||||
pool int
|
||||
mode string
|
||||
}{
|
||||
{0, "split"},
|
||||
{1, "split"},
|
||||
{0, "older-quorum"},
|
||||
{1, "older-quorum"},
|
||||
{0, "newer-quorum"},
|
||||
{1, "newer-quorum"},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("%s/pool-%d", tc.mode, tc.pool), func(t *testing.T) {
|
||||
broken := tc.pool
|
||||
object := fmt.Sprintf("conflict-%s-%d", tc.mode, broken)
|
||||
base := UTCNow().Add(-time.Hour)
|
||||
putConsistencyObject(t, z, bucket, object, 1-broken, "old-acknowledged", ObjectOptions{MTime: base})
|
||||
set := z.serverPools[broken].getHashedSet(object)
|
||||
disks := set.getDisks()
|
||||
saved := make([][]byte, len(disks))
|
||||
savedParts := make([]map[string][]byte, len(disks))
|
||||
for generation := range 4 {
|
||||
putConsistencyObject(t, z, bucket, object, broken, fmt.Sprintf("newer-%d", generation),
|
||||
ObjectOptions{MTime: base.Add(time.Duration(generation+1) * time.Minute)})
|
||||
for i, disk := range disks {
|
||||
g := i % 4
|
||||
switch tc.mode {
|
||||
case "older-quorum":
|
||||
g = 0
|
||||
if i >= len(disks)-3 {
|
||||
g = i - (len(disks) - 3) + 1
|
||||
}
|
||||
case "newer-quorum":
|
||||
g = 3
|
||||
if i < 3 {
|
||||
g = i
|
||||
}
|
||||
}
|
||||
if g != generation {
|
||||
continue
|
||||
}
|
||||
buf, err := disk.ReadAll(ctx, bucket, pathJoin(object, xlStorageFormatFile))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved[i] = bytes.Clone(buf)
|
||||
savedParts[i] = readGenerationParts(t, disk, bucket, object, buf)
|
||||
}
|
||||
}
|
||||
for i, disk := range disks {
|
||||
for path, data := range savedParts[i] {
|
||||
if err := disk.WriteAll(ctx, bucket, path, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := disk.WriteAll(ctx, bucket, pathJoin(object, xlStorageFormatFile), saved[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, version := range []string{"", nullVersionID} {
|
||||
for _, method := range []string{http.MethodHead, http.MethodGet} {
|
||||
rec := consistencyRequest(t, router, method, bucket, object, version)
|
||||
want := http.StatusServiceUnavailable
|
||||
if tc.mode == "newer-quorum" {
|
||||
want = http.StatusOK
|
||||
}
|
||||
if rec.Code != want {
|
||||
t.Errorf("%s version=%q: want status=%d, got=%d body=%q", method, version, want, rec.Code, rec.Body.String())
|
||||
}
|
||||
if want == http.StatusOK && method == http.MethodGet && rec.Body.String() != "newer-3" {
|
||||
t.Errorf("latest quorum did not return its payload: %q", rec.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
if tc.mode != "newer-quorum" {
|
||||
for _, version := range []string{"", nullVersionID} {
|
||||
_, err := set.CopyObject(ctx, bucket, object, bucket, object,
|
||||
ObjectInfo{metadataOnly: true, UserDefined: map[string]string{"copy-test": "conflicting-source"}},
|
||||
ObjectOptions{VersionID: version}, ObjectOptions{})
|
||||
if !isErrReadQuorum(err) {
|
||||
t.Errorf("COPY-to-self version=%q: want read-quorum error, got %v", version, err)
|
||||
}
|
||||
for _, scan := range []madmin.HealScanMode{madmin.HealNormalScan, madmin.HealDeepScan} {
|
||||
_, err := set.HealObject(ctx, bucket, object, version, madmin.HealOpts{ScanMode: scan, Remove: true})
|
||||
if !isErrReadQuorum(err) {
|
||||
t.Errorf("heal version=%q scan=%v: want read-quorum error, got %v", version, scan, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for i, disk := range disks {
|
||||
buf, err := disk.ReadAll(ctx, bucket, pathJoin(object, xlStorageFormatFile))
|
||||
if err != nil || !bytes.Equal(buf, saved[i]) {
|
||||
t.Errorf("read or heal destroyed conflicting generation on disk %d: %v", i, err)
|
||||
}
|
||||
if tc.mode != "newer-quorum" {
|
||||
for path, data := range savedParts[i] {
|
||||
got, err := disk.ReadAll(ctx, bucket, path)
|
||||
if err != nil || !bytes.Equal(got, data) {
|
||||
t.Errorf("read or heal destroyed generation shard on disk %d: %s: %v", i, path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func readGenerationParts(t *testing.T, disk StorageAPI, bucket, object string, metadata []byte) map[string][]byte {
|
||||
t.Helper()
|
||||
fi, err := fileInfoFromRaw(RawFileInfo{Buf: metadata}, bucket, object, false, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parts := make(map[string][]byte)
|
||||
if fi.InlineData() || fi.Size == 0 {
|
||||
return parts
|
||||
}
|
||||
// Later PUTs remove the previous data directory. Save its real bitrot
|
||||
// shards too, so restoring metadata never fabricates missing-part damage.
|
||||
for _, part := range fi.Parts {
|
||||
path := pathJoin(object, fi.DataDir, fmt.Sprintf("part.%d", part.Number))
|
||||
data, err := disk.ReadAll(t.Context(), bucket, path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parts[path] = data
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
func TestPoolsReadUnavailablePoolDoesNotExposeOldValue(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
for broken := range 2 {
|
||||
t.Run(fmt.Sprintf("broken-pool-%d", broken), func(t *testing.T) {
|
||||
object := fmt.Sprintf("unavailable-%d", broken)
|
||||
putConsistencyObject(t, z, bucket, object, 1-broken, "old-acknowledged", ObjectOptions{})
|
||||
set := z.serverPools[broken].getHashedSet(object)
|
||||
disks, switchDisks := installListingDiskSwitch(set)
|
||||
defer switchDisks(disks)
|
||||
faulty := append([]StorageAPI(nil), disks...)
|
||||
for i := range faulty {
|
||||
faulty[i] = consistencyReadFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object}
|
||||
}
|
||||
switchDisks(faulty)
|
||||
if _, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); err == nil || isErrObjectNotFound(err) {
|
||||
t.Errorf("unknown pool state allowed an old value: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type generationReadGateDisk struct {
|
||||
StorageAPI
|
||||
ready chan<- struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (d generationReadGateDisk) ReadXL(ctx context.Context, volume, path string, data bool) (RawFileInfo, error) {
|
||||
fi, err := d.StorageAPI.ReadXL(ctx, volume, path, data)
|
||||
d.ready <- struct{}{}
|
||||
if d.release != nil {
|
||||
<-d.release
|
||||
}
|
||||
return fi, err
|
||||
}
|
||||
|
||||
func (d generationReadGateDisk) ReadVersion(ctx context.Context, orig, volume, path, version string, opts ReadOptions) (FileInfo, error) {
|
||||
fi, err := d.StorageAPI.ReadVersion(ctx, orig, volume, path, version, opts)
|
||||
d.ready <- struct{}{}
|
||||
if d.release != nil {
|
||||
<-d.release
|
||||
}
|
||||
return fi, err
|
||||
}
|
||||
|
||||
func TestReadGenerationFastInfoWaitsForSlowDisk(t *testing.T) {
|
||||
for _, version := range []string{"", nullVersionID} {
|
||||
t.Run("version="+version, func(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
const object = "slow-newer-generation"
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, switchDisks := installListingDiskSwitch(set)
|
||||
defer switchDisks(disks)
|
||||
base := UTCNow().Add(-time.Hour)
|
||||
putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{MTime: base})
|
||||
saved := make([][]byte, len(disks)-1)
|
||||
for i := range saved {
|
||||
saved[i] = mustReadNullQuorumMeta(t, disks[i], bucket, object)
|
||||
}
|
||||
putConsistencyObject(t, z, bucket, object, 0, "new", ObjectOptions{MTime: base.Add(time.Minute)})
|
||||
for i, meta := range saved {
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, pathJoin(object, xlStorageFormatFile), meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ready, release := make(chan struct{}, len(disks)), make(chan struct{})
|
||||
var once sync.Once
|
||||
unblock := func() { once.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, disk := range disks {
|
||||
gate := generationReadGateDisk{StorageAPI: disk, ready: ready}
|
||||
if i == len(disks)-1 {
|
||||
gate.release = release
|
||||
}
|
||||
wrapped[i] = gate
|
||||
}
|
||||
switchDisks(wrapped)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, _, _, err := set.getObjectFileInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: true, VersionID: version}, false)
|
||||
done <- err
|
||||
}()
|
||||
for range disks {
|
||||
select {
|
||||
case <-ready:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("metadata reads did not reach the barrier")
|
||||
}
|
||||
}
|
||||
early := false
|
||||
select {
|
||||
case err := <-done:
|
||||
early = true
|
||||
t.Errorf("fast read returned before the newer disk: %v", err)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
unblock()
|
||||
if !early {
|
||||
select {
|
||||
case err := <-done:
|
||||
if !isErrReadQuorum(err) {
|
||||
t.Fatalf("fast read exposed the old generation: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("fast read did not finish after the newer disk")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
madmin "github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
)
|
||||
|
||||
// objectPoolInfos reads the addressed version in every pool, including pools
|
||||
@@ -62,6 +63,125 @@ func (z *erasureServerPools) objectPoolInfos(ctx context.Context, bucket, object
|
||||
return copies, nil
|
||||
}
|
||||
|
||||
// checkDataMovementCopies rejects moving a version when another pool holds a
|
||||
// newer copy of it, which reads return instead. A cross-pool overwrite can
|
||||
// leave such a copy, and a client write can add one while the move copies.
|
||||
// Equal timestamps normally identify an interrupted move. Conflicting ties
|
||||
// retain the same pool-order winner as reads. While the caller holds the
|
||||
// pools-layer object lock, the returned resolver serves the set layer's
|
||||
// reconciliation of this version from the same reads. Without opts.NoLock
|
||||
// it takes the read lock itself.
|
||||
func (z *erasureServerPools) checkDataMovementCopies(ctx context.Context, bucket, object string, opts ObjectOptions) (GetObjectInfoFn, error) {
|
||||
if opts.MTime.IsZero() {
|
||||
// The set layer would stamp the moved copy as the newest one.
|
||||
return nil, errInvalidArgument
|
||||
}
|
||||
if !opts.NoLock {
|
||||
lk := z.NewNSLock(bucket, object)
|
||||
lkctx, err := lk.GetRLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.RUnlock(lkctx)
|
||||
}
|
||||
versionID := opts.VersionID
|
||||
if versionID == "" {
|
||||
versionID = nullVersionID
|
||||
}
|
||||
copies, err := z.objectPoolInfos(ctx, bucket, object, ObjectOptions{VersionID: versionID, NoAuditLog: true})
|
||||
if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
return nil, err
|
||||
}
|
||||
// The snapshot may have been purged or replaced after the reader opened.
|
||||
// In that case the walker must leave the current source state alone.
|
||||
sourcePresent := false
|
||||
for _, copy := range copies {
|
||||
if copy.Index == opts.SrcPoolIdx && copy.ObjInfo.ModTime.Equal(opts.MTime) && copy.ObjInfo.DeleteMarker == opts.DeleteMarker {
|
||||
sourcePresent = true
|
||||
}
|
||||
}
|
||||
if !sourcePresent {
|
||||
return nil, VersionNotFound{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID}
|
||||
}
|
||||
for _, copy := range copies {
|
||||
if copy.Index != opts.SrcPoolIdx && copy.ObjInfo.ModTime.After(opts.MTime) {
|
||||
return nil, DataMovementOverwriteErr{Bucket: bucket, Object: object, VersionID: opts.VersionID, Err: errDataMovementSuperseded}
|
||||
}
|
||||
}
|
||||
if winner := copies[0]; winner.Index != opts.SrcPoolIdx && winner.ObjInfo.ModTime.Equal(opts.MTime) {
|
||||
for _, copy := range copies {
|
||||
if copy.Index == opts.SrcPoolIdx && (copy.ObjInfo.ETag != winner.ObjInfo.ETag || copy.ObjInfo.DeleteMarker != winner.ObjInfo.DeleteMarker || copy.ObjInfo.Size != winner.ObjInfo.Size) {
|
||||
return nil, DataMovementOverwriteErr{Bucket: bucket, Object: object, VersionID: opts.VersionID, Err: errDataMovementSuperseded}
|
||||
}
|
||||
}
|
||||
}
|
||||
return func(ctx context.Context, b, o string, ropts ObjectOptions) (ObjectInfo, error) {
|
||||
if ropts.VersionID == "" {
|
||||
ropts.VersionID = nullVersionID
|
||||
}
|
||||
if b != bucket || o != object || ropts.VersionID != versionID {
|
||||
return z.replicaObjectInfo(ctx, b, o, ropts)
|
||||
}
|
||||
return replicaCopiesInfo(copies, err, bucket, object, versionID)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// removeMovedVersions retires only the source versions in the walker's
|
||||
// snapshot. A concurrent delete can create a new marker in the source pool;
|
||||
// deleting the whole prefix would remove that marker and expose moved data.
|
||||
func (z *erasureServerPools) removeMovedVersions(ctx context.Context, bucket, object string, source int, versions []FileInfo) error {
|
||||
object = encodeDirObject(object)
|
||||
lk := z.NewNSLock(bucket, object)
|
||||
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
for _, version := range versions {
|
||||
vid := version.VersionID
|
||||
if vid == "" {
|
||||
vid = nullVersionID
|
||||
}
|
||||
copies, err := z.objectPoolInfos(ctx, bucket, object, ObjectOptions{VersionID: vid, NoAuditLog: true})
|
||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var src *ObjectInfo
|
||||
var retained []PoolObjInfo
|
||||
for _, copy := range copies {
|
||||
if copy.Index == source {
|
||||
src = ©.ObjInfo
|
||||
} else {
|
||||
retained = append(retained, copy)
|
||||
}
|
||||
}
|
||||
if src == nil {
|
||||
continue
|
||||
}
|
||||
if !src.ModTime.Equal(version.ModTime) || src.DeleteMarker != version.Deleted {
|
||||
return PreConditionFailed{}
|
||||
}
|
||||
// Successful movement is not enough if the destination changed again.
|
||||
// Keep the source unless another copy still preserves its read result.
|
||||
if len(retained) == 0 || retained[0].ObjInfo.ModTime.Before(src.ModTime) {
|
||||
return PreConditionFailed{}
|
||||
}
|
||||
_, err = z.serverPools[source].DeleteObject(ctx, bucket, object, ObjectOptions{
|
||||
VersionID: vid, NoLock: true, NoAuditLog: true,
|
||||
SkipFreeVersion: true, // preserve the old prefix cleanup's tier-GC behavior
|
||||
})
|
||||
if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Healing also commits object metadata. Both queued healing and drive healing
|
||||
// must use the same namespace as pooled writes, rather than racing them under
|
||||
// a destination set's independent namespace.
|
||||
@@ -137,11 +257,15 @@ func (z *erasureServerPools) replicaObjectInfo(ctx context.Context, bucket, obje
|
||||
opts.VersionID = nullVersionID
|
||||
}
|
||||
copies, err := z.objectPoolInfos(ctx, bucket, object, opts)
|
||||
return replicaCopiesInfo(copies, err, bucket, object, opts.VersionID)
|
||||
}
|
||||
|
||||
func replicaCopiesInfo(copies []PoolObjInfo, err error, bucket, object, versionID string) (ObjectInfo, error) {
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
if copies[0].ObjInfo.DeleteMarker {
|
||||
return copies[0].ObjInfo, MethodNotAllowed{Bucket: bucket, Object: decodeDirObject(object), VersionID: opts.VersionID}
|
||||
return copies[0].ObjInfo, MethodNotAllowed{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID}
|
||||
}
|
||||
return mergedPoolObjectInfo(copies), nil
|
||||
}
|
||||
@@ -316,6 +440,11 @@ func (z *erasureServerPools) retireReplicaCopies(ctx context.Context, bucket, ob
|
||||
func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
|
||||
copies, err := z.objectPoolInfos(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
if opts.isVersionPurge() && (isErrObjectNotFound(err) || isErrVersionNotFound(err)) {
|
||||
if quorumErr := z.checkPurgeAbsent(ctx, bucket, object, opts.VersionID); quorumErr != nil {
|
||||
return ObjectInfo{}, quorumErr
|
||||
}
|
||||
}
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
primary := copies[0]
|
||||
@@ -365,6 +494,21 @@ func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket,
|
||||
opts.EvalMetadataFn = nil
|
||||
}
|
||||
}
|
||||
if opts.isVersionPurge() {
|
||||
// A missing pool may have only read-quorum absence. Verify every pool
|
||||
// omitted by lookup before mutating the known copies.
|
||||
for i, pool := range z.serverPools {
|
||||
found := false
|
||||
for _, copy := range copies {
|
||||
found = found || copy.Index == i
|
||||
}
|
||||
if !found {
|
||||
if err := pool.getHashedSet(object).checkPurgeAbsent(ctx, bucket, object, opts.VersionID); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if opts.VersionID == "" && (opts.Versioned || opts.VersionSuspended) {
|
||||
// A single new delete marker hides the current version. Older versions
|
||||
// remain history and must not be removed by an unqualified DELETE.
|
||||
@@ -390,3 +534,110 @@ func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket,
|
||||
oi.replicationDecision = opts.DeleteReplication.ReplicateDecisionStr
|
||||
return oi, err
|
||||
}
|
||||
|
||||
// deleteObjectsReconciled removes data while retaining each item's authoritative
|
||||
// copy until every other pool has completed its cleanup. The caller holds the
|
||||
// pools-layer write locks and separates marker creation from this batch.
|
||||
func (z *erasureServerPools) deleteObjectsReconciled(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
|
||||
deleted := make([]DeletedObject, len(objects))
|
||||
errs := make([]error, len(objects))
|
||||
primary := make([]int, len(objects))
|
||||
tiered := make([]bool, len(objects))
|
||||
lookup := errgroup.WithNErrs(len(objects)).WithConcurrency(32)
|
||||
for i, obj := range objects {
|
||||
// Preserve the full request identity even if lookup or cleanup fails.
|
||||
// The HTTP handler includes replication state in its response-index key.
|
||||
deleted[i] = DeletedObject{
|
||||
ObjectName: obj.ObjectName,
|
||||
VersionID: obj.VersionID,
|
||||
ReplicationState: obj.ReplicationState(),
|
||||
}
|
||||
primary[i] = -1
|
||||
lookup.Go(func() error {
|
||||
copies, err := z.objectPoolInfos(ctx, bucket, obj.ObjectName, ObjectOptions{VersionID: obj.VersionID, NoAuditLog: true})
|
||||
if err == nil {
|
||||
primary[i] = copies[0].Index
|
||||
for _, copy := range copies {
|
||||
tiered[i] = tiered[i] || copy.ObjInfo.TransitionedObject.Status == lifecycle.TransitionComplete
|
||||
}
|
||||
} else if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
if isErrReadQuorum(err) {
|
||||
err = InsufficientWriteQuorum{Bucket: bucket, Object: decodeDirObject(obj.ObjectName)}
|
||||
}
|
||||
errs[i] = err
|
||||
}
|
||||
return nil
|
||||
}, i)
|
||||
}
|
||||
lookup.Wait()
|
||||
|
||||
type poolBatch struct {
|
||||
objects []ObjectToDelete
|
||||
indices []int
|
||||
deleted []DeletedObject
|
||||
errs []error
|
||||
}
|
||||
for phase := 0; phase < 2; phase++ {
|
||||
batches := make([]poolBatch, len(z.serverPools))
|
||||
for i, obj := range objects {
|
||||
if errs[i] != nil || tiered[i] {
|
||||
continue
|
||||
}
|
||||
for p := range z.serverPools {
|
||||
if (p == primary[i]) != (phase == 1) {
|
||||
continue
|
||||
}
|
||||
batches[p].objects = append(batches[p].objects, obj)
|
||||
batches[p].indices = append(batches[p].indices, i)
|
||||
}
|
||||
}
|
||||
// Each phase still uses the set layer's bulk disk I/O and runs pools
|
||||
// concurrently. Include apparently absent pools in cleanup: read-quorum
|
||||
// absence is not proof that an offline drive holds no remaining copy.
|
||||
group := errgroup.WithNErrs(len(z.serverPools))
|
||||
for p, pool := range z.serverPools {
|
||||
if len(batches[p].objects) == 0 {
|
||||
continue
|
||||
}
|
||||
group.Go(func() error {
|
||||
batches[p].deleted, batches[p].errs = pool.DeleteObjects(ctx, bucket, batches[p].objects, opts)
|
||||
return nil
|
||||
}, p)
|
||||
}
|
||||
group.Wait()
|
||||
for _, batch := range batches {
|
||||
for j, i := range batch.indices {
|
||||
if errs[i] != nil {
|
||||
continue
|
||||
}
|
||||
if batch.errs[j] != nil {
|
||||
errs[i] = batch.errs[j]
|
||||
} else if phase == 1 || primary[i] == -1 {
|
||||
deleted[i] = batch.deleted[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Tiered copies can share one remote object, including restored copies.
|
||||
// Reuse the existing reconciliation which suppresses premature tier GC.
|
||||
// Serializing this uncommon path also keeps duplicate key/version entries
|
||||
// in one request from racing their metadata updates under the shared lock.
|
||||
for i, obj := range objects {
|
||||
if !tiered[i] || errs[i] != nil {
|
||||
continue
|
||||
}
|
||||
deleteOpts := opts
|
||||
deleteOpts.VersionID, deleteOpts.NoLock = obj.VersionID, true
|
||||
deleteOpts.DeleteReplication = obj.ReplicationState()
|
||||
if opts.PrefixEnabledFn != nil {
|
||||
deleteOpts.Versioned = opts.PrefixEnabledFn(obj.ObjectName)
|
||||
}
|
||||
_, err := z.deleteObjectReconciled(ctx, bucket, obj.ObjectName, deleteOpts)
|
||||
if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
errs[i] = err
|
||||
} else {
|
||||
deleted[i].found = true
|
||||
}
|
||||
}
|
||||
return deleted, errs
|
||||
}
|
||||
|
||||
@@ -1071,6 +1071,8 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
|
||||
restored bool
|
||||
unconditional bool
|
||||
skipFreeVersion bool
|
||||
bulk bool
|
||||
unversioned bool
|
||||
}{
|
||||
{name: "metadata-copy"},
|
||||
{name: "restored-metadata-copy", restored: true},
|
||||
@@ -1081,6 +1083,14 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
|
||||
{name: "ordinary-failed-primary-delete", deleting: true, unconditional: true, failPrimaryDelete: true},
|
||||
{name: "ordinary-successful-delete", deleting: true, unconditional: true},
|
||||
{name: "ordinary-skip-free-version", deleting: true, unconditional: true, skipFreeVersion: true},
|
||||
{name: "bulk-failed-primary-delete", deleting: true, bulk: true, failPrimaryDelete: true},
|
||||
{name: "bulk-restored-failed-primary-delete", deleting: true, bulk: true, restored: true, failPrimaryDelete: true},
|
||||
{name: "bulk-successful-delete", deleting: true, bulk: true},
|
||||
{name: "bulk-distinct-reference", deleting: true, bulk: true, differentRemote: true},
|
||||
{name: "bulk-skip-free-version", deleting: true, bulk: true, skipFreeVersion: true},
|
||||
{name: "unversioned-failed-primary-delete", deleting: true, unconditional: true, unversioned: true, failPrimaryDelete: true},
|
||||
{name: "unversioned-successful-delete", deleting: true, unconditional: true, unversioned: true},
|
||||
{name: "bulk-unversioned-failed-primary-delete", deleting: true, bulk: true, unversioned: true, failPrimaryDelete: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
@@ -1094,13 +1104,13 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
|
||||
if test.restored {
|
||||
metadata[xhttp.AmzRestore] = completedRestoreObj(time.Now().Add(time.Hour)).String()
|
||||
}
|
||||
oi := putConsistencyObject(t, z, bucket, object, 0, "data", ObjectOptions{Versioned: true, UserDefined: metadata})
|
||||
oi := putConsistencyObject(t, z, bucket, object, 0, "data", ObjectOptions{Versioned: !test.unversioned, UserDefined: metadata})
|
||||
secondaryMetadata := maps.Clone(metadata)
|
||||
if test.differentRemote {
|
||||
secondaryMetadata[ReservedMetadataPrefixLower+TransitionedObjectName] = "other-remote-object"
|
||||
}
|
||||
putConsistencyObject(t, z, bucket, object, 1, "data", ObjectOptions{
|
||||
Versioned: true, VersionID: oi.VersionID, MTime: oi.ModTime, UserDefined: secondaryMetadata,
|
||||
Versioned: !test.unversioned, VersionID: oi.VersionID, MTime: oi.ModTime, UserDefined: secondaryMetadata,
|
||||
})
|
||||
current, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: oi.VersionID})
|
||||
if err != nil || current.TransitionedObject.Status != "complete" || current.IsRemote() == test.restored {
|
||||
@@ -1113,7 +1123,11 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
getDisks := set.getDisks
|
||||
faulty := append([]StorageAPI(nil), getDisks()...)
|
||||
var failed, attempted atomic.Int32
|
||||
for i := range faulty {
|
||||
if test.bulk {
|
||||
faulty[i] = bulkDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object, mode: "eio", fail: true, failed: &failed, attempted: &attempted}
|
||||
}
|
||||
faulty[i] = consistencyDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object, version: oi.VersionID}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return faulty }
|
||||
@@ -1121,13 +1135,20 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
|
||||
}
|
||||
if test.deleting {
|
||||
opts := ObjectOptions{
|
||||
Versioned: true, VersionID: oi.VersionID, SkipFreeVersion: test.skipFreeVersion,
|
||||
Versioned: !test.unversioned, VersionID: oi.VersionID, SkipFreeVersion: test.skipFreeVersion,
|
||||
CheckPrecondFn: func(info ObjectInfo) bool { return info.ETag != current.ETag },
|
||||
}
|
||||
if test.unconditional {
|
||||
opts.CheckPrecondFn = nil
|
||||
}
|
||||
_, err := z.DeleteObject(t.Context(), bucket, object, opts)
|
||||
var err error
|
||||
if test.bulk {
|
||||
opts.CheckPrecondFn = nil
|
||||
_, errs := z.DeleteObjects(t.Context(), bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: object, VersionID: oi.VersionID}}}, opts)
|
||||
err = errs[0]
|
||||
} else {
|
||||
_, err = z.DeleteObject(t.Context(), bucket, object, opts)
|
||||
}
|
||||
if (err != nil) != test.failPrimaryDelete {
|
||||
t.Fatalf("unexpected authoritative delete result: %v", err)
|
||||
}
|
||||
@@ -1145,7 +1166,7 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
|
||||
_, err = z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: oi.VersionID})
|
||||
primaryDeleted := test.deleting && !test.failPrimaryDelete
|
||||
if primaryDeleted {
|
||||
if !isErrVersionNotFound(err) {
|
||||
if !isErrVersionNotFound(err) && (!test.unversioned || !isErrObjectNotFound(err)) {
|
||||
t.Fatalf("authoritative copy survived successful delete: %v", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
@@ -1306,3 +1327,182 @@ func TestPoolsDeleteDirectoryMarker(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An unconditional DELETE of an object without versions removes its copies in
|
||||
// every pool. When the delete fails in one pool, the latest copy must stay in
|
||||
// place: removing it anyway serves an older write of the object.
|
||||
func TestPoolsDeleteUnversionedReportsOtherPoolFailure(t *testing.T) {
|
||||
for _, offline := range []bool{false, true} {
|
||||
for latest := range 2 {
|
||||
t.Run(fmt.Sprintf("offline=%v/latest=%d", offline, latest), func(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
const object = "unversioned-failure"
|
||||
older := 1 - latest
|
||||
putConsistencyObject(t, z, bucket, object, older, "older", ObjectOptions{MaxParity: true, MTime: time.Now().Add(-time.Hour)})
|
||||
oi := putConsistencyObject(t, z, bucket, object, latest, "latest", ObjectOptions{MaxParity: true})
|
||||
set := z.serverPools[older].getHashedSet(object)
|
||||
getDisks := set.getDisks
|
||||
disks := append([]StorageAPI(nil), getDisks()...)
|
||||
for i := range disks {
|
||||
switch {
|
||||
case !offline:
|
||||
disks[i] = consistencyDeleteFaultDisk{StorageAPI: disks[i], bucket: bucket, object: object}
|
||||
case i < len(disks)/2:
|
||||
// An EC 8+8 copy stays readable, but a delete needs 9 drives.
|
||||
disks[i] = nil
|
||||
}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
defer func() { set.getDisks = getDisks }()
|
||||
if _, err := z.DeleteObject(t.Context(), bucket, object, ObjectOptions{}); err == nil {
|
||||
t.Error("delete reported success despite the other pool's write failure")
|
||||
}
|
||||
set.getDisks = getDisks
|
||||
if got, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); err != nil || got.ETag != oi.ETag {
|
||||
t.Errorf("after the failed delete the object reads ETag %q, %v; want the latest write %q", got.ETag, err, oi.ETag)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// consistencyBulkDeleteFaultDisk fails the multi-object deletes of one object.
|
||||
type consistencyBulkDeleteFaultDisk struct {
|
||||
StorageAPI
|
||||
bucket, object string
|
||||
}
|
||||
|
||||
func (d consistencyBulkDeleteFaultDisk) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions, opts DeleteOptions) []error {
|
||||
errs := make([]error, len(versions))
|
||||
var rest []FileInfoVersions
|
||||
var restIdx []int
|
||||
for i, v := range versions {
|
||||
if volume == d.bucket && v.Name == d.object {
|
||||
errs[i] = errDiskFull
|
||||
continue
|
||||
}
|
||||
rest = append(rest, v)
|
||||
restIdx = append(restIdx, i)
|
||||
}
|
||||
if len(rest) > 0 {
|
||||
for i, err := range d.StorageAPI.DeleteVersions(ctx, volume, rest, opts) {
|
||||
errs[restIdx[i]] = err
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
// A multi-object delete must not report an object or version as deleted while
|
||||
// a pool that failed to delete it still holds a copy.
|
||||
func TestPoolsDeleteObjectsReportsOtherPoolFailure(t *testing.T) {
|
||||
for _, versioned := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("versioned=%v", versioned), func(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
const object = "bulk-delete-failure"
|
||||
opts := ObjectOptions{Versioned: versioned}
|
||||
oi := putConsistencyObject(t, z, bucket, object, 0, "latest", opts)
|
||||
if versioned {
|
||||
// The copy a data movement leaves until it removes the source.
|
||||
putConsistencyObject(t, z, bucket, object, 1, "latest", ObjectOptions{Versioned: true, VersionID: oi.VersionID, MTime: oi.ModTime})
|
||||
} else {
|
||||
putConsistencyObject(t, z, bucket, object, 1, "older", ObjectOptions{MTime: time.Now().Add(-time.Hour)})
|
||||
}
|
||||
set := z.serverPools[1].getHashedSet(object)
|
||||
getDisks := set.getDisks
|
||||
faulty := append([]StorageAPI(nil), getDisks()...)
|
||||
for i := range faulty {
|
||||
faulty[i] = consistencyBulkDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return faulty }
|
||||
defer func() { set.getDisks = getDisks }()
|
||||
_, errs := z.DeleteObjects(t.Context(), bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: object, VersionID: oi.VersionID}}}, opts)
|
||||
if errs[0] == nil {
|
||||
t.Error("multi-object delete reported success despite the other pool's write failure")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Pools delete a multi-object batch in parallel. With half the drives of the
|
||||
// older copy's set offline, an EC 8+8 copy stays readable but cannot be
|
||||
// deleted: the batch must fail the object and keep its latest copy, instead of
|
||||
// deleting it and serving the older write once the drives are back.
|
||||
func TestPoolsDeleteObjectsKeepsLatestWithoutDeleteQuorum(t *testing.T) {
|
||||
for latest := range 2 {
|
||||
t.Run(fmt.Sprintf("latest=%d", latest), func(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
const object = "bulk-delete-quorum"
|
||||
older := 1 - latest
|
||||
putConsistencyObject(t, z, bucket, object, older, "older", ObjectOptions{MaxParity: true, MTime: time.Now().Add(-time.Hour)})
|
||||
oi := putConsistencyObject(t, z, bucket, object, latest, "latest", ObjectOptions{MaxParity: true})
|
||||
set := z.serverPools[older].getHashedSet(object)
|
||||
getDisks := set.getDisks
|
||||
disks := append([]StorageAPI(nil), getDisks()...)
|
||||
for i := range len(disks) / 2 {
|
||||
disks[i] = nil
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
defer func() { set.getDisks = getDisks }()
|
||||
_, errs := z.DeleteObjects(t.Context(), bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: object}}}, ObjectOptions{})
|
||||
if !errors.As(errs[0], &InsufficientWriteQuorum{}) {
|
||||
t.Errorf("multi-object delete: got %v, want InsufficientWriteQuorum", errs[0])
|
||||
}
|
||||
set.getDisks = getDisks
|
||||
if got, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); err != nil || got.ETag != oi.ETag {
|
||||
t.Errorf("after the failed delete the object reads ETag %q, %v; want the latest write %q", got.ETag, err, oi.ETag)
|
||||
}
|
||||
// Nothing was deleted, so the retry removes every copy.
|
||||
if _, errs = z.DeleteObjects(t.Context(), bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: object}}}, ObjectOptions{}); errs[0] != nil {
|
||||
t.Fatal(errs[0])
|
||||
}
|
||||
for i, pool := range z.serverPools {
|
||||
if _, err := pool.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); !isErrObjectNotFound(err) {
|
||||
t.Errorf("pool %d retained the object after the retry: %v", i, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A delete marker hides the object in every pool. When the versions of an
|
||||
// object are in two pools and the set of one of them cannot take a write, the
|
||||
// marker goes to the other pool instead of failing the delete. The versions
|
||||
// stay below the marker.
|
||||
func TestPoolsDeleteMarkerSkipsSetWithoutQuorum(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
degraded int // pool whose set loses drives
|
||||
offline int
|
||||
}{
|
||||
{"older version readable", 0, 8},
|
||||
{"older version unreadable", 0, 9},
|
||||
{"latest version readable", 1, 8},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
const object = "marker-quorum"
|
||||
v1 := putConsistencyObject(t, z, bucket, object, 0, "v1", ObjectOptions{Versioned: true, MaxParity: true, MTime: time.Now().Add(-time.Hour)})
|
||||
v2 := putConsistencyObject(t, z, bucket, object, 1, "v2", ObjectOptions{Versioned: true, MaxParity: true})
|
||||
set := z.serverPools[tc.degraded].getHashedSet(object)
|
||||
getDisks := set.getDisks
|
||||
disks := append([]StorageAPI(nil), getDisks()...)
|
||||
for i := range tc.offline {
|
||||
disks[i] = nil
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
defer func() { set.getDisks = getDisks }()
|
||||
if _, err := z.DeleteObject(t.Context(), bucket, object, ObjectOptions{Versioned: true}); err != nil {
|
||||
t.Fatalf("delete with the set of pool %d without quorum: %v", tc.degraded, err)
|
||||
}
|
||||
set.getDisks = getDisks
|
||||
if _, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); !isErrObjectNotFound(err) && !isErrMethodNotAllowed(err) {
|
||||
t.Errorf("object still readable after the delete: %v", err)
|
||||
}
|
||||
for pool, v := range []ObjectInfo{v1, v2} {
|
||||
if _, err := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: v.VersionID}); err != nil {
|
||||
t.Errorf("pool %d lost version %s: %v", pool, v.VersionID, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -616,9 +616,15 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, idx int, bu
|
||||
}
|
||||
|
||||
if objInfo.isMultipart() {
|
||||
// Completion repeats this check under the object write lock.
|
||||
if _, err := z.checkDataMovementCopies(ctx, bucket, encodeDirObject(objInfo.Name), ObjectOptions{
|
||||
VersionID: objInfo.VersionID, MTime: objInfo.ModTime, SrcPoolIdx: idx,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
res, err := z.NewMultipartUpload(ctx, bucket, objInfo.Name, ObjectOptions{
|
||||
VersionID: objInfo.VersionID,
|
||||
UserDefined: objInfo.UserDefined,
|
||||
UserDefined: migrationObjectMetadata(objInfo),
|
||||
NoAuditLog: true,
|
||||
SrcPoolIdx: idx,
|
||||
DataMovement: true,
|
||||
@@ -656,6 +662,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, idx int, bu
|
||||
}
|
||||
}
|
||||
_, err = z.CompleteMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, parts, ObjectOptions{
|
||||
VersionID: objInfo.VersionID,
|
||||
SrcPoolIdx: idx,
|
||||
DataMovement: true,
|
||||
MTime: objInfo.ModTime,
|
||||
@@ -681,7 +688,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, idx int, bu
|
||||
SrcPoolIdx: idx,
|
||||
VersionID: objInfo.VersionID,
|
||||
MTime: objInfo.ModTime,
|
||||
UserDefined: objInfo.UserDefined,
|
||||
UserDefined: migrationObjectMetadata(objInfo),
|
||||
PreserveETag: objInfo.ETag, // Preserve original ETag to ensure same metadata.
|
||||
IndexCB: func() []byte {
|
||||
return objInfo.Parts[0].Index // Preserve part Index to ensure decompression works.
|
||||
@@ -825,27 +832,17 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
|
||||
// to create the appropriate stack.
|
||||
versionsSorter(fivs.Versions).reverse()
|
||||
|
||||
var decommissioned, expired int
|
||||
var decommissioned int
|
||||
for _, version := range fivs.Versions {
|
||||
stopFn := globalDecommissionMetrics.log(decomMetricDecommissionObject, idx, bi.Name, version.Name, version.VersionID)
|
||||
// Apply lifecycle rules on the objects that are expired.
|
||||
if filterLifecycle(bi.Name, version.Name, version) {
|
||||
expired++
|
||||
decommissioned++
|
||||
stopFn(version.Size, errors.New("ILM expired object/version will be skipped"))
|
||||
continue
|
||||
}
|
||||
|
||||
// any object with only single DEL marker we don't need
|
||||
// to decommission, just skip it, this also includes
|
||||
// any other versions that have already expired.
|
||||
remainingVersions := len(fivs.Versions) - expired
|
||||
if version.Deleted && remainingVersions == 1 && rcfg == nil {
|
||||
decommissioned++
|
||||
stopFn(version.Size, errors.New("DELETE marked object with no other non-current versions will be skipped"))
|
||||
continue
|
||||
}
|
||||
|
||||
// Preserve every marker: older versions may live in another pool.
|
||||
versionID := version.VersionID
|
||||
if versionID == "" {
|
||||
versionID = nullVersionID
|
||||
@@ -870,6 +867,9 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
|
||||
SkipDecommissioned: true, // make sure we skip the decommissioned pool
|
||||
NoAuditLog: true,
|
||||
})
|
||||
if isDataMovementSuperseded(err) {
|
||||
err = nil
|
||||
}
|
||||
if err != nil {
|
||||
// This can happen when rebalance stop races with ongoing rebalance workers.
|
||||
// These rebalance failures can be ignored.
|
||||
@@ -898,17 +898,19 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
|
||||
// gr.Close() is ensured by decommissionObject().
|
||||
for range 3 {
|
||||
if version.IsRemote() {
|
||||
if err := z.DecomTieredObject(ctx, bi.Name, version.Name, version, ObjectOptions{
|
||||
err := z.DecomTieredObject(ctx, bi.Name, version.Name, version, ObjectOptions{
|
||||
VersionID: versionID,
|
||||
MTime: version.ModTime,
|
||||
UserDefined: version.Metadata,
|
||||
SrcPoolIdx: idx,
|
||||
DataMovement: true,
|
||||
}); err != nil {
|
||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
|
||||
ignore = true
|
||||
stopFn(0, nil)
|
||||
}
|
||||
})
|
||||
if isDataMovementSuperseded(err) {
|
||||
err = nil
|
||||
}
|
||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
|
||||
ignore = true
|
||||
stopFn(0, nil)
|
||||
}
|
||||
if !ignore {
|
||||
stopFn(version.Size, err)
|
||||
@@ -949,7 +951,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
|
||||
stopFn(version.Size, err)
|
||||
continue
|
||||
}
|
||||
if err = z.decommissionObject(ctx, idx, bi.Name, gr); err != nil {
|
||||
if err = z.decommissionObject(ctx, idx, bi.Name, gr); err != nil && !isDataMovementSuperseded(err) {
|
||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
|
||||
ignore = true
|
||||
stopFn(0, nil)
|
||||
@@ -979,15 +981,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
|
||||
// if all versions were decommissioned, then we can delete the object versions.
|
||||
if decommissioned == len(fivs.Versions) {
|
||||
stopFn := globalDecommissionMetrics.log(decomMetricDecommissionRemoveObject, idx, bi.Name, entry.name)
|
||||
_, err := set.DeleteObject(ctx,
|
||||
bi.Name,
|
||||
encodeDirObject(entry.name),
|
||||
ObjectOptions{
|
||||
DeletePrefix: true, // use prefix delete to delete all versions at once.
|
||||
DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
|
||||
NoAuditLog: true,
|
||||
},
|
||||
)
|
||||
err := z.removeMovedVersions(ctx, bi.Name, entry.name, idx, fivs.Versions)
|
||||
stopFn(0, err)
|
||||
auditLogDecom(ctx, "DecomDeleteObject", bi.Name, entry.name, "", err)
|
||||
if err != nil {
|
||||
@@ -1196,10 +1190,6 @@ func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error
|
||||
ignored++
|
||||
continue
|
||||
}
|
||||
if version.Deleted {
|
||||
ignored++
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
versionsFound += len(fivs.Versions) - ignored
|
||||
|
||||
@@ -0,0 +1,582 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
minio "github.com/minio/minio-go/v7"
|
||||
)
|
||||
|
||||
func bulkDeletePools(t *testing.T, n int) (*erasureServerPools, string) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
dirs, err := getRandomDisks(4 * n)
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatal(err)
|
||||
}
|
||||
var endpoints EndpointServerPools
|
||||
for i := range n {
|
||||
endpoints = append(endpoints, mustGetPoolEndpoints(i, dirs[4*i:4*i+4]...)...)
|
||||
}
|
||||
obj, _, err := initObjectLayer(ctx, endpoints)
|
||||
if err != nil {
|
||||
cancel()
|
||||
removeRoots(dirs)
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
previous := newObjectLayerFn()
|
||||
setObjectLayer(z)
|
||||
t.Cleanup(func() { cancel(); z.Shutdown(context.Background()); removeRoots(dirs); setObjectLayer(previous) })
|
||||
bucket := "bulk-delete-consistency"
|
||||
if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return z, bucket
|
||||
}
|
||||
|
||||
func bulkDeleteBody(t *testing.T, z *erasureServerPools, bucket, object, version string) string {
|
||||
t.Helper()
|
||||
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: version})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
body, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(body)
|
||||
}
|
||||
|
||||
// All drives are online and all metadata reads succeed. Failure starts inside
|
||||
// DeleteVersions, after any preflight. Count attempts to delete the latest copy
|
||||
// instead of waiting for them: a safe implementation must never make that call.
|
||||
type bulkDeleteFaultDisk struct {
|
||||
StorageAPI
|
||||
bucket, object, mode string
|
||||
fail, latest bool
|
||||
failed, attempted *atomic.Int32
|
||||
}
|
||||
|
||||
func (d bulkDeleteFaultDisk) DeleteVersions(ctx context.Context, bucket string, versions []FileInfoVersions, opts DeleteOptions) []error {
|
||||
if bucket != d.bucket {
|
||||
return d.StorageAPI.DeleteVersions(ctx, bucket, versions, opts)
|
||||
}
|
||||
errs := make([]error, len(versions))
|
||||
for i, v := range versions {
|
||||
if v.Name != d.object {
|
||||
errs[i] = d.StorageAPI.DeleteVersions(ctx, bucket, []FileInfoVersions{v}, opts)[0]
|
||||
continue
|
||||
}
|
||||
if d.latest {
|
||||
d.attempted.Add(1)
|
||||
}
|
||||
if !d.fail {
|
||||
errs[i] = d.StorageAPI.DeleteVersions(ctx, bucket, []FileInfoVersions{v}, opts)[0]
|
||||
continue
|
||||
}
|
||||
d.failed.Add(1)
|
||||
switch d.mode {
|
||||
case "eio":
|
||||
errs[i] = syscall.EIO
|
||||
case "disconnect":
|
||||
errs[i] = errDiskNotFound
|
||||
case "permission":
|
||||
meta := filepath.Join(d.String(), bucket, v.Name, xlStorageFormatFile)
|
||||
info, err := os.Stat(meta)
|
||||
if err == nil {
|
||||
err = os.Chmod(meta, 0)
|
||||
}
|
||||
if err != nil {
|
||||
errs[i] = err
|
||||
continue
|
||||
}
|
||||
errs[i] = d.StorageAPI.DeleteVersions(ctx, bucket, []FileInfoVersions{v}, opts)[0]
|
||||
if err := os.Chmod(meta, info.Mode()); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func requireBulkDeletePermissionFailure(t *testing.T) {
|
||||
t.Helper()
|
||||
probe := filepath.Join(t.TempDir(), "permission-probe")
|
||||
if err := os.WriteFile(probe, []byte("metadata"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chmod(probe, 0); err != nil {
|
||||
t.Skipf("cannot inject filesystem permissions: %v", err)
|
||||
}
|
||||
_, err := os.ReadFile(probe)
|
||||
if restoreErr := os.Chmod(probe, 0o600); restoreErr != nil {
|
||||
t.Fatal(restoreErr)
|
||||
}
|
||||
if !os.IsPermission(err) {
|
||||
t.Skip("this user/filesystem can read mode-000 files")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsBulkDeleteKeepsLatestOnIOFailure(t *testing.T) {
|
||||
for _, mode := range []string{"eio", "disconnect", "permission"} {
|
||||
for _, n := range []int{2, 3} {
|
||||
for primary := range n {
|
||||
for _, kind := range []string{"unversioned", "null", "uuid"} {
|
||||
t.Run(fmt.Sprintf("%s/pools=%d/primary=%d/%s", mode, n, primary, kind), func(t *testing.T) {
|
||||
if mode == "permission" {
|
||||
requireBulkDeletePermissionFailure(t)
|
||||
}
|
||||
z, bucket := bulkDeletePools(t, n)
|
||||
object, old := "target", (primary+1)%n
|
||||
opts := ObjectOptions{MTime: UTCNow().Add(-time.Hour), MaxParity: true}
|
||||
oldBody := "old"
|
||||
if kind == "uuid" {
|
||||
opts.Versioned, opts.VersionID, oldBody = true, mustGetUUID(), "latest"
|
||||
}
|
||||
putConsistencyObject(t, z, bucket, object, old, oldBody, opts)
|
||||
opts.MTime = opts.MTime.Add(time.Minute)
|
||||
latest := putConsistencyObject(t, z, bucket, object, primary, "latest", opts)
|
||||
version := opts.VersionID
|
||||
if kind == "null" {
|
||||
version = nullVersionID
|
||||
}
|
||||
for _, key := range []string{"before", "after"} {
|
||||
putConsistencyObject(t, z, bucket, key, primary, "unrelated", ObjectOptions{})
|
||||
}
|
||||
var failed, attempted atomic.Int32
|
||||
var restore []func()
|
||||
for _, p := range []int{old, primary} {
|
||||
set := z.serverPools[p].getHashedSet(object)
|
||||
original := set.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i, disk := range disks {
|
||||
disks[i] = bulkDeleteFaultDisk{StorageAPI: disk, bucket: bucket, object: object, mode: mode, fail: p == old, latest: p == primary, failed: &failed, attempted: &attempted}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
restore = append(restore, func() { set.getDisks = original })
|
||||
}
|
||||
defer func() {
|
||||
for _, f := range restore {
|
||||
f()
|
||||
}
|
||||
}()
|
||||
objects := []ObjectToDelete{{ObjectV: ObjectV{ObjectName: "before"}}, {ObjectV: ObjectV{ObjectName: object, VersionID: version}}, {ObjectV: ObjectV{ObjectName: "after"}}}
|
||||
deleted, errs := z.DeleteObjects(t.Context(), bucket, objects, ObjectOptions{})
|
||||
if failed.Load() != 4 || attempted.Load() != 0 {
|
||||
t.Errorf("cleanup failures=%d; latest deletion attempts=%d, want 4/0", failed.Load(), attempted.Load())
|
||||
}
|
||||
if errs[1] == nil {
|
||||
t.Error("cleanup failure was reported as success")
|
||||
}
|
||||
for _, i := range []int{0, 2} {
|
||||
if errs[i] != nil || deleted[i].ObjectName != objects[i].ObjectName {
|
||||
t.Errorf("unrelated item %d: %+v, %v", i, deleted[i], errs[i])
|
||||
}
|
||||
}
|
||||
if body := bulkDeleteBody(t, z, bucket, object, version); body != "latest" {
|
||||
t.Errorf("GET during failure=%q", body)
|
||||
}
|
||||
for _, f := range restore {
|
||||
f()
|
||||
}
|
||||
if body := bulkDeleteBody(t, z, bucket, object, version); body != "latest" {
|
||||
t.Errorf("GET after recovery=%q", body)
|
||||
}
|
||||
listed, err := z.ListObjectVersions(t.Context(), bucket, object, "", "", "", 100)
|
||||
if err != nil || len(listed.Objects) != 1 || listed.Objects[0].ETag != latest.ETag {
|
||||
t.Errorf("recovered versions=%+v, %v", listed.Objects, err)
|
||||
}
|
||||
_, retry := z.DeleteObjects(t.Context(), bucket, objects[1:2], ObjectOptions{})
|
||||
if retry[0] != nil {
|
||||
t.Fatal(retry[0])
|
||||
}
|
||||
_, err = z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: version})
|
||||
if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
t.Errorf("retry retained the object: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsBulkDeleteReplicationIdentity(t *testing.T) {
|
||||
z, _ := bulkDeletePools(t, 2)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
bucket, router, err := initAPIHandlerTest(ctx, z, nil, MakeBucketOptions{VersioningEnabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Error("unexpected outbound replication for a failed deletion")
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
defer remote.Close()
|
||||
client, err := minio.New(strings.TrimPrefix(remote.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, restoreReplication := r5ReplicationFixture(t, z, bucket, client)
|
||||
defer restoreReplication()
|
||||
versions := make(map[string]string)
|
||||
for _, key := range []string{"first", "second"} {
|
||||
v := putConsistencyObject(t, z, bucket, key, 0, "payload", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
|
||||
putConsistencyObject(t, z, bucket, key, 1, "payload", ObjectOptions{Versioned: true, VersionID: v.VersionID, MTime: v.ModTime})
|
||||
versions[key] = v.VersionID
|
||||
}
|
||||
set := z.serverPools[0].getHashedSet("first")
|
||||
original := set.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
disks[0], disks[1] = nil, nil
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
defer func() { set.getDisks = original }()
|
||||
body := []byte(fmt.Sprintf("<Delete><Object><Key>first</Key><VersionId>%s</VersionId></Object><Object><Key>second</Key><VersionId>%s</VersionId></Object></Delete>", versions["first"], versions["second"]))
|
||||
req, err := newTestSignedRequestV4(http.MethodPost, getMultiDeleteObjectURL("", bucket), int64(len(body)), bytes.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
var result struct {
|
||||
Errors []struct {
|
||||
Key, Code string
|
||||
VersionID string `xml:"VersionId"`
|
||||
} `xml:"Error"`
|
||||
Deleted []struct{ Key string } `xml:"Deleted"`
|
||||
}
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec.Code != http.StatusOK || len(result.Errors) != 2 || len(result.Deleted) != 0 {
|
||||
t.Fatalf("lost response identity: HTTP %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
for _, e := range result.Errors {
|
||||
if versions[e.Key] == "" || e.VersionID != versions[e.Key] || e.Code != "SlowDownWrite" {
|
||||
t.Errorf("incorrect error identity: %+v", e)
|
||||
}
|
||||
delete(versions, e.Key)
|
||||
}
|
||||
if len(versions) != 0 {
|
||||
t.Errorf("missing errors: %v", versions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsBulkDeleteNullBeforeMarker(t *testing.T) {
|
||||
for _, n := range []int{1, 2, 3} {
|
||||
t.Run(fmt.Sprintf("pools=%d", n), func(t *testing.T) {
|
||||
z, bucket := bulkDeletePools(t, n)
|
||||
object := "suspended"
|
||||
history := putConsistencyObject(t, z, bucket, object, 0, "history", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
|
||||
for p := range n {
|
||||
putConsistencyObject(t, z, bucket, object, p, "null", ObjectOptions{MTime: UTCNow().Add(-time.Duration(p+1) * time.Minute)})
|
||||
}
|
||||
objects := []ObjectToDelete{{ObjectV: ObjectV{ObjectName: object}}, {ObjectV: ObjectV{ObjectName: object, VersionID: nullVersionID}}}
|
||||
deleted, errs := z.DeleteObjects(t.Context(), bucket, objects, ObjectOptions{VersionSuspended: true})
|
||||
if errs[0] != nil || errs[1] != nil || !deleted[0].DeleteMarker || deleted[1].VersionID != nullVersionID {
|
||||
t.Fatalf("results=%+v errors=%v", deleted, errs)
|
||||
}
|
||||
info, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||
if !info.DeleteMarker || !isErrObjectNotFound(err) {
|
||||
t.Errorf("replacement null marker lost: %+v, %v", info, err)
|
||||
}
|
||||
if body := bulkDeleteBody(t, z, bucket, object, history.VersionID); body != "history" {
|
||||
t.Errorf("UUID history changed: %q", body)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsBulkDeletePartialCleanupRetry(t *testing.T) {
|
||||
for _, kind := range []string{"unversioned", "null", "uuid"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
z, bucket := bulkDeletePools(t, 2)
|
||||
object := "partial-cleanup"
|
||||
opts := ObjectOptions{MTime: UTCNow().Add(-time.Hour)}
|
||||
oldBody := "old"
|
||||
if kind == "uuid" {
|
||||
opts.Versioned, opts.VersionID, oldBody = true, mustGetUUID(), "latest"
|
||||
}
|
||||
putConsistencyObject(t, z, bucket, object, 0, oldBody, opts)
|
||||
opts.MTime = UTCNow()
|
||||
putConsistencyObject(t, z, bucket, object, 1, "latest", opts)
|
||||
version := opts.VersionID
|
||||
if kind == "null" {
|
||||
version = nullVersionID
|
||||
}
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
original := set.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
var failed, attempted atomic.Int32
|
||||
for i := range 2 {
|
||||
disks[i] = bulkDeleteFaultDisk{StorageAPI: disks[i], bucket: bucket, object: object, mode: "eio", fail: true, failed: &failed, attempted: &attempted}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
defer func() { set.getDisks = original }()
|
||||
objects := []ObjectToDelete{{ObjectV: ObjectV{ObjectName: object, VersionID: version}}}
|
||||
_, errs := z.DeleteObjects(t.Context(), bucket, objects, ObjectOptions{})
|
||||
if errs[0] == nil || failed.Load() != 2 {
|
||||
t.Fatalf("did not partially fail cleanup: %v, failures=%d", errs, failed.Load())
|
||||
}
|
||||
set.getDisks = original
|
||||
if got := bulkDeleteBody(t, z, bucket, object, version); got != "latest" {
|
||||
t.Fatalf("latest lost after partial cleanup: %q", got)
|
||||
}
|
||||
_, errs = z.DeleteObjects(t.Context(), bucket, objects, ObjectOptions{})
|
||||
if errs[0] != nil {
|
||||
t.Fatalf("recovered retry must combine successful removals and already-absent replies: %v", errs[0])
|
||||
}
|
||||
_, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: version})
|
||||
if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
t.Errorf("retry retained object: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsBulkDeleteDifferentPrimaries(t *testing.T) {
|
||||
for _, n := range []int{2, 3} {
|
||||
t.Run(fmt.Sprintf("pools=%d", n), func(t *testing.T) {
|
||||
z, bucket := bulkDeletePools(t, n)
|
||||
objects := make([]ObjectToDelete, n)
|
||||
for primary := range n {
|
||||
object := fmt.Sprintf("key-%d", primary)
|
||||
objects[primary].ObjectName = object
|
||||
putConsistencyObject(t, z, bucket, object, (primary+1)%n, "old", ObjectOptions{MTime: UTCNow().Add(-time.Hour)})
|
||||
putConsistencyObject(t, z, bucket, object, primary, "latest", ObjectOptions{})
|
||||
}
|
||||
set := z.serverPools[0].getHashedSet(objects[n-1].ObjectName)
|
||||
original := set.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
var failed, attempted atomic.Int32
|
||||
for i, disk := range disks {
|
||||
disks[i] = bulkDeleteFaultDisk{StorageAPI: disk, bucket: bucket, object: objects[n-1].ObjectName, mode: "eio", fail: true, failed: &failed, attempted: &attempted}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
defer func() { set.getDisks = original }()
|
||||
deleted, errs := z.DeleteObjects(t.Context(), bucket, objects, ObjectOptions{})
|
||||
for i := range objects {
|
||||
if deleted[i].ObjectName != objects[i].ObjectName {
|
||||
t.Errorf("wrong result index %d: %+v", i, deleted[i])
|
||||
}
|
||||
if i == n-1 {
|
||||
if errs[i] == nil || bulkDeleteBody(t, z, bucket, objects[i].ObjectName, "") != "latest" {
|
||||
t.Errorf("failed item lost latest: %v", errs[i])
|
||||
}
|
||||
continue
|
||||
}
|
||||
_, err := z.GetObjectInfo(t.Context(), bucket, objects[i].ObjectName, ObjectOptions{})
|
||||
if errs[i] != nil || !isErrObjectNotFound(err) {
|
||||
t.Errorf("healthy item %d: delete=%v GET=%v", i, errs[i], err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsBulkDeleteTieAndUnreadable(t *testing.T) {
|
||||
for _, unreadable := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("unreadable=%t", unreadable), func(t *testing.T) {
|
||||
z, bucket := bulkDeletePools(t, 3)
|
||||
object := "same-time"
|
||||
stamp := UTCNow().Add(-time.Hour)
|
||||
for p := range 3 {
|
||||
putConsistencyObject(t, z, bucket, object, p, fmt.Sprintf("pool-%d", p), ObjectOptions{MTime: stamp})
|
||||
}
|
||||
if got := bulkDeleteBody(t, z, bucket, object, ""); got != "pool-0" {
|
||||
t.Fatalf("unexpected read winner: %q", got)
|
||||
}
|
||||
set := z.serverPools[2].getHashedSet(object)
|
||||
original := set.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
var failed, attempted atomic.Int32
|
||||
for i, disk := range disks {
|
||||
if unreadable {
|
||||
disks[i] = consistencyReadFaultDisk{StorageAPI: disk, bucket: bucket, object: object}
|
||||
} else {
|
||||
disks[i] = bulkDeleteFaultDisk{StorageAPI: disk, bucket: bucket, object: object, mode: "eio", fail: true, failed: &failed, attempted: &attempted}
|
||||
}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
defer func() { set.getDisks = original }()
|
||||
_, errs := z.DeleteObjects(t.Context(), bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: object}}}, ObjectOptions{})
|
||||
set.getDisks = original
|
||||
if errs[0] == nil || bulkDeleteBody(t, z, bucket, object, "") != "pool-0" {
|
||||
t.Errorf("lost tie winner or unreadable-pool error: %v", errs[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type bulkDeletePauseDisk struct {
|
||||
StorageAPI
|
||||
bucket, object string
|
||||
entered chan<- struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (d bulkDeletePauseDisk) DeleteVersions(ctx context.Context, bucket string, versions []FileInfoVersions, opts DeleteOptions) []error {
|
||||
for _, v := range versions {
|
||||
if bucket != d.bucket || v.Name != d.object {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case d.entered <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
select {
|
||||
case <-d.release:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
errs := make([]error, len(versions))
|
||||
for i := range errs {
|
||||
errs[i] = syscall.EIO
|
||||
}
|
||||
return errs
|
||||
}
|
||||
return d.StorageAPI.DeleteVersions(ctx, bucket, versions, opts)
|
||||
}
|
||||
|
||||
func TestPoolsBulkDeleteLockAndCancellation(t *testing.T) {
|
||||
for _, canceled := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("cancel=%t", canceled), func(t *testing.T) {
|
||||
z, bucket := bulkDeletePools(t, 2)
|
||||
object := "locked"
|
||||
putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{MTime: UTCNow().Add(-time.Hour)})
|
||||
putConsistencyObject(t, z, bucket, object, 1, "latest", ObjectOptions{})
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
original := set.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
entered, release := make(chan struct{}, 4), make(chan struct{})
|
||||
for i, disk := range disks {
|
||||
disks[i] = bulkDeletePauseDisk{StorageAPI: disk, bucket: bucket, object: object, entered: entered, release: release}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
defer func() { set.getDisks = original }()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, errs := z.DeleteObjects(ctx, bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: object}}}, ObjectOptions{})
|
||||
done <- errs[0]
|
||||
}()
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("old-copy cleanup did not start")
|
||||
}
|
||||
if canceled {
|
||||
cancel()
|
||||
if err := <-done; err == nil {
|
||||
t.Fatal("canceled cleanup succeeded")
|
||||
}
|
||||
set.getDisks = original
|
||||
if got := bulkDeleteBody(t, z, bucket, object, ""); got != "latest" {
|
||||
t.Errorf("cancellation lost latest: %q", got)
|
||||
}
|
||||
return
|
||||
}
|
||||
putDone := make(chan error, 1)
|
||||
reader := mustGetPutObjReader(t, bytes.NewBufferString("after-put"), 9, "", "")
|
||||
go func() {
|
||||
_, err := z.PutObject(t.Context(), bucket, object, reader, ObjectOptions{})
|
||||
putDone <- err
|
||||
}()
|
||||
select {
|
||||
case err := <-putDone:
|
||||
t.Fatalf("PUT passed the held DELETE lock: %v", err)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
close(release)
|
||||
if err := <-done; err == nil {
|
||||
t.Error("cleanup failure was lost")
|
||||
}
|
||||
if err := <-putDone; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := bulkDeleteBody(t, z, bucket, object, ""); got != "after-put" {
|
||||
t.Errorf("concurrent PUT lost: %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type bulkDeleteReplyDisk struct {
|
||||
StorageAPI
|
||||
err error
|
||||
}
|
||||
|
||||
func (d bulkDeleteReplyDisk) IsOnline() bool { return true }
|
||||
|
||||
func (d bulkDeleteReplyDisk) DeleteVersions(_ context.Context, _ string, versions []FileInfoVersions, _ DeleteOptions) []error {
|
||||
errs := make([]error, len(versions))
|
||||
for i := range errs {
|
||||
errs[i] = d.err
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func TestBulkDeleteQuorumEvidence(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
obj ObjectToDelete
|
||||
opts ObjectOptions
|
||||
ok bool
|
||||
}{
|
||||
{name: "unversioned", ok: true},
|
||||
{name: "null", obj: ObjectToDelete{ObjectV: ObjectV{VersionID: nullVersionID}}, ok: true},
|
||||
{name: "uuid", obj: ObjectToDelete{ObjectV: ObjectV{VersionID: mustGetUUID()}}, ok: true},
|
||||
{name: "marker", opts: ObjectOptions{Versioned: true}},
|
||||
{name: "suspended-marker", opts: ObjectOptions{VersionSuspended: true}},
|
||||
{name: "pending-purge", obj: ObjectToDelete{ObjectV: ObjectV{VersionID: mustGetUUID()}, VersionPurgeStatuses: "target=PENDING;"}},
|
||||
{name: "replicated-marker", obj: ObjectToDelete{DeleteMarkerReplicationStatus: "target=PENDING;"}, opts: ObjectOptions{Versioned: true}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
disks := []StorageAPI{
|
||||
bulkDeleteReplyDisk{},
|
||||
bulkDeleteReplyDisk{},
|
||||
bulkDeleteReplyDisk{err: errFileNotFound},
|
||||
bulkDeleteReplyDisk{err: errFileVersionNotFound},
|
||||
}
|
||||
er := erasureObjects{getDisks: func() []StorageAPI { return disks }}
|
||||
tc.obj.ObjectName, tc.opts.NoAuditLog = "object", true
|
||||
_, errs := er.DeleteObjects(t.Context(), "bucket", []ObjectToDelete{tc.obj}, tc.opts)
|
||||
if (errs[0] == nil) != tc.ok {
|
||||
t.Errorf("two writes + two absent replies: error=%v, want success=%t", errs[0], tc.ok)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Both walkers must preserve markers even when all older versions reside in
|
||||
// another pool. Local version counts cannot prove that a marker is redundant.
|
||||
func TestPoolsMigrationDeleteMarkers(t *testing.T) {
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for _, pools := range []int{2, 3} {
|
||||
for _, null := range []bool{false, true} {
|
||||
for _, history := range []string{"none", "source", "destination"} {
|
||||
t.Run(fmt.Sprintf("%s/pools=%d/null=%t/history=%s", kind, pools, null, history), func(t *testing.T) {
|
||||
var z *erasureServerPools
|
||||
var bucket string
|
||||
if pools == 2 {
|
||||
z, bucket = migrationTestPools(t)
|
||||
} else {
|
||||
z, bucket = migrationTestThreePools(t)
|
||||
}
|
||||
const object = "deleted"
|
||||
old, stamp := UTCNow().Add(-time.Hour), UTCNow().Add(-time.Minute)
|
||||
var prior ObjectInfo
|
||||
if history != "none" {
|
||||
pool := 0
|
||||
if history == "destination" {
|
||||
pool = 1
|
||||
}
|
||||
prior = putConsistencyObject(t, z, bucket, object, pool, "history", ObjectOptions{Versioned: true, MTime: old})
|
||||
}
|
||||
// A stale null copy is a normal consequence of cross-pool overwrites.
|
||||
putConsistencyObject(t, z, bucket, object, 1, "stale", ObjectOptions{MTime: old})
|
||||
vid := mustGetUUID()
|
||||
if null {
|
||||
vid = nullVersionID
|
||||
}
|
||||
opts := ObjectOptions{Versioned: true, VersionID: vid, DeleteMarker: true, MTime: stamp}
|
||||
marker, err := z.serverPools[0].DeleteObject(t.Context(), bucket, object, opts)
|
||||
mustIAM(t, err)
|
||||
if !marker.DeleteMarker {
|
||||
t.Fatal("fixture did not create marker")
|
||||
}
|
||||
if oi, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); !isErrObjectNotFound(err) || !oi.DeleteMarker {
|
||||
t.Fatalf("fixture visible: %+v %v", oi, err)
|
||||
}
|
||||
if failed := migrationTestWalk(t, z, bucket, 0, kind); failed != 0 {
|
||||
t.Fatalf("failed moves: %d", failed)
|
||||
}
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
if oi, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); !isErrObjectNotFound(err) || !oi.DeleteMarker || !oi.ModTime.Equal(stamp) {
|
||||
t.Fatalf("marker lost: %+v %v", oi, err)
|
||||
}
|
||||
if _, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: migrationTestVersion(marker)}); !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
t.Fatalf("source marker retained: %v", err)
|
||||
}
|
||||
if prior.VersionID != "" && migrationTestBody(t, z.GetObjectNInfo, bucket, object, prior.VersionID) != "history" {
|
||||
t.Fatal("lost historical version")
|
||||
}
|
||||
// Repeating a completed walk must preserve the same state.
|
||||
migrationTestWalk(t, z, bucket, 0, kind)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A reader opened before a public purge must not recreate the purged version.
|
||||
func TestPoolsMigrationDoesNotResurrectPurgedSource(t *testing.T) {
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for _, multipart := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/multipart=%t", kind, multipart), func(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
move := migrationTestMover(t, z, 0, kind)
|
||||
oi := migrationTestObject(t, z, bucket, "purged", 0, multipart, []string{"payload"}, ObjectOptions{Versioned: true})
|
||||
gr := migrationTestReader(t, z, 0, oi)
|
||||
_, err := z.DeleteObject(t.Context(), bucket, oi.Name, ObjectOptions{VersionID: oi.VersionID, Versioned: true})
|
||||
mustIAM(t, err)
|
||||
err = move(t.Context(), 0, bucket, gr)
|
||||
if err == nil {
|
||||
t.Error("move accepted a purged source")
|
||||
}
|
||||
if oi, err := z.GetObjectInfo(t.Context(), bucket, oi.Name, ObjectOptions{}); !isErrObjectNotFound(err) {
|
||||
t.Fatalf("purged version resurrected: %+v %v", oi, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup must not erase a marker created after the data was copied. It must
|
||||
// also fail closed if the destination becomes unreadable before source removal.
|
||||
func TestPoolsMigrationCleanupRechecksSource(t *testing.T) {
|
||||
for _, change := range []string{"delete-null", "delete-uuid", "unreadable-destination", "missing-destination"} {
|
||||
t.Run(change, func(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
move := migrationTestMover(t, z, 0, "decommission")
|
||||
oi := migrationTestObject(t, z, bucket, "cleanup", 0, false, []string{"payload"}, ObjectOptions{})
|
||||
snapshot := []FileInfo{{VersionID: oi.VersionID, ModTime: oi.ModTime}}
|
||||
mustIAM(t, move(t.Context(), 0, bucket, migrationTestReader(t, z, 0, oi)))
|
||||
switch change {
|
||||
case "delete-null", "delete-uuid":
|
||||
_, err := z.DeleteObject(t.Context(), bucket, oi.Name, ObjectOptions{VersionSuspended: change == "delete-null", Versioned: change == "delete-uuid"})
|
||||
mustIAM(t, err)
|
||||
case "unreadable-destination":
|
||||
restore := conditionalPutSwapDisks(z.serverPools[1], oi.Name, func(d StorageAPI) StorageAPI {
|
||||
return consistencyReadFaultDisk{StorageAPI: d, bucket: bucket, object: oi.Name}
|
||||
})
|
||||
defer restore()
|
||||
case "missing-destination":
|
||||
_, err := z.serverPools[1].DeleteObject(t.Context(), bucket, oi.Name, ObjectOptions{VersionID: nullVersionID})
|
||||
mustIAM(t, err)
|
||||
}
|
||||
err := z.removeMovedVersions(t.Context(), bucket, oi.Name, 0, snapshot)
|
||||
if change != "delete-uuid" && err == nil {
|
||||
t.Fatal("cleanup accepted changed or uncertain state")
|
||||
}
|
||||
if change == "delete-null" || change == "delete-uuid" {
|
||||
if got, err := z.GetObjectInfo(t.Context(), bucket, oi.Name, ObjectOptions{}); !isErrObjectNotFound(err) || !got.DeleteMarker {
|
||||
t.Fatalf("cleanup resurrected data: %v", err)
|
||||
}
|
||||
} else if _, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, oi.Name, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("cleanup lost source: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMigrationConflictingTimestampTie(t *testing.T) {
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for source := range 2 {
|
||||
t.Run(fmt.Sprintf("%s/source=%d", kind, source), func(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
stamp := UTCNow().Add(-time.Minute)
|
||||
for i := range 2 {
|
||||
putConsistencyObject(t, z, bucket, "tie", i, fmt.Sprintf("pool-%d", i), ObjectOptions{MTime: stamp})
|
||||
}
|
||||
if got := migrationTestBody(t, z.GetObjectNInfo, bucket, "tie", ""); got != "pool-0" {
|
||||
t.Fatalf("fixture winner: %s", got)
|
||||
}
|
||||
migrationTestWalk(t, z, bucket, source, kind)
|
||||
if got := migrationTestBody(t, z.GetObjectNInfo, bucket, "tie", ""); got != "pool-0" {
|
||||
t.Fatalf("changed tie winner: %s", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import xhttp "github.com/minio/minio/internal/http"
|
||||
|
||||
// migrationObjectMetadata restores tags removed from the read representation.
|
||||
// Keep the original revision, including ordered empty states, for the existing
|
||||
// locked reconciliation at PUT or multipart completion. Moving is not a new
|
||||
// tagging mutation, and the write must not modify the reader's metadata map.
|
||||
func migrationObjectMetadata(oi ObjectInfo) map[string]string {
|
||||
metadata := cloneMSS(oi.UserDefined)
|
||||
delete(metadata, xhttp.AmzObjectTagging)
|
||||
if oi.UserTags != "" {
|
||||
metadata[xhttp.AmzObjectTagging] = oi.UserTags
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
@@ -0,0 +1,928 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
const migrationTagRevision = ReservedMetadataPrefixLower + TaggingTimestamp
|
||||
|
||||
var migrationTagStates = []struct {
|
||||
name, tags, revision string
|
||||
hasRevision bool
|
||||
}{
|
||||
{name: "initial", tags: "team=storage&path=a%2Fb"},
|
||||
{name: "empty-revision", tags: "team=storage", hasRevision: true},
|
||||
{name: "invalid-revision", tags: "team=storage", revision: "invalid", hasRevision: true},
|
||||
{name: "ordered", tags: "team=storage", revision: "2026-09-16T09:00:00Z", hasRevision: true},
|
||||
{name: "cleared", revision: "2026-09-16T09:00:00Z", hasRevision: true},
|
||||
{name: "never-tagged"},
|
||||
{name: "empty-empty-revision", hasRevision: true},
|
||||
{name: "empty-invalid-revision", revision: "invalid", hasRevision: true},
|
||||
}
|
||||
|
||||
func TestMigrationObjectMetadata(t *testing.T) {
|
||||
for _, state := range migrationTagStates {
|
||||
for _, residual := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/residual=%t", state.name, residual), func(t *testing.T) {
|
||||
raw := map[string]string{
|
||||
xhttp.AmzObjectTagging: state.tags,
|
||||
"x-amz-meta-team": "storage",
|
||||
"x-amz-meta-empty": "",
|
||||
ReservedMetadataPrefix + "compression": "opaque-compression-state",
|
||||
ReservedMetadataPrefix + "sealed-key": "opaque-key-state",
|
||||
"x-amz-object-lock-retain-until-date": "2030-01-01T00:00:00Z",
|
||||
"x-amz-object-lock-legal-hold": "ON",
|
||||
"x-amz-tagging": "keep-noncanonical-metadata",
|
||||
ReservedMetadataPrefix + "actual-size": "42",
|
||||
ReservedMetadataPrefix + "replica-status": "opaque-replication-state",
|
||||
}
|
||||
if state.hasRevision {
|
||||
raw[migrationTagRevision] = state.revision
|
||||
}
|
||||
oi := (FileInfo{Metadata: raw}).ToObjectInfo("bucket", "object", false)
|
||||
if residual {
|
||||
oi.UserDefined[xhttp.AmzObjectTagging] = "stale=raw"
|
||||
}
|
||||
before := maps.Clone(oi.UserDefined)
|
||||
got := migrationObjectMetadata(oi)
|
||||
if got == nil || !maps.Equal(before, oi.UserDefined) {
|
||||
t.Fatal("helper must return a new non-nil map without changing the read snapshot")
|
||||
}
|
||||
value, exists := got[xhttp.AmzObjectTagging]
|
||||
if value != state.tags || exists != (state.tags != "") {
|
||||
t.Fatalf("raw tags=(%q,%t), want (%q,%t)", value, exists, state.tags, state.tags != "")
|
||||
}
|
||||
stamp, present := got[migrationTagRevision]
|
||||
if stamp != state.revision || present != state.hasRevision {
|
||||
t.Fatalf("revision=(%q,%t), want (%q,%t)", stamp, present, state.revision, state.hasRevision)
|
||||
}
|
||||
for key, value := range before {
|
||||
if stored, exists := got[key]; key != xhttp.AmzObjectTagging && (!exists || stored != value) {
|
||||
t.Errorf("lost metadata %q", key)
|
||||
}
|
||||
}
|
||||
got["x-amz-meta-team"] = "changed"
|
||||
if !maps.Equal(before, oi.UserDefined) {
|
||||
t.Fatal("output aliases the read snapshot")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, metadata := range []map[string]string{nil, {}} {
|
||||
got := migrationObjectMetadata(ObjectInfo{UserDefined: metadata})
|
||||
if got == nil || len(got) != 0 {
|
||||
t.Fatalf("empty input must yield a writable empty map: %#v", got)
|
||||
}
|
||||
got["new"] = "value"
|
||||
if len(metadata) != 0 {
|
||||
t.Fatal("empty map input was aliased")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Allocation ignores the host's used-space percentage, as in the existing tag
|
||||
// fixtures. All object data and metadata still use real erasure-storage disks.
|
||||
func migrationTestPools(t *testing.T) (*erasureServerPools, string) {
|
||||
t.Helper()
|
||||
z, bucket := consistencyPools(t)
|
||||
for _, pool := range z.serverPools {
|
||||
pool.erasureDisksMu.Lock()
|
||||
previous := pool.erasureDisks
|
||||
wrapped := make([][]StorageAPI, len(previous))
|
||||
for i, disks := range previous {
|
||||
wrapped[i] = append([]StorageAPI(nil), disks...)
|
||||
for j := range wrapped[i] {
|
||||
wrapped[i][j] = tagTestCapacityDisk{StorageAPI: wrapped[i][j]}
|
||||
}
|
||||
}
|
||||
pool.erasureDisks = wrapped
|
||||
pool.erasureDisksMu.Unlock()
|
||||
t.Cleanup(func() { pool.erasureDisksMu.Lock(); pool.erasureDisks = previous; pool.erasureDisksMu.Unlock() })
|
||||
}
|
||||
return z, bucket
|
||||
}
|
||||
|
||||
func migrationTestMover(t *testing.T, z *erasureServerPools, source int, kind string) func(context.Context, int, string, *GetObjectReader) error {
|
||||
t.Helper()
|
||||
if kind == "rebalance" {
|
||||
z.rebalMu.Lock()
|
||||
z.rebalMeta = &rebalanceMeta{PoolStats: make([]*rebalanceStats, len(z.serverPools))}
|
||||
for i := range z.rebalMeta.PoolStats {
|
||||
z.rebalMeta.PoolStats[i] = &rebalanceStats{}
|
||||
}
|
||||
z.rebalMeta.PoolStats[source] = &rebalanceStats{Participating: true, Info: rebalanceInfo{Status: rebalStarted}}
|
||||
z.rebalMu.Unlock()
|
||||
t.Cleanup(func() { z.rebalMu.Lock(); z.rebalMeta = nil; z.rebalMu.Unlock() })
|
||||
return z.rebalanceObject
|
||||
}
|
||||
z.poolMetaMutex.Lock()
|
||||
z.poolMeta.Pools[source].Decommission = &PoolDecommissionInfo{}
|
||||
z.poolMetaMutex.Unlock()
|
||||
t.Cleanup(func() { z.poolMetaMutex.Lock(); z.poolMeta.Pools[source].Decommission = nil; z.poolMetaMutex.Unlock() })
|
||||
return z.decommissionObject
|
||||
}
|
||||
|
||||
func migrationTestObject(t *testing.T, z *erasureServerPools, bucket, object string, source int, multipart bool, parts []string, opts ObjectOptions) ObjectInfo {
|
||||
t.Helper()
|
||||
if !multipart {
|
||||
oi := putConsistencyObject(t, z, bucket, object, source, strings.Join(parts, ""), opts)
|
||||
return migrationTestPersistedInfo(t, z, source, oi)
|
||||
}
|
||||
mp, err := z.serverPools[source].NewMultipartUpload(t.Context(), bucket, object, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
completed := make([]CompletePart, len(parts))
|
||||
for i, body := range parts {
|
||||
part, err := z.serverPools[source].PutObjectPart(t.Context(), bucket, object, mp.UploadID, i+1,
|
||||
mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
completed[i] = CompletePart{PartNumber: i + 1, ETag: part.ETag}
|
||||
}
|
||||
oi, err := z.serverPools[source].CompleteMultipartUpload(t.Context(), bucket, object, mp.UploadID, completed, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !oi.isMultipart() {
|
||||
t.Fatal("fixture did not create a multipart object")
|
||||
}
|
||||
return migrationTestPersistedInfo(t, z, source, oi)
|
||||
}
|
||||
|
||||
func migrationTestPersistedInfo(t *testing.T, z *erasureServerPools, pool int, oi ObjectInfo) ObjectInfo {
|
||||
t.Helper()
|
||||
// PUT can return transient fields such as tier-free-versionID that are not
|
||||
// stored. Take the same persisted read representation used by the movers.
|
||||
got, err := z.serverPools[pool].GetObjectInfo(t.Context(), oi.Bucket, oi.Name, ObjectOptions{VersionID: migrationTestVersion(oi)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func migrationTestVersion(oi ObjectInfo) string {
|
||||
if oi.VersionID == "" {
|
||||
return nullVersionID
|
||||
}
|
||||
return oi.VersionID
|
||||
}
|
||||
|
||||
func migrationTestReader(t *testing.T, z *erasureServerPools, source int, oi ObjectInfo) *GetObjectReader {
|
||||
t.Helper()
|
||||
gr, err := z.serverPools[source].GetObjectNInfo(t.Context(), oi.Bucket, oi.Name, nil, nil,
|
||||
ObjectOptions{VersionID: migrationTestVersion(oi), NoLock: true, NoDecryption: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { gr.Close() })
|
||||
return gr
|
||||
}
|
||||
|
||||
func migrationTestStored(t *testing.T, z *erasureServerPools, pool int, original ObjectInfo, tags, revision string, hasRevision bool, body string) {
|
||||
t.Helper()
|
||||
version := migrationTestVersion(original)
|
||||
got, err := z.serverPools[pool].GetObjectInfo(t.Context(), original.Bucket, original.Name, ObjectOptions{VersionID: version})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.UserTags != tags || got.ETag != original.ETag || !got.ModTime.Equal(original.ModTime) || got.VersionID != original.VersionID || got.Size != original.Size {
|
||||
t.Errorf("pool %d changed tags/object identity: tags=%q want=%q version=%q/%q etag=%q/%q size=%d/%d mtime=%v/%v", pool, got.UserTags, tags, got.VersionID, original.VersionID, got.ETag, original.ETag, got.Size, original.Size, got.ModTime, original.ModTime)
|
||||
}
|
||||
infos, errs := readAllFileInfo(t.Context(), z.serverPools[pool].getHashedSet(original.Name).getDisks(), "", original.Bucket, original.Name, version, false, false)
|
||||
for disk, fi := range infos {
|
||||
if errs[disk] != nil {
|
||||
t.Fatalf("pool %d disk %d: %v", pool, disk, errs[disk])
|
||||
}
|
||||
stamp, exists := fi.Metadata[migrationTagRevision]
|
||||
if fi.Metadata[xhttp.AmzObjectTagging] != tags || stamp != revision || exists != hasRevision {
|
||||
t.Errorf("pool %d disk %d: tags=%q revision=(%q,%t), want %q (%q,%t)", pool, disk, fi.Metadata[xhttp.AmzObjectTagging], stamp, exists, tags, revision, hasRevision)
|
||||
}
|
||||
for key, value := range original.UserDefined {
|
||||
if key != migrationTagRevision && fi.Metadata[key] != value {
|
||||
t.Errorf("pool %d disk %d lost metadata %q", pool, disk, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
gr, err := z.serverPools[pool].GetObjectNInfo(t.Context(), original.Bucket, original.Name, nil, nil, ObjectOptions{VersionID: version})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := io.ReadAll(gr)
|
||||
gr.Close()
|
||||
if err != nil || !bytes.Equal(data, []byte(body)) {
|
||||
t.Fatalf("pool %d content changed: bytes=%d err=%v", pool, len(data), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMigrationPreservesTagState(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for _, method := range []string{"put", "multipart"} {
|
||||
for source := range 2 {
|
||||
for _, state := range migrationTagStates {
|
||||
t.Run(fmt.Sprintf("%s/%s/source=%d/%s", kind, method, source, state.name), func(t *testing.T) {
|
||||
move := migrationTestMover(t, z, source, kind)
|
||||
meta := map[string]string{xhttp.AmzObjectTagging: state.tags, "x-amz-meta-owner": "retained"}
|
||||
if state.hasRevision {
|
||||
meta[migrationTagRevision] = state.revision
|
||||
}
|
||||
oi := migrationTestObject(t, z, bucket, t.Name(), source, method == "multipart", []string{"payload"}, ObjectOptions{Versioned: true, UserDefined: meta})
|
||||
migrationTestStored(t, z, source, oi, state.tags, state.revision, state.hasRevision, "payload")
|
||||
gr := migrationTestReader(t, z, source, oi)
|
||||
before := maps.Clone(gr.ObjInfo.UserDefined)
|
||||
if err := move(t.Context(), source, bucket, gr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !maps.Equal(before, gr.ObjInfo.UserDefined) {
|
||||
t.Error("migration modified its source snapshot")
|
||||
}
|
||||
migrationTestStored(t, z, 1-source, oi, state.tags, state.revision, state.hasRevision, "payload")
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type migrationTestGate struct {
|
||||
entered, resume chan struct{}
|
||||
once, release sync.Once
|
||||
}
|
||||
|
||||
func newMigrationTestGate() *migrationTestGate {
|
||||
return &migrationTestGate{entered: make(chan struct{}), resume: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (g *migrationTestGate) wait(ctx context.Context) error {
|
||||
g.once.Do(func() { close(g.entered) })
|
||||
select {
|
||||
case <-g.resume:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (g *migrationTestGate) unblock() { g.release.Do(func() { close(g.resume) }) }
|
||||
|
||||
type migrationTestGateReader struct {
|
||||
io.Reader
|
||||
ctx context.Context
|
||||
gate *migrationTestGate
|
||||
}
|
||||
|
||||
func (r migrationTestGateReader) Read(p []byte) (int, error) {
|
||||
if err := r.gate.wait(r.ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return r.Reader.Read(p)
|
||||
}
|
||||
|
||||
func TestPoolsMigrationRechecksTags(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
const recent = "2026-09-16T10:00:00Z"
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for _, method := range []string{"put", "multipart"} {
|
||||
for source := range 2 {
|
||||
for _, tags := range []string{"state=after", ""} {
|
||||
t.Run(fmt.Sprintf("%s/%s/source=%d/clear=%t", kind, method, source, tags == ""), func(t *testing.T) {
|
||||
move := migrationTestMover(t, z, source, kind)
|
||||
oi := migrationTestObject(t, z, bucket, t.Name(), source, method == "multipart", []string{"payload"}, ObjectOptions{
|
||||
Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "state=before"},
|
||||
})
|
||||
gr := migrationTestReader(t, z, source, oi)
|
||||
before := maps.Clone(gr.ObjInfo.UserDefined)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
gate := newMigrationTestGate()
|
||||
done, finished := make(chan error, 1), make(chan struct{})
|
||||
if method == "multipart" {
|
||||
// The first part read is after persisted upload initialization,
|
||||
// but before completion takes the object lock and reconciles.
|
||||
gr.Reader = migrationTestGateReader{Reader: gr.Reader, ctx: ctx, gate: gate}
|
||||
go func() { defer close(finished); done <- move(ctx, source, bucket, gr) }()
|
||||
defer func() { gate.unblock(); cancel(); <-finished }()
|
||||
select {
|
||||
case <-gate.entered:
|
||||
case err := <-done:
|
||||
t.Fatalf("migration finished before the part barrier: %v", err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
uploads, err := z.serverPools[1-source].listMultipartUploadsExact(ctx, bucket, oi.Name)
|
||||
if err != nil || len(uploads.Uploads) != 1 {
|
||||
t.Fatalf("upload not persisted at barrier: %+v, %v", uploads, err)
|
||||
}
|
||||
info, err := z.serverPools[1-source].GetMultipartInfo(ctx, bucket, oi.Name, uploads.Uploads[0].UploadID, ObjectOptions{})
|
||||
if err != nil || info.UserDefined[xhttp.AmzObjectTagging] != "state=before" {
|
||||
t.Fatalf("upload lost snapshot tags: %v, %v", info.UserDefined, err)
|
||||
}
|
||||
}
|
||||
// For ordinary PUT this must finish before calling the mover:
|
||||
// blocking its data Read would already hold the object lock.
|
||||
opts := ObjectOptions{VersionID: migrationTestVersion(oi), UserDefined: map[string]string{migrationTagRevision: recent}}
|
||||
var err error
|
||||
if tags == "" {
|
||||
_, err = z.DeleteObjectTags(ctx, bucket, oi.Name, opts)
|
||||
} else {
|
||||
_, err = z.PutObjectTags(ctx, bucket, oi.Name, tags, opts)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if gr.ObjInfo.UserTags != "state=before" || !maps.Equal(before, gr.ObjInfo.UserDefined) {
|
||||
t.Fatal("test must retain the old source snapshot")
|
||||
}
|
||||
migrationTestStored(t, z, source, oi, tags, recent, true, "payload")
|
||||
if method == "multipart" {
|
||||
gate.unblock()
|
||||
err = <-done
|
||||
} else {
|
||||
err = move(ctx, source, bucket, gr)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !maps.Equal(before, gr.ObjInfo.UserDefined) {
|
||||
t.Error("reconciliation changed the reader's map")
|
||||
}
|
||||
migrationTestStored(t, z, 1-source, oi, tags, recent, true, "payload")
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type migrationTestMetadataGateDisk struct {
|
||||
StorageAPI
|
||||
bucket, object string
|
||||
gate *migrationTestGate
|
||||
}
|
||||
|
||||
func (d migrationTestMetadataGateDisk) UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo, opts UpdateMetadataOpts) error {
|
||||
if volume == d.bucket && path == d.object {
|
||||
if err := d.gate.wait(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return d.StorageAPI.UpdateMetadata(ctx, volume, path, fi, opts)
|
||||
}
|
||||
|
||||
func TestPoolsMigrationTagsDuringCleanup(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
const recent = "2026-09-16T10:00:00Z"
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for source := range 2 {
|
||||
for _, tags := range []string{"state=after", ""} {
|
||||
for _, interrupt := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/source=%d/clear=%t/interrupt=%t", kind, source, tags == "", interrupt), func(t *testing.T) {
|
||||
move := migrationTestMover(t, z, source, kind)
|
||||
oi := migrationTestObject(t, z, bucket, t.Name(), source, false, []string{"payload"}, ObjectOptions{
|
||||
Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "state=before"},
|
||||
})
|
||||
if err := move(t.Context(), source, bucket, migrationTestReader(t, z, source, oi)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
migrationTestStored(t, z, 1-source, oi, "state=before", "", false, "payload")
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
mutate := func() (ObjectInfo, error) {
|
||||
opts := ObjectOptions{VersionID: migrationTestVersion(oi), UserDefined: map[string]string{migrationTagRevision: recent}}
|
||||
if tags == "" {
|
||||
return z.DeleteObjectTags(ctx, bucket, oi.Name, opts)
|
||||
}
|
||||
return z.PutObjectTags(ctx, bucket, oi.Name, tags, opts)
|
||||
}
|
||||
set := z.serverPools[source].getHashedSet(oi.Name)
|
||||
cleanup := func() {
|
||||
// Use the same source prefix cleanup as both outer movers.
|
||||
if _, err := set.DeleteObject(ctx, bucket, oi.Name, ObjectOptions{DeletePrefix: true, DeletePrefixObject: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var updated ObjectInfo
|
||||
if !interrupt {
|
||||
var err error
|
||||
updated, err = mutate()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cleanup()
|
||||
} else {
|
||||
gate := newMigrationTestGate()
|
||||
original := set.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i := range disks {
|
||||
disks[i] = migrationTestMetadataGateDisk{StorageAPI: disks[i], bucket: bucket, object: oi.Name, gate: gate}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
defer func() { set.getDisks = original }()
|
||||
done, finished := make(chan error, 1), make(chan struct{})
|
||||
go func() { defer close(finished); _, err := mutate(); done <- err }()
|
||||
defer func() { gate.unblock(); cancel(); <-finished }()
|
||||
select {
|
||||
case <-gate.entered:
|
||||
case err := <-done:
|
||||
t.Fatalf("metadata update missed the barrier: %v", err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
cleanup()
|
||||
gate.unblock()
|
||||
if err := <-done; err == nil {
|
||||
t.Fatal("update reported success after its source copy was removed")
|
||||
}
|
||||
<-finished
|
||||
set.getDisks = original
|
||||
var err error
|
||||
updated, err = mutate()
|
||||
if err != nil {
|
||||
t.Fatalf("metadata retry did not converge: %v", err)
|
||||
}
|
||||
}
|
||||
if _, err := z.serverPools[source].GetObjectInfo(ctx, bucket, oi.Name, ObjectOptions{VersionID: oi.VersionID}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Fatalf("source cleanup left a copy: %v", err)
|
||||
}
|
||||
migrationTestStored(t, z, 1-source, oi, tags, updated.UserDefined[migrationTagRevision], true, "payload")
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMigrationVersionHistory(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for _, method := range []string{"put", "multipart"} {
|
||||
for _, versioning := range []string{"unversioned", "null", "history"} {
|
||||
t.Run(kind+"/"+method+"/"+versioning, func(t *testing.T) {
|
||||
move := migrationTestMover(t, z, 0, kind)
|
||||
opts := ObjectOptions{
|
||||
Versioned: versioning == "history", VersionSuspended: versioning == "null",
|
||||
MTime: time.Date(2026, 9, 16, 8, 0, 0, 0, time.UTC),
|
||||
UserDefined: map[string]string{xhttp.AmzObjectTagging: "version=old"},
|
||||
}
|
||||
versions := []ObjectInfo{migrationTestObject(t, z, bucket, t.Name(), 0, method == "multipart", []string{"old"}, opts)}
|
||||
if versioning == "history" {
|
||||
opts.MTime = opts.MTime.Add(time.Minute)
|
||||
versions = append(versions, migrationTestObject(t, z, bucket, t.Name(), 0, method == "multipart", []string{"old"}, opts))
|
||||
}
|
||||
var newest ObjectInfo
|
||||
if versioning != "unversioned" {
|
||||
newest = migrationTestObject(t, z, bucket, t.Name(), 0, false, []string{"latest"}, ObjectOptions{
|
||||
Versioned: true, MTime: opts.MTime.Add(time.Minute),
|
||||
UserDefined: map[string]string{xhttp.AmzObjectTagging: "version=new"},
|
||||
})
|
||||
}
|
||||
for _, oi := range versions {
|
||||
if err := move(t.Context(), 0, bucket, migrationTestReader(t, z, 0, oi)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
migrationTestStored(t, z, 1, oi, "version=old", "", false, "old")
|
||||
}
|
||||
if versioning != "unversioned" {
|
||||
migrationTestStored(t, z, 0, newest, "version=new", "", false, "latest")
|
||||
if _, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, t.Name(), ObjectOptions{VersionID: newest.VersionID}); !isErrVersionNotFound(err) {
|
||||
t.Fatalf("moving an addressed version affected the newer version: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMigrationMultipartParts(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
parts := []string{strings.Repeat("a", 5<<20), "tail-with-tags"}
|
||||
body := strings.Join(parts, "")
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for source := range 2 {
|
||||
t.Run(fmt.Sprintf("%s/source=%d", kind, source), func(t *testing.T) {
|
||||
move := migrationTestMover(t, z, source, kind)
|
||||
oi := migrationTestObject(t, z, bucket, t.Name(), source, true, parts, ObjectOptions{
|
||||
Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "multipart=kept"},
|
||||
})
|
||||
if err := move(t.Context(), source, bucket, migrationTestReader(t, z, source, oi)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
migrationTestStored(t, z, 1-source, oi, "multipart=kept", "", false, body)
|
||||
got, err := z.serverPools[1-source].GetObjectInfo(t.Context(), bucket, oi.Name, ObjectOptions{VersionID: oi.VersionID})
|
||||
if err != nil || len(got.Parts) != len(oi.Parts) {
|
||||
t.Fatalf("parts changed: %+v, %v", got.Parts, err)
|
||||
}
|
||||
for i, part := range got.Parts {
|
||||
old := oi.Parts[i]
|
||||
if part.Number != old.Number || part.Size != old.Size || part.ActualSize != old.ActualSize || part.ETag != old.ETag || !bytes.Equal(part.Index, old.Index) {
|
||||
t.Errorf("part %d changed: %+v / %+v", i, part, old)
|
||||
}
|
||||
}
|
||||
start := int64(len(parts[0]) - 3)
|
||||
gr, err := z.serverPools[1-source].GetObjectNInfo(t.Context(), bucket, oi.Name, &HTTPRangeSpec{Start: start, End: start + 7}, nil, ObjectOptions{VersionID: oi.VersionID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := io.ReadAll(gr)
|
||||
gr.Close()
|
||||
if err != nil || string(data) != body[start:start+8] {
|
||||
t.Fatalf("cross-part range changed: %q, %v", data, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMigrationUnreadablePool(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for _, method := range []string{"put", "multipart"} {
|
||||
t.Run(kind+"/"+method, func(t *testing.T) {
|
||||
move := migrationTestMover(t, z, 0, kind)
|
||||
oi := migrationTestObject(t, z, bucket, t.Name(), 0, method == "multipart", []string{"payload"}, ObjectOptions{
|
||||
Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "keep=source"},
|
||||
})
|
||||
gr := migrationTestReader(t, z, 0, oi)
|
||||
set := z.serverPools[0].getHashedSet(oi.Name)
|
||||
original := set.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i := range disks {
|
||||
disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: oi.Name}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
defer func() { set.getDisks = original }()
|
||||
err := move(t.Context(), 0, bucket, gr)
|
||||
var quorum InsufficientReadQuorum
|
||||
if !errors.As(err, &quorum) {
|
||||
t.Fatalf("unreadable pool must fail migration with read quorum error: %v", err)
|
||||
}
|
||||
set.getDisks = original
|
||||
migrationTestStored(t, z, 0, oi, "keep=source", "", false, "payload")
|
||||
if _, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, oi.Name, ObjectOptions{VersionID: oi.VersionID}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Fatalf("failed migration committed a target version: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Three pools give a move two candidate destinations.
|
||||
func migrationTestThreePools(t *testing.T) (*erasureServerPools, string) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
dirs, err := getRandomDisks(12)
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatal(err)
|
||||
}
|
||||
var endpoints EndpointServerPools
|
||||
for i := range 3 {
|
||||
endpoints = append(endpoints, mustGetPoolEndpoints(i, dirs[4*i:4*i+4]...)...)
|
||||
}
|
||||
obj, _, err := initObjectLayer(ctx, endpoints)
|
||||
if err != nil {
|
||||
cancel()
|
||||
removeRoots(dirs)
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
previous := newObjectLayerFn()
|
||||
setObjectLayer(z)
|
||||
t.Cleanup(func() { cancel(); z.Shutdown(context.Background()); removeRoots(dirs); setObjectLayer(previous) })
|
||||
bucket := "pool-migration"
|
||||
if err := z.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return z, bucket
|
||||
}
|
||||
|
||||
type migrationTestGetter func(context.Context, string, string, *HTTPRangeSpec, http.Header, ObjectOptions) (*GetObjectReader, error)
|
||||
|
||||
// migrationTestBody returns the content read for a version, or the read error.
|
||||
func migrationTestBody(t *testing.T, get migrationTestGetter, bucket, object, versionID string) string {
|
||||
t.Helper()
|
||||
gr, err := get(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: versionID})
|
||||
if err != nil {
|
||||
return fmt.Sprintf("<%v>", err)
|
||||
}
|
||||
defer gr.Close()
|
||||
data, err := io.ReadAll(gr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
// migrationTestWalk runs one bucket of a decommission or rebalance of source,
|
||||
// including the walker's accounting and source cleanup. It returns the number
|
||||
// of versions decommission counted as failed.
|
||||
func migrationTestWalk(t *testing.T, z *erasureServerPools, bucket string, source int, kind string) int64 {
|
||||
t.Helper()
|
||||
migrationTestMover(t, z, source, kind)
|
||||
if kind == "rebalance" {
|
||||
if err := z.rebalanceBucket(t.Context(), bucket, source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
z.poolMetaMutex.Lock()
|
||||
z.poolMeta.Pools[source].LastUpdate = UTCNow() // keep pool.bin and peers out of the walk
|
||||
z.poolMetaMutex.Unlock()
|
||||
if err := z.decommissionPool(t.Context(), source, z.serverPools[source], decomBucketInfo{Name: bucket}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z.poolMetaMutex.RLock()
|
||||
defer z.poolMetaMutex.RUnlock()
|
||||
return z.poolMeta.Pools[source].Decommission.ItemsDecommissionFailed
|
||||
}
|
||||
|
||||
// migrationTestTiered stores a null version whose data is in a remote tier, in
|
||||
// the form decommission moves it.
|
||||
func migrationTestTiered(t *testing.T, z *erasureServerPools, bucket, object string, pool int, mtime time.Time) ObjectInfo {
|
||||
t.Helper()
|
||||
oi := putConsistencyObject(t, z, bucket, object, pool, "old", ObjectOptions{MTime: mtime})
|
||||
infos, errs := readAllFileInfo(t.Context(), z.serverPools[pool].getHashedSet(oi.Name).getDisks(), "", bucket, oi.Name, nullVersionID, false, false)
|
||||
if errs[0] != nil {
|
||||
t.Fatal(errs[0])
|
||||
}
|
||||
fi := infos[0]
|
||||
fi.TransitionStatus, fi.TransitionTier, fi.TransitionedObjName = lifecycle.TransitionComplete, "WARM", "remote"
|
||||
if err := z.serverPools[pool].DecomTieredObject(t.Context(), bucket, oi.Name, fi, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return oi
|
||||
}
|
||||
|
||||
// A cross-pool overwrite can leave copies of one version in several pools.
|
||||
// Reads return the newest copy, and moving an older one must not replace it.
|
||||
// An equal ModTime marks the same write, as an interrupted move leaves it.
|
||||
func TestPoolsMigrationKeepsCurrentCopy(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
base := time.Date(2026, 9, 16, 8, 0, 0, 0, time.UTC)
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for _, method := range []string{"put", "multipart"} {
|
||||
for _, versioned := range []bool{false, true} {
|
||||
for source := range 2 {
|
||||
for _, offset := range []time.Duration{-time.Minute, 0, time.Minute} {
|
||||
t.Run(fmt.Sprintf("%s/%s/versioned=%t/source=%d/offset=%s", kind, method, versioned, source, offset), func(t *testing.T) {
|
||||
move := migrationTestMover(t, z, source, kind)
|
||||
opts := ObjectOptions{Versioned: versioned, MTime: base.Add(offset)}
|
||||
if versioned {
|
||||
opts.VersionID = mustGetUUID()
|
||||
}
|
||||
oi := migrationTestObject(t, z, bucket, t.Name(), source, method == "multipart", []string{"source"}, opts)
|
||||
opts.MTime = base
|
||||
want, other := "source", "other"
|
||||
if offset == 0 {
|
||||
other = "source"
|
||||
opts.PreserveETag = oi.ETag // interrupted copies preserve the original multipart ETag
|
||||
} else if offset < 0 {
|
||||
want = "other"
|
||||
}
|
||||
putConsistencyObject(t, z, bucket, t.Name(), 1-source, other, opts)
|
||||
version := migrationTestVersion(oi)
|
||||
if got := migrationTestBody(t, z.GetObjectNInfo, bucket, t.Name(), version); got != want {
|
||||
t.Fatalf("fixture reads %q, want %q", got, want)
|
||||
}
|
||||
err := move(t.Context(), source, bucket, migrationTestReader(t, z, source, oi))
|
||||
if want == "other" && !isDataMovementSuperseded(err) || want == "source" && err != nil {
|
||||
t.Fatalf("move: %v", err)
|
||||
}
|
||||
if err != nil && strings.Contains(err.Error(), "CompleteMultipartUpload") {
|
||||
t.Error("copied the parts of a superseded version")
|
||||
}
|
||||
if got := migrationTestBody(t, z.serverPools[1-source].GetObjectNInfo, bucket, t.Name(), version); got != want {
|
||||
t.Errorf("destination holds %q, want %q", got, want)
|
||||
}
|
||||
if got := migrationTestBody(t, z.GetObjectNInfo, bucket, t.Name(), version); got != want {
|
||||
t.Errorf("move changed the read result to %q, want %q", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A client overwrite, or a replica of the same version, committed while a move
|
||||
// copies the old version is newer than the moved copy. The move must yield to
|
||||
// it, also at multipart completion.
|
||||
func TestPoolsMigrationYieldsToNewerWrite(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for _, method := range []string{"put", "multipart"} {
|
||||
for _, versioned := range []bool{false, true} {
|
||||
for source := range 2 {
|
||||
t.Run(fmt.Sprintf("%s/%s/versioned=%t/source=%d", kind, method, versioned, source), func(t *testing.T) {
|
||||
migrationTestNewerWrite(t, z, bucket, kind, method, versioned, source)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func migrationTestNewerWrite(t *testing.T, z *erasureServerPools, bucket, kind, method string, versioned bool, source int) {
|
||||
move := migrationTestMover(t, z, source, kind)
|
||||
oi := migrationTestObject(t, z, bucket, t.Name(), source, method == "multipart", []string{"moved"}, ObjectOptions{Versioned: versioned, MTime: UTCNow().Add(-time.Hour)})
|
||||
gr := migrationTestReader(t, z, source, oi)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
gate := newMigrationTestGate()
|
||||
done, finished := make(chan error, 1), make(chan struct{})
|
||||
if method == "multipart" {
|
||||
// Hold the move after its initial check, while it copies parts.
|
||||
gr.Reader = migrationTestGateReader{Reader: gr.Reader, ctx: ctx, gate: gate}
|
||||
go func() { defer close(finished); done <- move(ctx, source, bucket, gr) }()
|
||||
defer func() { gate.unblock(); cancel(); <-finished }()
|
||||
select {
|
||||
case <-gate.entered:
|
||||
case err := <-done:
|
||||
t.Fatalf("migration finished before the part barrier: %v", err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
}
|
||||
// An ordinary PUT copies under the object lock, so this write
|
||||
// lands between the mover's source read and its write.
|
||||
if versioned {
|
||||
putConsistencyObject(t, z, bucket, t.Name(), 1-source, "newer", ObjectOptions{Versioned: true, VersionID: oi.VersionID, MTime: UTCNow()})
|
||||
} else if _, err := z.PutObject(ctx, bucket, t.Name(), mustGetPutObjReader(t, strings.NewReader("newer"), 5, "", ""), ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var err error
|
||||
if method == "multipart" {
|
||||
gate.unblock()
|
||||
err = <-done
|
||||
} else {
|
||||
err = move(ctx, source, bucket, gr)
|
||||
}
|
||||
if !isDataMovementSuperseded(err) {
|
||||
t.Fatalf("move over a newer write: %v", err)
|
||||
}
|
||||
if got := migrationTestBody(t, z.GetObjectNInfo, bucket, t.Name(), migrationTestVersion(oi)); got != "newer" {
|
||||
t.Fatalf("newer write replaced by %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// With two candidate destinations a superseded copy must neither replace the
|
||||
// newer copy nor become a new duplicate in the third pool.
|
||||
func TestPoolsMigrationThreePools(t *testing.T) {
|
||||
z, bucket := migrationTestThreePools(t)
|
||||
stale, recent := UTCNow().Add(-time.Hour), UTCNow().Add(-time.Minute)
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
for _, newest := range []int{0, 1} {
|
||||
t.Run(fmt.Sprintf("%s/newest=%d", kind, newest), func(t *testing.T) {
|
||||
move := migrationTestMover(t, z, 0, kind)
|
||||
mtime := func(pool int) time.Time {
|
||||
if pool == newest {
|
||||
return recent
|
||||
}
|
||||
return stale
|
||||
}
|
||||
oi := migrationTestObject(t, z, bucket, t.Name(), 0, false, []string{"pool-0"}, ObjectOptions{MTime: mtime(0)})
|
||||
putConsistencyObject(t, z, bucket, t.Name(), 1, "pool-1", ObjectOptions{MTime: mtime(1)})
|
||||
if newest == 0 {
|
||||
putConsistencyObject(t, z, bucket, t.Name(), 2, "pool-2", ObjectOptions{MTime: stale})
|
||||
}
|
||||
want := fmt.Sprintf("pool-%d", newest)
|
||||
err := move(t.Context(), 0, bucket, migrationTestReader(t, z, 0, oi))
|
||||
if newest == 1 && !isDataMovementSuperseded(err) || newest == 0 && err != nil {
|
||||
t.Fatalf("move: %v", err)
|
||||
}
|
||||
if got := migrationTestBody(t, z.GetObjectNInfo, bucket, t.Name(), ""); got != want {
|
||||
t.Errorf("reads return %q, want %q", got, want)
|
||||
}
|
||||
if newest == 1 {
|
||||
if _, err := z.serverPools[2].GetObjectInfo(t.Context(), bucket, t.Name(), ObjectOptions{}); !isErrObjectNotFound(err) {
|
||||
t.Errorf("superseded copy duplicated into pool 2: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The walkers count a superseded version as moved, so its stale source copy is
|
||||
// removed with the object and the source pool drains.
|
||||
func TestPoolsMigrationRetiresSupersededCopy(t *testing.T) {
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
ctx := t.Context()
|
||||
stale, recent := UTCNow().Add(-time.Hour), UTCNow().Add(-time.Minute)
|
||||
put := func(object string, pool int, body string, multipart bool, opts ObjectOptions) ObjectInfo {
|
||||
return migrationTestObject(t, z, bucket, object, pool, multipart, []string{body}, opts)
|
||||
}
|
||||
put("stale", 0, "old", false, ObjectOptions{MTime: stale})
|
||||
put("stale", 1, "new", false, ObjectOptions{MTime: recent})
|
||||
put("stale-multipart", 0, "old", true, ObjectOptions{MTime: stale})
|
||||
put("stale-multipart", 1, "new", false, ObjectOptions{MTime: recent})
|
||||
put("newer", 0, "new", false, ObjectOptions{MTime: recent})
|
||||
put("newer", 1, "old", false, ObjectOptions{MTime: stale})
|
||||
oldest := put("history", 0, "old", false, ObjectOptions{Versioned: true, MTime: stale})
|
||||
put("history", 1, "new", false, ObjectOptions{Versioned: true, MTime: recent})
|
||||
// A suspended bucket's older null delete marker must not hide a newer
|
||||
// null version.
|
||||
hidden := put("stale-marker", 0, "old", false, ObjectOptions{Versioned: true, MTime: stale.Add(-time.Minute)})
|
||||
if _, err := z.serverPools[0].DeleteObject(ctx, bucket, "stale-marker", ObjectOptions{VersionSuspended: true, MTime: stale}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put("stale-marker", 1, "new", false, ObjectOptions{VersionSuspended: true, MTime: recent})
|
||||
objects := []string{"stale", "stale-multipart", "newer", "history", "stale-marker"}
|
||||
if kind == "decommission" {
|
||||
// Rebalance leaves tiered versions in place; decommission moves
|
||||
// their metadata.
|
||||
migrationTestTiered(t, z, bucket, "stale-tiered", 0, stale)
|
||||
put("stale-tiered", 1, "new", false, ObjectOptions{MTime: recent})
|
||||
objects = append(objects, "stale-tiered")
|
||||
}
|
||||
want := map[string]string{}
|
||||
for _, object := range objects {
|
||||
want[object] = migrationTestBody(t, z.GetObjectNInfo, bucket, object, "")
|
||||
}
|
||||
if want["stale"] != "new" || want["stale-marker"] != "new" {
|
||||
t.Fatalf("fixture reads %v", want)
|
||||
}
|
||||
if failed := migrationTestWalk(t, z, bucket, 0, kind); failed != 0 {
|
||||
t.Fatalf("%d versions failed to move", failed)
|
||||
}
|
||||
for _, object := range objects {
|
||||
if got := migrationTestBody(t, z.GetObjectNInfo, bucket, object, ""); got != want[object] {
|
||||
t.Errorf("%s: reads %q after the move, want %q", object, got, want[object])
|
||||
}
|
||||
}
|
||||
var retained []string
|
||||
for _, set := range z.serverPools[0].sets {
|
||||
if err := set.listObjectsToDecommission(ctx, decomBucketInfo{Name: bucket}, func(entry metaCacheEntry) {
|
||||
retained = append(retained, entry.name)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(retained) > 0 {
|
||||
t.Errorf("source pool retained %v", retained)
|
||||
}
|
||||
for _, oi := range []ObjectInfo{oldest, hidden} {
|
||||
if got := migrationTestBody(t, z.GetObjectNInfo, bucket, oi.Name, oi.VersionID); got != "old" {
|
||||
t.Errorf("%s: lost version %s: %q", oi.Name, oi.VersionID, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A tiered move that fails, here because another pool cannot be read, must keep
|
||||
// its source like any other failed version.
|
||||
func TestPoolsMigrationTieredFailureKeepsSource(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
tiered := migrationTestTiered(t, z, bucket, t.Name(), 0, UTCNow().Add(-time.Hour))
|
||||
set := z.serverPools[1].getHashedSet(tiered.Name)
|
||||
original := set.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
for i := range disks {
|
||||
disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: tiered.Name}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
defer func() { set.getDisks = original }()
|
||||
if failed := migrationTestWalk(t, z, bucket, 0, "decommission"); failed != 1 {
|
||||
t.Fatalf("%d failed versions, want 1", failed)
|
||||
}
|
||||
if _, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, tiered.Name, ObjectOptions{VersionID: nullVersionID}); err != nil {
|
||||
t.Fatalf("failed move removed its source: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -265,6 +265,7 @@ func TestPoolsMultipartConditionalLatestVersionAndCallbackOnce(t *testing.T) {
|
||||
if explicit {
|
||||
current = putConsistencyObject(t, z, bucket, object, 1, "latest-other-version", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Minute)})
|
||||
}
|
||||
opts.VersionID = ""
|
||||
id, parts := multipartConditionUpload(t, z, bucket, object, 1, opts)
|
||||
called := 0
|
||||
opts.CheckPrecondFn = func(oi ObjectInfo) bool { called++; return oi.ETag != current.ETag }
|
||||
|
||||
@@ -355,7 +355,7 @@ func TestPoolsConditionalPutVersionSelection(t *testing.T) {
|
||||
opts.ReplicaLockReconcile, opts.ReplicationRequest = true, true
|
||||
want = addressed
|
||||
case "movement":
|
||||
opts.DataMovement, opts.SrcPoolIdx = true, 1
|
||||
opts.DataMovement, opts.SrcPoolIdx, opts.MTime = true, 1, addressed.ModTime
|
||||
want = addressed
|
||||
case "tie":
|
||||
want = putConsistencyObject(t, z, bucket, object, 0, "tie-winner", ObjectOptions{Versioned: true, MTime: current.ModTime})
|
||||
@@ -633,6 +633,8 @@ func TestPoolsConditionalPutReplicaAvailability(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A public client cannot address a destination version, with or without a
|
||||
// precondition; only trusted replication may (SN-2026-016).
|
||||
func TestPoolsConditionalPutDestinationVersionHTTP(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
bucket, router := conditionalPutBucket(t, z, "versioned")
|
||||
@@ -641,21 +643,14 @@ func TestPoolsConditionalPutDestinationVersionHTTP(t *testing.T) {
|
||||
current := putConsistencyObject(t, z, bucket, object, 1, "current", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Minute)})
|
||||
defer conditionalPutPool(t, z, object, 0)()
|
||||
url := getPutObjectURL("", bucket, object) + "?versionId=" + old.VersionID
|
||||
for _, stale := range []bool{true, false} {
|
||||
tag, status := current.ETag, http.StatusOK
|
||||
if stale {
|
||||
tag, status = old.ETag, http.StatusPreconditionFailed
|
||||
}
|
||||
for _, tag := range []string{old.ETag, current.ETag} {
|
||||
put := multipartConditionRequest(t, router, http.MethodPut, url, "replacement", map[string]string{xhttp.IfMatch: fmt.Sprintf("%q", tag)})
|
||||
if put.Code != status {
|
||||
t.Fatalf("version-addressed public PUT: %d want %d: %s", put.Code, status, put.Body.String())
|
||||
}
|
||||
if got := put.Header()[xhttp.AmzVersionID]; !stale && (len(got) != 1 || got[0] != old.VersionID) {
|
||||
t.Fatalf("write version changed: %v", put.Header())
|
||||
if put.Code != http.StatusForbidden {
|
||||
t.Fatalf("version-addressed public PUT: %d want 403: %s", put.Code, put.Body.String())
|
||||
}
|
||||
}
|
||||
get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil)
|
||||
if get.Code != http.StatusOK || get.Body.String() != "replacement" {
|
||||
if get.Code != http.StatusOK || get.Body.String() != "old" {
|
||||
t.Fatalf("addressed GET: %d %q", get.Code, get.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -623,7 +623,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
|
||||
// to create the appropriate stack.
|
||||
versionsSorter(fivs.Versions).reverse()
|
||||
|
||||
var rebalanced, expired int
|
||||
var rebalanced int
|
||||
for _, version := range fivs.Versions {
|
||||
stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceObject, poolIdx, bucket, version.Name, version.VersionID)
|
||||
|
||||
@@ -635,21 +635,11 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
|
||||
|
||||
// Apply lifecycle rules on the objects that are expired.
|
||||
if filterLifecycle(bucket, version.Name, version) {
|
||||
expired++
|
||||
stopFn(version.Size, errors.New("ILM expired object/version will be skipped"))
|
||||
continue
|
||||
}
|
||||
|
||||
// any object with only single DEL marker we don't need
|
||||
// to rebalance, just skip it, this also includes
|
||||
// any other versions that have already expired.
|
||||
remainingVersions := len(fivs.Versions) - expired
|
||||
if version.Deleted && remainingVersions == 1 {
|
||||
rebalanced++
|
||||
stopFn(version.Size, errors.New("DELETE marked object with no other non-current versions will be skipped"))
|
||||
continue
|
||||
}
|
||||
|
||||
// Preserve every marker: older versions may live in another pool.
|
||||
versionID := version.VersionID
|
||||
if versionID == "" {
|
||||
versionID = nullVersionID
|
||||
@@ -671,6 +661,9 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
|
||||
SkipRebalancing: true, // make sure we skip the decommissioned pool
|
||||
NoAuditLog: true,
|
||||
})
|
||||
if isDataMovementSuperseded(err) {
|
||||
err = nil
|
||||
}
|
||||
// This can happen when rebalance stop races with ongoing rebalance workers.
|
||||
// These rebalance failures can be ignored.
|
||||
if err != nil {
|
||||
@@ -719,7 +712,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
|
||||
continue
|
||||
}
|
||||
|
||||
if err = z.rebalanceObject(ctx, poolIdx, bucket, gr); err != nil {
|
||||
if err = z.rebalanceObject(ctx, poolIdx, bucket, gr); err != nil && !isDataMovementSuperseded(err) {
|
||||
// This can happen when rebalance stop races with ongoing rebalance workers.
|
||||
// These rebalance failures can be ignored.
|
||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
|
||||
@@ -750,15 +743,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
|
||||
// if all versions were rebalanced, we can delete the object versions.
|
||||
if rebalanced == len(fivs.Versions) {
|
||||
stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceRemoveObject, poolIdx, bucket, entry.name)
|
||||
_, err := set.DeleteObject(ctx,
|
||||
bucket,
|
||||
encodeDirObject(entry.name),
|
||||
ObjectOptions{
|
||||
DeletePrefix: true, // use prefix delete to delete all versions at once.
|
||||
DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
|
||||
NoAuditLog: true,
|
||||
},
|
||||
)
|
||||
err := z.removeMovedVersions(ctx, bucket, entry.name, poolIdx, fivs.Versions)
|
||||
stopFn(0, err)
|
||||
auditLogRebalance(ctx, "Rebalance:DeleteObject", bucket, entry.name, "", err)
|
||||
if err != nil {
|
||||
@@ -863,9 +848,15 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, poolIdx int, b
|
||||
}
|
||||
|
||||
if oi.isMultipart() {
|
||||
// Completion repeats this check under the object write lock.
|
||||
if _, err := z.checkDataMovementCopies(ctx, bucket, encodeDirObject(oi.Name), ObjectOptions{
|
||||
VersionID: oi.VersionID, MTime: oi.ModTime, SrcPoolIdx: poolIdx,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
res, err := z.NewMultipartUpload(ctx, bucket, oi.Name, ObjectOptions{
|
||||
VersionID: oi.VersionID,
|
||||
UserDefined: oi.UserDefined,
|
||||
UserDefined: migrationObjectMetadata(oi),
|
||||
NoAuditLog: true,
|
||||
DataMovement: true,
|
||||
SrcPoolIdx: poolIdx,
|
||||
@@ -900,6 +891,8 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, poolIdx int, b
|
||||
}
|
||||
}
|
||||
_, err = z.CompleteMultipartUpload(ctx, bucket, oi.Name, res.UploadID, parts, ObjectOptions{
|
||||
VersionID: oi.VersionID,
|
||||
SrcPoolIdx: poolIdx,
|
||||
DataMovement: true,
|
||||
MTime: oi.ModTime,
|
||||
NoAuditLog: true,
|
||||
@@ -924,7 +917,7 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, poolIdx int, b
|
||||
DataMovement: true,
|
||||
VersionID: oi.VersionID,
|
||||
MTime: oi.ModTime,
|
||||
UserDefined: oi.UserDefined,
|
||||
UserDefined: migrationObjectMetadata(oi),
|
||||
PreserveETag: oi.ETag, // Preserve original ETag to ensure same metadata.
|
||||
IndexCB: func() []byte {
|
||||
return oi.Parts[0].Index // Preserve part Index to ensure decompression works.
|
||||
@@ -1010,7 +1003,10 @@ const (
|
||||
rebalanceMetricSaveMetadata
|
||||
)
|
||||
|
||||
var errDataMovementSrcDstPoolSame = errors.New("source and destination pool are the same")
|
||||
var (
|
||||
errDataMovementSrcDstPoolSame = errors.New("source and destination pool are the same")
|
||||
errDataMovementSuperseded = errors.New("another pool holds a newer copy")
|
||||
)
|
||||
|
||||
func rebalanceTrace(r rebalanceMetric, poolIdx int, startTime time.Time, duration time.Duration, err error, path string, sz int64) madmin.TraceInfo {
|
||||
var errStr string
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||
// maintenance release gate; failures record intentionally unresolved risks.
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
)
|
||||
|
||||
// An acknowledged recovery overwrite must resolve the unreadable pool, even
|
||||
// when another pool already has a readable copy of the same object.
|
||||
// Real inline metadata is combined in the same 15-old/1-new state used by
|
||||
// that regression. Capacity adapters only make destination choice deterministic.
|
||||
func TestPoolsMinorityOverwriteRecovery(t *testing.T) {
|
||||
for _, versioning := range []string{"unversioned", "versioned", "suspended"} {
|
||||
for _, mode := range []string{"single-pool", "dual-pool-only-inconsistent-copy", "dual-pool-readable-other-copy"} {
|
||||
t.Run(versioning+"/"+mode, func(t *testing.T) {
|
||||
testPoolsMinorityOverwriteRecovery(t, mode, versioning, "")
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, encryption := range []string{"SSE-C", "SSE-S3", "SSE-KMS"} {
|
||||
t.Run(encryption, func(t *testing.T) {
|
||||
testPoolsMinorityOverwriteRecovery(t, "dual-pool-readable-other-copy", "unversioned", encryption)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func testPoolsMinorityOverwriteRecovery(t *testing.T, mode, versioning, encryption string) {
|
||||
t.Helper()
|
||||
var z *erasureServerPools
|
||||
if mode == "single-pool" {
|
||||
z, _ = mutationPools(t)
|
||||
} else {
|
||||
z, _ = consistencyPools(t)
|
||||
}
|
||||
const object = "review-retry"
|
||||
for _, pool := range z.serverPools {
|
||||
restore := conditionalPutSwapDisks(pool, object, func(d StorageAPI) StorageAPI { return conditionalPutCapacityDisk{StorageAPI: d} })
|
||||
defer restore()
|
||||
}
|
||||
bucket, router := conditionalPutBucket(t, z, versioning)
|
||||
oldTime := time.Now().UTC().Add(-time.Hour)
|
||||
oldBody := bytes.Repeat([]byte{'a'}, 8192)
|
||||
pendingBody := bytes.Repeat([]byte{'b'}, 8192)
|
||||
recoveryBody := bytes.Repeat([]byte{'c'}, 8192)
|
||||
seed := func(pool int, body []byte, when time.Time) {
|
||||
t.Helper()
|
||||
putConsistencyObject(t, z, bucket, object, pool, string(body), ObjectOptions{MTime: when})
|
||||
}
|
||||
seed(0, oldBody, oldTime)
|
||||
disks := z.serverPools[0].getHashedSet(object).getDisks()
|
||||
oldMeta := make([][]byte, len(disks))
|
||||
for i, disk := range disks {
|
||||
oldMeta[i] = mustReadNullQuorumMeta(t, disk, bucket, object)
|
||||
}
|
||||
seed(0, pendingBody, oldTime.Add(time.Minute))
|
||||
for i := 0; i < len(disks)-1; i++ {
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if mode == "dual-pool-readable-other-copy" {
|
||||
seed(1, oldBody, oldTime.Add(-time.Minute))
|
||||
// Ordinary allocation can select this healthy other pool.
|
||||
// Force that permitted choice without altering metadata reads.
|
||||
restore := conditionalPutPool(t, z, object, 1)
|
||||
defer restore()
|
||||
}
|
||||
before := consistencyRequest(t, router, http.MethodGet, bucket, object, "")
|
||||
if before.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("fixture must refuse ambiguous read; got %d", before.Code)
|
||||
}
|
||||
var headers, readHeaders map[string]string
|
||||
if encryption != "" {
|
||||
oldKMS, oldTLS := GlobalKMS, globalIsTLS
|
||||
GlobalKMS, globalIsTLS = kms.NewStub("recovery-key"), true
|
||||
defer func() { GlobalKMS, globalIsTLS = oldKMS, oldTLS }()
|
||||
headers = map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES}
|
||||
switch encryption {
|
||||
case "SSE-C":
|
||||
key := bytes.Repeat([]byte{0x42}, 32)
|
||||
digest := md5.Sum(key)
|
||||
headers = map[string]string{
|
||||
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key),
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(digest[:]),
|
||||
}
|
||||
readHeaders = headers
|
||||
case "SSE-KMS":
|
||||
headers[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
|
||||
headers[xhttp.AmzServerSideEncryptionKmsID] = "recovery-key"
|
||||
}
|
||||
}
|
||||
url := getPutObjectURL("", bucket, object)
|
||||
result := multipartConditionRequest(t, router, http.MethodPut, url, string(recoveryBody), headers)
|
||||
t.Logf("recovery PUT HTTP=%d response=%s", result.Code, result.Body.String())
|
||||
if result.Code != http.StatusOK {
|
||||
t.Fatalf("unconditional recovery PUT: %d %s", result.Code, result.Body.String())
|
||||
}
|
||||
after := multipartConditionRequest(t, router, http.MethodGet, url, "", readHeaders)
|
||||
t.Logf("after acknowledged recovery PUT: GET HTTP=%d matches_new=%t", after.Code, bytes.Equal(after.Body.Bytes(), recoveryBody))
|
||||
for i, pool := range z.serverPools {
|
||||
oi, err := pool.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||
t.Logf("pool=%d mtime=%s err=%v", i, oi.ModTime, err)
|
||||
}
|
||||
if after.Code != http.StatusOK || !bytes.Equal(after.Body.Bytes(), recoveryBody) {
|
||||
t.Fatalf("acknowledged recovery overwrite did not restore reads: HTTP=%d body=%s", after.Code, after.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Read failures activated by a completed disk mutation verify that a recovery
|
||||
// ACK is not issued from preflight information alone.
|
||||
type recoveryReadFaultDisk struct {
|
||||
StorageAPI
|
||||
bucket, object string
|
||||
fail *atomic.Bool
|
||||
afterRename bool
|
||||
}
|
||||
|
||||
func (d recoveryReadFaultDisk) ReadXL(ctx context.Context, bucket, object string, data bool) (RawFileInfo, error) {
|
||||
if bucket == d.bucket && object == d.object && d.fail.Load() {
|
||||
return RawFileInfo{}, errDiskNotFound
|
||||
}
|
||||
return d.StorageAPI.ReadXL(ctx, bucket, object, data)
|
||||
}
|
||||
|
||||
func (d recoveryReadFaultDisk) ReadVersion(ctx context.Context, orig, bucket, object, version string, opts ReadOptions) (FileInfo, error) {
|
||||
if bucket == d.bucket && object == d.object && d.fail.Load() {
|
||||
return FileInfo{}, errDiskNotFound
|
||||
}
|
||||
return d.StorageAPI.ReadVersion(ctx, orig, bucket, object, version, opts)
|
||||
}
|
||||
|
||||
func (d recoveryReadFaultDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
|
||||
res, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts)
|
||||
if err == nil && d.afterRename && dv == d.bucket && dp == d.object {
|
||||
d.fail.Store(true)
|
||||
}
|
||||
return res, err
|
||||
}
|
||||
|
||||
func TestPoolsRecoveryRefusesUnresolvedPools(t *testing.T) {
|
||||
for _, mode := range []string{"two-inconsistent", "suspended", "rebalancing", "offline", "read-fails-after-commit", "newer-readable-other-copy"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
const object = "refused-recovery"
|
||||
bucket, router := conditionalPutBucket(t, z, "unversioned")
|
||||
for _, pool := range z.serverPools {
|
||||
restore := conditionalPutSwapDisks(pool, object, func(d StorageAPI) StorageAPI { return conditionalPutCapacityDisk{StorageAPI: d} })
|
||||
defer restore()
|
||||
}
|
||||
base := time.Now().UTC().Add(-time.Hour)
|
||||
old := bytes.Repeat([]byte{'a'}, 8192)
|
||||
pending := bytes.Repeat([]byte{'b'}, 8192)
|
||||
replacement := bytes.Repeat([]byte{'c'}, 8192)
|
||||
inconsistent := func(pool int) {
|
||||
t.Helper()
|
||||
putConsistencyObject(t, z, bucket, object, pool, string(old), ObjectOptions{MTime: base})
|
||||
disks := z.serverPools[pool].getHashedSet(object).getDisks()
|
||||
saved := make([][]byte, len(disks))
|
||||
for i, d := range disks {
|
||||
saved[i] = mustReadNullQuorumMeta(t, d, bucket, object)
|
||||
}
|
||||
putConsistencyObject(t, z, bucket, object, pool, string(pending), ObjectOptions{MTime: base.Add(time.Minute)})
|
||||
for i := 0; i < len(disks)-1; i++ {
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, saved[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
inconsistent(0)
|
||||
otherTime := base.Add(-time.Minute)
|
||||
if mode == "newer-readable-other-copy" {
|
||||
otherTime = base.Add(2 * time.Hour)
|
||||
}
|
||||
putConsistencyObject(t, z, bucket, object, 1, string(old), ObjectOptions{MTime: otherTime})
|
||||
if mode == "two-inconsistent" {
|
||||
inconsistent(1)
|
||||
}
|
||||
// Select a healthy destination if the resolver incorrectly ignores pool0.
|
||||
postCommitFailure := mode == "read-fails-after-commit" || mode == "newer-readable-other-copy"
|
||||
if !postCommitFailure {
|
||||
restore := conditionalPutPool(t, z, object, 1)
|
||||
defer restore()
|
||||
}
|
||||
if mode == "suspended" {
|
||||
z.poolMetaMutex.Lock()
|
||||
z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{}
|
||||
z.poolMetaMutex.Unlock()
|
||||
}
|
||||
if mode == "rebalancing" {
|
||||
z.rebalMu.Lock()
|
||||
z.rebalMeta = &rebalanceMeta{PoolStats: []*rebalanceStats{{Participating: true, Info: rebalanceInfo{Status: rebalStarted}}, {}}}
|
||||
z.rebalMu.Unlock()
|
||||
}
|
||||
var fail atomic.Bool
|
||||
if mode == "offline" || mode == "read-fails-after-commit" {
|
||||
fail.Store(mode == "offline")
|
||||
restore := conditionalPutSwapDisks(z.serverPools[0], object, func(d StorageAPI) StorageAPI {
|
||||
return recoveryReadFaultDisk{StorageAPI: d, bucket: bucket, object: object, fail: &fail, afterRename: mode == "read-fails-after-commit"}
|
||||
})
|
||||
defer restore()
|
||||
}
|
||||
snapshot := func() [][32]byte {
|
||||
var result [][32]byte
|
||||
for _, pool := range z.serverPools {
|
||||
for _, disk := range pool.getHashedSet(object).getDisks() {
|
||||
b, err := disk.ReadAll(t.Context(), bucket, object+"/"+xlStorageFormatFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result = append(result, sha256.Sum256(b))
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
before := snapshot()
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, getGetObjectURL("", bucket, object), int64(len(replacement)), bytes.NewReader(replacement), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := httptest.NewRecorder()
|
||||
router.ServeHTTP(result, req.WithContext(t.Context()))
|
||||
if result.Code != http.StatusServiceUnavailable {
|
||||
t.Errorf("unresolved recovery returned HTTP=%d: %s", result.Code, result.Body.String())
|
||||
}
|
||||
switch {
|
||||
case !postCommitFailure:
|
||||
after := snapshot()
|
||||
for i := range before {
|
||||
if before[i] != after[i] {
|
||||
t.Errorf("refused preflight modified shard %d", i)
|
||||
}
|
||||
}
|
||||
case mode == "read-fails-after-commit":
|
||||
fail.Store(false)
|
||||
got := consistencyRequest(t, router, http.MethodGet, bucket, object, "")
|
||||
if got.Code != http.StatusOK || !bytes.Equal(got.Body.Bytes(), replacement) {
|
||||
t.Errorf("read validation failure must not undo committed recovery: HTTP=%d", got.Code)
|
||||
}
|
||||
default:
|
||||
// A readable but different latest copy cannot confirm recovery.
|
||||
// The committed replacement must still remain in its target pool.
|
||||
r, err := z.serverPools[0].GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil || !bytes.Equal(body, replacement) {
|
||||
t.Fatalf("failed recovery validation must preserve committed data: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+226
-116
@@ -39,6 +39,7 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/bpool"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
"github.com/minio/minio/internal/cachevalue"
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
@@ -1183,6 +1184,14 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
|
||||
opts.CheckPrecondFn = nil
|
||||
}
|
||||
|
||||
if opts.DataMovement {
|
||||
resolve, err := z.checkDataMovementCopies(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
opts.replicaObjectInfo = resolve
|
||||
}
|
||||
|
||||
idx, err := z.getWritePoolIdx(ctx, bucket, object, data.Size(), true)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
@@ -1199,6 +1208,10 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
|
||||
|
||||
if opts.ReplicaLockReconcile || opts.DataMovement {
|
||||
opts.ReplicaLockReconcile = true
|
||||
}
|
||||
// Public overwrites need all pools for Object Lock. Unaddressed replica
|
||||
// callbacks retain their set-local reads unless reconciliation is needed.
|
||||
if opts.replicaObjectInfo == nil && (opts.ReplicaLockReconcile || !opts.ReplicationRequest) {
|
||||
opts.replicaObjectInfo = z.replicaObjectInfo
|
||||
}
|
||||
oi, err := z.serverPools[idx].PutObject(ctx, bucket, object, data, opts)
|
||||
@@ -1239,9 +1252,31 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
||||
return ObjectInfo{}, z.deletePrefix(ctx, bucket, object)
|
||||
}
|
||||
|
||||
// Reconcile ordinary addressed-version deletes independently of pool movement.
|
||||
if opts.DataMovement {
|
||||
opts.NoLock = true
|
||||
if _, err := z.checkDataMovementCopies(ctx, bucket, object, opts); err != nil {
|
||||
return objInfo, err
|
||||
}
|
||||
// A marker still needs a destination when it is the only version of
|
||||
// this key outside the draining pool.
|
||||
idx, err := z.getPoolIdxNoLock(ctx, bucket, object, 0)
|
||||
if err != nil {
|
||||
return objInfo, err
|
||||
}
|
||||
if idx == opts.SrcPoolIdx {
|
||||
return objInfo, DataMovementOverwriteErr{Bucket: bucket, Object: object, VersionID: opts.VersionID, Err: errDataMovementSrcDstPoolSame}
|
||||
}
|
||||
objInfo, err = z.serverPools[idx].DeleteObject(ctx, bucket, object, opts)
|
||||
objInfo.Name = decodeDirObject(object)
|
||||
return objInfo, err
|
||||
}
|
||||
|
||||
// Resolve a physical purge by its addressed version even on a replica
|
||||
// receiver: latest-key routing can select a different pool or miss copies.
|
||||
// Marker creation and specialized movement/scanner operations retain their
|
||||
// existing routing.
|
||||
reconcileVersion := opts.VersionID != "" && !opts.DataMovement &&
|
||||
!opts.ReplicationRequest && !opts.Expiration.Expire && !opts.InclFreeVersions
|
||||
(!opts.ReplicationRequest || opts.isVersionPurge()) && !opts.Expiration.Expire && !opts.InclFreeVersions
|
||||
if !z.SinglePool() && (opts.CheckPrecondFn != nil || reconcileVersion) {
|
||||
return z.deleteObjectReconciled(ctx, bucket, object, opts)
|
||||
}
|
||||
@@ -1254,6 +1289,13 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
||||
if _, ok := err.(InsufficientReadQuorum); ok {
|
||||
return objInfo, InsufficientWriteQuorum{}
|
||||
}
|
||||
// Lookup can return before the set's purge confirmation. Check here,
|
||||
// before any callback can change the request into a metadata update.
|
||||
if opts.isVersionPurge() && (isErrObjectNotFound(err) || isErrVersionNotFound(err)) {
|
||||
if quorumErr := z.checkPurgeAbsent(ctx, bucket, object, opts.VersionID); quorumErr != nil {
|
||||
return objInfo, quorumErr
|
||||
}
|
||||
}
|
||||
// A conditional (If-Match) delete addressing a specific version treats an
|
||||
// absent key as an absent version. getPoolInfoExistingWithOpts strips
|
||||
// VersionID, so a missing key surfaces ObjectNotFound here even for a
|
||||
@@ -1284,6 +1326,11 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
||||
if verr != nil && (!isErrMethodNotAllowed(verr) || !vi.DeleteMarker) {
|
||||
// Genuine read failure for the addressed version: a missing
|
||||
// version -> VersionNotFound (NoSuchVersion), read-quorum loss, etc.
|
||||
if opts.isVersionPurge() && (isErrObjectNotFound(verr) || isErrVersionNotFound(verr)) {
|
||||
if quorumErr := z.checkPurgeAbsent(ctx, bucket, object, opts.VersionID); quorumErr != nil {
|
||||
return objInfo, quorumErr
|
||||
}
|
||||
}
|
||||
return objInfo, verr
|
||||
}
|
||||
// verr is nil for a live version, or MethodNotAllowed with a populated
|
||||
@@ -1309,24 +1356,13 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
||||
return pinfo.ObjInfo, nil
|
||||
}
|
||||
|
||||
// Datamovement must never be allowed on the same pool.
|
||||
if opts.DataMovement && opts.SrcPoolIdx == pinfo.Index {
|
||||
return pinfo.ObjInfo, DataMovementOverwriteErr{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
VersionID: opts.VersionID,
|
||||
Err: errDataMovementSrcDstPoolSame,
|
||||
}
|
||||
}
|
||||
|
||||
if opts.DataMovement {
|
||||
objInfo, err = z.serverPools[pinfo.Index].DeleteObject(ctx, bucket, object, opts)
|
||||
objInfo.Name = decodeDirObject(object)
|
||||
return objInfo, err
|
||||
}
|
||||
|
||||
// Delete concurrently in all server pools with read quorum error for unversioned objects.
|
||||
// Retire older copies before the newest copy for unversioned objects.
|
||||
if len(noReadQuorumPools) > 0 && !opts.Versioned && !opts.VersionSuspended {
|
||||
if pinfo.ObjInfo.TransitionedObject.Status == lifecycle.TransitionComplete {
|
||||
// Restored and remote copies can share a tier object. Reuse the
|
||||
// reconciliation which keeps that reference out of premature GC.
|
||||
return z.deleteObjectReconciled(ctx, bucket, object, opts)
|
||||
}
|
||||
return z.deleteObjectFromAllPools(ctx, bucket, object, opts, noReadQuorumPools)
|
||||
}
|
||||
|
||||
@@ -1337,6 +1373,21 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
||||
return objInfo, err
|
||||
}
|
||||
|
||||
// A delete marker hides the object in every pool. When more than one pool
|
||||
// holds the object, put the marker into the first one with a readable
|
||||
// version whose set can take the write, so that a set without quorum does
|
||||
// not fail the delete while another pool could take the marker.
|
||||
if opts.VersionID == "" && len(noReadQuorumPools) > 1 {
|
||||
for idx, pool := range z.serverPools {
|
||||
if !poolReadable(noReadQuorumPools, idx) || deleteQuorumMissing(pool, object) {
|
||||
continue
|
||||
}
|
||||
objInfo, err = pool.DeleteObject(ctx, bucket, object, opts)
|
||||
objInfo.Name = decodeDirObject(object)
|
||||
return objInfo, err
|
||||
}
|
||||
}
|
||||
|
||||
for _, pool := range z.serverPools {
|
||||
objInfo, err := pool.DeleteObject(ctx, bucket, object, opts)
|
||||
if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
@@ -1357,36 +1408,76 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
||||
}
|
||||
|
||||
func (z *erasureServerPools) deleteObjectFromAllPools(ctx context.Context, bucket string, object string, opts ObjectOptions, poolIndices []poolErrs) (objInfo ObjectInfo, err error) {
|
||||
derrs := make([]error, len(poolIndices))
|
||||
dobjects := make([]ObjectInfo, len(poolIndices))
|
||||
|
||||
// Delete concurrently in all server pools that reported no error or read quorum error
|
||||
// where the read quorum issue is from metadata inconsistency.
|
||||
var wg sync.WaitGroup
|
||||
for idx, pe := range poolIndices {
|
||||
if v, ok := pe.Err.(InsufficientReadQuorum); ok && v.Type != RQInconsistentMeta {
|
||||
derrs[idx] = InsufficientWriteQuorum{}
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
pool := z.serverPools[pe.Index]
|
||||
go func(idx int, pool *erasureSets) {
|
||||
defer wg.Done()
|
||||
dobjects[idx], derrs[idx] = pool.DeleteObject(ctx, bucket, object, opts)
|
||||
}(idx, pool)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// the poolIndices array is pre-sorted in order of latest ModTime, we care only about pool with latest object though
|
||||
// the delete call tries to clean up other pools during DeleteObject call.
|
||||
objInfo = dobjects[0]
|
||||
objInfo.Name = decodeDirObject(object)
|
||||
for _, derr := range derrs {
|
||||
|
||||
// Delete in all server pools that reported no error or read quorum error
|
||||
// where the read quorum issue is from metadata inconsistency. The
|
||||
// poolIndices array is pre-sorted in order of latest ModTime. Retire the
|
||||
// older copies first and stop at the first failure, as
|
||||
// deleteObjectReconciled does: a failed delete then keeps the latest copy
|
||||
// in place, instead of serving an older write of the object.
|
||||
for idx := len(poolIndices) - 1; idx >= 0; idx-- {
|
||||
pe := poolIndices[idx]
|
||||
if v, ok := pe.Err.(InsufficientReadQuorum); ok && v.Type != RQInconsistentMeta {
|
||||
return objInfo, InsufficientWriteQuorum{}
|
||||
}
|
||||
dobj, derr := z.serverPools[pe.Index].DeleteObject(ctx, bucket, object, opts)
|
||||
if idx == 0 {
|
||||
objInfo, err = dobj, derr
|
||||
break
|
||||
}
|
||||
if derr != nil && !isErrObjectNotFound(derr) && !isErrVersionNotFound(derr) {
|
||||
return objInfo, derr
|
||||
dobj.Name = decodeDirObject(object)
|
||||
return dobj, derr
|
||||
}
|
||||
}
|
||||
return objInfo, derrs[0]
|
||||
objInfo.Name = decodeDirObject(object)
|
||||
return objInfo, err
|
||||
}
|
||||
|
||||
// deleteQuorumMissing reports whether the set of object in pool has fewer
|
||||
// online drives than a delete needs (N/2+1, as in erasureObjects.DeleteObjects).
|
||||
func deleteQuorumMissing(pool *erasureSets, object string) bool {
|
||||
disks := pool.getHashedSet(object).getDisks()
|
||||
return countOnlineDisks(disks) < len(disks)/2+1
|
||||
}
|
||||
|
||||
// poolReadable reports whether pools lists pool idx with a readable version.
|
||||
func poolReadable(pools []poolErrs, idx int) bool {
|
||||
for _, pe := range pools {
|
||||
if pe.Index == idx {
|
||||
return pe.Err == nil
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// copyWithoutDeleteQuorum avoids starting a known unwritable cleanup. This
|
||||
// reduces partial deletion, but successful cleanup must still precede removing
|
||||
// the authoritative copy: drives can fail after this preflight.
|
||||
func (z *erasureServerPools) copyWithoutDeleteQuorum(ctx context.Context, bucket, object, versionID string) bool {
|
||||
for _, pool := range z.serverPools {
|
||||
if deleteQuorumMissing(pool, object) {
|
||||
_, err := pool.GetObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: versionID, NoLock: true})
|
||||
if err == nil || isErrReadQuorum(err) || isErrMethodNotAllowed(err) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// deletesData reports whether deleting obj removes an object or a version, as
|
||||
// opposed to adding a delete marker (see erasureObjects.DeleteObjects).
|
||||
func deletesData(obj ObjectToDelete, opts ObjectOptions) bool {
|
||||
if obj.VersionID != "" {
|
||||
return true
|
||||
}
|
||||
versioned := opts.Versioned
|
||||
if opts.PrefixEnabledFn != nil {
|
||||
versioned = opts.PrefixEnabledFn(obj.ObjectName)
|
||||
}
|
||||
return !versioned && !opts.VersionSuspended
|
||||
}
|
||||
|
||||
func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
|
||||
@@ -1395,6 +1486,11 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
|
||||
objSets := set.NewStringSet()
|
||||
for i := range derrs {
|
||||
objects[i].ObjectName = encodeDirObject(objects[i].ObjectName)
|
||||
dobjects[i] = DeletedObject{
|
||||
ObjectName: decodeDirObject(objects[i].ObjectName),
|
||||
VersionID: objects[i].VersionID,
|
||||
ReplicationState: objects[i].ReplicationState(),
|
||||
}
|
||||
|
||||
derrs[i] = checkDelObjArgs(ctx, bucket, objects[i].ObjectName)
|
||||
objSets.Add(objects[i].ObjectName)
|
||||
@@ -1412,25 +1508,64 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
|
||||
ctx = lkctx.Context()
|
||||
defer multiDeleteLock.Unlock(lkctx)
|
||||
|
||||
// Removing data needs the same authoritative-copy protection as a single
|
||||
// DELETE. An online-drive preflight cannot protect against later I/O errors.
|
||||
// Keep marker creation after data removal, including when the same request
|
||||
// explicitly removes a null version and creates its replacement marker.
|
||||
batch, batchIdx := objects, []int(nil)
|
||||
if !z.SinglePool() {
|
||||
batch = make([]ObjectToDelete, 0, len(objects))
|
||||
batchIdx = make([]int, 0, len(objects))
|
||||
data, dataIdx := make([]ObjectToDelete, 0, len(objects)), make([]int, 0, len(objects))
|
||||
for i, obj := range objects {
|
||||
if derrs[i] != nil {
|
||||
continue
|
||||
}
|
||||
if deletesData(obj, opts) {
|
||||
if z.copyWithoutDeleteQuorum(ctx, bucket, obj.ObjectName, obj.VersionID) {
|
||||
derrs[i] = InsufficientWriteQuorum{}
|
||||
continue
|
||||
}
|
||||
data = append(data, obj)
|
||||
dataIdx = append(dataIdx, i)
|
||||
continue
|
||||
}
|
||||
batch = append(batch, obj)
|
||||
batchIdx = append(batchIdx, i)
|
||||
}
|
||||
if len(data) > 0 {
|
||||
deleted, errs := z.deleteObjectsReconciled(ctx, bucket, data, opts)
|
||||
for j, i := range dataIdx {
|
||||
dobjects[i], derrs[i] = deleted[j], errs[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dObjectsByPool := make([][]DeletedObject, len(z.serverPools))
|
||||
dErrsByPool := make([][]error, len(z.serverPools))
|
||||
|
||||
eg := errgroup.WithNErrs(len(z.serverPools)).WithConcurrency(len(z.serverPools))
|
||||
for i, pool := range z.serverPools {
|
||||
pool := pool
|
||||
eg.Go(func() error {
|
||||
dObjectsByPool[i], dErrsByPool[i] = pool.DeleteObjects(ctx, bucket, objects, opts)
|
||||
return nil
|
||||
}, i)
|
||||
if len(batch) > 0 {
|
||||
eg := errgroup.WithNErrs(len(z.serverPools)).WithConcurrency(len(z.serverPools))
|
||||
for i, pool := range z.serverPools {
|
||||
pool := pool
|
||||
eg.Go(func() error {
|
||||
dObjectsByPool[i], dErrsByPool[i] = pool.DeleteObjects(ctx, bucket, batch, opts)
|
||||
return nil
|
||||
}, i)
|
||||
}
|
||||
eg.Wait() // wait to check all the pools.
|
||||
}
|
||||
eg.Wait() // wait to check all the pools.
|
||||
|
||||
for i := range dobjects {
|
||||
for j := range batch {
|
||||
i := j
|
||||
if batchIdx != nil {
|
||||
i = batchIdx[j]
|
||||
}
|
||||
// Iterate over pools
|
||||
for pool := range z.serverPools {
|
||||
if dErrsByPool[pool][i] == nil && dObjectsByPool[pool][i].found {
|
||||
if dErrsByPool[pool][j] == nil && dObjectsByPool[pool][j].found {
|
||||
// A fast exit when the object is found and removed
|
||||
dobjects[i] = dObjectsByPool[pool][i]
|
||||
dobjects[i] = dObjectsByPool[pool][j]
|
||||
derrs[i] = nil
|
||||
break
|
||||
}
|
||||
@@ -1438,8 +1573,8 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
|
||||
// No error related to this object is found, assign this pool result
|
||||
// whether it is nil because there is no object found or because of
|
||||
// some other errors such erasure quorum errors.
|
||||
dobjects[i] = dObjectsByPool[pool][i]
|
||||
derrs[i] = dErrsByPool[pool][i]
|
||||
dobjects[i] = dObjectsByPool[pool][j]
|
||||
derrs[i] = dErrsByPool[pool][j]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1495,9 +1630,9 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
|
||||
return objInfo, err
|
||||
}
|
||||
|
||||
if !z.SinglePool() && dstOpts.ReplicaLockReconcile {
|
||||
if !z.SinglePool() && (dstOpts.ReplicaLockReconcile || !dstOpts.ReplicationRequest) {
|
||||
dstOpts.replicaObjectInfo = z.replicaObjectInfo
|
||||
if !dstOpts.DataMovement {
|
||||
if dstOpts.ReplicaLockReconcile && !dstOpts.DataMovement {
|
||||
defer func() {
|
||||
if err == nil {
|
||||
err = z.retireReplicaCopies(ctx, dstBucket, dstObject, poolIdx, objInfo)
|
||||
@@ -1534,13 +1669,16 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
|
||||
ServerSideEncryption: dstOpts.ServerSideEncryption,
|
||||
UserDefined: srcInfo.UserDefined,
|
||||
Versioned: dstOpts.Versioned,
|
||||
VersionSuspended: dstOpts.VersionSuspended,
|
||||
VersionID: dstOpts.VersionID,
|
||||
MTime: dstOpts.MTime,
|
||||
NoLock: true,
|
||||
EncryptFn: dstOpts.EncryptFn,
|
||||
WantChecksum: dstOpts.WantChecksum,
|
||||
WantServerSideChecksumType: dstOpts.WantServerSideChecksumType,
|
||||
ReplicationRequest: dstOpts.ReplicationRequest,
|
||||
ReplicaLockReconcile: dstOpts.ReplicaLockReconcile,
|
||||
DataMovement: dstOpts.DataMovement,
|
||||
replicaObjectInfo: dstOpts.replicaObjectInfo,
|
||||
}
|
||||
|
||||
@@ -1894,50 +2032,17 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p
|
||||
if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
return ListMultipartsInfo{}, toObjectErr(err, bucket)
|
||||
}
|
||||
if globalAPIConfig.getMultipartListingLegacy() {
|
||||
return z.listMultipartUploadsLegacy(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
|
||||
}
|
||||
scan, err := startMultipartScan(ctx, false)
|
||||
if err != nil {
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
defer scan.close()
|
||||
|
||||
var uploads []MultipartInfo
|
||||
var keyless bool
|
||||
for idx, pool := range z.serverPools {
|
||||
if z.IsSuspended(idx) {
|
||||
continue
|
||||
}
|
||||
poolUploads, poolKeyless, err := pool.scanMultipartUploads(scan, bucket, idx)
|
||||
if err != nil {
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
uploads = append(uploads, poolUploads...)
|
||||
keyless = keyless || poolKeyless
|
||||
}
|
||||
|
||||
// The old format cannot be enumerated authoritatively. Migration mode is
|
||||
// explicit: another bucket must never silently change this API's semantics.
|
||||
if keyless {
|
||||
return ListMultipartsInfo{}, errMultipartListingLegacy
|
||||
}
|
||||
return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
|
||||
}
|
||||
|
||||
func (z *erasureServerPools) listMultipartUploadsLegacy(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
|
||||
poolResult := ListMultipartsInfo{}
|
||||
poolResult.MaxUploads = maxUploads
|
||||
poolResult.KeyMarker = keyMarker
|
||||
poolResult.Prefix = prefix
|
||||
poolResult.Delimiter = delimiter
|
||||
|
||||
// if no prefix provided, return the list from cache
|
||||
// Preserve the legacy cache view, including its incomplete pagination.
|
||||
// Persistent discovery remains an explicit admin diagnostic, not a
|
||||
// prerequisite for public listing or known-upload operations.
|
||||
if prefix == "" {
|
||||
if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
return ListMultipartsInfo{}, toObjectErr(err, bucket)
|
||||
}
|
||||
|
||||
z.mpCache.Range(func(_ string, mp MultipartInfo) bool {
|
||||
if mp.Bucket == bucket {
|
||||
poolResult.Uploads = append(poolResult.Uploads, mp)
|
||||
@@ -2177,28 +2282,15 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
|
||||
legacy := globalAPIConfig.getMultipartListingLegacy()
|
||||
found := false
|
||||
var firstErr error
|
||||
for idx, pool := range z.serverPools {
|
||||
if z.IsSuspended(idx) {
|
||||
continue
|
||||
}
|
||||
poolFound, err := pool.getHashedSet(object).abortMultipartUpload(ctx, bucket, object, uploadID, opts, legacy)
|
||||
if legacy && (poolFound || err != nil) {
|
||||
// Match the released first-matching-pool behavior.
|
||||
found, err := pool.getHashedSet(object).abortMultipartUpload(ctx, bucket, object, uploadID, opts)
|
||||
if found || err != nil {
|
||||
// Preserve the default first-matching-pool behavior.
|
||||
return err
|
||||
}
|
||||
found = found || poolFound
|
||||
if err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
if firstErr != nil {
|
||||
return firstErr
|
||||
}
|
||||
if found {
|
||||
return nil
|
||||
}
|
||||
return InvalidUploadID{
|
||||
Bucket: bucket,
|
||||
@@ -2272,6 +2364,13 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
|
||||
}
|
||||
opts.CheckPrecondFn = nil
|
||||
}
|
||||
if opts.DataMovement {
|
||||
resolve, err := z.checkDataMovementCopies(ctx, bucket, encodeDirObject(object), opts)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
opts.replicaObjectInfo = resolve
|
||||
}
|
||||
}
|
||||
|
||||
// Hold write locks to verify uploaded parts, also disallows any
|
||||
@@ -2290,6 +2389,8 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
|
||||
|
||||
if opts.ReplicaLockReconcile || opts.DataMovement {
|
||||
opts.ReplicaLockReconcile = true
|
||||
}
|
||||
if opts.replicaObjectInfo == nil && (opts.ReplicaLockReconcile || !opts.ReplicationRequest) {
|
||||
opts.replicaObjectInfo = z.replicaObjectInfo
|
||||
}
|
||||
|
||||
@@ -2627,9 +2728,9 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
|
||||
minDisks: listingQuorum,
|
||||
reportNotFound: false,
|
||||
agreed: send,
|
||||
partial: func(entries metaCacheEntries, _ []error) {
|
||||
entry, ok := entries.resolve(&resolver)
|
||||
if ok {
|
||||
partial: func(entries metaCacheEntries, errs []error) {
|
||||
entry := set.resolveListEntry(entries, errs, &resolver)
|
||||
if entry != nil {
|
||||
send(*entry)
|
||||
}
|
||||
},
|
||||
@@ -2731,7 +2832,11 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
|
||||
// Merge all entries from all disks.
|
||||
// We leave quorum at 1, since entries are already resolved to have the desired quorum.
|
||||
// mergeEntryChannels will close 'merged' channel upon completion or cancellation.
|
||||
errCh <- mergeEntryChannels(ctx, entries, merged, 1)
|
||||
errCh <- mergeEntryChannelsWithResolver(ctx, entries, merged, 1, func(int) error {
|
||||
return context.Cause(ctx)
|
||||
}, func(group []*metaCacheEntry) (*metaCacheEntry, error) {
|
||||
return z.resolveListingGroup(ctx, bucket, group)
|
||||
})
|
||||
}()
|
||||
|
||||
return nil
|
||||
@@ -3276,6 +3381,11 @@ func (z *erasureServerPools) DecomTieredObject(ctx context.Context, bucket, obje
|
||||
defer ns.Unlock(lkctx)
|
||||
opts.NoLock = true
|
||||
}
|
||||
if opts.DataMovement {
|
||||
if _, err := z.checkDataMovementCopies(ctx, bucket, object, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
idx, err := z.getPoolIdxNoLock(ctx, bucket, object, fi.Size)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+7
-30
@@ -869,47 +869,24 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
|
||||
ServerSideEncryption: dstOpts.ServerSideEncryption,
|
||||
UserDefined: srcInfo.UserDefined,
|
||||
Versioned: dstOpts.Versioned,
|
||||
VersionSuspended: dstOpts.VersionSuspended,
|
||||
VersionID: dstOpts.VersionID,
|
||||
MTime: dstOpts.MTime,
|
||||
NoLock: dstOpts.NoLock,
|
||||
EncryptFn: dstOpts.EncryptFn,
|
||||
WantChecksum: dstOpts.WantChecksum,
|
||||
WantServerSideChecksumType: dstOpts.WantServerSideChecksumType,
|
||||
ReplicationRequest: dstOpts.ReplicationRequest,
|
||||
ReplicaLockReconcile: dstOpts.ReplicaLockReconcile,
|
||||
DataMovement: dstOpts.DataMovement,
|
||||
replicaObjectInfo: dstOpts.replicaObjectInfo,
|
||||
}
|
||||
|
||||
return dstSet.putObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
|
||||
}
|
||||
|
||||
func (s *erasureSets) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
|
||||
if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil {
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
scan, err := startMultipartScan(ctx, false)
|
||||
if err != nil {
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
defer scan.close()
|
||||
uploads, legacy, err := s.scanMultipartUploads(scan, bucket, 0)
|
||||
if err != nil {
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
if legacy {
|
||||
return ListMultipartsInfo{}, errMultipartListingLegacy
|
||||
}
|
||||
return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
|
||||
}
|
||||
|
||||
func (s *erasureSets) scanMultipartUploads(scan *multipartScan, bucket string, poolIdx int) ([]MultipartInfo, bool, error) {
|
||||
var uploads []MultipartInfo
|
||||
var keyless bool
|
||||
for i, set := range s.sets {
|
||||
setUploads, setKeyless, err := set.scanMultipartUploads(scan, bucket, poolIdx, i)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
uploads = append(uploads, setUploads...)
|
||||
keyless = keyless || setKeyless
|
||||
}
|
||||
return uploads, keyless, nil
|
||||
return s.getHashedSet(prefix).ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
|
||||
}
|
||||
|
||||
func (s *erasureSets) listMultipartUploadsExact(ctx context.Context, bucket, object string) (ListMultipartsInfo, error) {
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
)
|
||||
|
||||
// isVersionPurge distinguishes physical removal from marker creation and
|
||||
// replication-state updates. Call it again after metadata callbacks, which
|
||||
// can change an ordinary deletion into a pending purge update.
|
||||
func (o ObjectOptions) isVersionPurge() bool {
|
||||
if o.VersionID == "" || o.VersionID == nullVersionID || o.DeleteMarker ||
|
||||
o.DeletePrefix || o.DataMovement || o.InclFreeVersions || o.Expiration.Expire ||
|
||||
o.Transition != (TransitionOptions{}) {
|
||||
return false
|
||||
}
|
||||
id, err := uuid.Parse(o.VersionID)
|
||||
if err != nil || id == uuid.Nil {
|
||||
return false
|
||||
}
|
||||
purge := o.VersionPurgeStatus()
|
||||
if purge == replication.VersionPurgeComplete {
|
||||
return true
|
||||
}
|
||||
if !purge.Empty() || o.DeleteReplication.VersionPurgeStatusInternal != "" {
|
||||
return false
|
||||
}
|
||||
status := o.DeleteMarkerReplicationStatus()
|
||||
return (status.Empty() && o.DeleteReplication.ReplicationStatusInternal == "") ||
|
||||
(status == replication.Replica && o.ReplicationRequest)
|
||||
}
|
||||
|
||||
// confirmVersionAbsent is a read-only proof for an already-missing purge.
|
||||
// Read quorum (or a synthesized NotFound) cannot acknowledge a write. Do not
|
||||
// delete here: unreadable minority copies may carry retention we cannot check.
|
||||
// The caller holds the object lock. This function never heals or enqueues work.
|
||||
func (er erasureObjects) confirmVersionAbsent(ctx context.Context, bucket, object, versionID string) (allAbsent bool, err error) {
|
||||
disks := er.getDisks()
|
||||
_, errs := readAllFileInfo(ctx, disks, "", bucket, object, versionID, false, false)
|
||||
absent := 0
|
||||
for _, err := range errs {
|
||||
if err == errFileNotFound || err == errFileVersionNotFound {
|
||||
absent++
|
||||
}
|
||||
}
|
||||
if absent < len(disks)/2+1 {
|
||||
return false, InsufficientWriteQuorum{}
|
||||
}
|
||||
return absent == len(disks), nil
|
||||
}
|
||||
|
||||
// checkPurgeAbsent schedules recovery explicitly, outside the read-only proof.
|
||||
func (er erasureObjects) checkPurgeAbsent(ctx context.Context, bucket, object, versionID string) error {
|
||||
allAbsent, err := er.confirmVersionAbsent(ctx, bucket, object, versionID)
|
||||
if !allAbsent {
|
||||
er.addPartial(bucket, object, versionID)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (z *erasureServerPools) checkPurgeAbsent(ctx context.Context, bucket, object, versionID string) error {
|
||||
for _, pool := range z.serverPools {
|
||||
if err := pool.getHashedSet(object).checkPurgeAbsent(ctx, bucket, object, versionID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPutObjectGenerationFollowsCommitLock(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
for _, preserve := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "generated", true: "preserved"}[preserve], func(t *testing.T) {
|
||||
const object = "ordered-generation"
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
lock := set.NewNSLock(bucket, object)
|
||||
lc, err := lock.GetLock(t.Context(), globalOperationTimeout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
locked := true
|
||||
defer func() {
|
||||
if locked {
|
||||
lock.Unlock(lc)
|
||||
}
|
||||
}()
|
||||
|
||||
opts := ObjectOptions{}
|
||||
if preserve {
|
||||
opts.MTime = time.Unix(1234567890, 0)
|
||||
}
|
||||
reader := mustGetPutObjReader(t, strings.NewReader("payload"), 7, "", "")
|
||||
type result struct {
|
||||
info ObjectInfo
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
info, err := set.PutObject(t.Context(), bucket, object, reader, opts)
|
||||
done <- result{info, err}
|
||||
}()
|
||||
|
||||
// Wait for the real writer to queue behind the held namespace
|
||||
// lock. No elapsed delay is used to infer its progress.
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
set.nsMutex.lockMapMutex.Lock()
|
||||
entry := set.nsMutex.lockMap[pathJoin(bucket, object)]
|
||||
waiting := entry != nil && entry.ref == 2
|
||||
set.nsMutex.lockMapMutex.Unlock()
|
||||
if waiting {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case got := <-done:
|
||||
t.Fatalf("PUT bypassed the held commit lock: %v", got.err)
|
||||
default:
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("PUT did not queue for the commit lock")
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
beforeUnlock := UTCNow()
|
||||
lock.Unlock(lc)
|
||||
locked = false
|
||||
got := <-done
|
||||
if got.err != nil {
|
||||
t.Fatal(got.err)
|
||||
}
|
||||
if preserve {
|
||||
if !got.info.ModTime.Equal(opts.MTime) {
|
||||
t.Fatalf("explicit modification time changed: %s", got.info.ModTime)
|
||||
}
|
||||
} else if got.info.ModTime.Before(beforeUnlock) {
|
||||
t.Fatalf("generation %s predates commit lock release %s", got.info.ModTime, beforeUnlock)
|
||||
}
|
||||
for i, disk := range set.getDisks() {
|
||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, object, nullVersionID, ReadOptions{})
|
||||
if err != nil || !fi.ModTime.Equal(got.info.ModTime) {
|
||||
t.Errorf("disk %d stored a different generation: %s, %v", i, fi.ModTime, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -319,6 +319,8 @@ var (
|
||||
|
||||
globalOperationTimeout = newDynamicTimeout(10*time.Minute, 5*time.Minute) // default timeout for general ops
|
||||
globalDeleteOperationTimeout = newDynamicTimeout(5*time.Minute, 1*time.Minute) // default time for delete ops
|
||||
// Listing recovery has its own failure history and a five-second caller budget.
|
||||
globalListingOperationTimeout = newDynamicTimeout(5*time.Second, 5*time.Second)
|
||||
|
||||
globalBucketObjectLockSys *BucketObjectLockSys
|
||||
globalBucketQuotaSys *BucketQuotaSys
|
||||
|
||||
@@ -50,7 +50,6 @@ type apiConfig struct {
|
||||
transitionWorkers int
|
||||
|
||||
staleUploadsExpiry time.Duration
|
||||
multipartListingStrict bool
|
||||
staleUploadsCleanupInterval time.Duration
|
||||
deleteCleanupInterval time.Duration
|
||||
enableODirect bool
|
||||
@@ -182,7 +181,6 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int, legacy bool) {
|
||||
t.transitionWorkers = cfg.TransitionWorkers
|
||||
|
||||
t.staleUploadsExpiry = cfg.StaleUploadsExpiry
|
||||
t.multipartListingStrict = cfg.MultipartListing == "strict"
|
||||
t.deleteCleanupInterval = cfg.DeleteCleanupInterval
|
||||
t.enableODirect = cfg.EnableODirect
|
||||
t.gzipObjects = cfg.GzipObjects
|
||||
@@ -208,12 +206,6 @@ func (t *apiConfig) odirectEnabled() bool {
|
||||
return t.enableODirect
|
||||
}
|
||||
|
||||
func (t *apiConfig) getMultipartListingLegacy() bool {
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
return !t.multipartListingStrict
|
||||
}
|
||||
|
||||
func (t *apiConfig) shouldGzipObjects() bool {
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
|
||||
@@ -85,15 +85,21 @@ func TestIAMPolicyDeletionRemainsExplicit(t *testing.T) {
|
||||
}
|
||||
p, err := sys.store.GetPolicy("readwrite")
|
||||
mustIAM(t, err)
|
||||
native, err := sys.InfoPolicy("readwrite")
|
||||
mustIAM(t, err)
|
||||
if err := sys.DeletePolicy(ctx, "readwrite", true); err == nil {
|
||||
t.Fatal("local pristine builtin policy became deletable")
|
||||
}
|
||||
_, err = sys.SetPolicy(ctx, "readwrite", p)
|
||||
mustIAM(t, err)
|
||||
mustIAM(t, sys.DeletePolicy(ctx, "readwrite", true))
|
||||
assertIAMPolicyResetInfo(t, sys, "readwrite", native)
|
||||
mustIAM(t, sys.store.LoadIAMCache(ctx, false))
|
||||
if _, err := sys.store.GetPolicy("readwrite"); !errors.Is(err, errNoSuchPolicy) {
|
||||
t.Fatalf("reload restored an explicitly deleted override: %v", err)
|
||||
assertIAMPolicyResetInfo(t, sys, "readwrite", native)
|
||||
r, err = loadIAMRevision(ctx, sys.store, getPolicyDocPath("readwrite"))
|
||||
mustIAM(t, err)
|
||||
if !r.Deleted {
|
||||
t.Fatal("native policy restoration removed the override tombstone")
|
||||
}
|
||||
_, err = sys.SetPolicy(ctx, "readwrite", p)
|
||||
mustIAM(t, err)
|
||||
|
||||
+15
-11
@@ -123,8 +123,8 @@ func (ies *IAMEtcdStore) saveIAMConfig(ctx context.Context, item any, itemPath s
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ies *IAMEtcdStore) decodeIAMConfig(item any, data []byte, path string) error {
|
||||
data, err := decryptData(data, path)
|
||||
func (ies *IAMEtcdStore) decodeIAMConfig(ctx context.Context, item any, data []byte, path string) error {
|
||||
data, err := decryptData(ctx, data, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -145,7 +145,7 @@ func (ies *IAMEtcdStore) loadIAMConfigBytes(ctx context.Context, path string) ([
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err = decryptData(data, path)
|
||||
data, err = decryptData(ctx, data, path)
|
||||
if err == nil {
|
||||
ies.index.observe(path, data)
|
||||
}
|
||||
@@ -187,7 +187,7 @@ func (ies *IAMEtcdStore) loadPolicyDoc(ctx context.Context, policy string, m map
|
||||
}
|
||||
|
||||
func (ies *IAMEtcdStore) getPolicyDocKV(ctx context.Context, kvs *mvccpb.KeyValue, m map[string]PolicyDoc) error {
|
||||
data, err := decryptData(kvs.Value, string(kvs.Key))
|
||||
data, err := decryptData(ctx, kvs.Value, string(kvs.Key))
|
||||
if err != nil {
|
||||
if err == errConfigNotFound {
|
||||
return errNoSuchPolicy
|
||||
@@ -211,27 +211,31 @@ func (ies *IAMEtcdStore) getPolicyDocKV(ctx context.Context, kvs *mvccpb.KeyValu
|
||||
}
|
||||
|
||||
func (ies *IAMEtcdStore) loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
|
||||
// Bound the etcd read without shortening the caller's decryption budget.
|
||||
cctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
|
||||
defer cancel()
|
||||
// Retrieve all keys and values to avoid too many calls to etcd in case of
|
||||
// a large number of policies
|
||||
r, err := ies.client.Get(ctx, iamConfigPoliciesPrefix, etcd.WithPrefix())
|
||||
r, err := ies.client.Get(cctx, iamConfigPoliciesPrefix, etcd.WithPrefix())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Parse all values to construct the policies data model.
|
||||
for _, kvs := range r.Kvs {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = ies.getPolicyDocKV(ctx, kvs, m); err != nil && !errors.Is(err, errNoSuchPolicy) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func (ies *IAMEtcdStore) getUserKV(ctx context.Context, userkv *mvccpb.KeyValue, userType IAMUserType, m map[string]UserIdentity, basePrefix string) error {
|
||||
var u UserIdentity
|
||||
err := ies.decodeIAMConfig(&u, userkv.Value, string(userkv.Key))
|
||||
err := ies.decodeIAMConfig(ctx, &u, userkv.Value, string(userkv.Key))
|
||||
if err != nil {
|
||||
if err == errConfigNotFound {
|
||||
return errNoSuchUser
|
||||
@@ -392,9 +396,9 @@ func (ies *IAMEtcdStore) loadMappedPolicy(ctx context.Context, name string, user
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ies *IAMEtcdStore) getMappedPolicy(kv *mvccpb.KeyValue, m *xsync.MapOf[string, MappedPolicy], basePrefix string) error {
|
||||
func (ies *IAMEtcdStore) getMappedPolicy(ctx context.Context, kv *mvccpb.KeyValue, m *xsync.MapOf[string, MappedPolicy], basePrefix string) error {
|
||||
var p MappedPolicy
|
||||
err := ies.decodeIAMConfig(&p, kv.Value, string(kv.Key))
|
||||
err := ies.decodeIAMConfig(ctx, &p, kv.Value, string(kv.Key))
|
||||
if err != nil {
|
||||
if err == errConfigNotFound {
|
||||
return errNoSuchPolicy
|
||||
@@ -434,7 +438,7 @@ func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUse
|
||||
|
||||
// Parse all policies mapping to create the proper data model
|
||||
for _, kv := range r.Kvs {
|
||||
if err = ies.getMappedPolicy(kv, m, basePrefix); err != nil && !errors.Is(err, errNoSuchPolicy) {
|
||||
if err = ies.getMappedPolicy(ctx, kv, m, basePrefix); err != nil && !errors.Is(err, errNoSuchPolicy) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
+40
-5
@@ -108,7 +108,10 @@ func (iamOS *IAMObjectStore) saveIAMConfig(ctx context.Context, item any, objPat
|
||||
return nil
|
||||
}
|
||||
|
||||
func decryptData(data []byte, objPath string) ([]byte, error) {
|
||||
func decryptData(ctx context.Context, data []byte, objPath string) ([]byte, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if utf8.Valid(data) {
|
||||
return data, nil
|
||||
}
|
||||
@@ -118,13 +121,17 @@ func decryptData(data []byte, objPath string) ([]byte, error) {
|
||||
return pdata, nil
|
||||
}
|
||||
if GlobalKMS != nil {
|
||||
pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
|
||||
pdata, err = config.DecryptBytes(ctx, GlobalKMS, data, kms.Context{
|
||||
minioMetaBucket: path.Join(minioMetaBucket, objPath),
|
||||
})
|
||||
if err == nil {
|
||||
return pdata, nil
|
||||
}
|
||||
pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
|
||||
// Cancellation must not trigger the legacy associated-data fallback.
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pdata, err = config.DecryptBytes(ctx, GlobalKMS, data, kms.Context{
|
||||
minioMetaBucket: objPath,
|
||||
})
|
||||
if err == nil {
|
||||
@@ -134,12 +141,37 @@ func decryptData(data []byte, objPath string) ([]byte, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// iamConfigNameTooLong identifies length errors for valid keys with an ASCII
|
||||
// component that cannot be stored. Normalized LDAP DNs escape non-ASCII bytes.
|
||||
// Byte length alone is insufficient for Unicode filenames on macOS/Windows,
|
||||
// and a total-path limit can make an existing record unreadable after a move.
|
||||
func iamConfigNameTooLong(objPath string, err error) bool {
|
||||
var invalid ObjectNameInvalid
|
||||
if !errors.Is(err, errFileNameTooLong) && !errors.As(err, &invalid) {
|
||||
return false
|
||||
}
|
||||
if checkBucketAndObjectNames(context.Background(), minioMetaBucket, objPath) != nil {
|
||||
return false
|
||||
}
|
||||
for component := range strings.SplitSeq(objPath, SlashSeparator) {
|
||||
if len(component) > 255 && utf8.RuneCountInString(component) == len(component) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (iamOS *IAMObjectStore) loadIAMConfigBytesWithMetadata(ctx context.Context, objPath string) ([]byte, ObjectInfo, error) {
|
||||
data, meta, err := readConfigWithMetadata(ctx, iamOS.objAPI, objPath, ObjectOptions{})
|
||||
if err != nil {
|
||||
if iamConfigNameTooLong(objPath, err) {
|
||||
// A missing record, not a storage failure: lookups of the
|
||||
// identity proceed, while saving its record still fails.
|
||||
err = errConfigNotFound
|
||||
}
|
||||
return nil, meta, err
|
||||
}
|
||||
data, err = decryptData(data, objPath)
|
||||
data, err = decryptData(ctx, data, objPath)
|
||||
if err != nil {
|
||||
return nil, meta, err
|
||||
}
|
||||
@@ -251,12 +283,15 @@ func (iamOS *IAMObjectStore) loadPolicyDocs(ctx context.Context, m map[string]Po
|
||||
return item.Err
|
||||
}
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
policyName := path.Dir(item.Item)
|
||||
if err := iamOS.loadPolicyDoc(ctx, policyName, m); err != nil && !errors.Is(err, errNoSuchPolicy) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func (iamOS *IAMObjectStore) loadSecretKey(ctx context.Context, user string, userType IAMUserType) (string, error) {
|
||||
|
||||
@@ -24,6 +24,8 @@ func TestIAMPeerDeleteNotificationReloadsCommittedState(t *testing.T) {
|
||||
resetTestGlobals()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
// Both caches use object storage, independent of earlier etcd suites.
|
||||
globalEtcdClient = nil
|
||||
obj, disk, err := prepareFS(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/puzpuzpuz/xsync/v3"
|
||||
etcd "go.etcd.io/etcd/client/v3"
|
||||
)
|
||||
|
||||
type iamPolicyBudgetGetObserver struct {
|
||||
etcd.KV
|
||||
elapsed time.Duration
|
||||
budget time.Duration
|
||||
block bool
|
||||
}
|
||||
|
||||
func (k *iamPolicyBudgetGetObserver) Get(ctx context.Context, key string, opts ...etcd.OpOption) (*etcd.GetResponse, error) {
|
||||
start := time.Now()
|
||||
if d, ok := ctx.Deadline(); ok {
|
||||
k.budget = time.Until(d)
|
||||
}
|
||||
defer func() { k.elapsed = time.Since(start) }()
|
||||
if k.block {
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
return k.KV.Get(ctx, key, opts...)
|
||||
}
|
||||
|
||||
func iamPolicyBudgetCall(ctx context.Context, store *IAMStoreSys, operation string) (int, error) {
|
||||
switch operation {
|
||||
case "policies":
|
||||
m, err := store.ListPolicies(ctx, "")
|
||||
return len(m), err
|
||||
case "docs":
|
||||
m, err := store.ListPolicyDocs(ctx, "")
|
||||
return len(m), err
|
||||
default:
|
||||
m := xsync.NewMapOf[string, MappedPolicy]()
|
||||
err := store.IAMStorageAPI.(*IAMEtcdStore).loadMappedPolicies(ctx, regUser, false, m)
|
||||
return m.Size(), err
|
||||
}
|
||||
}
|
||||
|
||||
func prepareIAMPolicyBudgetFixture(t *testing.T, operation string) (*IAMStoreSys, *iamPolicyBudgetGetObserver, *atomic.Int32) {
|
||||
t.Helper()
|
||||
ctx, sys, _ := prepareIAMRevisionFixture(t, "etcd")
|
||||
backend := sys.store.IAMStorageAPI.(*IAMEtcdStore)
|
||||
builtin, err := kms.NewBuiltin("cancellation-key", bytes.Repeat([]byte{43}, 32))
|
||||
mustIAM(t, err)
|
||||
var doc any
|
||||
path := getPolicyDocPath("budget-slow")
|
||||
if operation == "mappings" {
|
||||
path = getMappedPolicyPath("budget-slow", regUser, false)
|
||||
doc = newMappedPolicy("readonly")
|
||||
} else {
|
||||
p, err := sys.store.GetPolicy("readonly")
|
||||
mustIAM(t, err)
|
||||
doc = newPolicyDoc(p)
|
||||
}
|
||||
plain, err := json.Marshal(doc)
|
||||
mustIAM(t, err)
|
||||
encrypted, err := config.EncryptBytes(builtin, plain, kms.Context{minioMetaBucket: minioMetaBucket + "/" + path})
|
||||
mustIAM(t, err)
|
||||
_, err = backend.client.Put(ctx, path, string(encrypted))
|
||||
mustIAM(t, err)
|
||||
calls := &atomic.Int32{}
|
||||
priorKMS := GlobalKMS
|
||||
GlobalKMS = connectCancellationKES(t, cancellationDecryptHandler(t, builtin, func(r *http.Request) {
|
||||
calls.Add(1)
|
||||
select {
|
||||
case <-time.After(450 * time.Millisecond):
|
||||
case <-r.Context().Done():
|
||||
}
|
||||
}))
|
||||
t.Cleanup(func() { GlobalKMS = priorKMS })
|
||||
priorTimeout := defaultContextTimeout
|
||||
defaultContextTimeout = 200 * time.Millisecond
|
||||
t.Cleanup(func() { defaultContextTimeout = priorTimeout })
|
||||
observer := &iamPolicyBudgetGetObserver{KV: backend.client.KV}
|
||||
backend.client.KV = observer
|
||||
t.Cleanup(func() { backend.client.KV = observer.KV })
|
||||
return sys.store, observer, calls
|
||||
}
|
||||
|
||||
// A fast etcd Get followed by slow KES decryption must retain the caller's
|
||||
// budget, even when decryption lasts longer than the etcd read timeout.
|
||||
func TestIAMPolicyEtcdDecodeBudget(t *testing.T) {
|
||||
for _, operation := range []string{"policies", "docs", "mappings"} {
|
||||
t.Run(operation, func(t *testing.T) {
|
||||
store, observer, calls := prepareIAMPolicyBudgetFixture(t, operation)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
n, err := iamPolicyBudgetCall(ctx, store, operation)
|
||||
t.Logf("operation=%s total_ms=%.3f get_ms=%.3f get_budget_ms=%.3f decrypts=%d count=%d error=%v caller_error=%v", operation, float64(time.Since(start).Microseconds())/1000, float64(observer.elapsed.Microseconds())/1000, float64(observer.budget.Microseconds())/1000, calls.Load(), n, err, ctx.Err())
|
||||
if err != nil {
|
||||
t.Fatalf("valid caller budget was shortened after etcd Get completed: %v", err)
|
||||
}
|
||||
if observer.elapsed >= 200*time.Millisecond || observer.budget <= 0 || observer.budget > 210*time.Millisecond {
|
||||
t.Fatalf("etcd read budget invalid: %+v", observer)
|
||||
}
|
||||
if n == 0 || calls.Load() != 1 || ctx.Err() != nil {
|
||||
t.Fatal("successful result was incomplete")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The correction must retain bounded etcd RPCs and caller cancellation. A
|
||||
// failed list cannot publish its private map or advance the cache timestamp.
|
||||
func TestIAMPolicyEtcdBudgetGuards(t *testing.T) {
|
||||
for _, operation := range []string{"policies", "docs", "mappings"} {
|
||||
for _, mode := range []string{"read_timeout", "caller_deadline", "caller_cancel"} {
|
||||
t.Run(fmt.Sprintf("%s/%s", operation, mode), func(t *testing.T) {
|
||||
store, observer, calls := prepareIAMPolicyBudgetFixture(t, operation)
|
||||
cache := store.rlock()
|
||||
before, count := cache.updatedAt, len(cache.iamPolicyDocsMap)
|
||||
store.runlock()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
want := context.DeadlineExceeded
|
||||
if mode == "read_timeout" {
|
||||
observer.block = true
|
||||
}
|
||||
if mode == "caller_deadline" {
|
||||
cancel()
|
||||
ctx, cancel = context.WithTimeout(context.Background(), 80*time.Millisecond)
|
||||
defer cancel()
|
||||
}
|
||||
var timer *time.Timer
|
||||
if mode == "caller_cancel" {
|
||||
timer = time.AfterFunc(80*time.Millisecond, cancel)
|
||||
defer timer.Stop()
|
||||
want = context.Canceled
|
||||
}
|
||||
start := time.Now()
|
||||
_, err := iamPolicyBudgetCall(ctx, store, operation)
|
||||
t.Logf("operation=%s mode=%s elapsed_ms=%.3f decrypts=%d error=%v", operation, mode, float64(time.Since(start).Microseconds())/1000, calls.Load(), err)
|
||||
if !errors.Is(err, want) {
|
||||
t.Fatalf("expected %v, got %v", want, err)
|
||||
}
|
||||
if mode == "read_timeout" && (calls.Load() != 0 || ctx.Err() != nil) {
|
||||
t.Fatal("read timeout did not remain local to the etcd RPC")
|
||||
}
|
||||
cache = store.rlock()
|
||||
unchanged := cache.updatedAt.Equal(before) && len(cache.iamPolicyDocsMap) == count
|
||||
store.runlock()
|
||||
if !unchanged {
|
||||
t.Fatal("failed list published a cache replacement")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
type iamPolicyRefreshContext struct{}
|
||||
|
||||
// Keep IAMStorageAPI as *IAMObjectStore so LoadIAMCache exercises the actual
|
||||
// 32-worker object loader. Capture bytes and release the physical read lock
|
||||
// before pausing; concurrent writes can commit against real object storage.
|
||||
type iamPolicyRefreshReadGate struct {
|
||||
ObjectLayer
|
||||
policyPath, userPath string
|
||||
listEntered, listRelease, refreshEntered, refreshRelease chan struct{}
|
||||
listOnce, refreshOnce sync.Once
|
||||
}
|
||||
|
||||
func (g *iamPolicyRefreshReadGate) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
|
||||
r, err := g.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mode, _ := ctx.Value(iamPolicyRefreshContext{}).(string)
|
||||
var release <-chan struct{}
|
||||
if bucket == minioMetaBucket && mode == "list" && object == g.policyPath {
|
||||
release = g.listRelease
|
||||
}
|
||||
if bucket == minioMetaBucket && mode == "refresh" && object == g.userPath {
|
||||
release = g.refreshRelease
|
||||
}
|
||||
if release == nil {
|
||||
return r, nil
|
||||
}
|
||||
data, err := io.ReadAll(r)
|
||||
info := r.ObjInfo
|
||||
r.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if mode == "list" {
|
||||
g.listOnce.Do(func() { close(g.listEntered) })
|
||||
} else {
|
||||
g.refreshOnce.Do(func() { close(g.refreshEntered) })
|
||||
}
|
||||
select {
|
||||
case <-release:
|
||||
return NewGetObjectReaderFromReader(bytes.NewReader(data), info, opts)
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMPolicyReloadActualObjectRefresh(t *testing.T) {
|
||||
for _, docs := range []bool{false, true} {
|
||||
for _, canceled := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("docs=%t/cancel=%t", docs, canceled), func(t *testing.T) {
|
||||
ctx, sys, obj := prepareIAMRevisionFixture(t)
|
||||
store := sys.store
|
||||
rw, err := store.GetPolicy("readwrite")
|
||||
mustIAM(t, err)
|
||||
ro, err := store.GetPolicy("readonly")
|
||||
mustIAM(t, err)
|
||||
for _, name := range []string{"review-edit", "review-delete"} {
|
||||
_, err = store.SetPolicy(ctx, name, rw)
|
||||
mustIAM(t, err)
|
||||
}
|
||||
const parent = "review-parent"
|
||||
_, err = sys.CreateUser(ctx, parent, madmin.AddOrUpdateUserReq{SecretKey: "synthetic-review-password", Status: madmin.AccountEnabled})
|
||||
mustIAM(t, err)
|
||||
secret, err := getTokenSigningKey()
|
||||
mustIAM(t, err)
|
||||
issue := func(typ string) auth.Credentials {
|
||||
c, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent, tokenRevokeTypeClaim: typ}, secret)
|
||||
mustIAM(t, err)
|
||||
c.ParentUser = parent
|
||||
_, err = store.SetTempUser(ctx, c.AccessKey, c, "")
|
||||
mustIAM(t, err)
|
||||
return c
|
||||
}
|
||||
revoked, retained := issue("selected"), issue("other")
|
||||
gate := &iamPolicyRefreshReadGate{ObjectLayer: obj, policyPath: getPolicyDocPath("review-edit"), userPath: getUserIdentityPath(parent, regUser), listEntered: make(chan struct{}), listRelease: make(chan struct{}), refreshEntered: make(chan struct{}), refreshRelease: make(chan struct{})}
|
||||
store.IAMStorageAPI.(*IAMObjectStore).objAPI = gate
|
||||
freeList := sync.OnceFunc(func() { close(gate.listRelease) })
|
||||
freeRefresh := sync.OnceFunc(func() { close(gate.refreshRelease) })
|
||||
var wg sync.WaitGroup
|
||||
t.Cleanup(func() { freeList(); freeRefresh(); wg.Wait() })
|
||||
start := func(fn func() error) <-chan error {
|
||||
done := make(chan error, 1)
|
||||
wg.Go(func() { done <- fn() })
|
||||
return done
|
||||
}
|
||||
refreshDone := start(func() error {
|
||||
return store.LoadIAMCache(context.WithValue(ctx, iamPolicyRefreshContext{}, "refresh"), false)
|
||||
})
|
||||
awaitIAMPolicyReload(t, gate.refreshEntered)
|
||||
listCtx, cancel := context.WithCancel(context.WithValue(ctx, iamPolicyRefreshContext{}, "list"))
|
||||
defer cancel()
|
||||
listDone := start(func() error { return iamPolicyReloadCall(listCtx, store, docs) })
|
||||
awaitIAMPolicyReload(t, gate.listEntered)
|
||||
read := start(func() error {
|
||||
if _, ok := store.GetUser(parent); !ok {
|
||||
return errors.New("cached user lost")
|
||||
}
|
||||
_, err := store.GetPolicy("review-edit")
|
||||
return err
|
||||
})
|
||||
select {
|
||||
case err := <-read:
|
||||
mustIAM(t, err)
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("actual object reload blocked cached reads")
|
||||
}
|
||||
writes := []<-chan error{
|
||||
start(func() error { _, err := store.SetPolicy(ctx, "review-edit", ro); return err }),
|
||||
start(func() error { return store.DeletePolicy(ctx, "review-delete", false) }),
|
||||
start(func() error { _, err := store.RevokeTokens(ctx, parent, "selected"); return err }),
|
||||
}
|
||||
if canceled {
|
||||
cancel()
|
||||
} else {
|
||||
freeList()
|
||||
}
|
||||
err = <-listDone
|
||||
if canceled {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancel error: %v", err)
|
||||
}
|
||||
} else {
|
||||
mustIAM(t, err)
|
||||
}
|
||||
for _, done := range writes {
|
||||
mustIAM(t, <-done)
|
||||
}
|
||||
freeRefresh()
|
||||
mustIAM(t, <-refreshDone)
|
||||
p, err := store.GetPolicy("review-edit")
|
||||
mustIAM(t, err)
|
||||
if p.IsAllowed(policy.Args{Action: policy.PutObjectAction, BucketName: "bucket", ObjectName: "object"}) || !p.IsAllowed(policy.Args{Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}) {
|
||||
t.Fatal("actual object refresh restored stale policy")
|
||||
}
|
||||
if _, err := store.GetPolicy("review-delete"); !errors.Is(err, errNoSuchPolicy) {
|
||||
t.Fatal("deleted policy restored")
|
||||
}
|
||||
if _, ok := store.GetUser(revoked.AccessKey); ok {
|
||||
t.Fatal("revoked STS token restored")
|
||||
}
|
||||
if _, ok := store.GetUser(retained.AccessKey); !ok {
|
||||
t.Fatal("unrelated token lost")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,373 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
type iamPolicyReloadStore struct {
|
||||
IAMStorageAPI
|
||||
policies func(context.Context, map[string]PolicyDoc) error
|
||||
users func(context.Context, IAMUserType, map[string]UserIdentity) error
|
||||
}
|
||||
|
||||
func (s *iamPolicyReloadStore) loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error {
|
||||
return s.policies(ctx, m)
|
||||
}
|
||||
|
||||
func (s *iamPolicyReloadStore) loadUsers(ctx context.Context, typ IAMUserType, m map[string]UserIdentity) error {
|
||||
if s.users != nil {
|
||||
return s.users(ctx, typ, m)
|
||||
}
|
||||
return s.IAMStorageAPI.loadUsers(ctx, typ, m)
|
||||
}
|
||||
|
||||
func iamPolicyReloadCall(ctx context.Context, store *IAMStoreSys, docs bool) error {
|
||||
if docs {
|
||||
_, err := store.ListPolicyDocs(ctx, "")
|
||||
return err
|
||||
}
|
||||
_, err := store.ListPolicies(ctx, "")
|
||||
return err
|
||||
}
|
||||
|
||||
func awaitIAMPolicyReload(t *testing.T, ch <-chan struct{}) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-ch:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("IAM operation did not reach the controlled barrier")
|
||||
}
|
||||
}
|
||||
|
||||
// Both list entry points must allow cached readers while preserving the writer
|
||||
// ordering used by policy edits, peer notifications and credential revocation.
|
||||
// A refresh that captured the old state must not publish after those writes.
|
||||
func TestIAMPolicyReloadConcurrentChanges(t *testing.T) {
|
||||
for _, backend := range []string{"object", "etcd"} {
|
||||
for _, docs := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/docs=%t", backend, docs), func(t *testing.T) {
|
||||
ctx, sys, obj := prepareIAMRevisionFixture(t, backend)
|
||||
store := sys.store
|
||||
original := store.IAMStorageAPI
|
||||
rw, err := store.GetPolicy("readwrite")
|
||||
mustIAM(t, err)
|
||||
ro, err := store.GetPolicy("readonly")
|
||||
mustIAM(t, err)
|
||||
for _, name := range []string{"reload-edit", "reload-delete", "reload-peer"} {
|
||||
_, err := store.SetPolicy(ctx, name, rw)
|
||||
mustIAM(t, err)
|
||||
}
|
||||
const parent = "reload-parent"
|
||||
_, err = sys.CreateUser(ctx, parent, madmin.AddOrUpdateUserReq{SecretKey: "synthetic-reload-password", Status: madmin.AccountEnabled})
|
||||
mustIAM(t, err)
|
||||
secret, err := getTokenSigningKey()
|
||||
mustIAM(t, err)
|
||||
issue := func(typ string) auth.Credentials {
|
||||
c, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent, tokenRevokeTypeClaim: typ}, secret)
|
||||
mustIAM(t, err)
|
||||
c.ParentUser = parent
|
||||
_, err = store.SetTempUser(ctx, c.AccessKey, c, "")
|
||||
mustIAM(t, err)
|
||||
return c
|
||||
}
|
||||
revoked, retained := issue("selected"), issue("other")
|
||||
peer := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
|
||||
if backend == "etcd" {
|
||||
peer.IAMStorageAPI = newIAMEtcdStore(original.(*IAMEtcdStore).client, MinIOUsersSysType)
|
||||
}
|
||||
mustIAM(t, peer.LoadIAMCache(ctx, true))
|
||||
|
||||
type refreshKey struct{}
|
||||
loading, releaseLoad := make(chan struct{}), make(chan struct{})
|
||||
refreshing, releaseRefresh := make(chan struct{}), make(chan struct{})
|
||||
unblockLoad := sync.OnceFunc(func() { close(releaseLoad) })
|
||||
unblockRefresh := sync.OnceFunc(func() { close(releaseRefresh) })
|
||||
var wg sync.WaitGroup
|
||||
t.Cleanup(func() { unblockLoad(); unblockRefresh(); wg.Wait() })
|
||||
store.IAMStorageAPI = &iamPolicyReloadStore{
|
||||
IAMStorageAPI: original,
|
||||
policies: func(ctx context.Context, m map[string]PolicyDoc) error {
|
||||
if err := original.loadPolicyDocs(ctx, m); err != nil {
|
||||
return err
|
||||
}
|
||||
if ctx.Value(refreshKey{}) == nil {
|
||||
close(loading)
|
||||
select {
|
||||
case <-releaseLoad:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
users: func(ctx context.Context, typ IAMUserType, m map[string]UserIdentity) error {
|
||||
if err := original.loadUsers(ctx, typ, m); err != nil {
|
||||
return err
|
||||
}
|
||||
if typ == stsUser && ctx.Value(refreshKey{}) != nil {
|
||||
close(refreshing)
|
||||
<-releaseRefresh
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
start := func(fn func() error) <-chan error {
|
||||
out := make(chan error, 1)
|
||||
wg.Go(func() { out <- fn() })
|
||||
return out
|
||||
}
|
||||
refreshDone := start(func() error { return store.LoadIAMCache(context.WithValue(ctx, refreshKey{}, true), false) })
|
||||
awaitIAMPolicyReload(t, refreshing)
|
||||
listDone := start(func() error { return iamPolicyReloadCall(ctx, store, docs) })
|
||||
awaitIAMPolicyReload(t, loading)
|
||||
reader := start(func() error {
|
||||
if _, ok := store.GetUser(parent); !ok {
|
||||
return errors.New("committed user disappeared during policy loading")
|
||||
}
|
||||
_, err := store.GetPolicy("reload-edit")
|
||||
return err
|
||||
})
|
||||
select {
|
||||
case err := <-reader:
|
||||
mustIAM(t, err)
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("policy loading blocked cached authentication")
|
||||
}
|
||||
|
||||
// The peer commits independently; its notification waits alongside
|
||||
// local changes while the older list snapshot remains in flight.
|
||||
_, err = peer.SetPolicy(ctx, "reload-peer", ro)
|
||||
mustIAM(t, err)
|
||||
writersStarted := make(chan struct{}, 4)
|
||||
startWrite := func(fn func() error) <-chan error {
|
||||
return start(func() error { writersStarted <- struct{}{}; return fn() })
|
||||
}
|
||||
writes := []<-chan error{
|
||||
startWrite(func() error { _, err := store.SetPolicy(ctx, "reload-edit", ro); return err }),
|
||||
startWrite(func() error { return store.DeletePolicy(ctx, "reload-delete", false) }),
|
||||
startWrite(func() error { _, err := store.RevokeTokens(ctx, parent, "selected"); return err }),
|
||||
startWrite(func() error { return store.PolicyNotificationHandler(ctx, "reload-peer") }),
|
||||
}
|
||||
for range writes {
|
||||
<-writersStarted
|
||||
}
|
||||
select {
|
||||
case err := <-writes[0]:
|
||||
t.Fatalf("policy write overtook the pending reload: %v", err)
|
||||
case <-time.After(25 * time.Millisecond):
|
||||
}
|
||||
for _, done := range writes[1:] {
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("IAM mutation overtook the pending reload: %v", err)
|
||||
default:
|
||||
}
|
||||
}
|
||||
unblockLoad()
|
||||
mustIAM(t, <-listDone)
|
||||
for _, done := range writes {
|
||||
mustIAM(t, <-done)
|
||||
}
|
||||
unblockRefresh()
|
||||
mustIAM(t, <-refreshDone)
|
||||
for _, name := range []string{"reload-edit", "reload-peer"} {
|
||||
p, err := store.GetPolicy(name)
|
||||
mustIAM(t, err)
|
||||
if p.IsAllowed(policy.Args{Action: policy.PutObjectAction, BucketName: "bucket", ObjectName: "object"}) {
|
||||
t.Fatalf("stale list/refresh restored the old grant: %s", name)
|
||||
}
|
||||
if !p.IsAllowed(policy.Args{Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}) {
|
||||
t.Fatalf("reload lost the committed read grant: %s", name)
|
||||
}
|
||||
}
|
||||
if _, err := store.GetPolicy("reload-delete"); !errors.Is(err, errNoSuchPolicy) {
|
||||
t.Fatalf("deleted policy returned: %v", err)
|
||||
}
|
||||
if _, ok := store.GetUser(revoked.AccessKey); ok {
|
||||
t.Fatal("stale list/refresh restored a revoked token")
|
||||
}
|
||||
for _, key := range []string{retained.AccessKey, parent} {
|
||||
if _, ok := store.GetUser(key); !ok {
|
||||
t.Fatal("reload removed an unrelated credential")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMPolicyReloadFailurePreservesCache(t *testing.T) {
|
||||
for _, docs := range []bool{false, true} {
|
||||
for _, failure := range []string{"storage", "cancel_partial", "caller_deadline_partial"} {
|
||||
t.Run(fmt.Sprintf("docs=%t/%s", docs, failure), func(t *testing.T) {
|
||||
ctx, sys, _ := prepareIAMRevisionFixture(t)
|
||||
var cancel context.CancelFunc
|
||||
if failure == "caller_deadline_partial" {
|
||||
ctx, cancel = context.WithTimeout(ctx, 100*time.Millisecond)
|
||||
} else {
|
||||
ctx, cancel = context.WithCancel(ctx)
|
||||
}
|
||||
defer cancel()
|
||||
store := sys.store
|
||||
cache := store.rlock()
|
||||
updated := cache.updatedAt
|
||||
store.runlock()
|
||||
injected := errors.New("policy read failed")
|
||||
entered := make(chan struct{})
|
||||
original := store.IAMStorageAPI
|
||||
store.IAMStorageAPI = &iamPolicyReloadStore{IAMStorageAPI: original, policies: func(ctx context.Context, m map[string]PolicyDoc) error {
|
||||
m["partial-result"] = PolicyDoc{}
|
||||
close(entered)
|
||||
if failure == "storage" {
|
||||
return injected
|
||||
}
|
||||
<-ctx.Done()
|
||||
// A canceled enumeration can close without yielding an error.
|
||||
return nil
|
||||
}}
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- iamPolicyReloadCall(ctx, store, docs) }()
|
||||
awaitIAMPolicyReload(t, entered)
|
||||
want := injected
|
||||
switch failure {
|
||||
case "cancel_partial":
|
||||
cancel()
|
||||
want = context.Canceled
|
||||
case "caller_deadline_partial":
|
||||
want = context.DeadlineExceeded
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, want) {
|
||||
t.Fatalf("got %v, want %v", err, want)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
cancel()
|
||||
<-done
|
||||
t.Fatal("reload did not honor caller cancellation")
|
||||
}
|
||||
cache = store.rlock()
|
||||
_, partial := cache.iamPolicyDocsMap["partial-result"]
|
||||
unchanged := cache.updatedAt.Equal(updated)
|
||||
store.runlock()
|
||||
if partial || !unchanged {
|
||||
t.Fatal("failed/canceled reload published a partial policy map")
|
||||
}
|
||||
store.IAMStorageAPI = original
|
||||
// Cancellation must release both locks so the next operation works.
|
||||
mustIAM(t, iamPolicyReloadCall(context.Background(), store, docs))
|
||||
if _, err := store.GetPolicy("readwrite"); err != nil {
|
||||
t.Fatal("reload lost default policies")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type iamPolicyRelockObserver struct {
|
||||
IAMStorageAPI
|
||||
calls atomic.Int32
|
||||
reacquiring chan struct{}
|
||||
}
|
||||
|
||||
func (s *iamPolicyRelockObserver) lock() *iamCache {
|
||||
if s.calls.Add(1) == 2 {
|
||||
close(s.reacquiring)
|
||||
}
|
||||
return s.IAMStorageAPI.lock()
|
||||
}
|
||||
|
||||
func TestIAMPolicyReloadCallerBudgetPreserved(t *testing.T) {
|
||||
for _, docs := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("docs=%t", docs), func(t *testing.T) {
|
||||
_, sys, _ := prepareIAMRevisionFixture(t)
|
||||
original := sys.store.IAMStorageAPI
|
||||
prior := defaultContextTimeout
|
||||
defaultContextTimeout = 50 * time.Millisecond
|
||||
t.Cleanup(func() { defaultContextTimeout = prior })
|
||||
sys.store.IAMStorageAPI = &iamPolicyReloadStore{IAMStorageAPI: original, policies: func(ctx context.Context, m map[string]PolicyDoc) error {
|
||||
select {
|
||||
case <-time.After(120 * time.Millisecond):
|
||||
return original.loadPolicyDocs(ctx, m)
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
err := iamPolicyReloadCall(ctx, sys.store, docs)
|
||||
if err != nil {
|
||||
t.Fatalf("reload cut off a still-valid caller budget: err=%v callerErr=%v", err, ctx.Err())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMPolicyReloadCancelWhileReacquiringCache(t *testing.T) {
|
||||
for _, docs := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("docs=%t", docs), func(t *testing.T) {
|
||||
_, sys, _ := prepareIAMRevisionFixture(t)
|
||||
store := sys.store
|
||||
original := store.IAMStorageAPI
|
||||
observer := &iamPolicyRelockObserver{IAMStorageAPI: original, reacquiring: make(chan struct{})}
|
||||
entered, release := make(chan struct{}), make(chan struct{})
|
||||
store.IAMStorageAPI = &iamPolicyReloadStore{IAMStorageAPI: observer, policies: func(ctx context.Context, m map[string]PolicyDoc) error {
|
||||
m["partial-reload"] = PolicyDoc{}
|
||||
close(entered)
|
||||
<-release
|
||||
return nil
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- iamPolicyReloadCall(ctx, store, docs) }()
|
||||
awaitIAMPolicyReload(t, entered)
|
||||
cache := store.rlock()
|
||||
before := cache.updatedAt
|
||||
count := len(cache.iamPolicyDocsMap)
|
||||
close(release)
|
||||
awaitIAMPolicyReload(t, observer.reacquiring)
|
||||
// This reader forces the reload to wait before its commit step.
|
||||
// The request is canceled after I/O completed, before publishing.
|
||||
cancel()
|
||||
store.runlock()
|
||||
err := <-done
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancel before publish lost: %v", err)
|
||||
}
|
||||
cache = store.rlock()
|
||||
_, partial := cache.iamPolicyDocsMap["partial-reload"]
|
||||
unchanged := cache.updatedAt.Equal(before) && len(cache.iamPolicyDocsMap) == count
|
||||
store.runlock()
|
||||
if partial || !unchanged {
|
||||
t.Fatal("canceled complete load still published")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/grid"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
func newIAMPolicyResetSibling(ctx context.Context, t *testing.T, source *IAMSys, obj ObjectLayer) *IAMSys {
|
||||
t.Helper()
|
||||
store := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
|
||||
if original, ok := source.store.IAMStorageAPI.(*IAMEtcdStore); ok {
|
||||
store.IAMStorageAPI = newIAMEtcdStore(original.client, MinIOUsersSysType)
|
||||
}
|
||||
mustIAM(t, store.LoadIAMCache(ctx, true))
|
||||
return &IAMSys{store: store, usersSysType: MinIOUsersSysType}
|
||||
}
|
||||
|
||||
func assertIAMPolicyResetInfo(t *testing.T, sys *IAMSys, name string, want *madmin.PolicyInfo) {
|
||||
t.Helper()
|
||||
got, err := sys.InfoPolicy(name)
|
||||
mustIAM(t, err)
|
||||
var gotPolicy, wantPolicy policy.Policy
|
||||
mustIAM(t, json.Unmarshal(got.Policy, &gotPolicy))
|
||||
mustIAM(t, json.Unmarshal(want.Policy, &wantPolicy))
|
||||
// Actions and resources are sets, whose JSON array order is not stable.
|
||||
// Compare their exact contents while retaining all native metadata checks.
|
||||
gotMetadata, wantMetadata := *got, *want
|
||||
gotMetadata.Policy, wantMetadata.Policy = nil, nil
|
||||
if !gotPolicy.Equals(wantPolicy) || !reflect.DeepEqual(gotMetadata, wantMetadata) {
|
||||
t.Fatalf("policy definition or native metadata changed: got %+v, want %+v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The first InfoPolicy call after Reset must work without any inventory read,
|
||||
// refresh, or retry. The durable tombstone belongs to the removed override;
|
||||
// it must still reject stale overrides without suppressing the native policy.
|
||||
func TestIAMBuiltinPolicyResetRestoresNativePolicy(t *testing.T) {
|
||||
for _, backend := range []string{"object", "etcd"} {
|
||||
t.Run(backend, func(t *testing.T) {
|
||||
ctx, sys, obj := prepareIAMRevisionFixture(t, backend)
|
||||
for _, builtin := range policy.DefaultPolicies {
|
||||
t.Run(builtin.Name, func(t *testing.T) {
|
||||
name := builtin.Name
|
||||
native, err := sys.InfoPolicy(name)
|
||||
mustIAM(t, err)
|
||||
override := builtin.Definition
|
||||
override.ID = "reset-override"
|
||||
written, err := sys.SetPolicy(ctx, name, override)
|
||||
mustIAM(t, err)
|
||||
mustIAM(t, sys.DeletePolicy(ctx, name, true))
|
||||
assertIAMPolicyResetInfo(t, sys, name, native)
|
||||
|
||||
path := getPolicyDocPath(name)
|
||||
deleted, err := loadIAMRevision(ctx, sys.store, path)
|
||||
mustIAM(t, err)
|
||||
if !deleted.Deleted || !deleted.timestamp().After(written) {
|
||||
t.Fatalf("Reset lost the override deletion revision: %+v", deleted)
|
||||
}
|
||||
for _, at := range []time.Time{written, deleted.timestamp()} {
|
||||
_, err = sys.SetPolicy(withIAMReplicationTime(ctx, at), name, override)
|
||||
if !errors.Is(err, errIAMStaleUpdate) {
|
||||
t.Fatalf("Reset accepted a stale override at %s: %v", at, err)
|
||||
}
|
||||
assertIAMPolicyResetInfo(t, sys, name, native)
|
||||
}
|
||||
mustIAM(t, sys.store.LoadIAMCache(ctx, false))
|
||||
assertIAMPolicyResetInfo(t, sys, name, native)
|
||||
fresh := newIAMPolicyResetSibling(ctx, t, sys, obj)
|
||||
assertIAMPolicyResetInfo(t, fresh, name, native)
|
||||
for _, docs := range []bool{false, true} {
|
||||
mustIAM(t, iamPolicyReloadCall(ctx, fresh.store, docs))
|
||||
assertIAMPolicyResetInfo(t, fresh, name, native)
|
||||
}
|
||||
after, err := loadIAMRevision(ctx, fresh.store, path)
|
||||
mustIAM(t, err)
|
||||
if !reflect.DeepEqual(after, deleted) || !fresh.store.revisionIndex().get(path).Deleted {
|
||||
t.Fatal("default restoration changed the durable deletion or revision index")
|
||||
}
|
||||
// A deliberate new edit remains possible after Reset.
|
||||
_, err = sys.SetPolicy(ctx, name, override)
|
||||
mustIAM(t, err)
|
||||
current, err := sys.store.GetPolicyDoc(name)
|
||||
mustIAM(t, err)
|
||||
if current.Policy.ID != override.ID || current.UpdateDate.IsZero() {
|
||||
t.Fatal("Reset prevented a subsequent explicit override")
|
||||
}
|
||||
mustIAM(t, sys.DeletePolicy(ctx, name, true))
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Peer and watch notifications carry a name, not a version. Reload committed
|
||||
// storage so delayed deletion events cannot erase a recreated policy. Built-in
|
||||
// defaults retain mappings; a genuinely deleted custom policy does not.
|
||||
func TestIAMPolicyResetNotificationLoadsCommittedState(t *testing.T) {
|
||||
for _, backend := range []string{"object", "etcd"} {
|
||||
for _, delivery := range []string{"peer", "watch"} {
|
||||
for _, recreate := range []bool{false, true} {
|
||||
for _, name := range []string{"writeonly", "reset-custom"} {
|
||||
t.Run(fmt.Sprintf("%s/%s/recreate=%t/%s", backend, delivery, recreate, name), func(t *testing.T) {
|
||||
ctx, source, obj := prepareIAMRevisionFixture(t, backend)
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = obj
|
||||
globalObjLayerMutex.Unlock()
|
||||
builtin := name == "writeonly"
|
||||
var want *madmin.PolicyInfo
|
||||
if builtin {
|
||||
var err error
|
||||
want, err = source.InfoPolicy(name)
|
||||
mustIAM(t, err)
|
||||
}
|
||||
p, err := source.store.GetPolicy("readwrite")
|
||||
mustIAM(t, err)
|
||||
p.ID = "reset-original-override"
|
||||
_, err = source.SetPolicy(ctx, name, p)
|
||||
mustIAM(t, err)
|
||||
_, err = source.CreateUser(ctx, "reset-user", madmin.AddOrUpdateUserReq{SecretKey: "synthetic-reset-password", Status: madmin.AccountEnabled})
|
||||
mustIAM(t, err)
|
||||
_, err = source.AddUsersToGroup(ctx, "reset-group", []string{"reset-user"})
|
||||
mustIAM(t, err)
|
||||
for _, group := range []bool{false, true} {
|
||||
entity := "reset-user"
|
||||
if group {
|
||||
entity = "reset-group"
|
||||
}
|
||||
_, err = source.PolicyDBSet(ctx, entity, "readonly,"+name, regUser, group)
|
||||
mustIAM(t, err)
|
||||
}
|
||||
if err := source.DeletePolicy(ctx, name, true); !errors.Is(err, errPolicyInUse) {
|
||||
t.Fatalf("Reset bypassed policy-in-use protection: %v", err)
|
||||
}
|
||||
sibling := newIAMPolicyResetSibling(ctx, t, source, obj)
|
||||
mappings := make(map[bool]MappedPolicy)
|
||||
mappings[false], _ = sibling.store.GetMappedPolicy("reset-user", false)
|
||||
mappings[true], _ = sibling.store.GetMappedPolicy("reset-group", true)
|
||||
// A remote site may delete before this node has received the
|
||||
// corresponding mapping changes. Local deletion stays guarded.
|
||||
mustIAM(t, source.DeletePolicy(withIAMReplicationTime(ctx, UTCNow()), name, true))
|
||||
if recreate {
|
||||
p.ID = "reset-recreated-override"
|
||||
_, err = source.SetPolicy(ctx, name, p)
|
||||
mustIAM(t, err)
|
||||
want, err = source.InfoPolicy(name)
|
||||
mustIAM(t, err)
|
||||
}
|
||||
before, err := loadIAMRevision(ctx, source.store, getPolicyDocPath(name))
|
||||
mustIAM(t, err)
|
||||
globalIAMSys = sibling
|
||||
defer func() { globalIAMSys = source }()
|
||||
for range 2 {
|
||||
if delivery == "peer" {
|
||||
_, remoteErr := (&peerRESTServer{}).DeletePolicyHandler(grid.NewMSSWith(map[string]string{peerRESTPolicy: name}))
|
||||
if remoteErr != nil {
|
||||
t.Fatal(remoteErr)
|
||||
}
|
||||
} else {
|
||||
mustIAM(t, sibling.loadWatchedEvent(ctx, iamWatchEvent{keyPath: getPolicyDocPath(name)}))
|
||||
}
|
||||
if builtin || recreate {
|
||||
assertIAMPolicyResetInfo(t, sibling, name, want)
|
||||
} else if _, err := sibling.InfoPolicy(name); !errors.Is(err, errNoSuchPolicy) {
|
||||
t.Fatalf("deleted custom policy returned: %v", err)
|
||||
}
|
||||
for group, original := range mappings {
|
||||
entity := "reset-user"
|
||||
if group {
|
||||
entity = "reset-group"
|
||||
}
|
||||
got, _ := sibling.store.GetMappedPolicy(entity, group)
|
||||
if builtin || recreate {
|
||||
if !reflect.DeepEqual(got, original) {
|
||||
t.Fatalf("notification changed surviving mappings: %+v", got)
|
||||
}
|
||||
} else if got.Policies != "readonly" {
|
||||
t.Fatalf("custom deletion did not retain only the unrelated mapping: %+v", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
after, err := loadIAMRevision(ctx, source.store, getPolicyDocPath(name))
|
||||
mustIAM(t, err)
|
||||
if !reflect.DeepEqual(after, before) {
|
||||
t.Fatal("notification changed durable policy storage")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMPolicyResetNotificationSurvivesOlderRefresh(t *testing.T) {
|
||||
ctx, source, obj := prepareIAMRevisionFixture(t)
|
||||
p, err := source.store.GetPolicy("readwrite")
|
||||
mustIAM(t, err)
|
||||
p.ID = "old-override"
|
||||
_, err = source.SetPolicy(ctx, "writeonly", p)
|
||||
mustIAM(t, err)
|
||||
sibling := newIAMPolicyResetSibling(ctx, t, source, obj)
|
||||
original := sibling.store.IAMStorageAPI
|
||||
loading, release := make(chan struct{}), make(chan struct{})
|
||||
unblock := sync.OnceFunc(func() { close(release) })
|
||||
var wg sync.WaitGroup
|
||||
t.Cleanup(func() { unblock(); wg.Wait() })
|
||||
sibling.store.IAMStorageAPI = &iamPolicyReloadStore{
|
||||
IAMStorageAPI: original,
|
||||
policies: original.loadPolicyDocs,
|
||||
users: func(ctx context.Context, typ IAMUserType, m map[string]UserIdentity) error {
|
||||
if typ == regUser {
|
||||
close(loading)
|
||||
<-release
|
||||
}
|
||||
return original.loadUsers(ctx, typ, m)
|
||||
},
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
wg.Go(func() { done <- sibling.store.LoadIAMCache(ctx, false) })
|
||||
awaitIAMPolicyReload(t, loading)
|
||||
mustIAM(t, source.DeletePolicy(ctx, "writeonly", true))
|
||||
p.ID = "newer-override"
|
||||
_, err = source.SetPolicy(ctx, "writeonly", p)
|
||||
mustIAM(t, err)
|
||||
want, err := source.InfoPolicy("writeonly")
|
||||
mustIAM(t, err)
|
||||
// The delayed deletion notification must observe the newer edit, and
|
||||
// prevent a refresh that captured the old override from replacing it.
|
||||
mustIAM(t, sibling.DeletePolicy(ctx, "writeonly", false))
|
||||
unblock()
|
||||
mustIAM(t, <-done)
|
||||
assertIAMPolicyResetInfo(t, sibling, "writeonly", want)
|
||||
}
|
||||
|
||||
type iamPolicyResetFailureStore struct {
|
||||
IAMStorageAPI
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *iamPolicyResetFailureStore) saveIAMConfig(context.Context, any, string, ...options) error {
|
||||
return s.err
|
||||
}
|
||||
|
||||
func (s *iamPolicyResetFailureStore) loadPolicyDoc(context.Context, string, map[string]PolicyDoc) error {
|
||||
return s.err
|
||||
}
|
||||
|
||||
func TestIAMBuiltinPolicyResetStorageFailurePreservesOverride(t *testing.T) {
|
||||
for _, backend := range []string{"object", "etcd"} {
|
||||
t.Run(backend, func(t *testing.T) {
|
||||
ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
|
||||
p, err := sys.store.GetPolicy("readonly")
|
||||
mustIAM(t, err)
|
||||
_, err = sys.SetPolicy(ctx, "writeonly", p)
|
||||
mustIAM(t, err)
|
||||
before, err := sys.InfoPolicy("writeonly")
|
||||
mustIAM(t, err)
|
||||
stored, err := loadIAMRevision(ctx, sys.store, getPolicyDocPath("writeonly"))
|
||||
mustIAM(t, err)
|
||||
original := sys.store.IAMStorageAPI
|
||||
injected := errors.New("injected policy storage failure")
|
||||
sys.store.IAMStorageAPI = &iamPolicyResetFailureStore{IAMStorageAPI: original, err: injected}
|
||||
for _, notify := range []bool{true, false} {
|
||||
if err := sys.DeletePolicy(ctx, "writeonly", notify); !errors.Is(err, injected) {
|
||||
t.Fatalf("storage failure became a successful Reset, notify=%t: %v", notify, err)
|
||||
}
|
||||
assertIAMPolicyResetInfo(t, sys, "writeonly", before)
|
||||
}
|
||||
if err := sys.loadWatchedEvent(ctx, iamWatchEvent{keyPath: getPolicyDocPath("writeonly")}); !errors.Is(err, injected) {
|
||||
t.Fatalf("watch storage failure became a default policy: %v", err)
|
||||
}
|
||||
assertIAMPolicyResetInfo(t, sys, "writeonly", before)
|
||||
after, err := loadIAMRevision(ctx, original, getPolicyDocPath("writeonly"))
|
||||
mustIAM(t, err)
|
||||
if !reflect.DeepEqual(after, stored) {
|
||||
t.Fatal("failed Reset changed the durable override")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,7 @@ const (
|
||||
iamRevisionPeerPath = "/v3/site-replication/peer/iam-revisions"
|
||||
iamUserBoundaryType = "silo-user-revocation"
|
||||
iamGroupBoundaryType = "silo-group-revocation"
|
||||
iamSTSDeleteType = "silo-sts-revocation"
|
||||
maxIAMRevisionBatch = 128
|
||||
)
|
||||
|
||||
@@ -49,6 +50,7 @@ type iamGroupBoundary struct {
|
||||
// before applying any change that would lose revocation or member metadata.
|
||||
type iamReplicationItem struct {
|
||||
madmin.SRIAMItem
|
||||
STSExpiresAt time.Time `json:"stsExpiresAt,omitempty"`
|
||||
GroupGrants map[string]time.Time `json:"groupGrants,omitempty"`
|
||||
GroupSnapshot bool `json:"groupSnapshot,omitempty"`
|
||||
UserRevocation *iamUserBoundary `json:"userRevocation,omitempty"`
|
||||
@@ -155,6 +157,13 @@ type (
|
||||
|
||||
func (c *SiteReplicationSys) replicationItem(ctx context.Context, item madmin.SRIAMItem) (iamReplicationItem, error) {
|
||||
out := iamReplicationItem{SRIAMItem: item}
|
||||
if item.Type == iamSTSDeleteType {
|
||||
r, err := loadIAMRevision(ctx, globalIAMSys.store, getUserIdentityPath(item.Name, stsUser))
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.STSExpiresAt = r.ExpiresAt
|
||||
}
|
||||
if item.Type == madmin.SRIAMItemSvcAcc && item.SvcAccChange != nil {
|
||||
var key string
|
||||
if item.SvcAccChange.Create != nil {
|
||||
@@ -242,6 +251,15 @@ func applyIAMReplicationItem(ctx context.Context, item iamReplicationItem) error
|
||||
ctx = context.WithValue(ctx, iamRecordBoundaryKey{}, item.RevokedBefore)
|
||||
}
|
||||
switch item.Type {
|
||||
case iamSTSDeleteType:
|
||||
if item.Name == "" || strings.ContainsAny(item.Name, "/\\") || item.UpdatedAt.IsZero() {
|
||||
return errSRInvalidRequest(errInvalidArgument)
|
||||
}
|
||||
if err := globalIAMSys.store.revokeTempUser(withIAMReplicationTime(ctx, item.UpdatedAt), item.Name, item.STSExpiresAt); err != nil {
|
||||
return iamReplicationError(err)
|
||||
}
|
||||
globalIAMSys.notifyForUser(ctx, item.Name, true)
|
||||
return nil
|
||||
case iamUserBoundaryType:
|
||||
if item.UserRevocation == nil || item.UserRevocation.User == "" || item.UserRevocation.Before.IsZero() {
|
||||
return errSRInvalidRequest(errInvalidArgument)
|
||||
|
||||
+32
-8
@@ -348,6 +348,13 @@ func saveIAMRevision(ctx context.Context, store IAMStorageAPI, path string, item
|
||||
// STS access keys identify immutable tokens, not reusable user names.
|
||||
return errIAMStaleUpdate
|
||||
}
|
||||
if strings.HasPrefix(path, iamConfigSTSPrefix) && deleted {
|
||||
// STS keys are immutable. A delayed revocation must also defeat a
|
||||
// replayed creation that was recorded with a newer event timestamp.
|
||||
if origin, replicated := iamReplicationTime(ctx); replicated && previous.timestamp().After(origin) {
|
||||
ctx = withIAMReplicationTime(ctx, previous.timestamp())
|
||||
}
|
||||
}
|
||||
if origin, replicated := iamReplicationTime(ctx); replicated {
|
||||
*at = origin
|
||||
if previous.timestamp().After(origin) || (previous.Deleted && !deleted && !origin.After(previous.timestamp())) {
|
||||
@@ -394,11 +401,18 @@ func saveIAMRevision(ctx context.Context, store IAMStorageAPI, path string, item
|
||||
// Retain only the parent name for root-account exclusion during heal.
|
||||
v.Credentials = auth.Credentials{ParentUser: previous.Credentials.ParentUser}
|
||||
v.RevokedBefore = *at
|
||||
if strings.HasPrefix(path, iamConfigSTSPrefix) && !previous.Credentials.Expiration.IsZero() && !previous.Credentials.Expiration.Equal(timeSentinel) {
|
||||
// The signed STS token cannot authorize beyond this time, even
|
||||
// if an offline site replays it with a newer event timestamp.
|
||||
v.ExpiresAt = previous.Credentials.Expiration.Add(globalMaxSkewTime)
|
||||
opts = []options{{ttl: max(1, int64(time.Until(v.ExpiresAt).Seconds())+1)}}
|
||||
if strings.HasPrefix(path, iamConfigSTSPrefix) {
|
||||
// Carry expiry when a revocation arrives before the token itself.
|
||||
// It is safe to forget this key only after its signed lifetime.
|
||||
if expiration := previous.Credentials.Expiration; !expiration.IsZero() && !expiration.Equal(timeSentinel) && expiration.Add(globalMaxSkewTime).After(v.ExpiresAt) {
|
||||
v.ExpiresAt = expiration.Add(globalMaxSkewTime)
|
||||
}
|
||||
if previous.ExpiresAt.After(v.ExpiresAt) {
|
||||
v.ExpiresAt = previous.ExpiresAt
|
||||
}
|
||||
if !v.ExpiresAt.IsZero() {
|
||||
opts = []options{{ttl: max(1, int64(time.Until(v.ExpiresAt).Seconds())+1)}}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if v.Credentials.SessionToken != "" && v.Credentials.Claims == nil {
|
||||
@@ -489,6 +503,12 @@ func iamDeletionItem(path string, r iamRevision) (item madmin.SRIAMItem, ok bool
|
||||
name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigUsersPrefix), "/"+iamIdentityFile)
|
||||
item.Type = madmin.SRIAMItemIAMUser
|
||||
item.IAMUser = &madmin.SRIAMUser{AccessKey: name, IsDeleteReq: true}
|
||||
case strings.HasPrefix(path, iamConfigSTSPrefix):
|
||||
if !r.ExpiresAt.IsZero() && UTCNow().After(r.ExpiresAt) {
|
||||
return item, false
|
||||
}
|
||||
item.Type = iamSTSDeleteType
|
||||
item.Name = strings.TrimSuffix(strings.TrimPrefix(path, iamConfigSTSPrefix), "/"+iamIdentityFile)
|
||||
case strings.HasPrefix(path, iamConfigServiceAccountsPrefix):
|
||||
name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigServiceAccountsPrefix), "/"+iamIdentityFile)
|
||||
if name == siteReplicatorSvcAcc || r.Credentials.ParentUser == globalActiveCred.AccessKey {
|
||||
@@ -518,8 +538,7 @@ func iamDeletionItem(path string, r iamRevision) (item madmin.SRIAMItem, ok bool
|
||||
item.Type = madmin.SRIAMItemPolicyMapping
|
||||
item.PolicyMapping = &madmin.SRPolicyMapping{UserOrGroup: strings.TrimSuffix(name, ".json"), UserType: int(typ), IsGroup: prefix == "groups"}
|
||||
default:
|
||||
// Expired STS credentials are not replayed. Parent revocations and
|
||||
// their retained timestamp reject delayed copies of derived tokens.
|
||||
// Other config paths do not represent replicated IAM deletions.
|
||||
return item, false
|
||||
}
|
||||
return item, true
|
||||
@@ -527,6 +546,8 @@ func iamDeletionItem(path string, r iamRevision) (item madmin.SRIAMItem, ok bool
|
||||
|
||||
func iamDeletionPath(item madmin.SRIAMItem) string {
|
||||
switch item.Type {
|
||||
case iamSTSDeleteType:
|
||||
return getUserIdentityPath(item.Name, stsUser)
|
||||
case madmin.SRIAMItemIAMUser:
|
||||
if item.IAMUser != nil && item.IAMUser.IsDeleteReq {
|
||||
return getUserIdentityPath(item.IAMUser.AccessKey, regUser)
|
||||
@@ -582,6 +603,9 @@ func (c *SiteReplicationSys) healIAMDeletions(ctx context.Context) (err error) {
|
||||
continue
|
||||
}
|
||||
out := iamReplicationItem{SRIAMItem: item}
|
||||
if item.Type == iamSTSDeleteType {
|
||||
out.STSExpiresAt = r.ExpiresAt
|
||||
}
|
||||
if item.Type == madmin.SRIAMItemIAMUser && !r.Deleted {
|
||||
out.Type, out.IAMUser = iamUserBoundaryType, nil
|
||||
out.UserRevocation = &iamUserBoundary{User: item.IAMUser.AccessKey, Before: r.RevokedBefore}
|
||||
@@ -593,7 +617,7 @@ func (c *SiteReplicationSys) healIAMDeletions(ctx context.Context) (err error) {
|
||||
byType[item.Type] = append(byType[item.Type], out)
|
||||
}
|
||||
var items []iamReplicationItem
|
||||
for _, typ := range []string{madmin.SRIAMItemPolicyMapping, madmin.SRIAMItemIAMUser, madmin.SRIAMItemSvcAcc, madmin.SRIAMItemGroupInfo, madmin.SRIAMItemPolicy} {
|
||||
for _, typ := range []string{madmin.SRIAMItemPolicyMapping, madmin.SRIAMItemIAMUser, madmin.SRIAMItemSvcAcc, iamSTSDeleteType, madmin.SRIAMItemGroupInfo, madmin.SRIAMItemPolicy} {
|
||||
items = append(items, byType[typ]...)
|
||||
}
|
||||
if len(items) == 0 {
|
||||
|
||||
@@ -7,14 +7,19 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/grid"
|
||||
"github.com/pgsty/silo-pkg/v3/ldap"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
)
|
||||
|
||||
@@ -93,6 +98,211 @@ func TestIAMRevocationTombstoneReplayIsIdempotent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// An LDAP parent named by a normalized DN longer than NAME_MAX has no record
|
||||
// on object storage. Its lookups must find nothing instead of failing login.
|
||||
func TestLoadIAMRevisionNameTooLong(t *testing.T) {
|
||||
for _, backend := range []string{"object", "etcd"} {
|
||||
t.Run(backend, func(t *testing.T) {
|
||||
ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
|
||||
store := sys.store.IAMStorageAPI
|
||||
// 160 bytes of UTF-8 become 384 characters.
|
||||
parent, err := ldap.NormalizeDN("uid=Сотрудник,ou=Отдел разработки,ou=Дирекция информационных технологий,ou=people,ou=swengg,dc=min,dc=io")
|
||||
mustIAM(t, err)
|
||||
if len(parent) <= 255 {
|
||||
t.Fatalf("normalized DN has only %d characters", len(parent))
|
||||
}
|
||||
parentPath := getUserIdentityPath(parent, regUser)
|
||||
for _, path := range []string{parentPath, getUserIdentityPath("missing-parent", regUser)} {
|
||||
r, err := loadIAMRevision(ctx, store, path)
|
||||
if err != nil || r.Deleted || !r.timestamp().IsZero() || !r.RevokedBefore.IsZero() {
|
||||
t.Fatalf("absent revision %.40s...: %+v, %v", path, r, err)
|
||||
}
|
||||
}
|
||||
if backend == "object" {
|
||||
var invalid ObjectNameInvalid
|
||||
if _, err := loadIAMRevision(ctx, store, iamConfigUsersPrefix+"a//b/"+iamIdentityFile); !errors.As(err, &invalid) {
|
||||
t.Fatalf("invalid name not caused by length was masked: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
claims := map[string]any{}
|
||||
mustIAM(t, setIAMParentRevocationClaim(ctx, store, parent, claims))
|
||||
if _, ok := claims[iamParentRevocationClaim]; ok {
|
||||
t.Fatalf("unexpected revocation claim: %v", claims)
|
||||
}
|
||||
if _, err := sys.PolicyDBGet(parent); err != nil {
|
||||
t.Fatalf("policy lookup failed: %v", err)
|
||||
}
|
||||
// The call that failed in AssumeRoleWithLDAPIdentity.
|
||||
secret, err := getTokenSigningKey()
|
||||
mustIAM(t, err)
|
||||
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
|
||||
mustIAM(t, err)
|
||||
cred.ParentUser = parent
|
||||
_, err = sys.SetTempUser(ctx, cred.AccessKey, cred, "")
|
||||
mustIAM(t, err)
|
||||
if _, ok := sys.store.GetUser(cred.AccessKey); !ok {
|
||||
t.Fatal("temporary credential not cached")
|
||||
}
|
||||
|
||||
// Object storage still cannot record a revocation for this parent;
|
||||
// the failure must be reported. Etcd has no such limit.
|
||||
err = saveIAMRevision(ctx, store, parentPath, &iamUserRevocation{})
|
||||
if backend == "object" {
|
||||
if !iamConfigNameTooLong(parentPath, err) {
|
||||
t.Fatalf("expected name-too-long error, got %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
mustIAM(t, err)
|
||||
if err := checkIAMParentRevision(ctx, store, cred); !errors.Is(err, errIAMStaleUpdate) {
|
||||
t.Fatalf("revoked parent accepted: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// checkPathLength does not count backslashes outside Windows, so this escaped
|
||||
// DN passes it. Once config/iam/users exists, as after creating any built-in
|
||||
// user, the filesystem rejects the element with ENAMETOOLONG instead.
|
||||
func TestLoadIAMRevisionFilesystemNameTooLong(t *testing.T) {
|
||||
ctx, sys, obj := prepareIAMRevisionFixture(t)
|
||||
parent, err := ldap.NormalizeDN("cn=Сидоров П.П.,ou=Отдел ИТ-поддержки,ou=Пользователи,ou=people,ou=swengg,dc=min,dc=io")
|
||||
mustIAM(t, err)
|
||||
parentPath := getUserIdentityPath(parent, regUser)
|
||||
if len(parent) <= 255 || checkPathLength(parentPath) != nil {
|
||||
t.Fatalf("normalized DN must exceed NAME_MAX yet pass checkPathLength: %d characters, %v", len(parent), checkPathLength(parentPath))
|
||||
}
|
||||
_, err = sys.CreateUser(ctx, "existing-user", madmin.AddOrUpdateUserReq{SecretKey: "synthetic-user-password", Status: madmin.AccountEnabled})
|
||||
mustIAM(t, err)
|
||||
mustIAM(t, saveIAMRevision(ctx, sys.store.IAMStorageAPI, getMappedPolicyPath("existing-parent", stsUser, false), &MappedPolicy{Version: 1, Policies: "readwrite"}))
|
||||
store := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, LDAPUsersSysType)}
|
||||
mustIAM(t, store.LoadIAMCache(ctx, false))
|
||||
r, err := loadIAMRevision(ctx, store.IAMStorageAPI, parentPath)
|
||||
if err != nil || r.Deleted || !r.timestamp().IsZero() {
|
||||
t.Fatalf("absent revision: %+v, %v", r, err)
|
||||
}
|
||||
// LDAP mode reads policydb/sts-users/<DN>.json.
|
||||
if _, err := store.PolicyDBGet(parent); err != nil {
|
||||
t.Fatalf("policy lookup failed: %v", err)
|
||||
}
|
||||
secret, err := getTokenSigningKey()
|
||||
mustIAM(t, err)
|
||||
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
|
||||
mustIAM(t, err)
|
||||
cred.ParentUser = parent
|
||||
if _, err := store.SetTempUser(ctx, cred.AccessKey, cred, ""); err != nil {
|
||||
t.Fatalf("temporary credential rejected: %v", err)
|
||||
}
|
||||
err = saveIAMRevision(ctx, store.IAMStorageAPI, parentPath, &iamUserRevocation{})
|
||||
if !iamConfigNameTooLong(parentPath, err) {
|
||||
t.Fatalf("expected name-too-long error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMConfigNameTooLong(t *testing.T) {
|
||||
long := iamConfigUsersPrefix + strings.Repeat("x", 256) + "/" + iamIdentityFile
|
||||
short := iamConfigUsersPrefix + strings.Repeat("x", 200) + "/" + iamIdentityFile
|
||||
invalid := iamConfigUsersPrefix + "a//b/" + iamIdentityFile
|
||||
escaped := iamConfigUsersPrefix + strings.Repeat("\\d0", 86) + "/" + iamIdentityFile
|
||||
limit := iamConfigUsersPrefix + strings.Repeat("x", 255) + "/" + iamIdentityFile
|
||||
unicode := iamConfigUsersPrefix + strings.Repeat("界", 100) + "/" + iamIdentityFile
|
||||
deep := iamConfigUsersPrefix + strings.Repeat("nested/", 160) + iamIdentityFile
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
path string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{"NAME_MAX", long, ObjectNameInvalid{Bucket: minioMetaBucket, Object: long}, true},
|
||||
{"escaped DN", escaped, ObjectNameInvalid{Bucket: minioMetaBucket, Object: escaped}, runtime.GOOS != "windows"},
|
||||
{"short component", short, ObjectNameInvalid{Bucket: minioMetaBucket, Object: short}, false},
|
||||
{"exact component limit", limit, ObjectNameInvalid{Bucket: minioMetaBucket, Object: limit}, false},
|
||||
{"unicode byte length", unicode, ObjectNameInvalid{Bucket: minioMetaBucket, Object: unicode}, false},
|
||||
{"total path length", deep, ObjectNameInvalid{Bucket: minioMetaBucket, Object: deep}, false},
|
||||
{"short drive path", short, errFileNameTooLong, false},
|
||||
{"policy suffix at limit", getMappedPolicyPath(strings.Repeat("x", 250), stsUser, false), errFileNameTooLong, false},
|
||||
{"policy suffix exceeds limit", getMappedPolicyPath(strings.Repeat("x", 251), stsUser, false), errFileNameTooLong, true},
|
||||
{"drive", long, errFileNameTooLong, true},
|
||||
{"wrapped drive", long, fmt.Errorf("read: %w", errFileNameTooLong), true},
|
||||
{"wrapped object", long, fmt.Errorf("read: %w", ObjectNameInvalid{Bucket: minioMetaBucket, Object: long}), true},
|
||||
{"no error", long, nil, false},
|
||||
{"not found", long, errConfigNotFound, false},
|
||||
{"offline drive", long, errDiskNotFound, false},
|
||||
{"read quorum", long, InsufficientReadQuorum{}, false},
|
||||
{"access denied", long, PrefixAccessDenied{}, false},
|
||||
{"deadline", long, context.DeadlineExceeded, false},
|
||||
{"invalid name", invalid, ObjectNameInvalid{Bucket: minioMetaBucket, Object: invalid}, false},
|
||||
{"invalid long name", invalid + strings.Repeat("x", 256), errFileNameTooLong, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := iamConfigNameTooLong(tc.path, tc.err); got != tc.want {
|
||||
t.Errorf("got %t, want %t", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Moving a drive can make an existing record exceed the total path limit.
|
||||
// A length error must not turn that parent's revocation into a missing record.
|
||||
func TestLoadIAMRevisionDeepPath(t *testing.T) {
|
||||
if runtime.GOOS != "darwin" {
|
||||
t.Skip("exercises the Darwin total-path limit")
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
parent string
|
||||
}{
|
||||
{"ASCII", strings.Repeat("p", 200)},
|
||||
{"Unicode", strings.Repeat("界", 100)},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
resetTestGlobals()
|
||||
defer resetTestGlobals()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
base := t.TempDir()
|
||||
oldRoot := filepath.Join(base, "drive")
|
||||
mustIAM(t, os.Mkdir(oldRoot, 0o700))
|
||||
obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, oldRoot))
|
||||
mustIAM(t, err)
|
||||
defer func() { obj.Shutdown(ctx) }()
|
||||
store := newIAMObjectStore(obj, LDAPUsersSysType)
|
||||
cred := auth.Credentials{ParentUser: tc.parent}
|
||||
parentPath := getUserIdentityPath(tc.parent, regUser)
|
||||
mustIAM(t, store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Deleted: true, RevokedBefore: UTCNow()}, parentPath))
|
||||
if err := checkIAMParentRevision(ctx, store, cred); !errors.Is(err, errIAMStaleUpdate) {
|
||||
t.Fatalf("original parent revocation ineffective: %v", err)
|
||||
}
|
||||
mustIAM(t, obj.Shutdown(ctx))
|
||||
deep := base
|
||||
for len(deep) < 850 {
|
||||
deep = filepath.Join(deep, strings.Repeat("d", 80))
|
||||
}
|
||||
mustIAM(t, os.MkdirAll(deep, 0o700))
|
||||
newRoot := filepath.Join(deep, "drive")
|
||||
mustIAM(t, checkPathLength(newRoot))
|
||||
mustIAM(t, os.Rename(oldRoot, newRoot))
|
||||
// The record still exists; a relative read avoids the total-path limit.
|
||||
root, err := os.OpenRoot(newRoot)
|
||||
mustIAM(t, err)
|
||||
defer root.Close()
|
||||
data, err := root.ReadFile(filepath.Join(minioMetaBucket, parentPath, xlStorageFormatFile))
|
||||
mustIAM(t, err)
|
||||
if len(data) == 0 {
|
||||
t.Fatal("missing physical revocation record")
|
||||
}
|
||||
obj, _, err = initObjectLayer(ctx, mustGetPoolEndpoints(0, newRoot))
|
||||
mustIAM(t, err)
|
||||
store = newIAMObjectStore(obj, LDAPUsersSysType)
|
||||
var invalid ObjectNameInvalid
|
||||
if err := checkIAMParentRevision(ctx, store, cred); !errors.As(err, &invalid) {
|
||||
t.Fatalf("expected path error for an existing parent revocation, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Run with both object storage and etcd through TestIAMRevocation*Lifecycle.
|
||||
// The object-store case uses the real peer RPC and deletion handler, so a
|
||||
// spurious notification actually destroys the parent instead of only counting it.
|
||||
|
||||
+153
-76
@@ -647,6 +647,20 @@ func setDefaultCannedPolicies(policies map[string]PolicyDoc) {
|
||||
}
|
||||
}
|
||||
|
||||
// Remove a stored policy from cache. A built-in policy's deletion removes its
|
||||
// override, so its native definition remains available with native metadata.
|
||||
// The durable tombstone is retained separately to reject stale overrides.
|
||||
func removePolicyFromCache(name string, policies map[string]PolicyDoc) (restoredDefault bool) {
|
||||
delete(policies, name)
|
||||
for _, p := range policy.DefaultPolicies {
|
||||
if p.Name == name {
|
||||
policies[name] = defaultPolicyDoc(p.Definition)
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// LoadIAMCache reads all IAM items and populates a new iamCache object and
|
||||
// replaces the in-memory cache object.
|
||||
func (store *IAMStoreSys) LoadIAMCache(ctx context.Context, firstTime bool) error {
|
||||
@@ -711,13 +725,18 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context, firstTime bool) erro
|
||||
if err := store.loadUsers(ctx, svcUser, newCache.iamUsersMap); err != nil {
|
||||
return err
|
||||
}
|
||||
// Rebuild STS revisions as well: a missed etcd watch event must not
|
||||
// leave a cached token valid or hide its tombstone after a restart.
|
||||
if err := store.loadUsers(ctx, stsUser, newCache.iamSTSAccountsMap); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
newCache.buildUserGroupMemberships()
|
||||
}
|
||||
|
||||
for name := range newCache.iamPolicyDocsMap {
|
||||
if store.revisionIndex().get(getPolicyDocPath(name)).Deleted {
|
||||
delete(newCache.iamPolicyDocsMap, name)
|
||||
removePolicyFromCache(name, newCache.iamPolicyDocsMap)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -754,6 +773,14 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context, firstTime bool) erro
|
||||
cache.updatedAt = time.Now()
|
||||
}
|
||||
|
||||
// Revision loading is independent of cache replacement. Even if ongoing
|
||||
// IAM writes defer the full swap, a missed STS notification must converge.
|
||||
for key := range cache.iamSTSAccountsMap {
|
||||
if store.revisionIndex().get(getUserIdentityPath(key, stsUser)).Deleted {
|
||||
delete(cache.iamSTSAccountsMap, key)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1363,9 +1390,9 @@ func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string,
|
||||
}
|
||||
|
||||
// PolicyNotificationHandler - loads given policy from storage. If not present,
|
||||
// deletes from cache. This notification only reads from storage, and updates
|
||||
// cache. When the notification is for a policy deletion, it updates the
|
||||
// user-policy and group-policy maps as well.
|
||||
// removes its override from cache. This notification only reads from storage,
|
||||
// and updates cache. A deleted built-in override restores its native policy;
|
||||
// a deleted custom policy is removed from user-policy and group-policy maps.
|
||||
func (store *IAMStoreSys) PolicyNotificationHandler(ctx context.Context, policy string) error {
|
||||
if policy == "" {
|
||||
return errInvalidArgument
|
||||
@@ -1376,8 +1403,10 @@ func (store *IAMStoreSys) PolicyNotificationHandler(ctx context.Context, policy
|
||||
|
||||
err := store.loadPolicyDoc(ctx, policy, cache.iamPolicyDocsMap)
|
||||
if errors.Is(err, errNoSuchPolicy) {
|
||||
// policy was deleted, update cache.
|
||||
delete(cache.iamPolicyDocsMap, policy)
|
||||
if removePolicyFromCache(policy, cache.iamPolicyDocsMap) {
|
||||
cache.updatedAt = time.Now()
|
||||
return nil
|
||||
}
|
||||
|
||||
// update user policy map
|
||||
cache.iamUserPolicyMap.Range(func(u string, mp MappedPolicy) bool {
|
||||
@@ -1413,74 +1442,77 @@ func (store *IAMStoreSys) PolicyNotificationHandler(ctx context.Context, policy
|
||||
cache.updatedAt = time.Now()
|
||||
return nil
|
||||
}
|
||||
if err == nil {
|
||||
cache.updatedAt = time.Now()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// DeletePolicy - deletes policy from storage and cache. When this called in
|
||||
// response to a notification (i.e. isFromNotification = true), it skips the
|
||||
// validation of policy usage and the attempt to delete in the backend as well
|
||||
// (as this is already done by the notifying node).
|
||||
// DeletePolicy - deletes a stored policy and restores the native definition
|
||||
// when it was a built-in override. Notifications reload committed storage, so
|
||||
// a delayed deletion event cannot remove a subsequently recreated policy.
|
||||
func (store *IAMStoreSys) DeletePolicy(ctx context.Context, policy string, isFromNotification bool) error {
|
||||
if policy == "" {
|
||||
return errInvalidArgument
|
||||
}
|
||||
if isFromNotification {
|
||||
return store.PolicyNotificationHandler(ctx, policy)
|
||||
}
|
||||
|
||||
cache := store.lock()
|
||||
defer store.unlock()
|
||||
|
||||
if !isFromNotification {
|
||||
if err := store.checkIAMRevision(ctx, getPolicyDocPath(policy), true); err != nil {
|
||||
return err
|
||||
if err := store.checkIAMRevision(ctx, getPolicyDocPath(policy), true); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, replicated := iamReplicationTime(ctx); !replicated {
|
||||
if _, exists := cache.iamPolicyDocsMap[policy]; !exists {
|
||||
// Preserve idempotent local deletion without creating a durable
|
||||
// revocation for a misspelled/nonexistent policy name.
|
||||
return nil
|
||||
}
|
||||
if _, replicated := iamReplicationTime(ctx); !replicated {
|
||||
if _, exists := cache.iamPolicyDocsMap[policy]; !exists {
|
||||
// Preserve idempotent local deletion without creating a durable
|
||||
// revocation for a misspelled/nonexistent policy name.
|
||||
return nil
|
||||
// Check if policy is mapped to any existing user or group. If so, we do not
|
||||
// allow deletion of the policy. If the policy is mapped to an STS account,
|
||||
// we do allow deletion.
|
||||
users := []string{}
|
||||
groups := []string{}
|
||||
cache.iamUserPolicyMap.Range(func(u string, mp MappedPolicy) bool {
|
||||
pset := mp.policySet()
|
||||
if store.getUsersSysType() == MinIOUsersSysType {
|
||||
if _, ok := cache.iamUsersMap[u]; !ok {
|
||||
// This case can happen when a temporary account is
|
||||
// deleted or expired - remove it from userPolicyMap.
|
||||
cache.iamUserPolicyMap.Delete(u)
|
||||
return true
|
||||
}
|
||||
}
|
||||
// Check if policy is mapped to any existing user or group. If so, we do not
|
||||
// allow deletion of the policy. If the policy is mapped to an STS account,
|
||||
// we do allow deletion.
|
||||
users := []string{}
|
||||
groups := []string{}
|
||||
cache.iamUserPolicyMap.Range(func(u string, mp MappedPolicy) bool {
|
||||
pset := mp.policySet()
|
||||
if store.getUsersSysType() == MinIOUsersSysType {
|
||||
if _, ok := cache.iamUsersMap[u]; !ok {
|
||||
// This case can happen when a temporary account is
|
||||
// deleted or expired - remove it from userPolicyMap.
|
||||
cache.iamUserPolicyMap.Delete(u)
|
||||
return true
|
||||
}
|
||||
}
|
||||
if pset.Contains(policy) {
|
||||
users = append(users, u)
|
||||
}
|
||||
return true
|
||||
})
|
||||
cache.iamGroupPolicyMap.Range(func(g string, mp MappedPolicy) bool {
|
||||
pset := mp.policySet()
|
||||
if pset.Contains(policy) {
|
||||
groups = append(groups, g)
|
||||
}
|
||||
return true
|
||||
})
|
||||
if len(users) != 0 || len(groups) != 0 {
|
||||
return errPolicyInUse
|
||||
if pset.Contains(policy) {
|
||||
users = append(users, u)
|
||||
}
|
||||
}
|
||||
|
||||
err := store.deletePolicyDoc(ctx, policy)
|
||||
if errors.Is(err, errNoSuchPolicy) {
|
||||
// Ignore error if policy is already deleted.
|
||||
err = nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
return true
|
||||
})
|
||||
cache.iamGroupPolicyMap.Range(func(g string, mp MappedPolicy) bool {
|
||||
pset := mp.policySet()
|
||||
if pset.Contains(policy) {
|
||||
groups = append(groups, g)
|
||||
}
|
||||
return true
|
||||
})
|
||||
if len(users) != 0 || len(groups) != 0 {
|
||||
return errPolicyInUse
|
||||
}
|
||||
}
|
||||
|
||||
delete(cache.iamPolicyDocsMap, policy)
|
||||
err := store.deletePolicyDoc(ctx, policy)
|
||||
if errors.Is(err, errNoSuchPolicy) {
|
||||
// Ignore error if policy is already deleted.
|
||||
err = nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
removePolicyFromCache(policy, cache.iamPolicyDocsMap)
|
||||
cache.updatedAt = time.Now()
|
||||
|
||||
return nil
|
||||
@@ -1568,10 +1600,22 @@ func (store *IAMStoreSys) ListPolicies(ctx context.Context, bucketName string) (
|
||||
defer store.unlock()
|
||||
|
||||
m := map[string]PolicyDoc{}
|
||||
err := store.loadPolicyDocs(ctx, m)
|
||||
// Keep cache writers serialized, but let authentication use the committed
|
||||
// cache while the replacement is loaded into this private map.
|
||||
err := func() error {
|
||||
store.IAMStorageAPI.unlock()
|
||||
defer store.IAMStorageAPI.lock()
|
||||
if err := store.loadPolicyDocs(ctx, m); err != nil {
|
||||
return err
|
||||
}
|
||||
return ctx.Err()
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Sets default canned policies
|
||||
setDefaultCannedPolicies(m)
|
||||
@@ -1596,10 +1640,20 @@ func (store *IAMStoreSys) ListPolicyDocs(ctx context.Context, bucketName string)
|
||||
defer store.unlock()
|
||||
|
||||
m := map[string]PolicyDoc{}
|
||||
err := store.loadPolicyDocs(ctx, m)
|
||||
err := func() error {
|
||||
store.IAMStorageAPI.unlock()
|
||||
defer store.IAMStorageAPI.lock()
|
||||
if err := store.loadPolicyDocs(ctx, m); err != nil {
|
||||
return err
|
||||
}
|
||||
return ctx.Err()
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Sets default canned policies
|
||||
setDefaultCannedPolicies(m)
|
||||
@@ -2184,45 +2238,68 @@ func (store *IAMStoreSys) SetTempUser(ctx context.Context, accessKey string, cre
|
||||
|
||||
// RevokeTokens - revokes all temporary credentials, or those with matching type,
|
||||
// associated with the parent user.
|
||||
func (store *IAMStoreSys) RevokeTokens(ctx context.Context, parentUser string, tokenRevokeType string) error {
|
||||
func (store *IAMStoreSys) RevokeTokens(ctx context.Context, parentUser string, tokenRevokeType string) ([]string, error) {
|
||||
if parentUser == "" {
|
||||
return errInvalidArgument
|
||||
return nil, errInvalidArgument
|
||||
}
|
||||
|
||||
before := UTCNow()
|
||||
cache := store.lock()
|
||||
defer store.unlock()
|
||||
|
||||
secret, err := getTokenSigningKey()
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var revoked bool
|
||||
for _, ui := range cache.iamSTSAccountsMap {
|
||||
if ui.Credentials.ParentUser != parentUser {
|
||||
// Another node may have issued a token whose notification was lost.
|
||||
// Enumerate durable identities, retaining cached entries for retrying a
|
||||
// partially propagated revocation. Do not hold up cached authentication.
|
||||
accounts := maps.Clone(cache.iamSTSAccountsMap)
|
||||
if err := store.withIAMStorage(ctx, func(ctx context.Context) error {
|
||||
return store.loadUsers(ctx, stsUser, accounts)
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var revoked []string
|
||||
var failures []error
|
||||
for key, ui := range accounts {
|
||||
if ui.Credentials.ParentUser != parentUser || ui.UpdatedAt.After(before) || ui.Credentials.IsExpired() {
|
||||
continue
|
||||
}
|
||||
if tokenRevokeType != "" {
|
||||
claims, err := getClaimsFromTokenWithSecret(ui.Credentials.SessionToken, secret)
|
||||
if err != nil {
|
||||
continue // skip if token is invalid
|
||||
continue
|
||||
}
|
||||
// skip if token type is given and does not match
|
||||
if v, _ := claims.Lookup(tokenRevokeTypeClaim); v != tokenRevokeType {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := store.deleteUserIdentity(ctx, ui.Credentials.AccessKey, stsUser); err != nil {
|
||||
return err
|
||||
if err := store.deleteUserIdentity(ctx, key, stsUser); err != nil {
|
||||
failures = append(failures, err)
|
||||
continue
|
||||
}
|
||||
delete(cache.iamSTSAccountsMap, ui.Credentials.AccessKey)
|
||||
revoked = true
|
||||
delete(cache.iamSTSAccountsMap, key)
|
||||
revoked = append(revoked, key)
|
||||
}
|
||||
|
||||
if revoked {
|
||||
if len(revoked) != 0 {
|
||||
cache.updatedAt = time.Now()
|
||||
}
|
||||
return revoked, errors.Join(failures...)
|
||||
}
|
||||
|
||||
// revokeTempUser applies an immutable token's replicated tombstone without
|
||||
// touching the parent, its policies, or other (including newer) tokens.
|
||||
func (store *IAMStoreSys) revokeTempUser(ctx context.Context, key string, expiresAt time.Time) error {
|
||||
cache := store.lock()
|
||||
defer store.unlock()
|
||||
u := UserIdentity{Version: 1, Deleted: true, ExpiresAt: expiresAt}
|
||||
if err := store.saveUserIdentity(ctx, key, stsUser, &u); err != nil {
|
||||
return err
|
||||
}
|
||||
delete(cache.iamSTSAccountsMap, key)
|
||||
cache.updatedAt = time.Now()
|
||||
if store.group != nil {
|
||||
store.group.Forget(key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/grid"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
)
|
||||
|
||||
func TestIAMSTSRevocationDurableScope(t *testing.T) {
|
||||
for _, backend := range []string{"object", "etcd"} {
|
||||
for _, notify := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/notify=%t", backend, notify), func(t *testing.T) {
|
||||
ctx, sys, obj := prepareIAMRevisionFixture(t, backend)
|
||||
for _, parent := range []string{"sts-parent", "other-parent"} {
|
||||
_, err := sys.CreateUser(ctx, parent, madmin.AddOrUpdateUserReq{SecretKey: "synthetic-parent-password", Status: madmin.AccountEnabled})
|
||||
mustIAM(t, err)
|
||||
}
|
||||
sibling := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
|
||||
if backend == "etcd" {
|
||||
sibling.IAMStorageAPI = newIAMEtcdStore(sys.store.IAMStorageAPI.(*IAMEtcdStore).client, MinIOUsersSysType)
|
||||
}
|
||||
mustIAM(t, sibling.LoadIAMCache(ctx, false))
|
||||
secret, err := getTokenSigningKey()
|
||||
mustIAM(t, err)
|
||||
issue := func(store *IAMStoreSys, parent, typ string) auth.Credentials {
|
||||
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent, tokenRevokeTypeClaim: typ}, secret)
|
||||
mustIAM(t, err)
|
||||
cred.ParentUser = parent
|
||||
_, err = store.SetTempUser(ctx, cred.AccessKey, cred, "")
|
||||
mustIAM(t, err)
|
||||
return cred
|
||||
}
|
||||
first := issue(sys.store, "sts-parent", "selected")
|
||||
uncached := issue(sibling, "sts-parent", "selected")
|
||||
otherType := issue(sys.store, "sts-parent", "other-type")
|
||||
otherUser := issue(sys.store, "other-parent", "selected")
|
||||
if _, ok := sys.store.GetUser(uncached.AccessKey); ok {
|
||||
t.Fatal("fixture cached the remote token")
|
||||
}
|
||||
for _, c := range []auth.Credentials{first, otherType, otherUser} {
|
||||
mustIAM(t, sibling.UserNotificationHandler(ctx, c.AccessKey, stsUser))
|
||||
}
|
||||
prior := globalNotificationSys
|
||||
globalNotificationSys = &NotificationSys{}
|
||||
t.Cleanup(func() { globalNotificationSys = prior })
|
||||
if notify {
|
||||
tg, err := grid.SetupTestGrid(2)
|
||||
mustIAM(t, err)
|
||||
t.Cleanup(tg.Cleanup)
|
||||
mustIAM(t, loadUserRPC.Register(tg.Managers[1], func(r *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
|
||||
if err := sibling.UserNotificationHandler(ctx, r.Get(peerRESTUser), stsUser); err != nil {
|
||||
return grid.NoPayload{}, grid.NewRemoteErr(err)
|
||||
}
|
||||
return grid.NoPayload{}, nil
|
||||
}))
|
||||
host, err := xnet.ParseHost(strings.TrimPrefix(tg.Hosts[1], "http://"))
|
||||
mustIAM(t, err)
|
||||
globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{{host: host, gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[1]) }}}}
|
||||
}
|
||||
mustIAM(t, sys.RevokeTokens(ctx, "sts-parent", "selected"))
|
||||
for _, c := range []auth.Credentials{first, uncached} {
|
||||
r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(c.AccessKey, stsUser))
|
||||
mustIAM(t, err)
|
||||
if !r.Deleted {
|
||||
t.Errorf("durable token was omitted: %s", c.AccessKey)
|
||||
}
|
||||
if _, ok := iamDeletionItem(getUserIdentityPath(c.AccessKey, stsUser), r); !ok {
|
||||
t.Error("STS revocation cannot be replayed")
|
||||
}
|
||||
if notify && backend == "object" {
|
||||
if _, ok := sibling.GetUser(c.AccessKey); ok {
|
||||
t.Error("notified sibling accepted revoked token")
|
||||
}
|
||||
}
|
||||
}
|
||||
fresh := issue(sibling, "sts-parent", "selected")
|
||||
// A missed notification must converge through the normal cache refresh.
|
||||
mustIAM(t, sibling.LoadIAMCache(ctx, false))
|
||||
for _, c := range []auth.Credentials{first, uncached} {
|
||||
if _, ok := sibling.GetUser(c.AccessKey); ok {
|
||||
t.Error("refresh retained revoked token")
|
||||
}
|
||||
}
|
||||
for _, c := range []auth.Credentials{otherType, otherUser, fresh} {
|
||||
mustIAM(t, sibling.UserNotificationHandler(ctx, c.AccessKey, stsUser))
|
||||
if _, ok := sibling.GetUser(c.AccessKey); !ok {
|
||||
t.Error("revocation removed an out-of-scope or new token")
|
||||
}
|
||||
}
|
||||
if _, ok := sibling.GetUser("sts-parent"); !ok {
|
||||
t.Error("revocation removed parent")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Deletion is keyed by the immutable STS access key, so delayed delivery cannot
|
||||
// affect a newly issued token and a delayed creation cannot undo revocation.
|
||||
func TestIAMSTSRevocationReorderedReplay(t *testing.T) {
|
||||
for _, backend := range []string{"object", "etcd"} {
|
||||
for _, missing := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/missing=%t", backend, missing), func(t *testing.T) {
|
||||
ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
|
||||
const parent = "sts-replay-parent"
|
||||
_, err := sys.CreateUser(ctx, parent, madmin.AddOrUpdateUserReq{SecretKey: "synthetic-parent-password", Status: madmin.AccountEnabled})
|
||||
mustIAM(t, err)
|
||||
secret, err := getTokenSigningKey()
|
||||
mustIAM(t, err)
|
||||
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
|
||||
mustIAM(t, err)
|
||||
cred.ParentUser = parent
|
||||
created := UTCNow().Add(-time.Minute)
|
||||
peer := madmin.SRSTSCredential{AccessKey: cred.AccessKey, SecretKey: cred.SecretKey, SessionToken: cred.SessionToken, ParentUser: parent}
|
||||
if !missing {
|
||||
mustIAM(t, globalSiteReplicationSys.PeerSTSAccHandler(ctx, &peer, created))
|
||||
}
|
||||
fresh, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
|
||||
mustIAM(t, err)
|
||||
fresh.ParentUser = parent
|
||||
_, err = sys.SetTempUser(ctx, fresh.AccessKey, fresh, "")
|
||||
mustIAM(t, err)
|
||||
// Older than the live record on this peer: deletion must still win.
|
||||
item := iamReplicationItem{SRIAMItem: madmin.SRIAMItem{Type: iamSTSDeleteType, Name: cred.AccessKey, UpdatedAt: created.Add(-time.Second)}, STSExpiresAt: cred.Expiration.Add(globalMaxSkewTime)}
|
||||
for range 2 {
|
||||
mustIAM(t, applyIAMReplicationItem(ctx, item))
|
||||
mustIAM(t, globalSiteReplicationSys.PeerSTSAccHandler(ctx, &peer, UTCNow()))
|
||||
mustIAM(t, sys.store.LoadIAMCache(ctx, false))
|
||||
if _, ok := sys.GetUser(ctx, cred.AccessKey); ok {
|
||||
t.Fatal("delayed STS creation undid revocation")
|
||||
}
|
||||
if _, ok := sys.GetUser(ctx, fresh.AccessKey); !ok {
|
||||
t.Fatal("delayed revocation removed new token")
|
||||
}
|
||||
}
|
||||
r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(cred.AccessKey, stsUser))
|
||||
mustIAM(t, err)
|
||||
if !r.Deleted || !r.ExpiresAt.Equal(item.STSExpiresAt) || r.Credentials.SessionToken != "" || r.Credentials.SecretKey != "" {
|
||||
t.Fatal("revocation lost expiry or retained secret material")
|
||||
}
|
||||
for _, status := range []madmin.AccountStatus{madmin.AccountDisabled, madmin.AccountEnabled} {
|
||||
_, err = sys.SetUserStatus(ctx, parent, status)
|
||||
mustIAM(t, err)
|
||||
}
|
||||
if _, ok := sys.GetUser(ctx, cred.AccessKey); ok {
|
||||
t.Fatal("parent re-enable revived revoked token")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Pause the durable scan while another node persists a new credential. The
|
||||
// revocation's original cutoff must survive the scan and leave that token live.
|
||||
type iamSTSScanGateStore struct {
|
||||
IAMStorageAPI
|
||||
entered chan struct{}
|
||||
resume chan struct{}
|
||||
}
|
||||
|
||||
func (s *iamSTSScanGateStore) loadUsers(ctx context.Context, typ IAMUserType, users map[string]UserIdentity) error {
|
||||
close(s.entered)
|
||||
select {
|
||||
case <-s.resume:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
return s.IAMStorageAPI.loadUsers(ctx, typ, users)
|
||||
}
|
||||
|
||||
func TestIAMSTSRevocationConcurrentIssuance(t *testing.T) {
|
||||
ctx, sys, obj := prepareIAMRevisionFixture(t)
|
||||
_, err := sys.CreateUser(ctx, "sts-race-parent", madmin.AddOrUpdateUserReq{SecretKey: "synthetic-parent-password", Status: madmin.AccountEnabled})
|
||||
mustIAM(t, err)
|
||||
sibling := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
|
||||
secret, err := getTokenSigningKey()
|
||||
mustIAM(t, err)
|
||||
issue := func() auth.Credentials {
|
||||
c, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: "sts-race-parent"}, secret)
|
||||
mustIAM(t, err)
|
||||
c.ParentUser = "sts-race-parent"
|
||||
_, err = sibling.SetTempUser(ctx, c.AccessKey, c, "")
|
||||
mustIAM(t, err)
|
||||
return c
|
||||
}
|
||||
old := issue()
|
||||
gate := &iamSTSScanGateStore{IAMStorageAPI: sys.store.IAMStorageAPI, entered: make(chan struct{}), resume: make(chan struct{})}
|
||||
sys.store.IAMStorageAPI = gate
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- sys.RevokeTokens(ctx, "sts-race-parent", "") }()
|
||||
<-gate.entered
|
||||
fresh := issue()
|
||||
close(gate.resume)
|
||||
mustIAM(t, <-done)
|
||||
mustIAM(t, sibling.UserNotificationHandler(ctx, old.AccessKey, stsUser))
|
||||
if _, ok := sibling.GetUser(old.AccessKey); ok {
|
||||
t.Fatal("old uncached token survived")
|
||||
}
|
||||
mustIAM(t, sibling.UserNotificationHandler(ctx, fresh.AccessKey, stsUser))
|
||||
if _, ok := sibling.GetUser(fresh.AccessKey); !ok {
|
||||
t.Fatal("new token was included in earlier revocation")
|
||||
}
|
||||
}
|
||||
|
||||
type iamSTSFailureStore struct {
|
||||
IAMStorageAPI
|
||||
failedPath string
|
||||
}
|
||||
|
||||
func (s *iamSTSFailureStore) saveIAMConfig(ctx context.Context, item any, path string, opts ...options) error {
|
||||
if path == s.failedPath {
|
||||
return errIAMInjectedWrite
|
||||
}
|
||||
return s.IAMStorageAPI.saveIAMConfig(ctx, item, path, opts...)
|
||||
}
|
||||
|
||||
func TestIAMSTSRevocationPartialFailure(t *testing.T) {
|
||||
ctx, sys, _ := prepareIAMRevisionFixture(t)
|
||||
_, err := sys.CreateUser(ctx, "sts-partial-parent", madmin.AddOrUpdateUserReq{SecretKey: "synthetic-parent-password", Status: madmin.AccountEnabled})
|
||||
mustIAM(t, err)
|
||||
secret, err := getTokenSigningKey()
|
||||
mustIAM(t, err)
|
||||
var keys []string
|
||||
for range 2 {
|
||||
c, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: "sts-partial-parent"}, secret)
|
||||
mustIAM(t, err)
|
||||
c.ParentUser = "sts-partial-parent"
|
||||
_, err = sys.SetTempUser(ctx, c.AccessKey, c, "")
|
||||
mustIAM(t, err)
|
||||
keys = append(keys, c.AccessKey)
|
||||
}
|
||||
sys.store.IAMStorageAPI = &iamSTSFailureStore{IAMStorageAPI: sys.store.IAMStorageAPI, failedPath: getUserIdentityPath(keys[0], stsUser)}
|
||||
revoked, err := sys.store.RevokeTokens(ctx, "sts-partial-parent", "")
|
||||
if !errors.Is(err, errIAMInjectedWrite) || len(revoked) != 1 || revoked[0] != keys[1] {
|
||||
t.Fatalf("committed keys or failure lost: %v %v", revoked, err)
|
||||
}
|
||||
if _, ok := sys.store.GetUser(keys[1]); ok {
|
||||
t.Fatal("committed revocation retained cache entry")
|
||||
}
|
||||
if _, ok := sys.store.GetUser(keys[0]); !ok {
|
||||
t.Fatal("failed revocation claimed cache success")
|
||||
}
|
||||
}
|
||||
+8
-1
@@ -717,7 +717,14 @@ func (sys *IAMSys) RevokeTokens(ctx context.Context, accessKey, tokenRevokeType
|
||||
return errServerNotInitialized
|
||||
}
|
||||
|
||||
return sys.store.RevokeTokens(ctx, accessKey, tokenRevokeType)
|
||||
keys, err := sys.store.RevokeTokens(ctx, accessKey, tokenRevokeType)
|
||||
// Publish every committed tombstone even if another token failed to save.
|
||||
for _, key := range keys {
|
||||
sys.notifyForUser(ctx, key, true)
|
||||
// Offline peers recover this same record through IAM deletion healing.
|
||||
iamLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{Type: iamSTSDeleteType, Name: key}))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteUser - delete user (only for long-term users not STS users).
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/kms-go/kes"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
)
|
||||
|
||||
// Use the configured KES client over TLS, rather than a mocked KMS interface:
|
||||
// cancellation must survive every layer, including the HTTP request context.
|
||||
func connectCancellationKES(t *testing.T, handler http.Handler) *kms.KMS {
|
||||
t.Helper()
|
||||
for _, key := range []string{kms.EnvKMSEndpoint, kms.EnvKMSSecretKey, kms.EnvKMSSecretKeyFile} {
|
||||
t.Setenv(key, "")
|
||||
mustIAM(t, os.Unsetenv(key))
|
||||
}
|
||||
server := httptest.NewTLSServer(handler)
|
||||
t.Cleanup(server.Close)
|
||||
ca := filepath.Join(t.TempDir(), "ca.pem")
|
||||
mustIAM(t, os.WriteFile(ca, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}), 0o600))
|
||||
key, err := kes.GenerateAPIKey(nil)
|
||||
mustIAM(t, err)
|
||||
t.Setenv(kms.EnvKESEndpoint, server.URL)
|
||||
t.Setenv(kms.EnvKESAPIKey, key.String())
|
||||
t.Setenv(kms.EnvKESDefaultKey, "cancellation-key")
|
||||
t.Setenv(kms.EnvKESServerCA, ca)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
k, err := kms.Connect(ctx, nil)
|
||||
mustIAM(t, err)
|
||||
return k
|
||||
}
|
||||
|
||||
func cancellationDecryptHandler(t *testing.T, k *kms.KMS, before func(*http.Request)) http.Handler {
|
||||
t.Helper()
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// KMS Connect periodically describes the default key to keep it cached.
|
||||
// This maintenance request is independent of the decrypt under test.
|
||||
if r.Method == http.MethodGet && r.URL.Path == "/v1/key/describe/cancellation-key" {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"name": "cancellation-key"})
|
||||
return
|
||||
}
|
||||
if r.URL.Path != "/v1/key/decrypt/cancellation-key" {
|
||||
t.Errorf("unexpected KES path: %s", r.URL.Path)
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
var request struct {
|
||||
Ciphertext []byte `json:"ciphertext"`
|
||||
Context []byte `json:"context"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
// Drain the HTTP/1 body so net/http can observe a peer disconnect while
|
||||
// the handler is blocked in the simulated KES operation.
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
_ = r.Body.Close()
|
||||
before(r)
|
||||
if r.Context().Err() != nil {
|
||||
return
|
||||
}
|
||||
var aad kms.Context
|
||||
if err := json.Unmarshal(request.Context, &aad); err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
plain, err := k.Decrypt(r.Context(), &kms.DecryptRequest{Name: "cancellation-key", Ciphertext: request.Ciphertext, AssociatedData: aad})
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"message": kes.ErrDecrypt.Error()})
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string][]byte{"plaintext": plain})
|
||||
})
|
||||
}
|
||||
|
||||
func TestKMSRequestCancellation(t *testing.T) {
|
||||
for _, operation := range []string{"iam", "sse-list"} {
|
||||
for _, expiry := range []string{"cancel", "deadline"} {
|
||||
t.Run(operation+"/"+expiry, func(t *testing.T) {
|
||||
builtin, err := kms.NewBuiltin("cancellation-key", bytes.Repeat([]byte{42}, 32))
|
||||
mustIAM(t, err)
|
||||
entered, canceled, release := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
||||
var calls atomic.Int32
|
||||
var enteredOnce, canceledOnce sync.Once
|
||||
unblock := sync.OnceFunc(func() { close(release) })
|
||||
client := connectCancellationKES(t, cancellationDecryptHandler(t, builtin, func(r *http.Request) {
|
||||
calls.Add(1)
|
||||
enteredOnce.Do(func() { close(entered) })
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
canceledOnce.Do(func() { close(canceled) })
|
||||
case <-release:
|
||||
}
|
||||
}))
|
||||
t.Cleanup(unblock)
|
||||
prior := GlobalKMS
|
||||
GlobalKMS = client
|
||||
t.Cleanup(func() { GlobalKMS = prior })
|
||||
var run func(context.Context) error
|
||||
if operation == "iam" {
|
||||
const name = "config/iam/policies/cancel/policy.json"
|
||||
plaintext := []byte(`{"Version":1}`)
|
||||
ciphertext, err := config.EncryptBytes(builtin, plaintext, kms.Context{minioMetaBucket: minioMetaBucket + "/" + name})
|
||||
mustIAM(t, err)
|
||||
run = func(ctx context.Context) error {
|
||||
got, err := decryptData(ctx, ciphertext, name)
|
||||
if err == nil && !bytes.Equal(got, plaintext) {
|
||||
return errors.New("config plaintext changed")
|
||||
}
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
var metadata []map[string]string
|
||||
var buckets, names []string
|
||||
var expected []crypto.ObjectKey
|
||||
for i := range 3 {
|
||||
name := fmt.Sprintf("object-%d", i)
|
||||
dek, err := builtin.GenerateKey(context.Background(), &kms.GenerateKeyRequest{AssociatedData: kms.Context{"bucket": "bucket/" + name}})
|
||||
mustIAM(t, err)
|
||||
key := crypto.GenerateKey(dek.Plaintext, nil)
|
||||
sealed := key.Seal(dek.Plaintext, crypto.GenerateIV(nil), crypto.S3.String(), "bucket", name)
|
||||
metadata = append(metadata, crypto.S3.CreateMetadata(nil, dek.KeyID, dek.Ciphertext, sealed))
|
||||
buckets, names = append(buckets, "bucket"), append(names, name)
|
||||
expected = append(expected, key)
|
||||
}
|
||||
run = func(ctx context.Context) error {
|
||||
keys, err := crypto.S3.UnsealObjectKeys(ctx, client, metadata, buckets, names)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(keys) != len(expected) {
|
||||
return errors.New("partial key list returned as success")
|
||||
}
|
||||
for i := range keys {
|
||||
if keys[i] != expected[i] {
|
||||
return errors.New("object key or ordering changed")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
if expiry == "deadline" {
|
||||
cancel()
|
||||
ctx, cancel = context.WithTimeout(context.Background(), 300*time.Millisecond)
|
||||
}
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(ctx) }()
|
||||
select {
|
||||
case <-entered:
|
||||
case err := <-done:
|
||||
t.Fatalf("decrypt returned before reaching KES: %v (type=%v)", err, client.Type)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("decrypt did not reach KES")
|
||||
}
|
||||
want := context.DeadlineExceeded
|
||||
if expiry == "cancel" {
|
||||
cancel()
|
||||
want = context.Canceled
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, want) {
|
||||
t.Fatalf("got %v, want %v", err, want)
|
||||
}
|
||||
// The current KES SDK may sleep twice for up to 999ms after a
|
||||
// deadline error. Include that existing backoff plus the 300ms
|
||||
// request deadline; cancellation must still stop new decrypts.
|
||||
case <-time.After(3 * time.Second):
|
||||
unblock()
|
||||
<-done
|
||||
t.Fatal("request exceeded the cancellation and KES retry budget")
|
||||
}
|
||||
awaitIAMPolicyReload(t, canceled)
|
||||
if n := calls.Load(); n != 1 {
|
||||
t.Fatalf("cancellation launched a fallback or more object decryptions: %d calls", n)
|
||||
}
|
||||
unblock()
|
||||
mustIAM(t, run(context.Background()))
|
||||
before := calls.Load()
|
||||
cctx, ccancel := context.WithCancel(context.Background())
|
||||
ccancel()
|
||||
if err := run(cctx); !errors.Is(err, context.Canceled) || calls.Load() != before {
|
||||
t.Fatal("already-canceled request still issued decryption work")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMDecryptAssociatedDataCompatibility(t *testing.T) {
|
||||
builtin, err := kms.NewBuiltin("cancellation-key", bytes.Repeat([]byte{42}, 32))
|
||||
mustIAM(t, err)
|
||||
var calls atomic.Int32
|
||||
client := connectCancellationKES(t, cancellationDecryptHandler(t, builtin, func(*http.Request) { calls.Add(1) }))
|
||||
prior := GlobalKMS
|
||||
GlobalKMS = client
|
||||
t.Cleanup(func() { GlobalKMS = prior })
|
||||
const name = "config/iam/policies/compat/policy.json"
|
||||
plaintext := []byte(`{"Version":1}`)
|
||||
for _, legacy := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint(legacy), func(t *testing.T) {
|
||||
aad, wantCalls := minioMetaBucket+"/"+name, int32(1)
|
||||
if legacy {
|
||||
aad, wantCalls = name, 2
|
||||
}
|
||||
ciphertext, err := config.EncryptBytes(builtin, plaintext, kms.Context{minioMetaBucket: aad})
|
||||
mustIAM(t, err)
|
||||
before := calls.Load()
|
||||
got, err := decryptData(context.Background(), ciphertext, name)
|
||||
mustIAM(t, err)
|
||||
if !bytes.Equal(got, plaintext) || calls.Load()-before != wantCalls {
|
||||
t.Fatal("current/legacy encrypted IAM records no longer decrypt correctly")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -19,7 +19,6 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"testing"
|
||||
@@ -41,13 +40,13 @@ func requireMultipartUploadKeys(t *testing.T, got ListMultipartsInfo, want ...st
|
||||
}
|
||||
}
|
||||
|
||||
func TestListMultipartUploadsS3Compatibility(t *testing.T) {
|
||||
func TestMultipartDiagnosticScanPagination(t *testing.T) {
|
||||
obj, dirs, err := prepareErasureSets32(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
setMultipartListingTestMode(t, false)
|
||||
|
||||
t.Cleanup(func() {
|
||||
z.Shutdown(t.Context())
|
||||
removeRoots(dirs)
|
||||
@@ -82,14 +81,14 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
|
||||
uploadIDs[object] = mp.UploadID
|
||||
}
|
||||
|
||||
// Durable multipart metadata, rather than this node-local cache, must be
|
||||
// authoritative after a restart or when another node handles the request.
|
||||
// The diagnostic scanner must discover metadata independently of the cache.
|
||||
// These checks do not claim that public listing offers durable pagination.
|
||||
z.mpCache.Range(func(uploadID string, _ MultipartInfo) bool {
|
||||
z.mpCache.Delete(uploadID)
|
||||
return true
|
||||
})
|
||||
|
||||
all, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 100)
|
||||
all, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -98,7 +97,7 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
|
||||
t.Fatalf("complete listing has truncation state: %+v", all)
|
||||
}
|
||||
|
||||
first, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 1)
|
||||
first, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -109,31 +108,31 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
|
||||
objects[0], uploadIDs[objects[0]])
|
||||
}
|
||||
|
||||
rest, err := z.ListMultipartUploads(t.Context(), bucket, "", first.NextKeyMarker, first.NextUploadIDMarker, "", 100)
|
||||
rest, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", first.NextKeyMarker, first.NextUploadIDMarker, "", 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, rest, objects[1:]...)
|
||||
|
||||
afterKey, err := z.ListMultipartUploads(t.Context(), bucket, "", objects[1], "", "", 100)
|
||||
afterKey, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", objects[1], "", "", 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, afterKey, objects[2:]...)
|
||||
|
||||
prefixed, err := z.ListMultipartUploads(t.Context(), bucket, "t/", "", "", "", 100)
|
||||
prefixed, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "t/", "", "", "", 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, prefixed, objects[:3]...)
|
||||
|
||||
nested, err := z.ListMultipartUploads(t.Context(), bucket, "t/a_b/", "", "", "", 100)
|
||||
nested, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "t/a_b/", "", "", "", 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, nested, objects[:2]...)
|
||||
|
||||
grouped, err := z.ListMultipartUploads(t.Context(), bucket, "t/", "", "", SlashSeparator, 100)
|
||||
grouped, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "t/", "", "", SlashSeparator, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -142,7 +141,7 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
|
||||
t.Fatalf("common prefixes = %v, want %v", grouped.CommonPrefixes, want)
|
||||
}
|
||||
|
||||
groupPage, err := z.ListMultipartUploads(t.Context(), bucket, "t/", "", "", SlashSeparator, 1)
|
||||
groupPage, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "t/", "", "", SlashSeparator, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -154,7 +153,7 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
|
||||
groupPage.NextKeyMarker, groupPage.NextUploadIDMarker, groupPage.IsTruncated)
|
||||
}
|
||||
|
||||
groupRest, err := z.ListMultipartUploads(t.Context(), bucket, "t/", groupPage.NextKeyMarker, "", SlashSeparator, 1)
|
||||
groupRest, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "t/", groupPage.NextKeyMarker, "", SlashSeparator, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -182,8 +181,8 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
|
||||
}
|
||||
|
||||
// Simulate an upload written by a pre-upgrade server. Its key cannot be
|
||||
// recovered by scanning the hashed namespace. Strict listing must fail;
|
||||
// the old path is available only through an explicit migration setting.
|
||||
// recovered by scanning the hashed namespace. Omit only that record, and
|
||||
// preserve normal operations when its bucket, key and upload ID are known.
|
||||
legacyObject := objects[1]
|
||||
er := sets.getHashedSet(legacyObject)
|
||||
fi, metadata, err := er.checkUploadIDExists(t.Context(), bucket, legacyObject, uploadIDs[legacyObject], true)
|
||||
@@ -198,16 +197,25 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
|
||||
er.getUploadIDDir(bucket, legacyObject, uploadIDs[legacyObject]), metadata, fi.WriteQuorum(er.defaultWQuorum())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = z.ListMultipartUploads(t.Context(), bucket, legacyObject, "", "", "", 100)
|
||||
if !errors.Is(err, errMultipartListingLegacy) {
|
||||
t.Fatalf("legacy strict listing: %v", err)
|
||||
}
|
||||
setMultipartListingTestMode(t, true)
|
||||
legacy, err := z.ListMultipartUploads(t.Context(), bucket, legacyObject, "", "", "", 100)
|
||||
legacy, err := z.scanMultipartUploadsForTest(t.Context(), bucket, legacyObject, "", "", "", 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, legacy, legacyObject)
|
||||
requireMultipartUploadKeys(t, legacy)
|
||||
remaining, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requireMultipartUploadKeys(t, remaining, objects[2:]...)
|
||||
part, err = z.PutObjectPart(t.Context(), bucket, legacyObject, uploadIDs[legacyObject], 1,
|
||||
mustGetPutObjReader(t, bytes.NewBufferString("old part"), 8, "", ""), ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("continue historical upload: %v", err)
|
||||
}
|
||||
if _, err = z.CompleteMultipartUpload(t.Context(), bucket, legacyObject, uploadIDs[legacyObject],
|
||||
[]CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("complete historical upload: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPaginateMultipartUploads(t *testing.T) {
|
||||
@@ -260,7 +268,7 @@ func TestPaginateMultipartUploads(t *testing.T) {
|
||||
|
||||
func TestListMultipartUploadsGlobalPageAcrossPools(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
setMultipartListingTestMode(t, false)
|
||||
|
||||
objects := []string{"a/one", "b/two", "c/three", "d/four"}
|
||||
for i, object := range objects {
|
||||
if _, err := z.serverPools[i%len(z.serverPools)].NewMultipartUpload(t.Context(), bucket, object, ObjectOptions{}); err != nil {
|
||||
@@ -272,7 +280,7 @@ func TestListMultipartUploadsGlobalPageAcrossPools(t *testing.T) {
|
||||
return true
|
||||
})
|
||||
|
||||
page, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 2)
|
||||
page, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -281,7 +289,7 @@ func TestListMultipartUploadsGlobalPageAcrossPools(t *testing.T) {
|
||||
t.Fatalf("first global page = %+v", page)
|
||||
}
|
||||
|
||||
rest, err := z.ListMultipartUploads(t.Context(), bucket, "", page.NextKeyMarker, page.NextUploadIDMarker, "", 2)
|
||||
rest, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", page.NextKeyMarker, page.NextUploadIDMarker, "", 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,412 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
)
|
||||
|
||||
func listingLockerTestWrite(t *testing.T, l *localLocker, args dsync.LockArgs) {
|
||||
t.Helper()
|
||||
if ok, err := l.Lock(t.Context(), args); !ok || err != nil {
|
||||
t.Fatalf("Lock = %v, %v", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func listingLockerTestUnlock(t *testing.T, l *localLocker, args dsync.LockArgs) {
|
||||
t.Helper()
|
||||
if ok, err := l.Unlock(t.Context(), args); !ok || err != nil {
|
||||
t.Fatalf("Unlock = %v, %v", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalLockerListingReverseWaitRechecks(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
change func(*testing.T, *localLocker, dsync.LockArgs)
|
||||
want bool
|
||||
}{
|
||||
{"writer-released", func(t *testing.T, l *localLocker, w dsync.LockArgs) { listingLockerTestUnlock(t, l, w) }, true},
|
||||
{"writer-still-held", func(*testing.T, *localLocker, dsync.LockArgs) {}, false},
|
||||
{"replacement-writer", func(t *testing.T, l *localLocker, w dsync.LockArgs) {
|
||||
listingLockerTestUnlock(t, l, w)
|
||||
listingLockerTestWrite(t, l, listingLockerTestArgs("new-writer", "key", "writer"))
|
||||
}, false},
|
||||
{"ordinary-reader-arrives", func(t *testing.T, l *localLocker, w dsync.LockArgs) {
|
||||
listingLockerTestUnlock(t, l, w)
|
||||
listingLockerTestRead(t, l, listingLockerTestArgs("ordinary-reader", "key", "ordinary-reader"))
|
||||
}, true},
|
||||
{"listing-reader-arrives", func(t *testing.T, l *localLocker, w dsync.LockArgs) {
|
||||
listingLockerTestUnlock(t, l, w)
|
||||
listingLockerTestRead(t, l, listingLockerTestArgs("other-listing", "key", listingLockerTestSource))
|
||||
}, true},
|
||||
{"force-unlock", func(t *testing.T, l *localLocker, w dsync.LockArgs) {
|
||||
if ok, err := l.ForceUnlock(t.Context(), w); !ok || err != nil {
|
||||
t.Fatalf("ForceUnlock = %v, %v", ok, err)
|
||||
}
|
||||
}, true},
|
||||
{"expiry", func(_ *testing.T, l *localLocker, _ dsync.LockArgs) { l.expireOldLocks(-time.Minute) }, true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
writer := listingLockerTestArgs("writer", "key", "writer")
|
||||
reader := listingLockerTestArgs("listing", "key", listingLockerTestSource)
|
||||
listingLockerTestWrite(t, l, writer)
|
||||
var got bool
|
||||
var err error
|
||||
start := time.Now()
|
||||
go func() { got, err = l.RLock(t.Context(), reader) }()
|
||||
synctest.Wait()
|
||||
if state := listingLockerTestSnapshot(l); state.waiting != 1 {
|
||||
t.Fatalf("waiting = %d, want 1", state.waiting)
|
||||
}
|
||||
listingLockerTestWrite(t, l, listingLockerTestArgs("unrelated", "other-key", "writer"))
|
||||
if elapsed := time.Since(start); elapsed != 0 {
|
||||
t.Fatalf("unrelated key waited %v", elapsed)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
tc.change(t, l, writer)
|
||||
time.Sleep(time.Millisecond)
|
||||
synctest.Wait()
|
||||
if state := listingLockerTestSnapshot(l); got != tc.want || err != nil || state.waiting != 0 {
|
||||
t.Fatalf("RLock = %v, %v, waiting = %d; want %v, nil, 0", got, err, state.waiting, tc.want)
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed != 2*time.Millisecond {
|
||||
t.Fatalf("one attempt elapsed %v, want 2ms", elapsed)
|
||||
}
|
||||
found := false
|
||||
for _, holder := range l.DupLockMap()["key"] {
|
||||
if holder.UID == reader.UID {
|
||||
found = true
|
||||
if holder.Writer || holder.Timestamp != UTCNow().UnixNano() || holder.TimeLastRefresh != holder.Timestamp {
|
||||
t.Fatalf("invalid granted reader: %+v", holder)
|
||||
}
|
||||
}
|
||||
}
|
||||
if found != got {
|
||||
t.Fatalf("reader present = %v, granted = %v", found, got)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalLockerListingReverseWaitCancellation(t *testing.T) {
|
||||
for _, when := range []string{"before-call", "during-wait", "deadline"} {
|
||||
t.Run(when, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
writer := listingLockerTestArgs("writer", "key", "writer")
|
||||
listingLockerTestWrite(t, l, writer)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
if when == "deadline" {
|
||||
cancel()
|
||||
ctx, cancel = context.WithTimeout(t.Context(), time.Millisecond)
|
||||
}
|
||||
defer cancel()
|
||||
if when == "before-call" {
|
||||
cancel()
|
||||
}
|
||||
var got bool
|
||||
var err error
|
||||
start := time.Now()
|
||||
go func() { got, err = l.RLock(ctx, listingLockerTestArgs("listing", "key", listingLockerTestSource)) }()
|
||||
synctest.Wait()
|
||||
if when != "before-call" {
|
||||
if state := listingLockerTestSnapshot(l); state.waiting != 1 {
|
||||
t.Fatalf("waiting = %d, want 1", state.waiting)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
// Removal must not turn the canceled request into a grant.
|
||||
listingLockerTestUnlock(t, l, writer)
|
||||
cancel()
|
||||
} else {
|
||||
listingLockerTestUnlock(t, l, writer)
|
||||
}
|
||||
synctest.Wait()
|
||||
state := listingLockerTestSnapshot(l)
|
||||
if got || err == nil || err != ctx.Err() || state != (listingLockerTestState{}) {
|
||||
t.Fatalf("canceled RLock = %v, %v; context %v; state %+v", got, err, ctx.Err(), state)
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed >= 2*time.Millisecond {
|
||||
t.Fatalf("canceled request waited for full timer: %v", elapsed)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalLockerListingReverseWaitExcluded(t *testing.T) {
|
||||
for _, source := range []string{"ordinary-reader", "", "[<unknown>:0:()]", "listingRLock", "[metacache-server-pool.go:0:(*erasureServerPools).listingRLock()]"} {
|
||||
t.Run(source, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
listingLockerTestWrite(t, l, listingLockerTestArgs("writer", "key", "writer"))
|
||||
start := time.Now()
|
||||
if ok, err := l.RLock(t.Context(), listingLockerTestArgs("reader", "key", source)); ok || err != nil {
|
||||
t.Fatalf("excluded RLock = %v, %v", ok, err)
|
||||
}
|
||||
if state := listingLockerTestSnapshot(l); time.Since(start) != 0 || state.waiting != 0 || state.uids != 1 {
|
||||
t.Fatalf("excluded RLock changed timing or state: %+v", state)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalLockerListingReverseWaitFastPaths(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
start := time.Now()
|
||||
for i, source := range []string{listingLockerTestSource, "ordinary-reader", ""} {
|
||||
listingLockerTestRead(t, l, listingLockerTestArgs(fmt.Sprintf("shared-%d", i), "shared", source))
|
||||
listingLockerTestRead(t, l, listingLockerTestArgs(fmt.Sprintf("free-%d", i), fmt.Sprintf("free-%d", i), source))
|
||||
}
|
||||
if state := listingLockerTestSnapshot(l); time.Since(start) != 0 || state.waiting != 0 || state.uids != 6 {
|
||||
t.Fatalf("uncontended readers changed timing or state: %+v", state)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestLocalLockerListingReverseWaitOverloaded(t *testing.T) {
|
||||
for _, when := range []string{"before-call", "during-wait"} {
|
||||
t.Run(when, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
writer := listingLockerTestArgs("writer", "key", "writer")
|
||||
listingLockerTestWrite(t, l, writer)
|
||||
defer l.waitMutex.Store(0)
|
||||
if when == "before-call" {
|
||||
l.waitMutex.Store(lockMutexWaitLimit + 1)
|
||||
}
|
||||
var got bool
|
||||
var err error
|
||||
go func() {
|
||||
got, err = l.RLock(t.Context(), listingLockerTestArgs("listing", "key", listingLockerTestSource))
|
||||
}()
|
||||
synctest.Wait()
|
||||
if when == "during-wait" {
|
||||
if state := listingLockerTestSnapshot(l); state.waiting != 1 {
|
||||
t.Fatalf("waiting = %d, want 1", state.waiting)
|
||||
}
|
||||
listingLockerTestUnlock(t, l, writer)
|
||||
l.waitMutex.Store(lockMutexWaitLimit + 1)
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
synctest.Wait()
|
||||
} else {
|
||||
listingLockerTestUnlock(t, l, writer)
|
||||
}
|
||||
if state := listingLockerTestSnapshot(l); got || err != nil || state != (listingLockerTestState{}) || l.locksOverloaded.Load() != 1 {
|
||||
t.Fatalf("overloaded RLock = %v, %v; state %+v; overloads %d", got, err, state, l.locksOverloaded.Load())
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalLockerListingWaitSharedAdmission(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
listingLockerTestRead(t, l, listingLockerTestArgs("reader", "read-key", listingLockerTestSource))
|
||||
listingLockerTestWrite(t, l, listingLockerTestArgs("writer", "write-key", "writer"))
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
results := make(chan error, listingLockWaitLimit)
|
||||
for i := range listingLockWaitLimit {
|
||||
go func() {
|
||||
var ok bool
|
||||
var err error
|
||||
if i%2 == 0 {
|
||||
ok, err = l.Lock(ctx, listingLockerTestArgs(fmt.Sprintf("writer-%d", i), "read-key", "writer"))
|
||||
} else {
|
||||
ok, err = l.RLock(ctx, listingLockerTestArgs(fmt.Sprintf("listing-%d", i), "write-key", listingLockerTestSource))
|
||||
}
|
||||
if ok {
|
||||
err = fmt.Errorf("granted over a conflicting holder")
|
||||
}
|
||||
results <- err
|
||||
}()
|
||||
}
|
||||
synctest.Wait()
|
||||
if state := listingLockerTestSnapshot(l); state.waiting != listingLockWaitLimit {
|
||||
t.Fatalf("combined waiting = %d, want %d", state.waiting, listingLockWaitLimit)
|
||||
}
|
||||
start := time.Now()
|
||||
if ok, err := l.Lock(ctx, listingLockerTestArgs("excess-writer", "read-key", "writer")); ok || err != nil {
|
||||
t.Fatalf("excess Lock = %v, %v", ok, err)
|
||||
}
|
||||
if ok, err := l.RLock(ctx, listingLockerTestArgs("excess-listing", "write-key", listingLockerTestSource)); ok || err != nil {
|
||||
t.Fatalf("excess RLock = %v, %v", ok, err)
|
||||
}
|
||||
listingLockerTestWrite(t, l, listingLockerTestArgs("unrelated-writer", "unrelated-write", "writer"))
|
||||
listingLockerTestRead(t, l, listingLockerTestArgs("unrelated-listing", "unrelated-read", listingLockerTestSource))
|
||||
if state := listingLockerTestSnapshot(l); time.Since(start) != 0 || state.waiting != listingLockWaitLimit || state.uids != 4 {
|
||||
t.Fatalf("combined cap changed immediate paths: %+v", state)
|
||||
}
|
||||
cancel()
|
||||
synctest.Wait()
|
||||
for range listingLockWaitLimit {
|
||||
if err := <-results; err != context.Canceled {
|
||||
t.Fatalf("canceled waiter: %v", err)
|
||||
}
|
||||
}
|
||||
if state := listingLockerTestSnapshot(l); state.waiting != 0 || state.uids != 4 {
|
||||
t.Fatalf("combined wait leaked: %+v", state)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
type listingReverseCountLocker struct {
|
||||
*localLocker
|
||||
calls atomic.Int32
|
||||
grants atomic.Int32
|
||||
reject bool
|
||||
}
|
||||
|
||||
func (l *listingReverseCountLocker) RLock(ctx context.Context, args dsync.LockArgs) (bool, error) {
|
||||
l.calls.Add(1)
|
||||
if l.reject {
|
||||
return false, nil
|
||||
}
|
||||
ok, err := l.localLocker.RLock(ctx, args)
|
||||
if ok {
|
||||
l.grants.Add(1)
|
||||
}
|
||||
return ok, err
|
||||
}
|
||||
|
||||
func TestDistributedListingReverseWait(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
free, held int
|
||||
immediateRejects int
|
||||
cancel bool
|
||||
want bool
|
||||
}{
|
||||
{name: "all-writers-release", want: true},
|
||||
{name: "two-writers-remain", held: 2, want: true},
|
||||
{name: "three-writers-remain", held: 3},
|
||||
{name: "partial-grant-then-quorum", free: 1, want: true},
|
||||
{name: "partial-grant-without-quorum", free: 1, held: 3},
|
||||
{name: "late-grant-after-rejection", immediateRejects: 3},
|
||||
{name: "partial-and-late-grants-after-cancel", free: 1, cancel: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
clients := make([]dsync.NetLocker, 4)
|
||||
writer := listingLockerTestArgs("writer", "key", "writer")
|
||||
for i := range clients {
|
||||
l := &listingReverseCountLocker{localLocker: newLocker(), reject: i < tc.immediateRejects}
|
||||
clients[i] = l
|
||||
if i >= tc.free {
|
||||
listingLockerTestWrite(t, l.localLocker, writer)
|
||||
}
|
||||
}
|
||||
ds := &dsync.Dsync{GetLockers: func() ([]dsync.NetLocker, string) { return clients, writer.Owner }, Timeouts: dsync.DefaultTimeouts}
|
||||
dm := dsync.NewDRWMutex(ds, "key")
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
result := make(chan bool, 1)
|
||||
go func() {
|
||||
result <- dm.GetRLock(ctx, nil, "listing", listingLockerTestSource, dsync.Options{Timeout: time.Second, RetryInterval: -1})
|
||||
}()
|
||||
synctest.Wait()
|
||||
for i, c := range clients {
|
||||
l := c.(*listingReverseCountLocker)
|
||||
if i < tc.free && l.grants.Load() != 1 {
|
||||
t.Fatal("uncontended partial grant did not arrive immediately")
|
||||
}
|
||||
if i >= tc.free && !l.reject && listingLockerTestSnapshot(l.localLocker).waiting != 1 {
|
||||
t.Fatal("conflicting reader did not wait in its first RPC")
|
||||
}
|
||||
}
|
||||
earlyResult := tc.immediateRejects == 3 || tc.cancel
|
||||
if tc.cancel {
|
||||
cancel()
|
||||
synctest.Wait()
|
||||
}
|
||||
if earlyResult {
|
||||
select {
|
||||
case got := <-result:
|
||||
if got {
|
||||
t.Fatal("attempt succeeded before delayed replies")
|
||||
}
|
||||
default:
|
||||
t.Fatal("attempt did not return before delayed replies")
|
||||
}
|
||||
for i := range tc.free {
|
||||
if state := listingLockerTestSnapshot(clients[i].(*listingReverseCountLocker).localLocker); state.uids != 0 {
|
||||
t.Fatal("early partial grant was not released")
|
||||
}
|
||||
}
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
for i, c := range clients {
|
||||
if i >= tc.free+tc.held {
|
||||
listingLockerTestUnlock(t, c.(*listingReverseCountLocker).localLocker, writer)
|
||||
}
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
synctest.Wait()
|
||||
if !earlyResult {
|
||||
if got := <-result; got != tc.want {
|
||||
t.Fatalf("GetRLock = %v, want %v", got, tc.want)
|
||||
}
|
||||
}
|
||||
for i, c := range clients {
|
||||
l := c.(*listingReverseCountLocker)
|
||||
if l.calls.Load() != 1 || listingLockerTestSnapshot(l.localLocker).waiting != 0 {
|
||||
t.Fatalf("peer %d calls = %d or leaked admission", i, l.calls.Load())
|
||||
}
|
||||
wantGrant := !l.reject && (i < tc.free || i >= tc.free+tc.held)
|
||||
if (l.grants.Load() == 1) != wantGrant {
|
||||
t.Fatalf("peer %d grants = %d, want granted = %v", i, l.grants.Load(), wantGrant)
|
||||
}
|
||||
if !tc.want {
|
||||
for _, holder := range l.DupLockMap()["key"] {
|
||||
if holder.UID == "listing" {
|
||||
t.Fatalf("peer %d leaked partial or late read grant", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if tc.want {
|
||||
dm.RUnlock(t.Context())
|
||||
synctest.Wait()
|
||||
}
|
||||
for i, c := range clients {
|
||||
l := c.(*listingReverseCountLocker)
|
||||
if i >= tc.free && i < tc.free+tc.held {
|
||||
listingLockerTestUnlock(t, l.localLocker, writer)
|
||||
}
|
||||
if state := listingLockerTestSnapshot(l.localLocker); state != (listingLockerTestState{}) {
|
||||
t.Fatalf("peer %d final state is not empty: %+v", i, state)
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,432 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
)
|
||||
|
||||
const listingLockerTestSource = "[metacache-server-pool.go:274:(*erasureServerPools).listingRLock()]"
|
||||
|
||||
func listingLockerTestArgs(uid, resource, source string) dsync.LockArgs {
|
||||
quorum := 3
|
||||
return dsync.LockArgs{UID: uid, Resources: []string{resource}, Source: source, Owner: "listing-wait-test", Quorum: &quorum}
|
||||
}
|
||||
|
||||
func listingLockerTestRead(t *testing.T, l *localLocker, args dsync.LockArgs) {
|
||||
t.Helper()
|
||||
if ok, err := l.RLock(t.Context(), args); !ok || err != nil {
|
||||
t.Fatalf("RLock = %v, %v", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func listingLockerTestRUnlock(t *testing.T, l *localLocker, args dsync.LockArgs) {
|
||||
t.Helper()
|
||||
if ok, err := l.RUnlock(t.Context(), args); !ok || err != nil {
|
||||
t.Fatalf("RUnlock = %v, %v", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
type listingLockerTestState struct {
|
||||
waiting, resources, uids int
|
||||
}
|
||||
|
||||
func listingLockerTestSnapshot(l *localLocker) listingLockerTestState {
|
||||
defer l.getMutex()()
|
||||
return listingLockerTestState{waiting: l.listingWaiters, resources: len(l.lockMap), uids: len(l.lockUID)}
|
||||
}
|
||||
|
||||
func TestLocalLockerListingSource(t *testing.T) {
|
||||
for _, source := range []string{
|
||||
"", "listingRLock", "[<unknown>:0:()]",
|
||||
strings.Replace(listingLockerTestSource, ":274:", ":", 1),
|
||||
strings.Replace(listingLockerTestSource, "274", "", 1),
|
||||
strings.Replace(listingLockerTestSource, "274", "0", 1),
|
||||
strings.Replace(listingLockerTestSource, "274", "1x", 1),
|
||||
strings.Replace(listingLockerTestSource, "274", "-1", 1),
|
||||
strings.Replace(listingLockerTestSource, "listingRLock", "GetObjectNInfo", 1),
|
||||
strings.Replace(listingLockerTestSource, "listingRLock", "listingRLock.func1", 1),
|
||||
strings.Replace(listingLockerTestSource, "metacache-server-pool.go", "other.go", 1),
|
||||
listingLockerTestSource + "trailing",
|
||||
} {
|
||||
if isListingReadLock([]lockRequesterInfo{{Source: source}}) {
|
||||
t.Errorf("unrecognized source accepted: %q", source)
|
||||
}
|
||||
}
|
||||
if isListingReadLock(nil) || isListingReadLock([]lockRequesterInfo{{Source: listingLockerTestSource, Writer: true}}) {
|
||||
t.Fatal("empty or write holders accepted")
|
||||
}
|
||||
if isListingReadLock([]lockRequesterInfo{{Source: listingLockerTestSource}, {Source: "ordinary-reader"}}) {
|
||||
t.Fatal("mixed readers accepted")
|
||||
}
|
||||
|
||||
// Check the real runtime-generated source, not only a hand-written label.
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
lockers := []dsync.NetLocker{newLocker(), newLocker(), newLocker(), newLocker()}
|
||||
set := &erasureObjects{nsMutex: newNSLock(true), getLockers: func() ([]dsync.NetLocker, string) { return lockers, "listing-source-test" }}
|
||||
z := &erasureServerPools{serverPools: []*erasureSets{{sets: []*erasureObjects{set}}, {sets: []*erasureObjects{set}}}}
|
||||
_, unlock, err := z.listingRLock(t.Context(), "bucket", "object", set)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range lockers {
|
||||
l := c.(*localLocker)
|
||||
for _, holders := range l.DupLockMap() {
|
||||
if !isListingReadLock(holders) {
|
||||
t.Errorf("actual listingRLock source was not recognized: %#v", holders)
|
||||
}
|
||||
}
|
||||
}
|
||||
unlock()
|
||||
synctest.Wait()
|
||||
})
|
||||
}
|
||||
|
||||
func FuzzListingLockSource(f *testing.F) {
|
||||
f.Add(listingLockerTestSource)
|
||||
f.Add("[metacache-server-pool.go:(*erasureServerPools).listingRLock()]")
|
||||
f.Add("")
|
||||
f.Fuzz(func(t *testing.T, source string) {
|
||||
// Source is diagnostic text from a peer, including older versions. No
|
||||
// malformed string may panic or make a writer eligible for this wait.
|
||||
isListingReadLock([]lockRequesterInfo{{Source: source}})
|
||||
if isListingReadLock([]lockRequesterInfo{{Source: source, Writer: true}}) {
|
||||
t.Fatal("writer was accepted as a listing reader")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestLocalLockerListingWaitRechecks(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
change func(*testing.T, *localLocker, dsync.LockArgs)
|
||||
want bool
|
||||
}{
|
||||
{"reader-released", func(t *testing.T, l *localLocker, r dsync.LockArgs) { listingLockerTestRUnlock(t, l, r) }, true},
|
||||
{"reader-still-held", func(*testing.T, *localLocker, dsync.LockArgs) {}, false},
|
||||
{"one-of-two-readers-remains", func(t *testing.T, l *localLocker, r dsync.LockArgs) {
|
||||
other := r
|
||||
other.UID = "second-reader"
|
||||
listingLockerTestRead(t, l, other)
|
||||
listingLockerTestRUnlock(t, l, r)
|
||||
}, false},
|
||||
{"ordinary-reader-arrives", func(t *testing.T, l *localLocker, r dsync.LockArgs) {
|
||||
listingLockerTestRUnlock(t, l, r)
|
||||
listingLockerTestRead(t, l, listingLockerTestArgs("new-reader", "key", "ordinary-reader"))
|
||||
}, false},
|
||||
{"other-writer-wins", func(t *testing.T, l *localLocker, r dsync.LockArgs) {
|
||||
listingLockerTestRUnlock(t, l, r)
|
||||
if ok, err := l.Lock(t.Context(), listingLockerTestArgs("other-writer", "key", "writer")); !ok || err != nil {
|
||||
t.Fatalf("competing Lock = %v, %v", ok, err)
|
||||
}
|
||||
}, false},
|
||||
{"force-unlock", func(t *testing.T, l *localLocker, r dsync.LockArgs) {
|
||||
if ok, err := l.ForceUnlock(t.Context(), r); !ok || err != nil {
|
||||
t.Fatalf("ForceUnlock = %v, %v", ok, err)
|
||||
}
|
||||
}, true},
|
||||
{"expiry", func(_ *testing.T, l *localLocker, _ dsync.LockArgs) { l.expireOldLocks(-time.Minute) }, true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
reader := listingLockerTestArgs("reader", "key", listingLockerTestSource)
|
||||
listingLockerTestRead(t, l, reader)
|
||||
var got bool
|
||||
var err error
|
||||
start := time.Now()
|
||||
go func() { got, err = l.Lock(t.Context(), listingLockerTestArgs("writer", "key", "writer")) }()
|
||||
synctest.Wait()
|
||||
if state := listingLockerTestSnapshot(l); state.waiting != 1 {
|
||||
t.Fatalf("waiting = %d, want 1", state.waiting)
|
||||
}
|
||||
// The global mutex must be released throughout the timer wait.
|
||||
if ok, err := l.Lock(t.Context(), listingLockerTestArgs("unrelated", "other-key", "writer")); !ok || err != nil {
|
||||
t.Fatalf("unrelated key blocked: %v, %v", ok, err)
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed != 0 {
|
||||
t.Fatalf("unrelated key waited %v", elapsed)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
tc.change(t, l, reader)
|
||||
time.Sleep(time.Millisecond)
|
||||
synctest.Wait()
|
||||
if state := listingLockerTestSnapshot(l); got != tc.want || err != nil || state.waiting != 0 {
|
||||
t.Fatalf("Lock = %v, %v, waiting = %d; want %v, nil, 0", got, err, state.waiting, tc.want)
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed != 2*time.Millisecond {
|
||||
t.Fatalf("one attempt elapsed %v, want 2ms", elapsed)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalLockerListingWaitCancellation(t *testing.T) {
|
||||
for _, when := range []string{"before-call", "during-wait", "deadline"} {
|
||||
t.Run(when, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
reader := listingLockerTestArgs("reader", "key", listingLockerTestSource)
|
||||
listingLockerTestRead(t, l, reader)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
if when == "deadline" {
|
||||
cancel()
|
||||
ctx, cancel = context.WithTimeout(t.Context(), time.Millisecond)
|
||||
}
|
||||
defer cancel()
|
||||
if when == "before-call" {
|
||||
cancel()
|
||||
}
|
||||
var got bool
|
||||
var err error
|
||||
go func() { got, err = l.Lock(ctx, listingLockerTestArgs("writer", "key", "writer")) }()
|
||||
synctest.Wait()
|
||||
if when != "before-call" {
|
||||
if state := listingLockerTestSnapshot(l); state.waiting != 1 {
|
||||
t.Fatalf("waiting = %d", state.waiting)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
cancel()
|
||||
}
|
||||
synctest.Wait()
|
||||
if state := listingLockerTestSnapshot(l); got || err != ctx.Err() || err == nil || state.waiting != 0 {
|
||||
t.Fatalf("canceled Lock = %v, %v; context %v; waiting = %d", got, err, ctx.Err(), state.waiting)
|
||||
}
|
||||
listingLockerTestRUnlock(t, l, reader)
|
||||
if state := listingLockerTestSnapshot(l); state.resources != 0 || state.uids != 0 {
|
||||
t.Fatal("canceled wait left a lock")
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalLockerListingWaitExcluded(t *testing.T) {
|
||||
for _, kind := range []string{"ordinary-reader", "unknown-reader", "mixed-readers", "writer", "bulk", "overloaded"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
reader := listingLockerTestArgs("holder", "key", listingLockerTestSource)
|
||||
if kind == "ordinary-reader" {
|
||||
reader.Source = "[erasure-object.go:100:(*erasureObjects).GetObjectNInfo()]"
|
||||
}
|
||||
if kind == "unknown-reader" {
|
||||
reader.Source = ""
|
||||
}
|
||||
if kind == "writer" {
|
||||
if ok, err := l.Lock(t.Context(), reader); !ok || err != nil {
|
||||
t.Fatalf("initial write = %v, %v", ok, err)
|
||||
}
|
||||
} else {
|
||||
listingLockerTestRead(t, l, reader)
|
||||
}
|
||||
if kind == "mixed-readers" {
|
||||
listingLockerTestRead(t, l, listingLockerTestArgs("ordinary", "key", "ordinary-reader"))
|
||||
}
|
||||
writer := listingLockerTestArgs("writer", "key", "writer")
|
||||
if kind == "bulk" {
|
||||
writer.Resources = []string{"key", "other-key"}
|
||||
}
|
||||
if kind == "overloaded" {
|
||||
l.waitMutex.Store(lockMutexWaitLimit + 1)
|
||||
}
|
||||
start := time.Now()
|
||||
if ok, err := l.Lock(t.Context(), writer); ok || err != nil {
|
||||
t.Fatalf("conflicting Lock = %v, %v", ok, err)
|
||||
}
|
||||
state := listingLockerTestSnapshot(l)
|
||||
if elapsed := time.Since(start); elapsed != 0 || state.waiting != 0 {
|
||||
t.Fatalf("excluded path waited %v; waiters = %d", elapsed, state.waiting)
|
||||
}
|
||||
if _, ok := l.DupLockMap()["other-key"]; ok {
|
||||
t.Fatal("bulk request partially acquired an unrelated key")
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalLockerListingWaitAdmission(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
reader := listingLockerTestArgs("reader", "key", listingLockerTestSource)
|
||||
listingLockerTestRead(t, l, reader)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
results := make(chan error, listingLockWaitLimit)
|
||||
for i := range listingLockWaitLimit {
|
||||
go func() {
|
||||
ok, err := l.Lock(ctx, listingLockerTestArgs(fmt.Sprintf("writer-%d", i), "key", "writer"))
|
||||
if ok {
|
||||
err = fmt.Errorf("granted over a live reader")
|
||||
}
|
||||
results <- err
|
||||
}()
|
||||
}
|
||||
synctest.Wait()
|
||||
if state := listingLockerTestSnapshot(l); state.waiting != listingLockWaitLimit {
|
||||
t.Fatalf("waiting = %d, want %d", state.waiting, listingLockWaitLimit)
|
||||
}
|
||||
start := time.Now()
|
||||
if ok, err := l.Lock(ctx, listingLockerTestArgs("excess", "key", "writer")); ok || err != nil {
|
||||
t.Fatalf("excess waiter = %v, %v", ok, err)
|
||||
}
|
||||
if state := listingLockerTestSnapshot(l); time.Since(start) != 0 || state.waiting != listingLockWaitLimit {
|
||||
t.Fatal("excess waiter consumed time or admission")
|
||||
}
|
||||
if ok, err := l.Lock(ctx, listingLockerTestArgs("unrelated", "other-key", "writer")); !ok || err != nil {
|
||||
t.Fatalf("admission limit rejected an uncontended key: %v, %v", ok, err)
|
||||
}
|
||||
cancel()
|
||||
synctest.Wait()
|
||||
for range listingLockWaitLimit {
|
||||
if err := <-results; err != context.Canceled {
|
||||
t.Fatalf("canceled waiter: %v", err)
|
||||
}
|
||||
}
|
||||
if state := listingLockerTestSnapshot(l); state.waiting != 0 {
|
||||
t.Fatalf("admission leaked: %d", state.waiting)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestLocalLockerListingWaitContended(t *testing.T) {
|
||||
for _, n := range []int{2, 8, 32} {
|
||||
t.Run(fmt.Sprintf("%d-writers", n), func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
l := newLocker()
|
||||
reader := listingLockerTestArgs("reader", "key", listingLockerTestSource)
|
||||
listingLockerTestRead(t, l, reader)
|
||||
results := make(chan bool, n)
|
||||
for i := range n {
|
||||
go func() {
|
||||
ok, err := l.Lock(t.Context(), listingLockerTestArgs(fmt.Sprintf("writer-%d", i), "key", "writer"))
|
||||
if err != nil {
|
||||
t.Errorf("Lock: %v", err)
|
||||
}
|
||||
results <- ok
|
||||
}()
|
||||
}
|
||||
synctest.Wait()
|
||||
if state := listingLockerTestSnapshot(l); state.waiting != n {
|
||||
t.Fatalf("waiting = %d, want %d", state.waiting, n)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
listingLockerTestRUnlock(t, l, reader)
|
||||
time.Sleep(time.Millisecond)
|
||||
synctest.Wait()
|
||||
granted := 0
|
||||
for range n {
|
||||
if <-results {
|
||||
granted++
|
||||
}
|
||||
}
|
||||
if state := listingLockerTestSnapshot(l); granted != 1 || state.uids != 1 || state.waiting != 0 {
|
||||
t.Fatalf("grants = %d, indexed = %d, waiting = %d", granted, state.uids, state.waiting)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type listingCountLocker struct {
|
||||
*localLocker
|
||||
calls atomic.Int32
|
||||
}
|
||||
|
||||
func (l *listingCountLocker) Lock(ctx context.Context, args dsync.LockArgs) (bool, error) {
|
||||
l.calls.Add(1)
|
||||
return l.localLocker.Lock(ctx, args)
|
||||
}
|
||||
|
||||
// Exercise real DRWMutex quorum and late-response cleanup. The mixed case with
|
||||
// two immediate rejections fails before the upgraded lockers finish waiting.
|
||||
func TestDistributedListingWait(t *testing.T) {
|
||||
for _, ordinaryReaders := range []int{0, 1, 2} {
|
||||
t.Run(fmt.Sprintf("%d-immediate-rejections", ordinaryReaders), func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
clients := make([]dsync.NetLocker, 4)
|
||||
reader := listingLockerTestArgs("reader", "key", listingLockerTestSource)
|
||||
for i := range clients {
|
||||
l := &listingCountLocker{localLocker: newLocker()}
|
||||
clients[i] = l
|
||||
r := reader
|
||||
if i < ordinaryReaders {
|
||||
r.Source = "old-node-unknown-source"
|
||||
}
|
||||
listingLockerTestRead(t, l.localLocker, r)
|
||||
}
|
||||
ds := &dsync.Dsync{GetLockers: func() ([]dsync.NetLocker, string) { return clients, reader.Owner }, Timeouts: dsync.DefaultTimeouts}
|
||||
dm := dsync.NewDRWMutex(ds, "key")
|
||||
var got bool
|
||||
go func() {
|
||||
got = dm.GetLock(t.Context(), nil, "writer", "writer", dsync.Options{Timeout: time.Second, RetryInterval: -1})
|
||||
}()
|
||||
synctest.Wait()
|
||||
time.Sleep(time.Millisecond)
|
||||
for _, c := range clients {
|
||||
listingLockerTestRUnlock(t, c.(*listingCountLocker).localLocker, reader)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
synctest.Wait()
|
||||
if got != (ordinaryReaders < 2) {
|
||||
t.Fatalf("GetLock = %v", got)
|
||||
}
|
||||
granted := 0
|
||||
for _, c := range clients {
|
||||
l := c.(*listingCountLocker)
|
||||
if l.calls.Load() != 1 {
|
||||
t.Fatalf("lock calls = %d, want one round", l.calls.Load())
|
||||
}
|
||||
state := listingLockerTestSnapshot(l.localLocker)
|
||||
if state.resources != 0 {
|
||||
granted++
|
||||
}
|
||||
if state.waiting != 0 {
|
||||
t.Fatal("wait admission leaked")
|
||||
}
|
||||
}
|
||||
if got && granted < 3 {
|
||||
t.Fatalf("writer accepted without quorum: %d", granted)
|
||||
}
|
||||
if !got && granted != 0 {
|
||||
t.Fatalf("late grants leaked after rejected attempt: %d", granted)
|
||||
}
|
||||
if got {
|
||||
dm.Unlock(t.Context())
|
||||
synctest.Wait()
|
||||
}
|
||||
for _, c := range clients {
|
||||
l := c.(*listingCountLocker)
|
||||
if state := listingLockerTestSnapshot(l.localLocker); state.resources != 0 || state.uids != 0 {
|
||||
t.Fatal("final lock state not empty")
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
+89
-4
@@ -23,6 +23,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
@@ -38,6 +39,14 @@ import (
|
||||
// Rejected clients are expected to retry.
|
||||
const lockMutexWaitLimit = 1000
|
||||
|
||||
// A short listing recovery read or conflicting writer may finish before the
|
||||
// rejected request retries. Recheck once in the same request, sharing a bound
|
||||
// for timers outside the mutex. This is not a queue and gives neither priority.
|
||||
const (
|
||||
listingLockWait = 2 * time.Millisecond
|
||||
listingLockWaitLimit = 64
|
||||
)
|
||||
|
||||
// lockRequesterInfo stores various info from the client for each lock that is requested.
|
||||
type lockRequesterInfo struct {
|
||||
Name string // name of the resource lock was requested for
|
||||
@@ -65,6 +74,8 @@ type localLocker struct {
|
||||
waitMutex atomic.Int32
|
||||
lockMap map[string][]lockRequesterInfo
|
||||
lockUID map[string]string // UUID -> resource map.
|
||||
// Protected by mutex. These waiters are not included in waitMutex.
|
||||
listingWaiters int
|
||||
|
||||
// the following are updated on every cleanup defined in lockValidityDuration
|
||||
readers atomic.Int32
|
||||
@@ -96,6 +107,32 @@ func (l *localLocker) canTakeLock(resources ...string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// isListingReadLock recognizes only the existing diagnostic source emitted by
|
||||
// listingRLock. Unknown or changed source formats retain immediate rejection;
|
||||
// Source is not a new wire contract or proof that the read will finish quickly.
|
||||
func isListingReadLock(holders []lockRequesterInfo) bool {
|
||||
if len(holders) == 0 {
|
||||
return false
|
||||
}
|
||||
const prefix = "[metacache-server-pool.go:"
|
||||
const suffix = ":(*erasureServerPools).listingRLock()]"
|
||||
for _, holder := range holders {
|
||||
if holder.Writer || len(holder.Source) <= len(prefix)+len(suffix) || !strings.HasPrefix(holder.Source, prefix) || !strings.HasSuffix(holder.Source, suffix) {
|
||||
return false
|
||||
}
|
||||
line := holder.Source[len(prefix) : len(holder.Source)-len(suffix)]
|
||||
if len(line) == 0 || line[0] == '0' {
|
||||
return false
|
||||
}
|
||||
for _, c := range line {
|
||||
if c < '0' || c > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
|
||||
if len(args.Resources) > maxDeleteList {
|
||||
return false, fmt.Errorf("internal error: localLocker.Lock called with more than %d resources", maxDeleteList)
|
||||
@@ -112,9 +149,33 @@ func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool
|
||||
return false, ctx.Err()
|
||||
}
|
||||
if !l.canTakeLock(args.Resources...) {
|
||||
// Not all locks can be taken on resources,
|
||||
// reject it completely.
|
||||
return false, nil
|
||||
// Only a single-resource write blocked exclusively by listing recovery
|
||||
// readers gets a short, bounded opportunity to succeed in this attempt.
|
||||
if len(args.Resources) != 1 || l.listingWaiters >= listingLockWaitLimit || !isListingReadLock(l.lockMap[args.Resources[0]]) {
|
||||
return false, nil
|
||||
}
|
||||
l.listingWaiters++
|
||||
l.mutex.Unlock()
|
||||
timer := time.NewTimer(listingLockWait)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-timer.C:
|
||||
}
|
||||
timer.Stop()
|
||||
// Restore the mutex before returning to the existing deferred unlock.
|
||||
// Readers or another writer may have entered while it was released.
|
||||
l.getMutex()
|
||||
l.listingWaiters--
|
||||
if ctx.Err() != nil {
|
||||
return false, ctx.Err()
|
||||
}
|
||||
if l.waitMutex.Load() > lockMutexWaitLimit {
|
||||
l.locksOverloaded.Add(1)
|
||||
return false, nil
|
||||
}
|
||||
if !l.canTakeLock(args.Resources...) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// No locks held on the all resources, so claim write
|
||||
@@ -206,6 +267,31 @@ func (l *localLocker) RLock(ctx context.Context, args dsync.LockArgs) (reply boo
|
||||
return false, ctx.Err()
|
||||
}
|
||||
resource := args.Resources[0]
|
||||
lri, ok := l.lockMap[resource]
|
||||
if ok && isWriteLock(lri) && l.listingWaiters < listingLockWaitLimit && isListingReadLock([]lockRequesterInfo{{Source: args.Source}}) {
|
||||
// Only a positively identified listing read blocked by a writer may
|
||||
// wait once. A long or replacement writer still rejects normally.
|
||||
l.listingWaiters++
|
||||
l.mutex.Unlock()
|
||||
timer := time.NewTimer(listingLockWait)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-timer.C:
|
||||
}
|
||||
timer.Stop()
|
||||
// Restore the mutex before the existing deferred unlock, then use
|
||||
// the current holders in the normal read-lock grant path below.
|
||||
l.getMutex()
|
||||
l.listingWaiters--
|
||||
if ctx.Err() != nil {
|
||||
return false, ctx.Err()
|
||||
}
|
||||
if l.waitMutex.Load() > lockMutexWaitLimit {
|
||||
l.locksOverloaded.Add(1)
|
||||
return false, nil
|
||||
}
|
||||
lri, ok = l.lockMap[resource]
|
||||
}
|
||||
now := UTCNow()
|
||||
lrInfo := lockRequesterInfo{
|
||||
Name: resource,
|
||||
@@ -217,7 +303,6 @@ func (l *localLocker) RLock(ctx context.Context, args dsync.LockArgs) (reply boo
|
||||
TimeLastRefresh: now.UnixNano(),
|
||||
Quorum: *args.Quorum,
|
||||
}
|
||||
lri, ok := l.lockMap[resource]
|
||||
if ok {
|
||||
if reply = !isWriteLock(lri); reply {
|
||||
// Unless there is a write lock
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
)
|
||||
|
||||
type compatibilityFailedOverwrite struct {
|
||||
StorageAPI
|
||||
commit bool
|
||||
}
|
||||
|
||||
func (d compatibilityFailedOverwrite) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
|
||||
if d.commit {
|
||||
if _, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts); err != nil {
|
||||
return RenameDataResp{}, err
|
||||
}
|
||||
}
|
||||
return RenameDataResp{}, errDiskNotFound
|
||||
}
|
||||
|
||||
// Maintenance compatibility requirement: after a completely joined failed
|
||||
// overwrite with one residue, keep the previously acknowledged quorum readable.
|
||||
// This tests a known history; it does not claim the resulting snapshot proves it.
|
||||
func TestMaintenancePartialFailedOverwriteAvailability(t *testing.T) {
|
||||
// Lock RPC tests configure a single-drive backend (EC:0). Pin this
|
||||
// fixture to 12+4 so its fifteen old shards really have read quorum.
|
||||
previousStorageClass := globalStorageClass
|
||||
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 4}})
|
||||
t.Cleanup(func() { globalStorageClass.Update(previousStorageClass) })
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
dirs, err := getRandomDisks(16)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, dirs...))
|
||||
if err != nil {
|
||||
removeRoots(dirs)
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
previous := newObjectLayerFn()
|
||||
setObjectLayer(z)
|
||||
defer func() { cancel(); z.Shutdown(context.Background()); removeRoots(dirs); setObjectLayer(previous) }()
|
||||
bucket := "maintenance-compatibility"
|
||||
key := "partial-overwrite"
|
||||
if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldBody := bytes.Repeat([]byte("old-A"), 65536)
|
||||
newBody := bytes.Repeat([]byte("new-B"), 65536)
|
||||
put := func(body []byte) error {
|
||||
_, err := z.PutObject(ctx, bucket, key, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{})
|
||||
return err
|
||||
}
|
||||
if err := put(oldBody); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
set := z.serverPools[0].getHashedSet(key)
|
||||
getDisks := set.getDisks
|
||||
disks := getDisks()
|
||||
fi, err := disks[0].ReadVersion(ctx, "", bucket, key, "", ReadOptions{})
|
||||
if err != nil || fi.Erasure.DataBlocks != 12 || fi.Erasure.ParityBlocks != 4 {
|
||||
t.Fatalf("compatibility fixture must have 12+4 erasure coding: %+v, %v", fi.Erasure, err)
|
||||
}
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
wrapped[i] = compatibilityFailedOverwrite{d, i == 0}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return wrapped }
|
||||
err = put(newBody)
|
||||
set.getDisks = getDisks
|
||||
if err == nil {
|
||||
t.Fatal("partial overwrite must fail")
|
||||
}
|
||||
t.Logf("Overwrite completed with failure; one new residue, fifteen original shards: %v", err)
|
||||
r, err := z.GetObjectNInfo(ctx, bucket, key, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("MAINTENANCE COMPATIBILITY: previously readable acknowledged value now refused: %v", err)
|
||||
}
|
||||
data, err := io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil || !bytes.Equal(data, oldBody) {
|
||||
t.Fatalf("old acknowledged payload mismatch: bytes=%d err=%v", len(data), err)
|
||||
}
|
||||
}
|
||||
+132
-15
@@ -37,6 +37,10 @@ type metaCacheEntry struct {
|
||||
// Entries without metadata will only be present in non-recursive scans.
|
||||
metadata []byte
|
||||
|
||||
// listingSet marks an eligible unresolved walk entry. It is consumed by the
|
||||
// pool merger before any cache serialization or downstream filtering.
|
||||
listingSet *erasureObjects
|
||||
|
||||
// cached contains the metadata if decoded.
|
||||
cached *xlMetaV2
|
||||
|
||||
@@ -46,17 +50,17 @@ type metaCacheEntry struct {
|
||||
|
||||
// isDir returns if the entry is representing a prefix directory.
|
||||
func (e metaCacheEntry) isDir() bool {
|
||||
return len(e.metadata) == 0 && strings.HasSuffix(e.name, slashSeparator)
|
||||
return e.listingSet == nil && len(e.metadata) == 0 && strings.HasSuffix(e.name, slashSeparator)
|
||||
}
|
||||
|
||||
// isObject returns if the entry is representing an object.
|
||||
func (e metaCacheEntry) isObject() bool {
|
||||
return len(e.metadata) > 0
|
||||
return e.listingSet != nil || len(e.metadata) > 0
|
||||
}
|
||||
|
||||
// isObjectDir returns if the entry is representing an object/
|
||||
func (e metaCacheEntry) isObjectDir() bool {
|
||||
return len(e.metadata) > 0 && strings.HasSuffix(e.name, slashSeparator)
|
||||
return (e.listingSet != nil || len(e.metadata) > 0) && strings.HasSuffix(e.name, slashSeparator)
|
||||
}
|
||||
|
||||
// hasPrefix returns whether an entry has a specific prefix
|
||||
@@ -670,26 +674,82 @@ func (m *metaCacheEntriesSorted) forwardPast(s string) {
|
||||
// If the context is canceled the function will return the error,
|
||||
// otherwise the function will return nil.
|
||||
func mergeEntryChannels(ctx context.Context, in []chan metaCacheEntry, out chan<- metaCacheEntry, readQuorum int) error {
|
||||
return mergeEntryChannelsWithErrors(ctx, in, out, readQuorum, nil)
|
||||
}
|
||||
|
||||
// terminalErr is inspected after each input closes and before the merger
|
||||
// advances beyond it. Callers must publish the error before closing the input.
|
||||
func mergeEntryChannelsWithErrors(ctx context.Context, in []chan metaCacheEntry, out chan<- metaCacheEntry, readQuorum int, terminalErr func(int) error) error {
|
||||
return mergeEntryChannelsWithResolver(ctx, in, out, readQuorum, terminalErr, nil)
|
||||
}
|
||||
|
||||
// resolve is called only for groups containing a pending listing entry. All
|
||||
// selected inputs are advanced (and their terminal errors checked) before it is
|
||||
// called, so recovery never holds an object lock while waiting for input.
|
||||
func mergeEntryChannelsWithResolver(ctx context.Context, in []chan metaCacheEntry, out chan<- metaCacheEntry, readQuorum int, terminalErr func(int) error, resolve func([]*metaCacheEntry) (*metaCacheEntry, error)) error {
|
||||
defer xioutil.SafeClose(out)
|
||||
top := make([]*metaCacheEntry, len(in))
|
||||
nDone := 0
|
||||
ctxDone := ctx.Done()
|
||||
|
||||
// Use simpler forwarder.
|
||||
// Keep the single-input normal path allocation-free. Only a pending entry
|
||||
// requires looking ahead to observe its stream's terminal error first.
|
||||
if len(in) == 1 {
|
||||
var buffered metaCacheEntry
|
||||
haveBuffered, exhausted := false, false
|
||||
for {
|
||||
select {
|
||||
case <-ctxDone:
|
||||
return ctx.Err()
|
||||
case v, ok := <-in[0]:
|
||||
if !ok {
|
||||
return nil
|
||||
var v metaCacheEntry
|
||||
if haveBuffered {
|
||||
v, haveBuffered = buffered, false
|
||||
} else {
|
||||
select {
|
||||
case <-ctxDone:
|
||||
return ctx.Err()
|
||||
case entry, ok := <-in[0]:
|
||||
if !ok {
|
||||
if terminalErr != nil {
|
||||
return terminalErr(0)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
v = entry
|
||||
}
|
||||
}
|
||||
if v.listingSet != nil {
|
||||
if resolve == nil {
|
||||
return errInvalidArgument
|
||||
}
|
||||
select {
|
||||
case <-ctxDone:
|
||||
return ctx.Err()
|
||||
case out <- v:
|
||||
case entry, ok := <-in[0]:
|
||||
buffered, haveBuffered, exhausted = entry, ok, !ok
|
||||
if !ok && terminalErr != nil {
|
||||
if err := terminalErr(0); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
pending := v
|
||||
resolved, err := resolve([]*metaCacheEntry{&pending})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resolved == nil {
|
||||
if exhausted {
|
||||
return nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
v = *resolved
|
||||
}
|
||||
select {
|
||||
case <-ctxDone:
|
||||
return ctx.Err()
|
||||
case out <- v:
|
||||
}
|
||||
if exhausted {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -700,6 +760,11 @@ func mergeEntryChannels(ctx context.Context, in []chan metaCacheEntry, out chan<
|
||||
return ctx.Err()
|
||||
case entry, ok := <-in[idx]:
|
||||
if !ok {
|
||||
if terminalErr != nil {
|
||||
if err := terminalErr(idx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
top[idx] = nil
|
||||
nDone++
|
||||
} else {
|
||||
@@ -716,6 +781,15 @@ func mergeEntryChannels(ctx context.Context, in []chan metaCacheEntry, out chan<
|
||||
}
|
||||
last := ""
|
||||
var toMerge []int
|
||||
var directoryCollisions []int
|
||||
discardDirectories := func() error {
|
||||
for _, idx := range directoryCollisions {
|
||||
if err := selectFrom(idx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Choose the best to return.
|
||||
for {
|
||||
@@ -725,6 +799,7 @@ func mergeEntryChannels(ctx context.Context, in []chan metaCacheEntry, out chan<
|
||||
best := top[0]
|
||||
bestIdx := 0
|
||||
toMerge = toMerge[:0]
|
||||
directoryCollisions = directoryCollisions[:0]
|
||||
for i, other := range top[1:] {
|
||||
otherIdx := i + 1
|
||||
if other == nil {
|
||||
@@ -754,10 +829,9 @@ func mergeEntryChannels(ctx context.Context, in []chan metaCacheEntry, out chan<
|
||||
if serverDebugLog {
|
||||
console.Debugln("mergeEntryChannels: discarding directory", other.name, "for object", best.name)
|
||||
}
|
||||
// Discard the directory.
|
||||
if err := selectFrom(otherIdx); err != nil {
|
||||
return err
|
||||
}
|
||||
// A pending object can resolve to absence. Keep the
|
||||
// directory until the whole object group is confirmed.
|
||||
directoryCollisions = append(directoryCollisions, otherIdx)
|
||||
continue
|
||||
}
|
||||
// Replace directory with object.
|
||||
@@ -773,11 +847,54 @@ func mergeEntryChannels(ctx context.Context, in []chan metaCacheEntry, out chan<
|
||||
}
|
||||
if best.name > other.name {
|
||||
toMerge = toMerge[:0]
|
||||
directoryCollisions = directoryCollisions[:0]
|
||||
best = other
|
||||
bestIdx = otherIdx
|
||||
}
|
||||
}
|
||||
|
||||
pending := best.listingSet != nil
|
||||
for _, idx := range toMerge {
|
||||
pending = pending || top[idx].listingSet != nil
|
||||
}
|
||||
if pending {
|
||||
if resolve == nil {
|
||||
return errInvalidArgument
|
||||
}
|
||||
group := make([]*metaCacheEntry, 1, len(toMerge)+1)
|
||||
group[0] = best
|
||||
for _, idx := range toMerge {
|
||||
group = append(group, top[idx])
|
||||
if err := selectFrom(idx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := selectFrom(bestIdx); err != nil {
|
||||
return err
|
||||
}
|
||||
resolved, err := resolve(group)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resolved != nil {
|
||||
if err := discardDirectories(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if resolved != nil && resolved.name > last {
|
||||
select {
|
||||
case <-ctxDone:
|
||||
return ctx.Err()
|
||||
case out <- *resolved:
|
||||
last = resolved.name
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := discardDirectories(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Merge any unmerged
|
||||
if len(toMerge) > 0 {
|
||||
versions := make([][]xlMetaV2ShallowVersion, 0, len(toMerge)+1)
|
||||
|
||||
@@ -0,0 +1,403 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"slices"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/grid"
|
||||
)
|
||||
|
||||
type listingReadFailDisk struct {
|
||||
StorageAPI
|
||||
object string
|
||||
reads *atomic.Int64
|
||||
}
|
||||
|
||||
func (d *listingReadFailDisk) ReadXL(ctx context.Context, bucket, object string, readData bool) (RawFileInfo, error) {
|
||||
if object == d.object {
|
||||
d.reads.Add(1)
|
||||
return RawFileInfo{}, errDiskNotFound
|
||||
}
|
||||
return d.StorageAPI.ReadXL(ctx, bucket, object, readData)
|
||||
}
|
||||
|
||||
// Install before issuing requests: cache writers can outlive their HTTP
|
||||
// response, so disk faults must be switched without replacing getDisks.
|
||||
// Each published slice must remain immutable.
|
||||
func installListingDiskSwitch(set *erasureObjects) ([]StorageAPI, func([]StorageAPI)) {
|
||||
disks := set.getDisks()
|
||||
var current atomic.Value
|
||||
current.Store(disks)
|
||||
set.getDisks = func() []StorageAPI { return current.Load().([]StorageAPI) }
|
||||
return disks, func(disks []StorageAPI) { current.Store(disks) }
|
||||
}
|
||||
|
||||
func TestListingCachePreservesFallbackError(t *testing.T) {
|
||||
for _, continuation := range []bool{false, true} {
|
||||
name := "cold"
|
||||
if continuation {
|
||||
name = "continuation"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testListingCacheFallbackError(t, continuation, false)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestListingCacheCleanupKeepsOriginalID(t *testing.T) {
|
||||
testListingCacheFallbackError(t, true, true)
|
||||
}
|
||||
|
||||
// Use actual peer RPCs to verify the abandoned cache is updated after the
|
||||
// caller clears its own listing ID. The local-only path cannot observe this.
|
||||
func recordListingPeerCleanup(t *testing.T) func() {
|
||||
t.Helper()
|
||||
tg, err := grid.SetupTestGrid(2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previous := globalNotificationSys
|
||||
t.Cleanup(func() {
|
||||
tg.Cleanup()
|
||||
globalNotificationSys = previous
|
||||
})
|
||||
getIDs := make(chan string, 8)
|
||||
type cleanupResult struct {
|
||||
id string
|
||||
err *grid.RemoteErr
|
||||
}
|
||||
updated := make(chan cleanupResult, 1)
|
||||
server := &peerRESTServer{}
|
||||
if err := getMetacacheListingRPC.Register(tg.Managers[1], func(o *listPathOptions) (*metacache, *grid.RemoteErr) {
|
||||
getIDs <- o.ID
|
||||
return server.GetMetacacheListingHandler(o)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := updateMetacacheListingRPC.Register(tg.Managers[1], func(m *metacache) (*metacache, *grid.RemoteErr) {
|
||||
result, err := server.UpdateMetacacheListingHandler(m)
|
||||
if m.error == "no longer used" {
|
||||
updated <- cleanupResult{id: m.id, err: err}
|
||||
}
|
||||
return result, err
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &peerRESTClient{gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[1]) }}
|
||||
globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{client}, allPeerClients: []*peerRESTClient{client}}
|
||||
return func() {
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
var ids []string
|
||||
for range 2 {
|
||||
select {
|
||||
case id := <-getIDs:
|
||||
ids = append(ids, id)
|
||||
case <-ctx.Done():
|
||||
t.Fatal("peer did not receive the failed cache cleanup lookup")
|
||||
}
|
||||
}
|
||||
if ids[0] == "" || ids[1] != ids[0] {
|
||||
t.Fatalf("cleanup lookup lost the original cache ID: %q", ids)
|
||||
}
|
||||
select {
|
||||
case result := <-updated:
|
||||
if result.err != nil || result.id != ids[0] {
|
||||
t.Fatalf("cleanup update: id=%q err=%v, want id=%q", result.id, result.err, ids[0])
|
||||
}
|
||||
t.Logf("peer cleanup Get/Update retained cache ID %q", result.id)
|
||||
case <-ctx.Done():
|
||||
t.Fatal("peer did not finish the failed cache cleanup update")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testListingCacheFallbackError(t *testing.T, continuation, peer bool) {
|
||||
t.Helper()
|
||||
z, bucket, router := nullQuorumBackend(t)
|
||||
var waitPeerCleanup func()
|
||||
if peer {
|
||||
waitPeerCleanup = recordListingPeerCleanup(t)
|
||||
}
|
||||
set := z.serverPools[0].sets[0]
|
||||
disks, switchDisks := installListingDiskSwitch(set)
|
||||
body := bytes.Repeat([]byte{'a'}, 8192)
|
||||
mtime := time.Now().UTC().Add(-time.Hour)
|
||||
for _, object := range []string{"a", "b", "z"} {
|
||||
nullQuorumPut(t, z, bucket, object, body, ObjectOptions{MTime: mtime})
|
||||
}
|
||||
old := make([][]byte, 3)
|
||||
for i := range old {
|
||||
old[i] = mustReadNullQuorumMeta(t, disks[i], bucket, "z")
|
||||
}
|
||||
nullQuorumPut(t, z, bucket, "z", body, ObjectOptions{MTime: mtime.Add(time.Minute)})
|
||||
var token string
|
||||
if continuation {
|
||||
rec := nullQuorumRequest(t, router, http.MethodGet, getListObjectsV2URL("", bucket, "", "1", "", "", ""))
|
||||
var list ListObjectsV2Response
|
||||
if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &list) != nil || !list.IsTruncated || list.NextContinuationToken == "" {
|
||||
t.Fatalf("first page: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
token = list.NextContinuationToken
|
||||
}
|
||||
for i := range old {
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, "z/"+xlStorageFormatFile, old[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var reads atomic.Int64
|
||||
walkers := slices.Clone(disks)
|
||||
for i := range walkers {
|
||||
walkers[i] = &listingReadFailDisk{StorageAPI: disks[i], object: "z", reads: &reads}
|
||||
if i >= 8 {
|
||||
walkers[i] = &failedMetadataWalkDisk{StorageAPI: walkers[i]}
|
||||
}
|
||||
}
|
||||
switchDisks(walkers)
|
||||
target := getListObjectsV2URL("", bucket, "", "1000", "", "", "")
|
||||
if token != "" {
|
||||
target += "&continuation-token=" + url.QueryEscape(token)
|
||||
}
|
||||
rec := nullQuorumRequest(t, router, http.MethodGet, target)
|
||||
t.Logf("LIST=%d failed metadata reads=%d body=%s", rec.Code, reads.Load(), rec.Body.String())
|
||||
if reads.Load() == 0 {
|
||||
t.Fatal("fallback not exercised")
|
||||
}
|
||||
var apiErr APIErrorResponse
|
||||
if rec.Code != http.StatusServiceUnavailable || xml.Unmarshal(rec.Body.Bytes(), &apiErr) != nil || apiErr.Code != "SlowDownRead" {
|
||||
t.Fatalf("fallback failure lost its read-quorum error: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if waitPeerCleanup != nil {
|
||||
waitPeerCleanup()
|
||||
}
|
||||
if continuation {
|
||||
caches, _ := localMetacacheMgr.getBucket(t.Context(), bucket).cloneCaches()
|
||||
if len(caches) == 0 {
|
||||
t.Fatal("continuation did not create a cache")
|
||||
}
|
||||
for _, cache := range caches {
|
||||
if cache.status != scanStateError {
|
||||
t.Fatalf("incomplete cache was not marked failed: %+v", cache)
|
||||
}
|
||||
}
|
||||
}
|
||||
switchDisks(disks)
|
||||
retry := nullQuorumRequest(t, router, http.MethodGet, target)
|
||||
var recovered ListObjectsV2Response
|
||||
if retry.Code != http.StatusOK || xml.Unmarshal(retry.Body.Bytes(), &recovered) != nil {
|
||||
t.Fatalf("retry after metadata recovers: %d %s", retry.Code, retry.Body.String())
|
||||
}
|
||||
var keys []string
|
||||
for _, entry := range recovered.Contents {
|
||||
keys = append(keys, entry.Key)
|
||||
}
|
||||
want := []string{"a", "b", "z"}
|
||||
if continuation {
|
||||
want = []string{"b", "z"}
|
||||
}
|
||||
if !slices.Equal(keys, want) || recovered.IsTruncated {
|
||||
t.Fatalf("retry returned incomplete result: %+v", recovered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListingCachePagination(t *testing.T) {
|
||||
z, bucket, router := nullQuorumBackend(t)
|
||||
for _, key := range []string{"a", "b", "c", "d"} {
|
||||
nullQuorumPut(t, z, bucket, key, []byte(key), ObjectOptions{})
|
||||
}
|
||||
var keys []string
|
||||
var token string
|
||||
var finished bool
|
||||
for page := 0; page < 8; page++ {
|
||||
target := getListObjectsV2URL("", bucket, "", "1", "", "", "")
|
||||
if token != "" {
|
||||
target += "&continuation-token=" + url.QueryEscape(token)
|
||||
}
|
||||
rec := nullQuorumRequest(t, router, http.MethodGet, target)
|
||||
var list ListObjectsV2Response
|
||||
if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &list) != nil {
|
||||
t.Fatalf("page %d: %d %s", page, rec.Code, rec.Body.String())
|
||||
}
|
||||
for _, entry := range list.Contents {
|
||||
keys = append(keys, entry.Key)
|
||||
}
|
||||
if !list.IsTruncated {
|
||||
finished = true
|
||||
break
|
||||
}
|
||||
if list.NextContinuationToken == "" || list.NextContinuationToken == token {
|
||||
t.Fatal("pagination did not advance")
|
||||
}
|
||||
token = list.NextContinuationToken
|
||||
}
|
||||
if !finished || !slices.Equal(keys, []string{"a", "b", "c", "d"}) {
|
||||
t.Fatalf("pagination returned %v, finished=%v", keys, finished)
|
||||
}
|
||||
}
|
||||
|
||||
// A cached full page must remain streamable even if a later key is slow. The
|
||||
// completion barrier belongs at EOF, not before every page response.
|
||||
func TestListingCacheFullPageDoesNotWaitForTail(t *testing.T) {
|
||||
z, bucket, router := nullQuorumBackend(t)
|
||||
disks, switchDisks := installListingDiskSwitch(z.serverPools[0].sets[0])
|
||||
for _, key := range []string{"a", "b", "c", "d", "e", "zz"} {
|
||||
nullQuorumPut(t, z, bucket, key, []byte(key), ObjectOptions{})
|
||||
}
|
||||
first := nullQuorumRequest(t, router, http.MethodGet, getListObjectsV2URL("", bucket, "", "1", "", "", ""))
|
||||
var page ListObjectsV2Response
|
||||
if first.Code != http.StatusOK || xml.Unmarshal(first.Body.Bytes(), &page) != nil || !page.IsTruncated {
|
||||
t.Fatalf("first page: %d %s", first.Code, first.Body.String())
|
||||
}
|
||||
resume := make(chan struct{})
|
||||
release := sync.OnceFunc(func() { close(resume) })
|
||||
defer release()
|
||||
var blocked, active atomic.Int64
|
||||
walkers := slices.Clone(disks)
|
||||
for i, disk := range walkers {
|
||||
walkers[i] = &listingPausedTailDisk{StorageAPI: disk, bucket: bucket, tail: "zz", resume: resume, blocked: &blocked, active: &active}
|
||||
}
|
||||
switchDisks(walkers)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
target := getListObjectsV2URL("", bucket, "", "1", "", "", "") + "&continuation-token=" + url.QueryEscape(page.NextContinuationToken)
|
||||
req, err := newTestSignedRequestV4(http.MethodGet, target, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response := make(chan *httptest.ResponseRecorder, 1)
|
||||
go func() {
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req.WithContext(ctx))
|
||||
response <- rec
|
||||
}()
|
||||
select {
|
||||
case rec := <-response:
|
||||
page = ListObjectsV2Response{}
|
||||
if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &page) != nil || !page.IsTruncated || len(page.Contents) != 1 || page.Contents[0].Key != "b" || page.NextContinuationToken == "" {
|
||||
t.Fatalf("full page before tail: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if blocked.Load() == 0 || active.Load() == 0 {
|
||||
t.Fatal("full page did not overlap the paused tail")
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("full page waited for the paused tail")
|
||||
}
|
||||
release()
|
||||
waitForListingCondition(t, "disk walkers did not finish", func() bool { return active.Load() == 0 })
|
||||
target = getListObjectsV2URL("", bucket, "", "1000", "", "", "") + "&continuation-token=" + url.QueryEscape(page.NextContinuationToken)
|
||||
rec := nullQuorumRequest(t, router, http.MethodGet, target)
|
||||
page = ListObjectsV2Response{}
|
||||
if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &page) != nil || page.IsTruncated {
|
||||
t.Fatalf("page after tail resumes: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var keys []string
|
||||
for _, entry := range page.Contents {
|
||||
keys = append(keys, entry.Key)
|
||||
}
|
||||
if !slices.Equal(keys, []string{"c", "d", "e", "zz"}) {
|
||||
t.Fatalf("cached remainder lost keys: %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the error update pending at the actual peer RPC boundary. Other cache
|
||||
// operations, including readers and abandoned-cache cleanup, remain live.
|
||||
func listingCacheErrorPeer(t *testing.T, fail bool) (*peerRESTClient, <-chan struct{}, func()) {
|
||||
t.Helper()
|
||||
tg, err := grid.SetupTestGrid(2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ready, resume := make(chan struct{}), make(chan struct{})
|
||||
release := sync.OnceFunc(func() { close(resume) })
|
||||
t.Cleanup(func() {
|
||||
release()
|
||||
tg.Cleanup()
|
||||
})
|
||||
var entered sync.Once
|
||||
server := &peerRESTServer{}
|
||||
if err := getMetacacheListingRPC.Register(tg.Managers[1], server.GetMetacacheListingHandler); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := updateMetacacheListingRPC.Register(tg.Managers[1], func(m *metacache) (*metacache, *grid.RemoteErr) {
|
||||
if m.error != "" && m.error != "no longer used" {
|
||||
entered.Do(func() { close(ready) })
|
||||
select {
|
||||
case <-resume:
|
||||
case <-t.Context().Done():
|
||||
return nil, grid.NewRemoteErr(t.Context().Err())
|
||||
}
|
||||
if fail {
|
||||
return nil, grid.NewRemoteErr(errors.New("cache error update failed"))
|
||||
}
|
||||
}
|
||||
return server.UpdateMetacacheListingHandler(m)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &peerRESTClient{gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[1]) }}, ready, release
|
||||
}
|
||||
|
||||
func TestListingCacheRetainsErrorAfterPeerFailure(t *testing.T) {
|
||||
client, ready, release := listingCacheErrorPeer(t, true)
|
||||
o := listPathOptions{ID: mustGetUUID(), Bucket: "bucket"}
|
||||
m := o.newMetacache()
|
||||
mc := metaCacheRPC{meta: &m, rpc: client, o: o}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
mc.setErr("terminal listing failure")
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
release()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Error("setErr worker did not finish before grid cleanup")
|
||||
}
|
||||
})
|
||||
select {
|
||||
case <-ready:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("error update did not reach peer")
|
||||
}
|
||||
release()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("failed peer update did not return")
|
||||
}
|
||||
if m.id != o.ID || m.bucket != o.Bucket || m.status != scanStateError || m.error != "terminal listing failure" || mc.getErr() == nil {
|
||||
t.Fatalf("peer failure erased local failure: %+v", m)
|
||||
}
|
||||
mc.setErr("later failure")
|
||||
if m.error != "terminal listing failure" {
|
||||
t.Fatalf("later failure replaced the terminal error: %+v", m)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||
// maintenance release gate; failures record intentionally unresolved risks.
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestStrictListFallbackRejectsSplitGenerations(t *testing.T) {
|
||||
z, bucket, _ := nullQuorumBackend(t)
|
||||
set := z.serverPools[0].sets[0]
|
||||
disks := set.getDisks()
|
||||
const object = "object"
|
||||
mtime := time.Now().UTC().Add(-time.Hour)
|
||||
var generations [3][][]byte
|
||||
for generation := range generations {
|
||||
body := bytes.Repeat([]byte{byte('a' + generation)}, 8192)
|
||||
nullQuorumPut(t, z, bucket, object, body, ObjectOptions{MTime: mtime.Add(time.Duration(generation) * time.Minute)})
|
||||
generations[generation] = make([][]byte, len(disks))
|
||||
for i, disk := range disks {
|
||||
generations[generation][i] = mustReadNullQuorumMeta(t, disk, bucket, object)
|
||||
}
|
||||
}
|
||||
// All sixteen drives are online, but the 7/5/4 split leaves no generation
|
||||
// with the eight matching snapshots required by the listing resolver.
|
||||
for i, disk := range disks {
|
||||
generation := 0
|
||||
if i >= 7 {
|
||||
generation = 1
|
||||
}
|
||||
if i >= 12 {
|
||||
generation = 2
|
||||
}
|
||||
if err := disk.WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, generations[generation][i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, _, _, err := set.getObjectFileInfo(t.Context(), bucket, object, ObjectOptions{}, false); !isErrReadQuorum(err) {
|
||||
t.Fatalf("split metadata must remain a read-quorum failure: %v", err)
|
||||
}
|
||||
entries := make(metaCacheEntries, len(disks))
|
||||
for i, disk := range disks {
|
||||
entries[i] = metaCacheEntry{name: object, metadata: mustReadNullQuorumMeta(t, disk, bucket, object)}
|
||||
if _, err := entries[i].xlmeta(); err != nil {
|
||||
t.Fatalf("drive %d no longer has valid metadata: %v", i, err)
|
||||
}
|
||||
}
|
||||
resolver := metadataResolutionParams{dirQuorum: 8, objQuorum: 8, bucket: bucket, requestedVersions: 1}
|
||||
entry, err := z.resolveListingGroup(t.Context(), bucket, []*metaCacheEntry{set.resolveListEntry(entries, nil, &resolver)})
|
||||
if entry != nil || !isErrReadQuorum(err) {
|
||||
t.Fatalf("split live metadata became absence: entry=%v err=%v", entry, err)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user