mirror of
https://github.com/pgsty/minio.git
synced 2026-11-12 10:35:33 +03:00
Compare commits
66 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 44e848fedf | |||
| 52371b906c | |||
| 43122f76f1 | |||
| c83dbe0eaa | |||
| 72a6047bf2 | |||
| 939cb26001 | |||
| ea37a7e8b0 | |||
| 265c31b714 | |||
| 2ed720a7fb | |||
| c39605d4f8 | |||
| 7c0a99284f | |||
| 34ba7cc1ff | |||
| bae0ba0b75 | |||
| cd3e46a2b7 | |||
| ef6efd3120 | |||
| 4a9bdb5407 | |||
| 2f228284f3 | |||
| e36ea51d9c | |||
| 3ce07a943b | |||
| 3c35cebe9e | |||
| ae483baca1 | |||
| 99fb47fbb9 | |||
| 467ce760ed | |||
| b5d97e3e1f | |||
| a8824b341f | |||
| f746d07076 | |||
| 745fbf0437 | |||
| 8587e061bb | |||
| 293ab78b4c | |||
| 67a90dfef4 | |||
| 62b8784428 | |||
| 5dc4bd6a1d | |||
| cf486df661 | |||
| 1d714834c5 | |||
| 14ecb222e7 | |||
| e51bff35a3 | |||
| 20b3bfb6e7 | |||
| 322cd8d2c1 | |||
| c3340ec7f5 | |||
| 7c91dc17b2 | |||
| 2f93e0c6b3 | |||
| 65acda0114 | |||
| c6949a38c2 | |||
| 80dd3e3ce1 | |||
| 89028732ab | |||
| 55bd906d30 | |||
| 296c9710b1 | |||
| f3b42b0eb8 | |||
| da8d8a54b1 | |||
| 2b62df876a | |||
| 518915d241 | |||
| ce5d191077 | |||
| b404f78764 | |||
| d138996983 | |||
| d52ba0290a | |||
| 8f3657dc52 | |||
| 7efdedd5ba | |||
| 63650a31a1 | |||
| 6a65b43076 | |||
| 3269d2214e | |||
| b3e69bca2f | |||
| 2c06b28dbb | |||
| a2ff3c2ecd | |||
| 67bc32f401 | |||
| 3c4a6296f9 | |||
| afa35d16e9 |
@@ -1,6 +1,9 @@
|
|||||||
name: DCO
|
name: DCO
|
||||||
|
|
||||||
on:
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
@@ -17,18 +20,29 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
|
||||||
# Every non-merge commit in the pull request must carry a Signed-off-by
|
# Every non-merge commit in the pull request or push must carry a Signed-off-by
|
||||||
# trailer matching the commit author's email, certifying the Developer
|
# trailer matching the commit author's email, certifying the Developer
|
||||||
# Certificate of Origin 1.1 (https://developercertificate.org/).
|
# Certificate of Origin 1.1 (https://developercertificate.org/).
|
||||||
# Only commits authored from a GitHub-issued bot address are exempt; a
|
# Only commits authored from a GitHub-issued bot address are exempt; a
|
||||||
# display name is attacker-controlled and must never grant the exemption.
|
# display name is attacker-controlled and must never grant the exemption.
|
||||||
- name: Check Signed-off-by trailers
|
- name: Check Signed-off-by trailers
|
||||||
env:
|
env:
|
||||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
EVENT_NAME: ${{ github.event_name }}
|
||||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||||
|
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||||
|
PUSH_BEFORE_SHA: ${{ github.event.before }}
|
||||||
|
PUSH_AFTER_SHA: ${{ github.sha }}
|
||||||
run: |
|
run: |
|
||||||
|
if [ "${EVENT_NAME}" = "pull_request" ]; then
|
||||||
|
rev_range="${PR_BASE_SHA}..${PR_HEAD_SHA}"
|
||||||
|
elif [ -n "${PUSH_BEFORE_SHA}" ] && [ "${PUSH_BEFORE_SHA}" != "0000000000000000000000000000000000000000" ] && git cat-file -e "${PUSH_BEFORE_SHA}^{commit}" 2>/dev/null; then
|
||||||
|
rev_range="${PUSH_BEFORE_SHA}..${PUSH_AFTER_SHA}"
|
||||||
|
else
|
||||||
|
rev_range="${PUSH_AFTER_SHA}^..${PUSH_AFTER_SHA}"
|
||||||
|
fi
|
||||||
fail=0
|
fail=0
|
||||||
while read -r sha; do
|
while read -r sha; do
|
||||||
|
[ -n "${sha}" ] || continue
|
||||||
author_name="$(git log -1 --format='%an' "${sha}")"
|
author_name="$(git log -1 --format='%an' "${sha}")"
|
||||||
author_email="$(git log -1 --format='%ae' "${sha}")"
|
author_email="$(git log -1 --format='%ae' "${sha}")"
|
||||||
case "${author_email}" in
|
case "${author_email}" in
|
||||||
@@ -39,5 +53,5 @@ jobs:
|
|||||||
echo "::error::commit ${sha} by ${author_name} <${author_email}> lacks a matching Signed-off-by trailer; sign with 'git commit -s', repair with 'git rebase --signoff'"
|
echo "::error::commit ${sha} by ${author_name} <${author_email}> lacks a matching Signed-off-by trailer; sign with 'git commit -s', repair with 'git rebase --signoff'"
|
||||||
fail=1
|
fail=1
|
||||||
fi
|
fi
|
||||||
done < <(git rev-list --no-merges "${BASE_SHA}..${HEAD_SHA}")
|
done < <(git rev-list --no-merges "${rev_range}")
|
||||||
exit "${fail}"
|
exit "${fail}"
|
||||||
|
|||||||
@@ -35,7 +35,8 @@ jobs:
|
|||||||
- name: Check gofmt
|
- name: Check gofmt
|
||||||
run: |
|
run: |
|
||||||
mapfile -t unformatted < <(gofmt -l main.go cmd internal \
|
mapfile -t unformatted < <(gofmt -l main.go cmd internal \
|
||||||
buildscripts/rebrand-guard buildscripts/helm-migration-guard)
|
buildscripts/rebrand-guard buildscripts/helm-migration-guard \
|
||||||
|
buildscripts/release-dependency-guard)
|
||||||
if [ "${#unformatted[@]}" -ne 0 ]; then
|
if [ "${#unformatted[@]}" -ne 0 ]; then
|
||||||
echo "The following files are not gofmt-formatted:"
|
echo "The following files are not gofmt-formatted:"
|
||||||
printf '%s\n' "${unformatted[@]}"
|
printf '%s\n' "${unformatted[@]}"
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ permissions:
|
|||||||
id-token: write
|
id-token: write
|
||||||
attestations: write
|
attestations: write
|
||||||
artifact-metadata: write
|
artifact-metadata: write
|
||||||
|
actions: read
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: release-${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}
|
group: release-${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}
|
||||||
@@ -102,6 +103,12 @@ jobs:
|
|||||||
echo "Release tag ${TAG} resolves to ${TAG_COMMIT}, checkout is ${HEAD_COMMIT}" >&2
|
echo "Release tag ${TAG} resolves to ${TAG_COMMIT}, checkout is ${HEAD_COMMIT}" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
CHART_APP_VERSION="$(sed -n 's/^appVersion: *//p' helm/silo/Chart.yaml)"
|
||||||
|
CHART_IMAGE_TAG="$(sed -n '/^image:/,/^[^ ]/s/^ tag: *//p' helm/silo/values.yaml)"
|
||||||
|
if [ "${CHART_APP_VERSION}" != "${TAG}" ] || [ "${CHART_IMAGE_TAG}" != "${TAG}" ]; then
|
||||||
|
echo "helm/silo Chart.yaml appVersion (${CHART_APP_VERSION}) and values.yaml image.tag (${CHART_IMAGE_TAG}) must match release tag ${TAG}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
VERSION_HYPHEN="${TAG#RELEASE.}"
|
VERSION_HYPHEN="${TAG#RELEASE.}"
|
||||||
PKG_VERSION="$(echo "${VERSION_HYPHEN}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')"
|
PKG_VERSION="$(echo "${VERSION_HYPHEN}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')"
|
||||||
VERSION_COLON="$(echo "${VERSION_HYPHEN}" | sed -E 's/T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/T\1:\2:\3Z/')"
|
VERSION_COLON="$(echo "${VERSION_HYPHEN}" | sed -E 's/T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/T\1:\2:\3Z/')"
|
||||||
@@ -109,14 +116,29 @@ jobs:
|
|||||||
|
|
||||||
{
|
{
|
||||||
echo "RELEASE_TAG=${TAG}"
|
echo "RELEASE_TAG=${TAG}"
|
||||||
|
echo "RELEASE_COMMIT=${HEAD_COMMIT}"
|
||||||
echo "PKG_VERSION=${PKG_VERSION}"
|
echo "PKG_VERSION=${PKG_VERSION}"
|
||||||
echo "LDFLAGS=${LDFLAGS}"
|
echo "LDFLAGS=${LDFLAGS}"
|
||||||
} >> "${GITHUB_ENV}"
|
} >> "${GITHUB_ENV}"
|
||||||
|
|
||||||
echo "Release tag: ${TAG}"
|
echo "Release tag: ${TAG}"
|
||||||
|
echo "Release commit: ${HEAD_COMMIT}"
|
||||||
echo "Package version: ${PKG_VERSION}"
|
echo "Package version: ${PKG_VERSION}"
|
||||||
echo "LDFLAGS: ${LDFLAGS}"
|
echo "LDFLAGS: ${LDFLAGS}"
|
||||||
|
|
||||||
|
- name: Check release commit is the tip of main with green CI
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
buildscripts/check-release-commit.sh "${RELEASE_COMMIT}"
|
||||||
|
|
||||||
|
- name: Verify coordinated release dependencies
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
GOWORK: "off"
|
||||||
|
run: go run ./buildscripts/release-dependency-guard
|
||||||
|
|
||||||
- name: Check existing release state
|
- name: Check existing release state
|
||||||
env:
|
env:
|
||||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
@@ -140,7 +162,7 @@ jobs:
|
|||||||
- name: Build Draft release with GoReleaser
|
- name: Build Draft release with GoReleaser
|
||||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
|
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
|
||||||
with:
|
with:
|
||||||
version: v2.18.3
|
version: v2.20.1
|
||||||
args: release --clean --skip=validate --config .github/goreleaser.yml
|
args: release --clean --skip=validate --config .github/goreleaser.yml
|
||||||
env:
|
env:
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|||||||
@@ -25,6 +25,9 @@ on:
|
|||||||
- "buildscripts/minio-upgrade.sh"
|
- "buildscripts/minio-upgrade.sh"
|
||||||
- "buildscripts/sign-release-rpms.sh"
|
- "buildscripts/sign-release-rpms.sh"
|
||||||
- "buildscripts/verify-build-provenance.sh"
|
- "buildscripts/verify-build-provenance.sh"
|
||||||
|
- "buildscripts/check-release-commit.sh"
|
||||||
|
- "buildscripts/check-release-commit_test.sh"
|
||||||
|
- "buildscripts/release-dependency-guard/**"
|
||||||
- "buildscripts/check-release-state.sh"
|
- "buildscripts/check-release-state.sh"
|
||||||
- "buildscripts/check-release-state_test.sh"
|
- "buildscripts/check-release-state_test.sh"
|
||||||
- "buildscripts/verify-rebrand.sh"
|
- "buildscripts/verify-rebrand.sh"
|
||||||
@@ -97,10 +100,14 @@ jobs:
|
|||||||
- name: GoReleaser config check
|
- name: GoReleaser config check
|
||||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
|
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
|
||||||
with:
|
with:
|
||||||
version: v2.18.3
|
version: v2.20.1
|
||||||
args: check --config .github/goreleaser.yml
|
args: check --config .github/goreleaser.yml
|
||||||
|
|
||||||
- name: Validate Helm chart and legacy upgrade identity
|
- name: Validate Helm chart and legacy upgrade identity
|
||||||
|
env:
|
||||||
|
# Compute test variables sets a synthetic GoReleaser fixture tag;
|
||||||
|
# real tag equality is enforced in release.yml.
|
||||||
|
RELEASE_TAG: ""
|
||||||
run: buildscripts/verify-helm-migration.sh
|
run: buildscripts/verify-helm-migration.sh
|
||||||
|
|
||||||
- name: Install Syft
|
- name: Install Syft
|
||||||
@@ -111,7 +118,7 @@ jobs:
|
|||||||
- name: Build snapshot artifacts
|
- name: Build snapshot artifacts
|
||||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
|
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
|
||||||
with:
|
with:
|
||||||
version: v2.18.3
|
version: v2.20.1
|
||||||
# A pull-request snapshot has no trusted release identity. Exercise
|
# A pull-request snapshot has no trusted release identity. Exercise
|
||||||
# the SBOM/checksum pipeline here, and reserve keyless signing for
|
# the SBOM/checksum pipeline here, and reserve keyless signing for
|
||||||
# the tag-triggered release workflow with GitHub OIDC.
|
# the tag-triggered release workflow with GitHub OIDC.
|
||||||
@@ -537,6 +544,8 @@ jobs:
|
|||||||
bash -n buildscripts/minio-upgrade.sh
|
bash -n buildscripts/minio-upgrade.sh
|
||||||
bash -n buildscripts/sign-release-rpms.sh
|
bash -n buildscripts/sign-release-rpms.sh
|
||||||
bash -n buildscripts/verify-build-provenance.sh
|
bash -n buildscripts/verify-build-provenance.sh
|
||||||
|
bash -n buildscripts/check-release-commit.sh
|
||||||
|
bash -n buildscripts/check-release-commit_test.sh
|
||||||
bash -n buildscripts/check-release-state.sh
|
bash -n buildscripts/check-release-state.sh
|
||||||
bash -n buildscripts/check-release-state_test.sh
|
bash -n buildscripts/check-release-state_test.sh
|
||||||
bash -n buildscripts/verify-rebrand.sh
|
bash -n buildscripts/verify-rebrand.sh
|
||||||
@@ -553,6 +562,8 @@ jobs:
|
|||||||
test -x buildscripts/package-release.sh
|
test -x buildscripts/package-release.sh
|
||||||
test -x buildscripts/sign-release-rpms.sh
|
test -x buildscripts/sign-release-rpms.sh
|
||||||
test -x buildscripts/verify-build-provenance.sh
|
test -x buildscripts/verify-build-provenance.sh
|
||||||
|
test -x buildscripts/check-release-commit.sh
|
||||||
|
test -x buildscripts/check-release-commit_test.sh
|
||||||
test -x buildscripts/check-release-state.sh
|
test -x buildscripts/check-release-state.sh
|
||||||
test -x buildscripts/check-release-state_test.sh
|
test -x buildscripts/check-release-state_test.sh
|
||||||
test -x buildscripts/verify-rebrand.sh
|
test -x buildscripts/verify-rebrand.sh
|
||||||
@@ -561,4 +572,6 @@ jobs:
|
|||||||
test -x buildscripts/package/preremove.sh
|
test -x buildscripts/package/preremove.sh
|
||||||
test -x buildscripts/package/lifecycle_test.sh
|
test -x buildscripts/package/lifecycle_test.sh
|
||||||
test -x dockerscripts/docker-entrypoint_test.sh
|
test -x dockerscripts/docker-entrypoint_test.sh
|
||||||
|
buildscripts/check-release-commit_test.sh
|
||||||
|
GOWORK=off go test ./buildscripts/release-dependency-guard
|
||||||
buildscripts/check-release-state_test.sh
|
buildscripts/check-release-state_test.sh
|
||||||
|
|||||||
+177
-25
@@ -1,24 +1,153 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
## Unreleased
|
## RELEASE.2026-11-12T00-00-00Z
|
||||||
|
|
||||||
- An edit of a service account or access key that names no policy keeps its
|
Package version: `20261112000000.0.0`. Publication status and artifacts are
|
||||||
inline policy. `mcli admin user svcacct disable`, `enable` and `edit` without
|
listed in [GitHub Releases](https://github.com/pgsty/silo/releases). This
|
||||||
`--policy`, the same `idp ldap accesskey` commands, and any other
|
section covers the changes after the published 20261109 source `d998292387cc`,
|
||||||
`update-service-account` request without `newPolicy` removed the inline
|
adopting Go `1.27.4`, `silo-pkg v3.16.1`, embedded Console `v2.7.2`, and
|
||||||
policy at the site that took the request and, for an account of another user
|
bundled `mcli 20261112`. See the
|
||||||
than root, at its replication peers, and the account then had all its parent
|
[server compatibility notes](https://silo.pgsty.com/compatibility/server/)
|
||||||
is allowed. The inline policy is removed
|
for behavior and upgrade boundaries.
|
||||||
only on request, by an empty policy document such as `{"Statement":[]}`
|
|
||||||
(`mcli admin user svcacct edit --policy`); a client that left the policy out
|
- Release checks verify the selected pkg and embedded Console versions and
|
||||||
to remove it must send that document. A JSON `null` policy counts as none.
|
match the bundled mcli release to its source module, independently of local
|
||||||
In the Console, clearing the policy editor of an access key no longer removes
|
workspaces, before building release artifacts. Deploy SILO and mcli
|
||||||
the policy; enter `{}` to remove it. The fix does not bring back a policy
|
`RELEASE.2026-11-12T00-00-00Z` by default from Helm chart `7.0.5`.
|
||||||
that was removed, and a site of a replicated deployment that does not run the
|
- Replica writes and copies use the destination version's retention, legal
|
||||||
fix still removes the policy when such an edit reaches it, so sites can
|
hold and tags under its mutation lock, verify staged replica content and
|
||||||
differ. Upgrade every site, then review on each site the accounts that
|
SSE-C/compressed metadata against stored locked versions before committing,
|
||||||
`svcacct info` reports with `Policy: implied`, and set the policy again on
|
and prevent newer peer timestamps from weakening unexpired COMPLIANCE
|
||||||
those that were meant to be restricted.
|
retention. Cross-pool reconciliation preserves COMPLIANCE protection
|
||||||
|
regardless of pool order, while authorized changes to GOVERNANCE retention
|
||||||
|
and legal hold remain possible. Local lock revisions advance past stored
|
||||||
|
revisions, unaddressed deletes under versioning suspension or excluded
|
||||||
|
prefixes enforce Object Lock across single-pool and multi-pool deployments
|
||||||
|
and fail closed when versioning or Object Lock metadata cannot be read,
|
||||||
|
bucket metadata imports require versioning across prefix exclusions as well
|
||||||
|
as suspension, and peer replication-trust headers must be signed with
|
||||||
|
header-based SigV4 ([#336](https://github.com/pgsty/silo/issues/336)).
|
||||||
|
- Tagging proxies (`GET`, `PUT` and `DELETE`) authenticate and authorize every
|
||||||
|
present replica, including its stored tags, fail closed on unreadable targets,
|
||||||
|
and skip missing replicas caused by replication lag unless all targets are
|
||||||
|
absent; active-active `GET` and `HEAD` proxies enforce `s3:ExistingObjectTag`
|
||||||
|
and `s3:versionid` before returning peer responses, forward
|
||||||
|
`X-Amz-Tagging-Directive: ACCESS` on `HEAD`, gate proxied
|
||||||
|
`x-amz-tagging-count` and `X-Amz-Tagging` headers on tag-read authorization
|
||||||
|
without emitting duplicate headers, and filter canonical-case peer
|
||||||
|
`X-Amz-Object-Lock-*` headers when `s3:GetObjectRetention` or
|
||||||
|
`s3:GetObjectLegalHold` is denied. Authenticated unsigned-trailer
|
||||||
|
uploads enforce the request time window and header signatures, SigV4 header,
|
||||||
|
presigned, streaming and POST policy verifiers bind the credential-scope date
|
||||||
|
to the signed request timestamp, streaming trailer readers bound trailer memory
|
||||||
|
and surface non-EOF stream errors without hanging on negative
|
||||||
|
`X-Amz-Decoded-Content-Length`, and bucket metadata cache loads no longer
|
||||||
|
overwrite fresher concurrent updates
|
||||||
|
([#337](https://github.com/pgsty/silo/issues/337),
|
||||||
|
[#338](https://github.com/pgsty/silo/issues/338)).
|
||||||
|
- Object tag and version authorization now applies consistently across read,
|
||||||
|
copy, list and archive endpoints: `GetObject`, `HeadObject` and
|
||||||
|
`SelectObjectContent` evaluate `s3:ExistingObjectTag` and `s3:versionid`
|
||||||
|
(preserving retryable `5xx` server errors for callers authorized by tag
|
||||||
|
conditions when storage is unreadable, resolving the 20261109 limitation in
|
||||||
|
[#293](https://github.com/pgsty/silo/issues/293) via
|
||||||
|
[PR #294](https://github.com/pgsty/silo/pull/294)); `CopyObject` and
|
||||||
|
`UploadPartCopy` authorize source `s3:ExistingObjectTag` and `s3:versionid`
|
||||||
|
as well as destination `s3:RequestObjectTag` on copied tags; both
|
||||||
|
`x-amz-tagging-count` and `X-Amz-Tagging` (`X-Amz-Tagging-Directive: ACCESS`)
|
||||||
|
are omitted on `GET`/`HEAD` (including versioned reads, STS sessions,
|
||||||
|
presigned/anonymous requests and replication proxies) unless
|
||||||
|
`s3:GetObjectTagging` / `s3:GetObjectVersionTagging` (with no explicit `Deny`)
|
||||||
|
or signed `s3:ReplicateTags` permits; `ListObjectsV2` / `ListObjectVersions` with `metadata=true` evaluate
|
||||||
|
current-object and per-version metadata permissions respectively, using stored tags; and `GetObjectAttributes`,
|
||||||
|
tagging handlers (`GET`/`PUT`/`DELETE`) and ZIP `GET`/`HEAD`/member listings
|
||||||
|
authorize stored tags and version actions without mutating unindexed archives
|
||||||
|
([#342](https://github.com/pgsty/silo/issues/342)).
|
||||||
|
- Policy parsing rejects unknown JSON fields, trailing tokens and malformed
|
||||||
|
principals across admin IAM policy/user/service-account endpoints, STS
|
||||||
|
session policies, IAM imports/replication, bucket policies and bucket metadata
|
||||||
|
imports instead of silently accepting empty or wildcard fallbacks. Validate
|
||||||
|
the complete imported policy file before applying entries; retain supported
|
||||||
|
historical snapshot semantics and explicit null bucket-policy replication
|
||||||
|
tombstones. New session policies retain strict resource validation.
|
||||||
|
- Every explicit version removal in S3 `DeleteObjects`, including in buckets
|
||||||
|
without Object Lock, now uses the sequential single-object path to check
|
||||||
|
metadata, retention and replication under the mutation lock. Large version
|
||||||
|
purges can take longer. Unaddressed deletes that can replace a locked null
|
||||||
|
version also use the single-object path; other unaddressed deletes retain
|
||||||
|
bulk execution.
|
||||||
|
- Decommission rescans keys hidden below another key only while source
|
||||||
|
versions are actually removed, resolving the 20261109 limitation recorded
|
||||||
|
in [#334](https://github.com/pgsty/silo/issues/334)
|
||||||
|
([PR #335](https://github.com/pgsty/silo/pull/335)). Decommission and rebalance
|
||||||
|
clean up successfully migrated sibling versions after concurrent deletion
|
||||||
|
of another version. Source-removal and final empty verification check every
|
||||||
|
online drive and tolerate offline drives within the erasure set's read quorum,
|
||||||
|
while rejecting online metadata remnants, corrupt metadata, directory errors
|
||||||
|
and drive outages beyond parity. Cancellation returns an error, and a scan
|
||||||
|
without physical progress stops. Free-version tier ownership and the final
|
||||||
|
physical-empty check remain protected
|
||||||
|
([#340](https://github.com/pgsty/silo/issues/340)). See the
|
||||||
|
[decommission procedure](https://silo.pgsty.com/operations/deployments/baremetal-decommission-server-pool/).
|
||||||
|
- Site-replication bucket healing, metadata healing, status collection and
|
||||||
|
replication hooks snapshot site topology and release the state read lock
|
||||||
|
before storage I/O and peer RPCs, avoiding deadlocks and state-map races
|
||||||
|
when a configuration writer is waiting
|
||||||
|
([#341](https://github.com/pgsty/silo/issues/341)). IAM import preserves
|
||||||
|
disabled service accounts and groups, reports group failures, and excludes
|
||||||
|
derived session-policy claims from exported and replicated credentials
|
||||||
|
([#345](https://github.com/pgsty/silo/issues/345)).
|
||||||
|
- Lifecycle history revision counting treats old and healed `xl.meta` headers
|
||||||
|
for the same version as one revision ([#332](https://github.com/pgsty/silo/issues/332)),
|
||||||
|
and restore expiry no longer turns a time-derived data-directory flag into a
|
||||||
|
different version revision; mixed healed headers can reach quorum and expired
|
||||||
|
local restore data can be cleaned up ([#344](https://github.com/pgsty/silo/issues/344)).
|
||||||
|
Historical-version self-copy plans its destination pool and data rewrite
|
||||||
|
before encryption or compression, preserving multi-block compression seek
|
||||||
|
indexes so a source version in another pool creates a readable new version.
|
||||||
|
In-place metadata updates locate the exact requested version and preserve
|
||||||
|
authoritative content on moving pools, while copies that require encryption
|
||||||
|
or checksum recomputation select an active pool
|
||||||
|
([#343](https://github.com/pgsty/silo/issues/343)). Indexed `Range` reads into
|
||||||
|
compressed, server-side encrypted (`SSE-S3`, `SSE-KMS`, `SSE-C`) objects also
|
||||||
|
succeed when the target compressed block starts inside the first 64 KiB
|
||||||
|
encrypted frame (`decryptSkip > 0`), without rewriting stored objects.
|
||||||
|
- Batch KMS rotation decodes and validates the configured encryption context
|
||||||
|
as strict UTF-8 JSON before changing an object key and without adding
|
||||||
|
validation-probe fields to object metadata, supports `SSE-S3` to `SSE-KMS`
|
||||||
|
key migration and multipart metadata preservation, and rewrites legacy `XLV1`
|
||||||
|
objects onto `XLv2` streams instead of failing in-place metadata updates
|
||||||
|
([#346](https://github.com/pgsty/silo/issues/346)). Incoming HTTP
|
||||||
|
`X-Amz-Server-Side-Encryption-Context` headers on `PutObject`, `CopyObject`,
|
||||||
|
`CreateMultipartUpload` and browser `POST` uploads reject empty values,
|
||||||
|
malformed Base64/UTF-8/JSON, trailing tokens and non-string/non-`null` member
|
||||||
|
values with `400 InvalidArgument` instead of `500 InternalError`. Batch variable
|
||||||
|
preflight follows the pinned policy evaluator's literal and substitution
|
||||||
|
rules, including BinaryEquals, while retaining refusal of unavailable
|
||||||
|
request context. This does not add AWS special-variable escape semantics
|
||||||
|
([#347](https://github.com/pgsty/silo/issues/347)).
|
||||||
|
- Service-account and access-key create/edit normalize a missing policy Version
|
||||||
|
to `2012-10-17`, reject malformed or unknown policy fields, and distinguish an
|
||||||
|
omitted or `null` policy (preserve existing inline policy) from an explicit
|
||||||
|
empty policy document such as `{}` or `{"Statement":[]}` (reset to parent
|
||||||
|
inheritance), while keeping historical deny-all accounts carrying an empty
|
||||||
|
embedded policy restricted until explicitly reset
|
||||||
|
([#339](https://github.com/pgsty/silo/issues/339),
|
||||||
|
[Console #89](https://github.com/pgsty/silo-console/issues/89),
|
||||||
|
[mcli #78](https://github.com/pgsty/mc/issues/78)). Previously,
|
||||||
|
`mcli admin user svcacct disable`, `enable` and `edit` without `--policy`,
|
||||||
|
the matching `admin accesskey` and `idp ldap/openid accesskey` commands, and
|
||||||
|
any other `update-service-account` request without `newPolicy` removed the
|
||||||
|
inline policy on the receiving site and on site-replication peers for
|
||||||
|
non-root parent users. In the matching Console (`v2.7.2`), turn off **Restrict
|
||||||
|
beyond user policy** or enter `{}` to remove an inline policy; clearing the
|
||||||
|
editor while restriction is enabled prevents saving, save errors render inside
|
||||||
|
the edit modal, and HTTP-to-HTTPS browser redirects preserve the request host
|
||||||
|
or `CONSOLE_SECURE_TLS_HOST` when bound to `--host 0.0.0.0`
|
||||||
|
(`unrolled/secure v1.18.0`). Freeze service-account edits on un-upgraded
|
||||||
|
`20261109` nodes during rollout; after upgrading every site, review accounts
|
||||||
|
that `svcacct info` reports with `Policy: implied` and re-apply restrictions
|
||||||
|
where needed.
|
||||||
|
|
||||||
- S3 `DeleteObjects` on a single pool stops at an entry whose name fails the
|
- S3 `DeleteObjects` on a single pool stops at an entry whose name fails the
|
||||||
object name check: the entry returns `XMinioInvalidObjectName` and is not
|
object name check: the entry returns `XMinioInvalidObjectName` and is not
|
||||||
@@ -31,13 +160,35 @@
|
|||||||
such as a bucket's configuration. The other entries of a request are deleted
|
such as a bucket's configuration. The other entries of a request are deleted
|
||||||
as before.
|
as before.
|
||||||
|
|
||||||
|
- Site replication sends a service account or access key as the record its
|
||||||
|
site has stored, whole: for a create, for every edit, at the first sync of a
|
||||||
|
new site and when healing. IAM export writes that record too. Until now an
|
||||||
|
edit was sent as the fields it changed. A peer that an earlier edit had not
|
||||||
|
reached, or reached late, kept the old secret key, status or inline policy
|
||||||
|
under the revision of the later edit, and healing took the sites for equal:
|
||||||
|
a key that was rotated, disabled or restricted on one site stayed as it was
|
||||||
|
on the other. Sync, heal and export took the claims and the inline policy, and
|
||||||
|
heal the secret key and status as well, from the cache of the server that ran
|
||||||
|
them, which can predate an edit made through another server of the site, and
|
||||||
|
sent them under the revision of the stored record. A service account created
|
||||||
|
for a user that had been deleted and created again did not reach the peers
|
||||||
|
before healing. Peers on 20261109 and 20261026 apply the new items as they
|
||||||
|
apply a healed account, and an edit that names no policy no longer removes the
|
||||||
|
inline policy on such a peer; a site that is not upgraded still sends its own
|
||||||
|
edits the old way. An edit now gives every site the whole account of the site
|
||||||
|
that took it, so accounts that already differ between sites are not repaired
|
||||||
|
by the upgrade, but by the next edit: after upgrading every site, review such
|
||||||
|
accounts and edit each one at a site that has it right.
|
||||||
|
|
||||||
## RELEASE.2026-11-09T00-00-00Z
|
## RELEASE.2026-11-09T00-00-00Z
|
||||||
|
|
||||||
Package version: `20261109000000.0.0`. Publication status and artifacts are
|
Package version: `20261109000000.0.0`; published from source `d998292387cc`.
|
||||||
listed in [GitHub Releases](https://github.com/pgsty/silo/releases). This
|
Artifacts are listed in the [GitHub release](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-11-09T00-00-00Z).
|
||||||
section covers the changes after the published 20261026 source `c3ad836aed0d`.
|
The coordinated [English release notes](https://silo.pgsty.com/blog/release/silo-20261109/)
|
||||||
Most fixes come from reports and patches by
|
and [Chinese release notes](https://silo.pgsty.com/zh/blog/release/silo-20261109/)
|
||||||
[@botdeniska](https://github.com/botdeniska); see [CONTRIBUTORS.md](CONTRIBUTORS.md).
|
cover all components, upgrade guidance, and validation scope since the
|
||||||
|
published 20261026 source `c3ad836aed0d`. Most fixes come from reports and
|
||||||
|
patches by [@botdeniska](https://github.com/botdeniska); see [CONTRIBUTORS.md](CONTRIBUTORS.md).
|
||||||
|
|
||||||
Lifecycle, multi-pool and tiering:
|
Lifecycle, multi-pool and tiering:
|
||||||
|
|
||||||
@@ -52,7 +203,8 @@ Lifecycle, multi-pool and tiering:
|
|||||||
`retainVersions: 0`) on the expired key instead of the keys below it; a key
|
`retainVersions: 0`) on the expired key instead of the keys below it; a key
|
||||||
with a tiered or Object Lock protected version is refused
|
with a tiered or Object Lock protected version is refused
|
||||||
([#297](https://github.com/pgsty/silo/issues/297)).
|
([#297](https://github.com/pgsty/silo/issues/297)).
|
||||||
- Stop taking the free version of a deleted tiered version for a version
|
- Exclude the internal free-version marker of a deleted tiered version from
|
||||||
|
visible version history and lifecycle decisions
|
||||||
([#263](https://github.com/pgsty/silo/issues/263)), apply rules whose only
|
([#263](https://github.com/pgsty/silo/issues/263)), apply rules whose only
|
||||||
action is `DelMarkerExpiration` ([#268](https://github.com/pgsty/silo/issues/268)),
|
action is `DelMarkerExpiration` ([#268](https://github.com/pgsty/silo/issues/268)),
|
||||||
and report and account only the removals that happened
|
and report and account only the removals that happened
|
||||||
@@ -198,7 +350,7 @@ Known issues: `ListMultipartUploads` prefix and paging
|
|||||||
([#217](https://github.com/pgsty/silo/issues/217)), replayed bucket creation
|
([#217](https://github.com/pgsty/silo/issues/217)), replayed bucket creation
|
||||||
under site replication ([PR #241](https://github.com/pgsty/silo/pull/241)), and
|
under site replication ([PR #241](https://github.com/pgsty/silo/pull/241)), and
|
||||||
`403` instead of the server error for reads allowed only by object tags while
|
`403` instead of the server error for reads allowed only by object tags while
|
||||||
the object is unreadable ([#293](https://github.com/pgsty/silo/issues/293)).
|
the object is unreadable ([PR #294](https://github.com/pgsty/silo/pull/294)).
|
||||||
|
|
||||||
## RELEASE.2026-10-26T00-00-00Z
|
## RELEASE.2026-10-26T00-00-00Z
|
||||||
|
|
||||||
|
|||||||
+5
-2
@@ -165,8 +165,11 @@ license grant to SILO or upstream MinIO maintainers is required.
|
|||||||
|
|
||||||
### How does Silo manage dependencies?
|
### How does Silo manage dependencies?
|
||||||
|
|
||||||
Silo uses Go modules. Preserve the compatibility module and import paths in
|
Silo uses Go modules. Shared packages in `github.com/pgsty/silo-pkg/v3` are
|
||||||
`go.mod`; downstream forks are selected with explicit `replace` directives.
|
required and imported directly under their own module path, upstream
|
||||||
|
`github.com/minio/minio-go/v7` is pinned to a verified commit, and Console
|
||||||
|
(`github.com/minio/console`) and MC (`github.com/minio/mc`) are selected with
|
||||||
|
explicit `replace` directives in `go.mod`.
|
||||||
|
|
||||||
- Run `go get foo/bar` in the source folder to add the dependency to `go.mod` file.
|
- Run `go get foo/bar` in the source folder to add the dependency to `go.mod` file.
|
||||||
|
|
||||||
|
|||||||
+35
-23
File diff suppressed because one or more lines are too long
@@ -14,7 +14,7 @@ RUN mkdir -p /prep/data && chmod 0777 /prep/data
|
|||||||
# Debian's bundle in distroless still trusts the roots the classic image
|
# Debian's bundle in distroless still trusts the roots the classic image
|
||||||
# blocklists. Build the same bundle as Dockerfile.goreleaser so both variants
|
# blocklists. Build the same bundle as Dockerfile.goreleaser so both variants
|
||||||
# trust the same roots; it is architecture-neutral, so build it natively.
|
# trust the same roots; it is architecture-neutral, so build it natively.
|
||||||
FROM --platform=$BUILDPLATFORM registry.access.redhat.com/ubi9/ubi:latest@sha256:16e9e43d99e181463513cc84a514859b1abb9460f1c4d9e208da4b4e51cd5cc3 AS certs
|
FROM --platform=$BUILDPLATFORM registry.access.redhat.com/ubi9/ubi:latest@sha256:8b212f75a0e5a156ac28b1c5c030d2e8a07715254e19d522286118279399af81 AS certs
|
||||||
# Native root blocklist mitigation: https://access.redhat.com/security/vulnerabilities/RHSB-2026-012
|
# Native root blocklist mitigation: https://access.redhat.com/security/vulnerabilities/RHSB-2026-012
|
||||||
RUN dnf -y install ca-certificates && \
|
RUN dnf -y install ca-certificates && \
|
||||||
if [ -d /etc/pki/ca-trust/source/blocklist ]; then \
|
if [ -d /etc/pki/ca-trust/source/blocklist ]; then \
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
FROM golang:1.27.3-alpine@sha256:d530cf60723abf1ffae0cd16bb546b379ec0550d2f3815e5b16ed69dbc00fba4 AS curl-build
|
FROM golang:1.27.4-alpine@sha256:0a5f7fe796599ff487605987432e7619ae45beeb2fba4c2ce0b907879932aebf AS curl-build
|
||||||
ARG TARGETARCH
|
ARG TARGETARCH
|
||||||
COPY dockerscripts/build-static-curl.sh /build/build-static-curl
|
COPY dockerscripts/build-static-curl.sh /build/build-static-curl
|
||||||
RUN /bin/sh /build/build-static-curl
|
RUN /bin/sh /build/build-static-curl
|
||||||
@@ -9,7 +9,7 @@ COPY --from=curl-build /go/bin/curl /curl
|
|||||||
COPY --from=curl-build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
COPY --from=curl-build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||||
ENTRYPOINT ["/curl"]
|
ENTRYPOINT ["/curl"]
|
||||||
|
|
||||||
FROM golang:1.27.3-alpine@sha256:d530cf60723abf1ffae0cd16bb546b379ec0550d2f3815e5b16ed69dbc00fba4 AS build
|
FROM golang:1.27.4-alpine@sha256:0a5f7fe796599ff487605987432e7619ae45beeb2fba4c2ce0b907879932aebf AS build
|
||||||
|
|
||||||
ARG TARGETARCH
|
ARG TARGETARCH
|
||||||
|
|
||||||
@@ -17,9 +17,9 @@ ENV GOPATH=/go
|
|||||||
ENV CGO_ENABLED=0
|
ENV CGO_ENABLED=0
|
||||||
|
|
||||||
ARG MC_REPO=pgsty/mc
|
ARG MC_REPO=pgsty/mc
|
||||||
ARG MC_VERSION=RELEASE.2026-11-09T00-00-00Z
|
ARG MC_VERSION=RELEASE.2026-11-12T00-00-00Z
|
||||||
ARG MC_AMD64_SHA256=79e6cf4ad00a9e5eb2e7ceccc6e6496e4af4ec096e59695515ea8f223c287e4a
|
ARG MC_AMD64_SHA256=68875c7fdd06a76af12b74753506df65226d1bec65a405d2617d76448c229ba2
|
||||||
ARG MC_ARM64_SHA256=5473c4b39a31dd66478f5c44686a09f3e9e37861a30f1faf18564fbe66118387
|
ARG MC_ARM64_SHA256=4ea59d2f278c96b25e380289fd2ee9bc49faa342687074f031805cb7b34fbe1a
|
||||||
|
|
||||||
RUN apk add -U --no-cache \
|
RUN apk add -U --no-cache \
|
||||||
ca-certificates \
|
ca-certificates \
|
||||||
@@ -69,7 +69,7 @@ RUN apk add -U --no-cache \
|
|||||||
mkdir -p /go/share/mcli && \
|
mkdir -p /go/share/mcli && \
|
||||||
cp /tmp/mcli-extract/LICENSE /tmp/mcli-extract/NOTICE /tmp/mcli-extract/CREDITS /go/share/mcli/
|
cp /tmp/mcli-extract/LICENSE /tmp/mcli-extract/NOTICE /tmp/mcli-extract/CREDITS /go/share/mcli/
|
||||||
|
|
||||||
FROM registry.access.redhat.com/ubi9/ubi:latest@sha256:16e9e43d99e181463513cc84a514859b1abb9460f1c4d9e208da4b4e51cd5cc3 AS certs
|
FROM registry.access.redhat.com/ubi9/ubi:latest@sha256:8b212f75a0e5a156ac28b1c5c030d2e8a07715254e19d522286118279399af81 AS certs
|
||||||
# Native root blocklist mitigation: https://access.redhat.com/security/vulnerabilities/RHSB-2026-012
|
# Native root blocklist mitigation: https://access.redhat.com/security/vulnerabilities/RHSB-2026-012
|
||||||
RUN dnf -y install ca-certificates && \
|
RUN dnf -y install ca-certificates && \
|
||||||
if [ -d /etc/pki/ca-trust/source/blocklist ]; then \
|
if [ -d /etc/pki/ca-trust/source/blocklist ]; then \
|
||||||
@@ -88,7 +88,7 @@ RUN dnf -y install ca-certificates && \
|
|||||||
dnf clean all && \
|
dnf clean all && \
|
||||||
rm -rf /var/cache/dnf
|
rm -rf /var/cache/dnf
|
||||||
|
|
||||||
FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:5be877a2a280ae36eb9ab80ca4efb28bbf77c57abbba38137cdc37a03b8465e6
|
FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:b59ac4f291d86027c2681f7a7b1cbf7448972fc55a6cb8e25cab5aa5eef5ce5f
|
||||||
|
|
||||||
LABEL org.opencontainers.image.title="Silo" \
|
LABEL org.opencontainers.image.title="Silo" \
|
||||||
org.opencontainers.image.description="S3-Interface Libre Object Storage" \
|
org.opencontainers.image.description="S3-Interface Libre Object Storage" \
|
||||||
|
|||||||
@@ -42,7 +42,8 @@ PRs must flow smoothly through review to reach production. Large PRs should be s
|
|||||||
- Document testing steps in the PR summary if manual testing was performed.
|
- Document testing steps in the PR summary if manual testing was performed.
|
||||||
|
|
||||||
4. **Before Submitting**:
|
4. **Before Submitting**:
|
||||||
- Run `make verify` to check formatting, linting, and tests.
|
- Sign off every commit under the Developer Certificate of Origin (`git commit -s`).
|
||||||
|
- Run `make verifiers` to check formatting, linting, code generation, and `rebrand-guard`.
|
||||||
- Reference related issues (e.g., “Closes #1234”).
|
- Reference related issues (e.g., “Closes #1234”).
|
||||||
- Notify team members via GitHub `@mentions` if urgent or complex.
|
- Notify team members via GitHub `@mentions` if urgent or complex.
|
||||||
|
|
||||||
|
|||||||
@@ -36,7 +36,7 @@
|
|||||||
|
|
||||||
PGSTY SILO keeps one maintained release line of the open-source MinIO server alive after upstream ended community distribution: builds, packages, multi-arch images, security fixes, and the full web console. Pigsty runs it in production as its PostgreSQL backup repository.
|
PGSTY SILO keeps one maintained release line of the open-source MinIO server alive after upstream ended community distribution: builds, packages, multi-arch images, security fixes, and the full web console. Pigsty runs it in production as its PostgreSQL backup repository.
|
||||||
|
|
||||||
It follows one rule — **the product and its delivery surfaces are renamed; the protocol and your data are not.** Everything else lives on [silo.pgsty.com](https://silo.pgsty.com/).
|
It follows one rule: **the product and its delivery surfaces are renamed; the protocol and your data are not.** Full documentation is published at [silo.pgsty.com](https://silo.pgsty.com/).
|
||||||
|
|
||||||
**Related:** [`pgsty/mc`](https://github.com/pgsty/mc) · [`pgsty/silo-console`](https://github.com/pgsty/silo-console) · [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) · [`pgsty/kes`](https://github.com/pgsty/kes) · [`pgsty/pigsty`](https://github.com/pgsty/pigsty)
|
**Related:** [`pgsty/mc`](https://github.com/pgsty/mc) · [`pgsty/silo-console`](https://github.com/pgsty/silo-console) · [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) · [`pgsty/kes`](https://github.com/pgsty/kes) · [`pgsty/pigsty`](https://github.com/pgsty/pigsty)
|
||||||
|
|
||||||
@@ -44,10 +44,10 @@ It follows one rule — **the product and its delivery surfaces are renamed; the
|
|||||||
<img src="https://silo.pgsty.com/images/silo-console/console-metrics-simple.webp" alt="Silo Console">
|
<img src="https://silo.pgsty.com/images/silo-console/console-metrics-simple.webp" alt="Silo Console">
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
#### SILO 20261109
|
### SILO 20261109
|
||||||
|
|
||||||
> [!WARNING]
|
> [!WARNING]
|
||||||
> [SILO 20261109](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-11-09T00-00-00Z) fixes bugs and security vulnerabilities. We strongly recommend scheduling an upgrade to reduce security risks.
|
> [SILO 20261109](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-11-09T00-00-00Z) (`20261109000000.0.0`, source `d998292387cc`) fixes storage, lifecycle, native restore, Object Lock, batch authorization, and IAM defects, embedding Console `v2.7.0` with `mcli 20261109`, `silo-pkg v3.16.0`, and Go `1.27.3`. Review the [release notes and upgrade requirements](https://silo.pgsty.com/blog/release/silo-20261109/) before upgrading.
|
||||||
|
|
||||||
> [!NOTE]
|
> [!NOTE]
|
||||||
> On 2026-08-06, `pgsty/minio` became `pgsty/silo`, and the default branch changed from `master` to `main`. The last MinIO-branded release is [`20260804`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z) on the [`minio`](https://github.com/pgsty/silo/tree/minio) branch.
|
> On 2026-08-06, `pgsty/minio` became `pgsty/silo`, and the default branch changed from `master` to `main`. The last MinIO-branded release is [`20260804`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z) on the [`minio`](https://github.com/pgsty/silo/tree/minio) branch.
|
||||||
@@ -86,7 +86,7 @@ docker exec silo mcli mb local/demo && docker exec silo mcli ls local
|
|||||||
| Kubernetes | Helm chart, see [Download & Install](https://silo.pgsty.com/download/) |
|
| Kubernetes | Helm chart, see [Download & Install](https://silo.pgsty.com/download/) |
|
||||||
| Source | `go build -o silo . && ./silo --version` |
|
| Source | `go build -o silo . && ./silo --version` |
|
||||||
|
|
||||||
Every release ships checksums, SPDX SBOMs, Sigstore-signed manifests, and GitHub build attestations. Installation methods and verification commands are documented at [Download & Install](https://silo.pgsty.com/download/); migrating from upstream MinIO — taking over an existing `minio.service` and its `/etc/default/minio`, and keeping data ownership stable with a `/etc/systemd/system/silo.service.d/10-legacy-user.conf` drop-in — is covered by the [migration guide](https://silo.pgsty.com/compatibility/migration/) and the [binary & service notes](https://silo.pgsty.com/compatibility/binary/).
|
Every release ships checksums, SPDX SBOMs, Sigstore-signed manifests, and GitHub build attestations. Installation methods and verification commands are documented at [Download & Install](https://silo.pgsty.com/download/). To migrate from upstream MinIO, take over `minio.service` and `/etc/default/minio`, and preserve data ownership with a `/etc/systemd/system/silo.service.d/10-legacy-user.conf` drop-in. See the [migration guide](https://silo.pgsty.com/compatibility/migration/) and [binary & service notes](https://silo.pgsty.com/compatibility/binary/).
|
||||||
|
|
||||||
## Compatibility
|
## Compatibility
|
||||||
|
|
||||||
@@ -98,7 +98,7 @@ Every divergence from upstream is listed in the code-verified [compatibility aud
|
|||||||
|
|
||||||
### TLS and Go upgrades
|
### TLS and Go upgrades
|
||||||
|
|
||||||
Server 20260916 restored Go's default key exchange; Server 20261018 retains it with Go 1.27.2.
|
Server 20260916 restored Go's default key exchange; Server 20261109 retains it with Go 1.27.3.
|
||||||
TLS key exchange follows Go's defaults across the S3 listener, node links,
|
TLS key exchange follows Go's defaults across the S3 listener, node links,
|
||||||
replication, identity providers, etcd, and external HTTP services. If an endpoint
|
replication, identity providers, etcd, and external HTTP services. If an endpoint
|
||||||
cannot accept ML-KEM, `GODEBUG=tlsmlkem=0` disables the default hybrid exchanges
|
cannot accept ML-KEM, `GODEBUG=tlsmlkem=0` disables the default hybrid exchanges
|
||||||
|
|||||||
+6
-6
@@ -36,7 +36,7 @@
|
|||||||
|
|
||||||
上游停止社区发行后,Silo 为开源 MinIO 服务端维护一条持续可用的版本线:构建、软件包、多架构镜像、安全修复与完整 Web 控制台。Pigsty 在生产环境中用它承载 PostgreSQL 备份存储。
|
上游停止社区发行后,Silo 为开源 MinIO 服务端维护一条持续可用的版本线:构建、软件包、多架构镜像、安全修复与完整 Web 控制台。Pigsty 在生产环境中用它承载 PostgreSQL 备份存储。
|
||||||
|
|
||||||
它只遵循一条原则:**改名的是产品与交付物,不是协议与你的数据。** 其余内容都在 [silo.pgsty.com](https://silo.pgsty.com/zh/)。
|
它只遵循一条原则:**改名的是产品与交付物,不是协议与你的数据。** 完整文档发布在 [silo.pgsty.com](https://silo.pgsty.com/zh/)。
|
||||||
|
|
||||||
**相关项目:**[`pgsty/mc`](https://github.com/pgsty/mc) · [`pgsty/silo-console`](https://github.com/pgsty/silo-console) · [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) · [`pgsty/kes`](https://github.com/pgsty/kes) · [`pgsty/pigsty`](https://github.com/pgsty/pigsty)
|
**相关项目:**[`pgsty/mc`](https://github.com/pgsty/mc) · [`pgsty/silo-console`](https://github.com/pgsty/silo-console) · [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) · [`pgsty/kes`](https://github.com/pgsty/kes) · [`pgsty/pigsty`](https://github.com/pgsty/pigsty)
|
||||||
|
|
||||||
@@ -44,10 +44,10 @@
|
|||||||
<img src="https://silo.pgsty.com/images/silo-console/console-metrics-simple.webp" alt="Silo 控制台">
|
<img src="https://silo.pgsty.com/images/silo-console/console-metrics-simple.webp" alt="Silo 控制台">
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
#### SILO 20261109
|
### SILO 20261109
|
||||||
|
|
||||||
> [!WARNING]
|
> [!WARNING]
|
||||||
> [SILO 20261109](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-11-09T00-00-00Z) 修复了缺陷与安全漏洞,强烈建议尽快安排升级,以降低安全风险。
|
> [SILO 20261109](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-11-09T00-00-00Z)(包版本 `20261109000000.0.0`,源码 `d998292387cc`)修复了多池生命周期、原生恢复、Object Lock、批处理授权与 IAM 等缺陷,内嵌 Console `v2.7.0` 并配套 `mcli 20261109`、`silo-pkg v3.16.0` 与 Go `1.27.3`。升级前请阅读[发布说明与升级要求](https://silo.pgsty.com/zh/blog/release/silo-20261109/)。
|
||||||
|
|
||||||
> [!NOTE]
|
> [!NOTE]
|
||||||
> 2026-08-06,`pgsty/minio` 更名为 `pgsty/silo`,默认分支由 `master` 改为 `main`。最后一个 MinIO 品牌发布是 [`minio`](https://github.com/pgsty/silo/tree/minio) 分支上的 [`20260804`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z)。
|
> 2026-08-06,`pgsty/minio` 更名为 `pgsty/silo`,默认分支由 `master` 改为 `main`。最后一个 MinIO 品牌发布是 [`minio`](https://github.com/pgsty/silo/tree/minio) 分支上的 [`20260804`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z)。
|
||||||
@@ -98,9 +98,9 @@ Silo 保留 S3 与存储格式兼容性,包括既有 `MINIO_*` 环境变量、
|
|||||||
|
|
||||||
### TLS 与 Go 升级
|
### TLS 与 Go 升级
|
||||||
|
|
||||||
Server 20260916 已恢复 Go 默认密钥交换策略,20261018 在 Go 1.27.2 下保留该行为。
|
Server 20260916 恢复了 Go 默认密钥交换策略,Server 20261109 在 Go 1.27.3 下保留该行为。S3 监听端口、节点间连接、站点与桶复制、身份提供方、etcd 及外部 HTTP 服务均使用 Go 默认 TLS 密钥交换。若对端无法接受 ML-KEM,设置 `GODEBUG=tlsmlkem=0` 可在当前进程禁用默认混合密钥交换(证书校验保持开启);该选项不会禁用 ML-DSA 签名,也不能解决所有 TLS 连接重置问题,建议在升级不兼容对端后移除该临时配置。若仅新增的 SecP 混合组引发问题,可设置 `GODEBUG=tlssecpmlkem=0` 仅禁用这些组并保留 `X25519MLKEM768`。
|
||||||
`GODEBUG=tlsmlkem=0`、`tlssecpmlkem=0` 的适用范围、macOS 根证书来源变化,
|
|
||||||
以及 OIDC discovery 的诊断方法见 [Go 1.27 TLS 与 OIDC 指南](https://silo.pgsty.com/zh/blog/design/go127-tls-oidc-discovery/)。
|
在 Go 1.27 构建中,macOS 上设置 `SSL_CERT_FILE` 或 `SSL_CERT_DIR` 会以磁盘根证书和 Go 验证器替代系统钥匙串信任;陈旧或不完整的 CA 路径可能导致原先受信任的连接失败,清除继承的环境变量即可恢复钥匙串信任。配置的 `CAs` 目录中的显式证书仍会叠加加入所选根证书池。Go 1.27 二进制要求 macOS 13 或更高版本。详见 [Go 1.27 发行说明](https://go.dev/doc/go1.27)与 [Go 1.27 TLS 与 OIDC 发现指南](https://silo.pgsty.com/zh/blog/design/go127-tls-oidc-discovery/)。
|
||||||
|
|
||||||
## 文档归属
|
## 文档归属
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -38,7 +38,7 @@ For vulnerabilities in this fork:
|
|||||||
|
|
||||||
1. Follow the fork-specific expectations in [VULNERABILITY_REPORT.md](VULNERABILITY_REPORT.md).
|
1. Follow the fork-specific expectations in [VULNERABILITY_REPORT.md](VULNERABILITY_REPORT.md).
|
||||||
2. Prefer this repository's [private GitHub security advisory](https://github.com/pgsty/silo/security/advisories/new) workflow.
|
2. Prefer this repository's [private GitHub security advisory](https://github.com/pgsty/silo/security/advisories/new) workflow.
|
||||||
3. If you cannot use GitHub private reporting, email [ron@pgsty.com](mailto:ron@pgsty.com) or rh@vonng.com
|
3. If you cannot use GitHub private reporting, email [ron@pgsty.com](mailto:ron@pgsty.com) or [rh@vonng.com](mailto:rh@vonng.com)
|
||||||
directly. Do not publish vulnerability details or exploit code in public
|
directly. Do not publish vulnerability details or exploit code in public
|
||||||
issues or pull requests.
|
issues or pull requests.
|
||||||
4. If you confirm the issue also affects upstream `minio/minio`, report it upstream separately.
|
4. If you confirm the issue also affects upstream `minio/minio`, report it upstream separately.
|
||||||
|
|||||||
Executable
+116
@@ -0,0 +1,116 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
# Fail closed before a release job builds a commit that main never accepted or
|
||||||
|
# that CI never validated.
|
||||||
|
#
|
||||||
|
# check-release-state.sh answers "is this tag safe to write to". This answers
|
||||||
|
# the other half: "is this commit safe to ship". The release workflow already
|
||||||
|
# proves the tag resolves to the checkout, but a tag can be pushed to any
|
||||||
|
# commit at any time, including one that never landed on main and one whose
|
||||||
|
# tests have not finished.
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
release_commit="${1:-}"
|
||||||
|
fixture="${2:-}"
|
||||||
|
repository="${GITHUB_REPOSITORY:-pgsty/silo}"
|
||||||
|
release_branch="${RELEASE_BRANCH:-origin/main}"
|
||||||
|
|
||||||
|
# Workflows that must have a successful run for this exact commit, keyed by the
|
||||||
|
# workflow file path rather than the display name: a name is free text, so a
|
||||||
|
# second workflow could otherwise be added that calls itself "Go CI", succeeds
|
||||||
|
# trivially, and satisfies this check while the real one fails.
|
||||||
|
required_workflow_paths=(
|
||||||
|
".github/workflows/go.yml"
|
||||||
|
".github/workflows/vulncheck.yml"
|
||||||
|
)
|
||||||
|
|
||||||
|
if ! command -v jq >/dev/null 2>&1; then
|
||||||
|
echo "jq is required to inspect workflow runs" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ! "${release_commit}" =~ ^[0-9a-f]{40}$ ]]; then
|
||||||
|
echo "Invalid release commit: ${release_commit:-<empty>}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z "${fixture}" ]; then
|
||||||
|
if ! git rev-parse --verify --quiet "${release_branch}" >/dev/null; then
|
||||||
|
echo "Cannot resolve ${release_branch}; fetch it before releasing" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
# The exact tip, not merely an ancestor: an older commit on main has no
|
||||||
|
# push run of its own when it arrived as part of a multi-commit push, and a
|
||||||
|
# release must not trail behind what main already contains.
|
||||||
|
tip="$(git rev-parse "${release_branch}^{commit}")"
|
||||||
|
if [ "${tip}" != "${release_commit}" ]; then
|
||||||
|
echo "Refusing to release ${release_commit}: ${release_branch} is at ${tip}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "Release commit ${release_commit} is the tip of ${release_branch}."
|
||||||
|
|
||||||
|
error_file="$(mktemp)"
|
||||||
|
trap 'rm -f "${error_file}"' EXIT
|
||||||
|
# A pull_request run checks out GitHub's synthetic merge ref, so a green
|
||||||
|
# run proves the merge result was good, not that this commit was ever built
|
||||||
|
# on its own. exclude_pull_requests only trims the pull_requests field of
|
||||||
|
# each run; the event/head_branch filter below is what drops those runs.
|
||||||
|
if ! runs_json="$(
|
||||||
|
gh api --paginate \
|
||||||
|
"repos/${repository}/actions/runs?head_sha=${release_commit}&exclude_pull_requests=true&per_page=100" \
|
||||||
|
--jq '.workflow_runs[] | {path, event, head_branch, head_sha, status, conclusion, run_number, run_attempt}' \
|
||||||
|
2>"${error_file}" | jq -s '.'
|
||||||
|
)"; then
|
||||||
|
cat "${error_file}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
runs_json="$(<"${fixture}")"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! jq -e 'type == "array" and all(.[]; type == "object" and (.path | type == "string"))' \
|
||||||
|
<<<"${runs_json}" >/dev/null 2>&1; then
|
||||||
|
echo "Invalid workflow run response for ${release_commit}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
fail=0
|
||||||
|
for workflow in "${required_workflow_paths[@]}"; do
|
||||||
|
# Only runs that actually built this commit count: the right workflow file,
|
||||||
|
# triggered by the push that put this SHA on main. A manual dispatch is not
|
||||||
|
# substitute evidence - it can be run from any ref. Take the newest such
|
||||||
|
# run; an older success must not paper over a later failure.
|
||||||
|
latest="$(jq -c --arg path "${workflow}" --arg sha "${release_commit}" '
|
||||||
|
[ .[]
|
||||||
|
| select(.path == $path)
|
||||||
|
| select(.head_sha == $sha)
|
||||||
|
| select(.event == "push" and .head_branch == "main")
|
||||||
|
]
|
||||||
|
| sort_by([.run_number // 0, .run_attempt // 0])
|
||||||
|
| last // null' <<<"${runs_json}")"
|
||||||
|
|
||||||
|
if [ "${latest}" = "null" ]; then
|
||||||
|
echo "Required workflow ${workflow} has no qualifying run for ${release_commit} (no run)" >&2
|
||||||
|
fail=1
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
status="$(jq -r '.status // "unknown"' <<<"${latest}")"
|
||||||
|
conclusion="$(jq -r '.conclusion // "none"' <<<"${latest}")"
|
||||||
|
if [ "${status}" != "completed" ] || [ "${conclusion}" != "success" ]; then
|
||||||
|
observed="${conclusion}"
|
||||||
|
if [ "${status}" != "completed" ]; then
|
||||||
|
observed="${status}"
|
||||||
|
fi
|
||||||
|
echo "Required workflow ${workflow} did not succeed for ${release_commit} (${observed})" >&2
|
||||||
|
fail=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "${fail}" -ne 0 ]; then
|
||||||
|
echo "Refusing to release ${release_commit}: required checks are not green" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Required workflows are green for ${release_commit}."
|
||||||
Executable
+132
@@ -0,0 +1,132 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
checker="${script_dir}/check-release-commit.sh"
|
||||||
|
commit="0123456789abcdef0123456789abcdef01234567"
|
||||||
|
other_commit="89abcdef0123456789abcdef0123456789abcdef"
|
||||||
|
fixture="$(mktemp)"
|
||||||
|
stdout_file="$(mktemp)"
|
||||||
|
stderr_file="$(mktemp)"
|
||||||
|
trap 'rm -f "${fixture}" "${stdout_file}" "${stderr_file}"' EXIT
|
||||||
|
|
||||||
|
expect_success() {
|
||||||
|
if ! "${checker}" "$@" >"${stdout_file}" 2>"${stderr_file}"; then
|
||||||
|
cat "${stderr_file}" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
expect_failure() {
|
||||||
|
if "${checker}" "$@" >"${stdout_file}" 2>"${stderr_file}"; then
|
||||||
|
echo "Expected release-commit check to fail: $*" >&2
|
||||||
|
cat "${stdout_file}" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# run <path> <conclusion> [event] [branch] [sha] [run_number] [status]
|
||||||
|
run() {
|
||||||
|
local path="$1" conclusion="$2"
|
||||||
|
local event="${3:-push}" branch="${4:-main}" sha="${5:-${commit}}"
|
||||||
|
local number="${6:-1}" status="${7:-completed}"
|
||||||
|
local conclusion_json="\"${conclusion}\""
|
||||||
|
[ "${conclusion}" = "null" ] && conclusion_json=null
|
||||||
|
printf '{"path":"%s","event":"%s","head_branch":"%s","head_sha":"%s","status":"%s","conclusion":%s,"run_number":%s,"run_attempt":1}' \
|
||||||
|
"${path}" "${event}" "${branch}" "${sha}" "${status}" "${conclusion_json}" "${number}"
|
||||||
|
}
|
||||||
|
|
||||||
|
go_ok="$(run .github/workflows/go.yml success)"
|
||||||
|
vuln_ok="$(run .github/workflows/vulncheck.yml success)"
|
||||||
|
|
||||||
|
all_green() {
|
||||||
|
printf '[%s,%s]\n' "${go_ok}" "${vuln_ok}" >"${fixture}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Baseline: both required workflows green on a push to main.
|
||||||
|
all_green
|
||||||
|
expect_success "${commit}" "${fixture}"
|
||||||
|
grep -qF "Required workflows are green for ${commit}." "${stdout_file}"
|
||||||
|
|
||||||
|
# A manual dispatch is not evidence: it can be run from any ref.
|
||||||
|
printf '[%s,%s]\n' \
|
||||||
|
"$(run .github/workflows/go.yml success workflow_dispatch main)" \
|
||||||
|
"${vuln_ok}" >"${fixture}"
|
||||||
|
expect_failure "${commit}" "${fixture}"
|
||||||
|
grep -qF "no qualifying run" "${stderr_file}"
|
||||||
|
|
||||||
|
# A re-run that finally succeeded is enough; the earlier failure is history.
|
||||||
|
printf '[%s,%s,%s]\n' \
|
||||||
|
"$(run .github/workflows/go.yml failure push main "${commit}" 1)" \
|
||||||
|
"$(run .github/workflows/go.yml success push main "${commit}" 2)" \
|
||||||
|
"${vuln_ok}" >"${fixture}"
|
||||||
|
expect_success "${commit}" "${fixture}"
|
||||||
|
|
||||||
|
# ...but a LATER failure must not be papered over by an earlier success.
|
||||||
|
printf '[%s,%s,%s]\n' \
|
||||||
|
"$(run .github/workflows/go.yml success push main "${commit}" 1)" \
|
||||||
|
"$(run .github/workflows/go.yml failure push main "${commit}" 2)" \
|
||||||
|
"${vuln_ok}" >"${fixture}"
|
||||||
|
expect_failure "${commit}" "${fixture}"
|
||||||
|
grep -qF "Required workflow .github/workflows/go.yml did not succeed" "${stderr_file}"
|
||||||
|
grep -qF "(failure)" "${stderr_file}"
|
||||||
|
|
||||||
|
# A decoy workflow that merely calls itself "Go CI" must not satisfy the gate.
|
||||||
|
printf '[%s,%s]\n' \
|
||||||
|
"$(run .github/workflows/decoy.yml success)" "${vuln_ok}" >"${fixture}"
|
||||||
|
expect_failure "${commit}" "${fixture}"
|
||||||
|
grep -qF "Required workflow .github/workflows/go.yml has no qualifying run" "${stderr_file}"
|
||||||
|
|
||||||
|
# A pull_request run tests GitHub's merge ref, not this commit on its own.
|
||||||
|
printf '[%s,%s]\n' \
|
||||||
|
"$(run .github/workflows/go.yml success pull_request feature)" "${vuln_ok}" >"${fixture}"
|
||||||
|
expect_failure "${commit}" "${fixture}"
|
||||||
|
grep -qF "no qualifying run" "${stderr_file}"
|
||||||
|
|
||||||
|
# A push run on some other branch is not evidence that main accepted it.
|
||||||
|
printf '[%s,%s]\n' \
|
||||||
|
"$(run .github/workflows/go.yml success push feature)" "${vuln_ok}" >"${fixture}"
|
||||||
|
expect_failure "${commit}" "${fixture}"
|
||||||
|
grep -qF "no qualifying run" "${stderr_file}"
|
||||||
|
|
||||||
|
# A run reporting a different head_sha must be ignored.
|
||||||
|
printf '[%s,%s]\n' \
|
||||||
|
"$(run .github/workflows/go.yml success push main "${other_commit}")" "${vuln_ok}" >"${fixture}"
|
||||||
|
expect_failure "${commit}" "${fixture}"
|
||||||
|
grep -qF "no qualifying run" "${stderr_file}"
|
||||||
|
|
||||||
|
# Missing workflow entirely.
|
||||||
|
printf '[%s]\n' "${go_ok}" >"${fixture}"
|
||||||
|
expect_failure "${commit}" "${fixture}"
|
||||||
|
grep -qF "Required workflow .github/workflows/vulncheck.yml has no qualifying run" "${stderr_file}"
|
||||||
|
|
||||||
|
# Still running.
|
||||||
|
printf '[%s,%s]\n' "${go_ok}" \
|
||||||
|
"$(run .github/workflows/vulncheck.yml null push main "${commit}" 1 in_progress)" >"${fixture}"
|
||||||
|
expect_failure "${commit}" "${fixture}"
|
||||||
|
grep -qF "(in_progress)" "${stderr_file}"
|
||||||
|
|
||||||
|
# Cancelled counts as not green.
|
||||||
|
printf '[%s,%s]\n' "${go_ok}" \
|
||||||
|
"$(run .github/workflows/vulncheck.yml cancelled)" >"${fixture}"
|
||||||
|
expect_failure "${commit}" "${fixture}"
|
||||||
|
grep -qF "(cancelled)" "${stderr_file}"
|
||||||
|
|
||||||
|
# Malformed API response.
|
||||||
|
printf '{not-json}\n' >"${fixture}"
|
||||||
|
expect_failure "${commit}" "${fixture}"
|
||||||
|
grep -qF "Invalid workflow run response for ${commit}" "${stderr_file}"
|
||||||
|
|
||||||
|
# An empty array is a valid response that satisfies nothing.
|
||||||
|
printf '[]\n' >"${fixture}"
|
||||||
|
expect_failure "${commit}" "${fixture}"
|
||||||
|
grep -qF "Required workflow .github/workflows/go.yml has no qualifying run" "${stderr_file}"
|
||||||
|
|
||||||
|
expect_failure "not-a-commit" "${fixture}"
|
||||||
|
grep -qF "Invalid release commit" "${stderr_file}"
|
||||||
|
|
||||||
|
expect_failure "" "${fixture}"
|
||||||
|
grep -qF "Invalid release commit: <empty>" "${stderr_file}"
|
||||||
|
|
||||||
|
echo "release-commit decision tests passed"
|
||||||
@@ -7,7 +7,7 @@ _init() {
|
|||||||
|
|
||||||
## Minimum required versions for build dependencies
|
## Minimum required versions for build dependencies
|
||||||
GIT_VERSION="1.0"
|
GIT_VERSION="1.0"
|
||||||
GO_VERSION="1.27.3"
|
GO_VERSION="1.27.4"
|
||||||
OSX_VERSION="10.8"
|
OSX_VERSION="10.8"
|
||||||
KNAME=$(uname -s)
|
KNAME=$(uname -s)
|
||||||
ARCH=$(uname -m)
|
ARCH=$(uname -m)
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ if [ -n "${MCLI_BIN:-}" ]; then
|
|||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
release=${MCLI_RELEASE:-RELEASE.2026-11-09T00-00-00Z}
|
release=${MCLI_RELEASE:-RELEASE.2026-11-12T00-00-00Z}
|
||||||
version_hyphen=${release#RELEASE.}
|
version_hyphen=${release#RELEASE.}
|
||||||
package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')
|
package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')
|
||||||
if [ "${package_version}" = "${version_hyphen}" ]; then
|
if [ "${package_version}" = "${version_hyphen}" ]; then
|
||||||
|
|||||||
@@ -0,0 +1,156 @@
|
|||||||
|
// Copyright (c) 2026 PGSTY
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
// release-dependency-guard checks the modules and bundled client actually used
|
||||||
|
// by a release. A local workspace must not substitute unreleased source trees.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"golang.org/x/mod/semver"
|
||||||
|
)
|
||||||
|
|
||||||
|
type module struct {
|
||||||
|
Path, Version, Dir string
|
||||||
|
Main bool
|
||||||
|
Replace *module
|
||||||
|
}
|
||||||
|
|
||||||
|
func selected(m module, path string) (module, error) {
|
||||||
|
if m.Replace != nil {
|
||||||
|
m = *m.Replace
|
||||||
|
}
|
||||||
|
if m.Main || m.Path != path || m.Version == "" {
|
||||||
|
return module{}, fmt.Errorf("%s must select an immutable maintained module, got %s@%s", path, m.Path, m.Version)
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func atLeast(version, minimum string) bool {
|
||||||
|
return semver.IsValid(version) && semver.Prerelease(version) == "" && semver.Compare(version, minimum) >= 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func validate(modules map[string]module, consoleVersion, mcTag, mcCommit string) error {
|
||||||
|
pkg, err := selected(modules["github.com/pgsty/silo-pkg/v3"], "github.com/pgsty/silo-pkg/v3")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !atLeast(pkg.Version, "v3.16.1") {
|
||||||
|
return fmt.Errorf("silo-pkg %s lacks the strict policy release; need v3.16.1 or newer", pkg.Version)
|
||||||
|
}
|
||||||
|
if _, err := selected(modules["github.com/minio/console"], "github.com/pgsty/silo-console"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !atLeast("v"+consoleVersion, "v2.7.2") {
|
||||||
|
return fmt.Errorf("embedded Console %s lacks the v2.7.2 release; need 2.7.2 or newer", consoleVersion)
|
||||||
|
}
|
||||||
|
mc, err := selected(modules["github.com/minio/mc"], "github.com/pgsty/mc")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !regexp.MustCompile(`^RELEASE\.\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}Z$`).MatchString(mcTag) || mcTag <= "RELEASE.2026-11-09T00-00-00Z" {
|
||||||
|
return fmt.Errorf("bundled mcli %s must be a release after 20261109", mcTag)
|
||||||
|
}
|
||||||
|
if len(mcCommit) != 40 || !strings.HasSuffix(mc.Version, "-"+mcCommit[:12]) {
|
||||||
|
return fmt.Errorf("mc module %s does not match Dockerfile.goreleaser tag %s (%s)", mc.Version, mcTag, mcCommit)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func command(name string, args ...string) ([]byte, error) {
|
||||||
|
cmd := exec.Command(name, args...)
|
||||||
|
cmd.Stderr = os.Stderr
|
||||||
|
return cmd.Output()
|
||||||
|
}
|
||||||
|
|
||||||
|
func check() error {
|
||||||
|
if err := os.Setenv("GOWORK", "off"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
data, err := command("go", "list", "-mod=readonly", "-m", "-json", "github.com/pgsty/silo-pkg/v3", "github.com/minio/console", "github.com/minio/mc")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
modules := make(map[string]module)
|
||||||
|
dec := json.NewDecoder(bytes.NewReader(data))
|
||||||
|
for {
|
||||||
|
var m module
|
||||||
|
if err := dec.Decode(&m); err != nil {
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
modules[m.Path] = m
|
||||||
|
}
|
||||||
|
pkg, err := selected(modules["github.com/pgsty/silo-pkg/v3"], "github.com/pgsty/silo-pkg/v3")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !atLeast(pkg.Version, "v3.16.1") {
|
||||||
|
return fmt.Errorf("release blocked: select silo-pkg v3.16.1 or newer (currently %s)", pkg.Version)
|
||||||
|
}
|
||||||
|
console, err := selected(modules["github.com/minio/console"], "github.com/pgsty/silo-console")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if console.Dir == "" {
|
||||||
|
data, err = command("go", "mod", "download", "-json", console.Path+"@"+console.Version)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(data, &console); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
data, err = os.ReadFile(filepath.Join(console.Dir, "web-app", "package.json"))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var metadata struct{ Version string }
|
||||||
|
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
data, err = os.ReadFile("Dockerfile.goreleaser")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
match := regexp.MustCompile(`(?m)^ARG MC_VERSION=(\S+)\s*$`).FindSubmatch(data)
|
||||||
|
if len(match) != 2 {
|
||||||
|
return fmt.Errorf("Dockerfile.goreleaser must pin MC_VERSION")
|
||||||
|
}
|
||||||
|
tag := string(match[1])
|
||||||
|
data, err = command("gh", "api", "repos/pgsty/mc/commits/"+tag, "--jq", ".sha")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
commit := strings.TrimSpace(string(data))
|
||||||
|
if err := validate(modules, metadata.Version, tag, commit); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
data, err = command("gh", "api", "repos/pgsty/mc/compare/d7262c2a5fbdf2b9503cc0283f1d37ae8da41baf..."+commit, "--jq", ".status")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if status := strings.TrimSpace(string(data)); status != "ahead" && status != "identical" {
|
||||||
|
return fmt.Errorf("mcli %s does not include the access-key policy repair", tag)
|
||||||
|
}
|
||||||
|
fmt.Printf("Release dependencies verified: silo-pkg %s, Console %s, mcli %s (%s)\n", pkg.Version, metadata.Version, tag, commit)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if err := check(); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
// Copyright (c) 2026 PGSTY
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestReleaseDependencySelection(t *testing.T) {
|
||||||
|
const commit = "0123456789abcdef0123456789abcdef01234567"
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, pkg, console, tag, mcSHA string
|
||||||
|
local, upstream, wantError bool
|
||||||
|
}{
|
||||||
|
{name: "coordinated", pkg: "v3.16.1", console: "2.7.2", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: commit},
|
||||||
|
{name: "old-package", pkg: "v3.16.0", console: "2.7.2", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: commit, wantError: true},
|
||||||
|
{name: "prerelease-package", pkg: "v3.16.1-rc.1", console: "2.7.2", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: commit, wantError: true},
|
||||||
|
{name: "old-embedded-console", pkg: "v3.16.1", console: "2.7.1", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: commit, wantError: true},
|
||||||
|
{name: "old-container-client", pkg: "v3.16.1", console: "2.7.2", tag: "RELEASE.2026-11-09T00-00-00Z", mcSHA: commit, wantError: true},
|
||||||
|
{name: "source-binary-mismatch", pkg: "v3.16.1", console: "2.7.2", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: "ffffffffffffffffffffffffffffffffffffffff", wantError: true},
|
||||||
|
{name: "workspace-console", pkg: "v3.16.1", console: "2.7.2", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: commit, local: true, wantError: true},
|
||||||
|
{name: "upstream-console", pkg: "v3.16.1", console: "2.7.2", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: commit, upstream: true, wantError: true},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
modules := map[string]module{
|
||||||
|
"github.com/pgsty/silo-pkg/v3": {Path: "github.com/pgsty/silo-pkg/v3", Version: tc.pkg},
|
||||||
|
"github.com/minio/console": {Path: "github.com/minio/console", Replace: &module{
|
||||||
|
Path: "github.com/pgsty/silo-console", Version: "v0.0.0-20261112000000-0123456789ab",
|
||||||
|
}},
|
||||||
|
"github.com/minio/mc": {Path: "github.com/minio/mc", Replace: &module{
|
||||||
|
Path: "github.com/pgsty/mc", Version: "v0.0.0-20261112000000-0123456789ab",
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
if tc.local {
|
||||||
|
modules["github.com/minio/console"].Replace.Version = ""
|
||||||
|
}
|
||||||
|
if tc.upstream {
|
||||||
|
modules["github.com/minio/console"].Replace.Path = "github.com/minio/console"
|
||||||
|
}
|
||||||
|
if err := validate(modules, tc.console, tc.tag, tc.mcSHA); (err != nil) != tc.wantError {
|
||||||
|
t.Fatalf("got %v, wantError=%t", err, tc.wantError)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -120,6 +120,10 @@ if [ -z "${app_version}" ] || [ "${image_tag}" != "${app_version}" ]; then
|
|||||||
echo "helm/silo: image.tag '${image_tag}' must equal appVersion '${app_version}'" >&2
|
echo "helm/silo: image.tag '${image_tag}' must equal appVersion '${app_version}'" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
if [ -n "${RELEASE_TAG:-}" ] && [ "${image_tag}" != "${RELEASE_TAG}" ]; then
|
||||||
|
echo "helm/silo: image.tag '${image_tag}' must equal RELEASE_TAG '${RELEASE_TAG}'" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
helm_run package "${new_chart}" --destination "${output_dir}" >/dev/null
|
helm_run package "${new_chart}" --destination "${output_dir}" >/dev/null
|
||||||
test -s "${work_dir}/silo-${chart_version}.tgz"
|
test -s "${work_dir}/silo-${chart_version}.tgz"
|
||||||
|
|||||||
@@ -817,17 +817,17 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
bucketMap[bucket] = &v
|
bucketMap[bucket] = &v
|
||||||
}
|
}
|
||||||
|
|
||||||
if globalSiteReplicationSys.isEnabled() && v.Suspended() {
|
if globalSiteReplicationSys.isEnabled() && (v.Suspended() || v.PrefixesExcluded()) {
|
||||||
rpt.SetStatus(bucket, fileName, fmt.Errorf("Cluster replication is enabled for this site, so the versioning state cannot be suspended."))
|
rpt.SetStatus(bucket, fileName, fmt.Errorf("Cluster replication is enabled for this site, so the versioning state cannot be suspended or exclude prefixes."))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && v.Suspended() {
|
if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && (v.Suspended() || v.PrefixesExcluded()) {
|
||||||
rpt.SetStatus(bucket, fileName, fmt.Errorf("An Object Lock configuration is present on this bucket, so the versioning state cannot be suspended."))
|
rpt.SetStatus(bucket, fileName, fmt.Errorf("An Object Lock configuration is present on this bucket, so the versioning state cannot be suspended or exclude prefixes."))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if rcfg, _ := getReplicationConfig(ctx, bucket); rcfg != nil && v.Suspended() {
|
if rcfg, _ := getReplicationConfig(ctx, bucket); rcfg != nil && (v.Suspended() || v.PrefixesExcluded()) {
|
||||||
rpt.SetStatus(bucket, fileName, fmt.Errorf("A replication configuration is present on this bucket, so the versioning state cannot be suspended."))
|
rpt.SetStatus(bucket, fileName, fmt.Errorf("A replication configuration is present on this bucket, so the versioning state cannot be suspended or exclude prefixes."))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -907,7 +907,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
bucketPolicy, err := policy.ParseBucketPolicyConfig(bytes.NewReader(bucketPolicyBytes), bucket)
|
bucketPolicy, err := parseBucketPolicyStrictBytes(bucketPolicyBytes, bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
rpt.SetStatus(bucket, fileName, err)
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -165,10 +165,10 @@ func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.
|
|||||||
default:
|
default:
|
||||||
err = errSRInvalidRequest(errInvalidArgument)
|
err = errSRInvalidRequest(errInvalidArgument)
|
||||||
case madmin.SRIAMItemPolicy:
|
case madmin.SRIAMItemPolicy:
|
||||||
if item.Policy == nil {
|
if item.Policy == nil || bytes.Equal(bytes.TrimSpace(item.Policy), []byte("null")) {
|
||||||
err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
|
err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
|
||||||
} else {
|
} else {
|
||||||
policy, perr := policy.ParseConfig(bytes.NewReader(item.Policy))
|
policy, perr := parsePolicySnapshotBytes(item.Policy)
|
||||||
if perr != nil {
|
if perr != nil {
|
||||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, perr), r.URL)
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, perr), r.URL)
|
||||||
return
|
return
|
||||||
@@ -222,10 +222,10 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
|
|||||||
default:
|
default:
|
||||||
err = globalSiteReplicationSys.PeerBucketMetadataUpdateHandler(ctx, item)
|
err = globalSiteReplicationSys.PeerBucketMetadataUpdateHandler(ctx, item)
|
||||||
case madmin.SRBucketMetaTypePolicy:
|
case madmin.SRBucketMetaTypePolicy:
|
||||||
if item.Policy == nil {
|
if item.Policy == nil || bytes.Equal(bytes.TrimSpace(item.Policy), []byte("null")) {
|
||||||
err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt)
|
err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt)
|
||||||
} else {
|
} else {
|
||||||
bktPolicy, berr := policy.ParseBucketPolicyConfig(bytes.NewReader(item.Policy), item.Bucket)
|
bktPolicy, berr := parseBucketPolicyStrictBytes(item.Policy, item.Bucket)
|
||||||
if berr != nil {
|
if berr != nil {
|
||||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, berr), r.URL)
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, berr), r.URL)
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -169,8 +169,9 @@ func TestIAMAdminEmptyPolicyDelegation(t *testing.T) {
|
|||||||
{"{}", http.StatusOK, true},
|
{"{}", http.StatusOK, true},
|
||||||
{`{"Statement":null}`, http.StatusOK, true},
|
{`{"Statement":null}`, http.StatusOK, true},
|
||||||
{limitedPolicy, http.StatusOK, false},
|
{limitedPolicy, http.StatusOK, false},
|
||||||
{`{"Version":"2012-10-17"}`, http.StatusForbidden, false},
|
{`{"Version":"2012-10-17"}`, http.StatusOK, true},
|
||||||
{`{"Statement":[]}`, http.StatusForbidden, false},
|
{`{"Statement":[]}`, http.StatusOK, true},
|
||||||
|
{`{"Version":"2012-10-17","Statement":[]}`, http.StatusOK, true},
|
||||||
} {
|
} {
|
||||||
expires := UTCNow().Add(time.Hour)
|
expires := UTCNow().Add(time.Hour)
|
||||||
req := madmin.AddServiceAccountReq{Expiration: &expires}
|
req := madmin.AddServiceAccountReq{Expiration: &expires}
|
||||||
|
|||||||
+60
-63
@@ -889,8 +889,9 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
|
|||||||
// policy the account has. An empty policy document is passed on as
|
// policy the account has. An empty policy document is passed on as
|
||||||
// such: it is the request to remove the embedded policy.
|
// such: it is the request to remove the embedded policy.
|
||||||
var sp *policy.Policy
|
var sp *policy.Policy
|
||||||
if len(updateReq.NewPolicy) > 0 && !bytes.Equal(updateReq.NewPolicy, []byte("null")) {
|
trimmedPolicy := bytes.TrimSpace(updateReq.NewPolicy)
|
||||||
sp, err = policy.ParseConfigStrict(bytes.NewReader(updateReq.NewPolicy))
|
if len(trimmedPolicy) > 0 && !bytes.Equal(trimmedPolicy, []byte("null")) {
|
||||||
|
sp, err = parseSessionPolicyStrictBytes(trimmedPolicy)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
@@ -914,21 +915,13 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Call site replication hook - non-root user accounts are replicated.
|
// Call site replication hook - non-root user accounts are replicated.
|
||||||
|
// The hook sends the account as it is stored after this edit, not the
|
||||||
|
// fields the request changed (see replicationItem).
|
||||||
if svcAccount.ParentUser != globalActiveCred.AccessKey {
|
if svcAccount.ParentUser != globalActiveCred.AccessKey {
|
||||||
replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
|
replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
|
||||||
Type: madmin.SRIAMItemSvcAcc,
|
Type: madmin.SRIAMItemSvcAcc,
|
||||||
SvcAccChange: &madmin.SRSvcAccChange{
|
SvcAccChange: &madmin.SRSvcAccChange{Update: &madmin.SRSvcAccUpdate{AccessKey: accessKey}},
|
||||||
Update: &madmin.SRSvcAccUpdate{
|
UpdatedAt: updatedAt,
|
||||||
AccessKey: accessKey,
|
|
||||||
SecretKey: opts.secretKey,
|
|
||||||
Status: opts.status,
|
|
||||||
Name: opts.name,
|
|
||||||
Description: opts.description,
|
|
||||||
SessionPolicy: madmin.SRSessionPolicy(updateReq.NewPolicy),
|
|
||||||
Expiration: updateReq.NewExpiration,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
UpdatedAt: updatedAt,
|
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1774,7 +1767,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
iamPolicy, err := policy.ParseConfigStrict(bytes.NewReader(iamPolicyBytes))
|
iamPolicy, err := parsePolicyStrictBytes(iamPolicyBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
@@ -2399,37 +2392,13 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
|
|||||||
// skip site-replication service account.
|
// skip site-replication service account.
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
claims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, acc.Credentials.AccessKey)
|
// The stored record only: this server's cache may not have
|
||||||
|
// heard of an edit made through another server.
|
||||||
|
svcAccts[user], err = serviceAccountSnapshot(acc)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
|
writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
sa, policy, err := globalIAMSys.GetServiceAccount(ctx, acc.Credentials.AccessKey)
|
|
||||||
if err != nil {
|
|
||||||
writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
var policyJSON []byte
|
|
||||||
if policy != nil {
|
|
||||||
policyJSON, err = json.Marshal(policy)
|
|
||||||
if err != nil {
|
|
||||||
writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
svcAccts[user] = madmin.SRSvcAccCreate{
|
|
||||||
Parent: acc.Credentials.ParentUser,
|
|
||||||
AccessKey: user,
|
|
||||||
SecretKey: acc.Credentials.SecretKey,
|
|
||||||
Groups: acc.Credentials.Groups,
|
|
||||||
Claims: claims,
|
|
||||||
SessionPolicy: policyJSON,
|
|
||||||
Status: acc.Credentials.Status,
|
|
||||||
Name: sa.Name,
|
|
||||||
Description: sa.Description,
|
|
||||||
Expiration: &sa.Expiration,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
svcAccData, err := json.Marshal(svcAccts)
|
svcAccData, err := json.Marshal(svcAccts)
|
||||||
@@ -2542,23 +2511,34 @@ func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiV
|
|||||||
return
|
return
|
||||||
default:
|
default:
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
var allPolicies map[string]policy.Policy
|
var policyDocuments map[string]json.RawMessage
|
||||||
data, err = io.ReadAll(f)
|
data, err = io.ReadAll(f)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allPoliciesFile, ""), r.URL)
|
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allPoliciesFile, ""), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
err = json.Unmarshal(data, &allPolicies)
|
err = json.Unmarshal(data, &policyDocuments)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, allPoliciesFile, ""), r.URL)
|
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, allPoliciesFile, ""), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
// Validate the entire policy file before applying any entries. Keep
|
||||||
|
// legacy semantic validation, but never discard unknown wire fields.
|
||||||
|
allPolicies := make(map[string]*policy.Policy, len(policyDocuments))
|
||||||
|
for policyName, document := range policyDocuments {
|
||||||
|
parsed, parseErr := parsePolicySnapshotBytes(document)
|
||||||
|
if parseErr != nil {
|
||||||
|
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, parseErr, allPoliciesFile, policyName), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
allPolicies[policyName] = parsed
|
||||||
|
}
|
||||||
for policyName, policy := range allPolicies {
|
for policyName, policy := range allPolicies {
|
||||||
if policy.IsEmpty() {
|
if policy.IsEmpty() {
|
||||||
err = globalIAMSys.DeletePolicy(ctx, policyName, true)
|
err = globalIAMSys.DeletePolicy(ctx, policyName, true)
|
||||||
removed.Policies = append(removed.Policies, policyName)
|
removed.Policies = append(removed.Policies, policyName)
|
||||||
} else {
|
} else {
|
||||||
_, err = globalIAMSys.SetPolicy(ctx, policyName, policy)
|
_, err = globalIAMSys.SetPolicy(ctx, policyName, *policy)
|
||||||
added.Policies = append(added.Policies, policyName)
|
added.Policies = append(added.Policies, policyName)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -2650,8 +2630,17 @@ func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiV
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if _, gerr := globalIAMSys.AddUsersToGroup(ctx, group, grpInfo.Members); gerr != nil {
|
_, gerr := globalIAMSys.AddUsersToGroup(ctx, group, grpInfo.Members)
|
||||||
failed.Groups = append(failed.Groups, madmin.IAMErrEntity{Name: group, Error: err})
|
if gerr == nil {
|
||||||
|
switch grpInfo.Status {
|
||||||
|
case statusDisabled:
|
||||||
|
_, gerr = globalIAMSys.SetGroupStatus(ctx, group, false)
|
||||||
|
case statusEnabled:
|
||||||
|
_, gerr = globalIAMSys.SetGroupStatus(ctx, group, true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if gerr != nil {
|
||||||
|
failed.Groups = append(failed.Groups, madmin.IAMErrEntity{Name: group, Error: gerr})
|
||||||
} else {
|
} else {
|
||||||
added.Groups = append(added.Groups, group)
|
added.Groups = append(added.Groups, group)
|
||||||
}
|
}
|
||||||
@@ -2697,10 +2686,13 @@ func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiV
|
|||||||
var sp *policy.Policy
|
var sp *policy.Policy
|
||||||
var err error
|
var err error
|
||||||
if len(svcAcctReq.SessionPolicy) > 0 {
|
if len(svcAcctReq.SessionPolicy) > 0 {
|
||||||
sp, err = policy.ParseConfig(bytes.NewReader(svcAcctReq.SessionPolicy))
|
trimmedSP := bytes.TrimSpace(svcAcctReq.SessionPolicy)
|
||||||
if err != nil {
|
if len(trimmedSP) > 0 && !bytes.Equal(trimmedSP, []byte("null")) {
|
||||||
writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
|
sp, err = parsePolicySnapshotBytes(trimmedSP)
|
||||||
return
|
if err != nil {
|
||||||
|
writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// service account access key cannot have space characters
|
// service account access key cannot have space characters
|
||||||
@@ -2729,14 +2721,16 @@ func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiV
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
opts := newServiceAccountOpts{
|
opts := newServiceAccountOpts{
|
||||||
accessKey: user,
|
accessKey: user,
|
||||||
secretKey: svcAcctReq.SecretKey,
|
secretKey: svcAcctReq.SecretKey,
|
||||||
sessionPolicy: sp,
|
status: svcAcctReq.Status,
|
||||||
claims: svcAcctReq.Claims,
|
sessionPolicy: sp,
|
||||||
name: svcAcctReq.Name,
|
claims: svcAcctReq.Claims,
|
||||||
description: svcAcctReq.Description,
|
name: svcAcctReq.Name,
|
||||||
expiration: svcAcctReq.Expiration,
|
description: svcAcctReq.Description,
|
||||||
allowSiteReplicatorAccount: false,
|
expiration: svcAcctReq.Expiration,
|
||||||
|
allowSiteReplicatorAccount: false,
|
||||||
|
preserveEmptyEmbeddedPolicy: svcAcctClaimsHaveEmbeddedPolicy(svcAcctReq.Claims),
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, _, err = globalIAMSys.NewServiceAccount(ctx, svcAcctReq.Parent, svcAcctReq.Groups, opts); err != nil {
|
if _, _, err = globalIAMSys.NewServiceAccount(ctx, svcAcctReq.Parent, svcAcctReq.Groups, opts); err != nil {
|
||||||
@@ -3080,14 +3074,17 @@ func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.
|
|||||||
}
|
}
|
||||||
|
|
||||||
var sp *policy.Policy
|
var sp *policy.Policy
|
||||||
if len(createReq.Policy) > 0 {
|
trimmedCreatePolicy := bytes.TrimSpace(createReq.Policy)
|
||||||
sp, err = policy.ParseConfigStrict(bytes.NewReader(createReq.Policy))
|
if bytes.Equal(trimmedCreatePolicy, []byte("null")) {
|
||||||
|
createReq.Policy = nil
|
||||||
|
} else if len(trimmedCreatePolicy) > 0 {
|
||||||
|
sp, err = parseSessionPolicyStrictBytes(trimmedCreatePolicy)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err)
|
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err)
|
||||||
}
|
}
|
||||||
// Empty service policies mean inheritance, including for expiring keys.
|
// Empty service policies mean inheritance, including for expiring keys.
|
||||||
// Normalize the replication request as well as the local credential.
|
// Normalize the replication request as well as the local credential.
|
||||||
if sp.Version == "" && sp.Statements == nil && sp.ID == "" {
|
if sp.IsEmpty() {
|
||||||
sp = nil
|
sp = nil
|
||||||
createReq.Policy = nil
|
createReq.Policy = nil
|
||||||
}
|
}
|
||||||
|
|||||||
+7
-2
@@ -134,8 +134,8 @@ func setObjectHeaders(ctx context.Context, w http.ResponseWriter, objInfo Object
|
|||||||
w.Header().Set(xhttp.Expires, objInfo.Expires.UTC().Format(http.TimeFormat))
|
w.Header().Set(xhttp.Expires, objInfo.Expires.UTC().Format(http.TimeFormat))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set tag count if object has tags
|
// Both the count and the optional values require permission to read tags.
|
||||||
if len(objInfo.UserTags) > 0 {
|
if opts.TaggingCount && len(objInfo.UserTags) > 0 {
|
||||||
tags, _ := tags.ParseObjectTags(objInfo.UserTags)
|
tags, _ := tags.ParseObjectTags(objInfo.UserTags)
|
||||||
if tags != nil && tags.Count() > 0 {
|
if tags != nil && tags.Count() > 0 {
|
||||||
w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(tags.Count())}
|
w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(tags.Count())}
|
||||||
@@ -159,6 +159,11 @@ func setObjectHeaders(ctx context.Context, w http.ResponseWriter, objInfo Object
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Tag count and tag values are gated above by opts.TaggingCount / opts.Tagging.
|
||||||
|
if equals(k, xhttp.AmzTagCount, xhttp.AmzObjectTagging) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
|
// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
|
||||||
if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
|
if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
|
||||||
continue
|
continue
|
||||||
|
|||||||
+6
-9
@@ -568,18 +568,15 @@ func generateListVersionsResponse(ctx context.Context, bucket, prefix, marker, v
|
|||||||
DisplayName: "minio",
|
DisplayName: "minio",
|
||||||
}
|
}
|
||||||
data := ListVersionsResponse{}
|
data := ListVersionsResponse{}
|
||||||
var lastObjMetaName string
|
|
||||||
var tagErr, metaErr APIErrorCode = -1, -1
|
var tagErr, metaErr APIErrorCode = -1, -1
|
||||||
|
|
||||||
for _, object := range resp.Objects {
|
for _, object := range resp.Objects {
|
||||||
if object.Name == "" {
|
if object.Name == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// Cache checks for the same object
|
if metadata != nil {
|
||||||
if metadata != nil && lastObjMetaName != object.Name {
|
tagErr = metadata(object, policy.GetObjectTaggingAction)
|
||||||
tagErr = metadata(object.Name, policy.GetObjectTaggingAction)
|
metaErr = metadata(object, policy.GetObjectAction)
|
||||||
metaErr = metadata(object.Name, policy.GetObjectAction)
|
|
||||||
lastObjMetaName = object.Name
|
|
||||||
}
|
}
|
||||||
content := ObjectVersion{}
|
content := ObjectVersion{}
|
||||||
content.Key = s3EncodeName(object.Name, encodingType)
|
content.Key = s3EncodeName(object.Name, encodingType)
|
||||||
@@ -727,10 +724,10 @@ func generateListObjectsV2Response(ctx context.Context, bucket, prefix, token, n
|
|||||||
}
|
}
|
||||||
content.Owner = owner
|
content.Owner = owner
|
||||||
if metadata != nil {
|
if metadata != nil {
|
||||||
if metadata(object.Name, policy.GetObjectTaggingAction) == ErrNone {
|
if metadata(object, policy.GetObjectTaggingAction) == ErrNone {
|
||||||
content.UserTags = object.UserTags
|
content.UserTags = object.UserTags
|
||||||
}
|
}
|
||||||
if metadata(object.Name, policy.GetObjectAction) == ErrNone {
|
if metadata(object, policy.GetObjectAction) == ErrNone {
|
||||||
content.UserMetadata = &Metadata{}
|
content.UserMetadata = &Metadata{}
|
||||||
switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
|
switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
|
||||||
case crypto.S3:
|
case crypto.S3:
|
||||||
@@ -774,7 +771,7 @@ func generateListObjectsV2Response(ctx context.Context, bucket, prefix, token, n
|
|||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
|
|
||||||
type metaCheckFn = func(name string, action policy.Action) (s3Err APIErrorCode)
|
type metaCheckFn = func(object ObjectInfo, action policy.Action) (s3Err APIErrorCode)
|
||||||
|
|
||||||
// generates CopyObjectResponse from the committed object information.
|
// generates CopyObjectResponse from the committed object information.
|
||||||
func generateCopyObjectResponse(oi ObjectInfo, cs map[string]string) CopyObjectResponse {
|
func generateCopyObjectResponse(oi ObjectInfo, cs map[string]string) CopyObjectResponse {
|
||||||
|
|||||||
@@ -0,0 +1,716 @@
|
|||||||
|
// Copyright (c) 2026 Denis Babchuk
|
||||||
|
//
|
||||||
|
// This file is part of Silo Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"maps"
|
||||||
|
"net/http"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/minio/minio-go/v7/pkg/set"
|
||||||
|
"github.com/minio/minio-go/v7/pkg/tags"
|
||||||
|
"github.com/minio/minio/internal/arn"
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
"github.com/minio/minio/internal/logger"
|
||||||
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
|
"github.com/pgsty/silo-pkg/v3/policy/condition"
|
||||||
|
"github.com/puzpuzpuz/xsync/v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
func withScopedReqObject(ctx context.Context, bucket, object, versionID string, fn func() APIErrorCode) APIErrorCode {
|
||||||
|
reqInfo := logger.GetReqInfo(ctx)
|
||||||
|
if reqInfo == nil {
|
||||||
|
return ErrAccessDenied
|
||||||
|
}
|
||||||
|
prevBucket, prevObject, prevVID := reqInfo.BucketName, reqInfo.ObjectName, reqInfo.VersionID
|
||||||
|
reqInfo.BucketName = bucket
|
||||||
|
reqInfo.ObjectName = object
|
||||||
|
reqInfo.VersionID = versionID
|
||||||
|
defer func() {
|
||||||
|
reqInfo.BucketName = prevBucket
|
||||||
|
reqInfo.ObjectName = prevObject
|
||||||
|
reqInfo.VersionID = prevVID
|
||||||
|
}()
|
||||||
|
return fn()
|
||||||
|
}
|
||||||
|
|
||||||
|
func objectVersionAction(base policy.Action, versionID string) (policy.Action, bool) {
|
||||||
|
if versionID == "" {
|
||||||
|
return base, false
|
||||||
|
}
|
||||||
|
switch base {
|
||||||
|
case policy.GetObjectAction:
|
||||||
|
return policy.GetObjectVersionAction, true
|
||||||
|
case policy.GetObjectTaggingAction:
|
||||||
|
return policy.GetObjectVersionTaggingAction, true
|
||||||
|
case policy.PutObjectTaggingAction:
|
||||||
|
return policy.PutObjectVersionTaggingAction, true
|
||||||
|
case policy.DeleteObjectTaggingAction:
|
||||||
|
return policy.DeleteObjectVersionTaggingAction, true
|
||||||
|
default:
|
||||||
|
return base, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sys *IAMSys) policySetsForExplicitDeny(cred auth.Credentials) [][]policy.Policy {
|
||||||
|
derived := cred.IsTemp() || cred.IsServiceAccount()
|
||||||
|
name := cred.AccessKey
|
||||||
|
if derived {
|
||||||
|
name = cred.ParentUser
|
||||||
|
}
|
||||||
|
cache := sys.store.rlock()
|
||||||
|
var names []string
|
||||||
|
roleARN := (policy.Args{Claims: cred.Claims}).GetRoleArn()
|
||||||
|
switch {
|
||||||
|
case name == globalActiveCred.AccessKey:
|
||||||
|
case derived && roleARN != "":
|
||||||
|
parsed, err := arn.Parse(roleARN)
|
||||||
|
if err != nil {
|
||||||
|
sys.store.runlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
names = newMappedPolicy(sys.rolesMap[parsed]).toSlice()
|
||||||
|
default:
|
||||||
|
var err error
|
||||||
|
names, _, err = cache.policyDBGet(sys.store, name, false, false)
|
||||||
|
if err != nil && err != errNoSuchPolicy {
|
||||||
|
sys.store.runlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
groups, gerr := cache.policyDBGetGroups(sys.store, len(names) > 0, cred.Groups...)
|
||||||
|
if gerr != nil && gerr != errNoSuchPolicy {
|
||||||
|
sys.store.runlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
names = append(names, groups...)
|
||||||
|
if derived && len(names) == 0 {
|
||||||
|
claims, _ := policy.GetPoliciesFromClaims(cred.Claims, iamPolicyClaimNameOpenID())
|
||||||
|
names = claims.ToSlice()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sys.store.runlock()
|
||||||
|
|
||||||
|
sets := [][]policy.Policy{nil}
|
||||||
|
slices.Sort(names)
|
||||||
|
compactNames := slices.Compact(names)
|
||||||
|
if len(compactNames) > 0 {
|
||||||
|
if avail, merged, err := sys.store.MergePolicies(strings.Join(compactNames, ",")); err == nil && avail != "" {
|
||||||
|
sets[0] = append(sets[0], merged)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if derived && (cred.IsTemp() || cred.Claims[iamPolicyClaimNameSA()] != inheritedPolicyType) {
|
||||||
|
if sp, ok := cred.Claims[sessionPolicyNameExtracted].(string); ok {
|
||||||
|
if p, err := policy.ParseConfig(strings.NewReader(sp)); err == nil {
|
||||||
|
sets = append(sets, []policy.Policy{*p})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sets
|
||||||
|
}
|
||||||
|
|
||||||
|
func isActionExplicitlyDenied(ctx context.Context, r *http.Request, action policy.Action, existingTags string, requestTags *string) bool {
|
||||||
|
reqInfo := logger.GetReqInfo(ctx)
|
||||||
|
if reqInfo == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if reqInfo.Owner {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
cred := reqInfo.Cred
|
||||||
|
bucket := reqInfo.BucketName
|
||||||
|
object := reqInfo.ObjectName
|
||||||
|
versionID := reqInfo.VersionID
|
||||||
|
buildValues := func(loc string, c auth.Credentials) map[string][]string {
|
||||||
|
values := getConditionValuesWithTags(r, loc, c, existingTags, requestTags)
|
||||||
|
if actionUsesObjectVersion(action) {
|
||||||
|
if versionID == "" {
|
||||||
|
delete(values, "versionid")
|
||||||
|
} else {
|
||||||
|
values["versionid"] = []string{versionID}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return values
|
||||||
|
}
|
||||||
|
|
||||||
|
if cred.AccessKey == "" {
|
||||||
|
p, err := globalPolicySys.Get(bucket)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
bArgs := policy.BucketPolicyArgs{
|
||||||
|
Action: action,
|
||||||
|
BucketName: bucket,
|
||||||
|
ObjectName: object,
|
||||||
|
ConditionValues: buildValues(reqInfo.Region, auth.AnonymousCredentials),
|
||||||
|
IsOwner: false,
|
||||||
|
}
|
||||||
|
for _, st := range p.Statements {
|
||||||
|
if st.Effect != policy.Deny {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
probe := st
|
||||||
|
probe.Effect = policy.Allow
|
||||||
|
if probe.IsAllowed(bArgs) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
values := buildValues("", cred)
|
||||||
|
if newGlobalAuthZPluginFn() != nil {
|
||||||
|
return !globalIAMSys.IsAllowed(policy.Args{
|
||||||
|
AccountName: cred.AccessKey,
|
||||||
|
Groups: cred.Groups,
|
||||||
|
Action: action,
|
||||||
|
BucketName: bucket,
|
||||||
|
ObjectName: object,
|
||||||
|
ConditionValues: values,
|
||||||
|
IsOwner: false,
|
||||||
|
Claims: cred.Claims,
|
||||||
|
DenyOnly: true,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if !globalIAMSys.Initialized() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
args := policy.Args{
|
||||||
|
AccountName: cred.AccessKey,
|
||||||
|
Groups: cred.Groups,
|
||||||
|
Action: action,
|
||||||
|
BucketName: bucket,
|
||||||
|
ObjectName: object,
|
||||||
|
ConditionValues: values,
|
||||||
|
IsOwner: false,
|
||||||
|
Claims: cred.Claims,
|
||||||
|
DenyOnly: true,
|
||||||
|
}
|
||||||
|
for _, policies := range globalIAMSys.policySetsForExplicitDeny(cred) {
|
||||||
|
if len(policies) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
merged := policy.MergePolicies(policies...)
|
||||||
|
if !merged.IsAllowed(args) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func authorizeVersionedRequestWithTags(ctx context.Context, r *http.Request, baseAction policy.Action, versionID, existingTags string, requestTags *string) APIErrorCode {
|
||||||
|
reqInfo := logger.GetReqInfo(ctx)
|
||||||
|
if reqInfo == nil {
|
||||||
|
return ErrAccessDenied
|
||||||
|
}
|
||||||
|
prevVID := reqInfo.VersionID
|
||||||
|
reqInfo.VersionID = versionID
|
||||||
|
defer func() { reqInfo.VersionID = prevVID }()
|
||||||
|
|
||||||
|
if verAction, ok := objectVersionAction(baseAction, versionID); ok {
|
||||||
|
if authorizeRequestWithTags(ctx, r, verAction, existingTags, requestTags) == ErrNone {
|
||||||
|
return ErrNone
|
||||||
|
}
|
||||||
|
if isActionExplicitlyDenied(ctx, r, verAction, existingTags, requestTags) {
|
||||||
|
return ErrAccessDenied
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return authorizeRequestWithTags(ctx, r, baseAction, existingTags, requestTags)
|
||||||
|
}
|
||||||
|
|
||||||
|
func authorizeVersionedRequestWithExistingTags(ctx context.Context, r *http.Request, baseAction policy.Action, versionID, existingTags string) APIErrorCode {
|
||||||
|
return authorizeVersionedRequestWithTags(ctx, r, baseAction, versionID, existingTags, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func authorizeVersionedRequestWithUnreadTags(ctx context.Context, r *http.Request, baseAction policy.Action, versionID string, readErr error) APIErrorCode {
|
||||||
|
s3Err := authorizeVersionedRequestWithExistingTags(ctx, r, baseAction, versionID, "")
|
||||||
|
if s3Err == ErrNone || toAPIError(ctx, readErr).HTTPStatusCode < http.StatusInternalServerError {
|
||||||
|
return s3Err
|
||||||
|
}
|
||||||
|
return authorizeVersionedRequestWithTagCandidates(ctx, r, baseAction, versionID, s3Err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func authorizeVersionedRequestWithPotentialTags(ctx context.Context, r *http.Request, baseAction policy.Action, versionID string) APIErrorCode {
|
||||||
|
s3Err := authorizeVersionedRequestWithExistingTags(ctx, r, baseAction, versionID, "")
|
||||||
|
if s3Err == ErrNone {
|
||||||
|
return ErrNone
|
||||||
|
}
|
||||||
|
return authorizeVersionedRequestWithTagCandidates(ctx, r, baseAction, versionID, s3Err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func authorizeVersionedRequestWithTagCandidates(ctx context.Context, r *http.Request, baseAction policy.Action, versionID string, s3Err APIErrorCode) APIErrorCode {
|
||||||
|
reqInfo := logger.GetReqInfo(ctx)
|
||||||
|
if reqInfo == nil {
|
||||||
|
return ErrAccessDenied
|
||||||
|
}
|
||||||
|
prevVID := reqInfo.VersionID
|
||||||
|
reqInfo.VersionID = versionID
|
||||||
|
defer func() { reqInfo.VersionID = prevVID }()
|
||||||
|
|
||||||
|
candidates := policyExistingObjectTags(ctx, r, baseAction)
|
||||||
|
if verAction, ok := objectVersionAction(baseAction, versionID); ok {
|
||||||
|
candidates = append(candidates, policyExistingObjectTags(ctx, r, verAction)...)
|
||||||
|
}
|
||||||
|
for _, existingTags := range slices.Compact(candidates) {
|
||||||
|
if authorizeVersionedRequestWithExistingTags(ctx, r, baseAction, versionID, existingTags) == ErrNone {
|
||||||
|
return ErrNone
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return s3Err
|
||||||
|
}
|
||||||
|
|
||||||
|
// canPotentiallyAllowAction reports whether the authenticated caller has any
|
||||||
|
// grant capable of allowing action on bucket/object once condition values
|
||||||
|
// (such as copied source tags on CopyObject) are resolved.
|
||||||
|
func canPotentiallyAllowAction(ctx context.Context, r *http.Request, action policy.Action, bucket, object string) bool {
|
||||||
|
reqInfo := logger.GetReqInfo(ctx)
|
||||||
|
if reqInfo == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
emptyTags := ""
|
||||||
|
if withScopedReqObject(ctx, bucket, object, "", func() APIErrorCode {
|
||||||
|
return authorizeRequestWithTags(ctx, r, action, "", &emptyTags)
|
||||||
|
}) == ErrNone {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if reqInfo.Owner {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
cred := reqInfo.Cred
|
||||||
|
if cred.AccessKey == "" {
|
||||||
|
p, err := globalPolicySys.Get(bucket)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
values := getConditionValues(r, reqInfo.Region, auth.AnonymousCredentials)
|
||||||
|
for _, st := range p.Statements {
|
||||||
|
if st.Effect != policy.Allow {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
probe := st
|
||||||
|
probe.Conditions = nil
|
||||||
|
if probe.IsAllowed(policy.BucketPolicyArgs{Action: action, BucketName: bucket, ObjectName: object, ConditionValues: values}) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if newGlobalAuthZPluginFn() != nil || !globalIAMSys.Initialized() {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
values := getConditionValues(r, "", cred)
|
||||||
|
for _, policies := range globalIAMSys.policySetsForExplicitDeny(cred) {
|
||||||
|
setAllows := false
|
||||||
|
for _, p := range policies {
|
||||||
|
for _, st := range p.Statements {
|
||||||
|
if st.Effect != policy.Allow {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
probe := st
|
||||||
|
probe.Conditions = nil
|
||||||
|
if probe.IsAllowed(policy.Args{
|
||||||
|
AccountName: cred.AccessKey,
|
||||||
|
Groups: cred.Groups,
|
||||||
|
Action: action,
|
||||||
|
BucketName: bucket,
|
||||||
|
ObjectName: object,
|
||||||
|
ConditionValues: values,
|
||||||
|
Claims: cred.Claims,
|
||||||
|
}) {
|
||||||
|
setAllows = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if setAllows {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(policies) > 0 && !setAllows {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if len(policies) == 0 && cred.ParentUser != globalActiveCred.AccessKey && cred.AccessKey != globalActiveCred.AccessKey {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func canReturnObjectTagsHeader(ctx context.Context, r *http.Request, versionID, existingTags string) bool {
|
||||||
|
reqInfo := logger.GetReqInfo(ctx)
|
||||||
|
if reqInfo == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
prevVID := reqInfo.VersionID
|
||||||
|
reqInfo.VersionID = versionID
|
||||||
|
defer func() { reqInfo.VersionID = prevVID }()
|
||||||
|
|
||||||
|
if isActionExplicitlyDenied(ctx, r, policy.GetObjectTaggingAction, existingTags, nil) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if verAction, ok := objectVersionAction(policy.GetObjectTaggingAction, versionID); ok {
|
||||||
|
if isActionExplicitlyDenied(ctx, r, verAction, existingTags, nil) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, versionID, existingTags) == ErrNone {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return reqInfo.Cred.AccessKey != "" &&
|
||||||
|
isRequestSignatureV4(r) &&
|
||||||
|
!isRequestPresignedSignatureV4(r) &&
|
||||||
|
authorizeRequestWithExistingTags(ctx, r, policy.ReplicateTagsAction, existingTags) == ErrNone
|
||||||
|
}
|
||||||
|
|
||||||
|
// authorizeRequestWithUnreadTags decides whether to disclose readErr for an
|
||||||
|
// object whose stored tags could not be read. It must not grant object access.
|
||||||
|
//
|
||||||
|
// Without the tags, an Allow statement that tests s3:ExistingObjectTag cannot
|
||||||
|
// match, so a caller allowed only by such a statement would get AccessDenied,
|
||||||
|
// which clients do not retry, for a transient server error such as lost read
|
||||||
|
// quorum. For a server error, try a bounded set of tags from the caller's
|
||||||
|
// policies and return the storage error if normal authorization accepts one.
|
||||||
|
// If no candidate passes, keep AccessDenied. This is not an exhaustive policy
|
||||||
|
// solver. For any other error, NoSuchKey among them, the tags stay empty.
|
||||||
|
func authorizeRequestWithUnreadTags(ctx context.Context, r *http.Request, action policy.Action, readErr error) APIErrorCode {
|
||||||
|
s3Err := authorizeRequest(ctx, r, action)
|
||||||
|
if s3Err == ErrNone || toAPIError(ctx, readErr).HTTPStatusCode < http.StatusInternalServerError {
|
||||||
|
return s3Err
|
||||||
|
}
|
||||||
|
return authorizeRequestWithTagCandidates(ctx, r, action, s3Err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func authorizeRequestWithTagCandidates(ctx context.Context, r *http.Request, action policy.Action, s3Err APIErrorCode) APIErrorCode {
|
||||||
|
for _, existingTags := range policyExistingObjectTags(ctx, r, action) {
|
||||||
|
if authorizeRequestWithExistingTags(ctx, r, action, existingTags) == ErrNone {
|
||||||
|
return ErrNone
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return s3Err
|
||||||
|
}
|
||||||
|
|
||||||
|
// maxExistingTagCandidates bounds the tag sets tried for one request; each
|
||||||
|
// one costs an authorization.
|
||||||
|
const maxExistingTagCandidates = 64
|
||||||
|
|
||||||
|
// policyExistingObjectTags returns tag sets under which the policies that
|
||||||
|
// authorize the request may let action through on its object: the bucket
|
||||||
|
// policy for an anonymous request, else the cached IAM policies and, as a
|
||||||
|
// second policy set, the session policy of the credential. A tag set passes
|
||||||
|
// the s3:ExistingObjectTag tests of one Allow statement of each policy set and,
|
||||||
|
// as far as the same tags allow, keeps the Deny statements that test tags from
|
||||||
|
// matching. The tag sets are only candidates; the regular authorization
|
||||||
|
// decides what they allow.
|
||||||
|
func policyExistingObjectTags(ctx context.Context, r *http.Request, action policy.Action) []string {
|
||||||
|
reqInfo := logger.GetReqInfo(ctx)
|
||||||
|
if reqInfo == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
bucket, object := reqInfo.BucketName, reqInfo.ObjectName
|
||||||
|
var (
|
||||||
|
allows [][]map[string]string // per policy set, the tags its Allow statements ask for
|
||||||
|
denies [][]map[string]string // per Deny statement, alternative tags that keep it from matching
|
||||||
|
)
|
||||||
|
// add collects the tags of a statement that applies to action on the
|
||||||
|
// object, whatever its conditions.
|
||||||
|
add := func(effect policy.Effect, conditions condition.Functions, values map[string][]string, fromSet *[]map[string]string) {
|
||||||
|
candidates := existingObjectTagsCandidatesFor(effect, conditions, values)
|
||||||
|
if effect == policy.Allow {
|
||||||
|
*fromSet = append(*fromSet, candidates...)
|
||||||
|
} else if len(candidates) > 0 {
|
||||||
|
denies = append(denies, candidates)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case reqInfo.Cred.AccessKey == "":
|
||||||
|
p, err := globalPolicySys.Get(bucket)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
values := getConditionValues(r, reqInfo.Region, auth.AnonymousCredentials)
|
||||||
|
var fromSet []map[string]string
|
||||||
|
for _, st := range p.Statements {
|
||||||
|
match := st
|
||||||
|
match.Effect, match.Conditions = policy.Allow, nil
|
||||||
|
if match.IsAllowed(policy.BucketPolicyArgs{Action: action, BucketName: bucket, ObjectName: object, ConditionValues: values}) {
|
||||||
|
add(st.Effect, st.Conditions, values, &fromSet)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
allows = append(allows, fromSet)
|
||||||
|
case newGlobalAuthZPluginFn() == nil && globalIAMSys.Initialized():
|
||||||
|
values := getConditionValues(r, "", reqInfo.Cred)
|
||||||
|
policySets := globalIAMSys.unreadTagPolicySets(reqInfo.Cred)
|
||||||
|
for _, policies := range policySets {
|
||||||
|
var fromSet []map[string]string
|
||||||
|
for _, p := range policies {
|
||||||
|
for _, st := range p.Statements {
|
||||||
|
match := st
|
||||||
|
match.Effect, match.Conditions = policy.Allow, nil
|
||||||
|
if match.IsAllowed(policy.Args{Action: action, BucketName: bucket, ObjectName: object, ConditionValues: values}) {
|
||||||
|
add(st.Effect, st.Conditions, values, &fromSet)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
allows = append(allows, fromSet)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return combineExistingObjectTags(append(allows, denies...))
|
||||||
|
}
|
||||||
|
|
||||||
|
// unreadTagMappings reuses the normal user/group mapping lookup without lazy
|
||||||
|
// storage reads. Missing cache entries contribute no candidates; the regular
|
||||||
|
// authorization still decides whether any candidate is allowed.
|
||||||
|
type unreadTagMappings struct {
|
||||||
|
*IAMStoreSys
|
||||||
|
}
|
||||||
|
|
||||||
|
func (unreadTagMappings) loadMappedPolicy(context.Context, string, IAMUserType, bool, *xsync.MapOf[string, MappedPolicy]) error {
|
||||||
|
return errNoSuchPolicy
|
||||||
|
}
|
||||||
|
|
||||||
|
func (unreadTagMappings) loadGroup(context.Context, string, map[string]GroupInfo) error {
|
||||||
|
return errNoSuchGroup
|
||||||
|
}
|
||||||
|
|
||||||
|
// unreadTagPolicySets selects only this credential's cached policies. Base
|
||||||
|
// and session policies remain separate because both must allow the request.
|
||||||
|
// This only supplies candidates, so identity, claims and policy validity are
|
||||||
|
// left to the normal authorization on each candidate.
|
||||||
|
func (sys *IAMSys) unreadTagPolicySets(cred auth.Credentials) [][]policy.Policy {
|
||||||
|
derived := cred.IsTemp() || cred.IsServiceAccount()
|
||||||
|
name := cred.AccessKey
|
||||||
|
if derived {
|
||||||
|
name = cred.ParentUser
|
||||||
|
}
|
||||||
|
cache := sys.store.rlock()
|
||||||
|
defer sys.store.runlock()
|
||||||
|
var names []string
|
||||||
|
roleARN := (policy.Args{Claims: cred.Claims}).GetRoleArn()
|
||||||
|
switch {
|
||||||
|
case name == globalActiveCred.AccessKey:
|
||||||
|
case derived && roleARN != "":
|
||||||
|
parsed, err := arn.Parse(roleARN)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
names = newMappedPolicy(sys.rolesMap[parsed]).toSlice()
|
||||||
|
default:
|
||||||
|
loader := unreadTagMappings{sys.store}
|
||||||
|
var err error
|
||||||
|
names, _, err = cache.policyDBGet(loader, name, false, false)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
groups, err := cache.policyDBGetGroups(loader, len(names) > 0, cred.Groups...)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
names = append(names, groups...)
|
||||||
|
if derived && len(names) == 0 {
|
||||||
|
claims, _ := policy.GetPoliciesFromClaims(cred.Claims, iamPolicyClaimNameOpenID())
|
||||||
|
names = claims.ToSlice()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sets := [][]policy.Policy{nil}
|
||||||
|
slices.Sort(names)
|
||||||
|
for _, name := range slices.Compact(names) {
|
||||||
|
if doc, ok := cache.iamPolicyDocsMap[name]; ok {
|
||||||
|
sets[0] = append(sets[0], doc.Policy)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if derived && (cred.IsTemp() || cred.Claims[iamPolicyClaimNameSA()] != inheritedPolicyType) {
|
||||||
|
if sp, ok := cred.Claims[sessionPolicyNameExtracted].(string); ok {
|
||||||
|
if p, err := policy.ParseConfig(strings.NewReader(sp)); err == nil {
|
||||||
|
sets = append(sets, []policy.Policy{*p})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sets
|
||||||
|
}
|
||||||
|
|
||||||
|
// combineExistingObjectTags combines alternatives from each base/session
|
||||||
|
// Allow set and each tag-dependent Deny. A set may also contribute no tags:
|
||||||
|
// another statement or another set may already supply what it needs. First
|
||||||
|
// values win conflicts; the normal authorization decides whether they work.
|
||||||
|
func combineExistingObjectTags(sets [][]map[string]string) []string {
|
||||||
|
combos := []map[string]string{{}}
|
||||||
|
for _, fromSet := range sets {
|
||||||
|
fromSet = append(uniqueExistingObjectTagMaps(fromSet), nil)
|
||||||
|
var next []map[string]string
|
||||||
|
seen := set.NewStringSet()
|
||||||
|
combinations:
|
||||||
|
for _, combo := range combos {
|
||||||
|
for _, addition := range fromSet {
|
||||||
|
merged := maps.Clone(combo)
|
||||||
|
for k, v := range addition {
|
||||||
|
if _, ok := merged[k]; !ok {
|
||||||
|
merged[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t, err := tags.MapToObjectTags(merged)
|
||||||
|
if err != nil || seen.Contains(t.String()) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen.Add(t.String())
|
||||||
|
next = append(next, merged)
|
||||||
|
if len(next) == maxExistingTagCandidates {
|
||||||
|
break combinations
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
combos = next
|
||||||
|
}
|
||||||
|
var candidates []string
|
||||||
|
for _, combo := range combos {
|
||||||
|
if len(combo) != 0 { // empty tags were already authorized
|
||||||
|
t, _ := tags.MapToObjectTags(combo)
|
||||||
|
candidates = append(candidates, t.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return candidates
|
||||||
|
}
|
||||||
|
|
||||||
|
// existingObjectTagsCandidatesFor returns tag sets under which the
|
||||||
|
// s3:ExistingObjectTag tests of conditions all pass, for an Allow statement,
|
||||||
|
// or one of them fails, for a Deny statement; a tag left out is absent. For an
|
||||||
|
// Allow, values for different tag keys are combined, while a Deny needs only
|
||||||
|
// one failing key. It tries the values named in the conditions, with policy
|
||||||
|
// variables resolved, wildcard fillers, and absence, then lets the policy
|
||||||
|
// engine decide whether each candidate really satisfies the conditions.
|
||||||
|
func existingObjectTagsCandidatesFor(effect policy.Effect, conditions condition.Functions, values map[string][]string) []map[string]string {
|
||||||
|
// Each function tests one key, so the tests of one tag pass or fail
|
||||||
|
// whatever the other tags are.
|
||||||
|
byKey := make(map[string]condition.Functions)
|
||||||
|
for _, f := range conditions {
|
||||||
|
for key := range condition.NewFunctions(f).Keys() {
|
||||||
|
if key.Is(condition.ExistingObjectTag) {
|
||||||
|
byKey[key.Name()] = append(byKey[key.Name()], f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(byKey) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
keys := slices.Sorted(maps.Keys(byKey))
|
||||||
|
if effect == policy.Deny {
|
||||||
|
var candidates []map[string]string
|
||||||
|
for _, key := range keys {
|
||||||
|
candidates = append(candidates, existingObjectTagValues(key, strings.TrimPrefix(key, condition.ExistingObjectTag.Name()+"/"), byKey[key], values, false)...)
|
||||||
|
}
|
||||||
|
return uniqueExistingObjectTagMaps(candidates)
|
||||||
|
}
|
||||||
|
|
||||||
|
candidates := []map[string]string{{}}
|
||||||
|
for _, key := range keys {
|
||||||
|
keyCandidates := existingObjectTagValues(key, strings.TrimPrefix(key, condition.ExistingObjectTag.Name()+"/"), byKey[key], values, true)
|
||||||
|
if len(keyCandidates) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
next := make([]map[string]string, 0, min(maxExistingTagCandidates, len(candidates)*len(keyCandidates)))
|
||||||
|
for _, base := range candidates {
|
||||||
|
for _, keyCandidate := range keyCandidates {
|
||||||
|
merged := maps.Clone(base)
|
||||||
|
maps.Copy(merged, keyCandidate)
|
||||||
|
next = append(next, merged)
|
||||||
|
if len(next) == maxExistingTagCandidates {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(next) == maxExistingTagCandidates {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
candidates = next
|
||||||
|
}
|
||||||
|
return uniqueExistingObjectTagMaps(candidates)
|
||||||
|
}
|
||||||
|
|
||||||
|
func uniqueExistingObjectTagMaps(candidates []map[string]string) []map[string]string {
|
||||||
|
seen := set.NewStringSet()
|
||||||
|
unique := make([]map[string]string, 0, len(candidates))
|
||||||
|
for _, candidate := range candidates {
|
||||||
|
t, err := tags.MapToObjectTags(candidate)
|
||||||
|
if err != nil || seen.Contains(t.String()) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen.Add(t.String())
|
||||||
|
unique = append(unique, candidate)
|
||||||
|
}
|
||||||
|
return unique
|
||||||
|
}
|
||||||
|
|
||||||
|
// existingObjectTagValues looks for values of tag tagKey, or its absence,
|
||||||
|
// under which the functions that test the condition key key evaluate to want.
|
||||||
|
func existingObjectTagValues(key, tagKey string, tests condition.Functions, values map[string][]string, want bool) []map[string]string {
|
||||||
|
defer delete(values, key)
|
||||||
|
delete(values, key)
|
||||||
|
var candidates []map[string]string
|
||||||
|
if tests.Evaluate(values) == want {
|
||||||
|
candidates = append(candidates, map[string]string{})
|
||||||
|
}
|
||||||
|
// The functions keep their values unexported; JSON exposes them.
|
||||||
|
buf, err := json.Marshal(tests)
|
||||||
|
if err != nil {
|
||||||
|
return candidates
|
||||||
|
}
|
||||||
|
var byName map[string]map[string]condition.ValueSet
|
||||||
|
if err = json.Unmarshal(buf, &byName); err != nil {
|
||||||
|
return candidates
|
||||||
|
}
|
||||||
|
var tried []string
|
||||||
|
for name, byKey := range byName {
|
||||||
|
if name == "Null" {
|
||||||
|
continue // its operand says whether the tag is there, not a value
|
||||||
|
}
|
||||||
|
for _, valueSet := range byKey {
|
||||||
|
for _, v := range valueSet.ToSlice() {
|
||||||
|
s := substituteConditionVariables(v.String(), values)
|
||||||
|
tried = append(tried, s, strings.NewReplacer("*", "", "?", "_").Replace(s))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
slices.Sort(tried)
|
||||||
|
for _, v := range append(slices.Compact(tried), "", "_") {
|
||||||
|
if _, err := tags.MapToObjectTags(map[string]string{tagKey: v}); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
values[key] = []string{v}
|
||||||
|
if tests.Evaluate(values) == want {
|
||||||
|
candidates = append(candidates, map[string]string{tagKey: v})
|
||||||
|
if len(candidates) == maxExistingTagCandidates {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return uniqueExistingObjectTagMaps(candidates)
|
||||||
|
}
|
||||||
|
|
||||||
|
// substituteConditionVariables resolves the policy variables in v from the
|
||||||
|
// condition values as the policy engine does for the values of a condition;
|
||||||
|
// silo-pkg keeps its own function for this unexported.
|
||||||
|
func substituteConditionVariables(v string, values map[string][]string) string {
|
||||||
|
for _, key := range condition.CommonKeys {
|
||||||
|
if rvalues, ok := values[key.Name()]; ok && len(rvalues) > 0 && rvalues[0] != "" {
|
||||||
|
v = strings.ReplaceAll(v, key.VarName(), rvalues[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
@@ -0,0 +1,443 @@
|
|||||||
|
// Copyright (c) 2026 Denis Babchuk
|
||||||
|
//
|
||||||
|
// This file is part of Silo Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"encoding/xml"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio-go/v7/pkg/tags"
|
||||||
|
"github.com/minio/minio/internal/arn"
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
|
"github.com/pgsty/silo-pkg/v3/policy/condition"
|
||||||
|
)
|
||||||
|
|
||||||
|
// objectReadFault takes the first n drives of the set that holds object
|
||||||
|
// offline for the metadata reads of that object only, so that IAM and bucket
|
||||||
|
// metadata stay readable. It returns a function that restores the drives.
|
||||||
|
func objectReadFault(z *erasureServerPools, bucket, object string, n int) func() {
|
||||||
|
pool := z.serverPools[0]
|
||||||
|
setIndex := pool.getHashedSetIndex(object)
|
||||||
|
pool.erasureDisksMu.Lock()
|
||||||
|
defer pool.erasureDisksMu.Unlock()
|
||||||
|
previous := pool.erasureDisks[setIndex]
|
||||||
|
disks := append([]StorageAPI(nil), previous...)
|
||||||
|
for i := range n {
|
||||||
|
disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: object}
|
||||||
|
}
|
||||||
|
pool.erasureDisks[setIndex] = disks
|
||||||
|
return func() {
|
||||||
|
pool.erasureDisksMu.Lock()
|
||||||
|
pool.erasureDisks[setIndex] = previous
|
||||||
|
pool.erasureDisksMu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A caller allowed to read an object only through s3:ExistingObjectTag must
|
||||||
|
// get the storage error, which clients retry, and not AccessDenied, when the
|
||||||
|
// object's tags cannot be read because read quorum is lost. A caller that no
|
||||||
|
// tags would let through must still get AccessDenied, and nothing changes
|
||||||
|
// while the tags can be read.
|
||||||
|
func TestExistingObjectTagAuthReadQuorumLost(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
defer cancel()
|
||||||
|
obj, dirs, err := prepareErasure16(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
z := obj.(*erasureServerPools)
|
||||||
|
previous := newObjectLayerFn()
|
||||||
|
setObjectLayer(z)
|
||||||
|
defer func() {
|
||||||
|
z.Shutdown(context.Background())
|
||||||
|
removeRoots(dirs)
|
||||||
|
setObjectLayer(previous)
|
||||||
|
}()
|
||||||
|
bucket, router, err := initAPIHandlerTest(ctx, z, nil, MakeBucketOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
statement := func(effect, resource, conditions string) string {
|
||||||
|
st := fmt.Sprintf(`{"Effect": %q, "Action": ["s3:GetObject", "s3:GetObjectTagging", "s3:PutObjectTagging", "s3:DeleteObjectTagging"], "Resource": ["arn:aws:s3:::%s/%s"]`,
|
||||||
|
effect, bucket, resource)
|
||||||
|
if conditions != "" {
|
||||||
|
st += `, "Condition": ` + conditions
|
||||||
|
}
|
||||||
|
return st + "}"
|
||||||
|
}
|
||||||
|
tagStatement := func(effect, resource, value string) string {
|
||||||
|
return statement(effect, resource, fmt.Sprintf(`{"StringEquals": {"s3:ExistingObjectTag/k": %q}}`, value))
|
||||||
|
}
|
||||||
|
newUser := func(statements ...string) auth.Credentials {
|
||||||
|
return newDeleteAuthzPolicyUser(t, "", bucket, "["+strings.Join(statements, ",")+"]")
|
||||||
|
}
|
||||||
|
newServiceAccount := func(parent auth.Credentials, statements string) auth.Credentials {
|
||||||
|
sessionPolicy, err := policy.ParseConfig(strings.NewReader(`{"Version": "2012-10-17", "Statement": [` + statements + `]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cred, _, err := globalIAMSys.NewServiceAccount(ctx, parent.AccessKey, nil, newServiceAccountOpts{sessionPolicy: sessionPolicy})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return cred
|
||||||
|
}
|
||||||
|
newTempUser := func(parent auth.Credentials, statements string) auth.Credentials {
|
||||||
|
session := `{"Version": "2012-10-17", "Statement": [` + statements + `]}`
|
||||||
|
secret, err := getTokenSigningKey()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{
|
||||||
|
parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(),
|
||||||
|
policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(session)),
|
||||||
|
}, secret)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cred.ParentUser = parent.AccessKey
|
||||||
|
if _, err := globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return cred
|
||||||
|
}
|
||||||
|
// An unrelated cached policy must not consume the candidate limit. The
|
||||||
|
// target user's policy is created below and sorts after this policy.
|
||||||
|
fillerStatements := make([]string, 0, maxExistingTagCandidates)
|
||||||
|
for i := range maxExistingTagCandidates {
|
||||||
|
fillerStatements = append(fillerStatements, statement("Allow", "*",
|
||||||
|
fmt.Sprintf(`{"StringEquals": {"s3:ExistingObjectTag/fill%02d": "v"}}`, i)))
|
||||||
|
}
|
||||||
|
fillerPolicy, err := policy.ParseConfig(strings.NewReader(`{"Version": "2012-10-17", "Statement": [` + strings.Join(fillerStatements, ",") + `]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fillerPolicyName := "000-unread-tags-filler-" + mustGetUUID()
|
||||||
|
if _, err = globalIAMSys.SetPolicy(ctx, fillerPolicyName, *fillerPolicy); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if err := globalIAMSys.DeletePolicy(ctx, fillerPolicyName, false); err != nil {
|
||||||
|
t.Logf("delete filler policy: %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
type user struct {
|
||||||
|
name string
|
||||||
|
creds auth.Credentials
|
||||||
|
// The status with the object's tags readable and with read quorum
|
||||||
|
// lost: 200 and 503 for a caller that some tags let through, 403
|
||||||
|
// and 403 for a caller that none do.
|
||||||
|
healthy, quorumLost int
|
||||||
|
}
|
||||||
|
allowedByTags := func(name string, creds auth.Credentials) user {
|
||||||
|
return user{name, creds, http.StatusOK, http.StatusServiceUnavailable}
|
||||||
|
}
|
||||||
|
deniedByAnyTags := func(name string, creds auth.Credentials) user {
|
||||||
|
return user{name, creds, http.StatusForbidden, http.StatusForbidden}
|
||||||
|
}
|
||||||
|
// Each case below that tests a tag key of its own uses a key no other
|
||||||
|
// policy asks for.
|
||||||
|
ownerCondition := `{"StringEquals": {"s3:ExistingObjectTag/owner": "${aws:username}"}}`
|
||||||
|
owner := newUser(statement("Allow", "*", ownerCondition))
|
||||||
|
users := []user{
|
||||||
|
// Allowed only while the object carries k=v.
|
||||||
|
allowedByTags("tagged", newUser(tagStatement("Allow", "*", "v"))),
|
||||||
|
// Allowed while the object carries a tag np, whatever its value.
|
||||||
|
allowedByTags("nullFalse", newUser(statement("Allow", "*", `{"Null": {"s3:ExistingObjectTag/np": "false"}}`))),
|
||||||
|
// Allowed while a tag ne is there and is not w.
|
||||||
|
allowedByTags("notEquals", newUser(statement("Allow", "*",
|
||||||
|
`{"Null": {"s3:ExistingObjectTag/ne": "false"}, "StringNotEquals": {"s3:ExistingObjectTag/ne": "w"}}`))),
|
||||||
|
// Allowed while every value of tag fa differs from w; that needs a tag fa.
|
||||||
|
allowedByTags("forAllNotEquals", newUser(statement("Allow", "*", `{"ForAllValues:StringNotEquals": {"s3:ExistingObjectTag/fa": "w"}}`))),
|
||||||
|
// Allowed while the object is tagged with the caller's name.
|
||||||
|
allowedByTags("owner", owner),
|
||||||
|
// Allowed, but denied unless the object carries du=1.
|
||||||
|
allowedByTags("denyUnlessTagged", newUser(statement("Allow", "*", ""),
|
||||||
|
statement("Deny", "*", `{"StringNotEquals": {"s3:ExistingObjectTag/du": "1"}}`))),
|
||||||
|
// A service account whose parent asks for k=v and whose own policy
|
||||||
|
// asks for j=w: only an object carrying both lets it through.
|
||||||
|
allowedByTags("serviceAccount", newServiceAccount(newUser(tagStatement("Allow", "*", "v")),
|
||||||
|
statement("Allow", "*", `{"StringEquals": {"s3:ExistingObjectTag/j": "w"}}`))),
|
||||||
|
allowedByTags("temporary", newTempUser(newUser(tagStatement("Allow", "*", "v")),
|
||||||
|
statement("Allow", "*", `{"StringEquals": {"s3:ExistingObjectTag/j": "w"}}`))),
|
||||||
|
allowedByTags("rootTemporary", newTempUser(globalActiveCred, tagStatement("Allow", "*", "v"))),
|
||||||
|
allowedByTags("rootServiceAccount", newServiceAccount(globalActiveCred, tagStatement("Allow", "*", "v"))),
|
||||||
|
// The effective policy must be selected before applying the candidate
|
||||||
|
// limit; the unrelated cached policy above has 64 other tag statements.
|
||||||
|
allowedByTags("unrelatedPolicies", newUser(statement("Allow", "*",
|
||||||
|
`{"StringEquals": {"s3:ExistingObjectTag/target": "ok"}}`))),
|
||||||
|
// An alternative allowed value must survive a Deny that excludes the
|
||||||
|
// first value. With unread tags, k=v is the only valid candidate.
|
||||||
|
allowedByTags("alternativeAllowedValue", newUser(
|
||||||
|
statement("Allow", "*", `{"StringEquals": {"s3:ExistingObjectTag/k": ["a", "v"]}}`),
|
||||||
|
tagStatement("Deny", "*", "a"))),
|
||||||
|
allowedByTags("alternativeValuesAcrossKeys", newUser(
|
||||||
|
statement("Allow", "*", `{"StringEquals": {"s3:ExistingObjectTag/k": ["a", "v"], "s3:ExistingObjectTag/j": ["b", "w"]}}`),
|
||||||
|
tagStatement("Deny", "*", "a"),
|
||||||
|
statement("Deny", "*", `{"StringEquals": {"s3:ExistingObjectTag/j": "b"}}`))),
|
||||||
|
// Both Deny statements must be avoided. The first candidate (k="")
|
||||||
|
// avoids the Null Deny but still matches the second Deny.
|
||||||
|
allowedByTags("multipleDenies", newUser(statement("Allow", "*", ""),
|
||||||
|
statement("Deny", "*", `{"Null": {"s3:ExistingObjectTag/k": "true"}}`),
|
||||||
|
tagStatement("Deny", "*", ""))),
|
||||||
|
// The Deny can be avoided by k=v OR j=w. Only j=w is compatible
|
||||||
|
// with this Allow, so stopping at the first Deny key loses it.
|
||||||
|
{
|
||||||
|
"alternativeDenyKey", newUser(tagStatement("Allow", "*", "a"),
|
||||||
|
statement("Deny", "*", `{"StringNotEquals": {"s3:ExistingObjectTag/k": "v", "s3:ExistingObjectTag/j": "w"}}`)),
|
||||||
|
http.StatusForbidden, http.StatusServiceUnavailable,
|
||||||
|
},
|
||||||
|
|
||||||
|
// Allowed only for k=w: the decision still depends on the tags.
|
||||||
|
{"otherValue", newUser(tagStatement("Allow", "*", "w")), http.StatusForbidden, http.StatusServiceUnavailable},
|
||||||
|
|
||||||
|
// Allowed for k=v under another prefix only.
|
||||||
|
deniedByAnyTags("otherPrefix", newUser(tagStatement("Allow", "other/*", "v"))),
|
||||||
|
// Allowed and denied for k=v alike.
|
||||||
|
deniedByAnyTags("allowDeny", newUser(tagStatement("Allow", "*", "v"), tagStatement("Deny", "*", "v"))),
|
||||||
|
// Allowed only without a tag nc that is 1.
|
||||||
|
deniedByAnyTags("nullConflict", newUser(statement("Allow", "*",
|
||||||
|
`{"Null": {"s3:ExistingObjectTag/nc": "true"}, "StringEquals": {"s3:ExistingObjectTag/nc": "1"}}`))),
|
||||||
|
// Tagged with the caller's name, under another prefix only.
|
||||||
|
deniedByAnyTags("ownerPrefix", newUser(statement("Allow", "other/*", ownerCondition))),
|
||||||
|
// Denied unless the object carries db=1, and denied if it does.
|
||||||
|
deniedByAnyTags("denyBoth", newUser(statement("Allow", "*", ""),
|
||||||
|
statement("Deny", "*", `{"StringNotEquals": {"s3:ExistingObjectTag/db": "1"}}`),
|
||||||
|
statement("Deny", "*", `{"StringEquals": {"s3:ExistingObjectTag/db": "1"}}`))),
|
||||||
|
// The parent asks for k=v and the service account for k=x.
|
||||||
|
deniedByAnyTags("serviceAccountConflict", newServiceAccount(newUser(tagStatement("Allow", "*", "v")),
|
||||||
|
tagStatement("Allow", "*", "x"))),
|
||||||
|
}
|
||||||
|
|
||||||
|
const object, missing = "tagged", "missing"
|
||||||
|
if _, err := z.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("data")), 4, "", ""),
|
||||||
|
ObjectOptions{UserDefined: map[string]string{xhttp.AmzObjectTagging: "k=v&j=w&np=1&ne=1&fa=1&du=1&target=ok&owner=" + owner.AccessKey}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Anonymous reads are allowed by the bucket policy while the object carries k=v.
|
||||||
|
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, []byte(`{"Version": "2012-10-17", "Statement": [
|
||||||
|
{"Effect": "Allow", "Principal": {"AWS": ["*"]}, "Action": ["s3:GetObject"], "Resource": ["arn:aws:s3:::`+bucket+`/*"],
|
||||||
|
"Condition": {"StringEquals": {"s3:ExistingObjectTag/k": "v"}}}]}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
type request struct {
|
||||||
|
name string
|
||||||
|
method, target string
|
||||||
|
header map[string]string
|
||||||
|
body []byte
|
||||||
|
}
|
||||||
|
objectURL := getGetObjectURL("", bucket, object)
|
||||||
|
get := request{"GET", http.MethodGet, objectURL, nil, nil}
|
||||||
|
head := request{"HEAD", http.MethodHead, objectURL, nil, nil}
|
||||||
|
taggingURL := makeTestTargetURL("", bucket, object, url.Values{"tagging": []string{""}})
|
||||||
|
taggingBody := []byte(`<Tagging><TagSet><Tag><Key>k</Key><Value>v</Value></Tag></TagSet></Tagging>`)
|
||||||
|
putTagging := request{"PutObjectTagging", http.MethodPut, taggingURL, nil, taggingBody}
|
||||||
|
getTagging := request{"GetObjectTagging", http.MethodGet, taggingURL, nil, nil}
|
||||||
|
deleteTagging := request{"DeleteObjectTagging", http.MethodDelete, taggingURL, nil, nil}
|
||||||
|
check := func(stage string, creds auth.Credentials, name string, req request, want int) {
|
||||||
|
t.Helper()
|
||||||
|
var rec *httptest.ResponseRecorder
|
||||||
|
if creds.AccessKey == "" {
|
||||||
|
anonymous, err := newTestRequest(req.method, req.target, 0, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec = httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, anonymous)
|
||||||
|
} else {
|
||||||
|
var body io.ReadSeeker
|
||||||
|
if req.body != nil {
|
||||||
|
body = bytes.NewReader(req.body)
|
||||||
|
}
|
||||||
|
header := make(map[string]string)
|
||||||
|
for k, v := range req.header {
|
||||||
|
header[k] = v
|
||||||
|
}
|
||||||
|
if creds.IsTemp() {
|
||||||
|
header[xhttp.AmzSecurityToken] = creds.SessionToken
|
||||||
|
}
|
||||||
|
signed, err := newTestSignedRequestV4(req.method, req.target, int64(len(req.body)), body, creds.AccessKey, creds.SecretKey, header)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec = httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, signed)
|
||||||
|
}
|
||||||
|
if rec.Code != want {
|
||||||
|
var apiErr APIErrorResponse
|
||||||
|
_ = xml.Unmarshal(rec.Body.Bytes(), &apiErr)
|
||||||
|
t.Errorf("%s: %s by %s: status %d %s, want %d", stage, req.name, name, rec.Code, apiErr.Code, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var anonymous auth.Credentials
|
||||||
|
for _, u := range users {
|
||||||
|
check("healthy", u.creds, u.name, get, u.healthy)
|
||||||
|
check("healthy", u.creds, u.name, getTagging, u.healthy)
|
||||||
|
}
|
||||||
|
check("healthy", anonymous, "anonymous", get, http.StatusOK)
|
||||||
|
// The tags were read and allow the request: the range error is the answer.
|
||||||
|
check("healthy", users[0].creds, users[0].name,
|
||||||
|
request{"GET bytes=100-200", http.MethodGet, objectURL, map[string]string{xhttp.Range: "bytes=100-200"}, nil},
|
||||||
|
http.StatusRequestedRangeNotSatisfiable)
|
||||||
|
// Without an object there are no tags: AccessDenied, as before.
|
||||||
|
missingURL := getGetObjectURL("", bucket, missing)
|
||||||
|
check("missing key", users[0].creds, users[0].name, request{"GET", http.MethodGet, missingURL, nil, nil}, http.StatusForbidden)
|
||||||
|
check("missing key", users[0].creds, users[0].name, request{"HEAD", http.MethodHead, missingURL, nil, nil}, http.StatusForbidden)
|
||||||
|
check("missing key", users[0].creds, users[0].name, request{"GetObjectTagging", http.MethodGet, missingURL + "?tagging", nil, nil}, http.StatusForbidden)
|
||||||
|
|
||||||
|
defer objectReadFault(z, bucket, object, 10)()
|
||||||
|
if _, err := z.GetObjectInfo(ctx, bucket, object, ObjectOptions{}); !isErrReadQuorum(err) {
|
||||||
|
t.Fatalf("GetObjectInfo with 10 of 16 drives failing: %v, want InsufficientReadQuorum", err)
|
||||||
|
}
|
||||||
|
for _, req := range []request{get, head, getTagging, putTagging, deleteTagging} {
|
||||||
|
for _, u := range users {
|
||||||
|
check("read quorum lost", u.creds, u.name, req, u.quorumLost)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
check("read quorum lost", anonymous, "anonymous", get, http.StatusServiceUnavailable)
|
||||||
|
check("read quorum lost", anonymous, "anonymous", head, http.StatusServiceUnavailable)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExistingObjectTagsFor(t *testing.T) {
|
||||||
|
values := map[string][]string{"username": {"alice"}, "sub": {"u1"}}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
effect policy.Effect
|
||||||
|
conditions string
|
||||||
|
want string // the tags to set, encoded as in x-amz-tagging
|
||||||
|
ok bool // false: no tags for this statement
|
||||||
|
}{
|
||||||
|
{policy.Allow, `{"StringEquals": {"s3:ExistingObjectTag/k": "v"}}`, "k=v", true},
|
||||||
|
{policy.Allow, `{"StringEqualsIgnoreCase": {"s3:ExistingObjectTag/k": "V"}, "IpAddress": {"aws:SourceIp": "10.0.0.0/8"}}`, "k=V", true},
|
||||||
|
{policy.Allow, `{"ForAnyValue:StringEquals": {"s3:ExistingObjectTag/k": ["b", "a"]}}`, "k=a", true},
|
||||||
|
{policy.Allow, `{"StringLike": {"s3:ExistingObjectTag/k": "a?c*"}}`, "k=a_c", true},
|
||||||
|
{policy.Allow, `{"StringEquals": {"s3:ExistingObjectTag/b": "2", "s3:ExistingObjectTag/a": "1"}}`, "a=1&b=2", true},
|
||||||
|
// Passed without the tag.
|
||||||
|
{policy.Allow, `{"StringNotEquals": {"s3:ExistingObjectTag/k": "v"}}`, "", true},
|
||||||
|
{policy.Allow, `{"Null": {"s3:ExistingObjectTag/k": "true"}}`, "", true},
|
||||||
|
// Null "false" asks for the tag, with any value.
|
||||||
|
{policy.Allow, `{"Null": {"s3:ExistingObjectTag/k": "false"}}`, "k=", true},
|
||||||
|
{policy.Allow, `{"Null": {"s3:ExistingObjectTag/k": "false"}, "StringNotEquals": {"s3:ExistingObjectTag/k": ["", "w"]}}`, "k=_", true},
|
||||||
|
{policy.Allow, `{"Null": {"s3:ExistingObjectTag/k": "false"}, "StringNotLike": {"s3:ExistingObjectTag/k": "*w"}}`, "k=", true},
|
||||||
|
{policy.Allow, `{"ForAllValues:StringNotEquals": {"s3:ExistingObjectTag/k": "w"}}`, "k=", true},
|
||||||
|
{policy.Allow, `{"Null": {"s3:ExistingObjectTag/k": "true"}, "StringEquals": {"s3:ExistingObjectTag/k": "v"}}`, "", false},
|
||||||
|
// Policy variables take the values of the request.
|
||||||
|
{policy.Allow, `{"StringEquals": {"s3:ExistingObjectTag/owner": "${aws:username}"}}`, "owner=alice", true},
|
||||||
|
{policy.Allow, `{"StringLike": {"s3:ExistingObjectTag/owner": "${jwt:sub}-*"}}`, "owner=u1-", true},
|
||||||
|
{policy.Allow, `{"StringEquals": {"s3:ExistingObjectTag/owner": "${ldap:user}"}}`, "", false},
|
||||||
|
// A Deny is kept from matching.
|
||||||
|
{policy.Deny, `{"StringNotEquals": {"s3:ExistingObjectTag/k": "v"}}`, "k=v", true},
|
||||||
|
{policy.Deny, `{"Null": {"s3:ExistingObjectTag/k": "true"}}`, "k=", true},
|
||||||
|
{policy.Deny, `{"StringEquals": {"s3:ExistingObjectTag/k": "v"}}`, "", true},
|
||||||
|
{policy.Deny, `{"StringEquals": {"s3:prefix": "v"}}`, "", false},
|
||||||
|
// No tag tests.
|
||||||
|
{policy.Allow, `{"StringEquals": {"s3:prefix": "v"}}`, "", false},
|
||||||
|
{policy.Allow, "", "", false},
|
||||||
|
} {
|
||||||
|
var conditions condition.Functions // nil: a statement without a Condition
|
||||||
|
if tc.conditions != "" {
|
||||||
|
if err := json.Unmarshal([]byte(tc.conditions), &conditions); err != nil {
|
||||||
|
t.Fatalf("%s: %v", tc.conditions, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
candidates := existingObjectTagsCandidatesFor(tc.effect, conditions, values)
|
||||||
|
ok := len(candidates) > 0
|
||||||
|
got := ""
|
||||||
|
if ok {
|
||||||
|
encoded, err := tags.MapToObjectTags(candidates[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: %v", tc.conditions, err)
|
||||||
|
}
|
||||||
|
got = encoded.String()
|
||||||
|
}
|
||||||
|
if got != tc.want || ok != tc.ok {
|
||||||
|
t.Errorf("%s %s: got %q %v, want %q %v", tc.effect, tc.conditions, got, ok, tc.want, tc.ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Group-only and external identities need mapping lookups that normally
|
||||||
|
// fall back to storage. Candidate collection must stay within the cache even
|
||||||
|
// for those identities, while still finding their group/claim/role policy.
|
||||||
|
func TestUnreadTagPolicySetsCacheOnly(t *testing.T) {
|
||||||
|
ctx, sys, _ := prepareIAMRevisionFixture(t)
|
||||||
|
const policyName = "unread-tag-policy"
|
||||||
|
batchAuthzPolicy(ctx, t, sys, policyName, `{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::bucket/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/k":"v"}}}`)
|
||||||
|
member := iamAdminUser(ctx, t, sys, "unread-group-member", "")
|
||||||
|
_, err := sys.AddUsersToGroup(ctx, "unread-group", []string{member.AccessKey})
|
||||||
|
mustIAM(t, err)
|
||||||
|
_, err = sys.PolicyDBSet(ctx, "unread-group", policyName, regUser, true)
|
||||||
|
mustIAM(t, err)
|
||||||
|
role, err := arn.NewIAMRoleARN("unread-role", "")
|
||||||
|
mustIAM(t, err)
|
||||||
|
sys.rolesMap = map[arn.ARN]string{role: policyName}
|
||||||
|
claimUser := auth.Credentials{
|
||||||
|
AccessKey: "unread-sts", ParentUser: "external-parent", SessionToken: "token",
|
||||||
|
Expiration: UTCNow().Add(time.Hour), Claims: map[string]any{iamPolicyClaimNameOpenID(): policyName},
|
||||||
|
}
|
||||||
|
roleUser := claimUser
|
||||||
|
roleUser.Claims = map[string]any{roleArnClaim: role.String()}
|
||||||
|
original := sys.store.IAMStorageAPI
|
||||||
|
defer func() { sys.store.IAMStorageAPI = original }()
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
cred auth.Credentials
|
||||||
|
}{
|
||||||
|
{"group", member},
|
||||||
|
{"claims", claimUser},
|
||||||
|
{"role", roleUser},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
name := tc.cred.AccessKey
|
||||||
|
if tc.cred.ParentUser != "" {
|
||||||
|
name = tc.cred.ParentUser
|
||||||
|
}
|
||||||
|
probe := &batchBlockedIAMStore{
|
||||||
|
IAMStorageAPI: original, mapping: name, doc: policyName,
|
||||||
|
started: make(chan struct{}), release: make(chan struct{}),
|
||||||
|
}
|
||||||
|
close(probe.release) // report an accidental read without blocking the test
|
||||||
|
sys.store.IAMStorageAPI = probe
|
||||||
|
sets := sys.unreadTagPolicySets(tc.cred)
|
||||||
|
select {
|
||||||
|
case <-probe.started:
|
||||||
|
t.Error("candidate collection attempted a storage read")
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
if len(sets) != 1 || len(sets[0]) != 1 {
|
||||||
|
t.Fatalf("expected the cached policy through %s, got %v", tc.name, sets)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
+31
-14
@@ -340,16 +340,6 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
|
|||||||
return s3Err
|
return s3Err
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkRequestAuthTypeWithExistingTags(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName, existingTags string) (s3Err APIErrorCode) {
|
|
||||||
logger.GetReqInfo(ctx).BucketName = bucketName
|
|
||||||
logger.GetReqInfo(ctx).ObjectName = objectName
|
|
||||||
|
|
||||||
if s3Err = authenticateRequest(ctx, r, action); s3Err != ErrNone {
|
|
||||||
return s3Err
|
|
||||||
}
|
|
||||||
return authorizeRequestWithExistingTags(ctx, r, action, existingTags)
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkRequestAuthTypeWithRequestTags(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string, requestTags *string) (s3Err APIErrorCode) {
|
func checkRequestAuthTypeWithRequestTags(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string, requestTags *string) (s3Err APIErrorCode) {
|
||||||
logger.GetReqInfo(ctx).BucketName = bucketName
|
logger.GetReqInfo(ctx).BucketName = bucketName
|
||||||
logger.GetReqInfo(ctx).ObjectName = objectName
|
logger.GetReqInfo(ctx).ObjectName = objectName
|
||||||
@@ -425,6 +415,19 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
|||||||
return authorizeRequestWithExistingTags(ctx, r, action, "")
|
return authorizeRequestWithExistingTags(ctx, r, action, "")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// authorizeBucketRequest authorizes a bucket action, such as s3:ListBucket,
|
||||||
|
// on the bucket of a request made for one of its objects.
|
||||||
|
func authorizeBucketRequest(ctx context.Context, r *http.Request, action policy.Action) APIErrorCode {
|
||||||
|
reqInfo := logger.GetReqInfo(ctx)
|
||||||
|
if reqInfo == nil {
|
||||||
|
return ErrAccessDenied
|
||||||
|
}
|
||||||
|
objectName := reqInfo.ObjectName
|
||||||
|
reqInfo.ObjectName = ""
|
||||||
|
defer func() { reqInfo.ObjectName = objectName }()
|
||||||
|
return authorizeRequest(ctx, r, action)
|
||||||
|
}
|
||||||
|
|
||||||
func deleteObjectAction(versionID string) policy.Action {
|
func deleteObjectAction(versionID string) policy.Action {
|
||||||
if versionID != "" {
|
if versionID != "" {
|
||||||
return policy.DeleteObjectVersionAction
|
return policy.DeleteObjectVersionAction
|
||||||
@@ -435,7 +438,12 @@ func deleteObjectAction(versionID string) policy.Action {
|
|||||||
func actionUsesObjectVersion(action policy.Action) bool {
|
func actionUsesObjectVersion(action policy.Action) bool {
|
||||||
switch action {
|
switch action {
|
||||||
case policy.DeleteObjectAction, policy.DeleteObjectVersionAction,
|
case policy.DeleteObjectAction, policy.DeleteObjectVersionAction,
|
||||||
policy.ReplicateDeleteAction, policy.BypassGovernanceRetentionAction:
|
policy.ReplicateDeleteAction, policy.BypassGovernanceRetentionAction,
|
||||||
|
policy.GetObjectAction, policy.GetObjectVersionAction,
|
||||||
|
policy.GetObjectAttributesAction, policy.GetObjectVersionAttributesAction,
|
||||||
|
policy.GetObjectTaggingAction, policy.GetObjectVersionTaggingAction,
|
||||||
|
policy.PutObjectTaggingAction, policy.PutObjectVersionTaggingAction,
|
||||||
|
policy.DeleteObjectTaggingAction, policy.DeleteObjectVersionTaggingAction:
|
||||||
return true
|
return true
|
||||||
default:
|
default:
|
||||||
return false
|
return false
|
||||||
@@ -676,7 +684,15 @@ func setAuthMiddleware(h http.Handler) http.Handler {
|
|||||||
|
|
||||||
aType := getRequestAuthType(r)
|
aType := getRequestAuthType(r)
|
||||||
switch aType {
|
switch aType {
|
||||||
case authTypeSigned, authTypeSignedV2, authTypeStreamingSigned, authTypeStreamingSignedTrailer:
|
case authTypeSigned, authTypeSignedV2, authTypeStreamingSigned, authTypeStreamingSignedTrailer, authTypeStreamingUnsignedTrailer:
|
||||||
|
// STREAMING-UNSIGNED-PAYLOAD-TRAILER is valid for a genuinely
|
||||||
|
// anonymous request. Once Authorization is present, it is a
|
||||||
|
// signed REST-HEADER request and must obey the same date/skew
|
||||||
|
// bound as the other SigV4 streaming forms.
|
||||||
|
if aType == authTypeStreamingUnsignedTrailer && r.Header.Get(xhttp.Authorization) == "" {
|
||||||
|
h.ServeHTTP(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
// Verify if date headers are set, if not reject the request
|
// Verify if date headers are set, if not reject the request
|
||||||
amzDate, errCode := parseAmzDateHeader(r)
|
amzDate, errCode := parseAmzDateHeader(r)
|
||||||
if errCode != ErrNone {
|
if errCode != ErrNone {
|
||||||
@@ -774,11 +790,12 @@ func isPutActionAllowedWithRequestTags(ctx context.Context, atype authType, buck
|
|||||||
|
|
||||||
func isPutActionAllowedWithCred(bucketName, objectName string, r *http.Request, action policy.Action, requestTags *string, cred auth.Credentials, owner bool) APIErrorCode {
|
func isPutActionAllowedWithCred(bucketName, objectName string, r *http.Request, action policy.Action, requestTags *string, cred auth.Credentials, owner bool) APIErrorCode {
|
||||||
// Do not check for PutObjectRetentionAction permission,
|
// Do not check for PutObjectRetentionAction permission,
|
||||||
// if mode and retain until date are not set.
|
// if neither retention values nor a replica retention revision are set.
|
||||||
// Can happen when bucket has default lock config set
|
// Can happen when bucket has default lock config set
|
||||||
if action == policy.PutObjectRetentionAction &&
|
if action == policy.PutObjectRetentionAction &&
|
||||||
r.Header.Get(xhttp.AmzObjectLockMode) == "" &&
|
r.Header.Get(xhttp.AmzObjectLockMode) == "" &&
|
||||||
r.Header.Get(xhttp.AmzObjectLockRetainUntilDate) == "" {
|
r.Header.Get(xhttp.AmzObjectLockRetainUntilDate) == "" &&
|
||||||
|
r.Header.Get(xhttp.MinIOSourceObjectRetentionTimestamp) == "" {
|
||||||
return ErrNone
|
return ErrNone
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -330,7 +330,7 @@ func TestBatchReviewRotationKeepsMigrationCleanup(t *testing.T) {
|
|||||||
t.Errorf("rotation changed a draining source after its version was copied (rotation error: %v)", rotationErr)
|
t.Errorf("rotation changed a draining source after its version was copied (rotation error: %v)", rotationErr)
|
||||||
}
|
}
|
||||||
snapshot := []FileInfo{{VersionID: before.VersionID, ModTime: beforeInfo.ModTime}}
|
snapshot := []FileInfo{{VersionID: before.VersionID, ModTime: beforeInfo.ModTime}}
|
||||||
if err := z.removeMovedVersions(t.Context(), bucket, object, 0, snapshot, nil); err != nil {
|
if _, err := z.removeMovedVersions(t.Context(), bucket, object, 0, snapshot, nil); err != nil {
|
||||||
t.Fatalf("rotation prevented migration cleanup: %v", err)
|
t.Fatalf("rotation prevented migration cleanup: %v", err)
|
||||||
}
|
}
|
||||||
if _, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}); !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
if _, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}); !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||||
|
|||||||
@@ -0,0 +1,152 @@
|
|||||||
|
// Copyright (c) 2026 Feng Ruohang
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/base64"
|
||||||
|
"fmt"
|
||||||
|
"maps"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Use explicit condition values to exercise the evaluator, including literal
|
||||||
|
// spellings that S3 tag validation does not permit in stored tags.
|
||||||
|
func TestBatchPolicyConditionVariableParity(t *testing.T) {
|
||||||
|
f := newBatchAuthzFixture(t)
|
||||||
|
cases := []struct {
|
||||||
|
value, actual string
|
||||||
|
matches bool
|
||||||
|
requestOnly bool
|
||||||
|
}{
|
||||||
|
{"${$}100", "${$}100", true, false},
|
||||||
|
{"${$}100", "$100", false, false},
|
||||||
|
{"${*}", "${*}", true, false},
|
||||||
|
{"${?}", "${?}", true, false},
|
||||||
|
{"${custom:unknown}", "${custom:unknown}", true, false},
|
||||||
|
{"${aws:Unknown}", "${aws:Unknown}", true, false},
|
||||||
|
{"${aws:UserAgent", "${aws:UserAgent", true, false},
|
||||||
|
{"${aws:username}", "", true, false},
|
||||||
|
{"${aws:UserAgent}", "test-agent", true, true},
|
||||||
|
{"${prefix${aws:UserAgent}}", "${prefixtest-agent}", true, true},
|
||||||
|
}
|
||||||
|
for i, tc := range cases {
|
||||||
|
for opIndex, op := range []string{
|
||||||
|
"StringEquals", "StringNotEquals", "StringEqualsIgnoreCase", "StringNotEqualsIgnoreCase",
|
||||||
|
"StringLike", "StringNotLike", "ForAnyValue:StringEquals", "ForAllValues:StringEquals", "BinaryEquals",
|
||||||
|
} {
|
||||||
|
for _, effect := range []string{"Allow", "Deny"} {
|
||||||
|
t.Run(fmt.Sprintf("%d/%s/%s", i, op, effect), func(t *testing.T) {
|
||||||
|
name := fmt.Sprintf("var-%d-%d-%s", i, opIndex, effect)
|
||||||
|
conditionValue := tc.value
|
||||||
|
if op == "BinaryEquals" {
|
||||||
|
conditionValue = base64.StdEncoding.EncodeToString([]byte(conditionValue))
|
||||||
|
}
|
||||||
|
statement := fmt.Sprintf(`{"Effect":%q,"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/data/*"],"Condition":{%q:{"s3:ExistingObjectTag/label":%q}}}`, effect, op, conditionValue)
|
||||||
|
if effect == "Deny" {
|
||||||
|
statement += `,{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/data/*"]}`
|
||||||
|
}
|
||||||
|
cred, sub := batchContextUser(t, f, name, statement)
|
||||||
|
actual := tc.actual
|
||||||
|
if tc.value == "${aws:username}" {
|
||||||
|
actual = cred.AccessKey
|
||||||
|
}
|
||||||
|
values := map[string][]string{"ExistingObjectTag/label": {actual}}
|
||||||
|
ordinaryValues := maps.Clone(values)
|
||||||
|
ordinaryValues["username"] = []string{cred.AccessKey}
|
||||||
|
ordinaryValues["UserAgent"] = []string{"test-agent"}
|
||||||
|
ordinaryAllowed := f.sys.isAllowedLocally(policy.Args{
|
||||||
|
AccountName: cred.AccessKey,
|
||||||
|
Action: policy.GetObjectAction,
|
||||||
|
BucketName: f.bucket,
|
||||||
|
ObjectName: "data/" + name,
|
||||||
|
ConditionValues: ordinaryValues,
|
||||||
|
})
|
||||||
|
matches := tc.matches
|
||||||
|
if strings.Contains(op, "Not") {
|
||||||
|
matches = !matches
|
||||||
|
}
|
||||||
|
if want := matches == (effect == "Allow"); ordinaryAllowed != want {
|
||||||
|
t.Fatalf("ordinary evaluator allowed=%v want %v", ordinaryAllowed, want)
|
||||||
|
}
|
||||||
|
batchErr := sub.authorize(f.ctx, batchAuthzRequest{
|
||||||
|
action: policy.GetObjectAction, bucket: f.bucket, object: "data/" + name, values: values,
|
||||||
|
})
|
||||||
|
if tc.requestOnly {
|
||||||
|
if !isBatchJobAuthzError(batchErr) {
|
||||||
|
t.Fatalf("missing request context must deny: %v", batchErr)
|
||||||
|
}
|
||||||
|
if (effect == "Deny") != isBatchContextRefusal(batchErr) {
|
||||||
|
t.Fatalf("only a Deny must refuse missing context: %v", batchErr)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (batchErr == nil) != ordinaryAllowed {
|
||||||
|
t.Fatalf("ordinary allowed=%v; batch=%v", ordinaryAllowed, batchErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Condition keys themselves are never expanded.
|
||||||
|
const statement = `{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/${aws:UserAgent}":"literal"}}}`
|
||||||
|
cred, sub := batchContextUser(t, f, "literal-condition-key", statement)
|
||||||
|
values := map[string][]string{"ExistingObjectTag/${aws:UserAgent}": {"literal"}}
|
||||||
|
if !f.sys.isAllowedLocally(policy.Args{AccountName: cred.AccessKey, Action: policy.GetObjectAction, BucketName: f.bucket, ObjectName: "data/key", ConditionValues: values}) {
|
||||||
|
t.Fatal("ordinary evaluator did not match the literal condition key")
|
||||||
|
}
|
||||||
|
if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "data/key", values: values}); err != nil {
|
||||||
|
t.Fatalf("batch treated a condition key as a variable: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Object names permit these spellings, so also compare real S3 GETs with batch.
|
||||||
|
func TestBatchPolicyResourceVariableParity(t *testing.T) {
|
||||||
|
f := newBatchAuthzFixture(t)
|
||||||
|
router := initTestAPIEndPoints(f.obj, []string{"GetObject"})
|
||||||
|
for i, raw := range []string{"${$}100", "${*}", "${?}", "${custom:unknown}", "${prefix${aws:UserAgent}}", "${aws:username}", "${aws:UserAgent}"} {
|
||||||
|
for _, effect := range []string{"Allow", "Deny"} {
|
||||||
|
t.Run(fmt.Sprintf("%d/%s", i, effect), func(t *testing.T) {
|
||||||
|
name := fmt.Sprintf("res-%d-%s", i, effect)
|
||||||
|
statement := fmt.Sprintf(`{"Effect":%q,"Action":["s3:GetObject"],"Resource":[%q]}`, effect, "arn:aws:s3:::batch-authz/"+raw)
|
||||||
|
if effect == "Deny" {
|
||||||
|
statement += `,{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/*"]}`
|
||||||
|
}
|
||||||
|
cred, sub := batchContextUser(t, f, name, statement)
|
||||||
|
object := strings.ReplaceAll(raw, "${aws:username}", cred.AccessKey)
|
||||||
|
if raw == "${aws:UserAgent}" {
|
||||||
|
object = "test-agent"
|
||||||
|
}
|
||||||
|
_, err := f.obj.PutObject(f.ctx, f.bucket, object, mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), ObjectOptions{})
|
||||||
|
mustIAM(t, err)
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", f.bucket, object), 0, nil, cred.AccessKey, cred.SecretKey, map[string]string{"User-Agent": "test-agent"})
|
||||||
|
mustIAM(t, err)
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, req)
|
||||||
|
wantStatus := http.StatusOK
|
||||||
|
if effect == "Deny" {
|
||||||
|
wantStatus = http.StatusForbidden
|
||||||
|
}
|
||||||
|
if rec.Code != wantStatus {
|
||||||
|
t.Fatalf("S3 read: %d want %d: %s", rec.Code, wantStatus, rec.Body.String())
|
||||||
|
}
|
||||||
|
err = sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: object})
|
||||||
|
if raw == "${aws:UserAgent}" {
|
||||||
|
if !isBatchJobAuthzError(err) || (effect == "Deny") != isBatchContextRefusal(err) {
|
||||||
|
t.Fatalf("batch did not reject missing request context: %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (err == nil) != (rec.Code == http.StatusOK) {
|
||||||
|
t.Fatalf("resource %q %s: S3=%d batch=%v", raw, effect, rec.Code, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+44
-4
@@ -829,15 +829,15 @@ func batchStatementNeedsRequest(st policy.Statement, args policy.Args, supplied
|
|||||||
// Policy variables in condition values.
|
// Policy variables in condition values.
|
||||||
if len(st.Conditions) > 0 {
|
if len(st.Conditions) > 0 {
|
||||||
data, err := json.Marshal(st.Conditions)
|
data, err := json.Marshal(st.Conditions)
|
||||||
if err != nil || batchUnsuppliedVariable(string(data), supplied) {
|
if err != nil || batchConditionsNeedVariables(data, supplied) {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// batchUnsuppliedVariable reports whether s holds a policy variable, ${key},
|
// batchUnsuppliedVariable follows Resource.Match: only CommonKeys are expanded,
|
||||||
// whose value the job does not supply.
|
// and an unsupported ${...} sequence (including its nested text) stays literal.
|
||||||
func batchUnsuppliedVariable(s string, supplied map[string]bool) bool {
|
func batchUnsuppliedVariable(s string, supplied map[string]bool) bool {
|
||||||
for {
|
for {
|
||||||
i := strings.Index(s, "${")
|
i := strings.Index(s, "${")
|
||||||
@@ -849,13 +849,53 @@ func batchUnsuppliedVariable(s string, supplied map[string]bool) bool {
|
|||||||
if j < 0 {
|
if j < 0 {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
if !batchKeySupplied(supplied, s[:j]) {
|
if condition.CommonKeysMap[condition.KeyName(s[:j])] && !batchKeySupplied(supplied, s[:j]) {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
s = s[j+1:]
|
s = s[j+1:]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Conditions replace each recognized variable wherever it occurs in a string
|
||||||
|
// value, unlike the resource parser. Inspect values only, not condition keys,
|
||||||
|
// and decode BinaryEquals as the policy evaluator does before substitution.
|
||||||
|
func batchConditionsNeedVariables(data []byte, supplied map[string]bool) bool {
|
||||||
|
var conditions map[string]map[string]condition.ValueSet
|
||||||
|
if err := json.Unmarshal(data, &conditions); err != nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for name, keys := range conditions {
|
||||||
|
name = strings.TrimPrefix(strings.TrimPrefix(name, "ForAnyValue:"), "ForAllValues:")
|
||||||
|
name = strings.TrimSuffix(name, "IfExists")
|
||||||
|
switch name {
|
||||||
|
case "StringEquals", "StringNotEquals", "StringEqualsIgnoreCase", "StringNotEqualsIgnoreCase", "StringLike", "StringNotLike", "BinaryEquals":
|
||||||
|
default:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, values := range keys {
|
||||||
|
for value := range values {
|
||||||
|
s, err := value.GetString()
|
||||||
|
if err != nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if name == "BinaryEquals" {
|
||||||
|
decoded, err := base64.StdEncoding.DecodeString(s)
|
||||||
|
if err != nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
s = string(decoded)
|
||||||
|
}
|
||||||
|
for _, key := range condition.CommonKeys {
|
||||||
|
if strings.Contains(s, key.VarName()) && !batchKeySupplied(supplied, string(key)) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// batchStatementMayApply matches the resources of a statement, in both forms
|
// batchStatementMayApply matches the resources of a statement, in both forms
|
||||||
// a bucket request can take. A statement whose resources the evaluator
|
// a bucket request can take. A statement whose resources the evaluator
|
||||||
// translates (S3 Tables and vectors actions cover S3 data actions) is taken
|
// translates (S3 Tables and vectors actions cover S3 data actions) is taken
|
||||||
|
|||||||
@@ -0,0 +1,177 @@
|
|||||||
|
// Copyright (c) 2026 Feng Ruohang
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"maps"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/crypto"
|
||||||
|
"github.com/minio/minio/internal/kms"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBatchRotationContextValidation(t *testing.T) {
|
||||||
|
previous := GlobalKMS
|
||||||
|
GlobalKMS = nil
|
||||||
|
t.Cleanup(func() { GlobalKMS = previous })
|
||||||
|
for _, encoded := range []string{"not-base64!", base64.StdEncoding.EncodeToString([]byte(`{`)), base64.StdEncoding.EncodeToString([]byte(`{"tenant":1}`)), base64.StdEncoding.EncodeToString([]byte(`[]`))} {
|
||||||
|
enc := BatchJobKeyRotateEncryption{Type: ssekms, Key: "key", Context: encoded}
|
||||||
|
if err := enc.Validate(); err == nil {
|
||||||
|
t.Fatalf("accepted invalid context %q", encoded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, raw := range []string{"", `{}`, `null`, `{"tenant":"acme"}`} {
|
||||||
|
encoded := base64.StdEncoding.EncodeToString([]byte(raw))
|
||||||
|
if raw == "" {
|
||||||
|
encoded = ""
|
||||||
|
}
|
||||||
|
if err := (BatchJobKeyRotateEncryption{Type: ssekms, Key: "key", Context: encoded}).Validate(); err != nil {
|
||||||
|
t.Fatalf("context %q: %v", raw, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBatchRotationContextInitialRetryAndReload(t *testing.T) {
|
||||||
|
for _, mode := range []string{"initial", "retry", "reload"} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
f := newBatchAuthzFixture(t)
|
||||||
|
previous := GlobalKMS
|
||||||
|
t.Cleanup(func() { GlobalKMS = previous })
|
||||||
|
configured, err := kms.NewBuiltin("context-key", bytes.Repeat([]byte{0x42}, 32))
|
||||||
|
mustIAM(t, err)
|
||||||
|
GlobalKMS = configured
|
||||||
|
const object = "rotation/context"
|
||||||
|
const payload = "a context-bound encryption key keeps this readable"
|
||||||
|
metadata := map[string]string{}
|
||||||
|
key, err := newEncryptMetadata(f.ctx, crypto.S3KMS, "context-key", nil, f.bucket, object, metadata, kms.Context{"tenant": "old"})
|
||||||
|
mustIAM(t, err)
|
||||||
|
ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader(payload), key))
|
||||||
|
mustIAM(t, err)
|
||||||
|
original, err := f.obj.PutObject(f.ctx, f.bucket, object, mustGetPutObjReader(t, bytes.NewReader(ciphertext), int64(len(ciphertext)), "", ""), ObjectOptions{Versioned: true, UserDefined: metadata})
|
||||||
|
mustIAM(t, err)
|
||||||
|
contextJSON := `{"tenant":"acme","purpose":"batch-rotation","区域":"杭州","�":"replacement","emoji":"🔑"}`
|
||||||
|
encoded := base64.StdEncoding.EncodeToString([]byte(contextJSON))
|
||||||
|
rec, job := f.submit(t, globalActiveCred, fmt.Sprintf("keyrotate:\n apiVersion: v1\n bucket: %s\n prefix: rotation/\n encryption:\n type: sse-kms\n key: context-key\n context: %s\n", f.bucket, encoded))
|
||||||
|
if job == nil {
|
||||||
|
t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if mode == "retry" {
|
||||||
|
GlobalKMS = nil
|
||||||
|
err := job.KeyRotate.KeyRotate(f.ctx, f.obj, original, job.Submitter)
|
||||||
|
GlobalKMS = configured
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("rotation succeeded while KMS was unavailable")
|
||||||
|
}
|
||||||
|
before, err := f.obj.GetObjectInfo(f.ctx, f.bucket, object, ObjectOptions{VersionID: original.VersionID})
|
||||||
|
mustIAM(t, err)
|
||||||
|
if before.UserDefined[crypto.MetaContext] != metadata[crypto.MetaContext] {
|
||||||
|
t.Fatal("failed attempt changed context")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if mode == "reload" {
|
||||||
|
loaded := &BatchJobRequest{}
|
||||||
|
mustIAM(t, loaded.load(f.ctx, f.obj, getJobPath(*job)))
|
||||||
|
job = loaded
|
||||||
|
}
|
||||||
|
mustIAM(t, job.KeyRotate.Start(f.ctx, f.obj, *job))
|
||||||
|
progress := f.jobInfo(t, job)
|
||||||
|
if !progress.Complete || progress.Failed || progress.Objects != 1 {
|
||||||
|
t.Fatalf("rotation incomplete: %+v", progress)
|
||||||
|
}
|
||||||
|
after, err := f.obj.GetObjectInfo(f.ctx, f.bucket, object, ObjectOptions{VersionID: original.VersionID})
|
||||||
|
mustIAM(t, err)
|
||||||
|
raw, err := base64.StdEncoding.DecodeString(after.UserDefined[crypto.MetaContext])
|
||||||
|
mustIAM(t, err)
|
||||||
|
var got kms.Context
|
||||||
|
mustIAM(t, json.Unmarshal(raw, &got))
|
||||||
|
if !maps.Equal(got, kms.Context{"tenant": "acme", "purpose": "batch-rotation", "区域": "杭州", "�": "replacement", "emoji": "🔑"}) {
|
||||||
|
t.Fatalf("stored context %v", got)
|
||||||
|
}
|
||||||
|
actual, err := crypto.S3KMS.UnsealObjectKey(configured, after.UserDefined, f.bucket, object)
|
||||||
|
mustIAM(t, err)
|
||||||
|
if actual != key {
|
||||||
|
t.Fatal("rotation changed the plaintext data key")
|
||||||
|
}
|
||||||
|
if got := readCopyChecksumObject(t, f.obj, f.bucket, object, ObjectOptions{VersionID: original.VersionID}); string(got) != payload {
|
||||||
|
t.Fatal("rotated object is unreadable")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBatchRotationInvalidUTF8LeavesObjectReadable(t *testing.T) {
|
||||||
|
for _, tc := range []struct{ name, raw string }{
|
||||||
|
{"colliding-keys", "{\"\xff\":\"a\",\"\xfe\":\"b\"}"},
|
||||||
|
{"replacement-key", "{\"\xff\":\"a\",\"�\":\"b\"}"},
|
||||||
|
{"invalid-value", "{\"tenant\":\"\xff\"}"},
|
||||||
|
} {
|
||||||
|
for _, phase := range []string{"submit", "reload"} {
|
||||||
|
t.Run(tc.name+"/"+phase, func(t *testing.T) {
|
||||||
|
f := newBatchAuthzFixture(t)
|
||||||
|
previous := GlobalKMS
|
||||||
|
t.Cleanup(func() { GlobalKMS = previous })
|
||||||
|
configured, err := kms.NewBuiltin("context-key", bytes.Repeat([]byte{0x42}, 32))
|
||||||
|
mustIAM(t, err)
|
||||||
|
GlobalKMS = configured
|
||||||
|
const object, payload = "rotation/context", "the original encrypted payload must remain readable"
|
||||||
|
metadata := map[string]string{}
|
||||||
|
key, err := newEncryptMetadata(f.ctx, crypto.S3KMS, "context-key", nil, f.bucket, object, metadata, kms.Context{"tenant": "old"})
|
||||||
|
mustIAM(t, err)
|
||||||
|
ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader(payload), key))
|
||||||
|
mustIAM(t, err)
|
||||||
|
original, err := f.obj.PutObject(f.ctx, f.bucket, object, mustGetPutObjReader(t, bytes.NewReader(ciphertext), int64(len(ciphertext)), "", ""), ObjectOptions{Versioned: true, UserDefined: metadata})
|
||||||
|
mustIAM(t, err)
|
||||||
|
encoded := base64.StdEncoding.EncodeToString([]byte(tc.raw))
|
||||||
|
submitContext := encoded
|
||||||
|
if phase == "reload" {
|
||||||
|
submitContext = base64.StdEncoding.EncodeToString([]byte(`{"tenant":"safe"}`))
|
||||||
|
}
|
||||||
|
rec, job := f.submit(t, globalActiveCred, fmt.Sprintf("keyrotate:\n apiVersion: v1\n bucket: %s\n prefix: rotation/\n encryption:\n type: sse-kms\n key: context-key\n context: %s\n", f.bucket, submitContext))
|
||||||
|
if phase == "submit" {
|
||||||
|
if job != nil || rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("invalid UTF-8 context accepted: %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if job == nil {
|
||||||
|
t.Fatalf("valid submit: %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
// Simulate a manifest saved by a version which accepted these
|
||||||
|
// bytes. Execution must validate again before changing the key.
|
||||||
|
job.KeyRotate.Encryption.Context = encoded
|
||||||
|
job.KeyRotate.Flags.Retry.Attempts = 1
|
||||||
|
manifest, err := job.MarshalMsg([]byte(batchJobEnvelope))
|
||||||
|
mustIAM(t, err)
|
||||||
|
mustIAM(t, saveConfig(f.ctx, f.obj, getJobPath(*job), manifest))
|
||||||
|
loaded := &BatchJobRequest{}
|
||||||
|
mustIAM(t, loaded.load(f.ctx, f.obj, getJobPath(*job)))
|
||||||
|
mustIAM(t, loaded.KeyRotate.Start(f.ctx, f.obj, *loaded))
|
||||||
|
progress := f.jobInfo(t, loaded)
|
||||||
|
if progress.Complete || !progress.Failed || progress.Objects != 0 || progress.ObjectsFailed != 1 {
|
||||||
|
t.Fatalf("invalid context reported as success: %+v", progress)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
after, err := f.obj.GetObjectInfo(f.ctx, f.bucket, object, ObjectOptions{VersionID: original.VersionID})
|
||||||
|
mustIAM(t, err)
|
||||||
|
if after.UserDefined[crypto.MetaContext] != metadata[crypto.MetaContext] {
|
||||||
|
t.Fatal("rejected rotation changed stored context")
|
||||||
|
}
|
||||||
|
actual, err := crypto.S3KMS.UnsealObjectKey(configured, after.UserDefined, f.bucket, object)
|
||||||
|
mustIAM(t, err)
|
||||||
|
if actual != key {
|
||||||
|
t.Fatal("rejected rotation changed data key")
|
||||||
|
}
|
||||||
|
if got := readCopyChecksumObject(t, f.obj, f.bucket, object, ObjectOptions{VersionID: original.VersionID}); string(got) != payload {
|
||||||
|
t.Fatal("rejected rotation damaged stored content")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+155
-48
@@ -29,10 +29,13 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
jsoniter "github.com/json-iterator/go"
|
jsoniter "github.com/json-iterator/go"
|
||||||
"github.com/minio/minio-go/v7/pkg/tags"
|
"github.com/minio/minio-go/v7/pkg/tags"
|
||||||
"github.com/minio/minio/internal/crypto"
|
"github.com/minio/minio/internal/crypto"
|
||||||
|
"github.com/minio/minio/internal/etag"
|
||||||
|
"github.com/minio/minio/internal/hash"
|
||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/minio/minio/internal/kms"
|
"github.com/minio/minio/internal/kms"
|
||||||
"github.com/pgsty/silo-pkg/v3/env"
|
"github.com/pgsty/silo-pkg/v3/env"
|
||||||
@@ -83,43 +86,67 @@ const (
|
|||||||
|
|
||||||
// BatchJobKeyRotateEncryption defines key rotation encryption options passed
|
// BatchJobKeyRotateEncryption defines key rotation encryption options passed
|
||||||
type BatchJobKeyRotateEncryption struct {
|
type BatchJobKeyRotateEncryption struct {
|
||||||
Type BatchKeyRotationType `yaml:"type" json:"type"`
|
Type BatchKeyRotationType `yaml:"type" json:"type"`
|
||||||
Key string `yaml:"key" json:"key"`
|
Key string `yaml:"key" json:"key"`
|
||||||
Context string `yaml:"context" json:"context"`
|
Context string `yaml:"context" json:"context"`
|
||||||
kmsContext kms.Context `msg:"-"`
|
}
|
||||||
|
|
||||||
|
// batchKeyRotateContext decodes the persisted configuration on every run. It
|
||||||
|
// must not depend on unpersisted derived state or mutate shared job options.
|
||||||
|
func batchKeyRotateContext(encoded string) (kms.Context, error) {
|
||||||
|
ctx := kms.Context{}
|
||||||
|
if encoded == "" {
|
||||||
|
return ctx, nil
|
||||||
|
}
|
||||||
|
b, err := base64.StdEncoding.DecodeString(encoded)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Saving the context replaces invalid UTF-8 and can collapse distinct
|
||||||
|
// keys, changing the associated data required to decrypt the object.
|
||||||
|
if !utf8.Valid(b) {
|
||||||
|
return nil, fmt.Errorf("KMS context must be valid UTF-8: %w", errInvalidArgument)
|
||||||
|
}
|
||||||
|
if err = jsoniter.ConfigCompatibleWithStandardLibrary.Unmarshal(b, &ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if ctx == nil {
|
||||||
|
ctx = kms.Context{}
|
||||||
|
}
|
||||||
|
return ctx, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate validates input key rotation encryption options.
|
// Validate validates input key rotation encryption options.
|
||||||
func (e BatchJobKeyRotateEncryption) Validate() error {
|
func (e BatchJobKeyRotateEncryption) Validate() error {
|
||||||
|
_, err := e.validate()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// validate also returns a fresh context for the actual rotation. Validation's
|
||||||
|
// probe entry is added to a separate map, never to the object's context.
|
||||||
|
func (e BatchJobKeyRotateEncryption) validate() (kms.Context, error) {
|
||||||
if e.Type != sses3 && e.Type != ssekms {
|
if e.Type != sses3 && e.Type != ssekms {
|
||||||
return errInvalidArgument
|
return nil, errInvalidArgument
|
||||||
}
|
}
|
||||||
spaces := strings.HasPrefix(e.Key, " ") || strings.HasSuffix(e.Key, " ")
|
spaces := strings.HasPrefix(e.Key, " ") || strings.HasSuffix(e.Key, " ")
|
||||||
if e.Type == ssekms && spaces {
|
if e.Type == ssekms && spaces {
|
||||||
return crypto.ErrInvalidEncryptionKeyID
|
return nil, crypto.ErrInvalidEncryptionKeyID
|
||||||
}
|
}
|
||||||
|
if e.Type != ssekms {
|
||||||
if e.Type == ssekms && GlobalKMS != nil {
|
return nil, nil
|
||||||
ctx := kms.Context{}
|
}
|
||||||
if e.Context != "" {
|
ctx, err := batchKeyRotateContext(e.Context)
|
||||||
b, err := base64.StdEncoding.DecodeString(e.Context)
|
if err != nil {
|
||||||
if err != nil {
|
return nil, err
|
||||||
return err
|
}
|
||||||
}
|
if GlobalKMS != nil {
|
||||||
|
probe := maps.Clone(ctx)
|
||||||
json := jsoniter.ConfigCompatibleWithStandardLibrary
|
probe["MinIO batch API"] = "batchrotate" // Context for a test key operation
|
||||||
if err := json.Unmarshal(b, &ctx); err != nil {
|
if _, err := GlobalKMS.GenerateKey(GlobalContext, &kms.GenerateKeyRequest{Name: e.Key, AssociatedData: probe}); err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
|
||||||
}
|
|
||||||
e.kmsContext = kms.Context{}
|
|
||||||
maps.Copy(e.kmsContext, ctx)
|
|
||||||
ctx["MinIO batch API"] = "batchrotate" // Context for a test key operation
|
|
||||||
if _, err := GlobalKMS.GenerateKey(GlobalContext, &kms.GenerateKeyRequest{Name: e.Key, AssociatedData: ctx}); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return ctx, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// BatchKeyRotateFilter holds all the filters currently supported for batch replication
|
// BatchKeyRotateFilter holds all the filters currently supported for batch replication
|
||||||
@@ -203,11 +230,11 @@ func (r *BatchJobKeyRotateV1) rotateAuthzRequests(oi ObjectInfo) []batchAuthzReq
|
|||||||
if versionID == "" {
|
if versionID == "" {
|
||||||
versionID = nullVersionID
|
versionID = nullVersionID
|
||||||
}
|
}
|
||||||
// The encryption the rewrite leaves: rotation keeps the kind of the
|
// The encryption the rewrite leaves: SSE-KMS takes the requested key,
|
||||||
// version (rotateKey), and SSE-KMS takes the requested key.
|
// including when migrating an SSE-S3 version to SSE-KMS.
|
||||||
values := map[string][]string{}
|
values := map[string][]string{}
|
||||||
switch {
|
switch {
|
||||||
case crypto.S3KMS.IsEncrypted(oi.UserDefined):
|
case crypto.S3KMS.IsEncrypted(oi.UserDefined) || r.Encryption.Type == ssekms:
|
||||||
values[strings.ToLower(xhttp.AmzServerSideEncryption)] = []string{xhttp.AmzEncryptionKMS}
|
values[strings.ToLower(xhttp.AmzServerSideEncryption)] = []string{xhttp.AmzEncryptionKMS}
|
||||||
if r.Encryption.Type == ssekms && r.Encryption.Key != "" {
|
if r.Encryption.Type == ssekms && r.Encryption.Key != "" {
|
||||||
values[strings.ToLower(xhttp.AmzServerSideEncryptionKmsID)] = []string{r.Encryption.Key}
|
values[strings.ToLower(xhttp.AmzServerSideEncryptionKmsID)] = []string{r.Encryption.Key}
|
||||||
@@ -285,31 +312,113 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
|
|||||||
}
|
}
|
||||||
oi := obj.Clone()
|
oi := obj.Clone()
|
||||||
var (
|
var (
|
||||||
newKeyID string
|
newKeyID string
|
||||||
newKeyContext kms.Context
|
newKeyContext kms.Context
|
||||||
|
replaceContext bool
|
||||||
|
targetKind crypto.Type = crypto.S3
|
||||||
)
|
)
|
||||||
encMetadata := make(map[string]string)
|
if sseKMS || r.Encryption.Type == ssekms {
|
||||||
for k, v := range oi.UserDefined {
|
targetKind = crypto.S3KMS
|
||||||
if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
|
|
||||||
encMetadata[k] = v
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (sseKMS || sseS3) && r.Encryption.Type == ssekms {
|
if (sseKMS || sseS3) && r.Encryption.Type == ssekms {
|
||||||
if err = r.Encryption.Validate(); err != nil {
|
newKeyContext, err = r.Encryption.validate()
|
||||||
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
newKeyID = strings.TrimPrefix(r.Encryption.Key, crypto.ARNPrefix)
|
newKeyID = strings.TrimPrefix(r.Encryption.Key, crypto.ARNPrefix)
|
||||||
newKeyContext = r.Encryption.kmsContext
|
replaceContext = r.Encryption.Context != ""
|
||||||
}
|
|
||||||
if err = rotateKey(ctx, []byte{}, newKeyID, []byte{}, r.Bucket, oi.Name, encMetadata, newKeyContext); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Since we are rotating the keys, make sure to update the metadata.
|
srcOpts := ObjectOptions{VersionID: oi.VersionID}
|
||||||
oi.metadataOnly = true
|
dstOpts := ObjectOptions{
|
||||||
oi.keyRotation = true
|
VersionID: oi.VersionID,
|
||||||
maps.Copy(oi.UserDefined, encMetadata)
|
Versioned: versioned,
|
||||||
|
VersionSuspended: versionSuspended,
|
||||||
|
NoLock: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
if oi.Legacy {
|
||||||
|
if targetKind == crypto.S3KMS && !replaceContext && len(newKeyContext) == 0 && sseKMS {
|
||||||
|
_, _, _, newKeyContext, err = crypto.S3KMS.ParseMetadata(oi.UserDefined)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
readOpts := ObjectOptions{
|
||||||
|
VersionID: oi.VersionID,
|
||||||
|
Versioned: versioned,
|
||||||
|
VersionSuspended: versionSuspended,
|
||||||
|
NoLock: true,
|
||||||
|
NoAuditLog: true,
|
||||||
|
}
|
||||||
|
var gr *GetObjectReader
|
||||||
|
if z, ok := api.(*erasureServerPools); ok && pool >= 0 {
|
||||||
|
gr, err = z.serverPools[pool].GetObjectNInfo(ctx, r.Bucket, encodeDirObject(oi.Name), nil, http.Header{}, readOpts)
|
||||||
|
} else {
|
||||||
|
gr, err = api.GetObjectNInfo(ctx, r.Bucket, oi.Name, nil, http.Header{}, readOpts)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer gr.Close()
|
||||||
|
|
||||||
|
actualSize, err := gr.ObjInfo.GetActualSize()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rawReader, err := hash.NewReader(ctx, gr, actualSize, "", "", actualSize)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
pReader := NewPutObjReader(rawReader)
|
||||||
|
encMetadata := make(map[string]string)
|
||||||
|
encReader, objEncKey, err := newEncryptReader(ctx, rawReader, targetKind, newKeyID, nil, r.Bucket, oi.Name, encMetadata, newKeyContext)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
crypto.RemoveInternalEntries(oi.UserDefined)
|
||||||
|
delete(oi.UserDefined, crypto.MetaContext)
|
||||||
|
delete(oi.UserDefined, ReservedMetadataPrefix+"compression")
|
||||||
|
delete(oi.UserDefined, ReservedMetadataPrefix+"actual-size")
|
||||||
|
maps.Copy(oi.UserDefined, encMetadata)
|
||||||
|
|
||||||
|
targetSize := (&ObjectInfo{Size: actualSize}).EncryptedSize()
|
||||||
|
wrapped := etag.Wrap(encReader, rawReader)
|
||||||
|
hReader, err := hash.NewReader(ctx, wrapped, targetSize, "", "", actualSize)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
pReader, err = pReader.WithEncryption(hReader, &objEncKey)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
oi.PutObjReader = pReader
|
||||||
|
oi.metadataOnly = false
|
||||||
|
oi.keyRotation = true
|
||||||
|
dstOpts.EncryptFn = metadataEncrypter(objEncKey)
|
||||||
|
if dstOpts.VersionID == "" && (versioned || versionSuspended) {
|
||||||
|
dstOpts.VersionID = nullVersionID
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
encMetadata := make(map[string]string)
|
||||||
|
for k, v := range oi.UserDefined {
|
||||||
|
if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
|
||||||
|
encMetadata[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err = rotateKeyWithTarget(ctx, []byte{}, targetKind, newKeyID, []byte{}, r.Bucket, oi.Name, encMetadata, newKeyContext, replaceContext); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Since we are rotating the keys, make sure to update the metadata.
|
||||||
|
oi.metadataOnly = true
|
||||||
|
oi.keyRotation = true
|
||||||
|
delete(oi.UserDefined, crypto.MetaSealedKeyS3)
|
||||||
|
delete(oi.UserDefined, crypto.MetaSealedKeyKMS)
|
||||||
|
delete(oi.UserDefined, crypto.MetaKeyID)
|
||||||
|
delete(oi.UserDefined, crypto.MetaDataEncryptionKey)
|
||||||
|
delete(oi.UserDefined, crypto.MetaContext)
|
||||||
|
maps.Copy(oi.UserDefined, encMetadata)
|
||||||
|
}
|
||||||
// The rewrite replaces the stored metadata with UserDefined, which a
|
// The rewrite replaces the stored metadata with UserDefined, which a
|
||||||
// read returns without the tags and the expiry: keep them.
|
// read returns without the tags and the expiry: keep them.
|
||||||
if oi.UserTags != "" {
|
if oi.UserTags != "" {
|
||||||
@@ -318,8 +427,6 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
|
|||||||
if !oi.Expires.IsZero() {
|
if !oi.Expires.IsZero() {
|
||||||
oi.UserDefined["expires"] = oi.ExpiresStr()
|
oi.UserDefined["expires"] = oi.ExpiresStr()
|
||||||
}
|
}
|
||||||
srcOpts := ObjectOptions{VersionID: oi.VersionID}
|
|
||||||
dstOpts := ObjectOptions{VersionID: oi.VersionID, NoLock: true}
|
|
||||||
if z, ok := api.(*erasureServerPools); ok && pool >= 0 {
|
if z, ok := api.(*erasureServerPools); ok && pool >= 0 {
|
||||||
// Rewrite the copy that was read and authorized. The pools layer
|
// Rewrite the copy that was read and authorized. The pools layer
|
||||||
// would route the rewrite by the latest version of the key, which
|
// would route the rewrite by the latest version of the key, which
|
||||||
|
|||||||
@@ -33,6 +33,9 @@ func newBatchAuthzFixture(t *testing.T) *batchAuthzFixture {
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
ctx, sys := iamAdminFixture(t)
|
ctx, sys := iamAdminFixture(t)
|
||||||
obj := newObjectLayerFn()
|
obj := newObjectLayerFn()
|
||||||
|
// Match storage-class parity to this fixture's single drive, even after
|
||||||
|
// an API test has loaded the configuration of a larger erasure set.
|
||||||
|
mustIAM(t, newTestConfig(globalMinioDefaultRegion, obj))
|
||||||
batchTestJobPool(ctx, t, obj)
|
batchTestJobPool(ctx, t, obj)
|
||||||
f := &batchAuthzFixture{ctx: ctx, sys: sys, obj: obj, bucket: "batch-authz"}
|
f := &batchAuthzFixture{ctx: ctx, sys: sys, obj: obj, bucket: "batch-authz"}
|
||||||
mustIAM(t, obj.MakeBucket(ctx, f.bucket, MakeBucketOptions{VersioningEnabled: true}))
|
mustIAM(t, obj.MakeBucket(ctx, f.bucket, MakeBucketOptions{VersioningEnabled: true}))
|
||||||
|
|||||||
+38
-20
@@ -478,15 +478,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
|||||||
getObjectInfoFn := objectAPI.GetObjectInfo
|
getObjectInfoFn := objectAPI.GetObjectInfo
|
||||||
|
|
||||||
var (
|
var (
|
||||||
hasLockEnabled bool
|
dsc ReplicateDecision
|
||||||
dsc ReplicateDecision
|
goi ObjectInfo
|
||||||
goi ObjectInfo
|
gerr error
|
||||||
gerr error
|
|
||||||
)
|
)
|
||||||
replicateDeletes := hasReplicationRules(ctx, bucket, deleteObjectsReq.Objects)
|
replicateDeletes := hasReplicationRules(ctx, bucket, deleteObjectsReq.Objects)
|
||||||
if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
|
|
||||||
hasLockEnabled = true
|
|
||||||
}
|
|
||||||
|
|
||||||
type deleteResult struct {
|
type deleteResult struct {
|
||||||
delInfo DeletedObject
|
delInfo DeletedObject
|
||||||
@@ -496,7 +492,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
|||||||
deleteResults := make([]deleteResult, len(deleteObjectsReq.Objects))
|
deleteResults := make([]deleteResult, len(deleteObjectsReq.Objects))
|
||||||
duplicateOf := make(map[int]int)
|
duplicateOf := make(map[int]int)
|
||||||
|
|
||||||
vc, _ := globalBucketVersioningSys.Get(bucket)
|
vc, rcfg, metaErr := checkBucketDeleteMetadata(bucket)
|
||||||
|
if metaErr != nil {
|
||||||
|
writeErrorResponse(ctx, w, toAPIError(ctx, metaErr), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
lockedNullDelete := func(obj ObjectToDelete) bool {
|
||||||
|
return obj.VersionID == "" && rcfg.LockEnabled && !vc.PrefixEnabled(obj.ObjectName)
|
||||||
|
}
|
||||||
oss := make([]*objSweeper, len(deleteObjectsReq.Objects))
|
oss := make([]*objSweeper, len(deleteObjectsReq.Objects))
|
||||||
for index, object := range deleteObjectsReq.Objects {
|
for index, object := range deleteObjectsReq.Objects {
|
||||||
reqInfo.ObjectName = object.ObjectName
|
reqInfo.ObjectName = object.ObjectName
|
||||||
@@ -537,10 +540,10 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
|||||||
VersionSuspended: vc.Suspended(),
|
VersionSuspended: vc.Suspended(),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Physical version deletes in a locked bucket are checked by the
|
// Every physical version delete checks the version's own protection
|
||||||
// single-object path below, while its mutation lock is held. A lookup
|
// under its mutation lock. Bucket metadata may be stale or unavailable
|
||||||
// here could miss a later protection update or a protected pool copy.
|
// and cannot establish that a stored version is unprotected.
|
||||||
if hasLockEnabled && object.VersionID != "" {
|
if object.VersionID != "" || lockedNullDelete(object) {
|
||||||
if first, ok := objectsToDelete[object]; ok {
|
if first, ok := objectsToDelete[object]; ok {
|
||||||
duplicateOf[index] = first
|
duplicateOf[index] = first
|
||||||
} else {
|
} else {
|
||||||
@@ -605,23 +608,32 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
|||||||
bulk := make([]ObjectToDelete, 0, len(deleteList))
|
bulk := make([]ObjectToDelete, 0, len(deleteList))
|
||||||
bulkIndex := make([]int, 0, len(deleteList))
|
bulkIndex := make([]int, 0, len(deleteList))
|
||||||
for i, object := range deleteList {
|
for i, object := range deleteList {
|
||||||
if !hasLockEnabled || object.VersionID == "" {
|
if object.VersionID == "" && !lockedNullDelete(object) {
|
||||||
bulk = append(bulk, object)
|
bulk = append(bulk, object)
|
||||||
bulkIndex = append(bulkIndex, i)
|
bulkIndex = append(bulkIndex, i)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
reqInfo.ObjectName, reqInfo.VersionID = object.ObjectName, object.VersionID
|
reqInfo.ObjectName, reqInfo.VersionID = object.ObjectName, object.VersionID
|
||||||
opts := ObjectOptions{
|
opts := ObjectOptions{
|
||||||
VersionID: object.VersionID,
|
VersionID: object.VersionID,
|
||||||
Versioned: vc.PrefixEnabled(object.ObjectName),
|
Versioned: vc.PrefixEnabled(object.ObjectName),
|
||||||
VersionSuspended: vc.Suspended(),
|
VersionSuspended: vc.Suspended(),
|
||||||
|
RefuseLockedOverwrite: lockedNullDelete(object),
|
||||||
}
|
}
|
||||||
index := objectsToDelete[object]
|
index := objectsToDelete[object]
|
||||||
if !globalTierConfigMgr.Empty() {
|
if !globalTierConfigMgr.Empty() {
|
||||||
oss[index] = newObjSweeper(bucket, object.ObjectName).WithVersion(object.VersionID).WithVersioning(opts.Versioned, opts.VersionSuspended)
|
sweepVID := object.VersionID
|
||||||
|
if sweepVID == "" && opts.VersionSuspended {
|
||||||
|
sweepVID = nullVersionID
|
||||||
|
}
|
||||||
|
oss[index] = newObjSweeper(bucket, object.ObjectName).WithVersion(sweepVID).WithVersioning(opts.Versioned, opts.VersionSuspended)
|
||||||
}
|
}
|
||||||
opts.SetEvalRetentionBypassFn(func(oi ObjectInfo, err error) error {
|
opts.SetEvalRetentionBypassFn(func(oi ObjectInfo, err error) error {
|
||||||
return enforceRetentionBypassForDelete(ctx, r, bucket, object, oi, err)
|
checkObj := object
|
||||||
|
if checkObj.VersionID == "" {
|
||||||
|
checkObj.VersionID = nullVersionID
|
||||||
|
}
|
||||||
|
return enforceRetentionBypassForDelete(ctx, r, bucket, checkObj, oi, err)
|
||||||
})
|
})
|
||||||
opts.SetEvalMetadataFn(func(oi *ObjectInfo, err error) (ReplicateDecision, error) {
|
opts.SetEvalMetadataFn(func(oi *ObjectInfo, err error) (ReplicateDecision, error) {
|
||||||
if err == nil && oss[index] != nil {
|
if err == nil && oss[index] != nil {
|
||||||
@@ -635,11 +647,17 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
|||||||
// DeleteObject merges protection across pools before evaluating these
|
// DeleteObject merges protection across pools before evaluating these
|
||||||
// callbacks and holds the same lock as retention and legal-hold writes.
|
// callbacks and holds the same lock as retention and legal-hold writes.
|
||||||
oi, err := objectAPI.DeleteObject(ctx, bucket, object.ObjectName, opts)
|
oi, err := objectAPI.DeleteObject(ctx, bucket, object.ObjectName, opts)
|
||||||
dObjects[i] = DeletedObject{
|
delObj := DeletedObject{
|
||||||
ObjectName: object.ObjectName,
|
ObjectName: object.ObjectName,
|
||||||
VersionID: object.VersionID,
|
VersionID: object.VersionID,
|
||||||
ReplicationState: oi.ReplicationState(),
|
ReplicationState: oi.ReplicationState(),
|
||||||
}
|
}
|
||||||
|
if object.VersionID == "" && oi.DeleteMarker {
|
||||||
|
delObj.DeleteMarker = true
|
||||||
|
delObj.DeleteMarkerVersionID = oi.VersionID
|
||||||
|
delObj.DeleteMarkerMTime = DeleteMarkerMTime{oi.ModTime}
|
||||||
|
}
|
||||||
|
dObjects[i] = delObj
|
||||||
errs[i] = err
|
errs[i] = err
|
||||||
}
|
}
|
||||||
if len(bulk) > 0 {
|
if len(bulk) > 0 {
|
||||||
@@ -663,7 +681,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
|||||||
// under the mutation lock, and a missing version has no stored identity.
|
// under the mutation lock, and a missing version has no stored identity.
|
||||||
dindex := objectsToDelete[deleteList[i]]
|
dindex := objectsToDelete[deleteList[i]]
|
||||||
if errs[i] == nil || isErrObjectNotFound(errs[i]) || isErrVersionNotFound(errs[i]) {
|
if errs[i] == nil || isErrObjectNotFound(errs[i]) || isErrVersionNotFound(errs[i]) {
|
||||||
if replicateDeletes && (!hasLockEnabled || deleteList[i].VersionID == "") {
|
if replicateDeletes && deleteList[i].VersionID == "" && !lockedNullDelete(deleteList[i]) {
|
||||||
dObjects[i].ReplicationState = deleteList[i].ReplicationState()
|
dObjects[i].ReplicationState = deleteList[i].ReplicationState()
|
||||||
}
|
}
|
||||||
deleteResults[dindex].delInfo = dObjects[i]
|
deleteResults[dindex].delInfo = dObjects[i]
|
||||||
|
|||||||
@@ -90,8 +90,14 @@ func (api objectAPIHandlers) listObjectVersionsHandler(w http.ResponseWriter, r
|
|||||||
}
|
}
|
||||||
var checkObjMeta metaCheckFn
|
var checkObjMeta metaCheckFn
|
||||||
if metadata {
|
if metadata {
|
||||||
checkObjMeta = func(name string, action policy.Action) (s3Err APIErrorCode) {
|
checkObjMeta = func(obj ObjectInfo, action policy.Action) (s3Err APIErrorCode) {
|
||||||
return checkRequestAuthType(ctx, r, action, bucket, name)
|
vid := obj.VersionID
|
||||||
|
if vid == "" {
|
||||||
|
vid = nullVersionID
|
||||||
|
}
|
||||||
|
return withScopedReqObject(ctx, bucket, obj.Name, vid, func() APIErrorCode {
|
||||||
|
return authorizeVersionedRequestWithExistingTags(ctx, r, action, vid, obj.UserTags)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
urlValues := r.Form
|
urlValues := r.Form
|
||||||
@@ -176,8 +182,10 @@ func (api objectAPIHandlers) listObjectsV2Handler(ctx context.Context, w http.Re
|
|||||||
|
|
||||||
var checkObjMeta metaCheckFn
|
var checkObjMeta metaCheckFn
|
||||||
if metadata {
|
if metadata {
|
||||||
checkObjMeta = func(name string, action policy.Action) (s3Err APIErrorCode) {
|
checkObjMeta = func(obj ObjectInfo, action policy.Action) (s3Err APIErrorCode) {
|
||||||
return checkRequestAuthType(ctx, r, action, bucket, name)
|
return withScopedReqObject(ctx, bucket, obj.Name, "", func() APIErrorCode {
|
||||||
|
return authorizeVersionedRequestWithExistingTags(ctx, r, action, "", obj.UserTags)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
urlValues := r.Form
|
urlValues := r.Form
|
||||||
@@ -202,7 +210,7 @@ func (api objectAPIHandlers) listObjectsV2Handler(ctx context.Context, w http.Re
|
|||||||
|
|
||||||
if r.Header.Get(xMinIOExtract) == "true" && strings.Contains(prefix, archivePattern) {
|
if r.Header.Get(xMinIOExtract) == "true" && strings.Contains(prefix, archivePattern) {
|
||||||
// Initiate a list objects operation inside a zip file based in the input params
|
// Initiate a list objects operation inside a zip file based in the input params
|
||||||
listObjectsV2Info, err = listObjectsV2InArchive(ctx, objectAPI, bucket, prefix, token, delimiter, maxKeys, startAfter, r.Header)
|
listObjectsV2Info, err = listObjectsV2InArchive(ctx, objectAPI, bucket, prefix, token, delimiter, maxKeys, startAfter, r)
|
||||||
} else {
|
} else {
|
||||||
// Initiate a list objects operation based on the input params.
|
// Initiate a list objects operation based on the input params.
|
||||||
// On success would return back ListObjectsInfo object to be
|
// On success would return back ListObjectsInfo object to be
|
||||||
|
|||||||
@@ -0,0 +1,125 @@
|
|||||||
|
// Copyright (c) 2026 Feng Ruohang
|
||||||
|
//
|
||||||
|
// This file is part of Silo Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/xml"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
)
|
||||||
|
|
||||||
|
type metadataCacheLoadBarrier struct {
|
||||||
|
ObjectLayer
|
||||||
|
key string
|
||||||
|
read, resume chan struct{}
|
||||||
|
once sync.Once
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *metadataCacheLoadBarrier) GetObjectNInfo(ctx context.Context, bucket, key string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
|
||||||
|
r, err := b.ObjectLayer.GetObjectNInfo(ctx, bucket, key, rs, h, opts)
|
||||||
|
if bucket == minioMetaBucket && key == b.key {
|
||||||
|
b.once.Do(func() { close(b.read); <-b.resume })
|
||||||
|
}
|
||||||
|
return r, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBucketMetadataLoadPreservesConcurrentPublication(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||||
|
// Disk has the old unlocked config; a concurrent publication enables
|
||||||
|
// lock and versioning after the cache miss has started loading it.
|
||||||
|
enableBucketObjectLock(t, bucket)
|
||||||
|
newer, err := globalBucketMetadataSys.Get(bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
globalBucketMetadataSys.Remove(bucket)
|
||||||
|
barrier := &metadataCacheLoadBarrier{ObjectLayer: obj, key: pathJoin(bucketMetaPrefix, bucket, bucketMetadataFile), read: make(chan struct{}), resume: make(chan struct{})}
|
||||||
|
previous := newObjectLayerFn()
|
||||||
|
setObjectLayer(barrier)
|
||||||
|
defer setObjectLayer(previous)
|
||||||
|
release := sync.OnceFunc(func() { close(barrier.resume) })
|
||||||
|
defer release()
|
||||||
|
type result struct {
|
||||||
|
meta BucketMetadata
|
||||||
|
reloaded bool
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
done := make(chan result, 1)
|
||||||
|
go func() {
|
||||||
|
meta, reloaded, err := globalBucketMetadataSys.GetConfig(t.Context(), bucket)
|
||||||
|
done <- result{meta, reloaded, err}
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-barrier.read:
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("metadata load did not reach disk")
|
||||||
|
}
|
||||||
|
globalBucketMetadataSys.Set(bucket, newer)
|
||||||
|
release()
|
||||||
|
select {
|
||||||
|
case got := <-done:
|
||||||
|
if got.err != nil || got.reloaded || got.meta.objectLockConfig == nil || got.meta.objectLockConfig.ObjectLockEnabled != "Enabled" {
|
||||||
|
t.Fatalf("%s: stale load replaced the publication: %+v", backend, got)
|
||||||
|
}
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("metadata load did not finish")
|
||||||
|
}
|
||||||
|
cached, err := globalBucketMetadataSys.Get(bucket)
|
||||||
|
if err != nil || cached.objectLockConfig == nil || cached.objectLockConfig.ObjectLockEnabled != "Enabled" {
|
||||||
|
t.Fatalf("cached lock lost: %+v %v", cached, err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBulkDeleteVersionLockWithStaleBucketCache(t *testing.T) {
|
||||||
|
for _, pools := range []int{1, 2} {
|
||||||
|
z, _ := bulkDeletePools(t, pools)
|
||||||
|
bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{LockEnabled: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, protection := range batchLockKinds {
|
||||||
|
oi := batchPutVersion(t, z, bucket, protection, batchLockMeta(protection))
|
||||||
|
saved, err := globalBucketMetadataSys.Get(bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
globalBucketMetadataSys.Set(bucket, newBucketMetadata(bucket))
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, bulkDeleteLockRequest(t, bucket, protection, oi.VersionID))
|
||||||
|
globalBucketMetadataSys.Set(bucket, saved)
|
||||||
|
var response DeleteObjectsResponse
|
||||||
|
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if rec.Code != http.StatusOK || len(response.Errors) != 1 || len(response.DeletedObjects) != 0 || !batchVersionExists(t, z, bucket, protection, oi.VersionID) {
|
||||||
|
t.Fatalf("pools=%d %s: stale bucket cache bypassed version lock: %s", pools, protection, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -182,8 +182,13 @@ func bucketConfigPayload(bucket, file string, data []byte, lockEnabled bool) ([]
|
|||||||
key := data
|
key := data
|
||||||
switch file {
|
switch file {
|
||||||
case bucketPolicyConfig:
|
case bucketPolicyConfig:
|
||||||
|
// Replication uses explicit null as a tombstone; ordinary policy PUTs
|
||||||
|
// validate their JSON before entering this metadata path.
|
||||||
|
if bytes.Equal(bytes.TrimSpace(data), []byte("null")) {
|
||||||
|
return nil, nil, nil
|
||||||
|
}
|
||||||
var cfg *policy.BucketPolicy
|
var cfg *policy.BucketPolicy
|
||||||
cfg, err = policy.ParseBucketPolicyConfig(bytes.NewReader(data), bucket)
|
cfg, err = parseBucketPolicyStrictBytes(data, bucket)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
if cfg.IsEmpty() {
|
if cfg.IsEmpty() {
|
||||||
return nil, nil, nil
|
return nil, nil, nil
|
||||||
|
|||||||
@@ -697,7 +697,7 @@ func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (met
|
|||||||
}
|
}
|
||||||
|
|
||||||
val, err, _ := sys.group.Do(bucket, func() (val any, err error) {
|
val, err, _ := sys.group.Do(bucket, func() (val any, err error) {
|
||||||
meta, err = loadBucketMetadata(ctx, objAPI, bucket)
|
meta, err := loadBucketMetadata(ctx, objAPI, bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !sys.Initialized() {
|
if !sys.Initialized() {
|
||||||
// bucket metadata not yet initialized
|
// bucket metadata not yet initialized
|
||||||
@@ -707,13 +707,18 @@ func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (met
|
|||||||
return meta, err
|
return meta, err
|
||||||
})
|
})
|
||||||
meta, _ = val.(BucketMetadata)
|
meta, _ = val.(BucketMetadata)
|
||||||
|
sys.Lock()
|
||||||
|
defer sys.Unlock()
|
||||||
|
// A writer may have published a newer configuration while the disk read
|
||||||
|
// was in flight. Never replace that publication with the older load.
|
||||||
|
if current, ok := sys.metadataMap[bucket]; ok {
|
||||||
|
return current, false, nil
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return meta, false, err
|
return meta, false, err
|
||||||
}
|
}
|
||||||
sys.Lock()
|
|
||||||
sys.metadataMap[bucket] = meta
|
sys.metadataMap[bucket] = meta
|
||||||
sys.clearLoadFailure(bucket)
|
sys.clearLoadFailure(bucket)
|
||||||
sys.Unlock()
|
|
||||||
|
|
||||||
return meta, true, nil
|
return meta, true, nil
|
||||||
}
|
}
|
||||||
|
|||||||
+145
-5
@@ -28,11 +28,27 @@ import (
|
|||||||
"github.com/minio/minio/internal/amztime"
|
"github.com/minio/minio/internal/amztime"
|
||||||
"github.com/minio/minio/internal/auth"
|
"github.com/minio/minio/internal/auth"
|
||||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||||
|
"github.com/minio/minio/internal/bucket/versioning"
|
||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/pgsty/silo-pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// checkBucketDeleteMetadata loads bucket versioning and Object Lock configs
|
||||||
|
// fail-closed so delete handlers never fall back to unversioned/unlocked
|
||||||
|
// defaults when bucket metadata cannot be read.
|
||||||
|
func checkBucketDeleteMetadata(bucketName string) (*versioning.Versioning, objectlock.Retention, error) {
|
||||||
|
vc, err := globalBucketVersioningSys.Get(bucketName)
|
||||||
|
if err != nil {
|
||||||
|
return nil, objectlock.Retention{}, err
|
||||||
|
}
|
||||||
|
rcfg, err := globalBucketObjectLockSys.Get(bucketName)
|
||||||
|
if err != nil {
|
||||||
|
return nil, objectlock.Retention{}, err
|
||||||
|
}
|
||||||
|
return vc, rcfg, nil
|
||||||
|
}
|
||||||
|
|
||||||
// BucketObjectLockSys - map of bucket and retention configuration.
|
// BucketObjectLockSys - map of bucket and retention configuration.
|
||||||
type BucketObjectLockSys struct{}
|
type BucketObjectLockSys struct{}
|
||||||
|
|
||||||
@@ -354,6 +370,8 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
|||||||
var legalHold objectlock.ObjectLegalHold
|
var legalHold objectlock.ObjectLegalHold
|
||||||
|
|
||||||
retentionRequested := objectlock.IsObjectLockRetentionRequested(rq.Header)
|
retentionRequested := objectlock.IsObjectLockRetentionRequested(rq.Header)
|
||||||
|
// A timestamp without a value is a replicated retention removal.
|
||||||
|
retentionRequested = retentionRequested || replicaTrusted && rq.Header.Get(xhttp.MinIOSourceObjectRetentionTimestamp) != ""
|
||||||
legalHoldRequested := objectlock.IsObjectLockLegalHoldRequested(rq.Header)
|
legalHoldRequested := objectlock.IsObjectLockLegalHoldRequested(rq.Header)
|
||||||
|
|
||||||
retentionCfg, err := globalBucketObjectLockSys.Get(bucket)
|
retentionCfg, err := globalBucketObjectLockSys.Get(bucket)
|
||||||
@@ -472,6 +490,103 @@ func storedObjectLockState(metadata map[string]string) objectLockState {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// reducePoolLockState selects each field independently using a total order,
|
||||||
|
// so neither pool enumeration nor the starting copy can change the result.
|
||||||
|
// Active COMPLIANCE takes precedence over revisions; among those copies the
|
||||||
|
// longest retention wins. Only expired COMPLIANCE may lose to a removal.
|
||||||
|
func reducePoolLockState(copies []PoolObjInfo, readState func(map[string]string) objectLockState) objectLockState {
|
||||||
|
var now time.Time
|
||||||
|
clockRead := false
|
||||||
|
state := readState(copies[0].ObjInfo.UserDefined)
|
||||||
|
for _, copy := range copies[1:] {
|
||||||
|
next := readState(copy.ObjInfo.UserDefined)
|
||||||
|
if !clockRead && (state.mode == string(objectlock.RetCompliance) || next.mode == string(objectlock.RetCompliance)) {
|
||||||
|
var err error
|
||||||
|
now, err = objectlock.UTCNowNTP()
|
||||||
|
if err != nil {
|
||||||
|
now = time.Time{} // An unavailable clock cannot prove retention expired.
|
||||||
|
}
|
||||||
|
clockRead = true
|
||||||
|
}
|
||||||
|
if comparePoolRetention(next, state, now) > 0 {
|
||||||
|
state.mode, state.retainUntil, state.retentionTimestamp = next.mode, next.retainUntil, next.retentionTimestamp
|
||||||
|
}
|
||||||
|
order := compareLockRevision(next.legalHoldTimestamp, state.legalHoldTimestamp)
|
||||||
|
// Equal or unordered conflicting holds resolve to ON, then OFF, then
|
||||||
|
// absence. A strictly newer valid OFF still releases a prior hold.
|
||||||
|
if order == 0 {
|
||||||
|
order = strings.Compare(next.legalHold, state.legalHold)
|
||||||
|
}
|
||||||
|
if order == 0 {
|
||||||
|
order = strings.Compare(next.legalHoldTimestamp, state.legalHoldTimestamp)
|
||||||
|
}
|
||||||
|
if order > 0 {
|
||||||
|
state.legalHold, state.legalHoldTimestamp = next.legalHold, next.legalHoldTimestamp
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return state
|
||||||
|
}
|
||||||
|
|
||||||
|
func compareLockRevision(a, b string) int {
|
||||||
|
ta, aok := lockRevisionTime(a)
|
||||||
|
tb, bok := lockRevisionTime(b)
|
||||||
|
if aok != bok {
|
||||||
|
if aok {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
if !aok {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return ta.Compare(tb)
|
||||||
|
}
|
||||||
|
|
||||||
|
func comparePoolRetention(a, b objectLockState, now time.Time) int {
|
||||||
|
au, ae := time.Parse(time.RFC3339Nano, a.retainUntil)
|
||||||
|
bu, be := time.Parse(time.RFC3339Nano, b.retainUntil)
|
||||||
|
ap := a.mode == string(objectlock.RetCompliance) && (ae != nil || now.IsZero() || au.After(now))
|
||||||
|
bp := b.mode == string(objectlock.RetCompliance) && (be != nil || now.IsZero() || bu.After(now))
|
||||||
|
if ap != bp {
|
||||||
|
if ap {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
if ap { // An unreadable COMPLIANCE deadline cannot be shortened either.
|
||||||
|
if (ae != nil) != (be != nil) {
|
||||||
|
if ae != nil {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
if ae == nil && au.Compare(bu) != 0 {
|
||||||
|
return au.Compare(bu)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if order := compareLockRevision(a.retentionTimestamp, b.retentionTimestamp); order != 0 {
|
||||||
|
return order
|
||||||
|
}
|
||||||
|
// Ties and legacy unordered metadata must also have a stable winner.
|
||||||
|
// A retained value takes precedence over an unordered absence.
|
||||||
|
if order := strings.Compare(a.mode, b.mode); order != 0 {
|
||||||
|
return order
|
||||||
|
}
|
||||||
|
if (ae != nil) != (be != nil) {
|
||||||
|
if ae != nil {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
if ae == nil && be == nil && au.Compare(bu) != 0 {
|
||||||
|
return au.Compare(bu)
|
||||||
|
}
|
||||||
|
if order := strings.Compare(a.retainUntil, b.retainUntil); order != 0 {
|
||||||
|
return order
|
||||||
|
}
|
||||||
|
return strings.Compare(a.retentionTimestamp, b.retentionTimestamp)
|
||||||
|
}
|
||||||
|
|
||||||
// olderThan reports whether a stored replication timestamp is missing,
|
// olderThan reports whether a stored replication timestamp is missing,
|
||||||
// unreadable, or earlier than the source timestamp, in which case the
|
// unreadable, or earlier than the source timestamp, in which case the
|
||||||
// replica update wins. A zero source timestamp never wins.
|
// replica update wins. A zero source timestamp never wins.
|
||||||
@@ -491,6 +606,25 @@ func (s objectLockState) legalHoldIsOlderThan(src time.Time) bool {
|
|||||||
return olderThan(s.legalHoldTimestamp, src)
|
return olderThan(s.legalHoldTimestamp, src)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A peer's ordering timestamp cannot override an unexpired COMPLIANCE lock.
|
||||||
|
// Otherwise it could first remove retention with a same-content retransmit,
|
||||||
|
// then replace the now-unprotected version in a second request.
|
||||||
|
func (s objectLockState) complianceWouldWeaken(incoming objectLockState) bool {
|
||||||
|
if s.mode != string(objectlock.RetCompliance) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
until, err := time.Parse(time.RFC3339Nano, s.retainUntil)
|
||||||
|
if err != nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
now, err := objectlock.UTCNowNTP()
|
||||||
|
if err == nil && !until.After(now) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
nextUntil, nextErr := time.Parse(time.RFC3339Nano, incoming.retainUntil)
|
||||||
|
return incoming.mode != string(objectlock.RetCompliance) || nextErr != nil || nextUntil.Before(until)
|
||||||
|
}
|
||||||
|
|
||||||
// restoreRetention and restoreLegalHold put the stored state back into
|
// restoreRetention and restoreLegalHold put the stored state back into
|
||||||
// metadata that was rebuilt from a request whose update was not applied.
|
// metadata that was rebuilt from a request whose update was not applied.
|
||||||
func (s objectLockState) restoreRetention(metadata map[string]string) {
|
func (s objectLockState) restoreRetention(metadata map[string]string) {
|
||||||
@@ -524,14 +658,20 @@ func (s objectLockState) restoreLegalHold(metadata map[string]string) {
|
|||||||
// (an older incoming value must not win over a newer stored one just because the
|
// (an older incoming value must not win over a newer stored one just because the
|
||||||
// read timed out).
|
// read timed out).
|
||||||
func replicaStoredLock(ctx context.Context, getObjectInfo GetObjectInfoFn, bucket, object, versionID string) (objectLockState, error) {
|
func replicaStoredLock(ctx context.Context, getObjectInfo GetObjectInfoFn, bucket, object, versionID string) (objectLockState, error) {
|
||||||
oi, err := getObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: versionID})
|
oi, err := replicaStoredObjectInfo(ctx, getObjectInfo, bucket, object, versionID)
|
||||||
|
return storedObjectLockState(oi.UserDefined), err
|
||||||
|
}
|
||||||
|
|
||||||
|
// The same destination lookup supplies both Object Lock and tag revisions.
|
||||||
|
func replicaStoredObjectInfo(ctx context.Context, getObjectInfo GetObjectInfoFn, bucket, object, versionID string) (ObjectInfo, error) {
|
||||||
|
oi, err := getObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: versionID, ReplicationRequest: true})
|
||||||
switch {
|
switch {
|
||||||
case err == nil:
|
case err == nil:
|
||||||
return storedObjectLockState(oi.UserDefined), nil
|
return oi, nil
|
||||||
case isErrObjectNotFound(err) || isErrVersionNotFound(err):
|
case isErrObjectNotFound(err) || isErrVersionNotFound(err):
|
||||||
return objectLockState{}, nil
|
return ObjectInfo{}, nil
|
||||||
default:
|
default:
|
||||||
return objectLockState{}, err
|
return ObjectInfo{}, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -610,7 +750,7 @@ func reconcileStoredObjectLock(metadata map[string]string, stored objectLockStat
|
|||||||
incoming := storedObjectLockState(metadata)
|
incoming := storedObjectLockState(metadata)
|
||||||
|
|
||||||
incomingRetentionTS, _ := time.Parse(time.RFC3339Nano, incoming.retentionTimestamp)
|
incomingRetentionTS, _ := time.Parse(time.RFC3339Nano, incoming.retentionTimestamp)
|
||||||
if !stored.retentionIsOlderThan(incomingRetentionTS) {
|
if !stored.retentionIsOlderThan(incomingRetentionTS) || stored.complianceWouldWeaken(incoming) {
|
||||||
// The stored retention is at least as new as the incoming one (or the
|
// The stored retention is at least as new as the incoming one (or the
|
||||||
// incoming update is unordered): drop the incoming value and put the stored
|
// incoming update is unordered): drop the incoming value and put the stored
|
||||||
// state back, which may itself be a removal (value keys absent, timestamp
|
// state back, which may itself be a removal (value keys absent, timestamp
|
||||||
|
|||||||
@@ -18,7 +18,6 @@
|
|||||||
package cmd
|
package cmd
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
@@ -83,7 +82,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
bucketPolicy, err := policy.ParseBucketPolicyConfig(bytes.NewReader(bucketPolicyBytes), bucket)
|
bucketPolicy, err := parseBucketPolicyStrictBytes(bucketPolicyBytes, bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponse(ctx, w, APIError{
|
writeErrorResponse(ctx, w, APIError{
|
||||||
Code: "MalformedPolicy",
|
Code: "MalformedPolicy",
|
||||||
|
|||||||
+13
-2
@@ -213,16 +213,27 @@ func getConditionValuesWithTags(r *http.Request, lc string, cred auth.Credential
|
|||||||
switch authType {
|
switch authType {
|
||||||
case authTypeSignedV2, authTypePresignedV2:
|
case authTypeSignedV2, authTypePresignedV2:
|
||||||
signatureVersion = signV2Algorithm
|
signatureVersion = signV2Algorithm
|
||||||
case authTypeSigned, authTypePresigned, authTypeStreamingSigned, authTypePostPolicy:
|
case authTypeSigned, authTypePresigned, authTypeStreamingSigned,
|
||||||
|
authTypeStreamingSignedTrailer, authTypePostPolicy:
|
||||||
signatureVersion = signV4Algorithm
|
signatureVersion = signV4Algorithm
|
||||||
|
case authTypeStreamingUnsignedTrailer:
|
||||||
|
// This form is also valid for a genuinely anonymous request. Only
|
||||||
|
// expose SigV4 condition values when header authentication is present.
|
||||||
|
if r.Header.Get(xhttp.Authorization) != "" {
|
||||||
|
signatureVersion = signV4Algorithm
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var authtype string
|
var authtype string
|
||||||
switch authType {
|
switch authType {
|
||||||
case authTypePresignedV2, authTypePresigned:
|
case authTypePresignedV2, authTypePresigned:
|
||||||
authtype = "REST-QUERY-STRING"
|
authtype = "REST-QUERY-STRING"
|
||||||
case authTypeSignedV2, authTypeSigned, authTypeStreamingSigned:
|
case authTypeSignedV2, authTypeSigned, authTypeStreamingSigned, authTypeStreamingSignedTrailer:
|
||||||
authtype = "REST-HEADER"
|
authtype = "REST-HEADER"
|
||||||
|
case authTypeStreamingUnsignedTrailer:
|
||||||
|
if r.Header.Get(xhttp.Authorization) != "" {
|
||||||
|
authtype = "REST-HEADER"
|
||||||
|
}
|
||||||
case authTypePostPolicy:
|
case authTypePostPolicy:
|
||||||
authtype = "POST"
|
authtype = "POST"
|
||||||
}
|
}
|
||||||
|
|||||||
+117
-105
@@ -2673,6 +2673,7 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRa
|
|||||||
},
|
},
|
||||||
PartNumber: opts.PartNumber,
|
PartNumber: opts.PartNumber,
|
||||||
}
|
}
|
||||||
|
gopts.Set(xhttp.AmzTagDirective, accessDirective)
|
||||||
if rs != nil {
|
if rs != nil {
|
||||||
h, err := rs.ToHeader()
|
h, err := rs.ToHeader()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -2710,6 +2711,9 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRa
|
|||||||
}
|
}
|
||||||
oi.UserDefined = make(map[string]string, len(objInfo.Metadata))
|
oi.UserDefined = make(map[string]string, len(objInfo.Metadata))
|
||||||
for k, v := range objInfo.Metadata {
|
for k, v := range objInfo.Metadata {
|
||||||
|
if equals(k, xhttp.AmzTagCount, xhttp.AmzObjectTagging) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
oi.UserDefined[k] = v[0]
|
oi.UserDefined[k] = v[0]
|
||||||
}
|
}
|
||||||
ce, ok := oi.UserDefined[xhttp.ContentEncoding]
|
ce, ok := oi.UserDefined[xhttp.ContentEncoding]
|
||||||
@@ -2773,127 +2777,135 @@ func scheduleReplication(ctx context.Context, oi ObjectInfo, o ObjectLayer, dsc
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// proxyTaggingToRepTarget proxies tagging requests to remote targets for
|
// taggingProxyTarget keeps the exact client and tags that were authorized.
|
||||||
// active-active replicated setups
|
// A second discovery must not introduce an unauthorized destination.
|
||||||
func proxyTaggingToRepTarget(ctx context.Context, bucket, object string, tags *tags.Tags, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (proxy proxyResult) {
|
type taggingProxyTarget struct {
|
||||||
// this option is set when active-active replication is in place between site A -> B,
|
client *TargetClient
|
||||||
// and request hits site B that does not have the object yet.
|
tags *tags.Tags
|
||||||
if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { // true only when site B sets MinIOSourceProxyRequest header
|
}
|
||||||
return proxy
|
|
||||||
|
// readTaggingProxyTargets authorizes all present replicas before returning any
|
||||||
|
// tags or allowing any mutation. Missing replicas may simply lag replication;
|
||||||
|
// unreadable replicas must still fail closed.
|
||||||
|
func readTaggingProxyTargets(ctx context.Context, bucket, object string, opts ObjectOptions, proxyTargets *madmin.BucketTargets, authorize func(string, error) APIErrorCode) ([]taggingProxyTarget, proxyResult, APIErrorCode) {
|
||||||
|
var targets []taggingProxyTarget
|
||||||
|
for _, target := range proxyTargets.Targets {
|
||||||
|
tgt := globalBucketTargetSys.GetRemoteTargetClient(bucket, target.Arn)
|
||||||
|
if tgt == nil || globalBucketTargetSys.isOffline(tgt.EndpointURL()) || tgt.disableProxy {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
targets = append(targets, taggingProxyTarget{client: tgt})
|
||||||
}
|
}
|
||||||
|
if len(targets) == 0 {
|
||||||
|
err := RemoteTargetConnectionErr{Bucket: bucket, Err: errors.New("no available remote replication target")}
|
||||||
|
return nil, proxyResult{Err: err}, authorize("", err)
|
||||||
|
}
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
errs := make([]error, len(proxyTargets.Targets))
|
errs := make([]error, len(targets))
|
||||||
for idx, t := range proxyTargets.Targets {
|
for i := range targets {
|
||||||
tgt := globalBucketTargetSys.GetRemoteTargetClient(bucket, t.Arn)
|
wg.Go(func() {
|
||||||
if tgt == nil || globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
|
tgt := &targets[i]
|
||||||
continue
|
tgt.tags, errs[i] = tgt.client.GetObjectTagging(ctx, tgt.client.Bucket, object, minio.GetObjectTaggingOptions{
|
||||||
}
|
VersionID: opts.VersionID,
|
||||||
// if proxying explicitly disabled on remote target
|
Internal: minio.AdvancedObjectTaggingOptions{ReplicationProxyRequest: "true"},
|
||||||
if tgt.disableProxy {
|
})
|
||||||
continue
|
})
|
||||||
}
|
|
||||||
idx := idx
|
|
||||||
wg.Add(1)
|
|
||||||
go func(idx int, tgt *TargetClient) {
|
|
||||||
defer wg.Done()
|
|
||||||
var err error
|
|
||||||
if tags != nil {
|
|
||||||
popts := minio.PutObjectTaggingOptions{
|
|
||||||
VersionID: opts.VersionID,
|
|
||||||
Internal: minio.AdvancedObjectTaggingOptions{
|
|
||||||
ReplicationProxyRequest: "true",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
err = tgt.PutObjectTagging(ctx, tgt.Bucket, object, tags, popts)
|
|
||||||
} else {
|
|
||||||
dopts := minio.RemoveObjectTaggingOptions{
|
|
||||||
VersionID: opts.VersionID,
|
|
||||||
Internal: minio.AdvancedObjectTaggingOptions{
|
|
||||||
ReplicationProxyRequest: "true",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
err = tgt.RemoveObjectTagging(ctx, tgt.Bucket, object, dopts)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
errs[idx] = err
|
|
||||||
}
|
|
||||||
}(idx, tgt)
|
|
||||||
}
|
}
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
|
||||||
var (
|
var firstNotFound, readErr error
|
||||||
terr error
|
|
||||||
taggedCount int
|
|
||||||
)
|
|
||||||
for _, err := range errs {
|
for _, err := range errs {
|
||||||
if err == nil {
|
switch {
|
||||||
taggedCount++
|
case err == nil:
|
||||||
|
case isRemoteTaggingNotFound(err):
|
||||||
|
if firstNotFound == nil {
|
||||||
|
firstNotFound = err
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
if code := authorize("", err); code != ErrNone {
|
||||||
|
return nil, proxyResult{}, code
|
||||||
|
}
|
||||||
|
if readErr == nil {
|
||||||
|
readErr = err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
present := make([]taggingProxyTarget, 0, len(targets))
|
||||||
|
for i, tgt := range targets {
|
||||||
|
if errs[i] != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if err != nil {
|
if tgt.tags == nil {
|
||||||
terr = err
|
tgt.tags, _ = tags.MapToObjectTags(nil)
|
||||||
}
|
}
|
||||||
|
if code := authorize(tgt.tags.String(), nil); code != ErrNone {
|
||||||
|
return nil, proxyResult{}, code
|
||||||
|
}
|
||||||
|
present = append(present, tgt)
|
||||||
}
|
}
|
||||||
// don't return error if at least one target was tagged successfully
|
if readErr != nil {
|
||||||
if taggedCount == 0 && terr != nil {
|
return nil, proxyResult{Err: readErr}, ErrNone
|
||||||
proxy.Err = terr
|
|
||||||
}
|
}
|
||||||
return proxy
|
if len(present) == 0 {
|
||||||
|
// With no object anywhere, authorize the empty tag set once before
|
||||||
|
// disclosing its absence. No write RPC can help a missing replica.
|
||||||
|
return nil, proxyResult{Err: firstNotFound}, authorize("", firstNotFound)
|
||||||
|
}
|
||||||
|
return present, proxyResult{}, ErrNone
|
||||||
|
}
|
||||||
|
|
||||||
|
// proxyTaggingToRepTarget proxies tagging requests to remote targets for
|
||||||
|
// active-active replicated setups.
|
||||||
|
func proxyTaggingToRepTarget(ctx context.Context, bucket, object string, tags *tags.Tags, opts ObjectOptions, proxyTargets *madmin.BucketTargets, authorize func(string, error) APIErrorCode) (proxy proxyResult, authErr APIErrorCode) {
|
||||||
|
if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) {
|
||||||
|
return proxy, ErrNone
|
||||||
|
}
|
||||||
|
targets, proxy, authErr := readTaggingProxyTargets(ctx, bucket, object, opts, proxyTargets, authorize)
|
||||||
|
if authErr != ErrNone || proxy.Err != nil {
|
||||||
|
return proxy, authErr
|
||||||
|
}
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
writeErrs := make([]error, len(targets))
|
||||||
|
for i, target := range targets {
|
||||||
|
wg.Go(func() {
|
||||||
|
tgt := target.client
|
||||||
|
if tags != nil {
|
||||||
|
writeErrs[i] = tgt.PutObjectTagging(ctx, tgt.Bucket, object, tags, minio.PutObjectTaggingOptions{
|
||||||
|
VersionID: opts.VersionID,
|
||||||
|
Internal: minio.AdvancedObjectTaggingOptions{ReplicationProxyRequest: "true"},
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
writeErrs[i] = tgt.RemoveObjectTagging(ctx, tgt.Bucket, object, minio.RemoveObjectTaggingOptions{
|
||||||
|
VersionID: opts.VersionID,
|
||||||
|
Internal: minio.AdvancedObjectTaggingOptions{ReplicationProxyRequest: "true"},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
var lastErr error
|
||||||
|
for _, err := range writeErrs {
|
||||||
|
if err == nil {
|
||||||
|
return proxy, ErrNone
|
||||||
|
}
|
||||||
|
lastErr = err
|
||||||
|
}
|
||||||
|
return proxyResult{Err: lastErr}, ErrNone
|
||||||
}
|
}
|
||||||
|
|
||||||
// proxyGetTaggingToRepTarget proxies get tagging requests to remote targets for
|
// proxyGetTaggingToRepTarget proxies get tagging requests to remote targets for
|
||||||
// active-active replicated setups
|
// active-active replicated setups.
|
||||||
func proxyGetTaggingToRepTarget(ctx context.Context, bucket, object string, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (tgs *tags.Tags, proxy proxyResult) {
|
func proxyGetTaggingToRepTarget(ctx context.Context, bucket, object string, opts ObjectOptions, proxyTargets *madmin.BucketTargets, authorize func(string, error) APIErrorCode) (tgs *tags.Tags, proxy proxyResult, authErr APIErrorCode) {
|
||||||
// this option is set when active-active replication is in place between site A -> B,
|
if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) {
|
||||||
// and request hits site B that does not have the object yet.
|
return nil, proxy, ErrNone
|
||||||
if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { // true only when site B sets MinIOSourceProxyRequest header
|
|
||||||
return nil, proxy
|
|
||||||
}
|
}
|
||||||
var wg sync.WaitGroup
|
targets, proxy, authErr := readTaggingProxyTargets(ctx, bucket, object, opts, proxyTargets, authorize)
|
||||||
errs := make([]error, len(proxyTargets.Targets))
|
if authErr != ErrNone || proxy.Err != nil {
|
||||||
tagSlc := make([]map[string]string, len(proxyTargets.Targets))
|
return nil, proxy, authErr
|
||||||
for idx, t := range proxyTargets.Targets {
|
|
||||||
tgt := globalBucketTargetSys.GetRemoteTargetClient(bucket, t.Arn)
|
|
||||||
if tgt == nil || globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// if proxying explicitly disabled on remote target
|
|
||||||
if tgt.disableProxy {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
idx := idx
|
|
||||||
wg.Add(1)
|
|
||||||
go func(idx int, tgt *TargetClient) {
|
|
||||||
defer wg.Done()
|
|
||||||
var err error
|
|
||||||
gopts := minio.GetObjectTaggingOptions{
|
|
||||||
VersionID: opts.VersionID,
|
|
||||||
Internal: minio.AdvancedObjectTaggingOptions{
|
|
||||||
ReplicationProxyRequest: "true",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
tgs, err = tgt.GetObjectTagging(ctx, tgt.Bucket, object, gopts)
|
|
||||||
if err != nil {
|
|
||||||
errs[idx] = err
|
|
||||||
} else {
|
|
||||||
tagSlc[idx] = tgs.ToMap()
|
|
||||||
}
|
|
||||||
}(idx, tgt)
|
|
||||||
}
|
}
|
||||||
wg.Wait()
|
return targets[0].tags, proxy, ErrNone
|
||||||
for idx, err := range errs {
|
|
||||||
errCode := minio.ToErrorResponse(err).Code
|
|
||||||
if err != nil && errCode != "NoSuchKey" && errCode != "NoSuchVersion" {
|
|
||||||
return nil, proxyResult{Err: err}
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
tgs, _ = tags.MapToObjectTags(tagSlc[idx])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(errs) == 1 {
|
|
||||||
proxy.Err = errs[0]
|
|
||||||
}
|
|
||||||
return tgs, proxy
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func scheduleReplicationDelete(ctx context.Context, dv DeletedObjectReplicationInfo, o ObjectLayer) {
|
func scheduleReplicationDelete(ctx context.Context, dv DeletedObjectReplicationInfo, o ObjectLayer) {
|
||||||
|
|||||||
+63
-41
@@ -260,6 +260,15 @@ func ParseSSECustomerHeader(header http.Header) (key []byte, err error) {
|
|||||||
// This function rotates old to new key.
|
// This function rotates old to new key.
|
||||||
func rotateKey(ctx context.Context, oldKey []byte, newKeyID string, newKey []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context) error {
|
func rotateKey(ctx context.Context, oldKey []byte, newKeyID string, newKey []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context) error {
|
||||||
kind, _ := crypto.IsEncrypted(metadata)
|
kind, _ := crypto.IsEncrypted(metadata)
|
||||||
|
return rotateKeyWithTarget(ctx, oldKey, kind, newKeyID, newKey, bucket, object, metadata, cryptoCtx, len(cryptoCtx) > 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func rotateKeyWithTarget(ctx context.Context, oldKey []byte, targetKind crypto.Type, newKeyID string, newKey []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context, replaceContext bool) error {
|
||||||
|
kind, _ := crypto.IsEncrypted(metadata)
|
||||||
|
if targetKind == nil {
|
||||||
|
targetKind = kind
|
||||||
|
}
|
||||||
|
var objectKey crypto.ObjectKey
|
||||||
switch kind {
|
switch kind {
|
||||||
case crypto.S3:
|
case crypto.S3:
|
||||||
if GlobalKMS == nil {
|
if GlobalKMS == nil {
|
||||||
@@ -269,7 +278,7 @@ func rotateKey(ctx context.Context, oldKey []byte, newKeyID string, newKey []byt
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
oldKey, err := GlobalKMS.Decrypt(ctx, &kms.DecryptRequest{
|
unsealKey, err := GlobalKMS.Decrypt(ctx, &kms.DecryptRequest{
|
||||||
Name: keyID,
|
Name: keyID,
|
||||||
Ciphertext: kmsKey,
|
Ciphertext: kmsKey,
|
||||||
AssociatedData: kms.Context{bucket: path.Join(bucket, object)},
|
AssociatedData: kms.Context{bucket: path.Join(bucket, object)},
|
||||||
@@ -277,66 +286,30 @@ func rotateKey(ctx context.Context, oldKey []byte, newKeyID string, newKey []byt
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
var objectKey crypto.ObjectKey
|
if err = objectKey.Unseal(unsealKey, sealedKey, crypto.S3.String(), bucket, object); err != nil {
|
||||||
if err = objectKey.Unseal(oldKey, sealedKey, crypto.S3.String(), bucket, object); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
newKey, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{
|
|
||||||
Name: GlobalKMS.DefaultKey,
|
|
||||||
AssociatedData: kms.Context{bucket: path.Join(bucket, object)},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
sealedKey = objectKey.Seal(newKey.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
|
|
||||||
crypto.S3.CreateMetadata(metadata, newKey.KeyID, newKey.Ciphertext, sealedKey)
|
|
||||||
return nil
|
|
||||||
case crypto.S3KMS:
|
case crypto.S3KMS:
|
||||||
if GlobalKMS == nil {
|
if GlobalKMS == nil {
|
||||||
return errKMSNotConfigured
|
return errKMSNotConfigured
|
||||||
}
|
}
|
||||||
objectKey, err := crypto.S3KMS.UnsealObjectKey(GlobalKMS, metadata, bucket, object)
|
var err error
|
||||||
|
objectKey, err = crypto.S3KMS.UnsealObjectKey(GlobalKMS, metadata, bucket, object)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if targetKind == crypto.S3KMS && !replaceContext && len(cryptoCtx) == 0 {
|
||||||
if len(cryptoCtx) == 0 {
|
|
||||||
_, _, _, cryptoCtx, err = crypto.S3KMS.ParseMetadata(metadata)
|
_, _, _, cryptoCtx, err = crypto.S3KMS.ParseMetadata(metadata)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// If the context does not contain the bucket key
|
|
||||||
// we must add it for key generation. However,
|
|
||||||
// the context must be stored exactly like the
|
|
||||||
// client provided it. Therefore, we create a copy
|
|
||||||
// of the client provided context and add the bucket
|
|
||||||
// key, if not present.
|
|
||||||
kmsCtx := kms.Context{}
|
|
||||||
maps.Copy(kmsCtx, cryptoCtx)
|
|
||||||
if _, ok := kmsCtx[bucket]; !ok {
|
|
||||||
kmsCtx[bucket] = path.Join(bucket, object)
|
|
||||||
}
|
|
||||||
newKey, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{
|
|
||||||
Name: newKeyID,
|
|
||||||
AssociatedData: kmsCtx,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
sealedKey := objectKey.Seal(newKey.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3KMS.String(), bucket, object)
|
|
||||||
crypto.S3KMS.CreateMetadata(metadata, newKey.KeyID, newKey.Ciphertext, sealedKey, cryptoCtx)
|
|
||||||
return nil
|
|
||||||
case crypto.SSEC:
|
case crypto.SSEC:
|
||||||
sealedKey, err := crypto.SSEC.ParseMetadata(metadata)
|
sealedKey, err := crypto.SSEC.ParseMetadata(metadata)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
var objectKey crypto.ObjectKey
|
|
||||||
if err = objectKey.Unseal(oldKey, sealedKey, crypto.SSEC.String(), bucket, object); err != nil {
|
if err = objectKey.Unseal(oldKey, sealedKey, crypto.SSEC.String(), bucket, object); err != nil {
|
||||||
if subtle.ConstantTimeCompare(oldKey, newKey) == 1 {
|
if subtle.ConstantTimeCompare(oldKey, newKey) == 1 {
|
||||||
return errInvalidSSEParameters // AWS returns special error for equal but invalid keys.
|
return errInvalidSSEParameters // AWS returns special error for equal but invalid keys.
|
||||||
@@ -353,6 +326,55 @@ func rotateKey(ctx context.Context, oldKey []byte, newKeyID string, newKey []byt
|
|||||||
default:
|
default:
|
||||||
return errObjectTampered
|
return errObjectTampered
|
||||||
}
|
}
|
||||||
|
|
||||||
|
switch targetKind {
|
||||||
|
case crypto.S3:
|
||||||
|
genKey, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{
|
||||||
|
Name: GlobalKMS.DefaultKey,
|
||||||
|
AssociatedData: kms.Context{bucket: path.Join(bucket, object)},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
delete(metadata, crypto.MetaSealedKeyKMS)
|
||||||
|
delete(metadata, crypto.MetaKeyID)
|
||||||
|
delete(metadata, crypto.MetaDataEncryptionKey)
|
||||||
|
delete(metadata, crypto.MetaContext)
|
||||||
|
sealedKey := objectKey.Seal(genKey.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
|
||||||
|
crypto.S3.CreateMetadata(metadata, genKey.KeyID, genKey.Ciphertext, sealedKey)
|
||||||
|
return nil
|
||||||
|
case crypto.S3KMS:
|
||||||
|
// If the context does not contain the bucket key
|
||||||
|
// we must add it for key generation. However,
|
||||||
|
// the context must be stored exactly like the
|
||||||
|
// client provided it. Therefore, we create a copy
|
||||||
|
// of the client provided context and add the bucket
|
||||||
|
// key, if not present.
|
||||||
|
kmsCtx := kms.Context{}
|
||||||
|
maps.Copy(kmsCtx, cryptoCtx)
|
||||||
|
if _, ok := kmsCtx[bucket]; !ok {
|
||||||
|
kmsCtx[bucket] = path.Join(bucket, object)
|
||||||
|
}
|
||||||
|
genKey, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{
|
||||||
|
Name: newKeyID,
|
||||||
|
AssociatedData: kmsCtx,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, kes.ErrKeyNotFound) {
|
||||||
|
return errKMSKeyNotFound
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
delete(metadata, crypto.MetaSealedKeyS3)
|
||||||
|
delete(metadata, crypto.MetaKeyID)
|
||||||
|
delete(metadata, crypto.MetaDataEncryptionKey)
|
||||||
|
delete(metadata, crypto.MetaContext)
|
||||||
|
sealedKey := objectKey.Seal(genKey.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3KMS.String(), bucket, object)
|
||||||
|
crypto.S3KMS.CreateMetadata(metadata, genKey.KeyID, genKey.Ciphertext, sealedKey, cryptoCtx)
|
||||||
|
return nil
|
||||||
|
default:
|
||||||
|
return errInvalidEncryptionParameters
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func newEncryptMetadata(ctx context.Context, kind crypto.Type, keyID string, key []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context) (crypto.ObjectKey, error) {
|
func newEncryptMetadata(ctx context.Context, kind crypto.Type, keyID string, key []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context) (crypto.ObjectKey, error) {
|
||||||
|
|||||||
@@ -1722,6 +1722,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := er.checkReplicaContent(ctx, bucket, object, minioMetaMultipartBucket, uploadIDPath, partsMetadata, onlineDisks, opts); err != nil {
|
||||||
|
return oi, toObjectErr(err, bucket, object)
|
||||||
|
}
|
||||||
er.cleanupMultipartPath(ctx, paths...) // cleanup all part.N.meta, and skipped part.N's before final rename().
|
er.cleanupMultipartPath(ctx, paths...) // cleanup all part.N.meta, and skipped part.N's before final rename().
|
||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
|
|||||||
+60
-6
@@ -133,8 +133,25 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
|
|||||||
}
|
}
|
||||||
|
|
||||||
if dstOpts.ReplicaLockReconcile {
|
if dstOpts.ReplicaLockReconcile {
|
||||||
reconcileStoredObjectLock(srcInfo.UserDefined, storedObjectLockState(fi.Metadata))
|
stored := fi.ToObjectInfo(dstBucket, dstObject, dstOpts.Versioned || dstOpts.VersionSuspended)
|
||||||
reconcileStoredObjectTags(srcInfo.UserDefined, fi.Metadata[xhttp.AmzObjectTagging], fi.Metadata[ReservedMetadataPrefixLower+TaggingTimestamp])
|
if srcInfo.versionOnly || dstOpts.replicaObjectInfo != nil {
|
||||||
|
// fi describes the source of the new reference, not the version
|
||||||
|
// being replaced. A pooled lookup also preserves protection merged
|
||||||
|
// from another copy, even when this set holds a newer weak revision.
|
||||||
|
getInfo := er.getObjectInfo
|
||||||
|
if dstOpts.replicaObjectInfo != nil {
|
||||||
|
getInfo = dstOpts.replicaObjectInfo
|
||||||
|
}
|
||||||
|
stored, err = replicaStoredObjectInfo(ctx, func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
|
||||||
|
opts.NoLock = true
|
||||||
|
return getInfo(ctx, bucket, object, opts)
|
||||||
|
}, dstBucket, dstObject, dstOpts.VersionID)
|
||||||
|
if err != nil {
|
||||||
|
return oi, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
reconcileStoredObjectLock(srcInfo.UserDefined, storedObjectLockState(stored.UserDefined))
|
||||||
|
reconcileStoredObjectTags(srcInfo.UserDefined, stored.UserTags, stored.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp])
|
||||||
}
|
}
|
||||||
|
|
||||||
filterOnlineDisksInplace(fi, metaArr, onlineDisks)
|
filterOnlineDisksInplace(fi, metaArr, onlineDisks)
|
||||||
@@ -148,6 +165,17 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
|
|||||||
fi.IsLatest = true // we are creating a new version so this is latest.
|
fi.IsLatest = true // we are creating a new version so this is latest.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if srcInfo.versionOnly && (dstOpts.ReplicaLockReconcile || dstOpts.ReplicationRequest) && !dstOpts.DataMovement {
|
||||||
|
candidate := slices.Clone(metaArr)
|
||||||
|
for i := range candidate {
|
||||||
|
candidate[i].VersionID = versionID
|
||||||
|
candidate[i].Metadata = srcInfo.UserDefined
|
||||||
|
}
|
||||||
|
readDisks, readMeta := shuffleDisksAndPartsMetadataByIndex(onlineDisks, candidate, fi)
|
||||||
|
if err := er.checkReplicaContent(ctx, dstBucket, dstObject, srcBucket, srcObject, readMeta, readDisks, dstOpts); err != nil {
|
||||||
|
return oi, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if !dstOpts.ReplicationRequest && !dstOpts.ReplicaLockReconcile && !dstOpts.DataMovement &&
|
if !dstOpts.ReplicationRequest && !dstOpts.ReplicaLockReconcile && !dstOpts.DataMovement &&
|
||||||
(versionID == "" || versionID == nullVersionID) {
|
(versionID == "" || versionID == nullVersionID) {
|
||||||
@@ -935,11 +963,14 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
|
|||||||
// to the disks; this value is not accurate because we do
|
// to the disks; this value is not accurate because we do
|
||||||
// not know the storage class of the object yet
|
// not know the storage class of the object yet
|
||||||
minDisks := 0
|
minDisks := 0
|
||||||
if p := globalStorageClass.GetParityForSC(""); p > -1 {
|
if p := globalStorageClass.GetParityForSC(""); p > -1 && p < er.setDriveCount {
|
||||||
minDisks = er.setDriveCount - p
|
minDisks = er.setDriveCount - p
|
||||||
} else {
|
} else {
|
||||||
minDisks = er.setDriveCount - er.defaultParityCount
|
minDisks = er.setDriveCount - er.defaultParityCount
|
||||||
}
|
}
|
||||||
|
if minDisks < 1 {
|
||||||
|
minDisks = 1
|
||||||
|
}
|
||||||
|
|
||||||
if minDisks == er.setDriveCount/2 {
|
if minDisks == er.setDriveCount/2 {
|
||||||
// when data and parity are same we must atleast
|
// when data and parity are same we must atleast
|
||||||
@@ -1722,6 +1753,9 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Rename the successfully written temporary object to final location.
|
// Rename the successfully written temporary object to final location.
|
||||||
|
if err := er.checkReplicaContent(ctx, bucket, object, minioMetaTmpBucket, tempObj, partsMetadata, onlineDisks, opts); err != nil {
|
||||||
|
return ObjectInfo{}, toObjectErr(err, bucket, object)
|
||||||
|
}
|
||||||
onlineDisks, versions, oldDataDir, err := renameData(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, bucket, object, writeQuorum)
|
onlineDisks, versions, oldDataDir, err := renameData(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, bucket, object, writeQuorum)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, errFileNotFound) {
|
if errors.Is(err, errFileNotFound) {
|
||||||
@@ -2186,7 +2220,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
|
|||||||
// A restored-copy expiry needs fresh metadata to confirm its target
|
// A restored-copy expiry needs fresh metadata to confirm its target
|
||||||
// and deadline, a checked expiry to guard its protection; ordinary
|
// and deadline, a checked expiry to guard its protection; ordinary
|
||||||
// unversioned DELETE may still try a blind delete.
|
// unversioned DELETE may still try a blind delete.
|
||||||
if opts.Transition.ExpireRestored || checkedExpiry || opts.Versioned || opts.VersionSuspended || countOnlineDisks(storageDisks) < len(storageDisks)/2+1 {
|
if opts.Transition.ExpireRestored || checkedExpiry || opts.Versioned || opts.VersionSuspended || opts.RefuseLockedOverwrite || countOnlineDisks(storageDisks) < len(storageDisks)/2+1 {
|
||||||
// Add an MRF heal for next time.
|
// Add an MRF heal for next time.
|
||||||
er.addPartial(bucket, object, opts.VersionID)
|
er.addPartial(bucket, object, opts.VersionID)
|
||||||
return objInfo, InsufficientWriteQuorum{}
|
return objInfo, InsufficientWriteQuorum{}
|
||||||
@@ -2237,8 +2271,28 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// When versioning is disabled or suspended for this key, an unaddressed
|
||||||
|
// delete overwrites or removes the null version even if a newer non-null
|
||||||
|
// version is returned as latest by getObjectInfoAndQuorum above.
|
||||||
|
retOI, retErr := goi, gerr
|
||||||
|
if !opts.Versioned && !opts.DeleteMarker && opts.VersionID == "" && (opts.EvalRetentionBypassFn != nil || opts.RefuseLockedOverwrite) {
|
||||||
|
if gerr == nil && goi.VersionID != "" && goi.VersionID != nullVersionID {
|
||||||
|
nullOpts := opts
|
||||||
|
nullOpts.VersionID = nullVersionID
|
||||||
|
retOI, _, retErr = er.getObjectInfoAndQuorum(ctx, bucket, object, nullOpts)
|
||||||
|
if _, ok := retErr.(InsufficientReadQuorum); ok && opts.RefuseLockedOverwrite {
|
||||||
|
er.addPartial(bucket, object, nullVersionID)
|
||||||
|
return objInfo, InsufficientWriteQuorum{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if opts.EvalRetentionBypassFn != nil {
|
if opts.EvalRetentionBypassFn != nil {
|
||||||
if err := opts.EvalRetentionBypassFn(goi, gerr); err != nil {
|
if err := opts.EvalRetentionBypassFn(retOI, retErr); err != nil {
|
||||||
|
return ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
} else if opts.RefuseLockedOverwrite && !opts.Versioned && !opts.DeleteMarker && opts.VersionID == "" {
|
||||||
|
if err := refuseLockedMarkerOverwrite(ctx, bucket, object, nullVersionID, retOI, retErr); err != nil {
|
||||||
return ObjectInfo{}, err
|
return ObjectInfo{}, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2560,7 +2614,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
|
|||||||
|
|
||||||
fi.Metadata[xhttp.AmzObjectTagging] = tags
|
fi.Metadata[xhttp.AmzObjectTagging] = tags
|
||||||
fi.ReplicationState = opts.PutReplicationState()
|
fi.ReplicationState = opts.PutReplicationState()
|
||||||
stamp := monotonicTaggingTimestamp(opts.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp], fi.Metadata[ReservedMetadataPrefixLower+TaggingTimestamp])
|
stamp := monotonicMetadataTimestamp(opts.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp], fi.Metadata[ReservedMetadataPrefixLower+TaggingTimestamp])
|
||||||
maps.Copy(fi.Metadata, opts.UserDefined)
|
maps.Copy(fi.Metadata, opts.UserDefined)
|
||||||
if stamp != "" {
|
if stamp != "" {
|
||||||
fi.Metadata[ReservedMetadataPrefixLower+TaggingTimestamp] = stamp
|
fi.Metadata[ReservedMetadataPrefixLower+TaggingTimestamp] = stamp
|
||||||
|
|||||||
@@ -271,11 +271,33 @@ func (z *erasureServerPools) applyMigrationExpiry(ctx context.Context, bucket, o
|
|||||||
// in the source until the ordinary expiry path has deleted them: that path
|
// in the source until the ordinary expiry path has deleted them: that path
|
||||||
// owns replication state and tier cleanup, and it re-evaluates the current
|
// owns replication state and tier cleanup, and it re-evaluates the current
|
||||||
// lifecycle rule. A migration cleanup must never turn an asynchronous ILM
|
// lifecycle rule. A migration cleanup must never turn an asynchronous ILM
|
||||||
// decision into an unconditional physical purge.
|
// decision into an unconditional physical purge. It returns how many source
|
||||||
func (z *erasureServerPools) removeMovedVersions(ctx context.Context, bucket, object string, source int, versions []FileInfo, expired []migrationExpiredVersion) error {
|
// versions were removed, including versions deleted by ILM before cleanup.
|
||||||
|
func (z *erasureServerPools) removeMovedVersions(ctx context.Context, bucket, object string, source int, versions []FileInfo, expired []migrationExpiredVersion) (int, error) {
|
||||||
|
err := z.retireMovedVersions(ctx, bucket, object, source, versions, expired)
|
||||||
|
removed, countErr := z.countAbsentMigrationVersions(ctx, bucket, object, source, versions)
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return 0, ctx.Err()
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return removed, err
|
||||||
|
}
|
||||||
|
return removed, countErr
|
||||||
|
}
|
||||||
|
|
||||||
|
// retireMovedVersions does not infer physical progress from a pool-level read
|
||||||
|
// or delete result. Decommission verifies the entire walk snapshot once after
|
||||||
|
// this cleanup and tier GC, including versions concurrently deleted by clients.
|
||||||
|
func (z *erasureServerPools) retireMovedVersions(ctx context.Context, bucket, object string, source int, versions []FileInfo, expired []migrationExpiredVersion) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if err := z.applyMigrationExpiry(ctx, bucket, object, source, expired); err != nil {
|
if err := z.applyMigrationExpiry(ctx, bucket, object, source, expired); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if len(versions) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
object = encodeDirObject(object)
|
object = encodeDirObject(object)
|
||||||
lk := z.NewNSLock(bucket, object)
|
lk := z.NewNSLock(bucket, object)
|
||||||
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||||
@@ -289,6 +311,7 @@ func (z *erasureServerPools) removeMovedVersions(ctx context.Context, bucket, ob
|
|||||||
if vid == "" {
|
if vid == "" {
|
||||||
vid = nullVersionID
|
vid = nullVersionID
|
||||||
}
|
}
|
||||||
|
expiredVersion := isExpiredVersion(expired, version)
|
||||||
copies, err := z.objectPoolInfos(ctx, bucket, object, ObjectOptions{VersionID: vid, NoAuditLog: true})
|
copies, err := z.objectPoolInfos(ctx, bucket, object, ObjectOptions{VersionID: vid, NoAuditLog: true})
|
||||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||||
continue
|
continue
|
||||||
@@ -311,7 +334,7 @@ func (z *erasureServerPools) removeMovedVersions(ctx context.Context, bucket, ob
|
|||||||
if !src.ModTime.Equal(version.ModTime) || src.DeleteMarker != version.Deleted {
|
if !src.ModTime.Equal(version.ModTime) || src.DeleteMarker != version.Deleted {
|
||||||
return PreConditionFailed{}
|
return PreConditionFailed{}
|
||||||
}
|
}
|
||||||
if isExpiredVersion(expired, version) {
|
if expiredVersion {
|
||||||
// The logical expiry path owns this version. Keeping the source is
|
// The logical expiry path owns this version. Keeping the source is
|
||||||
// also the retry point if that path failed or the pool check races it.
|
// also the retry point if that path failed or the pool check races it.
|
||||||
continue
|
continue
|
||||||
@@ -370,19 +393,7 @@ func (z *erasureServerPools) healObjectInPool(ctx context.Context, er *erasureOb
|
|||||||
// mergePoolLockState resolves each independently ordered field. An ordered
|
// mergePoolLockState resolves each independently ordered field. An ordered
|
||||||
// removal is a value in its own right; an absent unordered field is not one.
|
// removal is a value in its own right; an absent unordered field is not one.
|
||||||
func mergePoolLockState(copies []PoolObjInfo) objectLockState {
|
func mergePoolLockState(copies []PoolObjInfo) objectLockState {
|
||||||
state := storedObjectLockState(copies[0].ObjInfo.UserDefined)
|
return reducePoolLockState(copies, storedObjectLockState)
|
||||||
for _, copy := range copies[1:] {
|
|
||||||
next := storedObjectLockState(copy.ObjInfo.UserDefined)
|
|
||||||
retentionTime, _ := time.Parse(time.RFC3339Nano, next.retentionTimestamp)
|
|
||||||
if state.retentionIsOlderThan(retentionTime) || (state.retentionTimestamp == "" && state.mode == "" && next.mode != "") {
|
|
||||||
state.mode, state.retainUntil, state.retentionTimestamp = next.mode, next.retainUntil, next.retentionTimestamp
|
|
||||||
}
|
|
||||||
holdTime, _ := time.Parse(time.RFC3339Nano, next.legalHoldTimestamp)
|
|
||||||
if state.legalHoldIsOlderThan(holdTime) || (state.legalHoldTimestamp == "" && state.legalHold == "" && next.legalHold != "") {
|
|
||||||
state.legalHold, state.legalHoldTimestamp = next.legalHold, next.legalHoldTimestamp
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return state
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func replaceObjectLockMetadata(metadata map[string]string, state objectLockState) {
|
func replaceObjectLockMetadata(metadata map[string]string, state objectLockState) {
|
||||||
@@ -534,10 +545,10 @@ func reconcileStoredObjectTags(metadata map[string]string, storedTags, storedTim
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Local tagging mutations must advance the revision they overwrite, even when
|
// Local metadata mutations must advance the revision they overwrite, even when
|
||||||
// a request's clock or lock acquisition order is behind the stored revision.
|
// a request's clock or lock acquisition order is behind the stored revision.
|
||||||
// Replica writes use reconcileStoredObjectTags instead of minting a revision.
|
// Replica writes use reconcileStoredObjectTags instead of minting a revision.
|
||||||
func monotonicTaggingTimestamp(incoming, stored string) string {
|
func monotonicMetadataTimestamp(incoming, stored string) string {
|
||||||
requested, err := time.Parse(time.RFC3339Nano, incoming)
|
requested, err := time.Parse(time.RFC3339Nano, incoming)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return incoming
|
return incoming
|
||||||
@@ -635,6 +646,26 @@ func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket,
|
|||||||
case opts.VersionID != "":
|
case opts.VersionID != "":
|
||||||
// An addressed version already resolved every copy above.
|
// An addressed version already resolved every copy above.
|
||||||
logical = mergedPoolObjectInfo(copies)
|
logical = mergedPoolObjectInfo(copies)
|
||||||
|
case !opts.Versioned && !opts.DeleteMarker:
|
||||||
|
// An unqualified delete when versioning is not active for this key
|
||||||
|
// removes or replaces the null version in every pool that holds one,
|
||||||
|
// even if a newer non-null version is current for reads.
|
||||||
|
nullOpts := opts
|
||||||
|
nullOpts.VersionID = nullVersionID
|
||||||
|
nullCopies, err := z.objectPoolInfos(ctx, bucket, object, nullOpts)
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
if nullCopies[0].ObjInfo.DeleteMarker {
|
||||||
|
gerr = toObjectErr(errFileNotFound, bucket, object)
|
||||||
|
logical = nullCopies[0].ObjInfo
|
||||||
|
} else {
|
||||||
|
logical = mergedPoolObjectInfo(nullCopies)
|
||||||
|
}
|
||||||
|
case isErrObjectNotFound(err) || isErrVersionNotFound(err):
|
||||||
|
gerr = toObjectErr(errFileNotFound, bucket, object)
|
||||||
|
default:
|
||||||
|
return ObjectInfo{}, err
|
||||||
|
}
|
||||||
default:
|
default:
|
||||||
versions, err := z.metadataPoolInfos(ctx, bucket, object, opts)
|
versions, err := z.metadataPoolInfos(ctx, bucket, object, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -650,6 +681,14 @@ func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket,
|
|||||||
return ObjectInfo{}, err
|
return ObjectInfo{}, err
|
||||||
}
|
}
|
||||||
opts.EvalRetentionBypassFn = nil
|
opts.EvalRetentionBypassFn = nil
|
||||||
|
if !opts.Versioned && !opts.DeleteMarker && opts.VersionID == "" {
|
||||||
|
opts.RefuseLockedOverwrite = false
|
||||||
|
}
|
||||||
|
} else if opts.RefuseLockedOverwrite && !opts.Versioned && !opts.DeleteMarker && opts.VersionID == "" {
|
||||||
|
if err := refuseLockedMarkerOverwrite(ctx, bucket, object, nullVersionID, logical, gerr); err != nil {
|
||||||
|
return ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
opts.RefuseLockedOverwrite = false
|
||||||
}
|
}
|
||||||
if opts.EvalMetadataFn != nil {
|
if opts.EvalMetadataFn != nil {
|
||||||
decision, err := opts.EvalMetadataFn(&logical, gerr)
|
decision, err := opts.EvalMetadataFn(&logical, gerr)
|
||||||
|
|||||||
@@ -0,0 +1,102 @@
|
|||||||
|
// Copyright (c) 2026 Denis Babchuk
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A key below another key ("k/b" under "k") is a directory inside the
|
||||||
|
// directory of that key. The listing treats the directory of an object as a
|
||||||
|
// leaf, so a decommission walk does not see "k/b" while "k" is still in the
|
||||||
|
// pool. The walk has to be repeated once the keys above are gone, or the
|
||||||
|
// check after the bucket finds them and the decommission fails.
|
||||||
|
|
||||||
|
func keysBelowDecommission(t *testing.T, z *erasureServerPools) PoolDecommissionInfo {
|
||||||
|
t.Helper()
|
||||||
|
if err := z.StartDecommission(t.Context(), 0); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
z.doDecommissionInRoutine(t.Context(), 0)
|
||||||
|
z.poolMetaMutex.RLock()
|
||||||
|
defer z.poolMetaMutex.RUnlock()
|
||||||
|
return *z.poolMeta.Pools[0].Decommission
|
||||||
|
}
|
||||||
|
|
||||||
|
func keysBelowPools(t *testing.T) (*erasureServerPools, string, []string) {
|
||||||
|
t.Helper()
|
||||||
|
z, _ := migrationTestPools(t)
|
||||||
|
const bucket = "keys-below"
|
||||||
|
if err := z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
keys := []string{"k", "k/b", "k/b/c"}
|
||||||
|
for _, key := range keys {
|
||||||
|
putConsistencyObject(t, z, bucket, key, 0, key, ObjectOptions{Versioned: true})
|
||||||
|
}
|
||||||
|
return z, bucket, keys
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecommissionMovesKeysBelowAKey(t *testing.T) {
|
||||||
|
z, bucket, keys := keysBelowPools(t)
|
||||||
|
if info := keysBelowDecommission(t, z); !info.Complete || info.Failed {
|
||||||
|
t.Errorf("decommission: complete %v, failed %v; want complete", info.Complete, info.Failed)
|
||||||
|
}
|
||||||
|
for _, key := range keys {
|
||||||
|
if _, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, key, ObjectOptions{}); !isErrObjectNotFound(err) {
|
||||||
|
t.Errorf("%s still in pool 0: %v", key, err)
|
||||||
|
}
|
||||||
|
if body := migrationTestBody(t, z.GetObjectNInfo, bucket, key, ""); body != key {
|
||||||
|
t.Errorf("%s reads %q", key, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An ILM expiry can remove the parent key during a decommission walk. That
|
||||||
|
// removal must count as progress, or the next key below the parent remains
|
||||||
|
// hidden from the listing and the source pool is left non-empty.
|
||||||
|
func TestDecommissionContinuesAfterLifecycleRemovesParent(t *testing.T) {
|
||||||
|
z, _ := migrationTestPools(t)
|
||||||
|
const bucket = "decommission-ilm-keys-below"
|
||||||
|
if err := z.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
meta, err := globalBucketMetadataSys.Get(bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
meta.LifecycleConfigXML = []byte(`<LifecycleConfiguration><Rule><ID>expire-old</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter><Expiration><Days>1</Days></Expiration></Rule></LifecycleConfiguration>`)
|
||||||
|
if err := meta.Save(t.Context(), z); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
globalBucketMetadataSys.Set(bucket, meta)
|
||||||
|
|
||||||
|
putConsistencyObject(t, z, bucket, "k", 0, "parent", ObjectOptions{MTime: UTCNow().Add(-72 * time.Hour)})
|
||||||
|
putConsistencyObject(t, z, bucket, "k/b", 0, "child", ObjectOptions{MTime: UTCNow()})
|
||||||
|
|
||||||
|
if info := keysBelowDecommission(t, z); !info.Complete || info.Failed {
|
||||||
|
t.Fatalf("decommission: complete %v, failed %v; want complete", info.Complete, info.Failed)
|
||||||
|
}
|
||||||
|
if _, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, "k/b", ObjectOptions{}); !isErrObjectNotFound(err) {
|
||||||
|
t.Fatalf("child still in source pool: %v", err)
|
||||||
|
}
|
||||||
|
if body := migrationTestBody(t, z.GetObjectNInfo, bucket, "k/b", ""); body != "child" {
|
||||||
|
t.Fatalf("child reads %q", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -28,6 +28,8 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
@@ -752,7 +754,34 @@ func (set *erasureObjects) listObjectsToDecommission(ctx context.Context, bi dec
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// decommissionPool moves bucket bi out of pool idx. A key below another key
|
||||||
|
// ("k/b" under "k") is a directory inside the directory of that key, and the
|
||||||
|
// listing does not descend into the directory of an object: a walk sees "k/b"
|
||||||
|
// only once "k" has left the pool. The walk is repeated while it removes
|
||||||
|
// source versions, so each pass moves the keys that the previous one
|
||||||
|
// uncovered. Each repeat needs a version removed from the pool, which takes
|
||||||
|
// no new writes, so the repeats end.
|
||||||
func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool *erasureSets, bi decomBucketInfo) error {
|
func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool *erasureSets, bi decomBucketInfo) error {
|
||||||
|
for {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
removed, err := z.decommissionPoolWalk(ctx, idx, pool, bi)
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
if err != nil || removed == 0 {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// decommissionPoolWalk walks bucket bi of pool idx once and returns the number
|
||||||
|
// of snapshot versions verified absent from every online source drive afterwards.
|
||||||
|
func (z *erasureServerPools) decommissionPoolWalk(ctx context.Context, idx int, pool *erasureSets, bi decomBucketInfo) (int64, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
ctx = logger.SetReqInfo(ctx, &logger.ReqInfo{})
|
ctx = logger.SetReqInfo(ctx, &logger.ReqInfo{})
|
||||||
|
|
||||||
const envDecomWorkers = "_MINIO_DECOMMISSION_WORKERS"
|
const envDecomWorkers = "_MINIO_DECOMMISSION_WORKERS"
|
||||||
@@ -766,9 +795,13 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
|
|||||||
// add that many extra workers.
|
// add that many extra workers.
|
||||||
workerSize += len(pool.sets)
|
workerSize += len(pool.sets)
|
||||||
|
|
||||||
|
var removed atomic.Int64
|
||||||
|
var verifyErr error
|
||||||
|
var verifyOnce sync.Once
|
||||||
|
|
||||||
wk, err := workers.New(workerSize)
|
wk, err := workers.New(workerSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var vc *versioning.Versioning
|
var vc *versioning.Versioning
|
||||||
@@ -779,24 +812,24 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
|
|||||||
if bi.Name != minioMetaBucket {
|
if bi.Name != minioMetaBucket {
|
||||||
vc, err = globalBucketVersioningSys.Get(bi.Name)
|
vc, err = globalBucketVersioningSys.Get(bi.Name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if the current bucket has a configured lifecycle policy
|
// Check if the current bucket has a configured lifecycle policy
|
||||||
lc, lifecycleUpdatedAt, err = globalBucketMetadataSys.GetLifecycleConfig(bi.Name)
|
lc, lifecycleUpdatedAt, err = globalBucketMetadataSys.GetLifecycleConfig(bi.Name)
|
||||||
if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bi.Name}) {
|
if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bi.Name}) {
|
||||||
return err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if bucket is object locked.
|
// Check if bucket is object locked.
|
||||||
lr, err = globalBucketObjectLockSys.Get(bi.Name)
|
lr, err = globalBucketObjectLockSys.Get(bi.Name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|
||||||
rcfg, err = getReplicationConfig(ctx, bi.Name)
|
rcfg, err = getReplicationConfig(ctx, bi.Name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return 0, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -855,7 +888,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
|
|||||||
// to create the appropriate stack.
|
// to create the appropriate stack.
|
||||||
versionsSorter(fivs.Versions).reverse()
|
versionsSorter(fivs.Versions).reverse()
|
||||||
|
|
||||||
var decommissioned int
|
var candidates []FileInfo
|
||||||
var expired []migrationExpiredVersion
|
var expired []migrationExpiredVersion
|
||||||
// The key's history in all pools, read once for its versions.
|
// The key's history in all pools, read once for its versions.
|
||||||
var history migrationHistory
|
var history migrationHistory
|
||||||
@@ -863,8 +896,8 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
|
|||||||
stopFn := globalDecommissionMetrics.log(decomMetricDecommissionObject, idx, bi.Name, version.Name, version.VersionID)
|
stopFn := globalDecommissionMetrics.log(decomMetricDecommissionObject, idx, bi.Name, version.Name, version.VersionID)
|
||||||
// Apply lifecycle rules on the objects that are expired.
|
// Apply lifecycle rules on the objects that are expired.
|
||||||
if action := filterLifecycle(bi.Name, version.Name, version, &history); action.Delete() {
|
if action := filterLifecycle(bi.Name, version.Name, version, &history); action.Delete() {
|
||||||
decommissioned++
|
|
||||||
if ilmDeletesVersionOnly(action) || (action.DeleteAll() && !version.IsRemote()) {
|
if ilmDeletesVersionOnly(action) || (action.DeleteAll() && !version.IsRemote()) {
|
||||||
|
candidates = append(candidates, version)
|
||||||
// Run the logical expiry before retiring the source.
|
// Run the logical expiry before retiring the source.
|
||||||
// A full-object expiry is safe here only for hot-tier
|
// A full-object expiry is safe here only for hot-tier
|
||||||
// metadata; transitioned objects retain their tier cleanup
|
// metadata; transitioned objects retain their tier cleanup
|
||||||
@@ -912,7 +945,6 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
|
|||||||
// This can happen when rebalance stop races with ongoing rebalance workers.
|
// This can happen when rebalance stop races with ongoing rebalance workers.
|
||||||
// These rebalance failures can be ignored.
|
// These rebalance failures can be ignored.
|
||||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
|
if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
|
||||||
ignore = true
|
|
||||||
stopFn(0, nil)
|
stopFn(0, nil)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -926,8 +958,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
|
|||||||
z.poolMeta.CountItem(idx, 0, failure)
|
z.poolMeta.CountItem(idx, 0, failure)
|
||||||
z.poolMetaMutex.Unlock()
|
z.poolMetaMutex.Unlock()
|
||||||
if !failure {
|
if !failure {
|
||||||
// Success keep a count.
|
candidates = append(candidates, version)
|
||||||
decommissioned++
|
|
||||||
}
|
}
|
||||||
auditLogDecom(ctx, "DecomCopyDeleteMarker", bi.Name, version.Name, versionID, err)
|
auditLogDecom(ctx, "DecomCopyDeleteMarker", bi.Name, version.Name, versionID, err)
|
||||||
continue
|
continue
|
||||||
@@ -1013,16 +1044,32 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
|
|||||||
if failure {
|
if failure {
|
||||||
break // break out on first error
|
break // break out on first error
|
||||||
}
|
}
|
||||||
decommissioned++
|
candidates = append(candidates, version)
|
||||||
}
|
}
|
||||||
|
|
||||||
// if all versions were decommissioned, then we can delete the object versions.
|
// Retire only successful moves and eligible expiries. A concurrently
|
||||||
if decommissioned == len(fivs.Versions) {
|
// deleted or failed sibling must not strand the successful copies.
|
||||||
|
// Skipped versions are checked for absence, never purged here.
|
||||||
|
{
|
||||||
stopFn := globalDecommissionMetrics.log(decomMetricDecommissionRemoveObject, idx, bi.Name, entry.name)
|
stopFn := globalDecommissionMetrics.log(decomMetricDecommissionRemoveObject, idx, bi.Name, entry.name)
|
||||||
err := z.removeMovedVersions(ctx, bi.Name, entry.name, idx, fivs.Versions, expired)
|
err := z.retireMovedVersions(ctx, bi.Name, entry.name, idx, candidates, expired)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
err = z.removeFreeVersions(ctx, bi.Name, entry.name, idx, fivs.FreeVersions)
|
err = z.removeFreeVersions(ctx, bi.Name, entry.name, idx, fivs.FreeVersions)
|
||||||
}
|
}
|
||||||
|
// One metadata-only read of each source drive per entry, regardless
|
||||||
|
// of its version count. No read error can become progress or trigger
|
||||||
|
// another walk; leave the bucket queued for an explicit retry.
|
||||||
|
snapshot := slices.Concat(fivs.Versions, fivs.FreeVersions)
|
||||||
|
n, countErr := z.countAbsentMigrationVersions(ctx, bi.Name, entry.name, idx, snapshot)
|
||||||
|
removed.Add(int64(n))
|
||||||
|
// The disposable usage cache is also excluded by the final
|
||||||
|
// verification. Its unreadability still never counts as progress.
|
||||||
|
if countErr != nil && (bi.Name != minioMetaBucket || !strings.Contains(entry.name, dataUsageCacheName)) {
|
||||||
|
verifyOnce.Do(func() { verifyErr = countErr })
|
||||||
|
if err == nil {
|
||||||
|
err = countErr
|
||||||
|
}
|
||||||
|
}
|
||||||
stopFn(0, err)
|
stopFn(0, err)
|
||||||
auditLogDecom(ctx, "DecomDeleteObject", bi.Name, entry.name, "", err)
|
auditLogDecom(ctx, "DecomDeleteObject", bi.Name, entry.name, "", err)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1057,12 +1104,19 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
|
|||||||
setN := humanize.Ordinal(setIdx + 1)
|
setN := humanize.Ordinal(setIdx + 1)
|
||||||
retryDur := time.Duration(rand.Float64() * float64(5*time.Second))
|
retryDur := time.Duration(rand.Float64() * float64(5*time.Second))
|
||||||
decomLogOnceIf(ctx, fmt.Errorf("listing objects from %s set failed with %v, retrying in %v", setN, err, retryDur), "decom-listing-failed"+setN)
|
decomLogOnceIf(ctx, fmt.Errorf("listing objects from %s set failed with %v, retrying in %v", setN, err, retryDur), "decom-listing-failed"+setN)
|
||||||
time.Sleep(retryDur)
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(retryDur):
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}(setIdx)
|
}(setIdx)
|
||||||
}
|
}
|
||||||
wk.Wait()
|
wk.Wait()
|
||||||
return nil
|
if err := ctx.Err(); err != nil {
|
||||||
|
return removed.Load(), err
|
||||||
|
}
|
||||||
|
return removed.Load(), verifyErr
|
||||||
}
|
}
|
||||||
|
|
||||||
//msgp:ignore decomMetrics
|
//msgp:ignore decomMetrics
|
||||||
@@ -1113,6 +1167,9 @@ func (m *decomMetrics) log(d decomMetric, poolIdx int, paths ...string) func(z i
|
|||||||
// decommission fails and a retry resumes with what is left. A canceled
|
// decommission fails and a retry resumes with what is left. A canceled
|
||||||
// context returns at once.
|
// context returns at once.
|
||||||
func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx int) error {
|
func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx int) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
pool := z.serverPools[idx]
|
pool := z.serverPools[idx]
|
||||||
z.poolMetaMutex.RLock()
|
z.poolMetaMutex.RLock()
|
||||||
pending := z.poolMeta.PendingBuckets(idx)
|
pending := z.poolMeta.PendingBuckets(idx)
|
||||||
@@ -1151,7 +1208,7 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
|
|||||||
if err := z.decommissionPool(ctx, idx, pool, bucket); err != nil {
|
if err := z.decommissionPool(ctx, idx, pool, bucket); err != nil {
|
||||||
stopFn(0, err)
|
stopFn(0, err)
|
||||||
if contextCanceled(ctx) {
|
if contextCanceled(ctx) {
|
||||||
return err
|
return ctx.Err()
|
||||||
}
|
}
|
||||||
failed(bucket, err)
|
failed(bucket, err)
|
||||||
continue
|
continue
|
||||||
@@ -1164,7 +1221,7 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
|
|||||||
if err := z.checkAfterDecomBucket(ctx, idx, bucket); err != nil {
|
if err := z.checkAfterDecomBucket(ctx, idx, bucket); err != nil {
|
||||||
stopFn(0, err)
|
stopFn(0, err)
|
||||||
if contextCanceled(ctx) {
|
if contextCanceled(ctx) {
|
||||||
return err
|
return ctx.Err()
|
||||||
}
|
}
|
||||||
failed(bucket, err)
|
failed(bucket, err)
|
||||||
continue
|
continue
|
||||||
@@ -1177,6 +1234,9 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
|
|||||||
}
|
}
|
||||||
z.poolMetaMutex.Unlock()
|
z.poolMetaMutex.Unlock()
|
||||||
}
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
return firstErr
|
return firstErr
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1211,34 +1271,74 @@ func (z *erasureServerPools) removeFreeVersions(ctx context.Context, bucket, obj
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkAfterDecomBucket verifies that a source bucket is physically empty
|
// checkAfterDecomBucket verifies that a source bucket's online drives are empty
|
||||||
// before it is removed from the resumable decommission queue. ILM decisions
|
// before it is removed from the resumable decommission queue. ILM decisions
|
||||||
// are deliberately not treated as an exemption: an expired version remains
|
// are deliberately not treated as an exemption: an expired version remains
|
||||||
// until the expiry path has performed its logical delete, replication, and
|
// until the expiry path has performed its logical delete, replication, and
|
||||||
// tier cleanup.
|
// tier cleanup.
|
||||||
func (z *erasureServerPools) checkAfterDecomBucket(ctx context.Context, idx int, bi decomBucketInfo) error {
|
func (z *erasureServerPools) checkAfterDecomBucket(ctx context.Context, idx int, bi decomBucketInfo) error {
|
||||||
versionsFound := 0
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
for _, set := range z.serverPools[idx].sets {
|
for _, set := range z.serverPools[idx].sets {
|
||||||
if err := set.listObjectsToDecommission(ctx, bi, func(entry metaCacheEntry) {
|
disks := set.getDisks()
|
||||||
if !entry.isObject() {
|
if len(disks) == 0 {
|
||||||
return
|
return toObjectErr(errErasureReadQuorum, bi.Name)
|
||||||
|
}
|
||||||
|
readQuorum := set.migrationReadQuorum(len(disks))
|
||||||
|
onlineCount := 0
|
||||||
|
for _, disk := range disks {
|
||||||
|
if disk == nil {
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
if bi.Name == minioMetaBucket && strings.Contains(entry.name, dataUsageCacheName) {
|
// Quorum listings omit minority versions, and WalkDir can skip
|
||||||
return
|
// unreadable metadata. Inspect physical directory entries instead:
|
||||||
|
// even a corrupt or truncated xl.meta must keep this pool queued.
|
||||||
|
pending := []string{bi.Prefix}
|
||||||
|
diskOffline := false
|
||||||
|
for len(pending) > 0 {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
dir := pending[len(pending)-1]
|
||||||
|
pending = pending[:len(pending)-1]
|
||||||
|
entries, err := disk.ListDir(ctx, "", bi.Name, dir, -1)
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) {
|
||||||
|
continue // A concurrent deletion can remove the directory.
|
||||||
|
}
|
||||||
|
if errors.Is(err, errDiskNotFound) {
|
||||||
|
diskOffline = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, entry := range entries {
|
||||||
|
name := pathJoin(dir, entry)
|
||||||
|
if bi.Name == minioMetaBucket && strings.Contains(name, dataUsageCacheName) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if entry == xlStorageFormatFile || entry == xlStorageFormatFileV1 {
|
||||||
|
return fmt.Errorf("object metadata %s remains in bucket %s after decommissioning", name, bi.Name)
|
||||||
|
}
|
||||||
|
if strings.HasSuffix(entry, SlashSeparator) {
|
||||||
|
pending = append(pending, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
fivs, err := entry.fileInfoVersions(bi.Name)
|
if diskOffline {
|
||||||
if err != nil {
|
continue
|
||||||
return
|
|
||||||
}
|
}
|
||||||
versionsFound += len(fivs.Versions) + len(fivs.FreeVersions)
|
onlineCount++
|
||||||
}); err != nil {
|
}
|
||||||
return err
|
if onlineCount < readQuorum {
|
||||||
|
return toObjectErr(errErasureReadQuorum, bi.Name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if versionsFound > 0 {
|
return ctx.Err()
|
||||||
return fmt.Errorf("at least %d object(s)/version(s) were found in bucket `%s` after decommissioning", versionsFound, bi.Name)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error {
|
func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error {
|
||||||
|
|||||||
@@ -696,8 +696,10 @@ func TestPoolsLifecycleHeaderEras(t *testing.T) {
|
|||||||
if versionRevision(old(0), xlMetaVersion) != versionRevision(current(7), xlMetaVersion) {
|
if versionRevision(old(0), xlMetaVersion) != versionRevision(current(7), xlMetaVersion) {
|
||||||
t.Error("one version under the headers of two releases has two keys")
|
t.Error("one version under the headers of two releases has two keys")
|
||||||
}
|
}
|
||||||
// The header of the key orders the merge: it is the current one.
|
// The normalized current header orders the merge.
|
||||||
if versionRevision(old(0), xlMetaVersion).header != current(0).header {
|
wantHeader := current(0).header
|
||||||
|
wantHeader.Flags &^= xlFlagUsesDataDir
|
||||||
|
if versionRevision(old(0), xlMetaVersion).header != wantHeader {
|
||||||
t.Error("the key does not carry the header of the current release")
|
t.Error("the key does not carry the header of the current release")
|
||||||
}
|
}
|
||||||
for healed := range 17 {
|
for healed := range 17 {
|
||||||
@@ -753,8 +755,10 @@ func TestPoolsLifecycleHealedOldHeaders(t *testing.T) {
|
|||||||
rewriteTierTestMeta(t, disk, s.bucket, key, func(xl *xlMetaV2) {
|
rewriteTierTestMeta(t, disk, s.bucket, key, func(xl *xlMetaV2) {
|
||||||
for i := range xl.versions {
|
for i := range xl.versions {
|
||||||
// Control: the key of a version this release wrote
|
// Control: the key of a version this release wrote
|
||||||
// carries its header as stored.
|
// carries its header with only the time-derived flag removed.
|
||||||
if versionRevision(xl.versions[i], xl.metaV).header != xl.versions[i].header {
|
wantHeader := xl.versions[i].header
|
||||||
|
wantHeader.Flags &^= xlFlagUsesDataDir
|
||||||
|
if versionRevision(xl.versions[i], xl.metaV).header != wantHeader {
|
||||||
t.Fatal("the key of a current version carries another header")
|
t.Fatal("the key of a current version carries another header")
|
||||||
}
|
}
|
||||||
xl.versions[i].header.EcN, xl.versions[i].header.EcM = 0, 0
|
xl.versions[i].header.EcN, xl.versions[i].header.EcM = 0, 0
|
||||||
@@ -1260,6 +1264,7 @@ func TestPoolsLifecycleCheckedExpirySafeguard(t *testing.T) {
|
|||||||
{"hold-equal-stamps", poolLockMetadata("", "ON", old, recent), objectLockState{legalHold: "OFF", legalHoldTimestamp: recent}, false},
|
{"hold-equal-stamps", poolLockMetadata("", "ON", old, recent), objectLockState{legalHold: "OFF", legalHoldTimestamp: recent}, false},
|
||||||
{"hold-missing-approved-stamp", poolLockMetadata("", "ON", old, old), objectLockState{legalHold: "OFF"}, false},
|
{"hold-missing-approved-stamp", poolLockMetadata("", "ON", old, old), objectLockState{legalHold: "OFF"}, false},
|
||||||
{"retention-newer-locally", poolLockMetadata("GOVERNANCE", "OFF", recent, old), objectLockState{retentionTimestamp: old, legalHold: "OFF", legalHoldTimestamp: old}, false},
|
{"retention-newer-locally", poolLockMetadata("GOVERNANCE", "OFF", recent, old), objectLockState{retentionTimestamp: old, legalHold: "OFF", legalHoldTimestamp: old}, false},
|
||||||
|
{"compliance-cannot-release", poolLockMetadata("COMPLIANCE", "OFF", old, old), objectLockState{retentionTimestamp: recent, legalHold: "OFF", legalHoldTimestamp: old}, false},
|
||||||
// Controls: the approval strictly supersedes the local protection.
|
// Controls: the approval strictly supersedes the local protection.
|
||||||
{"hold-released", poolLockMetadata("", "ON", old, old), objectLockState{legalHold: "OFF", legalHoldTimestamp: recent}, true},
|
{"hold-released", poolLockMetadata("", "ON", old, old), objectLockState{legalHold: "OFF", legalHoldTimestamp: recent}, true},
|
||||||
{"retention-released", poolLockMetadata("GOVERNANCE", "OFF", old, old), objectLockState{retentionTimestamp: recent, legalHold: "OFF", legalHoldTimestamp: old}, true},
|
{"retention-released", poolLockMetadata("GOVERNANCE", "OFF", old, old), objectLockState{retentionTimestamp: recent, legalHold: "OFF", legalHoldTimestamp: old}, true},
|
||||||
@@ -1666,7 +1671,7 @@ func TestPoolsMigrationLifecycleRefusedExpiryKeepsSource(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
version := FileInfo{Name: key, VersionID: id, ModTime: source.ModTime}
|
version := FileInfo{Name: key, VersionID: id, ModTime: source.ModTime}
|
||||||
err = s.z.removeMovedVersions(ctx, s.bucket, key, 0, []FileInfo{version}, []migrationExpiredVersion{{
|
_, err = s.z.removeMovedVersions(ctx, s.bucket, key, 0, []FileInfo{version}, []migrationExpiredVersion{{
|
||||||
version: version,
|
version: version,
|
||||||
event: lifecycle.Event{Action: lifecycle.DeleteAction},
|
event: lifecycle.Event{Action: lifecycle.DeleteAction},
|
||||||
lifecycleUpdatedAt: updatedAt,
|
lifecycleUpdatedAt: updatedAt,
|
||||||
|
|||||||
@@ -70,6 +70,8 @@ func TestPoolsLifecycleDuplicateLockMetadata(t *testing.T) {
|
|||||||
{name: "independent-fields", copies: [2]map[string]string{poolLockMetadata("GOVERNANCE", "OFF", recent, old), poolLockMetadata("", "ON", old, recent)}, mode: "GOVERNANCE", hold: "ON"},
|
{name: "independent-fields", copies: [2]map[string]string{poolLockMetadata("GOVERNANCE", "OFF", recent, old), poolLockMetadata("", "ON", old, recent)}, mode: "GOVERNANCE", hold: "ON"},
|
||||||
{name: "hold-released", copies: [2]map[string]string{poolLockMetadata("", "ON", old, old), poolLockMetadata("", "OFF", old, recent)}, hold: "OFF", delete: true},
|
{name: "hold-released", copies: [2]map[string]string{poolLockMetadata("", "ON", old, old), poolLockMetadata("", "OFF", old, recent)}, hold: "OFF", delete: true},
|
||||||
{name: "retention-released", copies: [2]map[string]string{poolLockMetadata("GOVERNANCE", "OFF", old, old), poolLockMetadata("", "OFF", recent, old)}, hold: "OFF", delete: true},
|
{name: "retention-released", copies: [2]map[string]string{poolLockMetadata("GOVERNANCE", "OFF", old, old), poolLockMetadata("", "OFF", recent, old)}, hold: "OFF", delete: true},
|
||||||
|
{name: "compliance-before-removal", copies: [2]map[string]string{poolLockMetadata("COMPLIANCE", "OFF", old, old), poolLockMetadata("", "OFF", recent, old)}, mode: "COMPLIANCE", hold: "OFF"},
|
||||||
|
{name: "compliance-after-removal", copies: [2]map[string]string{poolLockMetadata("", "OFF", recent, old), poolLockMetadata("COMPLIANCE", "OFF", old, old)}, mode: "COMPLIANCE", hold: "OFF"},
|
||||||
} {
|
} {
|
||||||
for _, batch := range []bool{false, true} {
|
for _, batch := range []bool{false, true} {
|
||||||
path := "single"
|
path := "single"
|
||||||
|
|||||||
@@ -195,8 +195,9 @@ type revisionKey struct {
|
|||||||
// empty part ETags until RELEASE.2022-01-25. Reads and heal take such drives
|
// empty part ETags until RELEASE.2022-01-25. Reads and heal take such drives
|
||||||
// for one version (matchesNotStrict), and nothing rewrites them. The key has
|
// for one version (matchesNotStrict), and nothing rewrites them. The key has
|
||||||
// both as the current release computes them from the metadata, so that the
|
// both as the current release computes them from the metadata, so that the
|
||||||
// header of a version it wrote stays as stored: the signature also orders
|
// header of a version it wrote stays as stored, except UsesDataDir, which can
|
||||||
// versions of one time in the merge. The signature of a legacy version is
|
// change solely because a restored copy expired before healing. The signature
|
||||||
|
// also orders versions of one time in the merge. The signature of a legacy version is
|
||||||
// over its per-shard checksums; the key has one from the digest.
|
// over its per-shard checksums; the key has one from the digest.
|
||||||
func versionRevision(ver xlMetaV2ShallowVersion, metaV uint8) revisionKey {
|
func versionRevision(ver xlMetaV2ShallowVersion, metaV uint8) revisionKey {
|
||||||
h := sha256.New()
|
h := sha256.New()
|
||||||
@@ -281,6 +282,7 @@ func versionRevision(ver xlMetaV2ShallowVersion, metaV uint8) revisionKey {
|
|||||||
case ObjectType:
|
case ObjectType:
|
||||||
key.header.Signature = v.ObjectV2.Signature()
|
key.header.Signature = v.ObjectV2.Signature()
|
||||||
key.header.EcN, key.header.EcM = uint8(v.ObjectV2.ErasureN), uint8(v.ObjectV2.ErasureM)
|
key.header.EcN, key.header.EcM = uint8(v.ObjectV2.ErasureN), uint8(v.ObjectV2.ErasureM)
|
||||||
|
key.header.Flags &^= xlFlagUsesDataDir
|
||||||
}
|
}
|
||||||
return key
|
return key
|
||||||
}
|
}
|
||||||
@@ -587,20 +589,10 @@ func setLifecycleLockState(metadata map[string]string, state objectLockState) {
|
|||||||
set(ReservedMetadataPrefix+ObjectLockLegalHoldTimestamp, state.legalHoldTimestamp)
|
set(ReservedMetadataPrefix+ObjectLockLegalHoldTimestamp, state.legalHoldTimestamp)
|
||||||
}
|
}
|
||||||
|
|
||||||
// newestLockState returns the newest hold and the newest retention of the
|
// newestLockState orders independently revised fields without weakening a
|
||||||
// copies of a version, which lockRevisionsOrdered found equal or ordered.
|
// COMPLIANCE retention that any copy still protects.
|
||||||
func newestLockState(copies []PoolObjInfo) objectLockState {
|
func newestLockState(copies []PoolObjInfo) objectLockState {
|
||||||
state := lifecycleLockState(copies[0].ObjInfo.UserDefined)
|
return reducePoolLockState(copies, lifecycleLockState)
|
||||||
for _, copy := range copies[1:] {
|
|
||||||
next := lifecycleLockState(copy.ObjInfo.UserDefined)
|
|
||||||
if lockRevisionNewer(next.legalHoldTimestamp, state.legalHoldTimestamp) {
|
|
||||||
state.legalHold, state.legalHoldTimestamp = next.legalHold, next.legalHoldTimestamp
|
|
||||||
}
|
|
||||||
if lockRevisionNewer(next.retentionTimestamp, state.retentionTimestamp) {
|
|
||||||
state.mode, state.retainUntil, state.retentionTimestamp = next.mode, next.retainUntil, next.retentionTimestamp
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return state
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkedExpiryRefused reports whether a set must keep a version whose ILM
|
// checkedExpiryRefused reports whether a set must keep a version whose ILM
|
||||||
@@ -623,7 +615,7 @@ func checkedExpiryRefused(lc lifecycle.Lifecycle, lr objectlock.Retention, rcfg
|
|||||||
if lockRevisionNewer(approved.legalHoldTimestamp, local.legalHoldTimestamp) {
|
if lockRevisionNewer(approved.legalHoldTimestamp, local.legalHoldTimestamp) {
|
||||||
obj.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = approved.legalHold
|
obj.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = approved.legalHold
|
||||||
}
|
}
|
||||||
if lockRevisionNewer(approved.retentionTimestamp, local.retentionTimestamp) {
|
if lockRevisionNewer(approved.retentionTimestamp, local.retentionTimestamp) && !local.complianceWouldWeaken(approved) {
|
||||||
obj.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = approved.mode
|
obj.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = approved.mode
|
||||||
obj.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = approved.retainUntil
|
obj.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = approved.retainUntil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -132,7 +132,7 @@ func TestPoolsMigrationCleanupRechecksSource(t *testing.T) {
|
|||||||
_, err := z.serverPools[1].DeleteObject(t.Context(), bucket, oi.Name, ObjectOptions{VersionID: nullVersionID})
|
_, err := z.serverPools[1].DeleteObject(t.Context(), bucket, oi.Name, ObjectOptions{VersionID: nullVersionID})
|
||||||
mustIAM(t, err)
|
mustIAM(t, err)
|
||||||
}
|
}
|
||||||
err := z.removeMovedVersions(t.Context(), bucket, oi.Name, 0, snapshot, nil)
|
_, err := z.removeMovedVersions(t.Context(), bucket, oi.Name, 0, snapshot, nil)
|
||||||
if change != "delete-uuid" && err == nil {
|
if change != "delete-uuid" && err == nil {
|
||||||
t.Fatal("cleanup accepted changed or uncertain state")
|
t.Fatal("cleanup accepted changed or uncertain state")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -414,6 +414,46 @@ func testMigrationLifecycleExpiry(t *testing.T, kind string) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Physical expiry advances the walk; creating a delete marker does not remove
|
||||||
|
// the addressed version and must not make a walk repeat without progress.
|
||||||
|
func TestPoolsMigrationLifecycleRemovalProgress(t *testing.T) {
|
||||||
|
for _, versioned := range []bool{false, true} {
|
||||||
|
status := versioning.State("")
|
||||||
|
name, wantRemoved := "physical-delete", 1
|
||||||
|
if versioned {
|
||||||
|
status = versioning.Enabled
|
||||||
|
name, wantRemoved = "delete-marker-only", 0
|
||||||
|
}
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
m := newMigrationILM(t, status, false, migrationILMExpireCurrent())
|
||||||
|
oi := m.put("obj-01", "v1", ObjectOptions{Versioned: versioned, MTime: UTCNow().Add(-72 * time.Hour)})
|
||||||
|
_, updatedAt, err := globalBucketMetadataSys.GetLifecycleConfig(m.bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
version := FileInfo{Name: oi.Name, VersionID: oi.VersionID, ModTime: oi.ModTime}
|
||||||
|
removed, err := m.z.removeMovedVersions(t.Context(), m.bucket, oi.Name, 0, []FileInfo{version}, []migrationExpiredVersion{{
|
||||||
|
version: version,
|
||||||
|
event: lifecycle.Event{Action: lifecycle.DeleteAction},
|
||||||
|
lifecycleUpdatedAt: updatedAt,
|
||||||
|
source: lcEventSrc_Decom,
|
||||||
|
}})
|
||||||
|
if err != nil || removed != wantRemoved {
|
||||||
|
t.Fatalf("removed %d versions, err %v; want %d, nil", removed, err, wantRemoved)
|
||||||
|
}
|
||||||
|
if versioned {
|
||||||
|
if body := migrationTestBody(t, m.z.GetObjectNInfo, m.bucket, oi.Name, oi.VersionID); body != "obj-01 v1" {
|
||||||
|
t.Fatalf("noncurrent version reads %q", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
current, err := m.z.GetObjectInfo(t.Context(), m.bucket, oi.Name, ObjectOptions{})
|
||||||
|
if !isErrObjectNotFound(err) || current.DeleteMarker != versioned {
|
||||||
|
t.Fatalf("after expiry: delete marker %t, err %v", current.DeleteMarker, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// A lifecycle action found during the migration walk must not be applied
|
// A lifecycle action found during the migration walk must not be applied
|
||||||
// after the bucket's lifecycle configuration changes. The source copy stays
|
// after the bucket's lifecycle configuration changes. The source copy stays
|
||||||
// available for a fresh scan under the new policy.
|
// available for a fresh scan under the new policy.
|
||||||
|
|||||||
@@ -0,0 +1,601 @@
|
|||||||
|
// Copyright (c) 2026 Feng Ruohang
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
//
|
||||||
|
// This file is part of Silo Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"slices"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The walk has already captured the version when its data read reaches this
|
||||||
|
// hook. Metadata-only checks made by a concurrent DELETE do not reenter it.
|
||||||
|
type migrationProgressReadDisk struct {
|
||||||
|
StorageAPI
|
||||||
|
bucket, object, version string
|
||||||
|
before func() error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d migrationProgressReadDisk) ReadVersion(ctx context.Context, orig, bucket, object, version string, opts ReadOptions) (FileInfo, error) {
|
||||||
|
if opts.ReadData && bucket == d.bucket && object == d.object && version == d.version {
|
||||||
|
if err := d.before(); err != nil {
|
||||||
|
return FileInfo{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return d.StorageAPI.ReadVersion(ctx, orig, bucket, object, version, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Buffer every drive's first walk before publishing any entry. Thus deleting
|
||||||
|
// the parent cannot accidentally let this same walk discover the child.
|
||||||
|
type migrationSnapshotWalk struct {
|
||||||
|
bucket string
|
||||||
|
drives int32
|
||||||
|
calls atomic.Int32
|
||||||
|
ready chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
type migrationSnapshotWalkDisk struct {
|
||||||
|
StorageAPI
|
||||||
|
snapshot *migrationSnapshotWalk
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d migrationSnapshotWalkDisk) WalkDir(ctx context.Context, opts WalkDirOptions, out io.Writer) error {
|
||||||
|
if opts.Bucket != d.snapshot.bucket {
|
||||||
|
return d.StorageAPI.WalkDir(ctx, opts, out)
|
||||||
|
}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
if err := d.StorageAPI.WalkDir(ctx, opts, &buf); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if d.snapshot.calls.Add(1) == d.snapshot.drives {
|
||||||
|
close(d.snapshot.ready)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-d.snapshot.ready:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
_, err := io.Copy(out, &buf)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func migrationProgressSnapshot(t *testing.T, z *erasureServerPools, bucket, key string) *migrationSnapshotWalk {
|
||||||
|
t.Helper()
|
||||||
|
s := &migrationSnapshotWalk{bucket: bucket, drives: int32(len(z.serverPools[0].getHashedSet(key).getDisks())), ready: make(chan struct{})}
|
||||||
|
t.Cleanup(conditionalPutSwapDisks(z.serverPools[0], key, func(d StorageAPI) StorageAPI {
|
||||||
|
return migrationSnapshotWalkDisk{StorageAPI: d, snapshot: s}
|
||||||
|
}))
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func migrationProgressPrepare(t *testing.T, z *erasureServerPools, kind string) {
|
||||||
|
t.Helper()
|
||||||
|
migrationTestMover(t, z, 0, kind)
|
||||||
|
z.poolMetaMutex.Lock()
|
||||||
|
z.poolMeta.Pools[0].LastUpdate = UTCNow()
|
||||||
|
z.poolMetaMutex.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecommissionSplitMetadataDoesNotAdvance(t *testing.T) {
|
||||||
|
m := newMigrationILM(t, "", false, migrationILMExpireCurrent())
|
||||||
|
const key = "obj-01"
|
||||||
|
m.put(key, "old", ObjectOptions{MTime: UTCNow().Add(-96 * time.Hour)})
|
||||||
|
disks := m.z.serverPools[0].getHashedSet(key).getDisks()
|
||||||
|
old := make([][]byte, len(disks)/2)
|
||||||
|
for i := range old {
|
||||||
|
var err error
|
||||||
|
old[i], err = disks[i].ReadAll(t.Context(), m.bucket, pathJoin(key, xlStorageFormatFile))
|
||||||
|
mustIAM(t, err)
|
||||||
|
}
|
||||||
|
m.put(key, "new-and-longer", ObjectOptions{MTime: UTCNow().Add(-72 * time.Hour)})
|
||||||
|
for i := range old {
|
||||||
|
mustIAM(t, disks[i].WriteAll(t.Context(), m.bucket, pathJoin(key, xlStorageFormatFile), old[i]))
|
||||||
|
}
|
||||||
|
if _, err := m.z.serverPools[0].GetObjectInfo(t.Context(), m.bucket, key, ObjectOptions{VersionID: nullVersionID}); !isErrVersionNotFound(err) {
|
||||||
|
t.Fatalf("fixture must expose the ambiguous logical NotFound, got %v", err)
|
||||||
|
}
|
||||||
|
migrationProgressPrepare(t, m.z, "decommission")
|
||||||
|
walk := migrationProgressSnapshot(t, m.z, m.bucket, key)
|
||||||
|
bi := decomBucketInfo{Name: m.bucket}
|
||||||
|
// Check a single pass first, so the unfixed implementation fails rather
|
||||||
|
// than looping until the test timeout.
|
||||||
|
removed, err := m.z.decommissionPoolWalk(t.Context(), 0, m.z.serverPools[0], bi)
|
||||||
|
if err != nil || removed != 0 {
|
||||||
|
t.Fatalf("split metadata reported progress: removed=%d err=%v", removed, err)
|
||||||
|
}
|
||||||
|
mustIAM(t, m.z.decommissionPool(t.Context(), 0, m.z.serverPools[0], bi))
|
||||||
|
if got := walk.calls.Load(); got != 2*walk.drives {
|
||||||
|
t.Fatalf("unchanged metadata repeated the walk: calls=%d", got)
|
||||||
|
}
|
||||||
|
if err := m.z.checkAfterDecomBucket(t.Context(), 0, bi); err == nil {
|
||||||
|
t.Fatal("final verification accepted split metadata still on disk")
|
||||||
|
}
|
||||||
|
for _, disk := range disks {
|
||||||
|
if _, err := disk.ReadVersion(t.Context(), "", m.bucket, key, nullVersionID, ReadOptions{}); err != nil {
|
||||||
|
t.Fatalf("migration removed an ambiguous source revision: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPoolsMigrationConcurrentDeleteCleanup(t *testing.T) {
|
||||||
|
for _, kind := range []string{"decommission", "rebalance"} {
|
||||||
|
t.Run(kind, func(t *testing.T) {
|
||||||
|
z, bucket := migrationTestPools(t)
|
||||||
|
v1 := putConsistencyObject(t, z, bucket, "k", 0, "older", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
|
||||||
|
v2 := putConsistencyObject(t, z, bucket, "k", 0, "newer", ObjectOptions{Versioned: true})
|
||||||
|
var once sync.Once
|
||||||
|
var deleteErr error
|
||||||
|
before := func() error {
|
||||||
|
once.Do(func() {
|
||||||
|
_, deleteErr = z.DeleteObject(t.Context(), bucket, "k", ObjectOptions{Versioned: true, VersionID: v2.VersionID})
|
||||||
|
})
|
||||||
|
return deleteErr
|
||||||
|
}
|
||||||
|
restore := conditionalPutSwapDisks(z.serverPools[0], "k", func(d StorageAPI) StorageAPI {
|
||||||
|
return migrationProgressReadDisk{StorageAPI: d, bucket: bucket, object: "k", version: v2.VersionID, before: before}
|
||||||
|
})
|
||||||
|
defer restore()
|
||||||
|
if failed := migrationTestWalk(t, z, bucket, 0, kind); failed != 0 {
|
||||||
|
t.Fatalf("failed moves: %d", failed)
|
||||||
|
}
|
||||||
|
restore()
|
||||||
|
for _, v := range []ObjectInfo{v1, v2} {
|
||||||
|
for _, disk := range z.serverPools[0].getHashedSet("k").getDisks() {
|
||||||
|
_, err := disk.ReadVersion(t.Context(), "", bucket, "k", v.VersionID, ReadOptions{})
|
||||||
|
if !errors.Is(err, errFileVersionNotFound) && !errors.Is(err, errFileNotFound) {
|
||||||
|
t.Fatalf("source version %s remained: %v", v.VersionID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if body := migrationTestBody(t, z.GetObjectNInfo, bucket, "k", v1.VersionID); body != "older" {
|
||||||
|
t.Fatalf("moved version reads %q", body)
|
||||||
|
}
|
||||||
|
if _, err := z.GetObjectInfo(t.Context(), bucket, "k", ObjectOptions{VersionID: v2.VersionID}); !isErrVersionNotFound(err) {
|
||||||
|
t.Fatalf("concurrently deleted version resurrected: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPoolsMigrationSiblingFailureCleanup(t *testing.T) {
|
||||||
|
for _, kind := range []string{"decommission", "rebalance"} {
|
||||||
|
for _, loseDestination := range []bool{false, true} {
|
||||||
|
name := kind + "/destination-retained"
|
||||||
|
if loseDestination {
|
||||||
|
name = kind + "/destination-removed"
|
||||||
|
}
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
z, bucket := migrationTestPools(t)
|
||||||
|
v1 := putConsistencyObject(t, z, bucket, "k", 0, "older", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
|
||||||
|
v2 := putConsistencyObject(t, z, bucket, "k", 0, "newer", ObjectOptions{Versioned: true})
|
||||||
|
var once sync.Once
|
||||||
|
var deleteErr error
|
||||||
|
before := func() error {
|
||||||
|
once.Do(func() {
|
||||||
|
if loseDestination {
|
||||||
|
_, deleteErr = z.serverPools[1].DeleteObject(t.Context(), bucket, "k", ObjectOptions{VersionID: v1.VersionID})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return errDiskNotFound // only the sibling's data read fails
|
||||||
|
}
|
||||||
|
restore := conditionalPutSwapDisks(z.serverPools[0], "k", func(d StorageAPI) StorageAPI {
|
||||||
|
return migrationProgressReadDisk{StorageAPI: d, bucket: bucket, object: "k", version: v2.VersionID, before: before}
|
||||||
|
})
|
||||||
|
defer restore()
|
||||||
|
migrationProgressPrepare(t, z, kind)
|
||||||
|
if kind == "rebalance" {
|
||||||
|
mustIAM(t, z.rebalanceBucket(t.Context(), bucket, 0))
|
||||||
|
} else {
|
||||||
|
_, err := z.decommissionPoolWalk(t.Context(), 0, z.serverPools[0], decomBucketInfo{Name: bucket})
|
||||||
|
mustIAM(t, err)
|
||||||
|
if err := z.checkAfterDecomBucket(t.Context(), 0, decomBucketInfo{Name: bucket}); err == nil {
|
||||||
|
t.Fatal("failed sibling must block final decommission verification")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
restore()
|
||||||
|
mustIAM(t, deleteErr)
|
||||||
|
if body := migrationTestBody(t, z.GetObjectNInfo, bucket, "k", v2.VersionID); body != "newer" {
|
||||||
|
t.Fatalf("failed sibling reads %q", body)
|
||||||
|
}
|
||||||
|
if body := migrationTestBody(t, z.GetObjectNInfo, bucket, "k", v1.VersionID); body != "older" {
|
||||||
|
t.Fatalf("successful candidate lost: %q", body)
|
||||||
|
}
|
||||||
|
_, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, "k", ObjectOptions{VersionID: v1.VersionID})
|
||||||
|
if loseDestination && err != nil || !loseDestination && !isErrVersionNotFound(err) {
|
||||||
|
t.Fatalf("source cleanup ignored destination state: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecommissionConcurrentParentDeleteRescansChildren(t *testing.T) {
|
||||||
|
z, bucket := migrationTestPools(t)
|
||||||
|
putConsistencyObject(t, z, bucket, "k", 0, "parent", ObjectOptions{})
|
||||||
|
putConsistencyObject(t, z, bucket, "k/b", 0, "child", ObjectOptions{})
|
||||||
|
migrationProgressSnapshot(t, z, bucket, "k")
|
||||||
|
var once sync.Once
|
||||||
|
var deleteErr error
|
||||||
|
t.Cleanup(conditionalPutSwapDisks(z.serverPools[0], "k", func(d StorageAPI) StorageAPI {
|
||||||
|
return migrationProgressReadDisk{StorageAPI: d, bucket: bucket, object: "k", version: nullVersionID, before: func() error {
|
||||||
|
once.Do(func() {
|
||||||
|
_, deleteErr = z.DeleteObject(t.Context(), bucket, "k", ObjectOptions{VersionID: nullVersionID})
|
||||||
|
})
|
||||||
|
return deleteErr
|
||||||
|
}}
|
||||||
|
}))
|
||||||
|
if failed := migrationTestWalk(t, z, bucket, 0, "decommission"); failed != 0 {
|
||||||
|
t.Fatalf("failed moves: %d", failed)
|
||||||
|
}
|
||||||
|
mustIAM(t, z.checkAfterDecomBucket(t.Context(), 0, decomBucketInfo{Name: bucket}))
|
||||||
|
if body := migrationTestBody(t, z.GetObjectNInfo, bucket, "k/b", ""); body != "child" {
|
||||||
|
t.Fatalf("child reads %q", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecommissionDegradedPoolRescansChildren(t *testing.T) {
|
||||||
|
z, bucket := migrationTestPools(t)
|
||||||
|
putConsistencyObject(t, z, bucket, "k", 0, "parent", ObjectOptions{})
|
||||||
|
putConsistencyObject(t, z, bucket, "k/b", 0, "child", ObjectOptions{})
|
||||||
|
var wrapped int
|
||||||
|
t.Cleanup(conditionalPutSwapDisks(z.serverPools[0], "k", func(d StorageAPI) StorageAPI {
|
||||||
|
wrapped++
|
||||||
|
if wrapped == 1 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}))
|
||||||
|
if failed := migrationTestWalk(t, z, bucket, 0, "decommission"); failed != 0 {
|
||||||
|
t.Fatalf("failed moves: %d", failed)
|
||||||
|
}
|
||||||
|
mustIAM(t, z.checkAfterDecomBucket(t.Context(), 0, decomBucketInfo{Name: bucket}))
|
||||||
|
if body := migrationTestBody(t, z.GetObjectNInfo, bucket, "k/b", ""); body != "child" {
|
||||||
|
t.Fatalf("child reads %q", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFreeVersionDecommissionRescansChildren(t *testing.T) {
|
||||||
|
f := newFreeVersionTest(t, "")
|
||||||
|
f.bucket = "free-parent"
|
||||||
|
mustIAM(t, f.z.MakeBucket(t.Context(), f.bucket, MakeBucketOptions{VersioningEnabled: true}))
|
||||||
|
f.deleteVersion(f.tiered("k", "parent", UTCNow().Add(-time.Hour)))
|
||||||
|
f.put("k/b", "child", UTCNow())
|
||||||
|
migrationProgressSnapshot(t, f.z, f.bucket, "k")
|
||||||
|
migrationProgressPrepare(t, f.z, "decommission")
|
||||||
|
bi := decomBucketInfo{Name: f.bucket}
|
||||||
|
removed, err := f.z.decommissionPoolWalk(t.Context(), 0, f.z.serverPools[0], bi)
|
||||||
|
if err != nil || removed != 1 {
|
||||||
|
t.Fatalf("free-only parent removal: removed=%d err=%v", removed, err)
|
||||||
|
}
|
||||||
|
if err := f.z.checkAfterDecomBucket(t.Context(), 0, bi); err == nil {
|
||||||
|
t.Fatal("the newly exposed child must still block completion")
|
||||||
|
}
|
||||||
|
mustIAM(t, f.z.decommissionPool(t.Context(), 0, f.z.serverPools[0], bi))
|
||||||
|
mustIAM(t, f.z.checkAfterDecomBucket(t.Context(), 0, bi))
|
||||||
|
if body := migrationTestBody(t, f.z.GetObjectNInfo, f.bucket, "k/b", ""); body != "k/b child" {
|
||||||
|
t.Fatalf("child reads %q", body)
|
||||||
|
}
|
||||||
|
if !slices.Contains(f.tier.removed, f.remote("k", "parent")) {
|
||||||
|
t.Fatal("free version cleanup did not delete its remote object")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPoolsMigrationCancellation(t *testing.T) {
|
||||||
|
for _, kind := range []string{"decommission", "rebalance"} {
|
||||||
|
for _, phase := range []string{"before-walk", "during-read", "deadline"} {
|
||||||
|
name := kind + "/" + phase
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
z, bucket := migrationTestPools(t)
|
||||||
|
oi := putConsistencyObject(t, z, bucket, "k", 0, "source", ObjectOptions{Versioned: true})
|
||||||
|
migrationProgressPrepare(t, z, kind)
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
wantErr := context.Canceled
|
||||||
|
if phase == "deadline" {
|
||||||
|
cancel()
|
||||||
|
ctx, cancel = context.WithDeadline(t.Context(), time.Time{})
|
||||||
|
wantErr = context.DeadlineExceeded
|
||||||
|
}
|
||||||
|
defer cancel()
|
||||||
|
if phase == "during-read" {
|
||||||
|
t.Cleanup(conditionalPutSwapDisks(z.serverPools[0], "k", func(d StorageAPI) StorageAPI {
|
||||||
|
return migrationProgressReadDisk{StorageAPI: d, bucket: bucket, object: "k", version: oi.VersionID, before: func() error { cancel(); return ctx.Err() }}
|
||||||
|
}))
|
||||||
|
} else {
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
if kind == "decommission" {
|
||||||
|
err = z.decommissionPool(ctx, 0, z.serverPools[0], decomBucketInfo{Name: bucket})
|
||||||
|
} else {
|
||||||
|
err = z.rebalanceBucket(ctx, bucket, 0)
|
||||||
|
}
|
||||||
|
if !errors.Is(err, wantErr) {
|
||||||
|
t.Fatalf("cancellation returned %v", err)
|
||||||
|
}
|
||||||
|
if err := z.decommissionInBackground(ctx, 0); !errors.Is(err, wantErr) {
|
||||||
|
t.Fatalf("canceled background migration returned %v", err)
|
||||||
|
}
|
||||||
|
if err := z.rebalanceBuckets(ctx, 0); !errors.Is(err, wantErr) {
|
||||||
|
t.Fatalf("canceled background rebalance returned %v", err)
|
||||||
|
}
|
||||||
|
if n, err := z.countAbsentMigrationVersions(ctx, bucket, "k", 0, []FileInfo{{VersionID: oi.VersionID}}); n != 0 || !errors.Is(err, wantErr) {
|
||||||
|
t.Fatalf("canceled progress verification: removed=%d err=%v", n, err)
|
||||||
|
}
|
||||||
|
if _, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, "k", ObjectOptions{VersionID: oi.VersionID}); err != nil {
|
||||||
|
t.Fatalf("canceled migration lost source: %v", err)
|
||||||
|
}
|
||||||
|
if err := z.checkAfterDecomBucket(ctx, 0, decomBucketInfo{Name: bucket}); !errors.Is(err, wantErr) {
|
||||||
|
t.Fatalf("canceled verification returned %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type migrationProgressRawDisk struct {
|
||||||
|
StorageAPI
|
||||||
|
buf []byte
|
||||||
|
err error
|
||||||
|
reads *atomic.Int32
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d migrationProgressRawDisk) ReadXL(ctx context.Context, bucket, object string, data bool) (RawFileInfo, error) {
|
||||||
|
d.reads.Add(1)
|
||||||
|
if data {
|
||||||
|
return RawFileInfo{}, errors.New("progress check requested object data")
|
||||||
|
}
|
||||||
|
return RawFileInfo{Buf: d.buf}, d.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d migrationProgressRawDisk) ListDir(context.Context, string, string, string, int) ([]string, error) {
|
||||||
|
return nil, errFileNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPoolsMigrationPhysicalProgress(t *testing.T) {
|
||||||
|
f := newFreeVersionTest(t, "")
|
||||||
|
buf := f.xlMeta(0, "obj-01")
|
||||||
|
versions, err := getFileInfoVersions(buf, f.bucket, "obj-01", true)
|
||||||
|
mustIAM(t, err)
|
||||||
|
if len(versions.Versions) != 3 || !versions.Versions[0].TierFreeVersion() {
|
||||||
|
t.Fatal("fixture must contain two live versions and one free version")
|
||||||
|
}
|
||||||
|
missing := FileInfo{VersionID: mustGetUUID()}
|
||||||
|
snapshot := append(slices.Clone(versions.Versions), missing)
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
buf []byte
|
||||||
|
err error
|
||||||
|
want int
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "all-absent", err: errFileNotFound, want: 4},
|
||||||
|
{name: "missing-volume", err: errVolumeNotFound, want: 4},
|
||||||
|
{name: "minority-live-and-free", buf: buf, want: 1},
|
||||||
|
{name: "unavailable", err: errDiskNotFound, wantErr: true},
|
||||||
|
{name: "read-quorum", err: errErasureReadQuorum, wantErr: true},
|
||||||
|
{name: "corrupt-read", err: errFileCorrupt, wantErr: true},
|
||||||
|
{name: "corrupt-metadata", buf: []byte("broken xl.meta"), wantErr: true},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
var reads atomic.Int32
|
||||||
|
disks := make([]StorageAPI, 4)
|
||||||
|
for i := range disks {
|
||||||
|
disks[i] = migrationProgressRawDisk{err: errFileNotFound, reads: &reads}
|
||||||
|
}
|
||||||
|
disks[3] = migrationProgressRawDisk{buf: tc.buf, err: tc.err, reads: &reads}
|
||||||
|
er := erasureObjects{getDisks: func() []StorageAPI { return disks }}
|
||||||
|
n, err := er.countAbsentMigrationVersions(t.Context(), f.bucket, "obj-01", snapshot)
|
||||||
|
if n != tc.want || (err != nil) != tc.wantErr {
|
||||||
|
t.Fatalf("removed=%d err=%v, want %d / error=%t", n, err, tc.want, tc.wantErr)
|
||||||
|
}
|
||||||
|
if reads.Load() != int32(len(disks)) {
|
||||||
|
t.Fatalf("read count %d must not depend on version count", reads.Load())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
putConsistencyObject(t, f.z, f.bucket, "null", 0, "null-data", ObjectOptions{})
|
||||||
|
nullBuf := f.xlMeta(0, "null")
|
||||||
|
var reads atomic.Int32
|
||||||
|
er := erasureObjects{getDisks: func() []StorageAPI {
|
||||||
|
return []StorageAPI{migrationProgressRawDisk{buf: nullBuf, reads: &reads}}
|
||||||
|
}}
|
||||||
|
if n, err := er.countAbsentMigrationVersions(t.Context(), f.bucket, "null", []FileInfo{{VersionID: nullVersionID}, {VersionID: ""}}); n != 0 || err != nil {
|
||||||
|
t.Fatalf("null version normalization: removed=%d err=%v", n, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCountAbsentMigrationVersionsToleratesConfiguredOfflineDrives(t *testing.T) {
|
||||||
|
makeDisks := func(offlineCount int, asError bool) []StorageAPI {
|
||||||
|
disks := make([]StorageAPI, 4)
|
||||||
|
for i := range disks {
|
||||||
|
disks[i] = migrationProgressRawDisk{err: errFileNotFound, reads: &atomic.Int32{}}
|
||||||
|
}
|
||||||
|
for i := range offlineCount {
|
||||||
|
if asError {
|
||||||
|
disks[i] = migrationProgressRawDisk{err: errDiskNotFound, reads: &atomic.Int32{}}
|
||||||
|
} else {
|
||||||
|
disks[i] = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return disks
|
||||||
|
}
|
||||||
|
snapshot := []FileInfo{{VersionID: mustGetUUID()}}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
offline int
|
||||||
|
parity int
|
||||||
|
asError bool
|
||||||
|
wantCount int
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "nil-drive-within-parity", offline: 1, parity: 1, wantCount: 1},
|
||||||
|
{name: "read-error-within-parity", offline: 1, parity: 1, asError: true, wantCount: 1},
|
||||||
|
{name: "half-offline-within-parity", offline: 2, parity: 2, wantCount: 1},
|
||||||
|
{name: "offline-beyond-parity", offline: 2, parity: 1, wantErr: true},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
disks := makeDisks(tc.offline, tc.asError)
|
||||||
|
er := erasureObjects{
|
||||||
|
setDriveCount: len(disks),
|
||||||
|
defaultParityCount: tc.parity,
|
||||||
|
getDisks: func() []StorageAPI { return disks },
|
||||||
|
}
|
||||||
|
got, err := er.countAbsentMigrationVersions(t.Context(), "bucket", "object", snapshot)
|
||||||
|
if got != tc.wantCount || (err != nil) != tc.wantErr {
|
||||||
|
t.Fatalf("removed=%d err=%v, want %d / error=%t", got, err, tc.wantCount, tc.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A successful quorum deletion can leave one source drive behind. It must not
|
||||||
|
// count as physical removal or pass the final check, nor cause a retry loop.
|
||||||
|
func TestDecommissionQuorumDeletionIsNotPhysicalProgress(t *testing.T) {
|
||||||
|
z, bucket := migrationTestPools(t)
|
||||||
|
oi := putConsistencyObject(t, z, bucket, "k", 0, "payload", ObjectOptions{Versioned: true})
|
||||||
|
move := migrationTestMover(t, z, 0, "decommission")
|
||||||
|
mustIAM(t, move(t.Context(), 0, bucket, migrationTestReader(t, z, 0, oi)))
|
||||||
|
first := true
|
||||||
|
restore := conditionalPutSwapDisks(z.serverPools[0], "k", func(d StorageAPI) StorageAPI {
|
||||||
|
if first {
|
||||||
|
first = false
|
||||||
|
return consistencyDeleteFaultDisk{StorageAPI: d, bucket: bucket, object: "k", version: oi.VersionID}
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
})
|
||||||
|
defer restore()
|
||||||
|
snapshot := []FileInfo{{VersionID: oi.VersionID, ModTime: oi.ModTime}}
|
||||||
|
for range 2 {
|
||||||
|
n, err := z.removeMovedVersions(t.Context(), bucket, "k", 0, snapshot, nil)
|
||||||
|
if err != nil || n != 0 {
|
||||||
|
t.Fatalf("partial deletion reported progress: removed=%d err=%v", n, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := z.checkAfterDecomBucket(t.Context(), 0, decomBucketInfo{Name: bucket}); err == nil {
|
||||||
|
t.Fatal("final verification accepted a minority-drive source copy")
|
||||||
|
}
|
||||||
|
if body := migrationTestBody(t, z.GetObjectNInfo, bucket, "k", oi.VersionID); body != "payload" {
|
||||||
|
t.Fatalf("destination copy lost: %q", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecommissionProgressReadFailureStopsWalk(t *testing.T) {
|
||||||
|
z, bucket := migrationTestPools(t)
|
||||||
|
putConsistencyObject(t, z, bucket, "k", 0, "payload", ObjectOptions{Versioned: true})
|
||||||
|
migrationProgressPrepare(t, z, "decommission")
|
||||||
|
walk := migrationProgressSnapshot(t, z, bucket, "k")
|
||||||
|
restore := conditionalPutSwapDisks(z.serverPools[0], "k", func(d StorageAPI) StorageAPI {
|
||||||
|
return tierReadFaultDisk{StorageAPI: d, bucket: bucket, object: "k", err: errDiskNotFound}
|
||||||
|
})
|
||||||
|
defer restore()
|
||||||
|
if err := z.decommissionPool(t.Context(), 0, z.serverPools[0], decomBucketInfo{Name: bucket}); err == nil {
|
||||||
|
t.Fatal("unreadable source progress check reported success")
|
||||||
|
}
|
||||||
|
if got := walk.calls.Load(); got != walk.drives {
|
||||||
|
t.Fatalf("verification failure repeated the walk: calls=%d", got)
|
||||||
|
}
|
||||||
|
restore()
|
||||||
|
if body := migrationTestBody(t, z.GetObjectNInfo, bucket, "k", ""); body != "payload" {
|
||||||
|
t.Fatalf("quorum move lost destination: %q", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type migrationProgressListFaultDisk struct {
|
||||||
|
StorageAPI
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d migrationProgressListFaultDisk) ListDir(context.Context, string, string, string, int) ([]string, error) {
|
||||||
|
return nil, errDiskAccessDenied
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecommissionFinalVerificationFailsClosed(t *testing.T) {
|
||||||
|
for _, state := range []string{"offline-drive", "directory-error", "corrupt-metadata", "truncated-metadata", "legacy-metadata"} {
|
||||||
|
t.Run(state, func(t *testing.T) {
|
||||||
|
z, bucket := migrationTestPools(t)
|
||||||
|
switch state {
|
||||||
|
case "offline-drive":
|
||||||
|
set := z.serverPools[0].getHashedSet("k")
|
||||||
|
maxOffline := set.defaultParityCount + 1
|
||||||
|
wrapped := 0
|
||||||
|
t.Cleanup(conditionalPutSwapDisks(z.serverPools[0], "k", func(d StorageAPI) StorageAPI {
|
||||||
|
wrapped++
|
||||||
|
if wrapped <= maxOffline {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}))
|
||||||
|
case "directory-error":
|
||||||
|
t.Cleanup(conditionalPutSwapDisks(z.serverPools[0], "k", func(d StorageAPI) StorageAPI {
|
||||||
|
return migrationProgressListFaultDisk{StorageAPI: d}
|
||||||
|
}))
|
||||||
|
default:
|
||||||
|
// Real metadata on only one drive, without a readable quorum.
|
||||||
|
// This also covers errors that WalkDir deliberately skips.
|
||||||
|
disk := z.serverPools[0].getHashedSet("k").getDisks()[0]
|
||||||
|
file, data := xlStorageFormatFile, []byte("broken xl.meta")
|
||||||
|
if state == "truncated-metadata" {
|
||||||
|
data = nil
|
||||||
|
}
|
||||||
|
if state == "legacy-metadata" {
|
||||||
|
file = xlStorageFormatFileV1
|
||||||
|
}
|
||||||
|
mustIAM(t, disk.WriteAll(t.Context(), bucket, pathJoin("k", file), data))
|
||||||
|
}
|
||||||
|
if err := z.checkAfterDecomBucket(t.Context(), 0, decomBucketInfo{Name: bucket}); err == nil {
|
||||||
|
t.Fatal("unreadable storage was accepted as physically empty")
|
||||||
|
}
|
||||||
|
if n, err := z.countAbsentMigrationVersions(t.Context(), bucket, "k", 0, []FileInfo{{VersionID: mustGetUUID()}}); n != 0 || err == nil {
|
||||||
|
t.Fatalf("unreadable storage was counted as progress: removed=%d err=%v", n, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFreeVersionDecommissionPreservesLiveRemoteOwner(t *testing.T) {
|
||||||
|
f := newFreeVersionTest(t, "")
|
||||||
|
const key = "obj-02"
|
||||||
|
fv, tuple := f.freeRecord(0, key)
|
||||||
|
owner := putConsistencyObject(t, f.z, f.bucket, key, 1, "live-owner", ObjectOptions{Versioned: true})
|
||||||
|
infos, errs := readAllFileInfo(t.Context(), f.z.serverPools[1].getHashedSet(key).getDisks(), "", f.bucket, key, owner.VersionID, false, false)
|
||||||
|
mustIAM(t, errs[0])
|
||||||
|
fi := infos[0]
|
||||||
|
fi.TransitionStatus, fi.TransitionTier = lifecycle.TransitionComplete, tuple.Tier
|
||||||
|
fi.TransitionedObjName, fi.TransitionVersionID = tuple.Object, tuple.VersionID
|
||||||
|
mustIAM(t, f.z.serverPools[1].DecomTieredObject(t.Context(), f.bucket, key, fi, ObjectOptions{}))
|
||||||
|
if err := f.z.removeFreeVersions(t.Context(), f.bucket, key, 0, []FileInfo{fv}); !errors.Is(err, errTierStillReferenced) {
|
||||||
|
t.Fatalf("cleanup with a live remote owner: %v", err)
|
||||||
|
}
|
||||||
|
if n, err := f.z.countAbsentMigrationVersions(t.Context(), f.bucket, key, 0, []FileInfo{fv}); err != nil || n != 0 {
|
||||||
|
t.Fatalf("retained free record reported progress: removed=%d err=%v", n, err)
|
||||||
|
}
|
||||||
|
if slices.Contains(f.tier.removed, tuple.Object) {
|
||||||
|
t.Fatal("cleanup deleted a live remote object")
|
||||||
|
}
|
||||||
|
if err := f.z.checkAfterDecomBucket(t.Context(), 0, decomBucketInfo{Name: f.bucket}); err == nil {
|
||||||
|
t.Fatal("retained free record must block final decommission verification")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -17,7 +17,147 @@
|
|||||||
|
|
||||||
package cmd
|
package cmd
|
||||||
|
|
||||||
import xhttp "github.com/minio/minio/internal/http"
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||||
|
)
|
||||||
|
|
||||||
|
func migrationVersionKey(versionID string) string {
|
||||||
|
if versionID == "" || versionID == nullVersionID {
|
||||||
|
return nullVersionID
|
||||||
|
}
|
||||||
|
return versionID
|
||||||
|
}
|
||||||
|
|
||||||
|
// migrationReadQuorum requires enough online drives to intersect every
|
||||||
|
// committed write quorum. All of those drives must be checked: a minority
|
||||||
|
// remnant still prevents progress. Unknown set geometry tolerates no outages.
|
||||||
|
func (er erasureObjects) migrationReadQuorum(diskCount int) int {
|
||||||
|
if er.setDriveCount <= 0 || er.setDriveCount != diskCount || er.defaultParityCount < 0 || er.defaultParityCount > er.setDriveCount/2 {
|
||||||
|
return diskCount
|
||||||
|
}
|
||||||
|
return er.defaultRQuorum()
|
||||||
|
}
|
||||||
|
|
||||||
|
// countAbsentMigrationVersions counts snapshot versions that no online source drive
|
||||||
|
// holds anymore, including free versions. A pool-level NotFound is insufficient:
|
||||||
|
// the ordinary read path can return it for inconsistent metadata. Only readable
|
||||||
|
// per-drive metadata (or physical absence) proves progress for another walk.
|
||||||
|
// The caller holds the object's writer lock while checking the source.
|
||||||
|
func (er erasureObjects) countAbsentMigrationVersions(ctx context.Context, bucket, object string, versions []FileInfo) (int, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if len(versions) == 0 {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
absent := make(map[string]struct{}, len(versions))
|
||||||
|
for _, version := range versions {
|
||||||
|
absent[migrationVersionKey(version.VersionID)] = struct{}{}
|
||||||
|
}
|
||||||
|
disks := er.getDisks()
|
||||||
|
if len(disks) == 0 {
|
||||||
|
return 0, toObjectErr(errErasureReadQuorum, bucket, object)
|
||||||
|
}
|
||||||
|
readQuorum := er.migrationReadQuorum(len(disks))
|
||||||
|
offline := make([]bool, len(disks))
|
||||||
|
stored := make([][]FileInfo, len(disks))
|
||||||
|
g := errgroup.WithNErrs(len(disks))
|
||||||
|
for i := range disks {
|
||||||
|
g.Go(func() error {
|
||||||
|
if disks[i] == nil {
|
||||||
|
offline[i] = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
raw, err := disks[i].ReadXL(ctx, bucket, object, false)
|
||||||
|
if errors.Is(err, errDiskNotFound) {
|
||||||
|
offline[i] = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if errors.Is(err, errFileNotFound) {
|
||||||
|
// ReadXL also returns NotFound for an empty xl.meta, and may
|
||||||
|
// ignore legacy metadata. Confirm physical absence before using
|
||||||
|
// that result as progress. This does not descend into child keys.
|
||||||
|
// Both reads run in parallel across drives, once per object.
|
||||||
|
entries, err := disks[i].ListDir(ctx, "", bucket, object, -1)
|
||||||
|
if errors.Is(err, errDiskNotFound) {
|
||||||
|
offline[i] = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil && !errors.Is(err, errFileNotFound) && !errors.Is(err, errVolumeNotFound) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry == xlStorageFormatFile || entry == xlStorageFormatFileV1 {
|
||||||
|
return errFileCorrupt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if errors.Is(err, errVolumeNotFound) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// allParts=false omits part details, not free versions. Decode every
|
||||||
|
// version before concluding absence, so corrupt metadata fails closed.
|
||||||
|
fivs, err := getAllFileInfoVersions(raw.Buf, bucket, object, false)
|
||||||
|
stored[i] = fivs.Versions
|
||||||
|
return err
|
||||||
|
}, i)
|
||||||
|
}
|
||||||
|
errs := g.Wait()
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
for i, err := range errs {
|
||||||
|
if err != nil {
|
||||||
|
return 0, toObjectErr(err, bucket, object)
|
||||||
|
}
|
||||||
|
for _, version := range stored[i] {
|
||||||
|
delete(absent, migrationVersionKey(version.VersionID))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
offlineCount := 0
|
||||||
|
for _, unavailable := range offline {
|
||||||
|
if unavailable {
|
||||||
|
offlineCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(disks)-offlineCount < readQuorum {
|
||||||
|
return 0, toObjectErr(errErasureReadQuorum, bucket, object)
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return len(absent), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// countAbsentMigrationVersions checks ignored walk entries without authorizing
|
||||||
|
// their deletion. A concurrent client delete may have uncovered child keys.
|
||||||
|
func (z *erasureServerPools) countAbsentMigrationVersions(ctx context.Context, bucket, object string, source int, versions []FileInfo) (int, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if len(versions) == 0 {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
object = encodeDirObject(object)
|
||||||
|
lk := z.NewNSLock(bucket, object)
|
||||||
|
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return 0, ctx.Err()
|
||||||
|
}
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
defer lk.Unlock(lkctx)
|
||||||
|
return z.serverPools[source].getHashedSet(object).countAbsentMigrationVersions(lkctx.Context(), bucket, object, versions)
|
||||||
|
}
|
||||||
|
|
||||||
// migrationObjectMetadata restores tags removed from the read representation.
|
// migrationObjectMetadata restores tags removed from the read representation.
|
||||||
// Keep the original revision, including ordered empty states, for the existing
|
// Keep the original revision, including ordered empty states, for the existing
|
||||||
|
|||||||
@@ -0,0 +1,205 @@
|
|||||||
|
// Copyright (c) 2026 PGSTY
|
||||||
|
//
|
||||||
|
// This file is part of Silo Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
type copyPartSourceContextKey struct{}
|
||||||
|
|
||||||
|
// Pause the HTTP copy after the real reader has acquired its source lock.
|
||||||
|
// IAM and other background readers do not carry the request's context key.
|
||||||
|
type copyPartSourceGate struct {
|
||||||
|
ObjectLayer
|
||||||
|
opened, release chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *copyPartSourceGate) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
|
||||||
|
gr, err := a.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
|
||||||
|
if err != nil || ctx.Value(copyPartSourceContextKey{}) != true {
|
||||||
|
return gr, err
|
||||||
|
}
|
||||||
|
close(a.opened)
|
||||||
|
select {
|
||||||
|
case <-a.release:
|
||||||
|
return gr, nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
gr.Close()
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyPartLockRefs(l *localLockInstance) int32 {
|
||||||
|
l.ns.lockMapMutex.Lock()
|
||||||
|
defer l.ns.lockMapMutex.Unlock()
|
||||||
|
if lock := l.ns.lockMap[pathJoin(l.volume, l.paths[0])]; lock != nil {
|
||||||
|
return lock.ref
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// A copy from the upload's own key holds object RLock before asking for the
|
||||||
|
// upload RLock. Complete must wait for that reader without holding upload
|
||||||
|
// Lock, on both single- and multi-pool backends. Cancellation of either
|
||||||
|
// request must release its locks and let the other request finish.
|
||||||
|
func TestUploadPartCopyConcurrentComplete(t *testing.T) {
|
||||||
|
for _, pools := range []int{1, 2} {
|
||||||
|
for _, action := range []string{"complete", "cancel-copy", "cancel-complete"} {
|
||||||
|
t.Run(fmt.Sprintf("pools=%d/%s", pools, action), func(t *testing.T) {
|
||||||
|
z, _ := bulkDeletePools(t, pools)
|
||||||
|
ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second)
|
||||||
|
var workers sync.WaitGroup
|
||||||
|
defer func() { cancel(); workers.Wait() }()
|
||||||
|
api := ©PartSourceGate{ObjectLayer: z, opened: make(chan struct{}), release: make(chan struct{})}
|
||||||
|
bucket, router, err := initAPIHandlerTest(ctx, api, nil, MakeBucketOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
setObjectLayer(api)
|
||||||
|
const key = "copy-part-self"
|
||||||
|
data := bytes.Repeat([]byte("existing source bytes"), 32768)
|
||||||
|
oi, err := z.PutObject(ctx, bucket, key, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if oi.Inlined {
|
||||||
|
t.Fatal("the source must retain its read lock until its reader closes")
|
||||||
|
}
|
||||||
|
upload, err := z.NewMultipartUpload(ctx, bucket, key, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
first := []byte("the completed part")
|
||||||
|
part, err := z.PutObjectPart(ctx, bucket, key, upload.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(first), int64(len(first)), "", ""), ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
objectLock := z.NewObjectNSLock(bucket, key).(*localLockInstance)
|
||||||
|
uploadLock := z.uploadIDNSLock(bucket, key, upload.UploadID).(*localLockInstance)
|
||||||
|
copyCtx, cancelCopy := context.WithCancel(context.WithValue(ctx, copyPartSourceContextKey{}, true))
|
||||||
|
defer cancelCopy()
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectPartURL("", bucket, key, upload.UploadID, "2"), 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, map[string]string{xhttp.AmzCopySource: "/" + bucket + "/" + key})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
copyDone := make(chan *httptest.ResponseRecorder, 1)
|
||||||
|
workers.Go(func() {
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, req.WithContext(copyCtx))
|
||||||
|
copyDone <- rec
|
||||||
|
})
|
||||||
|
select {
|
||||||
|
case <-api.opened:
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal("COPY did not open its source")
|
||||||
|
}
|
||||||
|
if refs := copyPartLockRefs(objectLock); refs != 1 {
|
||||||
|
t.Fatalf("COPY object lock references = %d, want 1", refs)
|
||||||
|
}
|
||||||
|
|
||||||
|
completeCtx, cancelComplete := context.WithCancel(ctx)
|
||||||
|
defer cancelComplete()
|
||||||
|
completeDone := make(chan error, 1)
|
||||||
|
workers.Go(func() {
|
||||||
|
_, err := z.CompleteMultipartUpload(completeCtx, bucket, key, upload.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{})
|
||||||
|
completeDone <- err
|
||||||
|
})
|
||||||
|
// One reference belongs to COPY, the second to Complete waiting
|
||||||
|
// for it. Check the actual lock resources, not elapsed time.
|
||||||
|
ticker := time.NewTicker(time.Millisecond)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for copyPartLockRefs(objectLock) != 2 {
|
||||||
|
select {
|
||||||
|
case err := <-completeDone:
|
||||||
|
t.Fatalf("Complete exited before waiting for COPY: %v", err)
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal("Complete did not wait for the source object lock")
|
||||||
|
case <-ticker.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if refs := copyPartLockRefs(uploadLock); refs != 0 {
|
||||||
|
t.Fatalf("Complete holds the upload lock while waiting for COPY's object lock: refs=%d", refs)
|
||||||
|
}
|
||||||
|
|
||||||
|
if action == "cancel-complete" {
|
||||||
|
cancelComplete()
|
||||||
|
select {
|
||||||
|
case err := <-completeDone:
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("canceled Complete: %v", err)
|
||||||
|
}
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal("Complete did not cancel while COPY held the source")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if action == "cancel-copy" {
|
||||||
|
cancelCopy()
|
||||||
|
} else {
|
||||||
|
close(api.release)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case rec := <-copyDone:
|
||||||
|
want := http.StatusOK
|
||||||
|
if action == "cancel-copy" {
|
||||||
|
want = 499
|
||||||
|
}
|
||||||
|
if rec.Code != want {
|
||||||
|
t.Fatalf("COPY: %d, want %d: %s", rec.Code, want, rec.Body.String())
|
||||||
|
}
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal("COPY did not finish")
|
||||||
|
}
|
||||||
|
if action != "cancel-complete" {
|
||||||
|
select {
|
||||||
|
case err := <-completeDone:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Complete after source release: %v", err)
|
||||||
|
}
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal("Complete did not finish after source release")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if copyPartLockRefs(objectLock) != 0 || copyPartLockRefs(uploadLock) != 0 {
|
||||||
|
t.Fatal("request left an object or upload lock behind")
|
||||||
|
}
|
||||||
|
wantBody := string(first)
|
||||||
|
if action == "cancel-complete" {
|
||||||
|
wantBody = string(data)
|
||||||
|
if _, err := z.GetMultipartInfo(ctx, bucket, key, upload.UploadID, ObjectOptions{}); err != nil {
|
||||||
|
t.Fatalf("canceled completion lost the upload: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if body := bulkDeleteBody(t, z, bucket, key, ""); body != wantBody {
|
||||||
|
t.Fatalf("object body length = %d, want %d", len(body), len(wantBody))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -590,9 +590,9 @@ func TestSinglePoolConditionalPutHTTP(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// An internal replica callback without an addressed version is not a public
|
// Replica overwrites must establish protection across every pool, including
|
||||||
// condition. Preserve its availability when another pool is unreadable. An
|
// the null version of an unversioned request. An unreadable pool may hold a
|
||||||
// addressed replica already requires all pools for lock/tag reconciliation.
|
// protected copy, so neither addressed nor unaddressed writes may replace it.
|
||||||
func TestPoolsConditionalPutReplicaAvailability(t *testing.T) {
|
func TestPoolsConditionalPutReplicaAvailability(t *testing.T) {
|
||||||
for _, addressed := range []bool{false, true} {
|
for _, addressed := range []bool{false, true} {
|
||||||
t.Run(fmt.Sprintf("addressed=%t", addressed), func(t *testing.T) {
|
t.Run(fmt.Sprintf("addressed=%t", addressed), func(t *testing.T) {
|
||||||
@@ -615,10 +615,9 @@ func TestPoolsConditionalPutReplicaAvailability(t *testing.T) {
|
|||||||
xhttp.MinIOSourceETag: fmt.Sprintf("%x", md5.Sum([]byte("replacement"))),
|
xhttp.MinIOSourceETag: fmt.Sprintf("%x", md5.Sum([]byte("replacement"))),
|
||||||
}
|
}
|
||||||
url := getPutObjectURL("", bucket, object)
|
url := getPutObjectURL("", bucket, object)
|
||||||
wantStatus, wantBody := http.StatusOK, "replacement"
|
wantStatus, wantBody := http.StatusServiceUnavailable, "old"
|
||||||
if addressed {
|
if addressed {
|
||||||
url += "?versionId=" + oi.VersionID
|
url += "?versionId=" + oi.VersionID
|
||||||
wantStatus, wantBody = http.StatusServiceUnavailable, "old"
|
|
||||||
}
|
}
|
||||||
put := multipartConditionRequest(t, router, http.MethodPut, url, "replacement", headers)
|
put := multipartConditionRequest(t, router, http.MethodPut, url, "replacement", headers)
|
||||||
if put.Code != wantStatus {
|
if put.Code != wantStatus {
|
||||||
|
|||||||
@@ -376,6 +376,9 @@ func (z *erasureServerPools) IsPoolRebalancing(poolIndex int) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int) (err error) {
|
func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int) (err error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
doneCh := make(chan error, 1)
|
doneCh := make(chan error, 1)
|
||||||
defer xioutil.SafeClose(doneCh)
|
defer xioutil.SafeClose(doneCh)
|
||||||
|
|
||||||
@@ -450,7 +453,7 @@ func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int)
|
|||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
doneCh <- ctx.Err()
|
doneCh <- ctx.Err()
|
||||||
return err
|
return ctx.Err()
|
||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -474,6 +477,10 @@ func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int)
|
|||||||
z.bucketRebalanceDone(bucket, poolIdx)
|
z.bucketRebalanceDone(bucket, poolIdx)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
doneCh <- err
|
||||||
|
return err
|
||||||
|
}
|
||||||
rebalanceLogEvent(GlobalContext, "Pool %d rebalancing is done", poolIdx+1)
|
rebalanceLogEvent(GlobalContext, "Pool %d rebalancing is done", poolIdx+1)
|
||||||
|
|
||||||
return err
|
return err
|
||||||
@@ -538,6 +545,9 @@ func (set *erasureObjects) listObjectsToRebalance(ctx context.Context, bucketNam
|
|||||||
|
|
||||||
// rebalanceBucket rebalances objects under bucket in poolIdx pool
|
// rebalanceBucket rebalances objects under bucket in poolIdx pool
|
||||||
func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string, poolIdx int) (err error) {
|
func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string, poolIdx int) (err error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
ctx = logger.SetReqInfo(ctx, &logger.ReqInfo{})
|
ctx = logger.SetReqInfo(ctx, &logger.ReqInfo{})
|
||||||
|
|
||||||
var vc *versioning.Versioning
|
var vc *versioning.Versioning
|
||||||
@@ -643,7 +653,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
|
|||||||
// to create the appropriate stack.
|
// to create the appropriate stack.
|
||||||
versionsSorter(fivs.Versions).reverse()
|
versionsSorter(fivs.Versions).reverse()
|
||||||
|
|
||||||
var rebalanced int
|
var candidates []FileInfo
|
||||||
var expired []migrationExpiredVersion
|
var expired []migrationExpiredVersion
|
||||||
// The key's history in all pools, read once for its versions.
|
// The key's history in all pools, read once for its versions.
|
||||||
var history migrationHistory
|
var history migrationHistory
|
||||||
@@ -658,12 +668,8 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
|
|||||||
|
|
||||||
// Apply lifecycle rules on the objects that are expired.
|
// Apply lifecycle rules on the objects that are expired.
|
||||||
if action := filterLifecycle(bucket, version.Name, version, &history); action.Delete() {
|
if action := filterLifecycle(bucket, version.Name, version, &history); action.Delete() {
|
||||||
// Count the version, or the source copies of the moved
|
|
||||||
// ones stay behind as duplicates. Its own source copy
|
|
||||||
// goes with them when ILM deletes it alone; any other
|
|
||||||
// version stays until the expiry has run.
|
|
||||||
rebalanced++
|
|
||||||
if ilmDeletesVersionOnly(action) || action.DeleteAll() {
|
if ilmDeletesVersionOnly(action) || action.DeleteAll() {
|
||||||
|
candidates = append(candidates, version)
|
||||||
expired = append(expired, migrationExpiredVersion{
|
expired = append(expired, migrationExpiredVersion{
|
||||||
version: version,
|
version: version,
|
||||||
event: lifecycle.Event{Action: action},
|
event: lifecycle.Event{Action: action},
|
||||||
@@ -706,7 +712,6 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
|
|||||||
// This can happen when rebalance stop races with ongoing rebalance workers.
|
// This can happen when rebalance stop races with ongoing rebalance workers.
|
||||||
// These rebalance failures can be ignored.
|
// These rebalance failures can be ignored.
|
||||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
|
if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
|
||||||
ignore = true
|
|
||||||
stopFn(0, nil)
|
stopFn(0, nil)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -716,7 +721,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
|
|||||||
failure = err != nil
|
failure = err != nil
|
||||||
if !failure {
|
if !failure {
|
||||||
z.updatePoolStats(poolIdx, bucket, version)
|
z.updatePoolStats(poolIdx, bucket, version)
|
||||||
rebalanced++
|
candidates = append(candidates, version)
|
||||||
}
|
}
|
||||||
auditLogRebalance(ctx, "Rebalance:DeleteMarker", bucket, version.Name, versionID, err)
|
auditLogRebalance(ctx, "Rebalance:DeleteMarker", bucket, version.Name, versionID, err)
|
||||||
continue
|
continue
|
||||||
@@ -773,13 +778,14 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
|
|||||||
break // break out on first error
|
break // break out on first error
|
||||||
}
|
}
|
||||||
z.updatePoolStats(poolIdx, bucket, version)
|
z.updatePoolStats(poolIdx, bucket, version)
|
||||||
rebalanced++
|
candidates = append(candidates, version)
|
||||||
}
|
}
|
||||||
|
|
||||||
// if all versions were rebalanced, we can delete the object versions.
|
// A skipped or failed sibling does not prevent retiring successful
|
||||||
if rebalanced == len(fivs.Versions) {
|
// moves. Cleanup still rechecks each candidate and its destination.
|
||||||
|
if len(candidates) > 0 {
|
||||||
stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceRemoveObject, poolIdx, bucket, entry.name)
|
stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceRemoveObject, poolIdx, bucket, entry.name)
|
||||||
err := z.removeMovedVersions(ctx, bucket, entry.name, poolIdx, fivs.Versions, expired)
|
_, err := z.removeMovedVersions(ctx, bucket, entry.name, poolIdx, candidates, expired)
|
||||||
stopFn(0, err)
|
stopFn(0, err)
|
||||||
auditLogRebalance(ctx, "Rebalance:DeleteObject", bucket, entry.name, "", err)
|
auditLogRebalance(ctx, "Rebalance:DeleteObject", bucket, entry.name, "", err)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -806,7 +812,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
|
|||||||
}
|
}
|
||||||
|
|
||||||
wk.Wait()
|
wk.Wait()
|
||||||
return nil
|
return ctx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
type rebalSaveOpts uint8
|
type rebalSaveOpts uint8
|
||||||
|
|||||||
+85
-37
@@ -1195,6 +1195,7 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
|
|||||||
if opts.replicaObjectInfo == nil && (opts.ReplicaLockReconcile || !opts.ReplicationRequest) {
|
if opts.replicaObjectInfo == nil && (opts.ReplicaLockReconcile || !opts.ReplicationRequest) {
|
||||||
opts.replicaObjectInfo = z.replicaObjectInfo
|
opts.replicaObjectInfo = z.replicaObjectInfo
|
||||||
}
|
}
|
||||||
|
opts.replicaContentGuard = z.guardReplicaContent
|
||||||
oi, err := z.serverPools[idx].PutObject(ctx, bucket, object, data, opts)
|
oi, err := z.serverPools[idx].PutObject(ctx, bucket, object, data, opts)
|
||||||
if err == nil && opts.ReplicaLockReconcile && !opts.DataMovement {
|
if err == nil && opts.ReplicaLockReconcile && !opts.DataMovement {
|
||||||
err = z.retireReplicaCopies(ctx, bucket, object, idx, oi)
|
err = z.retireReplicaCopies(ctx, bucket, object, idx, oi)
|
||||||
@@ -1319,7 +1320,9 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
|||||||
approvedExpiry := opts.Expiration.Expire && opts.Expiration.CheckedAllPools && !opts.Transition.ExpireRestored
|
approvedExpiry := opts.Expiration.Expire && opts.Expiration.CheckedAllPools && !opts.Transition.ExpireRestored
|
||||||
reconcileVersion := opts.VersionID != "" && !opts.DataMovement &&
|
reconcileVersion := opts.VersionID != "" && !opts.DataMovement &&
|
||||||
(!opts.ReplicationRequest || opts.isVersionPurge()) && (!opts.Expiration.Expire || approvedExpiry) && !opts.InclFreeVersions
|
(!opts.ReplicationRequest || opts.isVersionPurge()) && (!opts.Expiration.Expire || approvedExpiry) && !opts.InclFreeVersions
|
||||||
if !z.SinglePool() && (opts.CheckPrecondFn != nil || reconcileVersion) {
|
reconcileUnaddressedLock := opts.VersionID == "" && !opts.Versioned && !opts.DeleteMarker &&
|
||||||
|
opts.RefuseLockedOverwrite && !opts.DataMovement && !opts.ReplicationRequest && !opts.Expiration.Expire && !opts.InclFreeVersions
|
||||||
|
if !z.SinglePool() && (opts.CheckPrecondFn != nil || reconcileVersion || reconcileUnaddressedLock) {
|
||||||
return z.deleteObjectReconciled(ctx, bucket, object, opts)
|
return z.deleteObjectReconciled(ctx, bucket, object, opts)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1688,6 +1691,51 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
|
|||||||
return dobjects, derrs
|
return dobjects, derrs
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// copyObjectPlan fixes placement before a caller prepares encrypted or compressed
|
||||||
|
// bytes. A reference can only be made in the pool holding the source version.
|
||||||
|
type copyObjectPlan struct {
|
||||||
|
poolIdx int
|
||||||
|
rewrite bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// prepareCopyObject requires the same source/destination locks as CopyObject.
|
||||||
|
// In-place changes address the exact version; new versions retain the normal
|
||||||
|
// write placement and rewrite the data when it lives in a different pool.
|
||||||
|
func (z *erasureServerPools) prepareCopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (copyObjectPlan, error) {
|
||||||
|
if err := checkCopyObjArgs(ctx, dstBucket, dstObject); err != nil {
|
||||||
|
return copyObjectPlan{}, err
|
||||||
|
}
|
||||||
|
srcObject, dstObject = encodeDirObject(srcObject), encodeDirObject(dstObject)
|
||||||
|
sameObject := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
|
||||||
|
reference := sameObject && !copyRewritesObjectData(srcInfo.metadataOnly && !srcInfo.Legacy, srcOpts, dstOpts)
|
||||||
|
inPlace := reference && (dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID || !dstOpts.Versioned && srcOpts.VersionID == "")
|
||||||
|
if inPlace {
|
||||||
|
// Metadata must stay with the authoritative source bytes, including
|
||||||
|
// when that pool is draining. Write allocation could select a stale
|
||||||
|
// duplicate and later retire the only current copy.
|
||||||
|
sourceOpts := srcOpts
|
||||||
|
sourceOpts.NoLock = true
|
||||||
|
sourceOpts.Transition = dstOpts.Transition
|
||||||
|
_, idx, err := z.getLatestObjectInfoWithIdx(ctx, srcBucket, srcObject, sourceOpts)
|
||||||
|
return copyObjectPlan{poolIdx: idx}, err
|
||||||
|
}
|
||||||
|
idx, err := z.getPoolIdxNoLock(ctx, dstBucket, dstObject, srcInfo.Size)
|
||||||
|
if err != nil {
|
||||||
|
return copyObjectPlan{}, err
|
||||||
|
}
|
||||||
|
plan := copyObjectPlan{poolIdx: idx}
|
||||||
|
if reference && !z.SinglePool() {
|
||||||
|
sourceOpts := srcOpts
|
||||||
|
sourceOpts.NoLock = true
|
||||||
|
_, sourcePool, err := z.getLatestObjectInfoWithIdx(ctx, srcBucket, srcObject, sourceOpts)
|
||||||
|
if err != nil {
|
||||||
|
return copyObjectPlan{}, err
|
||||||
|
}
|
||||||
|
plan.rewrite = sourcePool != idx
|
||||||
|
}
|
||||||
|
return plan, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
|
func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
|
||||||
if err := checkCopyObjArgs(ctx, srcBucket, srcObject); err != nil {
|
if err := checkCopyObjArgs(ctx, srcBucket, srcObject); err != nil {
|
||||||
return ObjectInfo{}, err
|
return ObjectInfo{}, err
|
||||||
@@ -1714,6 +1762,15 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
|
|||||||
dstOpts.NoLock = true
|
dstOpts.NoLock = true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if dstOpts.copyPlan == nil {
|
||||||
|
plan, err := z.prepareCopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
|
||||||
|
if err != nil {
|
||||||
|
return ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
dstOpts.copyPlan = &plan
|
||||||
|
}
|
||||||
|
poolIdx := dstOpts.copyPlan.poolIdx
|
||||||
|
|
||||||
if !z.SinglePool() && cpSrcDstSame && srcInfo.metadataOnly && dstOpts.ReplicaLockReconcile && srcOpts.VersionID == dstOpts.VersionID {
|
if !z.SinglePool() && cpSrcDstSame && srcInfo.metadataOnly && dstOpts.ReplicaLockReconcile && srcOpts.VersionID == dstOpts.VersionID {
|
||||||
copies, err := z.metadataPoolInfos(ctx, dstBucket, dstObject, dstOpts)
|
copies, err := z.metadataPoolInfos(ctx, dstBucket, dstObject, dstOpts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1722,7 +1779,8 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
|
|||||||
stored := mergedPoolObjectInfo(copies)
|
stored := mergedPoolObjectInfo(copies)
|
||||||
reconcileStoredObjectLock(srcInfo.UserDefined, storedObjectLockState(stored.UserDefined))
|
reconcileStoredObjectLock(srcInfo.UserDefined, storedObjectLockState(stored.UserDefined))
|
||||||
reconcileStoredObjectTags(srcInfo.UserDefined, stored.UserTags, stored.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp])
|
reconcileStoredObjectTags(srcInfo.UserDefined, stored.UserTags, stored.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp])
|
||||||
idx := copies[0].Index
|
dstOpts.replicaObjectInfo = z.replicaObjectInfo
|
||||||
|
idx := poolIdx
|
||||||
oi, err := z.serverPools[idx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
|
oi, err := z.serverPools[idx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
err = z.retireReplicaCopies(ctx, dstBucket, dstObject, idx, oi)
|
err = z.retireReplicaCopies(ctx, dstBucket, dstObject, idx, oi)
|
||||||
@@ -1730,25 +1788,9 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
|
|||||||
return oi, err
|
return oi, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var poolIdx int
|
if !z.SinglePool() {
|
||||||
if dstOpts.MetadataChg && cpSrcDstSame && srcInfo.metadataOnly && srcOpts.VersionID == dstOpts.VersionID {
|
dstOpts.replicaContentGuard = z.guardReplicaContent
|
||||||
// Restore status updates address an existing version. Keep ordinary
|
|
||||||
// copy/write allocation unchanged, including its latest-version lookup.
|
|
||||||
poolIdx, err = z.getPoolIdxExistingWithOpts(ctx, dstBucket, dstObject, ObjectOptions{
|
|
||||||
VersionID: dstOpts.VersionID,
|
|
||||||
MetadataChg: true,
|
|
||||||
NoLock: true,
|
|
||||||
Transition: dstOpts.Transition,
|
|
||||||
SkipDecommissioned: true,
|
|
||||||
SkipRebalancing: true,
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
poolIdx, err = z.getPoolIdxNoLock(ctx, dstBucket, dstObject, srcInfo.Size)
|
|
||||||
}
|
}
|
||||||
if err != nil {
|
|
||||||
return objInfo, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if !z.SinglePool() && (dstOpts.ReplicaLockReconcile || !dstOpts.ReplicationRequest) {
|
if !z.SinglePool() && (dstOpts.ReplicaLockReconcile || !dstOpts.ReplicationRequest) {
|
||||||
dstOpts.replicaObjectInfo = z.replicaObjectInfo
|
dstOpts.replicaObjectInfo = z.replicaObjectInfo
|
||||||
if dstOpts.ReplicaLockReconcile && !dstOpts.DataMovement {
|
if dstOpts.ReplicaLockReconcile && !dstOpts.DataMovement {
|
||||||
@@ -1760,9 +1802,7 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// CopyObjectHandler predicts the outcome of this decision in
|
if cpSrcDstSame && !copyRewritesObjectData(srcInfo.metadataOnly && !srcInfo.Legacy, srcOpts, dstOpts) {
|
||||||
// copyRewritesObjectData(); keep the two in sync.
|
|
||||||
if cpSrcDstSame && srcInfo.metadataOnly {
|
|
||||||
// Version ID is set for the destination and source == destination version ID.
|
// Version ID is set for the destination and source == destination version ID.
|
||||||
if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID {
|
if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID {
|
||||||
return z.serverPools[poolIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
|
return z.serverPools[poolIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
|
||||||
@@ -1784,6 +1824,10 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if srcInfo.PutObjReader == nil {
|
||||||
|
return ObjectInfo{}, errInvalidArgument
|
||||||
|
}
|
||||||
|
|
||||||
putOpts := ObjectOptions{
|
putOpts := ObjectOptions{
|
||||||
ServerSideEncryption: dstOpts.ServerSideEncryption,
|
ServerSideEncryption: dstOpts.ServerSideEncryption,
|
||||||
UserDefined: srcInfo.UserDefined,
|
UserDefined: srcInfo.UserDefined,
|
||||||
@@ -1793,12 +1837,15 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
|
|||||||
MTime: dstOpts.MTime,
|
MTime: dstOpts.MTime,
|
||||||
NoLock: true,
|
NoLock: true,
|
||||||
EncryptFn: dstOpts.EncryptFn,
|
EncryptFn: dstOpts.EncryptFn,
|
||||||
|
IndexCB: dstOpts.IndexCB,
|
||||||
WantChecksum: dstOpts.WantChecksum,
|
WantChecksum: dstOpts.WantChecksum,
|
||||||
WantServerSideChecksumType: dstOpts.WantServerSideChecksumType,
|
WantServerSideChecksumType: dstOpts.WantServerSideChecksumType,
|
||||||
|
PreserveETag: dstOpts.PreserveETag,
|
||||||
ReplicationRequest: dstOpts.ReplicationRequest,
|
ReplicationRequest: dstOpts.ReplicationRequest,
|
||||||
ReplicaLockReconcile: dstOpts.ReplicaLockReconcile,
|
ReplicaLockReconcile: dstOpts.ReplicaLockReconcile,
|
||||||
DataMovement: dstOpts.DataMovement,
|
DataMovement: dstOpts.DataMovement,
|
||||||
replicaObjectInfo: dstOpts.replicaObjectInfo,
|
replicaObjectInfo: dstOpts.replicaObjectInfo,
|
||||||
|
replicaContentGuard: dstOpts.replicaContentGuard,
|
||||||
}
|
}
|
||||||
|
|
||||||
return z.serverPools[poolIdx].PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
|
return z.serverPools[poolIdx].PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
|
||||||
@@ -2459,21 +2506,21 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
// Several pools take the object lock first, then the upload lock below.
|
// Take the object lock before the upload lock for every pool count.
|
||||||
// A single pool takes the upload lock here and the set takes the object
|
// UploadPartCopy holds its source read lock while it reads upload
|
||||||
// lock, in the object's own erasure set, only right before it commits.
|
// metadata, including when the source and destination are the same key.
|
||||||
if !z.SinglePool() {
|
if !opts.NoLock {
|
||||||
if !opts.NoLock {
|
lk := z.NewObjectNSLock(bucket, encodeDirObject(object))
|
||||||
lk := z.NewObjectNSLock(bucket, encodeDirObject(object))
|
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||||
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
if err != nil {
|
||||||
if err != nil {
|
return objInfo, err
|
||||||
return objInfo, err
|
|
||||||
}
|
|
||||||
ctx = lkctx.Context()
|
|
||||||
defer lk.Unlock(lkctx)
|
|
||||||
}
|
}
|
||||||
opts.NoLock = true
|
ctx = lkctx.Context()
|
||||||
|
defer lk.Unlock(lkctx)
|
||||||
|
}
|
||||||
|
opts.NoLock = true
|
||||||
|
|
||||||
|
if !z.SinglePool() {
|
||||||
// A conditional completion must be evaluated against the logical
|
// A conditional completion must be evaluated against the logical
|
||||||
// latest object across pools, under the object write lock held for
|
// latest object across pools, under the object write lock held for
|
||||||
// this operation. The pool hosting the upload may only hold a stale
|
// this operation. The pool hosting the upload may only hold a stale
|
||||||
@@ -2524,6 +2571,7 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
|
|||||||
opts.replicaObjectInfo = z.replicaObjectInfo
|
opts.replicaObjectInfo = z.replicaObjectInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
|
opts.replicaContentGuard = z.guardReplicaContent
|
||||||
for idx, pool := range z.serverPools {
|
for idx, pool := range z.serverPools {
|
||||||
if z.IsSuspended(idx) {
|
if z.IsSuspended(idx) {
|
||||||
continue
|
continue
|
||||||
@@ -3367,7 +3415,7 @@ func (z *erasureServerPools) PutObjectTags(ctx context.Context, bucket, object s
|
|||||||
// revision beyond all copies, so the returned value and every pool agree.
|
// revision beyond all copies, so the returned value and every pool agree.
|
||||||
if stamp := opts.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]; stamp != "" {
|
if stamp := opts.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]; stamp != "" {
|
||||||
for _, copy := range copies {
|
for _, copy := range copies {
|
||||||
stamp = monotonicTaggingTimestamp(stamp, copy.ObjInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp])
|
stamp = monotonicMetadataTimestamp(stamp, copy.ObjInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp])
|
||||||
}
|
}
|
||||||
opts.UserDefined = cloneMSS(opts.UserDefined)
|
opts.UserDefined = cloneMSS(opts.UserDefined)
|
||||||
opts.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = stamp
|
opts.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = stamp
|
||||||
|
|||||||
+7
-1
@@ -844,7 +844,7 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
|
|||||||
// Check if this request is only metadata update.
|
// Check if this request is only metadata update.
|
||||||
// CopyObjectHandler predicts the outcome of this decision in
|
// CopyObjectHandler predicts the outcome of this decision in
|
||||||
// copyRewritesObjectData(); keep the two in sync.
|
// copyRewritesObjectData(); keep the two in sync.
|
||||||
if cpSrcDstSame && srcInfo.metadataOnly {
|
if cpSrcDstSame && srcInfo.metadataOnly && !srcInfo.Legacy {
|
||||||
// Version ID is set for the destination and source == destination version ID.
|
// Version ID is set for the destination and source == destination version ID.
|
||||||
// perform an in-place update.
|
// perform an in-place update.
|
||||||
if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID {
|
if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID {
|
||||||
@@ -868,6 +868,10 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if srcInfo.PutObjReader == nil {
|
||||||
|
return ObjectInfo{}, errInvalidArgument
|
||||||
|
}
|
||||||
|
|
||||||
putOpts := ObjectOptions{
|
putOpts := ObjectOptions{
|
||||||
ServerSideEncryption: dstOpts.ServerSideEncryption,
|
ServerSideEncryption: dstOpts.ServerSideEncryption,
|
||||||
UserDefined: srcInfo.UserDefined,
|
UserDefined: srcInfo.UserDefined,
|
||||||
@@ -879,10 +883,12 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
|
|||||||
EncryptFn: dstOpts.EncryptFn,
|
EncryptFn: dstOpts.EncryptFn,
|
||||||
WantChecksum: dstOpts.WantChecksum,
|
WantChecksum: dstOpts.WantChecksum,
|
||||||
WantServerSideChecksumType: dstOpts.WantServerSideChecksumType,
|
WantServerSideChecksumType: dstOpts.WantServerSideChecksumType,
|
||||||
|
PreserveETag: dstOpts.PreserveETag,
|
||||||
ReplicationRequest: dstOpts.ReplicationRequest,
|
ReplicationRequest: dstOpts.ReplicationRequest,
|
||||||
ReplicaLockReconcile: dstOpts.ReplicaLockReconcile,
|
ReplicaLockReconcile: dstOpts.ReplicaLockReconcile,
|
||||||
DataMovement: dstOpts.DataMovement,
|
DataMovement: dstOpts.DataMovement,
|
||||||
replicaObjectInfo: dstOpts.replicaObjectInfo,
|
replicaObjectInfo: dstOpts.replicaObjectInfo,
|
||||||
|
replicaContentGuard: dstOpts.replicaContentGuard,
|
||||||
}
|
}
|
||||||
|
|
||||||
return dstSet.putObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
|
return dstSet.putObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
|
||||||
|
|||||||
@@ -0,0 +1,196 @@
|
|||||||
|
// Copyright (c) 2026 PGSTY
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/zip"
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/madmin-go/v3"
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
|
)
|
||||||
|
|
||||||
|
func iamImportArchive(t *testing.T, assets map[string]any) []byte {
|
||||||
|
t.Helper()
|
||||||
|
var buf bytes.Buffer
|
||||||
|
zw := zip.NewWriter(&buf)
|
||||||
|
for name, value := range assets {
|
||||||
|
w, err := zw.Create(pathJoin(iamAssetsDir, name))
|
||||||
|
mustIAM(t, err)
|
||||||
|
mustIAM(t, json.NewEncoder(w).Encode(value))
|
||||||
|
}
|
||||||
|
mustIAM(t, zw.Close())
|
||||||
|
return buf.Bytes()
|
||||||
|
}
|
||||||
|
|
||||||
|
func iamImportCall(t *testing.T, data []byte) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut, "http://localhost/minio/admin/v3/import-iam-v2", int64(len(data)), bytes.NewReader(data), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
||||||
|
mustIAM(t, err)
|
||||||
|
mustIAM(t, req.ParseForm())
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
adminAPIHandlers{}.ImportIAMV2(rec, req.WithContext(newContext(req, rec, "ImportIAM")))
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("import-iam-v2: %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
return rec
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIAMImportExportState(t *testing.T) {
|
||||||
|
for _, target := range []string{"fresh", "existing"} {
|
||||||
|
t.Run(target, func(t *testing.T) {
|
||||||
|
ctx, sys := iamAdminFixture(t)
|
||||||
|
parent := iamAdminUser(ctx, t, sys, "import-parent", "")
|
||||||
|
for _, status := range []string{statusDisabled, statusEnabled} {
|
||||||
|
group := "import-" + status
|
||||||
|
_, err := sys.AddUsersToGroup(ctx, group, []string{parent.AccessKey})
|
||||||
|
mustIAM(t, err)
|
||||||
|
_, err = sys.SetGroupStatus(ctx, group, status == statusEnabled)
|
||||||
|
mustIAM(t, err)
|
||||||
|
}
|
||||||
|
_, err := sys.PolicyDBSet(ctx, "import-disabled", "readwrite", regUser, true)
|
||||||
|
mustIAM(t, err)
|
||||||
|
sp, err := policy.ParseConfig(strings.NewReader(svcAccEditPolicy))
|
||||||
|
mustIAM(t, err)
|
||||||
|
expires := UTCNow().Add(time.Hour).Truncate(time.Second)
|
||||||
|
_, _, err = sys.NewServiceAccount(ctx, parent.AccessKey, nil, newServiceAccountOpts{
|
||||||
|
accessKey: "import-service", secretKey: "import-service-secret", status: auth.AccountOff,
|
||||||
|
sessionPolicy: sp, expiration: &expires, name: "disabled service", claims: map[string]any{"identity-attribute": "preserved"},
|
||||||
|
})
|
||||||
|
mustIAM(t, err)
|
||||||
|
exported := iamAdminCall(t, globalActiveCred, http.MethodGet, "/minio/admin/v3/export-iam", nil, nil, adminAPIHandlers{}.ExportIAM)
|
||||||
|
if exported.Code != http.StatusOK {
|
||||||
|
t.Fatalf("export-iam: %d %s", exported.Code, exported.Body.String())
|
||||||
|
}
|
||||||
|
if target == "fresh" {
|
||||||
|
ctx, sys = iamAdminFixture(t)
|
||||||
|
} else {
|
||||||
|
// Import must also replace an existing enabled account/group and
|
||||||
|
// restore an enabled group that is disabled on the target.
|
||||||
|
_, err = sys.UpdateServiceAccount(ctx, "import-service", updateServiceAccountOpts{status: auth.AccountOn})
|
||||||
|
mustIAM(t, err)
|
||||||
|
_, err = sys.SetGroupStatus(ctx, "import-disabled", true)
|
||||||
|
mustIAM(t, err)
|
||||||
|
_, err = sys.SetGroupStatus(ctx, "import-enabled", false)
|
||||||
|
mustIAM(t, err)
|
||||||
|
}
|
||||||
|
rec := iamImportCall(t, exported.Body.Bytes())
|
||||||
|
var result madmin.ImportIAMResult
|
||||||
|
mustIAM(t, json.Unmarshal(rec.Body.Bytes(), &result))
|
||||||
|
if len(result.Failed.Groups) != 0 || len(result.Failed.ServiceAccounts) != 0 || len(result.Added.ServiceAccounts) != 1 {
|
||||||
|
t.Fatalf("import did not restore the exported entities: %s", rec.Body.String())
|
||||||
|
}
|
||||||
|
mustIAM(t, sys.store.LoadIAMCache(ctx, false))
|
||||||
|
for _, status := range []string{statusDisabled, statusEnabled} {
|
||||||
|
group, err := sys.GetGroupDescription("import-" + status)
|
||||||
|
mustIAM(t, err)
|
||||||
|
if group.Status != status || len(group.Members) != 1 || group.Members[0] != parent.AccessKey {
|
||||||
|
t.Fatalf("group status or members changed: %+v", group)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if sys.IsAllowed(policy.Args{AccountName: parent.AccessKey, Action: policy.GetObjectAction, BucketName: "edit", ObjectName: "object"}) {
|
||||||
|
t.Error("the disabled imported group grants access")
|
||||||
|
}
|
||||||
|
cr, gotPolicy, err := sys.GetServiceAccount(ctx, "import-service")
|
||||||
|
mustIAM(t, err)
|
||||||
|
if cr.Status != auth.AccountOff || !cr.Expiration.Equal(expires) || cr.Name != "disabled service" || gotPolicy == nil || !gotPolicy.Equals(*sp) {
|
||||||
|
t.Fatal("import changed the service account status, expiration, name or inline policy")
|
||||||
|
}
|
||||||
|
if _, valid, err := sys.CheckKey(ctx, "import-service"); err != nil || valid {
|
||||||
|
t.Fatalf("imported disabled credential is usable: valid=%v err=%v", valid, err)
|
||||||
|
}
|
||||||
|
var stored UserIdentity
|
||||||
|
mustIAM(t, sys.store.loadIAMConfig(ctx, &stored, getUserIdentityPath("import-service", svcUser)))
|
||||||
|
raw, err := auth.ExtractClaims(stored.Credentials.SessionToken, stored.Credentials.SecretKey)
|
||||||
|
mustIAM(t, err)
|
||||||
|
if _, ok := raw.Map()[sessionPolicyNameExtracted]; ok || raw.Map()["identity-attribute"] != "preserved" {
|
||||||
|
t.Fatal("import polluted the raw JWT or lost identity claims")
|
||||||
|
}
|
||||||
|
// Old exports may contain a derived claim and omit the policy version.
|
||||||
|
snapshot, err := serviceAccountSnapshot(stored)
|
||||||
|
mustIAM(t, err)
|
||||||
|
snapshot.Claims[sessionPolicyNameExtracted] = "stale decoded policy"
|
||||||
|
snapshot.SessionPolicy = []byte(strings.ReplaceAll(svcAccEditPolicy, `"Version":"2012-10-17",`, ""))
|
||||||
|
rec = iamImportCall(t, iamImportArchive(t, map[string]any{allSvcAcctsFile: map[string]madmin.SRSvcAccCreate{"import-service": snapshot}}))
|
||||||
|
mustIAM(t, json.Unmarshal(rec.Body.Bytes(), &result))
|
||||||
|
if len(result.Failed.ServiceAccounts) != 0 {
|
||||||
|
t.Fatalf("legacy snapshot import failed: %s", rec.Body.String())
|
||||||
|
}
|
||||||
|
_, gotPolicy, err = sys.GetServiceAccount(ctx, "import-service")
|
||||||
|
mustIAM(t, err)
|
||||||
|
if gotPolicy == nil || !gotPolicy.Equals(*sp) {
|
||||||
|
t.Fatal("import did not normalize the inline policy")
|
||||||
|
}
|
||||||
|
mustIAM(t, sys.store.loadIAMConfig(ctx, &stored, getUserIdentityPath("import-service", svcUser)))
|
||||||
|
raw, err = auth.ExtractClaims(stored.Credentials.SessionToken, stored.Credentials.SecretKey)
|
||||||
|
mustIAM(t, err)
|
||||||
|
if _, ok := raw.Map()[sessionPolicyNameExtracted]; ok {
|
||||||
|
t.Fatal("legacy import signed a derived claim into the JWT")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type iamImportGroupFailureStore struct {
|
||||||
|
IAMStorageAPI
|
||||||
|
statusOnly bool
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *iamImportGroupFailureStore) saveIAMConfig(ctx context.Context, item any, path string, opts ...options) error {
|
||||||
|
mutation, _ := ctx.Value(iamGroupMutationKey{}).(iamGroupMutation)
|
||||||
|
if path == getGroupInfoPath("import-failure") && mutation.StatusOnly == s.statusOnly {
|
||||||
|
return s.err
|
||||||
|
}
|
||||||
|
return s.IAMStorageAPI.saveIAMConfig(ctx, item, path, opts...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIAMImportGroupFailure(t *testing.T) {
|
||||||
|
for _, operation := range []string{"members", "status"} {
|
||||||
|
t.Run(operation, func(t *testing.T) {
|
||||||
|
_, sys := iamAdminFixture(t)
|
||||||
|
injected := madmin.ErrorResponse{Code: "InjectedGroupWrite", Message: operation + " write failed"}
|
||||||
|
sys.store.IAMStorageAPI = &iamImportGroupFailureStore{IAMStorageAPI: sys.store.IAMStorageAPI, statusOnly: operation == "status", err: injected}
|
||||||
|
rec := iamImportCall(t, iamImportArchive(t, map[string]any{allGroupsFile: map[string]GroupInfo{
|
||||||
|
"import-failure": {Status: statusDisabled},
|
||||||
|
}}))
|
||||||
|
var result struct {
|
||||||
|
Added madmin.IAMEntities `json:"added"`
|
||||||
|
Failed struct {
|
||||||
|
Groups []struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Error json.RawMessage `json:"error"`
|
||||||
|
} `json:"groups"`
|
||||||
|
} `json:"failed"`
|
||||||
|
}
|
||||||
|
mustIAM(t, json.Unmarshal(rec.Body.Bytes(), &result))
|
||||||
|
wantError, err := json.Marshal(injected)
|
||||||
|
mustIAM(t, err)
|
||||||
|
if len(result.Added.Groups) != 0 || len(result.Failed.Groups) != 1 || result.Failed.Groups[0].Name != "import-failure" || !bytes.Equal(result.Failed.Groups[0].Error, wantError) {
|
||||||
|
t.Fatalf("import did not report the actual %s error: %s", operation, rec.Body.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,7 +4,6 @@
|
|||||||
package cmd
|
package cmd
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
@@ -172,14 +171,41 @@ func (c *SiteReplicationSys) replicationItem(ctx context.Context, item madmin.SR
|
|||||||
key = item.SvcAccChange.Update.AccessKey
|
key = item.SvcAccChange.Update.AccessKey
|
||||||
}
|
}
|
||||||
if key != "" {
|
if key != "" {
|
||||||
r, err := loadIAMRevision(ctx, globalIAMSys.store, getUserIdentityPath(key, svcUser))
|
// A create or an edit goes out as the account stored now: the
|
||||||
|
// whole record, under its revision, whatever the caller's item
|
||||||
|
// says. A peer orders items by revision and drops an older one,
|
||||||
|
// so an edit sent as the fields it changed would lose every
|
||||||
|
// earlier edit that did not reach the peer, or reaches it late.
|
||||||
|
// A later write that won before sending goes out in place of
|
||||||
|
// this one.
|
||||||
|
stored := make(map[string]UserIdentity, 1)
|
||||||
|
err := globalIAMSys.store.loadUser(ctx, key, svcUser, stored)
|
||||||
|
if err != nil && !errors.Is(err, errNoSuchUser) {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
u, ok := stored[key]
|
||||||
|
if !ok {
|
||||||
|
// Nothing is sent for an account that is deleted, expired or
|
||||||
|
// of a revoked parent meanwhile; a deletion replicates itself.
|
||||||
|
// A stored token that cannot be read is an error, and the
|
||||||
|
// account is not taken for sent.
|
||||||
|
var raw UserIdentity
|
||||||
|
err := globalIAMSys.store.loadIAMConfig(ctx, &raw, getUserIdentityPath(key, svcUser))
|
||||||
|
if err == nil && !raw.Deleted && !raw.Credentials.IsExpired() {
|
||||||
|
_, err = extractJWTClaims(raw)
|
||||||
|
}
|
||||||
|
if err != nil && !errors.Is(err, errConfigNotFound) {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
return out, errIAMStaleUpdate
|
||||||
|
}
|
||||||
|
snapshot, err := serviceAccountSnapshot(u)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return out, err
|
return out, err
|
||||||
}
|
}
|
||||||
if r.Deleted || r.timestamp().After(item.UpdatedAt) {
|
out.SvcAccChange = &madmin.SRSvcAccChange{Create: &snapshot}
|
||||||
return out, errIAMStaleUpdate
|
out.UpdatedAt = u.UpdatedAt
|
||||||
}
|
out.RevokedBefore = u.RevokedBefore
|
||||||
out.RevokedBefore = r.RevokedBefore
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if item.Type == madmin.SRIAMItemGroupInfo && item.GroupInfo != nil && !item.GroupInfo.UpdateReq.IsRemove {
|
if item.Type == madmin.SRIAMItemGroupInfo && item.GroupInfo != nil && !item.GroupInfo.UpdateReq.IsRemove {
|
||||||
@@ -275,7 +301,7 @@ func applyIAMReplicationItem(ctx context.Context, item iamReplicationItem) error
|
|||||||
if len(item.Policy) == 0 {
|
if len(item.Policy) == 0 {
|
||||||
return globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
|
return globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
|
||||||
}
|
}
|
||||||
p, err := policy.ParseConfig(bytes.NewReader(item.Policy))
|
p, err := parsePolicySnapshotBytes(item.Policy)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
+5
-6
@@ -676,9 +676,8 @@ func (c *SiteReplicationSys) healIAMDeletions(ctx context.Context) (err error) {
|
|||||||
}()
|
}()
|
||||||
c.iamHealMu.Lock()
|
c.iamHealMu.Lock()
|
||||||
defer c.iamHealMu.Unlock()
|
defer c.iamHealMu.Unlock()
|
||||||
c.RLock()
|
state, ok := c.stateSnapshot()
|
||||||
defer c.RUnlock()
|
if !ok {
|
||||||
if !c.enabled {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
snapshot := globalIAMSys.store.revisionIndex().snapshot()
|
snapshot := globalIAMSys.store.revisionIndex().snapshot()
|
||||||
@@ -719,17 +718,17 @@ func (c *SiteReplicationSys) healIAMDeletions(ctx context.Context) (err error) {
|
|||||||
c.iamRevisionProgress = make(map[string]iamRevisionProgress)
|
c.iamRevisionProgress = make(map[string]iamRevisionProgress)
|
||||||
}
|
}
|
||||||
for id := range c.iamRevisionProgress {
|
for id := range c.iamRevisionProgress {
|
||||||
if _, present := c.state.Peers[id]; !present {
|
if _, present := state.Peers[id]; !present {
|
||||||
delete(c.iamRevisionProgress, id)
|
delete(c.iamRevisionProgress, id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
var progressMu sync.Mutex
|
var progressMu sync.Mutex
|
||||||
cerr := c.concDo(nil, func(id string, p madmin.PeerInfo) error {
|
cerr := state.concDo(nil, func(id string, p madmin.PeerInfo) error {
|
||||||
// Bound each pass, but retain acknowledgements independently of the
|
// Bound each pass, but retain acknowledgements independently of the
|
||||||
// pass deadline or unrelated changes at either site.
|
// pass deadline or unrelated changes at either site.
|
||||||
peerCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
|
peerCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
client, err := c.getAdminClient(peerCtx, id)
|
client, err := state.getAdminClient(id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
+7
-20
@@ -20,7 +20,6 @@ package cmd
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"maps"
|
"maps"
|
||||||
@@ -2934,27 +2933,15 @@ func updatedServiceAccountCredentials(ctx context.Context, accessKey string, cr
|
|||||||
// the embedded one stays. Only a policy that was given, and is empty,
|
// the embedded one stays. Only a policy that was given, and is empty,
|
||||||
// removes it: the account then has all its parent is allowed, which is
|
// removes it: the account then has all its parent is allowed, which is
|
||||||
// usually more.
|
// usually more.
|
||||||
reset := opts.sessionPolicy != nil && opts.sessionPolicy.Version == "" && len(opts.sessionPolicy.Statements) == 0
|
if opts.sessionPolicy != nil {
|
||||||
if _, ok := m.Lookup(policy.SessionPolicyName); ok && reset {
|
policyBuf, err := marshalServiceAccountPolicy(opts.sessionPolicy)
|
||||||
m.Delete(policy.SessionPolicyName)
|
if err != nil {
|
||||||
m.Set(iamPolicyClaimNameSA(), inheritedPolicyType)
|
|
||||||
}
|
|
||||||
|
|
||||||
if opts.sessionPolicy != nil { // session policies is being updated
|
|
||||||
if err := opts.sessionPolicy.Validate(); err != nil {
|
|
||||||
return cr, err
|
return cr, err
|
||||||
}
|
}
|
||||||
|
if len(policyBuf) == 0 {
|
||||||
if opts.sessionPolicy.Version != "" && len(opts.sessionPolicy.Statements) > 0 {
|
m.Delete(policy.SessionPolicyName)
|
||||||
policyBuf, err := json.Marshal(opts.sessionPolicy)
|
m.Set(iamPolicyClaimNameSA(), inheritedPolicyType)
|
||||||
if err != nil {
|
} else {
|
||||||
return cr, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(policyBuf) > maxSVCSessionPolicySize {
|
|
||||||
return cr, errSessionPolicyTooLarge
|
|
||||||
}
|
|
||||||
|
|
||||||
// Overwrite session policy claims.
|
// Overwrite session policy claims.
|
||||||
m.Set(policy.SessionPolicyName, base64.StdEncoding.EncodeToString(policyBuf))
|
m.Set(policy.SessionPolicyName, base64.StdEncoding.EncodeToString(policyBuf))
|
||||||
m.Set(iamPolicyClaimNameSA(), embeddedPolicyType)
|
m.Set(iamPolicyClaimNameSA(), embeddedPolicyType)
|
||||||
|
|||||||
@@ -4,9 +4,12 @@
|
|||||||
package cmd
|
package cmd
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"reflect"
|
||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -25,6 +28,133 @@ import (
|
|||||||
|
|
||||||
const svcAccEditPolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::edit/*"]}]}`
|
const svcAccEditPolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::edit/*"]}]}`
|
||||||
|
|
||||||
|
func TestIAMServiceAccountPolicySemantics(t *testing.T) {
|
||||||
|
for _, expiring := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprintf("expiring=%v", expiring), func(t *testing.T) {
|
||||||
|
f := newGroupRefreshFixture(t)
|
||||||
|
readPolicy, err := policy.ParseConfig(strings.NewReader(svcAccEditPolicy))
|
||||||
|
mustIAM(t, err)
|
||||||
|
writePolicy, err := policy.ParseConfig(strings.NewReader(strings.ReplaceAll(svcAccEditPolicy, "s3:GetObject", "s3:PutObject")))
|
||||||
|
mustIAM(t, err)
|
||||||
|
_, err = f.sys.PolicyDBSet(f.ctx, f.parent.AccessKey, "readwrite", regUser, false)
|
||||||
|
mustIAM(t, err)
|
||||||
|
for _, action := range []policy.Action{policy.GetObjectAction, policy.PutObjectAction} {
|
||||||
|
if !f.sys.IsAllowed(policy.Args{AccountName: f.parent.AccessKey, Action: action, BucketName: "edit", ObjectName: "object"}) {
|
||||||
|
t.Fatal("control: the parent must allow reads and writes to the test object")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
noVersion, invalidVersion, oversized := *writePolicy, *writePolicy, *writePolicy
|
||||||
|
noVersion.Version = ""
|
||||||
|
invalidVersion.Version = "2008-10-17"
|
||||||
|
oversized.ID = policy.ID(strings.Repeat("x", maxSVCSessionPolicySize))
|
||||||
|
var expiration *time.Time
|
||||||
|
if expiring {
|
||||||
|
later := UTCNow().Add(time.Hour)
|
||||||
|
expiration = &later
|
||||||
|
}
|
||||||
|
for i, tc := range []struct {
|
||||||
|
name string
|
||||||
|
input *policy.Policy
|
||||||
|
invalid bool
|
||||||
|
}{
|
||||||
|
{"nil", nil, false},
|
||||||
|
{"empty", &policy.Policy{}, false},
|
||||||
|
{"empty-statements", &policy.Policy{Statements: []policy.Statement{}}, false},
|
||||||
|
{"empty-with-version", &policy.Policy{Version: policy.DefaultVersion, Statements: []policy.Statement{}}, false},
|
||||||
|
{"missing-version", &noVersion, false},
|
||||||
|
{"version", writePolicy, false},
|
||||||
|
{"invalid-version", &invalidVersion, true},
|
||||||
|
{"empty-invalid-version", &policy.Policy{Version: "2008-10-17"}, true},
|
||||||
|
{"invalid-statement", &policy.Policy{Statements: []policy.Statement{{}}}, true},
|
||||||
|
{"oversized", &oversized, true},
|
||||||
|
} {
|
||||||
|
for _, edit := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprintf("%s/edit=%v", tc.name, edit), func(t *testing.T) {
|
||||||
|
key := fmt.Sprintf("policy-%d-%v", i, edit)
|
||||||
|
inputBefore, err := json.Marshal(tc.input)
|
||||||
|
mustIAM(t, err)
|
||||||
|
var before UserIdentity
|
||||||
|
if edit {
|
||||||
|
_, _, err = f.sys.NewServiceAccount(f.ctx, f.parent.AccessKey, nil, newServiceAccountOpts{
|
||||||
|
accessKey: key, secretKey: "policy-valid-secret", sessionPolicy: readPolicy, expiration: expiration,
|
||||||
|
})
|
||||||
|
mustIAM(t, err)
|
||||||
|
mustIAM(t, f.sys.store.loadIAMConfig(f.ctx, &before, getUserIdentityPath(key, svcUser)))
|
||||||
|
_, err = f.sys.UpdateServiceAccount(f.ctx, key, updateServiceAccountOpts{sessionPolicy: tc.input, description: "edited"})
|
||||||
|
} else {
|
||||||
|
// Snapshots can carry a stale copy of the policy in claims.
|
||||||
|
var claims map[string]any
|
||||||
|
if tc.input != nil {
|
||||||
|
claims = map[string]any{policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(svcAccEditPolicy))}
|
||||||
|
}
|
||||||
|
_, _, err = f.sys.NewServiceAccount(f.ctx, f.parent.AccessKey, nil, newServiceAccountOpts{
|
||||||
|
accessKey: key, secretKey: "policy-valid-secret", sessionPolicy: tc.input, expiration: expiration,
|
||||||
|
claims: claims,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if (err != nil) != tc.invalid {
|
||||||
|
t.Fatalf("operation error = %v, want rejected=%v", err, tc.invalid)
|
||||||
|
}
|
||||||
|
inputAfter, err := json.Marshal(tc.input)
|
||||||
|
mustIAM(t, err)
|
||||||
|
if !bytes.Equal(inputBefore, inputAfter) {
|
||||||
|
t.Fatal("operation changed the caller's policy")
|
||||||
|
}
|
||||||
|
var stored UserIdentity
|
||||||
|
loadErr := f.sys.store.loadIAMConfig(f.ctx, &stored, getUserIdentityPath(key, svcUser))
|
||||||
|
if tc.invalid {
|
||||||
|
if edit {
|
||||||
|
mustIAM(t, loadErr)
|
||||||
|
if !reflect.DeepEqual(stored, before) {
|
||||||
|
t.Fatal("rejected edit changed the stored account")
|
||||||
|
}
|
||||||
|
} else if loadErr != errConfigNotFound {
|
||||||
|
t.Fatalf("rejected create left an account: %v", loadErr)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
mustIAM(t, loadErr)
|
||||||
|
claims, err := auth.ExtractClaims(stored.Credentials.SessionToken, stored.Credentials.SecretKey)
|
||||||
|
mustIAM(t, err)
|
||||||
|
if _, ok := claims.Map()[sessionPolicyNameExtracted]; ok {
|
||||||
|
t.Fatal("raw JWT contains the derived policy claim")
|
||||||
|
}
|
||||||
|
var want *policy.Policy
|
||||||
|
if tc.input != nil && !tc.input.IsEmpty() {
|
||||||
|
want = writePolicy
|
||||||
|
} else if tc.input == nil && edit {
|
||||||
|
want = readPolicy
|
||||||
|
}
|
||||||
|
got, err := embeddedServiceAccountPolicy(claims)
|
||||||
|
mustIAM(t, err)
|
||||||
|
if (got == nil) != (want == nil) || (got != nil && !got.Equals(*want)) {
|
||||||
|
t.Fatalf("stored policy = %+v, want %+v", got, want)
|
||||||
|
}
|
||||||
|
if want == nil {
|
||||||
|
if _, ok := claims.Lookup(policy.SessionPolicyName); ok || claims.Map()[iamPolicyClaimNameSA()] != inheritedPolicyType {
|
||||||
|
t.Fatal("inheritance must remove the embedded policy claim")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
mustIAM(t, f.sys.store.LoadIAMCache(f.ctx, false))
|
||||||
|
u, valid, err := f.sys.CheckKey(f.ctx, key)
|
||||||
|
mustIAM(t, err)
|
||||||
|
if !valid || u.Credentials.IsTemp() != expiring {
|
||||||
|
t.Fatal("credential validity or expiration changed")
|
||||||
|
}
|
||||||
|
for _, action := range []policy.Action{policy.GetObjectAction, policy.PutObjectAction} {
|
||||||
|
allowed := f.sys.IsAllowed(policy.Args{AccountName: key, Claims: u.Credentials.Claims, Action: action, BucketName: "edit", ObjectName: "object"})
|
||||||
|
wantAllowed := want == nil || (want == readPolicy && action == policy.GetObjectAction) || (want == writePolicy && action == policy.PutObjectAction)
|
||||||
|
if allowed != wantAllowed {
|
||||||
|
t.Errorf("%s allowed=%v, want %v", action, allowed, wantAllowed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestIAMServiceAccountEditKeepsPolicy(t *testing.T) {
|
func TestIAMServiceAccountEditKeepsPolicy(t *testing.T) {
|
||||||
f := newGroupRefreshFixture(t)
|
f := newGroupRefreshFixture(t)
|
||||||
want, err := policy.ParseConfig(strings.NewReader(svcAccEditPolicy))
|
want, err := policy.ParseConfig(strings.NewReader(svcAccEditPolicy))
|
||||||
@@ -110,6 +240,7 @@ func TestIAMServiceAccountEditKeepsPolicy(t *testing.T) {
|
|||||||
{"peer-status", madmin.SRSvcAccUpdate{Status: auth.AccountOff}, true, func(c auth.Credentials) bool { return c.Status == auth.AccountOff }},
|
{"peer-status", madmin.SRSvcAccUpdate{Status: auth.AccountOff}, true, func(c auth.Credentials) bool { return c.Status == auth.AccountOff }},
|
||||||
{"peer-null", madmin.SRSvcAccUpdate{Description: "described", SessionPolicy: madmin.SRSessionPolicy("null")}, true, func(c auth.Credentials) bool { return c.Description == "described" }},
|
{"peer-null", madmin.SRSvcAccUpdate{Description: "described", SessionPolicy: madmin.SRSessionPolicy("null")}, true, func(c auth.Credentials) bool { return c.Description == "described" }},
|
||||||
{"peer-empty", madmin.SRSvcAccUpdate{SessionPolicy: madmin.SRSessionPolicy(`{"Statement":[]}`)}, false, nil},
|
{"peer-empty", madmin.SRSvcAccUpdate{SessionPolicy: madmin.SRSessionPolicy(`{"Statement":[]}`)}, false, nil},
|
||||||
|
{"peer-empty-v", madmin.SRSvcAccUpdate{SessionPolicy: madmin.SRSessionPolicy(`{"Version":"2012-10-17","Statement":[]}`)}, false, nil},
|
||||||
} {
|
} {
|
||||||
t.Run(edit.name, func(t *testing.T) {
|
t.Run(edit.name, func(t *testing.T) {
|
||||||
key := "edit-" + edit.name
|
key := "edit-" + edit.name
|
||||||
|
|||||||
@@ -0,0 +1,578 @@
|
|||||||
|
// Copyright (c) 2026 PGSTY
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/zip"
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"path"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/madmin-go/v3"
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Site replication sends a service account as the record its site has
|
||||||
|
// stored, whole. A peer that applies the last item it was sent then has
|
||||||
|
// the account of the sending site, whatever it received or lost before,
|
||||||
|
// and whatever the sending server holds in its cache.
|
||||||
|
|
||||||
|
const svcAccSnapshotPolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::snapshot/*"]}]}`
|
||||||
|
|
||||||
|
// svcAccRecord is what a site has of a service account.
|
||||||
|
type svcAccRecord struct {
|
||||||
|
Parent, Secret, Status, Name, Description, Policy string
|
||||||
|
Expiration, UpdatedAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// svcAccPolicyText is a policy document in the one form it is compared in;
|
||||||
|
// an account without an embedded policy has "".
|
||||||
|
func svcAccPolicyText(t *testing.T, document []byte) string {
|
||||||
|
t.Helper()
|
||||||
|
if len(document) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
p, err := policy.ParseConfig(bytes.NewReader(document))
|
||||||
|
mustIAM(t, err)
|
||||||
|
text, err := json.Marshal(p)
|
||||||
|
mustIAM(t, err)
|
||||||
|
return string(text)
|
||||||
|
}
|
||||||
|
|
||||||
|
// svcAccStored is the account as a server that starts now loads it.
|
||||||
|
func svcAccStored(t *testing.T, f *groupRefreshFixture, key string) svcAccRecord {
|
||||||
|
t.Helper()
|
||||||
|
u, ok := f.stored(t, key)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("service account %s is not stored", key)
|
||||||
|
}
|
||||||
|
cr := u.Credentials
|
||||||
|
rawClaims, err := auth.ExtractClaims(cr.SessionToken, cr.SecretKey)
|
||||||
|
mustIAM(t, err)
|
||||||
|
if _, ok := rawClaims.Map()[sessionPolicyNameExtracted]; ok {
|
||||||
|
t.Fatal("stored JWT contains the derived policy claim")
|
||||||
|
}
|
||||||
|
claims, err := getClaimsFromTokenWithSecret(cr.SessionToken, cr.SecretKey)
|
||||||
|
mustIAM(t, err)
|
||||||
|
rec := svcAccRecord{Parent: cr.ParentUser, Secret: cr.SecretKey, Status: cr.Status, Name: cr.Name, Description: cr.Description, Expiration: cr.Expiration, UpdatedAt: u.UpdatedAt}
|
||||||
|
if sp, ok := claims.Lookup(policy.SessionPolicyName); ok && claims.MapClaims[iamPolicyClaimNameSA()] == embeddedPolicyType {
|
||||||
|
document, err := base64.StdEncoding.DecodeString(sp)
|
||||||
|
mustIAM(t, err)
|
||||||
|
rec.Policy = svcAccPolicyText(t, document)
|
||||||
|
}
|
||||||
|
return rec
|
||||||
|
}
|
||||||
|
|
||||||
|
// svcAccSite is a site that replicates to one peer, which records the IAM
|
||||||
|
// items it is sent.
|
||||||
|
type svcAccSite struct {
|
||||||
|
*groupRefreshFixture
|
||||||
|
mu sync.Mutex
|
||||||
|
sent []iamReplicationItem
|
||||||
|
}
|
||||||
|
|
||||||
|
func newSvcAccSite(t *testing.T) *svcAccSite {
|
||||||
|
t.Helper()
|
||||||
|
s := &svcAccSite{groupRefreshFixture: newGroupRefreshFixture(t)}
|
||||||
|
globalObjLayerMutex.Lock()
|
||||||
|
globalObjectAPI = s.obj
|
||||||
|
globalObjLayerMutex.Unlock()
|
||||||
|
|
||||||
|
peer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method == http.MethodPut && strings.HasSuffix(r.URL.Path, iamRevisionPeerPath) {
|
||||||
|
var batch iamRevisionBatch
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&batch); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
s.sent = append(s.sent, batch.Items...)
|
||||||
|
s.mu.Unlock()
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: "node", Instance: "boot", Digest: "fixture"}})
|
||||||
|
}))
|
||||||
|
globalSiteReplicatorCred.Set("snapshot-site-signing-key")
|
||||||
|
globalSiteReplicationSys.Lock()
|
||||||
|
globalSiteReplicationSys.enabled = true
|
||||||
|
globalSiteReplicationSys.state = srState{ServiceAccountAccessKey: s.parent.AccessKey, Peers: s.peers(peer.URL)}
|
||||||
|
globalSiteReplicationSys.Unlock()
|
||||||
|
t.Cleanup(func() {
|
||||||
|
s.leave()
|
||||||
|
peer.Close()
|
||||||
|
})
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *svcAccSite) peers(endpoint string) map[string]madmin.PeerInfo {
|
||||||
|
return map[string]madmin.PeerInfo{
|
||||||
|
globalDeploymentID(): {Name: "local", DeploymentID: globalDeploymentID()},
|
||||||
|
"remote": {Name: "remote", DeploymentID: "remote", Endpoint: endpoint},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// leave ends the replication of this process, for the site that comes next.
|
||||||
|
func (s *svcAccSite) leave() {
|
||||||
|
globalSiteReplicationSys.Lock()
|
||||||
|
globalSiteReplicationSys.enabled = false
|
||||||
|
globalSiteReplicationSys.state = srState{}
|
||||||
|
globalSiteReplicationSys.Unlock()
|
||||||
|
globalSiteReplicatorCred.Set("")
|
||||||
|
}
|
||||||
|
|
||||||
|
// items are the creates and edits of account key the peer was sent, in the
|
||||||
|
// order of sending.
|
||||||
|
func (s *svcAccSite) items(key string) []iamReplicationItem {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
var items []iamReplicationItem
|
||||||
|
for _, item := range s.sent {
|
||||||
|
c := item.SvcAccChange
|
||||||
|
if c != nil && ((c.Create != nil && c.Create.AccessKey == key) || (c.Update != nil && c.Update.AccessKey == key)) {
|
||||||
|
items = append(items, item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return items
|
||||||
|
}
|
||||||
|
|
||||||
|
// create and edit are an administrator's requests to this site.
|
||||||
|
func (s *svcAccSite) create(t *testing.T, key string) {
|
||||||
|
t.Helper()
|
||||||
|
rec := iamAdminCall(t, globalActiveCred, http.MethodPut, "/minio/admin/v3/add-service-account",
|
||||||
|
madmin.AddServiceAccountReq{TargetUser: s.parent.AccessKey, AccessKey: key, SecretKey: "snapshot-original-secret"}, nil, adminAPIHandlers{}.AddServiceAccount)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("add-service-account: %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *svcAccSite) edit(t *testing.T, key string, req madmin.UpdateServiceAccountReq) {
|
||||||
|
t.Helper()
|
||||||
|
rec := iamAdminCall(t, globalActiveCred, http.MethodPost, "/minio/admin/v3/update-service-account?accessKey="+key,
|
||||||
|
req, map[string]string{"accessKey": key}, adminAPIHandlers{}.UpdateServiceAccount)
|
||||||
|
if rec.Code != http.StatusNoContent {
|
||||||
|
t.Fatalf("update-service-account: %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// svcAccPeer is the site that applies items, in the order given, and what
|
||||||
|
// it then has of account key. It has the parent user, as a peer does.
|
||||||
|
func svcAccPeer(t *testing.T, from *svcAccSite, key string, items ...iamReplicationItem) svcAccRecord {
|
||||||
|
t.Helper()
|
||||||
|
from.leave()
|
||||||
|
peer := newGroupRefreshFixture(t)
|
||||||
|
for i, item := range items {
|
||||||
|
// As the peer reads it off the wire.
|
||||||
|
raw, err := json.Marshal(item)
|
||||||
|
mustIAM(t, err)
|
||||||
|
var received iamReplicationItem
|
||||||
|
mustIAM(t, json.Unmarshal(raw, &received))
|
||||||
|
if create := received.SvcAccChange.Create; create != nil {
|
||||||
|
if _, ok := create.Claims[sessionPolicyNameExtracted]; ok {
|
||||||
|
t.Fatal("replication snapshot contains the derived policy claim")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := applyIAMReplicationItem(peer.ctx, received); err != nil {
|
||||||
|
t.Fatalf("the peer refused item %d: %v", i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return svcAccStored(t, peer, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func wantSvcAccRecord(t *testing.T, where string, got, want svcAccRecord) {
|
||||||
|
t.Helper()
|
||||||
|
if got != want {
|
||||||
|
t.Errorf("%s:\n got %+v\nwant %+v", where, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An edit of a service account reaches a peer whatever became of the edits
|
||||||
|
// before it: one that arrives late, or never, is not lost.
|
||||||
|
func TestSiteReplicationServiceAccountEdits(t *testing.T) {
|
||||||
|
const key = "snapshot-edits"
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
edits []madmin.UpdateServiceAccountReq
|
||||||
|
took func(svcAccRecord) bool // control: what the site has after them
|
||||||
|
}{
|
||||||
|
{"policy-secret-description", []madmin.UpdateServiceAccountReq{
|
||||||
|
{NewPolicy: json.RawMessage(svcAccSnapshotPolicy)},
|
||||||
|
{NewSecretKey: "snapshot-rotated-secret"},
|
||||||
|
{NewDescription: "described"},
|
||||||
|
}, func(r svcAccRecord) bool {
|
||||||
|
return r.Policy != "" && r.Secret == "snapshot-rotated-secret" && r.Description == "described"
|
||||||
|
}},
|
||||||
|
{"secret-disable", []madmin.UpdateServiceAccountReq{
|
||||||
|
{NewPolicy: json.RawMessage(svcAccSnapshotPolicy), NewSecretKey: "snapshot-rotated-secret"},
|
||||||
|
{NewStatus: string(madmin.AccountDisabled)},
|
||||||
|
}, func(r svcAccRecord) bool {
|
||||||
|
return r.Policy != "" && r.Secret == "snapshot-rotated-secret" && r.Status == auth.AccountOff
|
||||||
|
}},
|
||||||
|
// The last edit removes the embedded policy: the peer gets an
|
||||||
|
// account without one, and the description set before.
|
||||||
|
{"policy-removed", []madmin.UpdateServiceAccountReq{
|
||||||
|
{NewPolicy: json.RawMessage(svcAccSnapshotPolicy)},
|
||||||
|
{NewDescription: "described"},
|
||||||
|
{NewPolicy: json.RawMessage(`{"Statement":[]}`)},
|
||||||
|
}, func(r svcAccRecord) bool { return r.Policy == "" && r.Description == "described" }},
|
||||||
|
{"policy-removed-with-version", []madmin.UpdateServiceAccountReq{
|
||||||
|
{NewPolicy: json.RawMessage(svcAccSnapshotPolicy)},
|
||||||
|
{NewPolicy: json.RawMessage(`{"Version":"2012-10-17","Statement":[]}`), NewName: "named"},
|
||||||
|
{NewDescription: "described"},
|
||||||
|
}, func(r svcAccRecord) bool {
|
||||||
|
return r.Policy == "" && r.Name == "named" && r.Description == "described"
|
||||||
|
}},
|
||||||
|
{"policy-missing-version", []madmin.UpdateServiceAccountReq{
|
||||||
|
{NewPolicy: json.RawMessage(svcAccSnapshotPolicy)},
|
||||||
|
{NewPolicy: json.RawMessage(strings.ReplaceAll(strings.ReplaceAll(svcAccSnapshotPolicy, `"Version":"2012-10-17",`, ""), "s3:GetObject", "s3:PutObject"))},
|
||||||
|
{NewDescription: "described"},
|
||||||
|
}, func(r svcAccRecord) bool {
|
||||||
|
return r.Policy == svcAccPolicyText(t, []byte(strings.ReplaceAll(svcAccSnapshotPolicy, "s3:GetObject", "s3:PutObject"))) && r.Description == "described"
|
||||||
|
}},
|
||||||
|
} {
|
||||||
|
for _, delivery := range []string{"late", "lost"} {
|
||||||
|
t.Run(tc.name+"/"+delivery, func(t *testing.T) {
|
||||||
|
site := newSvcAccSite(t)
|
||||||
|
site.create(t, key)
|
||||||
|
for _, edit := range tc.edits {
|
||||||
|
site.edit(t, key, edit)
|
||||||
|
}
|
||||||
|
sent := site.items(key)
|
||||||
|
if len(sent) != 1+len(tc.edits) {
|
||||||
|
t.Fatalf("the site sent %d items for the account, want one for the create and each edit", len(sent))
|
||||||
|
}
|
||||||
|
want := svcAccStored(t, site.groupRefreshFixture, key)
|
||||||
|
if !tc.took(want) {
|
||||||
|
t.Fatalf("control: the site did not apply the edits: %+v", want)
|
||||||
|
}
|
||||||
|
// The peer has the account and gets the last edit. The edits
|
||||||
|
// before it were lost, or reach it after the last.
|
||||||
|
last := len(sent) - 1
|
||||||
|
deliver := []iamReplicationItem{sent[0], sent[last]}
|
||||||
|
if delivery == "late" {
|
||||||
|
deliver = append(deliver, sent[1:last]...)
|
||||||
|
}
|
||||||
|
wantSvcAccRecord(t, "peer", svcAccPeer(t, site, key, deliver...), want)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Two requests edit the account through this site at once, and the
|
||||||
|
// first comes to replicate when the second is stored.
|
||||||
|
t.Run("concurrent", func(t *testing.T) {
|
||||||
|
site := newSvcAccSite(t)
|
||||||
|
site.create(t, key)
|
||||||
|
sp, err := policy.ParseConfig(strings.NewReader(svcAccSnapshotPolicy))
|
||||||
|
mustIAM(t, err)
|
||||||
|
first, err := site.sys.UpdateServiceAccount(site.ctx, key, updateServiceAccountOpts{sessionPolicy: sp})
|
||||||
|
mustIAM(t, err)
|
||||||
|
second, err := site.sys.UpdateServiceAccount(site.ctx, key, updateServiceAccountOpts{description: "described"})
|
||||||
|
mustIAM(t, err)
|
||||||
|
for _, at := range []time.Time{first, second} {
|
||||||
|
mustIAM(t, globalSiteReplicationSys.IAMChangeHook(site.ctx, madmin.SRIAMItem{
|
||||||
|
Type: madmin.SRIAMItemSvcAcc,
|
||||||
|
SvcAccChange: &madmin.SRSvcAccChange{Update: &madmin.SRSvcAccUpdate{AccessKey: key}},
|
||||||
|
UpdatedAt: at,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
want := svcAccStored(t, site.groupRefreshFixture, key)
|
||||||
|
if want.Policy == "" || want.Description != "described" {
|
||||||
|
t.Fatalf("control: the site did not apply the edits: %+v", want)
|
||||||
|
}
|
||||||
|
wantSvcAccRecord(t, "peer", svcAccPeer(t, site, key, site.items(key)...), want)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Another server of the site restricted, rotated and disabled the account,
|
||||||
|
// and this server's cache has not heard of it. What this server sends, and
|
||||||
|
// exports, is the account that is stored.
|
||||||
|
func TestSiteReplicationServiceAccountStaleCache(t *testing.T) {
|
||||||
|
const key = "snapshot-stale"
|
||||||
|
// prepare returns the site, the stored account and what it sent for the
|
||||||
|
// create.
|
||||||
|
prepare := func(t *testing.T) (*svcAccSite, svcAccRecord, iamReplicationItem) {
|
||||||
|
t.Helper()
|
||||||
|
site := newSvcAccSite(t)
|
||||||
|
site.create(t, key)
|
||||||
|
created := site.items(key)
|
||||||
|
if len(created) != 1 {
|
||||||
|
t.Fatalf("the site sent %d items for the create", len(created))
|
||||||
|
}
|
||||||
|
sp, err := policy.ParseConfig(strings.NewReader(svcAccSnapshotPolicy))
|
||||||
|
mustIAM(t, err)
|
||||||
|
_, err = site.sibling(t).UpdateServiceAccount(site.ctx, key, updateServiceAccountOpts{
|
||||||
|
sessionPolicy: sp, secretKey: "snapshot-rotated-secret", status: auth.AccountOff, description: "restricted",
|
||||||
|
})
|
||||||
|
mustIAM(t, err)
|
||||||
|
if cached, _ := site.sys.store.GetUser(key); cached.Credentials.SecretKey != "snapshot-original-secret" || cached.Credentials.Status != auth.AccountOn {
|
||||||
|
t.Fatal("control: this server's cache must not have heard of the edit")
|
||||||
|
}
|
||||||
|
want := svcAccStored(t, site.groupRefreshFixture, key)
|
||||||
|
if want.Policy == "" || want.Secret != "snapshot-rotated-secret" || want.Status != auth.AccountOff {
|
||||||
|
t.Fatalf("control: the edit is not stored: %+v", want)
|
||||||
|
}
|
||||||
|
return site, want, created[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("heal", func(t *testing.T) {
|
||||||
|
site, want, created := prepare(t)
|
||||||
|
local := madmin.UserInfo{Status: madmin.AccountStatus(want.Status), UpdatedAt: want.UpdatedAt}
|
||||||
|
remote := madmin.UserInfo{Status: madmin.AccountStatus(auth.AccountOn), UpdatedAt: created.UpdatedAt}
|
||||||
|
info := srStatusInfo{Sites: globalSiteReplicationSys.state.Peers, UserStats: map[string]map[string]srUserStatsSummary{key: {
|
||||||
|
globalDeploymentID(): {userInfo: srUserInfo{UserInfo: local}},
|
||||||
|
"remote": {SRUserStatsSummary: madmin.SRUserStatsSummary{UserInfoMismatch: true}, userInfo: srUserInfo{UserInfo: remote}},
|
||||||
|
}}}
|
||||||
|
mustIAM(t, globalSiteReplicationSys.healUsers(site.ctx, site.obj, key, info))
|
||||||
|
sent := site.items(key)
|
||||||
|
if len(sent) != 2 {
|
||||||
|
t.Fatalf("heal sent %d items for the account, want 1", len(sent)-1)
|
||||||
|
}
|
||||||
|
wantSvcAccRecord(t, "healed peer", svcAccPeer(t, site, key, sent...), want)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("sync", func(t *testing.T) {
|
||||||
|
site, want, _ := prepare(t)
|
||||||
|
mustIAM(t, globalSiteReplicationSys.syncToAllPeers(site.ctx, madmin.SRAddOptions{}))
|
||||||
|
sent := site.items(key)
|
||||||
|
if len(sent) != 2 {
|
||||||
|
t.Fatalf("sync sent %d items for the account, want 1", len(sent)-1)
|
||||||
|
}
|
||||||
|
wantSvcAccRecord(t, "new peer", svcAccPeer(t, site, key, sent[1]), want)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("export", func(t *testing.T) {
|
||||||
|
_, want, _ := prepare(t)
|
||||||
|
rec := iamAdminCall(t, globalActiveCred, http.MethodGet, "/minio/admin/v3/export-iam", nil, nil, adminAPIHandlers{}.ExportIAM)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("export-iam: %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
archive, err := zip.NewReader(bytes.NewReader(rec.Body.Bytes()), int64(rec.Body.Len()))
|
||||||
|
mustIAM(t, err)
|
||||||
|
file, err := archive.Open(path.Join(iamAssetsDir, allSvcAcctsFile))
|
||||||
|
mustIAM(t, err)
|
||||||
|
raw, err := io.ReadAll(file)
|
||||||
|
mustIAM(t, err)
|
||||||
|
var accounts map[string]madmin.SRSvcAccCreate
|
||||||
|
mustIAM(t, json.Unmarshal(raw, &accounts))
|
||||||
|
a, ok := accounts[key]
|
||||||
|
if !ok || a.Expiration == nil {
|
||||||
|
t.Fatalf("the export has no account %s with an expiration (present=%v)", key, ok)
|
||||||
|
}
|
||||||
|
got := svcAccRecord{Parent: a.Parent, Secret: a.SecretKey, Status: a.Status, Name: a.Name, Description: a.Description, Policy: svcAccPolicyText(t, a.SessionPolicy), Expiration: *a.Expiration}
|
||||||
|
want.UpdatedAt = time.Time{} // an export has no revisions
|
||||||
|
wantSvcAccRecord(t, "export", got, want)
|
||||||
|
if a.Claims[iamPolicyClaimNameSA()] != embeddedPolicyType {
|
||||||
|
t.Errorf("exported claims say %v, want %s", a.Claims[iamPolicyClaimNameSA()], embeddedPolicyType)
|
||||||
|
}
|
||||||
|
if _, ok := a.Claims[sessionPolicyNameExtracted]; ok {
|
||||||
|
t.Error("exported claims contain the derived policy claim")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIAMServiceAccountSnapshotClaims(t *testing.T) {
|
||||||
|
f := newGroupRefreshFixture(t)
|
||||||
|
sp, err := policy.ParseConfig(strings.NewReader(svcAccSnapshotPolicy))
|
||||||
|
mustIAM(t, err)
|
||||||
|
_, _, err = f.sys.NewServiceAccount(f.ctx, f.parent.AccessKey, nil, newServiceAccountOpts{
|
||||||
|
accessKey: "snapshot-claims", secretKey: "snapshot-valid-secret", sessionPolicy: sp,
|
||||||
|
claims: map[string]any{"identity-attribute": "preserved"},
|
||||||
|
})
|
||||||
|
mustIAM(t, err)
|
||||||
|
u, ok := f.stored(t, "snapshot-claims")
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("service account missing")
|
||||||
|
}
|
||||||
|
cr := u.Credentials
|
||||||
|
rawClaims, err := auth.ExtractClaims(cr.SessionToken, cr.SecretKey)
|
||||||
|
mustIAM(t, err)
|
||||||
|
derived, ok := cr.Claims[sessionPolicyNameExtracted]
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("control: loaded claims must contain the derived policy")
|
||||||
|
}
|
||||||
|
snapshot, err := serviceAccountSnapshot(u)
|
||||||
|
mustIAM(t, err)
|
||||||
|
if !reflect.DeepEqual(snapshot.Claims, rawClaims.Map()) {
|
||||||
|
t.Fatal("snapshot claims differ from the raw JWT")
|
||||||
|
}
|
||||||
|
if u.Credentials.SessionToken != cr.SessionToken || u.Credentials.Claims[sessionPolicyNameExtracted] != derived {
|
||||||
|
t.Fatal("snapshot changed the source token or its cached claims")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The parent was deleted and created again on both sites. The token of a
|
||||||
|
// service account created afterwards names that revocation, and a peer
|
||||||
|
// refuses a credential of the parent that does not: the create must carry
|
||||||
|
// the claims of the token.
|
||||||
|
func TestSiteReplicationServiceAccountOfRecreatedParent(t *testing.T) {
|
||||||
|
const key = "snapshot-reparent"
|
||||||
|
site := newSvcAccSite(t)
|
||||||
|
mustIAM(t, site.sys.DeleteUser(site.ctx, site.parent.AccessKey, false))
|
||||||
|
iamAdminUser(site.ctx, t, site.sys, site.parent.AccessKey, "refresh-parent")
|
||||||
|
revoked, err := loadIAMRevision(site.ctx, site.sys.store, getUserIdentityPath(site.parent.AccessKey, regUser))
|
||||||
|
mustIAM(t, err)
|
||||||
|
if revoked.RevokedBefore.IsZero() {
|
||||||
|
t.Fatal("control: the parent must have a revocation boundary")
|
||||||
|
}
|
||||||
|
site.create(t, key)
|
||||||
|
sent := site.items(key)
|
||||||
|
if len(sent) != 1 {
|
||||||
|
t.Fatalf("the site sent %d items for the create", len(sent))
|
||||||
|
}
|
||||||
|
want := svcAccStored(t, site.groupRefreshFixture, key)
|
||||||
|
|
||||||
|
// The peer's parent is newer than the deletion, which leaves it live
|
||||||
|
// with the boundary: a parent that was deleted and created again.
|
||||||
|
site.leave()
|
||||||
|
peer := newGroupRefreshFixture(t)
|
||||||
|
mustIAM(t, peer.sys.DeleteUser(withIAMReplicationTime(peer.ctx, revoked.RevokedBefore), peer.parent.AccessKey, false))
|
||||||
|
if r, err := loadIAMRevision(peer.ctx, peer.sys.store, getUserIdentityPath(peer.parent.AccessKey, regUser)); err != nil || r.Deleted || !r.RevokedBefore.Equal(revoked.RevokedBefore) {
|
||||||
|
t.Fatalf("control: the peer's parent must be live with the boundary (err=%v deleted=%v)", err, r.Deleted)
|
||||||
|
}
|
||||||
|
mustIAM(t, applyIAMReplicationItem(peer.ctx, sent[0]))
|
||||||
|
if _, ok := peer.stored(t, key); !ok {
|
||||||
|
t.Fatal("the peer dropped the new service account")
|
||||||
|
}
|
||||||
|
wantSvcAccRecord(t, "peer", svcAccStored(t, peer, key), want)
|
||||||
|
}
|
||||||
|
|
||||||
|
// An account that is deleted is not sent, and that is not an error. An
|
||||||
|
// account whose stored token cannot be read is not taken for deleted: the
|
||||||
|
// site signing key is not available here, and no token is read without it.
|
||||||
|
func TestSiteReplicationServiceAccountUnreadable(t *testing.T) {
|
||||||
|
const key = "snapshot-unreadable"
|
||||||
|
site := newSvcAccSite(t)
|
||||||
|
site.create(t, key)
|
||||||
|
item := madmin.SRIAMItem{Type: madmin.SRIAMItemSvcAcc, SvcAccChange: &madmin.SRSvcAccChange{Update: &madmin.SRSvcAccUpdate{AccessKey: key}}}
|
||||||
|
|
||||||
|
globalSiteReplicatorCred.Set("")
|
||||||
|
if _, err := globalSiteReplicationSys.replicationItem(site.ctx, item); err == nil || errors.Is(err, errIAMStaleUpdate) {
|
||||||
|
t.Errorf("an account whose token cannot be read must be an error, got %v", err)
|
||||||
|
}
|
||||||
|
if err := globalSiteReplicationSys.IAMChangeHook(site.ctx, item); err == nil {
|
||||||
|
t.Error("the hook reported an account as sent that it could not read")
|
||||||
|
}
|
||||||
|
globalSiteReplicatorCred.Set("snapshot-site-signing-key")
|
||||||
|
|
||||||
|
mustIAM(t, site.sys.DeleteServiceAccount(site.ctx, key, false))
|
||||||
|
if _, err := globalSiteReplicationSys.replicationItem(site.ctx, item); !errors.Is(err, errIAMStaleUpdate) {
|
||||||
|
t.Errorf("a deleted account must not be sent, got %v", err)
|
||||||
|
}
|
||||||
|
if sent := site.items(key); len(sent) != 1 {
|
||||||
|
t.Errorf("the site sent %d items for the account, want only the create", len(sent))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// svcAccPausedStore holds every load of a user until proceed is closed.
|
||||||
|
type svcAccPausedStore struct {
|
||||||
|
IAMStorageAPI
|
||||||
|
once sync.Once
|
||||||
|
loading, proceed chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *svcAccPausedStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error {
|
||||||
|
s.once.Do(func() { close(s.loading) })
|
||||||
|
<-s.proceed
|
||||||
|
return s.IAMStorageAPI.loadUser(ctx, user, userType, m)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The hook reads the account from storage without the lock of the
|
||||||
|
// replication configuration. Loading an identity takes that lock for
|
||||||
|
// reading; if the hook held it too, a change of the configuration that
|
||||||
|
// arrives in between would wait for the hook, and the hook for it.
|
||||||
|
func TestSiteReplicationHookReadsStorageUnlocked(t *testing.T) {
|
||||||
|
const key = "snapshot-unlocked"
|
||||||
|
site := newSvcAccSite(t)
|
||||||
|
site.create(t, key)
|
||||||
|
paused := &svcAccPausedStore{IAMStorageAPI: site.sys.store.IAMStorageAPI, loading: make(chan struct{}), proceed: make(chan struct{})}
|
||||||
|
site.sys.store.IAMStorageAPI = paused
|
||||||
|
|
||||||
|
sent := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
sent <- globalSiteReplicationSys.IAMChangeHook(site.ctx, madmin.SRIAMItem{
|
||||||
|
Type: madmin.SRIAMItemSvcAcc,
|
||||||
|
SvcAccChange: &madmin.SRSvcAccChange{Update: &madmin.SRSvcAccUpdate{AccessKey: key}},
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-paused.loading:
|
||||||
|
case err := <-sent:
|
||||||
|
t.Fatalf("the hook did not read the account from storage (err=%v)", err)
|
||||||
|
}
|
||||||
|
// A writer that had to wait here would never get the lock, nor the
|
||||||
|
// hook its next read lock behind that writer: try, do not wait.
|
||||||
|
if globalSiteReplicationSys.TryLock() {
|
||||||
|
globalSiteReplicationSys.Unlock() //nolint:staticcheck // only the lock is of interest
|
||||||
|
} else {
|
||||||
|
t.Error("the hook holds the lock of the replication configuration while it reads storage")
|
||||||
|
}
|
||||||
|
close(paused.proceed)
|
||||||
|
select {
|
||||||
|
case err := <-sent:
|
||||||
|
mustIAM(t, err)
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("the hook did not return")
|
||||||
|
}
|
||||||
|
if got := site.items(key); len(got) != 2 {
|
||||||
|
t.Errorf("the site sent %d items for the account, want the create and the edit", len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Healing must release its own configuration lock before calling the hook.
|
||||||
|
// Otherwise a waiting configuration writer blocks the hook's next read lock.
|
||||||
|
func TestSiteReplicationHealReadsStorageUnlocked(t *testing.T) {
|
||||||
|
const key = "heal-unlocked"
|
||||||
|
site := newSvcAccSite(t)
|
||||||
|
site.create(t, key)
|
||||||
|
stored := svcAccStored(t, site.groupRefreshFixture, key)
|
||||||
|
local := madmin.UserInfo{Status: madmin.AccountEnabled, UpdatedAt: stored.UpdatedAt}
|
||||||
|
remote := madmin.UserInfo{Status: madmin.AccountDisabled, UpdatedAt: stored.UpdatedAt.Add(-time.Minute)}
|
||||||
|
info := srStatusInfo{Sites: globalSiteReplicationSys.state.Peers, UserStats: map[string]map[string]srUserStatsSummary{key: {
|
||||||
|
globalDeploymentID(): {userInfo: srUserInfo{UserInfo: local}},
|
||||||
|
"remote": {SRUserStatsSummary: madmin.SRUserStatsSummary{UserInfoMismatch: true}, userInfo: srUserInfo{UserInfo: remote}},
|
||||||
|
}}}
|
||||||
|
paused := &svcAccPausedStore{IAMStorageAPI: site.sys.store.IAMStorageAPI, loading: make(chan struct{}), proceed: make(chan struct{})}
|
||||||
|
site.sys.store.IAMStorageAPI = paused
|
||||||
|
release := sync.OnceFunc(func() { close(paused.proceed) })
|
||||||
|
defer release()
|
||||||
|
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
done <- globalSiteReplicationSys.healUsers(site.ctx, site.obj, key, info)
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-paused.loading:
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("heal did not read the account from storage (err=%v)", err)
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("heal did not reach the storage read")
|
||||||
|
}
|
||||||
|
// Testing the writer lock without waiting keeps a regression from leaving
|
||||||
|
// deadlocked goroutines behind in the test process.
|
||||||
|
if globalSiteReplicationSys.TryLock() {
|
||||||
|
globalSiteReplicationSys.Unlock() //nolint:staticcheck // only the lock is of interest
|
||||||
|
} else {
|
||||||
|
t.Error("heal holds the replication configuration lock during a storage read")
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
mustIAM(t, err)
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("heal did not return")
|
||||||
|
}
|
||||||
|
if got := site.items(key); len(got) != 2 {
|
||||||
|
t.Errorf("the site sent %d items, want the create and the healed snapshot", len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
+131
-53
@@ -1087,17 +1087,52 @@ func (sys *IAMSys) notifyForServiceAccount(ctx context.Context, accessKey string
|
|||||||
}
|
}
|
||||||
|
|
||||||
type newServiceAccountOpts struct {
|
type newServiceAccountOpts struct {
|
||||||
sessionPolicy *policy.Policy
|
sessionPolicy *policy.Policy
|
||||||
accessKey string
|
accessKey string
|
||||||
secretKey string
|
secretKey string
|
||||||
status string // Used by replication snapshots; local creates default to enabled.
|
status string // Used by replication snapshots; local creates default to enabled.
|
||||||
name, description string
|
name, description string
|
||||||
expiration *time.Time
|
expiration *time.Time
|
||||||
allowSiteReplicatorAccount bool // allow creating internal service account for site-replication.
|
allowSiteReplicatorAccount bool // allow creating internal service account for site-replication.
|
||||||
|
preserveEmptyEmbeddedPolicy bool // preserve historical deny-all empty embedded policies on snapshot import/replication.
|
||||||
|
|
||||||
claims map[string]any
|
claims map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func svcAcctClaimsHaveEmbeddedPolicy(claims map[string]any) bool {
|
||||||
|
if claims == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
pt, _ := claims[iamPolicyClaimNameSA()].(string)
|
||||||
|
return pt == embeddedPolicyType
|
||||||
|
}
|
||||||
|
|
||||||
|
// marshalServiceAccountPolicy validates a supplied policy and normalizes its
|
||||||
|
// version without changing the caller's policy. An empty policy means inheritance.
|
||||||
|
func marshalServiceAccountPolicy(sp *policy.Policy) ([]byte, error) {
|
||||||
|
if sp == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
p := *sp
|
||||||
|
if p.Version == "" {
|
||||||
|
p.Version = policy.DefaultVersion
|
||||||
|
}
|
||||||
|
if err := p.Validate(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if p.IsEmpty() {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
buf, err := json.Marshal(p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(buf) > maxSVCSessionPolicySize {
|
||||||
|
return nil, errSessionPolicyTooLarge
|
||||||
|
}
|
||||||
|
return buf, nil
|
||||||
|
}
|
||||||
|
|
||||||
// NewServiceAccount - create a new service account
|
// NewServiceAccount - create a new service account
|
||||||
func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, groups []string, opts newServiceAccountOpts) (auth.Credentials, time.Time, error) {
|
func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, groups []string, opts newServiceAccountOpts) (auth.Credentials, time.Time, error) {
|
||||||
if !sys.Initialized() {
|
if !sys.Initialized() {
|
||||||
@@ -1115,19 +1150,9 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
|
|||||||
return auth.Credentials{}, time.Time{}, auth.ErrNoAccessKeyWithSecretKey
|
return auth.Credentials{}, time.Time{}, auth.ErrNoAccessKeyWithSecretKey
|
||||||
}
|
}
|
||||||
|
|
||||||
var policyBuf []byte
|
policyBuf, err := marshalServiceAccountPolicy(opts.sessionPolicy)
|
||||||
if opts.sessionPolicy != nil {
|
if err != nil {
|
||||||
err := opts.sessionPolicy.Validate()
|
return auth.Credentials{}, time.Time{}, err
|
||||||
if err != nil {
|
|
||||||
return auth.Credentials{}, time.Time{}, err
|
|
||||||
}
|
|
||||||
policyBuf, err = json.Marshal(opts.sessionPolicy)
|
|
||||||
if err != nil {
|
|
||||||
return auth.Credentials{}, time.Time{}, err
|
|
||||||
}
|
|
||||||
if len(policyBuf) > maxSVCSessionPolicySize {
|
|
||||||
return auth.Credentials{}, time.Time{}, errSessionPolicyTooLarge
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// found newly requested service account, to be same as
|
// found newly requested service account, to be same as
|
||||||
@@ -1141,15 +1166,45 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
|
|||||||
m := make(map[string]any)
|
m := make(map[string]any)
|
||||||
m[parentClaim] = parentUser
|
m[parentClaim] = parentUser
|
||||||
|
|
||||||
if len(policyBuf) > 0 {
|
switch {
|
||||||
|
case len(policyBuf) > 0:
|
||||||
m[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString(policyBuf)
|
m[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString(policyBuf)
|
||||||
m[iamPolicyClaimNameSA()] = embeddedPolicyType
|
m[iamPolicyClaimNameSA()] = embeddedPolicyType
|
||||||
} else {
|
case opts.preserveEmptyEmbeddedPolicy:
|
||||||
|
switch {
|
||||||
|
case opts.sessionPolicy != nil:
|
||||||
|
p := *opts.sessionPolicy
|
||||||
|
if p.Version == "" {
|
||||||
|
p.Version = policy.DefaultVersion
|
||||||
|
}
|
||||||
|
buf, merr := json.Marshal(p)
|
||||||
|
if merr != nil {
|
||||||
|
return auth.Credentials{}, time.Time{}, merr
|
||||||
|
}
|
||||||
|
m[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString(buf)
|
||||||
|
m[iamPolicyClaimNameSA()] = embeddedPolicyType
|
||||||
|
default:
|
||||||
|
if raw, ok := opts.claims[policy.SessionPolicyName].(string); ok && raw != "" {
|
||||||
|
m[policy.SessionPolicyName] = raw
|
||||||
|
m[iamPolicyClaimNameSA()] = embeddedPolicyType
|
||||||
|
} else {
|
||||||
|
m[iamPolicyClaimNameSA()] = inheritedPolicyType
|
||||||
|
}
|
||||||
|
}
|
||||||
|
default:
|
||||||
m[iamPolicyClaimNameSA()] = inheritedPolicyType
|
m[iamPolicyClaimNameSA()] = inheritedPolicyType
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add all the necessary claims for the service account.
|
// Add all the necessary claims for the service account.
|
||||||
for k, v := range opts.claims {
|
for k, v := range opts.claims {
|
||||||
|
// Older snapshots may carry this cache-only, decoded policy claim.
|
||||||
|
if k == sessionPolicyNameExtracted {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// An explicit policy, including an empty one, owns this claim.
|
||||||
|
if k == policy.SessionPolicyName && opts.sessionPolicy != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
_, ok := m[k]
|
_, ok := m[k]
|
||||||
if !ok {
|
if !ok {
|
||||||
m[k] = v
|
m[k] = v
|
||||||
@@ -1162,7 +1217,6 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
|
|||||||
}
|
}
|
||||||
|
|
||||||
var accessKey, secretKey string
|
var accessKey, secretKey string
|
||||||
var err error
|
|
||||||
if len(opts.accessKey) > 0 || len(opts.secretKey) > 0 {
|
if len(opts.accessKey) > 0 || len(opts.secretKey) > 0 {
|
||||||
accessKey, secretKey = opts.accessKey, opts.secretKey
|
accessKey, secretKey = opts.accessKey, opts.secretKey
|
||||||
} else {
|
} else {
|
||||||
@@ -1311,22 +1365,65 @@ func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (Use
|
|||||||
return UserIdentity{}, nil, errNoSuchServiceAccount
|
return UserIdentity{}, nil, errNoSuchServiceAccount
|
||||||
}
|
}
|
||||||
|
|
||||||
var embeddedPolicy *policy.Policy
|
embeddedPolicy, err := embeddedServiceAccountPolicy(jwtClaims)
|
||||||
|
if err != nil {
|
||||||
|
return UserIdentity{}, nil, err
|
||||||
|
}
|
||||||
|
return sa, embeddedPolicy, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// embeddedServiceAccountPolicy returns the policy embedded in the token of a
|
||||||
|
// service account, or nil when the account has the policies of its parent.
|
||||||
|
func embeddedServiceAccountPolicy(jwtClaims *jwt.MapClaims) (*policy.Policy, error) {
|
||||||
pt, ptok := jwtClaims.Lookup(iamPolicyClaimNameSA())
|
pt, ptok := jwtClaims.Lookup(iamPolicyClaimNameSA())
|
||||||
sp, spok := jwtClaims.Lookup(policy.SessionPolicyName)
|
sp, spok := jwtClaims.Lookup(policy.SessionPolicyName)
|
||||||
if ptok && spok && pt == embeddedPolicyType {
|
if !ptok || !spok || pt != embeddedPolicyType {
|
||||||
policyBytes, err := base64.StdEncoding.DecodeString(sp)
|
return nil, nil
|
||||||
if err != nil {
|
}
|
||||||
return UserIdentity{}, nil, err
|
policyBytes, err := base64.StdEncoding.DecodeString(sp)
|
||||||
}
|
if err != nil {
|
||||||
embeddedPolicy, err = policy.ParseConfig(bytes.NewReader(policyBytes))
|
return nil, err
|
||||||
if err != nil {
|
}
|
||||||
return UserIdentity{}, nil, err
|
return policy.ParseConfig(bytes.NewReader(policyBytes))
|
||||||
|
}
|
||||||
|
|
||||||
|
// serviceAccountSnapshot returns service account u, a record loaded from
|
||||||
|
// storage, as the create that replication, the first sync, heal and export
|
||||||
|
// carry. Its claims and its embedded policy are read from the token of that
|
||||||
|
// record, so that every part of the snapshot is of one revision: a server's
|
||||||
|
// cache may hold the account as it was before an edit made through another
|
||||||
|
// server.
|
||||||
|
func serviceAccountSnapshot(u UserIdentity) (madmin.SRSvcAccCreate, error) {
|
||||||
|
jwtClaims, err := extractJWTClaims(u)
|
||||||
|
if err != nil {
|
||||||
|
return madmin.SRSvcAccCreate{}, err
|
||||||
|
}
|
||||||
|
// Extraction adds a decoded policy for local authorization. Transport only
|
||||||
|
// the token's claims, so peers and imports do not sign this derived value.
|
||||||
|
jwtClaims.Delete(sessionPolicyNameExtracted)
|
||||||
|
embeddedPolicy, err := embeddedServiceAccountPolicy(jwtClaims)
|
||||||
|
if err != nil {
|
||||||
|
return madmin.SRSvcAccCreate{}, err
|
||||||
|
}
|
||||||
|
var policyJSON []byte
|
||||||
|
if embeddedPolicy != nil {
|
||||||
|
if policyJSON, err = json.Marshal(embeddedPolicy); err != nil {
|
||||||
|
return madmin.SRSvcAccCreate{}, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
cr := u.Credentials
|
||||||
return sa, embeddedPolicy, nil
|
return madmin.SRSvcAccCreate{
|
||||||
|
Parent: cr.ParentUser,
|
||||||
|
AccessKey: cr.AccessKey,
|
||||||
|
SecretKey: cr.SecretKey,
|
||||||
|
Groups: cr.Groups,
|
||||||
|
Claims: jwtClaims.Map(),
|
||||||
|
SessionPolicy: policyJSON,
|
||||||
|
Status: cr.Status,
|
||||||
|
Name: cr.Name,
|
||||||
|
Description: cr.Description,
|
||||||
|
Expiration: &cr.Expiration,
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetTemporaryAccount - wrapper method to get information about a temporary account
|
// GetTemporaryAccount - wrapper method to get information about a temporary account
|
||||||
@@ -1398,25 +1495,6 @@ func (sys *IAMSys) getAccountWithClaims(ctx context.Context, accessKey string) (
|
|||||||
return acc, jwtClaims, nil
|
return acc, jwtClaims, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetClaimsForSvcAcc - gets the claims associated with the service account.
|
|
||||||
func (sys *IAMSys) GetClaimsForSvcAcc(ctx context.Context, accessKey string) (map[string]any, error) {
|
|
||||||
if !sys.Initialized() {
|
|
||||||
return nil, errServerNotInitialized
|
|
||||||
}
|
|
||||||
|
|
||||||
sa, ok := sys.store.GetUser(accessKey)
|
|
||||||
if !ok || !sa.Credentials.IsServiceAccount() {
|
|
||||||
return nil, errNoSuchServiceAccount
|
|
||||||
}
|
|
||||||
|
|
||||||
jwtClaims, err := extractJWTClaims(sa)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return jwtClaims.Map(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// DeleteServiceAccount - delete a service account
|
// DeleteServiceAccount - delete a service account
|
||||||
func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string, notifyPeers bool) error {
|
func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string, notifyPeers bool) error {
|
||||||
if !sys.Initialized() {
|
if !sys.Initialized() {
|
||||||
|
|||||||
@@ -71,6 +71,7 @@ type ObjectOptions struct {
|
|||||||
DeleteMarker bool // Is only set in DELETE operations for delete marker replication
|
DeleteMarker bool // Is only set in DELETE operations for delete marker replication
|
||||||
CheckDMReplicationReady bool // Is delete marker ready to be replicated - set only during HEAD
|
CheckDMReplicationReady bool // Is delete marker ready to be replicated - set only during HEAD
|
||||||
Tagging bool // Is only in GET/HEAD operations to return tagging metadata along with regular metadata and body.
|
Tagging bool // Is only in GET/HEAD operations to return tagging metadata along with regular metadata and body.
|
||||||
|
TaggingCount bool // GET/HEAD caller is authorized to receive object tag metadata.
|
||||||
|
|
||||||
UserDefined map[string]string // only set in case of POST/PUT operations
|
UserDefined map[string]string // only set in case of POST/PUT operations
|
||||||
ObjectAttributes map[string]struct{} // Attribute tags defined by the users for the GetObjectAttributes request
|
ObjectAttributes map[string]struct{} // Attribute tags defined by the users for the GetObjectAttributes request
|
||||||
@@ -122,6 +123,10 @@ type ObjectOptions struct {
|
|||||||
|
|
||||||
// Pools-layer version resolver; the caller holds the shared object lock.
|
// Pools-layer version resolver; the caller holds the shared object lock.
|
||||||
replicaObjectInfo GetObjectInfoFn
|
replicaObjectInfo GetObjectInfoFn
|
||||||
|
// Verifies protected content in every pool before a replica commits.
|
||||||
|
replicaContentGuard replicaContentGuardFn
|
||||||
|
// Selected under the object locks, before copy encryption/compression.
|
||||||
|
copyPlan *copyObjectPlan
|
||||||
|
|
||||||
// RefuseLockedOverwrite refuses, under the destination's write lock, a
|
// RefuseLockedOverwrite refuses, under the destination's write lock, a
|
||||||
// write that would replace a stored version Object Lock protects: the
|
// write that would replace a stored version Object Lock protects: the
|
||||||
@@ -297,6 +302,7 @@ type ObjectLayer interface {
|
|||||||
NewObjectNSLock(bucket, object string) RWLocker
|
NewObjectNSLock(bucket, object string) RWLocker
|
||||||
// lockObjects locks several objects together, see erasureServerPools.
|
// lockObjects locks several objects together, see erasureServerPools.
|
||||||
lockObjects(ctx context.Context, reqs ...objectLockRequest) (LockContext, func(), error)
|
lockObjects(ctx context.Context, reqs ...objectLockRequest) (LockContext, func(), error)
|
||||||
|
prepareCopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (copyObjectPlan, error)
|
||||||
|
|
||||||
// Storage operations.
|
// Storage operations.
|
||||||
Shutdown(context.Context) error
|
Shutdown(context.Context) error
|
||||||
|
|||||||
@@ -303,13 +303,18 @@ func delOpts(ctx context.Context, r *http.Request, bucket, object string) (opts
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
vc, _, err := checkBucketDeleteMetadata(bucket)
|
||||||
|
if err != nil {
|
||||||
|
return opts, err
|
||||||
|
}
|
||||||
|
|
||||||
opts.DeletePrefix = deletePrefix
|
opts.DeletePrefix = deletePrefix
|
||||||
opts.Versioned = globalBucketVersioningSys.PrefixEnabled(bucket, object)
|
opts.Versioned = vc.PrefixEnabled(object)
|
||||||
// Objects matching prefixes should not leave delete markers,
|
// Objects matching prefixes should not leave delete markers,
|
||||||
// dramatically reduces namespace pollution while keeping the
|
// dramatically reduces namespace pollution while keeping the
|
||||||
// benefits of replication, make sure to apply version suspension
|
// benefits of replication, make sure to apply version suspension
|
||||||
// only at bucket level instead.
|
// only at bucket level instead.
|
||||||
opts.VersionSuspended = globalBucketVersioningSys.Suspended(bucket)
|
opts.VersionSuspended = vc.Suspended()
|
||||||
// For directory objects, delete `null` version permanently.
|
// For directory objects, delete `null` version permanently.
|
||||||
if isDirObject(object) && opts.VersionID == "" {
|
if isDirObject(object) && opts.VersionID == "" {
|
||||||
opts.VersionID = nullVersionID
|
opts.VersionID = nullVersionID
|
||||||
|
|||||||
@@ -916,7 +916,7 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions, h
|
|||||||
}
|
}
|
||||||
// Decompression reader.
|
// Decompression reader.
|
||||||
var dopts []s2.ReaderOption
|
var dopts []s2.ReaderOption
|
||||||
if off > 0 || decOff > 0 {
|
if off > 0 || decOff > 0 || decryptSkip > 0 {
|
||||||
// We are not starting at the beginning, so ignore stream identifiers.
|
// We are not starting at the beginning, so ignore stream identifiers.
|
||||||
dopts = append(dopts, s2.ReaderIgnoreStreamIdentifier())
|
dopts = append(dopts, s2.ReaderIgnoreStreamIdentifier())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,170 @@
|
|||||||
|
// Copyright (c) 2026 PGSTY
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/zip"
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAPIAttributesAndArchiveStoredTags(t *testing.T) {
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t, makeBucketOptions: MakeBucketOptions{VersioningEnabled: true},
|
||||||
|
objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, root auth.Credentials, t *testing.T) {
|
||||||
|
_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var archive bytes.Buffer
|
||||||
|
zw := zip.NewWriter(&archive)
|
||||||
|
entry, err := zw.Create("entry.txt")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err = entry.Write([]byte("private entry")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err = zw.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, indexed := range []bool{false, true} {
|
||||||
|
key := fmt.Sprintf("tagged-%v.zip", indexed)
|
||||||
|
put := func(tag string) ObjectInfo {
|
||||||
|
oi, err := obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, bytes.NewReader(archive.Bytes()), int64(archive.Len()), "", ""), ObjectOptions{Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "tier=" + tag}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return oi
|
||||||
|
}
|
||||||
|
old := put("public")
|
||||||
|
latest := put("restricted")
|
||||||
|
if indexed {
|
||||||
|
if _, err := updateObjectMetadataWithZipInfo(t.Context(), obj, bucket, key, ObjectOptions{VersionID: latest.VersionID}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, anonymous := range []bool{false, true} {
|
||||||
|
for _, effect := range []string{"Allow", "Deny"} {
|
||||||
|
actions := `"s3:GetObject","s3:GetObjectAttributes","s3:GetObjectVersion","s3:GetObjectVersionAttributes"`
|
||||||
|
condition := `"Condition":{"StringEquals":{"s3:ExistingObjectTag/tier":"restricted"}}`
|
||||||
|
statement := fmt.Sprintf(`{"Effect":"%s","Action":[%s],"Resource":["arn:aws:s3:::%s/*"],%s}`, effect, actions, bucket, condition)
|
||||||
|
if effect == "Deny" {
|
||||||
|
statement += fmt.Sprintf(`,{"Effect":"Allow","Action":[%s],"Resource":["arn:aws:s3:::%s/*"]}`, actions, bucket)
|
||||||
|
}
|
||||||
|
statement += fmt.Sprintf(`,{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]}`, bucket)
|
||||||
|
creds := newDeleteAuthzPolicyUser(t, backend, bucket, "["+statement+"]")
|
||||||
|
if anonymous {
|
||||||
|
document := `{"Version":"2012-10-17","Statement":[` + strings.ReplaceAll(statement, `"Effect":`, `"Principal":"*","Effect":`) + `]}`
|
||||||
|
if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketPolicyConfig, []byte(document)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, operation := range []string{"attributes", "version-attributes", http.MethodGet, http.MethodHead, "LIST"} {
|
||||||
|
t.Run(fmt.Sprintf("%s/indexed=%v/anonymous=%v/%s/%s", backend, indexed, anonymous, effect, operation), func(t *testing.T) {
|
||||||
|
verb := http.MethodGet
|
||||||
|
target := getGetObjectURL("", bucket, key+"/entry.txt")
|
||||||
|
headers := map[string]string{xMinIOExtract: "true", xhttp.AmzObjectTagging: "tier=public"}
|
||||||
|
want := http.StatusOK
|
||||||
|
if effect == "Deny" {
|
||||||
|
want = http.StatusForbidden
|
||||||
|
}
|
||||||
|
switch operation {
|
||||||
|
case "attributes", "version-attributes":
|
||||||
|
q := url.Values{"attributes": {""}}
|
||||||
|
if operation == "version-attributes" {
|
||||||
|
q.Set("versionId", old.VersionID)
|
||||||
|
if effect == "Allow" {
|
||||||
|
want = http.StatusForbidden
|
||||||
|
} else {
|
||||||
|
want = http.StatusOK
|
||||||
|
}
|
||||||
|
}
|
||||||
|
target = makeTestTargetURL("", bucket, key, q)
|
||||||
|
headers[xhttp.AmzObjectAttributes] = "ETag,ObjectSize"
|
||||||
|
case http.MethodHead:
|
||||||
|
verb = http.MethodHead
|
||||||
|
case "LIST":
|
||||||
|
target = "/" + bucket + "?list-type=2&prefix=" + key + "/"
|
||||||
|
}
|
||||||
|
req, err := newTestSignedRequestV4(verb, target, 0, nil, creds.AccessKey, creds.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if anonymous {
|
||||||
|
req.Header.Del("Authorization")
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != want {
|
||||||
|
t.Fatalf("status %d, want %d: %s", rec.Code, want, rec.Body.String())
|
||||||
|
}
|
||||||
|
if rec.Code == http.StatusOK && operation == http.MethodGet && rec.Body.String() != "private entry" {
|
||||||
|
t.Fatalf("wrong extracted content: %s", rec.Body.String())
|
||||||
|
}
|
||||||
|
if rec.Code == http.StatusForbidden && strings.Contains(rec.Body.String(), "private entry") {
|
||||||
|
t.Fatal("denied response disclosed content")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Each Attributes action is necessary; a conditional Deny on either
|
||||||
|
// action must win even when the other action is unconditionally allowed.
|
||||||
|
for _, versioned := range []bool{false, true} {
|
||||||
|
for _, deniedAction := range []string{"GetObject", "GetObjectAttributes"} {
|
||||||
|
actions := `"s3:GetObject","s3:GetObjectAttributes"`
|
||||||
|
if versioned {
|
||||||
|
actions = `"s3:GetObjectVersion","s3:GetObjectVersionAttributes"`
|
||||||
|
deniedAction = strings.Replace(deniedAction, "GetObject", "GetObjectVersion", 1)
|
||||||
|
}
|
||||||
|
statements := fmt.Sprintf(`[{"Effect":"Allow","Action":[%s],"Resource":["arn:aws:s3:::%s/*"]},{"Effect":"Deny","Action":["s3:%s"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/tier":"restricted"}}}]`, actions, bucket, deniedAction, bucket)
|
||||||
|
creds := newDeleteAuthzPolicyUser(t, backend, bucket, statements)
|
||||||
|
q := url.Values{"attributes": {""}}
|
||||||
|
if versioned {
|
||||||
|
oi, err := obj.GetObjectInfo(t.Context(), bucket, "tagged-true.zip", ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
q.Set("versionId", oi.VersionID)
|
||||||
|
}
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodGet, makeTestTargetURL("", bucket, "tagged-true.zip", q), 0, nil, creds.AccessKey, creds.SecretKey, map[string]string{xhttp.AmzObjectAttributes: "ETag"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusForbidden {
|
||||||
|
t.Fatalf("%s Deny: %d %s", deniedAction, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
listOnly := newDeleteAuthzPolicyUser(t, backend, bucket, fmt.Sprintf(`[{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]}]`, bucket))
|
||||||
|
for _, extract := range []bool{false, true} {
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodGet, "/"+bucket+"?list-type=2&prefix=tagged-true.zip/", 0, nil, listOnly.AccessKey, listOnly.SecretKey, map[string]string{xMinIOExtract: fmt.Sprint(extract)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, req)
|
||||||
|
want := http.StatusOK
|
||||||
|
if extract {
|
||||||
|
want = http.StatusForbidden
|
||||||
|
}
|
||||||
|
if rec.Code != want {
|
||||||
|
t.Fatalf("list-only extract=%v: %d %s", extract, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
// Copyright (c) 2026 Denis Babchuk
|
||||||
|
//
|
||||||
|
// This file is part of Silo Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/xml"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GetObjectAttributes answers NoSuchKey with AccessDenied to a caller without
|
||||||
|
// s3:ListBucket on the bucket, as S3 does, so that a missing object cannot be
|
||||||
|
// told apart from a forbidden one. A server error, such as lost read quorum,
|
||||||
|
// must reach a caller allowed to read the object as the error it is.
|
||||||
|
func TestGetObjectAttributesReadQuorumLost(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
defer cancel()
|
||||||
|
obj, dirs, err := prepareErasure16(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
z := obj.(*erasureServerPools)
|
||||||
|
previous := newObjectLayerFn()
|
||||||
|
setObjectLayer(z)
|
||||||
|
defer func() {
|
||||||
|
z.Shutdown(context.Background())
|
||||||
|
removeRoots(dirs)
|
||||||
|
setObjectLayer(previous)
|
||||||
|
}()
|
||||||
|
bucket, router, err := initAPIHandlerTest(ctx, z, nil, MakeBucketOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
const object, missing = "object", "missing"
|
||||||
|
if _, err := z.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("data")), 4, "", ""), ObjectOptions{UserDefined: map[string]string{xhttp.AmzObjectTagging: "tier=restricted"}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
read := `{"Effect": "Allow", "Action": ["s3:GetObject", "s3:GetObjectAttributes"], "Resource": ["arn:aws:s3:::` + bucket + `/*"]}`
|
||||||
|
withoutList := newDeleteAuthzPolicyUser(t, "", bucket, "["+read+"]")
|
||||||
|
withList := newDeleteAuthzPolicyUser(t, "", bucket, "["+read+
|
||||||
|
`, {"Effect": "Allow", "Action": ["s3:ListBucket"], "Resource": ["arn:aws:s3:::`+bucket+`"]}]`)
|
||||||
|
|
||||||
|
tagged := newDeleteAuthzPolicyUser(t, "", bucket, `[{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectAttributes"],"Resource":["arn:aws:s3:::`+bucket+`/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/tier":"restricted"}}}]`)
|
||||||
|
unauthorized := newDeleteAuthzPolicyUser(t, "", bucket, `[{"Effect":"Allow","Action":["s3:GetObjectAttributes"],"Resource":["arn:aws:s3:::`+bucket+`/*"]}]`)
|
||||||
|
|
||||||
|
check := func(stage, user string, creds auth.Credentials, key string, want int) {
|
||||||
|
t.Helper()
|
||||||
|
target := makeTestTargetURL("", bucket, key, url.Values{"attributes": []string{""}})
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodGet, target, 0, nil, creds.AccessKey, creds.SecretKey,
|
||||||
|
map[string]string{xhttp.AmzObjectAttributes: "ETag"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != want {
|
||||||
|
var apiErr APIErrorResponse
|
||||||
|
_ = xml.Unmarshal(rec.Body.Bytes(), &apiErr)
|
||||||
|
t.Errorf("%s: GetObjectAttributes %s by %s: status %d %s, want %d", stage, key, user, rec.Code, apiErr.Code, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
check("healthy", "withoutList", withoutList, object, http.StatusOK)
|
||||||
|
check("healthy", "withList", withList, object, http.StatusOK)
|
||||||
|
check("healthy", "withoutList", withoutList, missing, http.StatusForbidden)
|
||||||
|
check("healthy", "withList", withList, missing, http.StatusNotFound)
|
||||||
|
|
||||||
|
check("healthy", "tagged", tagged, object, http.StatusOK)
|
||||||
|
check("healthy", "unauthorized", unauthorized, object, http.StatusForbidden)
|
||||||
|
|
||||||
|
defer objectReadFault(z, bucket, object, 10)()
|
||||||
|
check("read quorum lost", "withoutList", withoutList, object, http.StatusServiceUnavailable)
|
||||||
|
check("read quorum lost", "withList", withList, object, http.StatusServiceUnavailable)
|
||||||
|
check("read quorum lost", "tagged", tagged, object, http.StatusServiceUnavailable)
|
||||||
|
check("read quorum lost", "unauthorized", unauthorized, object, http.StatusForbidden)
|
||||||
|
}
|
||||||
@@ -0,0 +1,183 @@
|
|||||||
|
// Copyright (c) 2026 Feng Ruohang
|
||||||
|
//
|
||||||
|
// This file is part of Silo Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
"github.com/minio/minio/internal/crypto"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
"github.com/minio/minio/internal/kms"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Indexed reads may skip into the first decrypted block while both the disk
|
||||||
|
// offset and the remaining decompressed skip are zero. The reader must still
|
||||||
|
// accept a compression block without the original stream identifier.
|
||||||
|
func TestAPICompressedEncryptedRange(t *testing.T) {
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
|
||||||
|
previousKMS, previousTLS := GlobalKMS, globalIsTLS
|
||||||
|
var err error
|
||||||
|
GlobalKMS, err = kms.NewBuiltin("range-key", bytes.Repeat([]byte{0x42}, 32))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
globalIsTLS = true
|
||||||
|
defer func() { GlobalKMS, globalIsTLS = previousKMS, previousTLS }()
|
||||||
|
restore := setCopyChecksumCompression(true)
|
||||||
|
defer restore()
|
||||||
|
part := bytes.Repeat([]byte("compressible-A"), 1<<20)
|
||||||
|
for _, mode := range []string{"compressed", "sse-s3", "sse-kms"} {
|
||||||
|
for _, multipart := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprintf("%s/%s/multipart=%t", backend, mode, multipart), func(t *testing.T) {
|
||||||
|
key := fmt.Sprintf("range-%s-%t.txt", mode, multipart)
|
||||||
|
headers := map[string]string{}
|
||||||
|
switch mode {
|
||||||
|
case "sse-s3":
|
||||||
|
headers[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
|
||||||
|
case "sse-kms":
|
||||||
|
headers[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
|
||||||
|
headers[xhttp.AmzServerSideEncryptionKmsID] = "range-key"
|
||||||
|
}
|
||||||
|
data := part
|
||||||
|
if multipart {
|
||||||
|
data = attributesPartsUpload(t, router, cred, bucket, key, headers,
|
||||||
|
[][]byte{part, bytes.Repeat([]byte("compressible-B"), 1<<20)}, []int{1, 2})
|
||||||
|
} else {
|
||||||
|
putCopyChecksumSource(t, router, cred, bucket, key, data, headers)
|
||||||
|
}
|
||||||
|
oi, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, encrypted := crypto.IsEncrypted(oi.UserDefined)
|
||||||
|
if !oi.IsCompressed() || encrypted != (mode != "compressed") || len(oi.Parts) == 0 || len(oi.Parts[0].Index) == 0 {
|
||||||
|
t.Fatal("fixture must have indexed compression and the requested encryption")
|
||||||
|
}
|
||||||
|
if encrypted {
|
||||||
|
off, decOff, _, skip, _ := getCompressedOffsets(oi, 1<<20, func(b []byte) ([]byte, error) { return oi.compressionIndexDecrypt(b, nil) })
|
||||||
|
if off != 0 || decOff != 0 || skip <= 0 {
|
||||||
|
t.Fatalf("fixture misses boundary: off=%d decOff=%d decryptSkip=%d", off, decOff, skip)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
offsets := []int{0, 100, 1 << 20, 1<<20 + 1, len(data) - 1000}
|
||||||
|
if multipart {
|
||||||
|
offsets = append(offsets, len(part)-500, len(part), len(part)+1<<20)
|
||||||
|
}
|
||||||
|
for _, start := range offsets {
|
||||||
|
rec := attributesPartsSignedRequest(t, router, cred, http.MethodGet, getGetObjectURL("", bucket, key), nil,
|
||||||
|
map[string]string{"Range": fmt.Sprintf("bytes=%d-%d", start, start+999)})
|
||||||
|
if rec.Code != http.StatusPartialContent || !bytes.Equal(rec.Body.Bytes(), data[start:start+1000]) {
|
||||||
|
t.Errorf("range at %d: status=%d bytes=%d, want 206 and exact slice", start, rec.Code, rec.Body.Len())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rec := attributesPartsSignedRequest(t, router, cred, http.MethodGet, getGetObjectURL("", bucket, key), nil, nil)
|
||||||
|
if rec.Code != http.StatusOK || !bytes.Equal(rec.Body.Bytes(), data) {
|
||||||
|
t.Fatal("full read changed after ranged reads")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPICompressedEncryptedRangeCorruptBlockRejected(t *testing.T) {
|
||||||
|
resetTestGlobals()
|
||||||
|
ctx := GlobalContext
|
||||||
|
obj, fsDir, err := prepareFSWithDisks(ctx, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer removeAll(fsDir)
|
||||||
|
defer obj.Shutdown(ctx)
|
||||||
|
|
||||||
|
bucket, router, err := initAPIHandlerTest(ctx, obj, nil, MakeBucketOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := newTestConfig(globalMinioDefaultRegion, obj); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
previousKMS, previousTLS := GlobalKMS, globalIsTLS
|
||||||
|
GlobalKMS, err = kms.NewBuiltin("range-key", bytes.Repeat([]byte{0x42}, 32))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
globalIsTLS = true
|
||||||
|
defer func() { GlobalKMS, globalIsTLS = previousKMS, previousTLS }()
|
||||||
|
restore := setCopyChecksumCompression(true)
|
||||||
|
defer restore()
|
||||||
|
|
||||||
|
cred := globalActiveCred
|
||||||
|
payload := bytes.Repeat([]byte("compressible-A"), 1<<20)
|
||||||
|
|
||||||
|
for _, mode := range []string{"compressed", "sse-kms"} {
|
||||||
|
key := "corrupt-" + mode + ".txt"
|
||||||
|
headers := map[string]string{}
|
||||||
|
if mode == "sse-kms" {
|
||||||
|
headers[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
|
||||||
|
headers[xhttp.AmzServerSideEncryptionKmsID] = "range-key"
|
||||||
|
}
|
||||||
|
putCopyChecksumSource(t, router, cred, bucket, key, payload, headers)
|
||||||
|
|
||||||
|
rec := attributesPartsSignedRequest(t, router, cred, http.MethodGet, getGetObjectURL("", bucket, key), nil,
|
||||||
|
map[string]string{"Range": fmt.Sprintf("bytes=%d-%d", 1<<20, (1<<20)+999)})
|
||||||
|
if rec.Code != http.StatusPartialContent || !bytes.Equal(rec.Body.Bytes(), payload[1<<20:(1<<20)+1000]) {
|
||||||
|
t.Fatalf("%s pre-corrupt range failed: code=%d", mode, rec.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
corruptedAny := false
|
||||||
|
_ = filepath.Walk(fsDir, func(path string, info os.FileInfo, walkErr error) error {
|
||||||
|
if walkErr != nil || info.IsDir() || !strings.Contains(path, key) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
raw, rerr := os.ReadFile(path)
|
||||||
|
if rerr != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if strings.HasSuffix(path, "part.1") {
|
||||||
|
raw[len(raw)/2] ^= 0xff
|
||||||
|
_ = os.WriteFile(path, raw, info.Mode())
|
||||||
|
corruptedAny = true
|
||||||
|
} else if strings.HasSuffix(path, xlStorageFormatFile) && len(raw) > 256 {
|
||||||
|
raw[len(raw)-64] ^= 0xff
|
||||||
|
_ = os.WriteFile(path, raw, info.Mode())
|
||||||
|
corruptedAny = true
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if !corruptedAny {
|
||||||
|
t.Fatalf("%s: failed to locate backing storage file to corrupt", mode)
|
||||||
|
}
|
||||||
|
|
||||||
|
recAfter := attributesPartsSignedRequest(t, router, cred, http.MethodGet, getGetObjectURL("", bucket, key), nil,
|
||||||
|
map[string]string{"Range": fmt.Sprintf("bytes=%d-%d", 1<<20, (1<<20)+999)})
|
||||||
|
if recAfter.Code == http.StatusPartialContent && recAfter.Body.Len() == 1000 {
|
||||||
|
t.Fatalf("%s: corrupted range unexpectedly returned 206 with 1000 bytes", mode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,408 @@
|
|||||||
|
// Copyright (c) 2026 PGSTY
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/md5"
|
||||||
|
"encoding/base64"
|
||||||
|
"fmt"
|
||||||
|
"maps"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/hash"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
"github.com/minio/minio/internal/kms"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Place uploaded versions in specific pools, while retaining the real HTTP
|
||||||
|
// encryption/compression pipeline and the normal pools-layer copy operation.
|
||||||
|
type copyUploadPool struct {
|
||||||
|
*erasureServerPools
|
||||||
|
uploadPool int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (z *copyUploadPool) PutObject(ctx context.Context, bucket, object string, r *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
|
||||||
|
return z.serverPools[z.uploadPool].PutObject(ctx, bucket, object, r, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPICopyHistoricalVersionAcrossPools(t *testing.T) {
|
||||||
|
z, _ := consistencyPools(t)
|
||||||
|
placed := ©UploadPool{erasureServerPools: z}
|
||||||
|
bucket, router, err := initAPIHandlerTest(t.Context(), placed, nil, MakeBucketOptions{VersioningEnabled: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
setObjectLayer(placed)
|
||||||
|
if _, err = globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
previousKMS, previousTLS := GlobalKMS, globalIsTLS
|
||||||
|
GlobalKMS, err = kms.NewBuiltin("copy-key", bytes.Repeat([]byte{0x33}, 32))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
globalIsTLS = true
|
||||||
|
defer func() { GlobalKMS, globalIsTLS = previousKMS, previousTLS }()
|
||||||
|
customer := func(value byte) map[string]string {
|
||||||
|
key := bytes.Repeat([]byte{value}, 32)
|
||||||
|
sum := md5.Sum(key)
|
||||||
|
return map[string]string{xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES, xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key), xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(sum[:])}
|
||||||
|
}
|
||||||
|
for _, tc := range []struct{ mode, movement string }{
|
||||||
|
{"plain", "active"},
|
||||||
|
{"compressed", "active"},
|
||||||
|
{"sse-s3", "active"},
|
||||||
|
{"sse-c", "active"},
|
||||||
|
{"sse-kms", "active"},
|
||||||
|
{"sse-c", "decommission"},
|
||||||
|
{"sse-kms", "rebalance"},
|
||||||
|
} {
|
||||||
|
mode := tc.mode
|
||||||
|
for _, sourcePool := range []int{0, 1} {
|
||||||
|
t.Run(fmt.Sprintf("%s/%s/source=%d", mode, tc.movement, sourcePool), func(t *testing.T) {
|
||||||
|
compressed := mode == "compressed"
|
||||||
|
restore := setCopyChecksumCompression(true)
|
||||||
|
globalCompressConfigMu.Lock()
|
||||||
|
globalCompressConfig.Enabled = compressed
|
||||||
|
globalCompressConfigMu.Unlock()
|
||||||
|
defer restore()
|
||||||
|
sourceHeaders := map[string]string{xhttp.AmzChecksumCRC32: ""}
|
||||||
|
if mode == "sse-s3" {
|
||||||
|
sourceHeaders[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
|
||||||
|
}
|
||||||
|
if mode == "sse-kms" {
|
||||||
|
sourceHeaders[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
|
||||||
|
sourceHeaders[xhttp.AmzServerSideEncryptionKmsID] = "copy-key"
|
||||||
|
sourceHeaders[xhttp.AmzServerSideEncryptionKmsContext] = base64.StdEncoding.EncodeToString([]byte(`{"tenant":"source"}`))
|
||||||
|
}
|
||||||
|
if mode == "sse-c" {
|
||||||
|
maps.Copy(sourceHeaders, customer(0x11))
|
||||||
|
}
|
||||||
|
pattern := []byte("historical encrypted compressed content-")
|
||||||
|
repeats := 8192
|
||||||
|
if mode == "plain" {
|
||||||
|
repeats = compMinIndexSize/len(pattern) + 1
|
||||||
|
}
|
||||||
|
data := bytes.Repeat(pattern, repeats)
|
||||||
|
sourceHeaders[xhttp.AmzChecksumCRC32] = mustChecksum(t, hash.ChecksumCRC32, data)
|
||||||
|
key := fmt.Sprintf("copy-%s-%s-%d.txt", mode, tc.movement, sourcePool)
|
||||||
|
placed.uploadPool = sourcePool
|
||||||
|
putCopyChecksumSource(t, router, globalActiveCred, bucket, key, data, sourceHeaders)
|
||||||
|
original, err := z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if original.VersionID == "" {
|
||||||
|
t.Fatal("fixture has no version ID")
|
||||||
|
}
|
||||||
|
if _, err := z.serverPools[sourcePool].GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: original.VersionID}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if original.IsCompressed() != compressed {
|
||||||
|
t.Fatalf("fixture compressed=%v", original.IsCompressed())
|
||||||
|
}
|
||||||
|
placed.uploadPool = 1 - sourcePool
|
||||||
|
putCopyChecksumSource(t, router, globalActiveCred, bucket, key, []byte("newer version"), nil)
|
||||||
|
newer, err := z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
setCopyMovingPool(t, z, sourcePool, tc.movement)
|
||||||
|
// Toggle compression so metadata has to describe the bytes written now.
|
||||||
|
restore()
|
||||||
|
restoreTarget := setCopyChecksumCompression(true)
|
||||||
|
globalCompressConfigMu.Lock()
|
||||||
|
globalCompressConfig.Enabled = !compressed
|
||||||
|
globalCompressConfigMu.Unlock()
|
||||||
|
defer restoreTarget()
|
||||||
|
headers := map[string]string{xhttp.AmzMetadataDirective: "REPLACE"}
|
||||||
|
getHeaders := map[string]string{}
|
||||||
|
if mode == "sse-s3" {
|
||||||
|
headers[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
|
||||||
|
}
|
||||||
|
if mode == "sse-kms" {
|
||||||
|
headers[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
|
||||||
|
headers[xhttp.AmzServerSideEncryptionKmsID] = "copy-key"
|
||||||
|
headers[xhttp.AmzServerSideEncryptionKmsContext] = base64.StdEncoding.EncodeToString([]byte(`{"tenant":"destination"}`))
|
||||||
|
}
|
||||||
|
if mode == "sse-c" {
|
||||||
|
maps.Copy(headers, customer(0x22))
|
||||||
|
maps.Copy(getHeaders, customer(0x22))
|
||||||
|
old := customer(0x11)
|
||||||
|
headers[xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm] = old[xhttp.AmzServerSideEncryptionCustomerAlgorithm]
|
||||||
|
headers[xhttp.AmzServerSideEncryptionCopyCustomerKey] = old[xhttp.AmzServerSideEncryptionCustomerKey]
|
||||||
|
headers[xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5] = old[xhttp.AmzServerSideEncryptionCustomerKeyMD5]
|
||||||
|
}
|
||||||
|
rec := copyChecksumRequest(t, router, globalActiveCred, bucket, key+"?versionId="+original.VersionID, key, headers)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("copy: %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
copied, err := z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{})
|
||||||
|
if err != nil || copied.VersionID == "" || copied.VersionID == original.VersionID || copied.VersionID == newer.VersionID {
|
||||||
|
t.Fatalf("copy did not create a version: %+v %v", copied, err)
|
||||||
|
}
|
||||||
|
if copied.IsCompressed() != (!compressed && mode != "sse-c") {
|
||||||
|
t.Fatalf("copy did not use destination compression setting: %v", copied.IsCompressed())
|
||||||
|
}
|
||||||
|
if copied.IsCompressed() && len(data) > compMinIndexSize {
|
||||||
|
if len(copied.Parts) == 0 || len(copied.Parts[0].Index) == 0 {
|
||||||
|
t.Fatalf("compressed multi-block copy lost seek index: %+v", copied.Parts)
|
||||||
|
}
|
||||||
|
rangeHeaders := maps.Clone(getHeaders)
|
||||||
|
rangeHeaders["Range"] = "bytes=6000000-6000127"
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucket, key)+"?versionId="+copied.VersionID, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, rangeHeaders)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rrec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rrec, req)
|
||||||
|
if rrec.Code != http.StatusPartialContent || !bytes.Equal(rrec.Body.Bytes(), data[6000000:6000128]) {
|
||||||
|
t.Fatalf("ranged GET on compressed copy: %d len=%d", rrec.Code, rrec.Body.Len())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if mode == "sse-kms" {
|
||||||
|
assertFederationKMSContext(t, copied, headers[xhttp.AmzServerSideEncryptionKmsContext])
|
||||||
|
assertFederationKMSContext(t, original, sourceHeaders[xhttp.AmzServerSideEncryptionKmsContext])
|
||||||
|
}
|
||||||
|
if _, err = z.serverPools[1-sourcePool].GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: copied.VersionID}); err != nil {
|
||||||
|
t.Fatalf("copy was not placed with destination: %v", err)
|
||||||
|
}
|
||||||
|
read := func(version string, headers map[string]string, want []byte) {
|
||||||
|
t.Helper()
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucket, key)+"?versionId="+version, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK || !bytes.Equal(rec.Body.Bytes(), want) {
|
||||||
|
t.Fatalf("GET version %s: %d body length=%d, want %d", version, rec.Code, rec.Body.Len(), len(want))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
read(copied.VersionID, getHeaders, data)
|
||||||
|
oldHeaders := map[string]string{}
|
||||||
|
if mode == "sse-c" {
|
||||||
|
oldHeaders = customer(0x11)
|
||||||
|
}
|
||||||
|
read(original.VersionID, oldHeaders, data)
|
||||||
|
read(newer.VersionID, nil, []byte("newer version"))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Internal metadata updates do not all set MetadataChg. They must still
|
||||||
|
// address the requested historical version, and must never create a new one.
|
||||||
|
func TestPoolsCopyExactVersionMetadata(t *testing.T) {
|
||||||
|
z, bucket := consistencyPools(t)
|
||||||
|
for sourcePool := range 2 {
|
||||||
|
key := fmt.Sprintf("exact-%d", sourcePool)
|
||||||
|
older := putConsistencyObject(t, z, bucket, key, sourcePool, "old data", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
|
||||||
|
newer := putConsistencyObject(t, z, bucket, key, 1-sourcePool, "new data", ObjectOptions{Versioned: true, MTime: UTCNow()})
|
||||||
|
gr, err := z.GetObjectNInfo(t.Context(), bucket, key, nil, nil, ObjectOptions{VersionID: older.VersionID, NoLock: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
info := gr.ObjInfo
|
||||||
|
info.Reader, err = hash.NewReader(t.Context(), gr, info.Size, "", "", info.Size)
|
||||||
|
if err != nil {
|
||||||
|
gr.Close()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
info.PutObjReader = NewPutObjReader(info.Reader)
|
||||||
|
info.metadataOnly = true
|
||||||
|
info.UserDefined["updated"] = "yes"
|
||||||
|
_, err = z.CopyObject(t.Context(), bucket, key, bucket, key, info, ObjectOptions{VersionID: older.VersionID}, ObjectOptions{Versioned: true, VersionID: older.VersionID, MTime: older.ModTime})
|
||||||
|
gr.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := z.serverPools[sourcePool].GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: older.VersionID})
|
||||||
|
if err != nil || got.UserDefined["updated"] != "yes" {
|
||||||
|
t.Fatalf("historical metadata not updated: %v %v", got.UserDefined, err)
|
||||||
|
}
|
||||||
|
got, err = z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{})
|
||||||
|
if err != nil || got.VersionID != newer.VersionID || got.UserDefined["updated"] != "" {
|
||||||
|
t.Fatalf("latest changed: %+v %v", got, err)
|
||||||
|
}
|
||||||
|
if got := readCopyChecksumObject(t, z, bucket, key, ObjectOptions{VersionID: older.VersionID}); string(got) != "old data" {
|
||||||
|
t.Fatalf("historical body changed: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func setCopyMovingPool(t *testing.T, z *erasureServerPools, pool int, movement string) {
|
||||||
|
t.Helper()
|
||||||
|
switch movement {
|
||||||
|
case "decommission":
|
||||||
|
z.poolMetaMutex.Lock()
|
||||||
|
z.poolMeta.Pools[pool].Decommission = &PoolDecommissionInfo{}
|
||||||
|
z.poolMetaMutex.Unlock()
|
||||||
|
t.Cleanup(func() {
|
||||||
|
z.poolMetaMutex.Lock()
|
||||||
|
z.poolMeta.Pools[pool].Decommission = nil
|
||||||
|
z.poolMetaMutex.Unlock()
|
||||||
|
})
|
||||||
|
case "rebalance":
|
||||||
|
z.rebalMu.Lock()
|
||||||
|
z.rebalMeta = &rebalanceMeta{PoolStats: []*rebalanceStats{{}, {}}}
|
||||||
|
z.rebalMeta.PoolStats[pool] = &rebalanceStats{Participating: true, Info: rebalanceInfo{Status: rebalStarted}}
|
||||||
|
z.rebalMu.Unlock()
|
||||||
|
t.Cleanup(func() { z.rebalMu.Lock(); z.rebalMeta = nil; z.rebalMu.Unlock() })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The content that a metadata-only replica preserves must be the same content
|
||||||
|
// that reads selected, even when write allocation would skip its pool.
|
||||||
|
func TestCopyReplicaMetadataMovingPool(t *testing.T) {
|
||||||
|
z, _ := consistencyPools(t)
|
||||||
|
bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{LockEnabled: true})
|
||||||
|
mustIAM(t, err)
|
||||||
|
setObjectLayer(z)
|
||||||
|
_, err = globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
|
||||||
|
mustIAM(t, err)
|
||||||
|
const body = "current-body"
|
||||||
|
for _, api := range []bool{false, true} {
|
||||||
|
for sourcePool := range 2 {
|
||||||
|
for _, movement := range []string{"decommission", "rebalance"} {
|
||||||
|
for _, kind := range []string{"uuid", "null"} {
|
||||||
|
for _, duplicate := range []string{"absent", "older", "tie"} {
|
||||||
|
// The lower pool index wins equal ModTimes, just as on reads.
|
||||||
|
if duplicate == "tie" && sourcePool != 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
t.Run(fmt.Sprintf("api=%t/source=%d/%s/%s/%s", api, sourcePool, movement, kind, duplicate), func(t *testing.T) {
|
||||||
|
key := fmt.Sprintf("metadata-%t-%d-%s-%s-%s", api, sourcePool, movement, kind, duplicate)
|
||||||
|
oi := putConsistencyObject(t, z, bucket, key, sourcePool, body, ObjectOptions{
|
||||||
|
Versioned: kind == "uuid", MTime: UTCNow().Add(-time.Minute),
|
||||||
|
WantChecksum: hash.NewChecksumFromData(hash.ChecksumCRC32, []byte(body)),
|
||||||
|
})
|
||||||
|
version := oi.VersionID
|
||||||
|
if version == "" {
|
||||||
|
version = nullVersionID
|
||||||
|
}
|
||||||
|
if duplicate != "absent" {
|
||||||
|
stamp := oi.ModTime
|
||||||
|
if duplicate == "older" {
|
||||||
|
stamp = stamp.Add(-time.Hour)
|
||||||
|
}
|
||||||
|
putConsistencyObject(t, z, bucket, key, 1-sourcePool, "stale---body", ObjectOptions{
|
||||||
|
Versioned: kind == "uuid", VersionID: version, MTime: stamp,
|
||||||
|
UserDefined: poolLockMetadata("", "ON", "", objectLockTestStamp1000),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if got := readCopyChecksumObject(t, z, bucket, key, ObjectOptions{VersionID: version}); string(got) != body {
|
||||||
|
t.Fatalf("fixture: %q", got)
|
||||||
|
}
|
||||||
|
setCopyMovingPool(t, z, sourcePool, movement)
|
||||||
|
if api {
|
||||||
|
headers := map[string]string{
|
||||||
|
xhttp.AmzCopySource: "/" + bucket + "/" + key + "?versionId=" + version,
|
||||||
|
xhttp.AmzMetadataDirective: "REPLACE", "X-Amz-Meta-Updated": "yes",
|
||||||
|
xhttp.MinIOSourceReplicationRequest: "true", xhttp.AmzBucketReplicationStatus: "REPLICA",
|
||||||
|
}
|
||||||
|
rec := r5Request(t, router, globalActiveCred, http.MethodPut, "/"+bucket+"/"+key+"?versionId="+version, "", headers)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("metadata copy: %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, []byte(body))
|
||||||
|
} else {
|
||||||
|
oi.metadataOnly = true
|
||||||
|
oi.UserDefined["X-Amz-Meta-Updated"] = "yes"
|
||||||
|
_, err = z.CopyObject(t.Context(), bucket, key, bucket, key, oi,
|
||||||
|
ObjectOptions{VersionID: version}, ObjectOptions{Versioned: kind == "uuid", VersionID: version, MTime: oi.ModTime, ReplicaLockReconcile: true})
|
||||||
|
mustIAM(t, err)
|
||||||
|
}
|
||||||
|
if got := readCopyChecksumObject(t, z, bucket, key, ObjectOptions{VersionID: version}); string(got) != body {
|
||||||
|
t.Fatalf("metadata copy replaced content: %q", got)
|
||||||
|
}
|
||||||
|
stored, err := z.serverPools[sourcePool].GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: version})
|
||||||
|
mustIAM(t, err)
|
||||||
|
if stored.UserDefined["X-Amz-Meta-Updated"] != "yes" || !bytes.Equal(stored.Checksum, oi.Checksum) {
|
||||||
|
t.Fatalf("metadata/checksum: %v, %x; want checksum %x", stored.UserDefined, stored.Checksum, oi.Checksum)
|
||||||
|
}
|
||||||
|
if duplicate != "absent" && storedObjectLockState(stored.UserDefined).legalHold != "ON" {
|
||||||
|
t.Fatal("lost protection from the retired copy")
|
||||||
|
}
|
||||||
|
_, err = z.serverPools[1-sourcePool].GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: version})
|
||||||
|
if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||||
|
t.Fatalf("stale copy not retired: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPICopyRewriteMovingPool(t *testing.T) {
|
||||||
|
z, _ := consistencyPools(t)
|
||||||
|
placed := ©UploadPool{erasureServerPools: z}
|
||||||
|
bucket, router, err := initAPIHandlerTest(t.Context(), placed, nil, MakeBucketOptions{})
|
||||||
|
mustIAM(t, err)
|
||||||
|
setObjectLayer(placed)
|
||||||
|
previousKMS := GlobalKMS
|
||||||
|
GlobalKMS, err = kms.NewBuiltin("copy-key", bytes.Repeat([]byte{0x33}, 32))
|
||||||
|
mustIAM(t, err)
|
||||||
|
defer func() { GlobalKMS = previousKMS }()
|
||||||
|
for sourcePool := range 2 {
|
||||||
|
for _, movement := range []string{"decommission", "rebalance"} {
|
||||||
|
for _, mode := range []string{"sse-s3", "checksum", "default-checksum", "invalid-self-copy"} {
|
||||||
|
t.Run(fmt.Sprintf("source=%d/%s/%s", sourcePool, movement, mode), func(t *testing.T) {
|
||||||
|
key := fmt.Sprintf("rewrite-%d-%s-%s", sourcePool, movement, mode)
|
||||||
|
data := []byte("rewriting a source on a draining pool")
|
||||||
|
sourceHeaders := map[string]string{}
|
||||||
|
if mode != "default-checksum" {
|
||||||
|
sourceHeaders[xhttp.AmzChecksumCRC32] = mustChecksum(t, hash.ChecksumCRC32, data)
|
||||||
|
}
|
||||||
|
placed.uploadPool = sourcePool
|
||||||
|
putCopyChecksumSource(t, router, globalActiveCred, bucket, key, data, sourceHeaders)
|
||||||
|
setCopyMovingPool(t, z, sourcePool, movement)
|
||||||
|
headers := map[string]string{xhttp.AmzMetadataDirective: "REPLACE"}
|
||||||
|
wantChecksum := hash.ChecksumCRC32
|
||||||
|
switch mode {
|
||||||
|
case "sse-s3":
|
||||||
|
headers[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
|
||||||
|
case "checksum", "invalid-self-copy":
|
||||||
|
headers[xhttp.AmzChecksumAlgo] = "SHA256"
|
||||||
|
wantChecksum = hash.ChecksumSHA256
|
||||||
|
if mode == "invalid-self-copy" {
|
||||||
|
delete(headers, xhttp.AmzMetadataDirective)
|
||||||
|
}
|
||||||
|
case "default-checksum":
|
||||||
|
wantChecksum = hash.ChecksumCRC64NVME
|
||||||
|
}
|
||||||
|
rec := copyChecksumRequest(t, router, globalActiveCred, bucket, key, key, headers)
|
||||||
|
if mode == "invalid-self-copy" {
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("invalid self-copy: %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
_, err := z.serverPools[1-sourcePool].GetObjectInfo(t.Context(), bucket, key, ObjectOptions{})
|
||||||
|
if !isErrObjectNotFound(err) {
|
||||||
|
t.Fatalf("rejected copy wrote a destination: %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("rewrite: %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
assertCopyChecksumResponse(t, rec, wantChecksum, data)
|
||||||
|
if got := readCopyChecksumObject(t, z, bucket, key, ObjectOptions{}); !bytes.Equal(got, data) {
|
||||||
|
t.Fatalf("rewritten body: %q", got)
|
||||||
|
}
|
||||||
|
if _, err := z.serverPools[1-sourcePool].GetObjectInfo(t.Context(), bucket, key, ObjectOptions{}); err != nil {
|
||||||
|
t.Fatalf("rewrite did not reach active pool: %v", err)
|
||||||
|
}
|
||||||
|
assertCopyChecksum(t, z, bucket, key, wantChecksum, data, false, nil)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -35,22 +35,15 @@ import (
|
|||||||
// the single-disk and 16-disk fixtures are single-pool backends.
|
// the single-disk and 16-disk fixtures are single-pool backends.
|
||||||
func TestAPICopyObjectReplicaTaggingTimestampUnderKMS(t *testing.T) {
|
func TestAPICopyObjectReplicaTaggingTimestampUnderKMS(t *testing.T) {
|
||||||
defer DetectTestLeak(t)()
|
defer DetectTestLeak(t)()
|
||||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPICopyObjectReplicaTaggingTimestampUnderKMS})
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t, objAPITest: testAPICopyObjectReplicaTaggingTimestampUnderKMS,
|
||||||
|
// Ignore host free-space percentage while retaining real disk I/O.
|
||||||
|
// Install once before IAM starts and keep the disks until shutdown.
|
||||||
|
wrapDisk: func(disk StorageAPI) StorageAPI { return tagTestCapacityDisk{StorageAPI: disk} },
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func testAPICopyObjectReplicaTaggingTimestampUnderKMS(obj ObjectLayer, instance, bucket string, router http.Handler, creds auth.Credentials, t *testing.T) {
|
func testAPICopyObjectReplicaTaggingTimestampUnderKMS(obj ObjectLayer, instance, bucket string, router http.Handler, creds auth.Credentials, t *testing.T) {
|
||||||
// Ignore the host free-space percentage while retaining real disk I/O.
|
|
||||||
for _, pool := range obj.(*erasureServerPools).serverPools {
|
|
||||||
for _, set := range pool.sets {
|
|
||||||
original := set.getDisks
|
|
||||||
disks := append([]StorageAPI(nil), original()...)
|
|
||||||
for i := range disks {
|
|
||||||
disks[i] = tagTestCapacityDisk{StorageAPI: disks[i]}
|
|
||||||
}
|
|
||||||
set.getDisks = func() []StorageAPI { return disks }
|
|
||||||
defer func() { set.getDisks = original }()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
oldKMS, oldAuto := GlobalKMS, globalAutoEncryption
|
oldKMS, oldAuto := GlobalKMS, globalAutoEncryption
|
||||||
GlobalKMS = kms.NewStub("replica-tags-key")
|
GlobalKMS = kms.NewStub("replica-tags-key")
|
||||||
globalAutoEncryption = false
|
globalAutoEncryption = false
|
||||||
|
|||||||
+399
-104
@@ -139,7 +139,13 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
|
|||||||
|
|
||||||
// Check for auth type to return S3 compatible error.
|
// Check for auth type to return S3 compatible error.
|
||||||
// type to return the correct error (NoSuchKey vs AccessDenied)
|
// type to return the correct error (NoSuchKey vs AccessDenied)
|
||||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, bucket, object); s3Error != ErrNone {
|
logger.GetReqInfo(ctx).BucketName = bucket
|
||||||
|
logger.GetReqInfo(ctx).ObjectName = object
|
||||||
|
s3Error := authenticateRequest(ctx, r, policy.GetObjectAction)
|
||||||
|
if s3Error == ErrNone {
|
||||||
|
s3Error = authorizeVersionedRequestWithPotentialTags(ctx, r, policy.GetObjectAction, opts.VersionID)
|
||||||
|
}
|
||||||
|
if s3Error != ErrNone {
|
||||||
if getRequestAuthType(r) == authTypeAnonymous {
|
if getRequestAuthType(r) == authTypeAnonymous {
|
||||||
// As per "Permission" section in
|
// As per "Permission" section in
|
||||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html
|
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html
|
||||||
@@ -199,6 +205,19 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
|
|||||||
gopts.NoLock = true // We already have a lock, we can live with it.
|
gopts.NoLock = true // We already have a lock, we can live with it.
|
||||||
objInfo, err := getObjectInfo(ctx, bucket, object, gopts)
|
objInfo, err := getObjectInfo(ctx, bucket, object, gopts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if s3Error := authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, opts.VersionID, err); s3Error != ErrNone {
|
||||||
|
if getRequestAuthType(r) == authTypeAnonymous && toAPIError(ctx, err).Code == "NoSuchKey" &&
|
||||||
|
globalPolicySys.IsAllowed(policy.BucketPolicyArgs{
|
||||||
|
Action: policy.ListBucketAction,
|
||||||
|
BucketName: bucket,
|
||||||
|
ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
|
||||||
|
IsOwner: false,
|
||||||
|
}) {
|
||||||
|
s3Error = ErrNoSuchKey
|
||||||
|
}
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
// Versioning enabled quite possibly object is deleted might be delete-marker
|
// Versioning enabled quite possibly object is deleted might be delete-marker
|
||||||
// if present set the headers, no idea why AWS S3 sets these headers.
|
// if present set the headers, no idea why AWS S3 sets these headers.
|
||||||
if objInfo.VersionID != "" && objInfo.DeleteMarker {
|
if objInfo.VersionID != "" && objInfo.DeleteMarker {
|
||||||
@@ -208,10 +227,14 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
|
|||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, objInfo.UserTags); s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// filter object lock metadata if permission does not permit
|
// filter object lock metadata if permission does not permit
|
||||||
getRetPerms := checkRequestAuthType(ctx, r, policy.GetObjectRetentionAction, bucket, object)
|
getRetPerms := authorizeRequest(ctx, r, policy.GetObjectRetentionAction)
|
||||||
legalHoldPerms := checkRequestAuthType(ctx, r, policy.GetObjectLegalHoldAction, bucket, object)
|
legalHoldPerms := authorizeRequest(ctx, r, policy.GetObjectLegalHoldAction)
|
||||||
|
|
||||||
// filter object lock metadata if permission does not permit
|
// filter object lock metadata if permission does not permit
|
||||||
objInfo.UserDefined = objectlock.FilterObjectLockMetadata(objInfo.UserDefined, getRetPerms != ErrNone, legalHoldPerms != ErrNone)
|
objInfo.UserDefined = objectlock.FilterObjectLockMetadata(objInfo.UserDefined, getRetPerms != ErrNone, legalHoldPerms != ErrNone)
|
||||||
@@ -326,6 +349,8 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
|
|||||||
|
|
||||||
// Check for auth type to return S3 compatible error.
|
// Check for auth type to return S3 compatible error.
|
||||||
// type to return the correct error (NoSuchKey vs AccessDenied)
|
// type to return the correct error (NoSuchKey vs AccessDenied)
|
||||||
|
logger.GetReqInfo(ctx).BucketName = bucket
|
||||||
|
logger.GetReqInfo(ctx).ObjectName = object
|
||||||
if s3Error := authenticateRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone {
|
if s3Error := authenticateRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone {
|
||||||
if getRequestAuthType(r) == authTypeAnonymous {
|
if getRequestAuthType(r) == authTypeAnonymous {
|
||||||
// As per "Permission" section in
|
// As per "Permission" section in
|
||||||
@@ -395,14 +420,16 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Validate pre-conditions if any.
|
// Validate pre-conditions if any.
|
||||||
|
var storedTags *string // set once the object has been read far enough
|
||||||
opts.CheckPrecondFn = func(oi ObjectInfo) bool {
|
opts.CheckPrecondFn = func(oi ObjectInfo) bool {
|
||||||
if _, err := DecryptObjectInfo(&oi, r); err != nil {
|
storedTags = &oi.UserTags
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, oi.UserTags); s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectAction, oi.UserTags); s3Error != ErrNone {
|
if _, err := DecryptObjectInfo(&oi, r); err != nil {
|
||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -421,7 +448,7 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
|
|||||||
|
|
||||||
if (isErrObjectNotFound(err) || isErrVersionNotFound(err) || isErrReadQuorum(err)) && (gr == nil || !gr.ObjInfo.DeleteMarker) {
|
if (isErrObjectNotFound(err) || isErrVersionNotFound(err) || isErrReadQuorum(err)) && (gr == nil || !gr.ObjInfo.DeleteMarker) {
|
||||||
proxytgts := getProxyTargets(ctx, bucket, object, opts)
|
proxytgts := getProxyTargets(ctx, bucket, object, opts)
|
||||||
if !proxytgts.Empty() {
|
if !proxytgts.Empty() && authorizeVersionedRequestWithPotentialTags(ctx, r, policy.GetObjectAction, opts.VersionID) == ErrNone {
|
||||||
globalReplicationStats.Load().incProxy(bucket, getObjectAPI, false)
|
globalReplicationStats.Load().incProxy(bucket, getObjectAPI, false)
|
||||||
// proxy to replication target if active-active replication is in place.
|
// proxy to replication target if active-active replication is in place.
|
||||||
reader, proxy, perr = proxyGetToReplicationTarget(ctx, bucket, object, rs, r.Header, opts, proxytgts)
|
reader, proxy, perr = proxyGetToReplicationTarget(ctx, bucket, object, rs, r.Header, opts, proxytgts)
|
||||||
@@ -439,14 +466,29 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if reader == nil || !proxy.Proxy {
|
if reader == nil || !proxy.Proxy {
|
||||||
// The conditional callback has already written 304/412. Do not
|
// The conditional callback has already written 304/412 (or 403). Do not
|
||||||
// authorize again without the stored tags or write a second response.
|
// authorize again without the stored tags or write a second response.
|
||||||
if isErrPreconditionFailed(err) {
|
if isErrPreconditionFailed(err) || isErrPreconditionFailed(perr) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
// validate if the request indeed was authorized, if it wasn't we need to return "ErrAccessDenied"
|
// validate if the request indeed was authorized, if it wasn't we need to return "ErrAccessDenied"
|
||||||
// instead of any namespace related error.
|
// instead of any namespace related error. Use the stored tags if the callback read them.
|
||||||
if s3Error := authorizeRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone {
|
var s3Error APIErrorCode
|
||||||
|
if storedTags != nil {
|
||||||
|
s3Error = authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, *storedTags)
|
||||||
|
} else {
|
||||||
|
readErr := err
|
||||||
|
if proxy.Err != nil {
|
||||||
|
readErr = proxy.Err
|
||||||
|
}
|
||||||
|
if getRequestAuthType(r) == authTypeAnonymous && toAPIError(ctx, readErr).Code == "NoSuchKey" &&
|
||||||
|
authorizeBucketRequest(ctx, r, policy.ListBucketAction) == ErrNone {
|
||||||
|
s3Error = ErrNone
|
||||||
|
} else {
|
||||||
|
s3Error = authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, opts.VersionID, readErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if s3Error != ErrNone {
|
||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -478,6 +520,12 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
|
|||||||
defer gr.Close()
|
defer gr.Close()
|
||||||
|
|
||||||
objInfo := gr.ObjInfo
|
objInfo := gr.ObjInfo
|
||||||
|
if storedTags == nil {
|
||||||
|
if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, objInfo.UserTags); s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if !proxy.Proxy { // apply lifecycle rules only for local requests
|
if !proxy.Proxy { // apply lifecycle rules only for local requests
|
||||||
// Automatically remove the object/version if an expiry lifecycle rule can be applied
|
// Automatically remove the object/version if an expiry lifecycle rule can be applied
|
||||||
@@ -539,6 +587,8 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
|
|||||||
hash.AddChecksumHeader(w, cs)
|
hash.AddChecksumHeader(w, cs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
opts.TaggingCount = len(objInfo.UserTags) > 0 && canReturnObjectTagsHeader(ctx, r, opts.VersionID, objInfo.UserTags)
|
||||||
|
|
||||||
if err = setObjectHeaders(ctx, w, objInfo, rs, opts); err != nil {
|
if err = setObjectHeaders(ctx, w, objInfo, rs, opts); err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
@@ -597,20 +647,14 @@ func (api objectAPIHandlers) getObjectAttributesHandler(ctx context.Context, obj
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
var s3Error APIErrorCode
|
actions := [2]policy.Action{policy.GetObjectAttributesAction, policy.GetObjectAction}
|
||||||
if opts.VersionID != "" {
|
if opts.VersionID != "" {
|
||||||
s3Error = checkRequestAuthType(ctx, r, policy.GetObjectVersionAttributesAction, bucket, object)
|
actions = [2]policy.Action{policy.GetObjectVersionAttributesAction, policy.GetObjectVersionAction}
|
||||||
if s3Error == ErrNone {
|
|
||||||
s3Error = checkRequestAuthType(ctx, r, policy.GetObjectVersionAction, bucket, object)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
s3Error = checkRequestAuthType(ctx, r, policy.GetObjectAttributesAction, bucket, object)
|
|
||||||
if s3Error == ErrNone {
|
|
||||||
s3Error = checkRequestAuthType(ctx, r, policy.GetObjectAction, bucket, object)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
logger.GetReqInfo(ctx).BucketName = bucket
|
||||||
if s3Error != ErrNone {
|
logger.GetReqInfo(ctx).ObjectName = object
|
||||||
|
logger.GetReqInfo(ctx).VersionID = opts.VersionID
|
||||||
|
if s3Error := authenticateRequest(ctx, r, actions[0]); s3Error != ErrNone {
|
||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -622,13 +666,30 @@ func (api objectAPIHandlers) getObjectAttributesHandler(ctx context.Context, obj
|
|||||||
}
|
}
|
||||||
|
|
||||||
objInfo, err := objectAPI.GetObjectInfo(ctx, bucket, object, opts)
|
objInfo, err := objectAPI.GetObjectInfo(ctx, bucket, object, opts)
|
||||||
if err != nil {
|
// Both actions must be authorized with the stored tags. Authorizing first
|
||||||
s3Error = authorizeRequest(ctx, r, policy.ListBucketAction)
|
// with empty tags would reject conditional Allows and miss conditional Denies.
|
||||||
if s3Error == ErrNone {
|
for _, action := range actions {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
var s3Error APIErrorCode
|
||||||
|
if err != nil {
|
||||||
|
s3Error = authorizeRequestWithUnreadTags(ctx, r, action, err)
|
||||||
|
} else {
|
||||||
|
s3Error = authorizeRequestWithExistingTags(ctx, r, action, objInfo.UserTags)
|
||||||
|
}
|
||||||
|
if s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
}
|
||||||
|
if err != nil {
|
||||||
|
// As with S3, a caller without s3:ListBucket on the bucket gets
|
||||||
|
// AccessDenied in place of NoSuchKey. A server error, such as lost
|
||||||
|
// read quorum, tells nothing about the object and reaches the caller,
|
||||||
|
// who may read the object, as it is.
|
||||||
|
apiErr := toAPIError(ctx, err)
|
||||||
|
if apiErr.HTTPStatusCode < http.StatusInternalServerError && authorizeBucketRequest(ctx, r, policy.ListBucketAction) != ErrNone {
|
||||||
|
apiErr = errorCodes.ToAPIErr(ErrAccessDenied)
|
||||||
|
}
|
||||||
|
writeErrorResponse(ctx, w, apiErr, r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -833,6 +894,8 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
|
|||||||
|
|
||||||
// Check for auth type to return S3 compatible error.
|
// Check for auth type to return S3 compatible error.
|
||||||
// type to return the correct error (NoSuchKey vs AccessDenied)
|
// type to return the correct error (NoSuchKey vs AccessDenied)
|
||||||
|
logger.GetReqInfo(ctx).BucketName = bucket
|
||||||
|
logger.GetReqInfo(ctx).ObjectName = object
|
||||||
if s3Error := authenticateRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone {
|
if s3Error := authenticateRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone {
|
||||||
if getRequestAuthType(r) == authTypeAnonymous {
|
if getRequestAuthType(r) == authTypeAnonymous {
|
||||||
// As per "Permission" section in
|
// As per "Permission" section in
|
||||||
@@ -910,25 +973,43 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
|
|||||||
if err != nil && !objInfo.DeleteMarker && (isErrObjectNotFound(err) || isErrVersionNotFound(err) || isErrReadQuorum(err)) {
|
if err != nil && !objInfo.DeleteMarker && (isErrObjectNotFound(err) || isErrVersionNotFound(err) || isErrReadQuorum(err)) {
|
||||||
// proxy HEAD to replication target if active-active replication configured on bucket
|
// proxy HEAD to replication target if active-active replication configured on bucket
|
||||||
proxytgts := getProxyTargets(ctx, bucket, object, opts)
|
proxytgts := getProxyTargets(ctx, bucket, object, opts)
|
||||||
if !proxytgts.Empty() {
|
if !proxytgts.Empty() && authorizeVersionedRequestWithPotentialTags(ctx, r, policy.GetObjectAction, opts.VersionID) == ErrNone {
|
||||||
globalReplicationStats.Load().incProxy(bucket, headObjectAPI, false)
|
globalReplicationStats.Load().incProxy(bucket, headObjectAPI, false)
|
||||||
var oi ObjectInfo
|
var oi ObjectInfo
|
||||||
oi, proxy = proxyHeadToReplicationTarget(ctx, bucket, object, rs, opts, proxytgts)
|
oi, proxy = proxyHeadToReplicationTarget(ctx, bucket, object, rs, opts, proxytgts)
|
||||||
if proxy.Proxy {
|
if proxy.Proxy && proxy.Err == nil {
|
||||||
objInfo = oi
|
objInfo = oi
|
||||||
}
|
}
|
||||||
if proxy.Err != nil {
|
if proxy.Err != nil {
|
||||||
globalReplicationStats.Load().incProxy(bucket, headObjectAPI, true)
|
globalReplicationStats.Load().incProxy(bucket, headObjectAPI, true)
|
||||||
writeErrorResponseHeadersOnly(w, toAPIError(ctx, proxy.Err))
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectAction, objInfo.UserTags); s3Error != ErrNone {
|
// objInfo carries the stored tags unless reading the object failed.
|
||||||
|
var s3Error APIErrorCode
|
||||||
|
if (err != nil && !proxy.Proxy) || proxy.Err != nil {
|
||||||
|
readErr := err
|
||||||
|
if proxy.Err != nil {
|
||||||
|
readErr = proxy.Err
|
||||||
|
}
|
||||||
|
if getRequestAuthType(r) == authTypeAnonymous && toAPIError(ctx, readErr).Code == "NoSuchKey" &&
|
||||||
|
authorizeBucketRequest(ctx, r, policy.ListBucketAction) == ErrNone {
|
||||||
|
s3Error = ErrNone
|
||||||
|
} else {
|
||||||
|
s3Error = authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, opts.VersionID, readErr)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
s3Error = authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, objInfo.UserTags)
|
||||||
|
}
|
||||||
|
if s3Error != ErrNone {
|
||||||
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error))
|
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if proxy.Err != nil {
|
||||||
|
writeErrorResponseHeadersOnly(w, toAPIError(ctx, proxy.Err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if err != nil && !proxy.Proxy {
|
if err != nil && !proxy.Proxy {
|
||||||
switch {
|
switch {
|
||||||
@@ -1033,6 +1114,8 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
|
|||||||
hash.AddChecksumHeader(w, cs)
|
hash.AddChecksumHeader(w, cs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
opts.TaggingCount = len(objInfo.UserTags) > 0 && canReturnObjectTagsHeader(ctx, r, opts.VersionID, objInfo.UserTags)
|
||||||
|
|
||||||
// Set standard object headers.
|
// Set standard object headers.
|
||||||
if err = setObjectHeaders(ctx, w, objInfo, rs, opts); err != nil {
|
if err = setObjectHeaders(ctx, w, objInfo, rs, opts); err != nil {
|
||||||
writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
|
writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
|
||||||
@@ -1292,13 +1375,11 @@ func isRemoteCallRequired(ctx context.Context, bucket string, objAPI ObjectLayer
|
|||||||
|
|
||||||
// copyRewritesObjectData reports whether the object layer stores new object data
|
// copyRewritesObjectData reports whether the object layer stores new object data
|
||||||
// for this copy instead of updating metadata in place or adding a
|
// for this copy instead of updating metadata in place or adding a
|
||||||
// self-referential version. It mirrors the metadata-only decision taken by
|
// self-referential version. The selected pool must contain the source data to
|
||||||
// erasureServerPools.CopyObject and erasureSets.CopyObject. CopyObjectHandler
|
// add a reference. metadataOnly already excludes legacy sources, which the
|
||||||
// has to predict that decision because the compression metadata it records must
|
// object layer always rewrites.
|
||||||
// describe whichever bytes are finally stored. metadataOnly already excludes
|
|
||||||
// legacy sources, which the object layer always rewrites.
|
|
||||||
func copyRewritesObjectData(metadataOnly bool, srcOpts, dstOpts ObjectOptions) bool {
|
func copyRewritesObjectData(metadataOnly bool, srcOpts, dstOpts ObjectOptions) bool {
|
||||||
if !metadataOnly {
|
if !metadataOnly || dstOpts.copyPlan != nil && dstOpts.copyPlan.rewrite {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
switch {
|
switch {
|
||||||
@@ -1343,9 +1424,23 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, dstBucket, dstObject); s3Error != ErrNone {
|
if isDirectiveReplace(r.Header.Get(xhttp.AmzTagDirective)) {
|
||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, dstBucket, dstObject); s3Error != ErrNone {
|
||||||
return
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
logger.GetReqInfo(ctx).BucketName = dstBucket
|
||||||
|
logger.GetReqInfo(ctx).ObjectName = dstObject
|
||||||
|
logger.GetReqInfo(ctx).VersionID = ""
|
||||||
|
if s3Error := authenticateRequest(ctx, r, policy.PutObjectAction); s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !canPotentiallyAllowAction(ctx, r, policy.PutObjectAction, dstBucket, dstObject) {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read escaped copy source path to check for parameters.
|
// Read escaped copy source path to check for parameters.
|
||||||
@@ -1379,7 +1474,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, srcBucket, srcObject); s3Error != ErrNone {
|
if s3Error := withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode {
|
||||||
|
return authorizeVersionedRequestWithPotentialTags(ctx, r, policy.GetObjectAction, vid)
|
||||||
|
}); s3Error != ErrNone {
|
||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -1463,7 +1560,15 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
|||||||
|
|
||||||
getObjectNInfo := objectAPI.GetObjectNInfo
|
getObjectNInfo := objectAPI.GetObjectNInfo
|
||||||
|
|
||||||
|
var storedSrcTags *string
|
||||||
checkCopyPrecondFn := func(o ObjectInfo) bool {
|
checkCopyPrecondFn := func(o ObjectInfo) bool {
|
||||||
|
storedSrcTags = &o.UserTags
|
||||||
|
if s3Error := withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode {
|
||||||
|
return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, vid, o.UserTags)
|
||||||
|
}); s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return true
|
||||||
|
}
|
||||||
if _, err := DecryptObjectInfo(&o, r); err != nil {
|
if _, err := DecryptObjectInfo(&o, r); err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return true
|
return true
|
||||||
@@ -1515,6 +1620,20 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
|||||||
if isErrPreconditionFailed(err) {
|
if isErrPreconditionFailed(err) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
var s3Error APIErrorCode
|
||||||
|
if storedSrcTags != nil {
|
||||||
|
s3Error = withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode {
|
||||||
|
return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, vid, *storedSrcTags)
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
s3Error = withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode {
|
||||||
|
return authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, vid, err)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// Versioning enabled quite possibly object is deleted might be delete-marker
|
// Versioning enabled quite possibly object is deleted might be delete-marker
|
||||||
// if present set the headers, no idea why AWS S3 sets these headers.
|
// if present set the headers, no idea why AWS S3 sets these headers.
|
||||||
@@ -1532,6 +1651,23 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
|||||||
}
|
}
|
||||||
defer gr.Close()
|
defer gr.Close()
|
||||||
srcInfo := gr.ObjInfo
|
srcInfo := gr.ObjInfo
|
||||||
|
if storedSrcTags == nil {
|
||||||
|
if s3Error := withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode {
|
||||||
|
return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, vid, srcInfo.UserTags)
|
||||||
|
}); s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !isDirectiveReplace(r.Header.Get(xhttp.AmzTagDirective)) {
|
||||||
|
copiedTags := srcInfo.UserTags
|
||||||
|
if s3Error := withScopedReqObject(ctx, dstBucket, dstObject, "", func() APIErrorCode {
|
||||||
|
return authorizeRequestWithTags(ctx, r, policy.PutObjectAction, "", &copiedTags)
|
||||||
|
}); s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
// A trusted SSE-C replica reads raw ciphertext. Federation's ordinary PUT
|
// A trusted SSE-C replica reads raw ciphertext. Federation's ordinary PUT
|
||||||
// expects plaintext and does not forward the replica's sealed-key metadata.
|
// expects plaintext and does not forward the replica's sealed-key metadata.
|
||||||
if remoteCallRequired && replicaTrusted && crypto.SSEC.IsEncrypted(srcInfo.UserDefined) {
|
if remoteCallRequired && replicaTrusted && crypto.SSEC.IsEncrypted(srcInfo.UserDefined) {
|
||||||
@@ -1673,16 +1809,23 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
|||||||
// Without a requested destination checksum algorithm, an in-place SSE-C
|
// Without a requested destination checksum algorithm, an in-place SSE-C
|
||||||
// rotation preserves the stored checksum state, including absence; it does
|
// rotation preserves the stored checksum state, including absence; it does
|
||||||
// not add the default CRC64NVME.
|
// not add the default CRC64NVME.
|
||||||
canRotateKeyInPlace := !srcInfo.Legacy &&
|
canRotateKeyInPlace := cpSrcDstSame && sseCopyC && sseC && !chStorageClass && !srcInfo.Legacy &&
|
||||||
!copyRewritesObjectData(srcInfo.metadataOnly, copySrcOpts, dstOpts) &&
|
!copyRewritesObjectData(srcInfo.metadataOnly, copySrcOpts, dstOpts) &&
|
||||||
(replicaTrusted || !hash.NewChecksumHeader(r.Header).IsSet())
|
(replicaTrusted || !hash.NewChecksumHeader(r.Header).IsSet())
|
||||||
|
|
||||||
// If src == dst and either
|
if canRotateKeyInPlace {
|
||||||
// - the object is encrypted using SSE-C and two different SSE-C keys are present
|
// A new version may need another pool. Resolve that before rewrapping
|
||||||
// - the object is encrypted using SSE-S3 and the SSE-S3 header is present
|
// the key, since a cross-pool copy must re-encrypt the source stream.
|
||||||
// - the object storage class is not changing
|
plan, err := objectAPI.prepareCopyObject(lockCtx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, copySrcOpts, dstOpts)
|
||||||
// then execute a key rotation.
|
if err != nil {
|
||||||
if cpSrcDstSame && (sseCopyC && sseC) && !chStorageClass && canRotateKeyInPlace {
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
dstOpts.copyPlan = &plan
|
||||||
|
canRotateKeyInPlace = !plan.rewrite
|
||||||
|
}
|
||||||
|
|
||||||
|
if canRotateKeyInPlace {
|
||||||
oldKey, err = ParseSSECopyCustomerRequest(r.Header, srcInfo.UserDefined)
|
oldKey, err = ParseSSECopyCustomerRequest(r.Header, srcInfo.UserDefined)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
@@ -1773,6 +1916,20 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if !remoteCallRequired && dstOpts.copyPlan == nil {
|
||||||
|
// Encryption and checksum recomputation require new data. Plan
|
||||||
|
// with that final decision before starting either transform, but
|
||||||
|
// retain metadataOnly for the invalid self-copy check below.
|
||||||
|
planInfo := srcInfo
|
||||||
|
planInfo.metadataOnly = srcInfo.metadataOnly && !dstOpts.WantServerSideChecksumType.IsSet()
|
||||||
|
plan, err := objectAPI.prepareCopyObject(lockCtx, srcBucket, srcObject, dstBucket, dstObject, planInfo, copySrcOpts, dstOpts)
|
||||||
|
if err != nil {
|
||||||
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
dstOpts.copyPlan = &plan
|
||||||
|
}
|
||||||
|
|
||||||
if isDstCompressed {
|
if isDstCompressed {
|
||||||
checksumReader := srcInfo.Reader
|
checksumReader := srcInfo.Reader
|
||||||
wantEncryption := crypto.Requested(r.Header)
|
wantEncryption := crypto.Requested(r.Header)
|
||||||
@@ -1829,12 +1986,27 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
|||||||
|
|
||||||
srcInfo.PutObjReader = pReader
|
srcInfo.PutObjReader = pReader
|
||||||
|
|
||||||
// Object Lock state as stored on disk, captured before the metadata
|
// Reconcile against the addressed destination version under its write
|
||||||
// directive rebuilds the map. A replica update is applied only when its
|
// lock. The source can be a different object or version, including when
|
||||||
// source timestamp is newer than the stored one, and a stale update must
|
// the destination version has never been written.
|
||||||
// leave the stored state in place instead of erasing it.
|
getObjectInfo := func(_ context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
|
||||||
storedLock := storedObjectLockState(srcInfo.UserDefined)
|
opts.NoLock = true
|
||||||
storedTagTimestamp := srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]
|
return objectAPI.GetObjectInfo(lockCtx, bucket, object, opts)
|
||||||
|
}
|
||||||
|
var storedInfo ObjectInfo
|
||||||
|
if replicaTrusted && !remoteCallRequired {
|
||||||
|
storedInfo, err = replicaStoredObjectInfo(lockCtx, getObjectInfo, dstBucket, dstObject, dstOpts.VersionID)
|
||||||
|
if err != nil {
|
||||||
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
storedLock := storedObjectLockState(storedInfo.UserDefined)
|
||||||
|
storedTagTimestamp := storedInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]
|
||||||
|
localTagTimestamp := ""
|
||||||
|
if cpSrcDstSame {
|
||||||
|
localTagTimestamp = srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]
|
||||||
|
}
|
||||||
|
|
||||||
srcInfo.UserDefined, err = getCpObjMetadataFromHeader(ctx, r, srcInfo.UserDefined, allowReplicationMetadata)
|
srcInfo.UserDefined, err = getCpObjMetadataFromHeader(ctx, r, srcInfo.UserDefined, allowReplicationMetadata)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1859,9 +2031,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
|||||||
// the captured state even if metadata REPLACE rebuilt the map.
|
// the captured state even if metadata REPLACE rebuilt the map.
|
||||||
srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = srcTimestamp.UTC().Format(time.RFC3339Nano)
|
srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = srcTimestamp.UTC().Format(time.RFC3339Nano)
|
||||||
srcInfo.UserDefined[xhttp.AmzObjectTagging] = objTags
|
srcInfo.UserDefined[xhttp.AmzObjectTagging] = objTags
|
||||||
reconcileStoredObjectTags(srcInfo.UserDefined, srcInfo.UserTags, storedTagTimestamp)
|
reconcileStoredObjectTags(srcInfo.UserDefined, storedInfo.UserTags, storedTagTimestamp)
|
||||||
} else {
|
} else {
|
||||||
srcInfo.UserDefined[xhttp.AmzObjectTagging] = srcInfo.UserTags
|
srcInfo.UserDefined[xhttp.AmzObjectTagging] = storedInfo.UserTags
|
||||||
if storedTagTimestamp != "" {
|
if storedTagTimestamp != "" {
|
||||||
srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = storedTagTimestamp
|
srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = storedTagTimestamp
|
||||||
} else {
|
} else {
|
||||||
@@ -1870,7 +2042,18 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
|||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
srcInfo.UserDefined[xhttp.AmzObjectTagging] = objTags
|
srcInfo.UserDefined[xhttp.AmzObjectTagging] = objTags
|
||||||
srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = monotonicMetadataTimestamp(UTCNow().Format(time.RFC3339Nano), localTagTimestamp)
|
||||||
|
}
|
||||||
|
if !isDirectiveReplace(r.Header.Get(xhttp.AmzTagDirective)) {
|
||||||
|
finalDstTags := srcInfo.UserDefined[xhttp.AmzObjectTagging]
|
||||||
|
if finalDstTags != srcInfo.UserTags {
|
||||||
|
if s3Error := withScopedReqObject(ctx, dstBucket, dstObject, "", func() APIErrorCode {
|
||||||
|
return authorizeRequestWithTags(ctx, r, policy.PutObjectAction, "", &finalDstTags)
|
||||||
|
}); s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// SSE-C rotation snapshots reserved metadata before the tag decision. Its
|
// SSE-C rotation snapshots reserved metadata before the tag decision. Its
|
||||||
// later merge must not put the old timestamp back over the accepted state.
|
// later merge must not put the old timestamp back over the accepted state.
|
||||||
@@ -1878,14 +2061,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
|||||||
|
|
||||||
srcInfo.UserDefined = filterReplicationStatusMetadata(srcInfo.UserDefined)
|
srcInfo.UserDefined = filterReplicationStatusMetadata(srcInfo.UserDefined)
|
||||||
srcInfo.UserDefined = objectlock.FilterObjectLockMetadata(srcInfo.UserDefined, true, true)
|
srcInfo.UserDefined = objectlock.FilterObjectLockMetadata(srcInfo.UserDefined, true, true)
|
||||||
|
// COPY may have preserved the source's reserved timestamps. Only the
|
||||||
|
// addressed destination and the incoming update may order a replica.
|
||||||
|
delete(srcInfo.UserDefined, ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp)
|
||||||
|
delete(srcInfo.UserDefined, ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp)
|
||||||
retPerms := isPutActionAllowed(ctx, getRequestAuthType(r), dstBucket, dstObject, r, policy.PutObjectRetentionAction)
|
retPerms := isPutActionAllowed(ctx, getRequestAuthType(r), dstBucket, dstObject, r, policy.PutObjectRetentionAction)
|
||||||
holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), dstBucket, dstObject, r, policy.PutObjectLegalHoldAction)
|
holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), dstBucket, dstObject, r, policy.PutObjectLegalHoldAction)
|
||||||
// The destination is locked above: look it up without its lock, in the
|
|
||||||
// locks' context.
|
|
||||||
getObjectInfo := func(_ context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
|
|
||||||
opts.NoLock = true
|
|
||||||
return objectAPI.GetObjectInfo(lockCtx, bucket, object, opts)
|
|
||||||
}
|
|
||||||
|
|
||||||
// apply default bucket configuration/governance headers for dest side.
|
// apply default bucket configuration/governance headers for dest side.
|
||||||
retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, dstBucket, dstObject, getObjectInfo, retPerms, holdPerms, replicaTrusted)
|
retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, dstBucket, dstObject, getObjectInfo, retPerms, holdPerms, replicaTrusted)
|
||||||
@@ -2236,9 +2417,13 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
|||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if size < 0 {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidArgument), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if size == -1 {
|
if size < 0 {
|
||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -2701,10 +2886,14 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
|
|||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if size < 0 {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidArgument), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if size == -1 {
|
if size < 0 {
|
||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -3140,6 +3329,22 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Check before delOpts reads bucket versioning: unavailable bucket metadata
|
||||||
|
// must fail the request before a force-delete can reach the object layer.
|
||||||
|
if forceDelete, _ := strconv.ParseBool(r.Header.Get(xhttp.MinIOForceDelete)); forceDelete {
|
||||||
|
rcfg, err := globalBucketObjectLockSys.Get(bucket)
|
||||||
|
if err != nil {
|
||||||
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if rcfg.LockEnabled {
|
||||||
|
apiErr := toAPIError(ctx, errInvalidArgument)
|
||||||
|
apiErr.Description = "force-delete is forbidden on Object Locking enabled buckets"
|
||||||
|
writeErrorResponse(ctx, w, apiErr, r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
opts, err := delOpts(ctx, r, bucket, object)
|
opts, err := delOpts(ctx, r, bucket, object)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
@@ -3160,14 +3365,6 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
rcfg, _ := globalBucketObjectLockSys.Get(bucket)
|
|
||||||
if rcfg.LockEnabled && opts.DeletePrefix {
|
|
||||||
apiErr := toAPIError(ctx, errInvalidArgument)
|
|
||||||
apiErr.Description = "force-delete is forbidden on Object Locking enabled buckets"
|
|
||||||
writeErrorResponse(ctx, w, apiErr, r.URL)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
os := newObjSweeper(bucket, object).WithVersion(opts.VersionID).WithVersioning(opts.Versioned, opts.VersionSuspended)
|
os := newObjSweeper(bucket, object).WithVersion(opts.VersionID).WithVersioning(opts.Versioned, opts.VersionSuspended)
|
||||||
|
|
||||||
opts.SetEvalMetadataFn(func(oi *ObjectInfo, gerr error) (dsc ReplicateDecision, err error) {
|
opts.SetEvalMetadataFn(func(oi *ObjectInfo, gerr error) (dsc ReplicateDecision, err error) {
|
||||||
@@ -3190,6 +3387,12 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
|
|||||||
return dsc, nil
|
return dsc, nil
|
||||||
})
|
})
|
||||||
|
|
||||||
|
_, rcfg, err := checkBucketDeleteMetadata(bucket)
|
||||||
|
if err != nil {
|
||||||
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
vID := opts.VersionID
|
vID := opts.VersionID
|
||||||
if replica {
|
if replica {
|
||||||
opts.SetReplicaStatus(replication.Replica)
|
opts.SetReplicaStatus(replication.Replica)
|
||||||
@@ -3201,20 +3404,27 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
|
|||||||
opts.RefuseLockedOverwrite = true
|
opts.RefuseLockedOverwrite = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
checkUnaddressedLock := vID == "" && !opts.Versioned && !opts.DeleteMarker && rcfg.LockEnabled
|
||||||
|
if checkUnaddressedLock {
|
||||||
|
opts.RefuseLockedOverwrite = true
|
||||||
|
}
|
||||||
opts.SetEvalRetentionBypassFn(func(goi ObjectInfo, gerr error) (err error) {
|
opts.SetEvalRetentionBypassFn(func(goi ObjectInfo, gerr error) (err error) {
|
||||||
err = nil
|
if vID != "" || checkUnaddressedLock || (vID == "" && !opts.Versioned && !opts.DeleteMarker && gerr == nil) {
|
||||||
if vID != "" {
|
checkVID := vID
|
||||||
|
if checkVID == "" {
|
||||||
|
checkVID = nullVersionID
|
||||||
|
}
|
||||||
err := enforceRetentionBypassForDelete(ctx, r, bucket, ObjectToDelete{
|
err := enforceRetentionBypassForDelete(ctx, r, bucket, ObjectToDelete{
|
||||||
ObjectV: ObjectV{
|
ObjectV: ObjectV{
|
||||||
ObjectName: object,
|
ObjectName: object,
|
||||||
VersionID: vID,
|
VersionID: checkVID,
|
||||||
},
|
},
|
||||||
}, goi, gerr)
|
}, goi, gerr)
|
||||||
if err != nil && !isErrObjectNotFound(err) {
|
if err != nil && !isErrObjectNotFound(err) {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return err
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
deleteObject := objectAPI.DeleteObject
|
deleteObject := objectAPI.DeleteObject
|
||||||
@@ -3366,7 +3576,7 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
|
|||||||
VersionID: opts.VersionID,
|
VersionID: opts.VersionID,
|
||||||
EvalMetadataFn: func(oi *ObjectInfo, gerr error) (ReplicateDecision, error) {
|
EvalMetadataFn: func(oi *ObjectInfo, gerr error) (ReplicateDecision, error) {
|
||||||
oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = strings.ToUpper(string(legalHold.Status))
|
oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = strings.ToUpper(string(legalHold.Status))
|
||||||
oi.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
oi.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = monotonicMetadataTimestamp(UTCNow().Format(time.RFC3339Nano), oi.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp])
|
||||||
|
|
||||||
dsc := mustReplicate(ctx, bucket, object, oi.getMustReplicateOptions(replication.MetadataReplicationType, opts))
|
dsc := mustReplicate(ctx, bucket, object, oi.getMustReplicateOptions(replication.MetadataReplicationType, opts))
|
||||||
if dsc.ReplicateAny() {
|
if dsc.ReplicateAny() {
|
||||||
@@ -3543,7 +3753,7 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
|
|||||||
oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = ""
|
oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = ""
|
||||||
oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = ""
|
oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = ""
|
||||||
}
|
}
|
||||||
oi.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
oi.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = monotonicMetadataTimestamp(UTCNow().Format(time.RFC3339Nano), oi.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp])
|
||||||
dsc = mustReplicate(ctx, bucket, object, oi.getMustReplicateOptions(replication.MetadataReplicationType, opts))
|
dsc = mustReplicate(ctx, bucket, object, oi.getMustReplicateOptions(replication.MetadataReplicationType, opts))
|
||||||
if dsc.ReplicateAny() {
|
if dsc.ReplicateAny() {
|
||||||
oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||||
@@ -3668,6 +3878,8 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
logger.GetReqInfo(ctx).BucketName = bucket
|
||||||
|
logger.GetReqInfo(ctx).ObjectName = object
|
||||||
if s3Error := authenticateRequest(ctx, r, policy.GetObjectTaggingAction); s3Error != ErrNone {
|
if s3Error := authenticateRequest(ctx, r, policy.GetObjectTaggingAction); s3Error != ErrNone {
|
||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
return
|
return
|
||||||
@@ -3687,9 +3899,22 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
|
|||||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||||
proxytgts := getProxyTargets(ctx, bucket, object, opts)
|
proxytgts := getProxyTargets(ctx, bucket, object, opts)
|
||||||
if !proxytgts.Empty() {
|
if !proxytgts.Empty() {
|
||||||
|
if s3Error := authorizeVersionedRequestWithPotentialTags(ctx, r, policy.GetObjectTaggingAction, opts.VersionID); s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
globalReplicationStats.Load().incProxy(bucket, getObjectTaggingAPI, false)
|
globalReplicationStats.Load().incProxy(bucket, getObjectTaggingAPI, false)
|
||||||
// proxy to replication target if site replication is in place.
|
// proxy to replication target if site replication is in place.
|
||||||
tags, gerr := proxyGetTaggingToRepTarget(ctx, bucket, object, opts, proxytgts)
|
tags, gerr, s3Error := proxyGetTaggingToRepTarget(ctx, bucket, object, opts, proxytgts, func(existingTags string, readErr error) APIErrorCode {
|
||||||
|
if readErr != nil && !isRemoteTaggingNotFound(readErr) {
|
||||||
|
return authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectTaggingAction, opts.VersionID, readErr)
|
||||||
|
}
|
||||||
|
return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, opts.VersionID, existingTags)
|
||||||
|
})
|
||||||
|
if s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
if gerr.Err != nil || tags == nil {
|
if gerr.Err != nil || tags == nil {
|
||||||
globalReplicationStats.Load().incProxy(bucket, getObjectTaggingAPI, true)
|
globalReplicationStats.Load().incProxy(bucket, getObjectTaggingAPI, true)
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, gerr.Err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, gerr.Err), r.URL)
|
||||||
@@ -3698,16 +3923,24 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
|
|||||||
ot = tags
|
ot = tags
|
||||||
w.Header()[xhttp.MinIOTaggingProxied] = []string{"true"} // indicate that the request was proxied.
|
w.Header()[xhttp.MinIOTaggingProxied] = []string{"true"} // indicate that the request was proxied.
|
||||||
} else {
|
} else {
|
||||||
|
if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, opts.VersionID, ""); s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
if s3Error := authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectTaggingAction, opts.VersionID, err); s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, ot.String()); s3Error != ErrNone {
|
if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, opts.VersionID, ot.String()); s3Error != ErrNone {
|
||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -3736,6 +3969,18 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
|
|||||||
writeSuccessResponseXML(w, encodeResponse(otags))
|
writeSuccessResponseXML(w, encodeResponse(otags))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isRemoteTaggingNotFound(err error) bool {
|
||||||
|
if err == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
switch miniogo.ToErrorResponse(err).Code {
|
||||||
|
case "NoSuchKey", "NoSuchVersion":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// PutObjectTaggingHandler - PUT object tagging
|
// PutObjectTaggingHandler - PUT object tagging
|
||||||
func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "PutObjectTagging")
|
ctx := newContext(r, w, "PutObjectTagging")
|
||||||
@@ -3755,6 +4000,13 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
logger.GetReqInfo(ctx).BucketName = bucket
|
||||||
|
logger.GetReqInfo(ctx).ObjectName = object
|
||||||
|
if s3Error := authenticateRequest(ctx, r, policy.PutObjectTaggingAction); s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// Tags XML will not be bigger than 1MiB in size, fail if its bigger.
|
// Tags XML will not be bigger than 1MiB in size, fail if its bigger.
|
||||||
tags, err := tags.ParseObjectXML(io.LimitReader(r.Body, 1<<20))
|
tags, err := tags.ParseObjectXML(io.LimitReader(r.Body, 1<<20))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -3763,13 +4015,6 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
|
|||||||
}
|
}
|
||||||
tagsStr := tags.String()
|
tagsStr := tags.String()
|
||||||
|
|
||||||
logger.GetReqInfo(ctx).BucketName = bucket
|
|
||||||
logger.GetReqInfo(ctx).ObjectName = object
|
|
||||||
if s3Error := authenticateRequest(ctx, r, policy.PutObjectTaggingAction); s3Error != ErrNone {
|
|
||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set this such that authorization policies can be applied on the object
|
// Set this such that authorization policies can be applied on the object
|
||||||
// tags. This is derived from the request body, so it must be injected only
|
// tags. This is derived from the request body, so it must be injected only
|
||||||
// after signature verification: the SigV4 verifier now rejects unsigned
|
// after signature verification: the SigV4 verifier now rejects unsigned
|
||||||
@@ -3783,14 +4028,6 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
|
|||||||
}
|
}
|
||||||
|
|
||||||
objInfo, err := objAPI.GetObjectInfo(ctx, bucket, object, opts)
|
objInfo, err := objAPI.GetObjectInfo(ctx, bucket, object, opts)
|
||||||
existingTags := ""
|
|
||||||
if err == nil {
|
|
||||||
existingTags = objInfo.UserTags
|
|
||||||
}
|
|
||||||
if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.PutObjectTaggingAction, existingTags); s3Error != ErrNone {
|
|
||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// if object is not found locally, but exists on peer site - proxy
|
// if object is not found locally, but exists on peer site - proxy
|
||||||
// the tagging request to peer site. The response to client will
|
// the tagging request to peer site. The response to client will
|
||||||
@@ -3798,9 +4035,21 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
|
|||||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||||
proxytgts := getProxyTargets(ctx, bucket, object, opts)
|
proxytgts := getProxyTargets(ctx, bucket, object, opts)
|
||||||
if !proxytgts.Empty() {
|
if !proxytgts.Empty() {
|
||||||
|
if s3Error := authorizeVersionedRequestWithPotentialTags(ctx, r, policy.PutObjectTaggingAction, opts.VersionID); s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
perr, s3Error := proxyTaggingToRepTarget(ctx, bucket, object, tags, opts, proxytgts, func(existingTags string, readErr error) APIErrorCode {
|
||||||
|
if readErr != nil && !isRemoteTaggingNotFound(readErr) {
|
||||||
|
return authorizeVersionedRequestWithUnreadTags(ctx, r, policy.PutObjectTaggingAction, opts.VersionID, readErr)
|
||||||
|
}
|
||||||
|
return authorizeVersionedRequestWithExistingTags(ctx, r, policy.PutObjectTaggingAction, opts.VersionID, existingTags)
|
||||||
|
})
|
||||||
|
if s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
globalReplicationStats.Load().incProxy(bucket, putObjectTaggingAPI, false)
|
globalReplicationStats.Load().incProxy(bucket, putObjectTaggingAPI, false)
|
||||||
// proxy to replication target if site replication is in place.
|
|
||||||
perr := proxyTaggingToRepTarget(ctx, bucket, object, tags, opts, proxytgts)
|
|
||||||
if perr.Err != nil {
|
if perr.Err != nil {
|
||||||
globalReplicationStats.Load().incProxy(bucket, putObjectTaggingAPI, true)
|
globalReplicationStats.Load().incProxy(bucket, putObjectTaggingAPI, true)
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, perr.Err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, perr.Err), r.URL)
|
||||||
@@ -3821,9 +4070,26 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// A local read failure must never be turned into a write grant. A
|
||||||
|
// server error may still be disclosed to an otherwise authorized
|
||||||
|
// caller for retry, while a missing object is evaluated as untagged.
|
||||||
|
var s3Error APIErrorCode
|
||||||
|
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||||
|
s3Error = authorizeVersionedRequestWithExistingTags(ctx, r, policy.PutObjectTaggingAction, opts.VersionID, "")
|
||||||
|
} else {
|
||||||
|
s3Error = authorizeVersionedRequestWithUnreadTags(ctx, r, policy.PutObjectTaggingAction, opts.VersionID, err)
|
||||||
|
}
|
||||||
|
if s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.PutObjectTaggingAction, opts.VersionID, objInfo.UserTags); s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(objInfo.UserDefined, tagsStr, objInfo.ReplicationStatus, replication.MetadataReplicationType, opts))
|
dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(objInfo.UserDefined, tagsStr, objInfo.ReplicationStatus, replication.MetadataReplicationType, opts))
|
||||||
stamp := UTCNow().Format(time.RFC3339Nano)
|
stamp := UTCNow().Format(time.RFC3339Nano)
|
||||||
@@ -3880,6 +4146,13 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
logger.GetReqInfo(ctx).BucketName = bucket
|
||||||
|
logger.GetReqInfo(ctx).ObjectName = object
|
||||||
|
if s3Error := authenticateRequest(ctx, r, policy.DeleteObjectTaggingAction); s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
opts, err := getOpts(ctx, r, bucket, object)
|
opts, err := getOpts(ctx, r, bucket, object)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
@@ -3894,9 +4167,21 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
|
|||||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||||
proxytgts := getProxyTargets(ctx, bucket, object, opts)
|
proxytgts := getProxyTargets(ctx, bucket, object, opts)
|
||||||
if !proxytgts.Empty() {
|
if !proxytgts.Empty() {
|
||||||
|
if s3Error := authorizeVersionedRequestWithPotentialTags(ctx, r, policy.DeleteObjectTaggingAction, opts.VersionID); s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
perr, s3Error := proxyTaggingToRepTarget(ctx, bucket, object, nil, opts, proxytgts, func(existingTags string, readErr error) APIErrorCode {
|
||||||
|
if readErr != nil && !isRemoteTaggingNotFound(readErr) {
|
||||||
|
return authorizeVersionedRequestWithUnreadTags(ctx, r, policy.DeleteObjectTaggingAction, opts.VersionID, readErr)
|
||||||
|
}
|
||||||
|
return authorizeVersionedRequestWithExistingTags(ctx, r, policy.DeleteObjectTaggingAction, opts.VersionID, existingTags)
|
||||||
|
})
|
||||||
|
if s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
globalReplicationStats.Load().incProxy(bucket, removeObjectTaggingAPI, false)
|
globalReplicationStats.Load().incProxy(bucket, removeObjectTaggingAPI, false)
|
||||||
// proxy to replication target if active-active replication is in place.
|
|
||||||
perr := proxyTaggingToRepTarget(ctx, bucket, object, nil, opts, proxytgts)
|
|
||||||
if perr.Err != nil {
|
if perr.Err != nil {
|
||||||
globalReplicationStats.Load().incProxy(bucket, removeObjectTaggingAPI, true)
|
globalReplicationStats.Load().incProxy(bucket, removeObjectTaggingAPI, true)
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, perr.Err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, perr.Err), r.URL)
|
||||||
@@ -3918,12 +4203,22 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
var s3Error APIErrorCode
|
||||||
|
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||||
|
s3Error = authorizeVersionedRequestWithExistingTags(ctx, r, policy.DeleteObjectTaggingAction, opts.VersionID, "")
|
||||||
|
} else {
|
||||||
|
s3Error = authorizeVersionedRequestWithUnreadTags(ctx, r, policy.DeleteObjectTaggingAction, opts.VersionID, err)
|
||||||
|
}
|
||||||
|
if s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Allow deleteObjectTagging if policy action is set
|
// Allow deleteObjectTagging if policy action is set
|
||||||
if s3Error := checkRequestAuthTypeWithExistingTags(ctx, r, policy.DeleteObjectTaggingAction, bucket, object, oi.UserTags); s3Error != ErrNone {
|
if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.DeleteObjectTaggingAction, opts.VersionID, oi.UserTags); s3Error != ErrNone {
|
||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,421 @@
|
|||||||
|
// Copyright (c) 2026 Feng Ruohang
|
||||||
|
//
|
||||||
|
// This file is part of Silo Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/md5"
|
||||||
|
"encoding/base64"
|
||||||
|
"fmt"
|
||||||
|
"maps"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPoolLockReductionOrderIndependent(t *testing.T) {
|
||||||
|
const old = objectLockTestStamp0900
|
||||||
|
const recent = objectLockTestStamp1000
|
||||||
|
const newest = objectLockTestStamp1100
|
||||||
|
compliance := objectLockState{mode: "COMPLIANCE", retainUntil: "2035-01-01T00:00:00Z", retentionTimestamp: old}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
states [3]objectLockState
|
||||||
|
want objectLockState
|
||||||
|
}{
|
||||||
|
{"compliance-before-newer-removal", [3]objectLockState{compliance, {retentionTimestamp: recent}, {mode: "GOVERNANCE", retainUntil: "2036-01-01T00:00:00Z", retentionTimestamp: newest}}, compliance},
|
||||||
|
{"longer-compliance-before-newer-shortening", [3]objectLockState{compliance, {mode: "COMPLIANCE", retainUntil: "2034-01-01T00:00:00Z", retentionTimestamp: newest}, {retentionTimestamp: recent}}, compliance},
|
||||||
|
{"unordered-compliance", [3]objectLockState{{mode: "COMPLIANCE", retainUntil: compliance.retainUntil, retentionTimestamp: "invalid"}, {retentionTimestamp: newest}, {}}, objectLockState{mode: "COMPLIANCE", retainUntil: compliance.retainUntil, retentionTimestamp: "invalid"}},
|
||||||
|
{"expired-compliance-can-release", [3]objectLockState{{mode: "COMPLIANCE", retainUntil: "2000-01-01T00:00:00Z", retentionTimestamp: old}, {retentionTimestamp: newest}, {retentionTimestamp: recent}}, objectLockState{retentionTimestamp: newest}},
|
||||||
|
{"governance-can-release", [3]objectLockState{{mode: "GOVERNANCE", retainUntil: compliance.retainUntil, retentionTimestamp: old}, {retentionTimestamp: newest}, {}}, objectLockState{retentionTimestamp: newest}},
|
||||||
|
{"unordered-hold-conflict", [3]objectLockState{{legalHold: "OFF"}, {legalHold: "ON"}, {}}, objectLockState{legalHold: "ON"}},
|
||||||
|
{"newer-hold-release", [3]objectLockState{{legalHold: "ON", legalHoldTimestamp: old}, {legalHold: "OFF", legalHoldTimestamp: newest}, {legalHold: "ON", legalHoldTimestamp: "invalid"}}, objectLockState{legalHold: "OFF", legalHoldTimestamp: newest}},
|
||||||
|
{"independent-fields", [3]objectLockState{compliance, {retentionTimestamp: newest, legalHold: "ON", legalHoldTimestamp: recent}, {legalHold: "OFF", legalHoldTimestamp: old}}, objectLockState{mode: compliance.mode, retainUntil: compliance.retainUntil, retentionTimestamp: old, legalHold: "ON", legalHoldTimestamp: recent}},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
for _, order := range [][3]int{{0, 1, 2}, {0, 2, 1}, {1, 0, 2}, {1, 2, 0}, {2, 0, 1}, {2, 1, 0}} {
|
||||||
|
var copies []PoolObjInfo
|
||||||
|
for _, i := range order {
|
||||||
|
meta := make(map[string]string)
|
||||||
|
tc.states[i].restoreRetention(meta)
|
||||||
|
tc.states[i].restoreLegalHold(meta)
|
||||||
|
copies = append(copies, PoolObjInfo{Index: i, ObjInfo: ObjectInfo{UserDefined: meta}})
|
||||||
|
}
|
||||||
|
for name, reduce := range map[string]func([]PoolObjInfo) objectLockState{"merge": mergePoolLockState, "lifecycle": newestLockState} {
|
||||||
|
if got := reduce(copies); got != tc.want {
|
||||||
|
t.Errorf("%s order=%v: got %+v want %+v", name, order, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// All requests copy the same bytes. Revisions belong to the addressed
|
||||||
|
// destination, even if the source is the latest version of the same key.
|
||||||
|
func TestAPICopyReplicaDestinationState(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t, makeBucketOptions: MakeBucketOptions{LockEnabled: true},
|
||||||
|
objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
|
||||||
|
for _, sameKey := range []bool{true, false} {
|
||||||
|
for _, missing := range []bool{false, true} {
|
||||||
|
for _, directive := range []string{copyDirective, replaceDirective} {
|
||||||
|
name := fmt.Sprintf("copy-destination/%t/%t/%s", sameKey, missing, directive)
|
||||||
|
source := name
|
||||||
|
if !sameKey {
|
||||||
|
source += "-source"
|
||||||
|
}
|
||||||
|
targetMeta := poolLockMetadata("GOVERNANCE", "ON", objectLockTestStamp1000, objectLockTestStamp1000)
|
||||||
|
targetMeta[xhttp.AmzObjectTagging] = "state=destination"
|
||||||
|
targetMeta[r5TagStamp] = objectLockTestStamp1000
|
||||||
|
targetID := mustGetUUID()
|
||||||
|
if !missing {
|
||||||
|
targetID = putObjectLockVersion(t, obj, bucket, name, targetMeta)
|
||||||
|
}
|
||||||
|
sourceMeta := poolLockMetadata("", "OFF", objectLockTestStamp1100, objectLockTestStamp1100)
|
||||||
|
sourceMeta[xhttp.AmzObjectTagging] = "state=source"
|
||||||
|
sourceMeta[r5TagStamp] = objectLockTestStamp1100
|
||||||
|
sourceID := putObjectLockVersion(t, obj, bucket, source, sourceMeta)
|
||||||
|
headers := map[string]string{
|
||||||
|
xhttp.AmzCopySource: "/" + bucket + "/" + source + "?versionId=" + sourceID,
|
||||||
|
xhttp.AmzMetadataDirective: directive, xhttp.AmzTagDirective: replaceDirective,
|
||||||
|
xhttp.MinIOSourceReplicationRequest: "true", xhttp.AmzBucketReplicationStatus: "REPLICA",
|
||||||
|
}
|
||||||
|
// No source revision means no change, including no
|
||||||
|
// inheritance from a different source for a first write.
|
||||||
|
rec := r5Request(t, router, cred, http.MethodPut, "/"+bucket+"/"+name+"?versionId="+targetID, "", headers)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s %s: %d %s", backend, name, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
want := storedObjectLockState(targetMeta)
|
||||||
|
wantTags, wantStamp := "state=destination", objectLockTestStamp1000
|
||||||
|
if missing {
|
||||||
|
want = objectLockState{}
|
||||||
|
wantTags = ""
|
||||||
|
wantStamp = ""
|
||||||
|
}
|
||||||
|
got := r5Stored(t, obj, bucket, name, targetID, wantTags, wantStamp)
|
||||||
|
if state := storedObjectLockState(got.UserDefined); state != want {
|
||||||
|
t.Fatalf("%s absent revision: got %+v want %+v", name, state, want)
|
||||||
|
}
|
||||||
|
// The incoming update is older than the source but may
|
||||||
|
// still win at a new destination. A source lookup loses it.
|
||||||
|
headers[xhttp.MinIOSourceObjectRetentionTimestamp] = objectLockTestStamp0930
|
||||||
|
headers[xhttp.MinIOSourceObjectLegalHoldTimestamp] = objectLockTestStamp0930
|
||||||
|
headers[xhttp.AmzObjectLockMode] = "GOVERNANCE"
|
||||||
|
headers[xhttp.AmzObjectLockRetainUntilDate] = objectLockTestRetainUntil
|
||||||
|
headers[xhttp.AmzObjectLockLegalHold] = "ON"
|
||||||
|
headers[xhttp.MinIOSourceTaggingTimestamp] = objectLockTestStamp0930
|
||||||
|
headers[xhttp.AmzObjectTagging] = "state=incoming"
|
||||||
|
rec = r5Request(t, router, cred, http.MethodPut, "/"+bucket+"/"+name+"?versionId="+targetID, "", headers)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("incoming %s: %d %s", name, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if missing {
|
||||||
|
want = objectLockState{mode: "GOVERNANCE", retainUntil: "2030-01-01T00:00:00.000Z", retentionTimestamp: objectLockTestStamp0930, legalHold: "ON", legalHoldTimestamp: objectLockTestStamp0930}
|
||||||
|
wantTags = "state=incoming"
|
||||||
|
wantStamp = objectLockTestStamp0930
|
||||||
|
}
|
||||||
|
got = r5Stored(t, obj, bucket, name, targetID, wantTags, wantStamp)
|
||||||
|
if state := storedObjectLockState(got.UserDefined); state != want {
|
||||||
|
t.Fatalf("%s stale revision: got %+v want %+v", name, state, want)
|
||||||
|
}
|
||||||
|
// Authorized same-content release needs no governance
|
||||||
|
// bypass header; it must not be rejected by the data guard.
|
||||||
|
delete(headers, xhttp.AmzObjectLockMode)
|
||||||
|
delete(headers, xhttp.AmzObjectLockRetainUntilDate)
|
||||||
|
headers[xhttp.MinIOSourceObjectRetentionTimestamp] = "2026-09-03T10:30:00Z"
|
||||||
|
headers[xhttp.MinIOSourceObjectLegalHoldTimestamp] = "2026-09-03T10:30:00Z"
|
||||||
|
headers[xhttp.AmzObjectLockLegalHold] = "OFF"
|
||||||
|
rec = r5Request(t, router, cred, http.MethodPut, "/"+bucket+"/"+name+"?versionId="+targetID, "", headers)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("release %s: %d %s", name, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
got = r5Stored(t, obj, bucket, name, targetID, wantTags, wantStamp)
|
||||||
|
want = objectLockState{retentionTimestamp: "2026-09-03T10:30:00Z", legalHold: "OFF", legalHoldTimestamp: "2026-09-03T10:30:00Z"}
|
||||||
|
if state := storedObjectLockState(got.UserDefined); state != want {
|
||||||
|
t.Fatalf("%s release: got %+v want %+v", name, state, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReplicaReferenceCopyDestinationState(t *testing.T) {
|
||||||
|
z, bucket := consistencyPools(t)
|
||||||
|
for _, missing := range []bool{false, true} {
|
||||||
|
key := fmt.Sprintf("reference-copy-%t", missing)
|
||||||
|
id := mustGetUUID()
|
||||||
|
targetMeta := poolLockMetadata("GOVERNANCE", "ON", objectLockTestStamp1000, objectLockTestStamp1000)
|
||||||
|
targetMeta[xhttp.AmzObjectTagging] = "state=destination"
|
||||||
|
targetMeta[r5TagStamp] = objectLockTestStamp1000
|
||||||
|
if !missing {
|
||||||
|
putConsistencyObject(t, z, bucket, key, 0, "data", ObjectOptions{Versioned: true, VersionID: id, UserDefined: targetMeta})
|
||||||
|
}
|
||||||
|
sourceMeta := poolLockMetadata("", "OFF", objectLockTestStamp1100, objectLockTestStamp1100)
|
||||||
|
sourceMeta[xhttp.AmzObjectTagging] = "state=source"
|
||||||
|
sourceMeta[r5TagStamp] = objectLockTestStamp1100
|
||||||
|
src := putConsistencyObject(t, z, bucket, key, 0, "data", ObjectOptions{Versioned: true, UserDefined: sourceMeta})
|
||||||
|
src.metadataOnly = true
|
||||||
|
src.UserDefined = make(map[string]string)
|
||||||
|
_, err := z.CopyObject(t.Context(), bucket, key, bucket, key, src, ObjectOptions{VersionID: src.VersionID}, ObjectOptions{Versioned: true, VersionID: id, ReplicaLockReconcile: true, ReplicationRequest: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := storedObjectLockState(targetMeta)
|
||||||
|
wantTags, wantStamp := "state=destination", objectLockTestStamp1000
|
||||||
|
if missing {
|
||||||
|
want = objectLockState{}
|
||||||
|
wantTags = ""
|
||||||
|
wantStamp = ""
|
||||||
|
}
|
||||||
|
got := r5Stored(t, z, bucket, key, id, wantTags, wantStamp)
|
||||||
|
if state := storedObjectLockState(got.UserDefined); state != want {
|
||||||
|
t.Fatalf("missing=%t: got %+v want %+v", missing, state, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The set must not undo the all-pools decision with a newer weak revision
|
||||||
|
// from its own copy before the protected competing copy is retired.
|
||||||
|
func TestPoolsReplicaComplianceMetadataCopyOrder(t *testing.T) {
|
||||||
|
z, bucket := consistencyPools(t)
|
||||||
|
for _, protectedPool := range []int{0, 1} {
|
||||||
|
key := fmt.Sprintf("compliance-copy-%d", protectedPool)
|
||||||
|
id := mustGetUUID()
|
||||||
|
stamp := UTCNow().Add(-time.Hour)
|
||||||
|
var source ObjectInfo
|
||||||
|
for pool := range 2 {
|
||||||
|
meta := poolLockMetadata("", "OFF", objectLockTestStamp1100, objectLockTestStamp1000)
|
||||||
|
if pool == protectedPool {
|
||||||
|
meta = poolLockMetadata("COMPLIANCE", "OFF", objectLockTestStamp0900, objectLockTestStamp1000)
|
||||||
|
}
|
||||||
|
oi := putConsistencyObject(t, z, bucket, key, pool, "data", ObjectOptions{Versioned: true, VersionID: id, MTime: stamp, UserDefined: meta})
|
||||||
|
if pool == 0 {
|
||||||
|
source = oi
|
||||||
|
}
|
||||||
|
}
|
||||||
|
source.metadataOnly = true
|
||||||
|
source.UserDefined = map[string]string{}
|
||||||
|
_, err := z.CopyObject(t.Context(), bucket, key, bucket, key, source, ObjectOptions{VersionID: id}, ObjectOptions{VersionID: id, Versioned: true, ReplicaLockReconcile: true, ReplicationRequest: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := readObjectLockFields(t, z, bucket, key, id)
|
||||||
|
if got.mode != "COMPLIANCE" || got.retainUntil != "2030-01-01T00:00:00Z" {
|
||||||
|
t.Fatalf("protected pool %d lost COMPLIANCE: %+v", protectedPool, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPIReplicaRetentionTimestampAuthorization(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t, makeBucketOptions: MakeBucketOptions{LockEnabled: true},
|
||||||
|
objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, _ auth.Credentials, t *testing.T) {
|
||||||
|
basic := `"s3:GetObject","s3:PutObject","s3:ReplicateObject"`
|
||||||
|
noRetention := newObjectAttributesAuthzUser(t, backend, bucket, basic)
|
||||||
|
withRetention := newObjectAttributesAuthzUser(t, backend, bucket, basic+`,"s3:PutObjectRetention"`)
|
||||||
|
for _, operation := range []string{"put", "copy", "multipart"} {
|
||||||
|
for _, authorized := range []bool{false, true} {
|
||||||
|
for _, timestamp := range []bool{false, true} {
|
||||||
|
key := fmt.Sprintf("replica-permission/%s/%t/%t", operation, authorized, timestamp)
|
||||||
|
meta := poolLockMetadata("GOVERNANCE", "", objectLockTestStamp0900, "")
|
||||||
|
id := putObjectLockVersion(t, obj, bucket, key, meta)
|
||||||
|
headers := map[string]string{xhttp.MinIOSourceReplicationRequest: "true", xhttp.AmzBucketReplicationStatus: "REPLICA"}
|
||||||
|
if timestamp {
|
||||||
|
headers[xhttp.MinIOSourceObjectRetentionTimestamp] = objectLockTestStamp1000
|
||||||
|
}
|
||||||
|
method, path, body := http.MethodPut, "/"+bucket+"/"+key+"?versionId="+id, "data"
|
||||||
|
switch operation {
|
||||||
|
case "copy":
|
||||||
|
headers[xhttp.AmzCopySource] = "/" + bucket + "/" + key + "?versionId=" + id
|
||||||
|
headers[xhttp.AmzMetadataDirective] = replaceDirective
|
||||||
|
body = ""
|
||||||
|
case "multipart":
|
||||||
|
method = http.MethodPost
|
||||||
|
path += "&uploads"
|
||||||
|
body = ""
|
||||||
|
}
|
||||||
|
cred := noRetention
|
||||||
|
if authorized {
|
||||||
|
cred = withRetention
|
||||||
|
}
|
||||||
|
rec := r5Request(t, router, cred, method, path, body, headers)
|
||||||
|
want := http.StatusOK
|
||||||
|
if timestamp && !authorized {
|
||||||
|
want = http.StatusForbidden
|
||||||
|
}
|
||||||
|
if rec.Code != want {
|
||||||
|
t.Fatalf("%s: %d want %d: %s", key, rec.Code, want, rec.Body.String())
|
||||||
|
}
|
||||||
|
if want == http.StatusForbidden || !timestamp {
|
||||||
|
got := readObjectLockFields(t, obj, bucket, key, id)
|
||||||
|
if got.mode != "GOVERNANCE" || got.retentionStamp != objectLockTestStamp0900 {
|
||||||
|
t.Fatalf("unrequested/denied retention changed: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPILocalObjectLockTimestampsAdvance(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t, makeBucketOptions: MakeBucketOptions{LockEnabled: true},
|
||||||
|
objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
|
||||||
|
future := UTCNow().Add(48 * time.Hour)
|
||||||
|
meta := poolLockMetadata("GOVERNANCE", "ON", future.Format(time.RFC3339Nano), future.Format(time.RFC3339Nano))
|
||||||
|
id := putObjectLockVersion(t, obj, bucket, "local-lock", meta)
|
||||||
|
for n := 1; n <= 2; n++ {
|
||||||
|
if status := putRetention(t, router, cred, bucket, "local-lock", id, "GOVERNANCE", time.Date(2035, 1, n, 0, 0, 0, 0, time.UTC), false); status != http.StatusOK {
|
||||||
|
t.Fatalf("%s retention: %d", backend, status)
|
||||||
|
}
|
||||||
|
body := []byte(`<LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>OFF</Status></LegalHold>`)
|
||||||
|
sum := md5.Sum(body)
|
||||||
|
rec := retentionAuthzRequest(t, router, cred, http.MethodPut, makeTestTargetURL("", bucket, "local-lock", url.Values{"legal-hold": {""}, "versionId": {id}}), body, map[string]string{xhttp.ContentMD5: base64.StdEncoding.EncodeToString(sum[:])})
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s hold: %d %s", backend, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
got := readObjectLockFields(t, obj, bucket, "local-lock", id)
|
||||||
|
want := future.Add(time.Duration(n) * time.Nanosecond).Format(time.RFC3339Nano)
|
||||||
|
if got.retentionStamp != want || got.legalHoldStamp != want {
|
||||||
|
t.Fatalf("local mutation %d regressed: %+v want %s", n, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
type lockBoundaryReadFailure struct {
|
||||||
|
ObjectLayer
|
||||||
|
bucket, key, version string
|
||||||
|
metadataError error
|
||||||
|
destinationError error
|
||||||
|
deleteCalls int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *lockBoundaryReadFailure) GetObjectNInfo(ctx context.Context, bucket, key string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
|
||||||
|
if f.metadataError != nil && bucket == minioMetaBucket && strings.HasPrefix(key, pathJoin(bucketMetaPrefix, f.bucket)+"/") {
|
||||||
|
return nil, f.metadataError
|
||||||
|
}
|
||||||
|
return f.ObjectLayer.GetObjectNInfo(ctx, bucket, key, rs, h, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *lockBoundaryReadFailure) GetObjectInfo(ctx context.Context, bucket, key string, opts ObjectOptions) (ObjectInfo, error) {
|
||||||
|
if f.destinationError != nil && bucket == f.bucket && key == f.key && opts.VersionID == f.version {
|
||||||
|
return ObjectInfo{}, f.destinationError
|
||||||
|
}
|
||||||
|
return f.ObjectLayer.GetObjectInfo(ctx, bucket, key, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *lockBoundaryReadFailure) DeleteObject(ctx context.Context, bucket, key string, opts ObjectOptions) (ObjectInfo, error) {
|
||||||
|
f.deleteCalls++
|
||||||
|
return f.ObjectLayer.DeleteObject(ctx, bucket, key, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPICopyReplicaDestinationReadFailure(t *testing.T) {
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, makeBucketOptions: MakeBucketOptions{LockEnabled: true}, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||||
|
id := putObjectLockVersion(t, obj, bucket, "destination", poolLockMetadata("GOVERNANCE", "ON", objectLockTestStamp1000, objectLockTestStamp1000))
|
||||||
|
source := putObjectLockVersion(t, obj, bucket, "source", nil)
|
||||||
|
f := &lockBoundaryReadFailure{ObjectLayer: obj, bucket: bucket, key: "destination", version: id, destinationError: errErasureReadQuorum}
|
||||||
|
router := initTestAPIEndPoints(f, []string{"CopyObject"})
|
||||||
|
headers := map[string]string{xhttp.AmzCopySource: "/" + bucket + "/source?versionId=" + source, xhttp.MinIOSourceReplicationRequest: "true", xhttp.AmzBucketReplicationStatus: "REPLICA"}
|
||||||
|
rec := r5Request(t, router, cred, http.MethodPut, "/"+bucket+"/destination?versionId="+id, "", headers)
|
||||||
|
if rec.Code < 500 {
|
||||||
|
t.Fatalf("%s read failure allowed copy: %d %s", backend, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if got := readObjectLockFields(t, obj, bucket, "destination", id); got.legalHold != "ON" || got.mode != "GOVERNANCE" {
|
||||||
|
t.Fatalf("failed read changed state: %+v", got)
|
||||||
|
}
|
||||||
|
}})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPIForceDeleteLockConfigFailure(t *testing.T) {
|
||||||
|
for _, locked := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprintf("locked=%t", locked), func(t *testing.T) {
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, makeBucketOptions: MakeBucketOptions{LockEnabled: locked}, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||||
|
saved, err := globalBucketMetadataSys.Get(bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
restoreInit := markBucketMetadataInitialized(t)
|
||||||
|
defer restoreInit()
|
||||||
|
previous := newObjectLayerFn()
|
||||||
|
defer setObjectLayer(previous)
|
||||||
|
defer globalBucketMetadataSys.Set(bucket, saved)
|
||||||
|
for _, failure := range []string{"quorum", "unavailable", "cache-miss"} {
|
||||||
|
key := "force-" + failure + "/data"
|
||||||
|
putObjectLockVersion(t, obj, bucket, key, nil)
|
||||||
|
f := &lockBoundaryReadFailure{ObjectLayer: obj, bucket: bucket}
|
||||||
|
globalBucketMetadataSys.Remove(bucket)
|
||||||
|
switch failure {
|
||||||
|
case "quorum":
|
||||||
|
f.metadataError = errErasureReadQuorum
|
||||||
|
case "unavailable":
|
||||||
|
f.metadataError = errServerNotInitialized
|
||||||
|
}
|
||||||
|
router := initTestAPIEndPoints(f, []string{"DeleteObject"})
|
||||||
|
rec := r5Request(t, router, cred, http.MethodDelete, "/"+bucket+"/force-"+failure, "", map[string]string{xhttp.MinIOForceDelete: "true"})
|
||||||
|
setObjectLayer(previous)
|
||||||
|
globalBucketMetadataSys.Set(bucket, saved)
|
||||||
|
allowed := failure == "cache-miss" && !locked
|
||||||
|
if allowed {
|
||||||
|
if rec.Code != http.StatusNoContent || f.deleteCalls != 1 {
|
||||||
|
t.Fatalf("%s missing unlocked cache: %d calls=%d %s", backend, rec.Code, f.deleteCalls, rec.Body.String())
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if rec.Code < 400 || f.deleteCalls != 0 {
|
||||||
|
t.Fatalf("%s %s: force-delete proceeded: %d calls=%d %s", backend, failure, rec.Code, f.deleteCalls, rec.Body.String())
|
||||||
|
}
|
||||||
|
if _, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{}); err != nil {
|
||||||
|
t.Fatalf("rejected force-delete lost data: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPILocalCopyTagTimestampAdvances(t *testing.T) {
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
|
||||||
|
stamp := UTCNow().Add(48 * time.Hour)
|
||||||
|
meta := map[string]string{r5TagStamp: stamp.Format(time.RFC3339Nano), xhttp.AmzObjectTagging: "key=old"}
|
||||||
|
_, err := obj.PutObject(t.Context(), bucket, "local-copy", mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), ObjectOptions{UserDefined: maps.Clone(meta)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := r5Request(t, router, cred, http.MethodPut, "/"+bucket+"/local-copy", "", map[string]string{xhttp.AmzCopySource: "/" + bucket + "/local-copy", xhttp.AmzMetadataDirective: replaceDirective, xhttp.AmzTagDirective: replaceDirective})
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s copy: %d %s", backend, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
r5Stored(t, obj, bucket, "local-copy", "", "", stamp.Add(time.Nanosecond).Format(time.RFC3339Nano))
|
||||||
|
}})
|
||||||
|
}
|
||||||
@@ -427,7 +427,9 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, srcBucket, srcObject); s3Error != ErrNone {
|
if s3Error := withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode {
|
||||||
|
return authorizeVersionedRequestWithPotentialTags(ctx, r, policy.GetObjectAction, vid)
|
||||||
|
}); s3Error != ErrNone {
|
||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -491,7 +493,15 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var storedSrcTags *string
|
||||||
checkCopyPartPrecondFn := func(o ObjectInfo) bool {
|
checkCopyPartPrecondFn := func(o ObjectInfo) bool {
|
||||||
|
storedSrcTags = &o.UserTags
|
||||||
|
if s3Error := withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode {
|
||||||
|
return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, vid, o.UserTags)
|
||||||
|
}); s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return true
|
||||||
|
}
|
||||||
if _, err := DecryptObjectInfo(&o, r); err != nil {
|
if _, err := DecryptObjectInfo(&o, r); err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return true
|
return true
|
||||||
@@ -513,6 +523,20 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
|||||||
if isErrPreconditionFailed(err) {
|
if isErrPreconditionFailed(err) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
var s3Error APIErrorCode
|
||||||
|
if storedSrcTags != nil {
|
||||||
|
s3Error = withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode {
|
||||||
|
return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, vid, *storedSrcTags)
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
s3Error = withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode {
|
||||||
|
return authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, vid, err)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
if globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject) && gr != nil {
|
if globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject) && gr != nil {
|
||||||
// Versioning enabled quite possibly object is deleted might be delete-marker
|
// Versioning enabled quite possibly object is deleted might be delete-marker
|
||||||
// if present set the headers, no idea why AWS S3 sets these headers.
|
// if present set the headers, no idea why AWS S3 sets these headers.
|
||||||
@@ -521,11 +545,22 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
|||||||
w.Header()[xhttp.AmzDeleteMarker] = []string{strconv.FormatBool(gr.ObjInfo.DeleteMarker)}
|
w.Header()[xhttp.AmzDeleteMarker] = []string{strconv.FormatBool(gr.ObjInfo.DeleteMarker)}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
reqInfo := logger.GetReqInfo(ctx)
|
||||||
|
reqInfo.BucketName = srcBucket
|
||||||
|
reqInfo.ObjectName = srcObject
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer gr.Close()
|
defer gr.Close()
|
||||||
srcInfo := gr.ObjInfo
|
srcInfo := gr.ObjInfo
|
||||||
|
if storedSrcTags == nil {
|
||||||
|
if s3Error := withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode {
|
||||||
|
return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, vid, srcInfo.UserTags)
|
||||||
|
}); s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
actualPartSize, err := srcInfo.GetActualSize()
|
actualPartSize, err := srcInfo.GetActualSize()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -788,10 +823,14 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
|
|||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if size < 0 {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidArgument), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if size == -1 {
|
if size < 0 {
|
||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
// Copyright (c) 2026 PGSTY
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestIAMImportRejectsUnknownPolicyFields(t *testing.T) {
|
||||||
|
_, _ = iamAdminFixture(t)
|
||||||
|
for _, raw := range []string{
|
||||||
|
`{"Version":"2012-10-17","Statements":[]}`,
|
||||||
|
`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::b/*"],"Conditions":{"StringEquals":{"s3:ExistingObjectTag/classification":"public"}}}]}`,
|
||||||
|
} {
|
||||||
|
data := iamImportArchive(t, map[string]any{allPoliciesFile: map[string]json.RawMessage{"import-policy": json.RawMessage(raw)}})
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut, "http://localhost/minio/admin/v3/import-iam-v2", int64(len(data)), bytes.NewReader(data), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
||||||
|
mustIAM(t, err)
|
||||||
|
mustIAM(t, req.ParseForm())
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
adminAPIHandlers{}.ImportIAMV2(rec, req.WithContext(newContext(req, rec, "ImportIAM")))
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("import malformed policy: got %d %s, want 400", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSessionPolicyWriteKeepsStrictSemanticValidation(t *testing.T) {
|
||||||
|
raw := []byte(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:GetBucketQuota"],"Resource":["arn:aws:s3:::mybucket"],"NotResource":["arn:aws:s3:::otherbucket"]}]}`)
|
||||||
|
if _, err := parseSessionPolicyStrictBytes(raw); err == nil {
|
||||||
|
t.Fatal("new session policy accepted conflicting Resource and NotResource")
|
||||||
|
}
|
||||||
|
if _, err := parsePolicySnapshotBytes(raw); err != nil {
|
||||||
|
t.Fatalf("historical snapshot compatibility lost: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBucketPolicyRejectsUnsupportedNotPrincipal(t *testing.T) {
|
||||||
|
raw := []byte(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","NotPrincipal":{"AWS":"excluded-user"},"Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*"}]}`)
|
||||||
|
if _, err := parseBucketPolicyStrictBytes(raw, "b"); err == nil {
|
||||||
|
t.Fatal("unsupported NotPrincipal was silently discarded")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,159 @@
|
|||||||
|
// Copyright (c) 2026 PGSTY
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
|
||||||
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
|
)
|
||||||
|
|
||||||
|
type strictIAMStatementWire struct {
|
||||||
|
SID policy.ID `json:"Sid,omitempty"`
|
||||||
|
Effect policy.Effect `json:"Effect"`
|
||||||
|
Actions json.RawMessage `json:"Action,omitempty"`
|
||||||
|
NotActions json.RawMessage `json:"NotAction,omitempty"`
|
||||||
|
Resources json.RawMessage `json:"Resource,omitempty"`
|
||||||
|
NotResources json.RawMessage `json:"NotResource,omitempty"`
|
||||||
|
Conditions json.RawMessage `json:"Condition,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type strictIAMPolicyWire struct {
|
||||||
|
ID policy.ID `json:"ID,omitempty"`
|
||||||
|
Version string `json:"Version"`
|
||||||
|
Statements []strictIAMStatementWire `json:"Statement"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type strictPrincipalWire struct {
|
||||||
|
AWS json.RawMessage `json:"AWS"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type strictBucketStatementWire struct {
|
||||||
|
SID policy.ID `json:"Sid,omitempty"`
|
||||||
|
Effect policy.Effect `json:"Effect"`
|
||||||
|
Principal json.RawMessage `json:"Principal,omitempty"`
|
||||||
|
Actions json.RawMessage `json:"Action,omitempty"`
|
||||||
|
NotActions json.RawMessage `json:"NotAction,omitempty"`
|
||||||
|
Resources json.RawMessage `json:"Resource,omitempty"`
|
||||||
|
NotResources json.RawMessage `json:"NotResource,omitempty"`
|
||||||
|
Conditions json.RawMessage `json:"Condition,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type strictBucketPolicyWire struct {
|
||||||
|
ID policy.ID `json:"ID,omitempty"`
|
||||||
|
Version string `json:"Version"`
|
||||||
|
Statements []strictBucketStatementWire `json:"Statement"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateStrictIAMPolicyWire(data []byte) ([]byte, error) {
|
||||||
|
trimmed := bytes.TrimSpace(data)
|
||||||
|
if len(trimmed) == 0 || trimmed[0] != '{' {
|
||||||
|
return nil, policy.Errorf("policy must be a JSON object")
|
||||||
|
}
|
||||||
|
var wire strictIAMPolicyWire
|
||||||
|
dec := json.NewDecoder(bytes.NewReader(trimmed))
|
||||||
|
dec.DisallowUnknownFields()
|
||||||
|
if err := dec.Decode(&wire); err != nil {
|
||||||
|
return nil, policy.Errorf("%w", err)
|
||||||
|
}
|
||||||
|
var extra any
|
||||||
|
if err := dec.Decode(&extra); err != io.EOF {
|
||||||
|
if err == nil {
|
||||||
|
return nil, policy.Errorf("unexpected trailing JSON data in policy")
|
||||||
|
}
|
||||||
|
return nil, policy.Errorf("%w", err)
|
||||||
|
}
|
||||||
|
return trimmed, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parsePolicyStrictBytes(data []byte) (*policy.Policy, error) {
|
||||||
|
trimmed, err := validateStrictIAMPolicyWire(data)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
p, err := policy.ParseConfigStrict(bytes.NewReader(trimmed))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if p.IsEmpty() {
|
||||||
|
return nil, policy.Errorf("empty policies are not allowed")
|
||||||
|
}
|
||||||
|
return p, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseSessionPolicyStrictBytes(data []byte) (*policy.Policy, error) {
|
||||||
|
trimmed, err := validateStrictIAMPolicyWire(data)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return policy.ParseConfigStrict(bytes.NewReader(trimmed))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot imports retain supported historical policy semantics while rejecting
|
||||||
|
// unknown JSON fields that could silently discard restrictions.
|
||||||
|
func parsePolicySnapshotBytes(data []byte) (*policy.Policy, error) {
|
||||||
|
trimmed, err := validateStrictIAMPolicyWire(data)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return policy.ParseConfig(bytes.NewReader(trimmed))
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateStrictPrincipalWire(raw json.RawMessage) error {
|
||||||
|
trimmed := bytes.TrimSpace(raw)
|
||||||
|
if len(trimmed) == 0 || trimmed[0] != '{' {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var pw strictPrincipalWire
|
||||||
|
dec := json.NewDecoder(bytes.NewReader(trimmed))
|
||||||
|
dec.DisallowUnknownFields()
|
||||||
|
if err := dec.Decode(&pw); err != nil {
|
||||||
|
return policy.Errorf("%w", err)
|
||||||
|
}
|
||||||
|
var extra any
|
||||||
|
if err := dec.Decode(&extra); err != io.EOF {
|
||||||
|
if err == nil {
|
||||||
|
return policy.Errorf("unexpected trailing JSON data in principal")
|
||||||
|
}
|
||||||
|
return policy.Errorf("%w", err)
|
||||||
|
}
|
||||||
|
if len(bytes.TrimSpace(pw.AWS)) == 0 {
|
||||||
|
return policy.Errorf("invalid principal '%s'", string(raw))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseBucketPolicyStrictBytes(data []byte, bucket string) (*policy.BucketPolicy, error) {
|
||||||
|
trimmed := bytes.TrimSpace(data)
|
||||||
|
if len(trimmed) == 0 || trimmed[0] != '{' {
|
||||||
|
return nil, policy.Errorf("bucket policy must be a JSON object")
|
||||||
|
}
|
||||||
|
var wire strictBucketPolicyWire
|
||||||
|
dec := json.NewDecoder(bytes.NewReader(trimmed))
|
||||||
|
dec.DisallowUnknownFields()
|
||||||
|
if err := dec.Decode(&wire); err != nil {
|
||||||
|
return nil, policy.Errorf("%w", err)
|
||||||
|
}
|
||||||
|
var extra any
|
||||||
|
if err := dec.Decode(&extra); err != io.EOF {
|
||||||
|
if err == nil {
|
||||||
|
return nil, policy.Errorf("unexpected trailing JSON data in bucket policy")
|
||||||
|
}
|
||||||
|
return nil, policy.Errorf("%w", err)
|
||||||
|
}
|
||||||
|
for _, st := range wire.Statements {
|
||||||
|
if err := validateStrictPrincipalWire(st.Principal); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return policy.ParseBucketPolicyConfig(bytes.NewReader(trimmed), bucket)
|
||||||
|
}
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
// Copyright (c) 2026 PGSTY
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"encoding/base64"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
|
)
|
||||||
|
|
||||||
|
type trailerFaultReader struct {
|
||||||
|
err error
|
||||||
|
reads int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *trailerFaultReader) Read([]byte) (int, error) {
|
||||||
|
r.reads++
|
||||||
|
if r.reads > 2 {
|
||||||
|
return 0, io.EOF // Bound a broken parser rather than exhausting test memory.
|
||||||
|
}
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrailerReadPreservesConnectionErrors(t *testing.T) {
|
||||||
|
for _, signed := range []bool{false, true} {
|
||||||
|
for _, prefix := range []string{"", "checksum:value\r", "checksum:value\r\n", "checksum:value\r\nx-amz-trailer-signature:a\r"} {
|
||||||
|
if !signed && strings.Contains(prefix, "\r\n") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
t.Run(fmt.Sprintf("signed=%t/prefix=%q", signed, prefix), func(t *testing.T) {
|
||||||
|
failure := errors.New("injected connection reset")
|
||||||
|
fault := &trailerFaultReader{err: failure}
|
||||||
|
reader := bufio.NewReader(io.MultiReader(strings.NewReader(prefix), fault))
|
||||||
|
var err error
|
||||||
|
if signed {
|
||||||
|
err = (&s3ChunkedReader{reader: reader}).readTrailers()
|
||||||
|
} else {
|
||||||
|
err = (&s3UnsignedChunkedReader{reader: reader}).readTrailers()
|
||||||
|
}
|
||||||
|
if !errors.Is(err, failure) || fault.reads != 1 {
|
||||||
|
t.Fatalf("got err=%v reads=%d, want original connection error after one read", err, fault.reads)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLatestMetadataListingDoesNotRequireVersionGrants(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, _ auth.Credentials, t *testing.T) {
|
||||||
|
_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
|
||||||
|
mustIAM(t, err)
|
||||||
|
_, err = obj.PutObject(t.Context(), bucket, "latest", mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), ObjectOptions{
|
||||||
|
Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "classification=public"},
|
||||||
|
})
|
||||||
|
mustIAM(t, err)
|
||||||
|
cred := createTestPolicyUser(t, fmt.Sprintf(`{"Version":"2012-10-17","Statement":[
|
||||||
|
{"Effect":"Allow","Action":["s3:ListBucket","s3:ListBucketVersions"],"Resource":["arn:aws:s3:::%s"]},
|
||||||
|
{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectTagging"],"Resource":["arn:aws:s3:::%s/*"]},
|
||||||
|
{"Effect":"Deny","Action":["s3:GetObjectVersion","s3:GetObjectVersionTagging"],"Resource":["arn:aws:s3:::%s/*"]}]}`, bucket, bucket, bucket))
|
||||||
|
for _, query := range []string{"list-type=2", "versions="} {
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodGet, "/"+bucket+"?"+query+"&metadata=true&prefix=latest", 0, nil, cred.AccessKey, cred.SecretKey, nil)
|
||||||
|
mustIAM(t, err)
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, req)
|
||||||
|
wantTags := query == "list-type=2"
|
||||||
|
if rec.Code != http.StatusOK || strings.Contains(rec.Body.String(), "classification=public") != wantTags {
|
||||||
|
t.Fatalf("%s %s: status=%d body=%s, want tags=%t", backend, query, rec.Code, rec.Body.String(), wantTags)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVersionDeniesSurviveInheritedAndSessionPolicies(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, _ auth.Credentials, t *testing.T) {
|
||||||
|
ctx := t.Context()
|
||||||
|
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
|
||||||
|
mustIAM(t, err)
|
||||||
|
oi, err := obj.PutObject(ctx, bucket, "item", mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), ObjectOptions{
|
||||||
|
Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "classification=public"},
|
||||||
|
})
|
||||||
|
mustIAM(t, err)
|
||||||
|
allow := fmt.Sprintf(`{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectTagging"],"Resource":["arn:aws:s3:::%s/*"]}`, bucket)
|
||||||
|
deny := fmt.Sprintf(`{"Effect":"Deny","Action":["s3:GetObjectVersion","s3:GetObjectVersionTagging"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:versionid":%q}}}`, bucket, oi.VersionID)
|
||||||
|
document := func(statements string) string {
|
||||||
|
return `{"Version":"2012-10-17","Statement":[` + statements + `]}`
|
||||||
|
}
|
||||||
|
baseOnly := createTestPolicyUser(t, document(allow))
|
||||||
|
denied := createTestPolicyUser(t, document(allow+","+deny))
|
||||||
|
inherited, _, err := globalIAMSys.NewServiceAccount(ctx, denied.AccessKey, nil, newServiceAccountOpts{})
|
||||||
|
mustIAM(t, err)
|
||||||
|
session, err := policy.ParseConfig(strings.NewReader(document(allow + "," + deny)))
|
||||||
|
mustIAM(t, err)
|
||||||
|
inline, _, err := globalIAMSys.NewServiceAccount(ctx, globalActiveCred.AccessKey, nil, newServiceAccountOpts{sessionPolicy: session})
|
||||||
|
mustIAM(t, err)
|
||||||
|
secret, err := getTokenSigningKey()
|
||||||
|
mustIAM(t, err)
|
||||||
|
temporary, err := auth.GetNewCredentialsWithMetadata(map[string]any{
|
||||||
|
parentClaim: globalActiveCred.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(),
|
||||||
|
policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(document(allow + "," + deny))),
|
||||||
|
}, secret)
|
||||||
|
mustIAM(t, err)
|
||||||
|
temporary.ParentUser = globalActiveCred.AccessKey
|
||||||
|
_, err = globalIAMSys.SetTempUser(ctx, temporary.AccessKey, temporary, "")
|
||||||
|
mustIAM(t, err)
|
||||||
|
groupUser := createTestPolicyUser(t, document(allow))
|
||||||
|
groupPolicy, err := policy.ParseConfig(strings.NewReader(document(deny)))
|
||||||
|
mustIAM(t, err)
|
||||||
|
_, err = globalIAMSys.SetPolicy(ctx, "version-deny", *groupPolicy)
|
||||||
|
mustIAM(t, err)
|
||||||
|
_, err = globalIAMSys.AddUsersToGroup(ctx, "version-deny-group", []string{groupUser.AccessKey})
|
||||||
|
mustIAM(t, err)
|
||||||
|
_, err = globalIAMSys.PolicyDBSet(ctx, "version-deny-group", "version-deny", regUser, true)
|
||||||
|
mustIAM(t, err)
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
cred auth.Credentials
|
||||||
|
deny bool
|
||||||
|
}{
|
||||||
|
{"legacy-base-grant", baseOnly, false},
|
||||||
|
{"regular", denied, true},
|
||||||
|
{"inherited-service-account", inherited, true},
|
||||||
|
{"root-service-account-session", inline, true},
|
||||||
|
{"root-sts-session", temporary, true},
|
||||||
|
{"group", groupUser, true},
|
||||||
|
} {
|
||||||
|
for _, query := range []string{"", "?versionId=" + oi.VersionID, "?tagging=&versionId=" + oi.VersionID} {
|
||||||
|
headers := map[string]string{xhttp.AmzTagDirective: "ACCESS"}
|
||||||
|
if tc.cred.IsTemp() {
|
||||||
|
headers[xhttp.AmzSecurityToken] = tc.cred.SessionToken
|
||||||
|
}
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodGet, "/"+bucket+"/item"+query, 0, nil, tc.cred.AccessKey, tc.cred.SecretKey, headers)
|
||||||
|
mustIAM(t, err)
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, req)
|
||||||
|
want := http.StatusOK
|
||||||
|
if tc.deny && query != "" {
|
||||||
|
want = http.StatusForbidden
|
||||||
|
}
|
||||||
|
if rec.Code != want {
|
||||||
|
t.Errorf("%s %s %s: got %d %s, want %d", backend, tc.name, query, rec.Code, rec.Body.String(), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}})
|
||||||
|
}
|
||||||
@@ -0,0 +1,941 @@
|
|||||||
|
// Copyright (c) 2026 PGSTY
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/zip"
|
||||||
|
"bytes"
|
||||||
|
"crypto/hmac"
|
||||||
|
"crypto/md5"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"encoding/xml"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/madmin-go/v3"
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
"github.com/minio/minio/internal/bucket/versioning"
|
||||||
|
"github.com/minio/minio/internal/crypto"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
"github.com/minio/minio/internal/kms"
|
||||||
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestStrictPolicyParsingRejectsMalformedAndTypoJSON(t *testing.T) {
|
||||||
|
invalidDocs := []struct {
|
||||||
|
name string
|
||||||
|
raw string
|
||||||
|
}{
|
||||||
|
{"null-literal", "null"},
|
||||||
|
{"array-literal", "[]"},
|
||||||
|
{"string-literal", `"policy"`},
|
||||||
|
{"trailing-object", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::b/*"]}]} {"extra":1}`},
|
||||||
|
{"typo-statement-key", `{"Version":"2012-10-17","Statment":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::b/*"]}]}`}, //nolint:misspell
|
||||||
|
{"unknown-stmt-field", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Actions":["s3:GetObject"],"Resource":["arn:aws:s3:::b/*"]}]}`},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range invalidDocs {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if _, err := parsePolicyStrictBytes([]byte(tc.raw)); err == nil {
|
||||||
|
t.Fatalf("parsePolicyStrictBytes(%s) unexpectedly succeeded", tc.name)
|
||||||
|
}
|
||||||
|
if _, err := parseSessionPolicyStrictBytes([]byte(tc.raw)); err == nil {
|
||||||
|
t.Fatalf("parseSessionPolicyStrictBytes(%s) unexpectedly succeeded", tc.name)
|
||||||
|
}
|
||||||
|
if _, err := parseBucketPolicyStrictBytes([]byte(tc.raw), "b"); err == nil {
|
||||||
|
t.Fatalf("parseBucketPolicyStrictBytes(%s) unexpectedly succeeded", tc.name)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bucket policy must reject unknown Principal keys such as Service instead of treating AWS as empty.
|
||||||
|
servicePrincipalPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":["s3.amazonaws.com"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::b/*"]}]}`
|
||||||
|
if _, err := parseBucketPolicyStrictBytes([]byte(servicePrincipalPolicy), "b"); err == nil {
|
||||||
|
t.Fatal("parseBucketPolicyStrictBytes accepted Principal.Service without AWS key")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Explicit session-policy reset documents remain accepted for service accounts,
|
||||||
|
// while named IAM policies reject empty statement lists.
|
||||||
|
for _, resetDoc := range []string{`{}`, `{"Statement":[]}`, `{"Version":"2012-10-17","Statement":[]}`} {
|
||||||
|
sp, err := parseSessionPolicyStrictBytes([]byte(resetDoc))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parseSessionPolicyStrictBytes(%s) unexpected error: %v", resetDoc, err)
|
||||||
|
}
|
||||||
|
if !sp.IsEmpty() {
|
||||||
|
t.Fatalf("parseSessionPolicyStrictBytes(%s) expected empty policy", resetDoc)
|
||||||
|
}
|
||||||
|
if _, err := parsePolicyStrictBytes([]byte(resetDoc)); err == nil {
|
||||||
|
t.Fatalf("parsePolicyStrictBytes(%s) accepted empty named policy", resetDoc)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Versioning ExcludedPrefixes must be recognized as PrefixesExcluded so Object Lock import rejects it.
|
||||||
|
var v versioning.Versioning
|
||||||
|
if err := xml.Unmarshal([]byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status><ExcludedPrefixes><Prefix>tmp/</Prefix></ExcludedPrefixes></VersioningConfiguration>`), &v); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !v.PrefixesExcluded() {
|
||||||
|
t.Fatal("expected v.PrefixesExcluded() == true when ExcludedPrefixes is populated")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLegacyEmptyEmbeddedPolicyServiceAccountStaysDenyAll(t *testing.T) {
|
||||||
|
f := newGroupRefreshFixture(t)
|
||||||
|
_, err := f.sys.PolicyDBSet(f.ctx, f.parent.AccessKey, "readwrite", regUser, false)
|
||||||
|
mustIAM(t, err)
|
||||||
|
|
||||||
|
// Sanity check: parent has full access.
|
||||||
|
if !f.sys.IsAllowed(policy.Args{AccountName: f.parent.AccessKey, Action: policy.GetObjectAction, BucketName: "b", ObjectName: "k"}) {
|
||||||
|
t.Fatal("expected parent user to allow GetObject")
|
||||||
|
}
|
||||||
|
|
||||||
|
const legacyKey = "legacy-deny-all-sa"
|
||||||
|
legacyClaims := map[string]any{
|
||||||
|
iamPolicyClaimNameSA(): embeddedPolicyType,
|
||||||
|
policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(`{"Version":"2012-10-17","Statement":[]}`)),
|
||||||
|
}
|
||||||
|
_, _, err = f.sys.NewServiceAccount(f.ctx, f.parent.AccessKey, nil, newServiceAccountOpts{
|
||||||
|
accessKey: legacyKey,
|
||||||
|
secretKey: "legacy-secret-key-12345",
|
||||||
|
claims: legacyClaims,
|
||||||
|
preserveEmptyEmbeddedPolicy: true,
|
||||||
|
})
|
||||||
|
mustIAM(t, err)
|
||||||
|
mustIAM(t, f.sys.store.LoadIAMCache(f.ctx, false))
|
||||||
|
|
||||||
|
u, valid, err := f.sys.CheckKey(f.ctx, legacyKey)
|
||||||
|
mustIAM(t, err)
|
||||||
|
if !valid {
|
||||||
|
t.Fatal("expected legacy service account to remain valid")
|
||||||
|
}
|
||||||
|
if f.sys.IsAllowed(policy.Args{AccountName: legacyKey, Claims: u.Credentials.Claims, Action: policy.GetObjectAction, BucketName: "b", ObjectName: "k"}) {
|
||||||
|
t.Fatal("legacy empty embedded-policy service account was promoted to parent permissions")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Metadata-only edit (nil sessionPolicy) must preserve the deny-all embedded-policy state.
|
||||||
|
_, err = f.sys.UpdateServiceAccount(f.ctx, legacyKey, updateServiceAccountOpts{
|
||||||
|
description: "metadata-only-update",
|
||||||
|
})
|
||||||
|
mustIAM(t, err)
|
||||||
|
mustIAM(t, f.sys.store.LoadIAMCache(f.ctx, false))
|
||||||
|
|
||||||
|
u, _, err = f.sys.CheckKey(f.ctx, legacyKey)
|
||||||
|
mustIAM(t, err)
|
||||||
|
if u.Credentials.Description != "metadata-only-update" {
|
||||||
|
t.Fatalf("description = %q, want metadata-only-update", u.Credentials.Description)
|
||||||
|
}
|
||||||
|
if f.sys.IsAllowed(policy.Args{AccountName: legacyKey, Claims: u.Credentials.Claims, Action: policy.GetObjectAction, BucketName: "b", ObjectName: "k"}) {
|
||||||
|
t.Fatal("metadata-only edit promoted legacy deny-all service account to parent permissions")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Explicit empty reset document clears the inline policy to parent inheritance.
|
||||||
|
_, err = f.sys.UpdateServiceAccount(f.ctx, legacyKey, updateServiceAccountOpts{
|
||||||
|
sessionPolicy: &policy.Policy{},
|
||||||
|
})
|
||||||
|
mustIAM(t, err)
|
||||||
|
mustIAM(t, f.sys.store.LoadIAMCache(f.ctx, false))
|
||||||
|
|
||||||
|
u, _, err = f.sys.CheckKey(f.ctx, legacyKey)
|
||||||
|
mustIAM(t, err)
|
||||||
|
if !f.sys.IsAllowed(policy.Args{AccountName: legacyKey, Claims: u.Credentials.Claims, Action: policy.GetObjectAction, BucketName: "b", ObjectName: "k"}) {
|
||||||
|
t.Fatal("explicit empty-policy reset did not restore parent inheritance")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestObjectLockSuspendedAndExcludedNullVersionProtection(t *testing.T) {
|
||||||
|
t.Run("multi-pool-null-version-lock", func(t *testing.T) {
|
||||||
|
z, bucket := consistencyPools(t)
|
||||||
|
const key = "null-locked-obj"
|
||||||
|
retainUntil := UTCNow().Add(24 * time.Hour).Format(time.RFC3339)
|
||||||
|
|
||||||
|
// Pool 0 holds a COMPLIANCE-locked null version; Pool 1 holds an unlocked competing copy.
|
||||||
|
metaLocked := poolLockMetadata("COMPLIANCE", "OFF", objectLockTestStamp0900, "")
|
||||||
|
metaLocked[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retainUntil
|
||||||
|
metaUnlocked := poolLockMetadata("", "OFF", objectLockTestStamp1000, "")
|
||||||
|
stamp := UTCNow().Add(-time.Minute)
|
||||||
|
putConsistencyObject(t, z, bucket, key, 0, "locked-null", ObjectOptions{
|
||||||
|
VersionID: nullVersionID, VersionSuspended: true, MTime: stamp, UserDefined: metaLocked,
|
||||||
|
})
|
||||||
|
putConsistencyObject(t, z, bucket, key, 1, "locked-null", ObjectOptions{
|
||||||
|
VersionID: nullVersionID, VersionSuspended: true, MTime: stamp.Add(time.Second), UserDefined: metaUnlocked,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Unaddressed DeleteObject (prefix-excluded or unversioned delete with RefuseLockedOverwrite) must fail closed,
|
||||||
|
// even if a newer unlocked UUID version also exists on top of the locked null version.
|
||||||
|
putConsistencyObject(t, z, bucket, key, 1, "newer-unlocked-v2", ObjectOptions{
|
||||||
|
VersionID: mustGetUUID(), Versioned: true, MTime: stamp.Add(2 * time.Second),
|
||||||
|
})
|
||||||
|
_, err := z.DeleteObject(t.Context(), bucket, key, ObjectOptions{RefuseLockedOverwrite: true})
|
||||||
|
var lockedErr ObjectLocked
|
||||||
|
if !errors.As(err, &lockedErr) {
|
||||||
|
t.Fatalf("DeleteObject error = %v, want ObjectLocked", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify the COMPLIANCE-locked null version on pool 0 is untouched.
|
||||||
|
got, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: nullVersionID, VersionSuspended: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("locked null version lost from pool 0: %v", err)
|
||||||
|
}
|
||||||
|
if got.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] != "COMPLIANCE" {
|
||||||
|
t.Fatalf("unexpected retention mode after blocked delete: %+v", got.UserDefined)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("api-excluded-prefix-unaddressed-delete-and-bulk-delete", func(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
makeBucketOptions: MakeBucketOptions{LockEnabled: true},
|
||||||
|
objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
|
||||||
|
const key = "excl/worm-null"
|
||||||
|
retainUntil := UTCNow().Add(24 * time.Hour).Format(time.RFC3339)
|
||||||
|
meta := map[string]string{
|
||||||
|
strings.ToLower(xhttp.AmzObjectLockMode): "COMPLIANCE",
|
||||||
|
strings.ToLower(xhttp.AmzObjectLockRetainUntilDate): retainUntil,
|
||||||
|
}
|
||||||
|
const payload = "immutable-null-version-data"
|
||||||
|
// Write a COMPLIANCE-locked null version AND a newer unlocked UUID version on top of it,
|
||||||
|
// then simulate a legacy Object Lock bucket with ExcludedPrefixes loaded in memory.
|
||||||
|
_, err := obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, strings.NewReader(payload), int64(len(payload)), "", ""), ObjectOptions{
|
||||||
|
Versioned: false,
|
||||||
|
UserDefined: meta,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: PutObject null: %v", backend, err)
|
||||||
|
}
|
||||||
|
_, err = obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, strings.NewReader("newer-v2"), int64(len("newer-v2")), "", ""), ObjectOptions{
|
||||||
|
Versioned: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: PutObject v2: %v", backend, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
bm, err := globalBucketMetadataSys.Get(bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: get bucket metadata: %v", backend, err)
|
||||||
|
}
|
||||||
|
bm.versioningConfig = &versioning.Versioning{
|
||||||
|
XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
|
||||||
|
Status: versioning.Enabled,
|
||||||
|
ExcludedPrefixes: []versioning.ExcludedPrefix{{Prefix: "excl/"}},
|
||||||
|
}
|
||||||
|
globalBucketMetadataSys.Set(bucket, bm)
|
||||||
|
|
||||||
|
// 1. Unaddressed DELETE /bucket/excl/worm-null must fail closed with ErrObjectLocked (400 InvalidRequest).
|
||||||
|
delReq, err := newTestSignedRequestV4(http.MethodDelete, "/"+bucket+"/"+key, 0, nil, cred.AccessKey, cred.SecretKey, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, delReq)
|
||||||
|
if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "WORM protected") {
|
||||||
|
t.Fatalf("%s: unaddressed DELETE status = %d, want 400 WORM protected; body=%s", backend, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Unaddressed Multi-Object Delete POST /bucket?delete must report WORM error per key.
|
||||||
|
delBody := `<Delete><Object><Key>` + key + `</Key></Object></Delete>`
|
||||||
|
sum := md5.Sum([]byte(delBody))
|
||||||
|
headers := map[string]string{
|
||||||
|
xhttp.ContentMD5: base64.StdEncoding.EncodeToString(sum[:]),
|
||||||
|
}
|
||||||
|
bulkReq, err := newTestSignedRequestV4(http.MethodPost, "/"+bucket+"?delete", int64(len(delBody)), strings.NewReader(delBody), cred.AccessKey, cred.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec = httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, bulkReq)
|
||||||
|
if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "WORM protected") {
|
||||||
|
t.Fatalf("%s: DeleteObjects status=%d body=%s, want WORM protected error entry", backend, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Locked null-version object must remain intact and readable.
|
||||||
|
if _, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: nullVersionID, VersionSuspended: true}); err != nil {
|
||||||
|
t.Fatalf("%s: locked null-version object disappeared: %v", backend, err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStreamingTrailerEOFAndNegativeDecodedLength(t *testing.T) {
|
||||||
|
t.Run("unsigned-trailer-truncated-and-oversized", func(t *testing.T) {
|
||||||
|
// Truncated mid-trailer (missing final \r\n\r\n) must return ErrUnexpectedEOF, never nil/EOF.
|
||||||
|
truncatedBody := "5\r\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA=="
|
||||||
|
req := httptest.NewRequest(http.MethodPut, "http://minio.local/b/k", strings.NewReader(truncatedBody))
|
||||||
|
req.Header.Set(xhttp.AmzContentSha256, unsignedPayloadTrailer)
|
||||||
|
req.Header.Set(xhttp.AmzDecodedContentLength, "5")
|
||||||
|
req.Header.Set(awsTrailerHeader, "x-amz-checksum-crc32")
|
||||||
|
|
||||||
|
rd, code := newUnsignedV4ChunkedReader(req, true)
|
||||||
|
if code != ErrNone {
|
||||||
|
t.Fatalf("newUnsignedV4ChunkedReader code = %v", code)
|
||||||
|
}
|
||||||
|
_, err := io.ReadAll(rd)
|
||||||
|
rd.Close()
|
||||||
|
if !errors.Is(err, io.ErrUnexpectedEOF) {
|
||||||
|
t.Fatalf("truncated trailer ReadAll error = %v, want io.ErrUnexpectedEOF", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Oversized trailer (> 64 KiB without \r\n\r\n) must reject cleanly without unbounded growth.
|
||||||
|
oversizedBody := "1\r\na\r\n0\r\nx-amz-checksum-crc32:" + strings.Repeat("A", 64*1024+1024)
|
||||||
|
req2 := httptest.NewRequest(http.MethodPut, "http://minio.local/b/k", strings.NewReader(oversizedBody))
|
||||||
|
req2.Header.Set(xhttp.AmzContentSha256, unsignedPayloadTrailer)
|
||||||
|
req2.Header.Set(xhttp.AmzDecodedContentLength, "1")
|
||||||
|
req2.Header.Set(awsTrailerHeader, "x-amz-checksum-crc32")
|
||||||
|
|
||||||
|
rd2, code := newUnsignedV4ChunkedReader(req2, true)
|
||||||
|
if code != ErrNone {
|
||||||
|
t.Fatalf("newUnsignedV4ChunkedReader oversized code = %v", code)
|
||||||
|
}
|
||||||
|
_, err = io.ReadAll(rd2)
|
||||||
|
rd2.Close()
|
||||||
|
if err == nil || errors.Is(err, io.EOF) {
|
||||||
|
t.Fatalf("oversized trailer ReadAll error = %v, want non-EOF error", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("negative-decoded-content-length-rejected", func(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
endpoints: []string{"PutObject"},
|
||||||
|
objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
|
||||||
|
req, err := newStreamingUnsignedTrailerRequest(http.MethodPut, getPutObjectURL("", bucket, "neg-len"), []byte("abc"), UTCNow())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
req.Header.Set(xhttp.AmzDecodedContentLength, "-5")
|
||||||
|
if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("%s: negative X-Amz-Decoded-Content-Length status = %d, want 400; body=%s", backend, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if _, err := obj.GetObjectInfo(t.Context(), bucket, "neg-len", ObjectOptions{}); err == nil {
|
||||||
|
t.Fatalf("%s: object stored despite negative X-Amz-Decoded-Content-Length", backend)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func signV4WithScopeDate(req *http.Request, accessKey, secretKey, region string, reqTime, scopeTime time.Time, payloadHash string) {
|
||||||
|
req.Header.Set(xhttp.AmzDate, reqTime.Format(iso8601Format))
|
||||||
|
req.Header.Set(xhttp.AmzContentSha256, payloadHash)
|
||||||
|
if req.Header.Get("Host") == "" {
|
||||||
|
req.Header.Set("Host", req.Host)
|
||||||
|
}
|
||||||
|
signedHeaders := []string{"host", "x-amz-content-sha256", "x-amz-date"}
|
||||||
|
canonicalHeaders := fmt.Sprintf("host:%s\nx-amz-content-sha256:%s\nx-amz-date:%s\n",
|
||||||
|
req.Host, payloadHash, reqTime.Format(iso8601Format))
|
||||||
|
canonicalRequest := strings.Join([]string{
|
||||||
|
req.Method,
|
||||||
|
req.URL.EscapedPath(),
|
||||||
|
req.URL.RawQuery,
|
||||||
|
canonicalHeaders,
|
||||||
|
strings.Join(signedHeaders, ";"),
|
||||||
|
payloadHash,
|
||||||
|
}, "\n")
|
||||||
|
scope := fmt.Sprintf("%s/%s/s3/aws4_request", scopeTime.Format(yyyymmdd), region)
|
||||||
|
sum := sha256.Sum256([]byte(canonicalRequest))
|
||||||
|
stringToSign := strings.Join([]string{
|
||||||
|
signV4Algorithm,
|
||||||
|
reqTime.Format(iso8601Format),
|
||||||
|
scope,
|
||||||
|
hex.EncodeToString(sum[:]),
|
||||||
|
}, "\n")
|
||||||
|
signingKey := getSigningKey(secretKey, scopeTime, region, serviceS3)
|
||||||
|
sig := getSignature(signingKey, stringToSign)
|
||||||
|
req.Header.Set(xhttp.Authorization, fmt.Sprintf("%s Credential=%s/%s, SignedHeaders=%s, Signature=%s",
|
||||||
|
signV4Algorithm, accessKey, scope, strings.Join(signedHeaders, ";"), sig))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSigV4ScopeDateMismatchRejected(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
endpoints: []string{"GetObject", "PutObject"},
|
||||||
|
objAPITest: func(_ ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||||
|
now := UTCNow()
|
||||||
|
yesterday := now.Add(-24 * time.Hour)
|
||||||
|
const region = globalMinioDefaultRegion
|
||||||
|
|
||||||
|
// 1. Standard Header SigV4
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "http://minio.local/"+bucket+"/obj", nil)
|
||||||
|
signV4WithScopeDate(req, cred.AccessKey, cred.SecretKey, region, now, yesterday, emptySHA256)
|
||||||
|
if code := doesSignatureMatch(emptySHA256, req, region, serviceS3); code == ErrNone {
|
||||||
|
t.Fatalf("%s: doesSignatureMatch accepted mismatched scope date", backend)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Streaming Seed SigV4
|
||||||
|
sreq := httptest.NewRequest(http.MethodPut, "http://minio.local/"+bucket+"/obj", strings.NewReader("0\r\n\r\n"))
|
||||||
|
sreq.Header.Set(xhttp.AmzDecodedContentLength, "0")
|
||||||
|
signV4WithScopeDate(sreq, cred.AccessKey, cred.SecretKey, region, now, yesterday, streamingContentSHA256)
|
||||||
|
if _, _, _, _, code := calculateSeedSignature(sreq, false); code == ErrNone {
|
||||||
|
t.Fatalf("%s: calculateSeedSignature accepted mismatched scope date", backend)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Presigned SigV4
|
||||||
|
q := url.Values{}
|
||||||
|
scope := fmt.Sprintf("%s/%s/s3/aws4_request", yesterday.Format(yyyymmdd), region)
|
||||||
|
q.Set(xhttp.AmzAlgorithm, signV4Algorithm)
|
||||||
|
q.Set(xhttp.AmzCredential, cred.AccessKey+"/"+scope)
|
||||||
|
q.Set(xhttp.AmzDate, now.Format(iso8601Format))
|
||||||
|
q.Set(xhttp.AmzExpires, "300")
|
||||||
|
q.Set(xhttp.AmzSignedHeaders, "host")
|
||||||
|
preq := httptest.NewRequest(http.MethodGet, "http://minio.local/"+bucket+"/obj?"+q.Encode(), nil)
|
||||||
|
canonicalPresigned := strings.Join([]string{
|
||||||
|
http.MethodGet,
|
||||||
|
preq.URL.EscapedPath(),
|
||||||
|
preq.URL.Query().Encode(),
|
||||||
|
"host:" + preq.Host + "\n",
|
||||||
|
"host",
|
||||||
|
unsignedPayload,
|
||||||
|
}, "\n")
|
||||||
|
psum := sha256.Sum256([]byte(canonicalPresigned))
|
||||||
|
pSTS := strings.Join([]string{
|
||||||
|
signV4Algorithm,
|
||||||
|
now.Format(iso8601Format),
|
||||||
|
scope,
|
||||||
|
hex.EncodeToString(psum[:]),
|
||||||
|
}, "\n")
|
||||||
|
pSig := getSignature(getSigningKey(cred.SecretKey, yesterday, region, serviceS3), pSTS)
|
||||||
|
q.Set(xhttp.AmzSignature, pSig)
|
||||||
|
preq.URL.RawQuery = q.Encode()
|
||||||
|
if code := doesPresignedSignatureMatch(unsignedPayload, preq, region, serviceS3); code == ErrNone {
|
||||||
|
t.Fatalf("%s: doesPresignedSignatureMatch accepted mismatched scope date", backend)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. POST Policy SigV4
|
||||||
|
policyDoc := fmt.Sprintf(`{"expiration":%q,"conditions":[["eq","$bucket",%q],["eq","$x-amz-date",%q],["eq","$x-amz-credential",%q],["eq","$x-amz-algorithm","AWS4-HMAC-SHA256"]]}`,
|
||||||
|
now.Add(10*time.Minute).Format(iso8601Format), bucket, now.Format(iso8601Format), cred.AccessKey+"/"+scope)
|
||||||
|
policyB64 := base64.StdEncoding.EncodeToString([]byte(policyDoc))
|
||||||
|
mac := hmac.New(sha256.New, getSigningKey(cred.SecretKey, yesterday, region, serviceS3))
|
||||||
|
mac.Write([]byte(policyB64))
|
||||||
|
postSig := hex.EncodeToString(mac.Sum(nil))
|
||||||
|
formValues := http.Header{}
|
||||||
|
formValues.Set(xhttp.AmzAlgorithm, signV4Algorithm)
|
||||||
|
formValues.Set(xhttp.AmzCredential, cred.AccessKey+"/"+scope)
|
||||||
|
formValues.Set(xhttp.AmzDate, now.Format(iso8601Format))
|
||||||
|
formValues.Set(xhttp.AmzSignature, postSig)
|
||||||
|
formValues.Set("Policy", policyB64)
|
||||||
|
if _, code := doesPolicySignatureV4Match(formValues); code == ErrNone {
|
||||||
|
t.Fatalf("%s: doesPolicySignatureV4Match accepted mismatched scope date", backend)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBatchKeyRotateLegacyXLV1AndSSES3ToKMS(t *testing.T) {
|
||||||
|
f := newBatchAuthzFixture(t)
|
||||||
|
previous := GlobalKMS
|
||||||
|
t.Cleanup(func() { GlobalKMS = previous })
|
||||||
|
configured, err := kms.NewBuiltin("rotate-key", bytes.Repeat([]byte{0x37}, 32))
|
||||||
|
mustIAM(t, err)
|
||||||
|
GlobalKMS = configured
|
||||||
|
|
||||||
|
t.Run("sse-s3-multipart-to-sse-kms-empty-context", func(t *testing.T) {
|
||||||
|
const object = "sses3-mp/item"
|
||||||
|
const payload = "multipart sse-s3 migrated to sse-kms without losing multipart flag"
|
||||||
|
metadata := map[string]string{
|
||||||
|
crypto.MetaMultipart: "1",
|
||||||
|
}
|
||||||
|
key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, f.bucket, object, metadata, nil)
|
||||||
|
mustIAM(t, err)
|
||||||
|
ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader(payload), key))
|
||||||
|
mustIAM(t, err)
|
||||||
|
original, err := f.obj.PutObject(f.ctx, f.bucket, object, mustGetPutObjReader(t, bytes.NewReader(ciphertext), int64(len(ciphertext)), "", ""), ObjectOptions{
|
||||||
|
Versioned: true,
|
||||||
|
UserDefined: metadata,
|
||||||
|
})
|
||||||
|
mustIAM(t, err)
|
||||||
|
|
||||||
|
rec, job := f.submit(t, globalActiveCred, fmt.Sprintf("keyrotate:\n apiVersion: v1\n bucket: %s\n prefix: sses3-mp/\n encryption:\n type: sse-kms\n key: rotate-key\n", f.bucket))
|
||||||
|
if job == nil {
|
||||||
|
t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
mustIAM(t, job.KeyRotate.Start(f.ctx, f.obj, *job))
|
||||||
|
progress := f.jobInfo(t, job)
|
||||||
|
if !progress.Complete || progress.Failed || progress.Objects != 1 {
|
||||||
|
t.Fatalf("SSE-S3 -> SSE-KMS rotation failed: %+v", progress)
|
||||||
|
}
|
||||||
|
|
||||||
|
after, err := f.obj.GetObjectInfo(f.ctx, f.bucket, object, ObjectOptions{VersionID: original.VersionID})
|
||||||
|
mustIAM(t, err)
|
||||||
|
if !crypto.S3KMS.IsEncrypted(after.UserDefined) || crypto.S3.IsEncrypted(after.UserDefined) {
|
||||||
|
t.Fatalf("unexpected encryption headers after SSE-S3 -> SSE-KMS: %+v", after.UserDefined)
|
||||||
|
}
|
||||||
|
if after.UserDefined[crypto.MetaMultipart] != "1" {
|
||||||
|
t.Fatalf("MetaMultipart stripped during SSE-S3 -> SSE-KMS rotation: %+v", after.UserDefined)
|
||||||
|
}
|
||||||
|
actual, err := crypto.S3KMS.UnsealObjectKey(configured, after.UserDefined, f.bucket, object)
|
||||||
|
mustIAM(t, err)
|
||||||
|
if actual != key {
|
||||||
|
t.Fatal("data encryption key changed during metadata rotation")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("legacy-xlv1-stream-rewrite-rotation", func(t *testing.T) {
|
||||||
|
const legacyBucket = "legacy-xlv1-bucket"
|
||||||
|
mustIAM(t, f.obj.MakeBucket(f.ctx, legacyBucket, MakeBucketOptions{}))
|
||||||
|
|
||||||
|
const object = "legacy/obj"
|
||||||
|
payload := strings.Repeat("legacy XLV1 object rotated cleanly to XLV2 without panic! ", 4096)
|
||||||
|
metadata := map[string]string{
|
||||||
|
xhttp.AmzObjectTagging: "tier=legacy",
|
||||||
|
}
|
||||||
|
key, err := newEncryptMetadata(f.ctx, crypto.S3KMS, "rotate-key", nil, legacyBucket, object, metadata, kms.Context{"stage": "v1"})
|
||||||
|
mustIAM(t, err)
|
||||||
|
ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader(payload), key))
|
||||||
|
mustIAM(t, err)
|
||||||
|
plainMD5 := md5.Sum([]byte(payload))
|
||||||
|
sealedETag := hex.EncodeToString(key.SealETag(plainMD5[:]))
|
||||||
|
metadata["etag"] = sealedETag
|
||||||
|
|
||||||
|
_, err = f.obj.PutObject(f.ctx, legacyBucket, object, mustGetPutObjReader(t, bytes.NewReader(ciphertext), int64(len(ciphertext)), "", ""), ObjectOptions{
|
||||||
|
UserDefined: metadata,
|
||||||
|
})
|
||||||
|
mustIAM(t, err)
|
||||||
|
|
||||||
|
// Convert stored XLV2 layout on the single test disk into a real XLV1 xl.json + part.1 layout.
|
||||||
|
disk := f.obj.(*erasureServerPools).serverPools[0].sets[0].getDisks()[0]
|
||||||
|
xlDisk := disk.(*xlStorageDiskIDCheck).storage
|
||||||
|
xlDisk.Lock()
|
||||||
|
xlDisk.formatLegacy = true
|
||||||
|
xlDisk.Unlock()
|
||||||
|
t.Cleanup(func() {
|
||||||
|
xlDisk.Lock()
|
||||||
|
xlDisk.formatLegacy = false
|
||||||
|
xlDisk.Unlock()
|
||||||
|
})
|
||||||
|
|
||||||
|
fi, err := disk.ReadVersion(f.ctx, "", legacyBucket, object, "", ReadOptions{})
|
||||||
|
mustIAM(t, err)
|
||||||
|
|
||||||
|
partPath := pathJoin(object, fi.DataDir, "part.1")
|
||||||
|
rawPartBytes, err := disk.ReadAll(f.ctx, legacyBucket, partPath)
|
||||||
|
mustIAM(t, err)
|
||||||
|
|
||||||
|
mustIAM(t, disk.Delete(f.ctx, legacyBucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{}))
|
||||||
|
mustIAM(t, disk.Delete(f.ctx, legacyBucket, partPath, DeleteOptions{}))
|
||||||
|
mustIAM(t, disk.AppendFile(f.ctx, legacyBucket, pathJoin(object, "part.1"), rawPartBytes))
|
||||||
|
|
||||||
|
fi.Metadata["etag"] = sealedETag
|
||||||
|
v1Obj := xlMetaV1Object{
|
||||||
|
Version: xlMetaVersion101,
|
||||||
|
Format: xlMetaFormat,
|
||||||
|
Stat: StatInfo{
|
||||||
|
Size: fi.Size,
|
||||||
|
ModTime: fi.ModTime,
|
||||||
|
},
|
||||||
|
Erasure: fi.Erasure,
|
||||||
|
Meta: fi.Metadata,
|
||||||
|
Parts: fi.Parts,
|
||||||
|
}
|
||||||
|
v1Obj.Minio.Release = "RELEASE.2019-12-30T05-45-39Z"
|
||||||
|
v1Bytes, err := json.Marshal(v1Obj)
|
||||||
|
mustIAM(t, err)
|
||||||
|
mustIAM(t, disk.AppendFile(f.ctx, legacyBucket, pathJoin(object, xlStorageFormatFileV1), v1Bytes))
|
||||||
|
|
||||||
|
legacyOI, err := f.obj.GetObjectInfo(f.ctx, legacyBucket, object, ObjectOptions{})
|
||||||
|
mustIAM(t, err)
|
||||||
|
if !legacyOI.Legacy {
|
||||||
|
t.Fatal("expected test fixture object to report Legacy (XLV1) == true")
|
||||||
|
}
|
||||||
|
|
||||||
|
newCtxB64 := base64.StdEncoding.EncodeToString([]byte(`{"stage":"v2"}`))
|
||||||
|
rec, job := f.submit(t, globalActiveCred, fmt.Sprintf("keyrotate:\n apiVersion: v1\n bucket: %s\n prefix: legacy/\n encryption:\n type: sse-kms\n key: rotate-key\n context: %s\n", legacyBucket, newCtxB64))
|
||||||
|
if job == nil {
|
||||||
|
t.Fatalf("submit legacy keyrotate: %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
mustIAM(t, job.KeyRotate.Start(f.ctx, f.obj, *job))
|
||||||
|
progress := f.jobInfo(t, job)
|
||||||
|
if !progress.Complete || progress.Failed || progress.Objects != 1 {
|
||||||
|
t.Fatalf("legacy XLV1 rotation failed: %+v", progress)
|
||||||
|
}
|
||||||
|
|
||||||
|
after, err := f.obj.GetObjectInfo(f.ctx, legacyBucket, object, ObjectOptions{})
|
||||||
|
mustIAM(t, err)
|
||||||
|
if after.Legacy {
|
||||||
|
t.Fatal("expected rotated legacy object to be upgraded to XLV2 (Legacy == false)")
|
||||||
|
}
|
||||||
|
if after.UserTags != "tier=legacy" {
|
||||||
|
t.Fatalf("UserTags = %q, want tier=legacy", after.UserTags)
|
||||||
|
}
|
||||||
|
if got := readCopyChecksumObject(t, f.obj, legacyBucket, object, ObjectOptions{}); string(got) != payload {
|
||||||
|
t.Fatalf("decrypted content = %q, want %q", string(got), payload)
|
||||||
|
}
|
||||||
|
newObjKey, err := crypto.S3KMS.UnsealObjectKey(configured, after.UserDefined, legacyBucket, object)
|
||||||
|
mustIAM(t, err)
|
||||||
|
unsealedETag, err := DecryptETag(newObjKey, ObjectInfo{ETag: after.ETag})
|
||||||
|
mustIAM(t, err)
|
||||||
|
if wantETag := hex.EncodeToString(plainMD5[:]); unsealedETag != wantETag {
|
||||||
|
t.Fatalf("unsealed ETag = %q, want %q", unsealedETag, wantETag)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func createTestPolicyUser(t *testing.T, policyJSON string) auth.Credentials {
|
||||||
|
t.Helper()
|
||||||
|
ctx := t.Context()
|
||||||
|
ak, sk, err := auth.GenerateCredentials()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
creds := auth.Credentials{AccessKey: ak, SecretKey: sk}
|
||||||
|
if _, err = globalIAMSys.CreateUser(ctx, creds.AccessKey, madmin.AddOrUpdateUserReq{
|
||||||
|
SecretKey: creds.SecretKey,
|
||||||
|
Status: madmin.AccountEnabled,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
parsed, err := policy.ParseConfig(strings.NewReader(policyJSON))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse policy: %v\n%s", err, policyJSON)
|
||||||
|
}
|
||||||
|
pName := "audit-policy-" + mustGetUUID()
|
||||||
|
if _, err = globalIAMSys.SetPolicy(ctx, pName, *parsed); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err = globalIAMSys.PolicyDBSet(ctx, creds.AccessKey, pName, regUser, false); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return creds
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestObjectTagAndVersionAuthSymmetry(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, adminCred auth.Credentials, t *testing.T) {
|
||||||
|
if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
|
||||||
|
t.Fatalf("%s: enable versioning: %v", backend, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create v1 (env=prod) and v2 (env=dev) of src-obj.
|
||||||
|
v1, err := obj.PutObject(t.Context(), bucket, "src-obj", mustGetPutObjReader(t, strings.NewReader("prod-secret"), int64(len("prod-secret")), "", ""), ObjectOptions{
|
||||||
|
Versioned: true,
|
||||||
|
UserDefined: map[string]string{xhttp.AmzObjectTagging: "env=prod"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
v2, err := obj.PutObject(t.Context(), bucket, "src-obj", mustGetPutObjReader(t, strings.NewReader("dev-public"), int64(len("dev-public")), "", ""), ObjectOptions{
|
||||||
|
Versioned: true,
|
||||||
|
UserDefined: map[string]string{xhttp.AmzObjectTagging: "env=dev"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also create a ZIP object with env=prod tag to verify ZIP sub-handlers enforce ExistingObjectTag.
|
||||||
|
var zipBuf bytes.Buffer
|
||||||
|
zw := zip.NewWriter(&zipBuf)
|
||||||
|
zf, err := zw.Create("inside.txt")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, _ = zf.Write([]byte("zip-inner"))
|
||||||
|
if err := zw.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, err = obj.PutObject(t.Context(), bucket, "archive.zip", mustGetPutObjReader(t, bytes.NewReader(zipBuf.Bytes()), int64(zipBuf.Len()), "", ""), ObjectOptions{
|
||||||
|
Versioned: true,
|
||||||
|
UserDefined: map[string]string{xhttp.AmzObjectTagging: "env=prod"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// User A:
|
||||||
|
// - Can read/tag ONLY objects/versions tagged env=dev.
|
||||||
|
// - Can write dst/* ONLY when RequestObjectTag/env == dev (plus unconditional PutObject on prod-readable copy test).
|
||||||
|
userAPolicy := fmt.Sprintf(`{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": [
|
||||||
|
"s3:GetObject",
|
||||||
|
"s3:GetObjectVersion",
|
||||||
|
"s3:GetObjectTagging",
|
||||||
|
"s3:GetObjectVersionTagging",
|
||||||
|
"s3:PutObjectTagging",
|
||||||
|
"s3:PutObjectVersionTagging",
|
||||||
|
"s3:DeleteObjectTagging",
|
||||||
|
"s3:DeleteObjectVersionTagging"
|
||||||
|
],
|
||||||
|
"Resource": ["arn:aws:s3:::%s/*"],
|
||||||
|
"Condition": {"StringEquals": {"s3:ExistingObjectTag/env": ["dev"]}}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": ["s3:ListBucket", "s3:ListBucketVersions", "s3:ListBucketMultipartUploads"],
|
||||||
|
"Resource": ["arn:aws:s3:::%s"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": ["s3:PutObject"],
|
||||||
|
"Resource": ["arn:aws:s3:::%s/dst-*"],
|
||||||
|
"Condition": {"StringEquals": {"s3:RequestObjectTag/env": ["dev"]}}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": ["s3:PutObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts"],
|
||||||
|
"Resource": ["arn:aws:s3:::%s/mp-*"]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}`, bucket, bucket, bucket, bucket)
|
||||||
|
userA := createTestPolicyUser(t, userAPolicy)
|
||||||
|
|
||||||
|
// 1. CopyObject from v1 (env=prod) must fail 403 due to source ExistingObjectTag.
|
||||||
|
cpDenied, err := newTestSignedRequestV4(http.MethodPut, "/"+bucket+"/dst-denied", 0, nil, userA.AccessKey, userA.SecretKey, map[string]string{
|
||||||
|
xhttp.AmzCopySource: "/" + bucket + "/src-obj?versionId=" + v1.VersionID,
|
||||||
|
xhttp.AmzTagDirective: "REPLACE",
|
||||||
|
xhttp.AmzObjectTagging: "env=dev",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, cpDenied)
|
||||||
|
if rec.Code != http.StatusForbidden {
|
||||||
|
t.Fatalf("%s: CopyObject from env=prod version status=%d, want 403", backend, rec.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. CopyObject from v2 (env=dev) with COPY directive succeeds (RequestObjectTag/env == dev from copied tags).
|
||||||
|
cpOK, err := newTestSignedRequestV4(http.MethodPut, "/"+bucket+"/dst-ok", 0, nil, userA.AccessKey, userA.SecretKey, map[string]string{
|
||||||
|
xhttp.AmzCopySource: "/" + bucket + "/src-obj?versionId=" + v2.VersionID,
|
||||||
|
xhttp.AmzTagDirective: "COPY",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec = httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, cpOK)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: CopyObject from env=dev with COPY directive status=%d body=%s, want 200", backend, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. RequestObjectTag spoofing defense on COPY directive:
|
||||||
|
// User B can read all versions of src-obj, but can PutObject to dst-* ONLY when RequestObjectTag/env == dev.
|
||||||
|
// Copying v1 (env=prod) with Tagging-Directive: COPY and a spoofed X-Amz-Tagging: env=dev header MUST fail 403!
|
||||||
|
userBPolicy := fmt.Sprintf(`{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": ["s3:GetObject", "s3:GetObjectVersion", "s3:GetObjectTagging", "s3:GetObjectVersionTagging"],
|
||||||
|
"Resource": ["arn:aws:s3:::%s/src-obj"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": ["s3:PutObject", "s3:PutObjectTagging"],
|
||||||
|
"Resource": ["arn:aws:s3:::%s/dst-*"],
|
||||||
|
"Condition": {"StringEquals": {"s3:RequestObjectTag/env": ["dev"]}}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": ["s3:GetObject"],
|
||||||
|
"Resource": ["arn:aws:s3:::%s/src-obj"]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}`, bucket, bucket, bucket)
|
||||||
|
userB := createTestPolicyUser(t, userBPolicy)
|
||||||
|
|
||||||
|
cpSpoof, err := newTestSignedRequestV4(http.MethodPut, "/"+bucket+"/dst-spoof", 0, nil, userB.AccessKey, userB.SecretKey, map[string]string{
|
||||||
|
xhttp.AmzCopySource: "/" + bucket + "/src-obj?versionId=" + v1.VersionID,
|
||||||
|
xhttp.AmzTagDirective: "COPY",
|
||||||
|
xhttp.AmzObjectTagging: "env=dev",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec = httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, cpSpoof)
|
||||||
|
if rec.Code != http.StatusForbidden {
|
||||||
|
t.Fatalf("%s: CopyObject accepted spoofed X-Amz-Tagging under COPY directive: status=%d body=%s", backend, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. UploadPartCopy enforces ExistingObjectTag on versioned source!
|
||||||
|
initMP, err := newTestSignedRequestV4(http.MethodPost, "/"+bucket+"/mp-target?uploads", 0, nil, userA.AccessKey, userA.SecretKey, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec = httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, initMP)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: CreateMultipartUpload status=%d body=%s", backend, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
var initResp InitiateMultipartUploadResponse
|
||||||
|
if err := xml.Unmarshal(rec.Body.Bytes(), &initResp); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
upcDenied, err := newTestSignedRequestV4(http.MethodPut, fmt.Sprintf("/%s/mp-target?partNumber=1&uploadId=%s", bucket, url.QueryEscape(initResp.UploadID)), 0, nil, userA.AccessKey, userA.SecretKey, map[string]string{
|
||||||
|
xhttp.AmzCopySource: "/" + bucket + "/src-obj?versionId=" + v1.VersionID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec = httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, upcDenied)
|
||||||
|
if rec.Code != http.StatusForbidden {
|
||||||
|
t.Fatalf("%s: UploadPartCopy from env=prod status=%d, want 403", backend, rec.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
upcOK, err := newTestSignedRequestV4(http.MethodPut, fmt.Sprintf("/%s/mp-target?partNumber=1&uploadId=%s", bucket, url.QueryEscape(initResp.UploadID)), 0, nil, userA.AccessKey, userA.SecretKey, map[string]string{
|
||||||
|
xhttp.AmzCopySource: "/" + bucket + "/src-obj?versionId=" + v2.VersionID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec = httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, upcOK)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: UploadPartCopy from env=dev status=%d body=%s, want 200", backend, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tag counts require the same permission as values, even without ACCESS.
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, extra string
|
||||||
|
currentTags, versionTags bool
|
||||||
|
}{
|
||||||
|
{"implicit-deny", "", false, false},
|
||||||
|
{"allow", `,{"Effect":"Allow","Action":["s3:GetObjectTagging","s3:GetObjectVersionTagging"],"Resource":"arn:aws:s3:::*"}`, true, true},
|
||||||
|
{"explicit-deny", `,{"Effect":"Allow","Action":"s3:*","Resource":"arn:aws:s3:::*"},{"Effect":"Deny","Action":["s3:GetObjectTagging","s3:GetObjectVersionTagging"],"Resource":"arn:aws:s3:::*"}`, false, false},
|
||||||
|
{"current-only", `,{"Effect":"Allow","Action":"s3:GetObjectTagging","Resource":"arn:aws:s3:::*"},{"Effect":"Deny","Action":"s3:GetObjectVersionTagging","Resource":"arn:aws:s3:::*"}`, true, false},
|
||||||
|
{"replicate-tags", `,{"Effect":"Allow","Action":"s3:ReplicateTags","Resource":"arn:aws:s3:::*"}`, true, true},
|
||||||
|
} {
|
||||||
|
creds := createTestPolicyUser(t, fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":"arn:aws:s3:::%s/*"}%s]}`, bucket, tc.extra))
|
||||||
|
for _, method := range []string{http.MethodGet, http.MethodHead} {
|
||||||
|
for _, access := range []bool{false, true} {
|
||||||
|
for _, version := range []string{"", v1.VersionID} {
|
||||||
|
t.Run(fmt.Sprintf("%s/headers/%s/%s/access=%t/version=%t", backend, tc.name, method, access, version != ""), func(t *testing.T) {
|
||||||
|
target := "/" + bucket + "/src-obj"
|
||||||
|
wantTags, allowed := "env=dev", tc.currentTags
|
||||||
|
if version != "" {
|
||||||
|
target += "?versionId=" + version
|
||||||
|
wantTags, allowed = "env=prod", tc.versionTags
|
||||||
|
}
|
||||||
|
headers := map[string]string{}
|
||||||
|
if access {
|
||||||
|
headers[xhttp.AmzTagDirective] = "ACCESS"
|
||||||
|
}
|
||||||
|
req, err := newTestSignedRequestV4(method, target, 0, nil, creds.AccessKey, creds.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status=%d: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
// Normalize literal handler keys as net/http does on the wire.
|
||||||
|
responseHeaders := http.Header{}
|
||||||
|
for key, values := range rec.Header() {
|
||||||
|
for _, value := range values {
|
||||||
|
responseHeaders.Add(key, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
wantCount := ""
|
||||||
|
if allowed {
|
||||||
|
wantCount = "1"
|
||||||
|
}
|
||||||
|
if !allowed || !access {
|
||||||
|
wantTags = ""
|
||||||
|
}
|
||||||
|
if got := responseHeaders.Get(xhttp.AmzTagCount); got != wantCount {
|
||||||
|
t.Errorf("tag count=%q, want %q", got, wantCount)
|
||||||
|
}
|
||||||
|
if got := responseHeaders.Get(xhttp.AmzObjectTagging); got != wantTags {
|
||||||
|
t.Errorf("tag values=%q, want %q", got, wantTags)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 6. ListObjectVersions with metadata=true filters <UserTags> per version's ExistingObjectTag!
|
||||||
|
listVerReq, err := newTestSignedRequestV4(http.MethodGet, "/"+bucket+"?versions=&metadata=true&prefix=src-obj", 0, nil, userA.AccessKey, userA.SecretKey, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec = httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, listVerReq)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: ListObjectVersions status=%d body=%s", backend, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
bodyStr := rec.Body.String()
|
||||||
|
if strings.Contains(bodyStr, "env=prod") {
|
||||||
|
t.Fatalf("%s: ListObjectVersions metadata=true leaked unauthorized version tags env=prod: %s", backend, bodyStr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(bodyStr, "env=dev") {
|
||||||
|
t.Fatalf("%s: ListObjectVersions metadata=true omitted authorized version tags env=dev: %s", backend, bodyStr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 7. GetObjectTagging & DeleteObjectTagging with ?versionId= enforce ExistingObjectTag on that version!
|
||||||
|
for _, method := range []string{http.MethodGet, http.MethodDelete} {
|
||||||
|
tagDenied, err := newTestSignedRequestV4(method, "/"+bucket+"/src-obj?tagging=&versionId="+v1.VersionID, 0, nil, userA.AccessKey, userA.SecretKey, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec = httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, tagDenied)
|
||||||
|
if rec.Code != http.StatusForbidden {
|
||||||
|
t.Fatalf("%s: %s tagging on v1 (env=prod) status=%d, want 403", backend, method, rec.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
getTagOK, err := newTestSignedRequestV4(http.MethodGet, "/"+bucket+"/src-obj?tagging=&versionId="+v2.VersionID, 0, nil, userA.AccessKey, userA.SecretKey, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec = httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, getTagOK)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: GET tagging on v2 (env=dev) status=%d, want 200", backend, rec.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 8. Archive x-minio-extract sub-handlers (HEAD/GET/ListObjectsV2) reject access when archive ExistingObjectTag mismatches!
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, method, target string
|
||||||
|
}{
|
||||||
|
{"zip-get", http.MethodGet, "/" + bucket + "/archive.zip/inside.txt"},
|
||||||
|
{"zip-head", http.MethodHead, "/" + bucket + "/archive.zip/inside.txt"},
|
||||||
|
{"zip-list", http.MethodGet, "/" + bucket + "?list-type=2&prefix=archive.zip/"},
|
||||||
|
} {
|
||||||
|
zreq, err := newTestSignedRequestV4(tc.method, tc.target, 0, nil, userA.AccessKey, userA.SecretKey, map[string]string{
|
||||||
|
"x-minio-extract": "true",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec = httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, zreq)
|
||||||
|
if rec.Code != http.StatusForbidden {
|
||||||
|
t.Fatalf("%s: %s on env=prod archive status=%d, want 403", backend, tc.name, rec.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up multipart upload with admin credentials so DetectTestLeak sees no background state.
|
||||||
|
abortReq, err := newTestSignedRequestV4(http.MethodDelete, fmt.Sprintf("/%s/mp-target?uploadId=%s", bucket, url.QueryEscape(initResp.UploadID)), 0, nil, adminCred.AccessKey, adminCred.SecretKey, nil)
|
||||||
|
if err == nil {
|
||||||
|
router.ServeHTTP(httptest.NewRecorder(), abortReq)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,205 @@
|
|||||||
|
// Copyright (c) 2026 Feng Ruohang
|
||||||
|
//
|
||||||
|
// This file is part of Silo Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/crypto"
|
||||||
|
"github.com/minio/minio/internal/hash/sha256"
|
||||||
|
xioutil "github.com/minio/minio/internal/ioutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
type replicaContentGuardFn func(context.Context, string, string, ObjectInfo, func() (*GetObjectReader, error)) error
|
||||||
|
|
||||||
|
// verifyLockedReplica compares the actual data, never a sender's ETag or
|
||||||
|
// checksum assertion. A peer may retransmit a protected version to update its
|
||||||
|
// metadata, but may not replace its content. Call under the object write lock.
|
||||||
|
func verifyLockedReplica(ctx context.Context, incoming, stored ObjectInfo, openIncoming, openStored func() (*GetObjectReader, error)) error {
|
||||||
|
if !enforceRetentionForDeletion(ctx, stored) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
locked := ObjectLocked{Bucket: stored.Bucket, Object: stored.Name, VersionID: stored.VersionID}
|
||||||
|
if storedObjectLockState(stored.UserDefined).complianceWouldWeaken(storedObjectLockState(incoming.UserDefined)) {
|
||||||
|
return locked
|
||||||
|
}
|
||||||
|
inSSEC, oldSSEC := crypto.SSEC.IsEncrypted(incoming.UserDefined), crypto.SSEC.IsEncrypted(stored.UserDefined)
|
||||||
|
if inSSEC != oldSSEC {
|
||||||
|
return locked
|
||||||
|
}
|
||||||
|
if inSSEC {
|
||||||
|
inSize, inErr := incoming.GetActualSize()
|
||||||
|
oldSize, oldErr := stored.GetActualSize()
|
||||||
|
if inErr != nil || oldErr != nil || inSize != oldSize {
|
||||||
|
return locked
|
||||||
|
}
|
||||||
|
// Without the customer key, compare ciphertext and its interpretation.
|
||||||
|
// Equal ciphertext with a changed seal or multipart framing is not the
|
||||||
|
// same object, even if the supplied ETag and size are unchanged.
|
||||||
|
for _, key := range []string{
|
||||||
|
crypto.MetaIV, crypto.MetaAlgorithm, crypto.MetaSealedKeySSEC,
|
||||||
|
ReservedMetadataPrefix + "compression",
|
||||||
|
} {
|
||||||
|
if incoming.UserDefined[key] != stored.UserDefined[key] {
|
||||||
|
return locked
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if crypto.IsMultiPart(incoming.UserDefined) != crypto.IsMultiPart(stored.UserDefined) || incoming.isMultipart() != stored.isMultipart() {
|
||||||
|
return locked
|
||||||
|
}
|
||||||
|
if crypto.IsMultiPart(incoming.UserDefined) {
|
||||||
|
if len(incoming.Parts) != len(stored.Parts) {
|
||||||
|
return locked
|
||||||
|
}
|
||||||
|
for i, p := range incoming.Parts {
|
||||||
|
old := stored.Parts[i]
|
||||||
|
if p.Number != old.Number || p.Size != old.Size {
|
||||||
|
return locked
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
in, err := openIncoming()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer in.Close()
|
||||||
|
old, err := openStored()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer old.Close()
|
||||||
|
ih, oh := sha256.New(), sha256.New()
|
||||||
|
inSize, err := io.Copy(ih, in)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
oldSize, err := io.Copy(oh, old)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if inSize != oldSize || !bytes.Equal(ih.Sum(nil), oh.Sum(nil)) {
|
||||||
|
return locked
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (er erasureObjects) guardReplicaContent(ctx context.Context, bucket, object string, incoming ObjectInfo, openIncoming func() (*GetObjectReader, error)) error {
|
||||||
|
opts := ObjectOptions{VersionID: incoming.VersionID, NoLock: true, NoAuditLog: true, ReplicationRequest: true}
|
||||||
|
if opts.VersionID == "" {
|
||||||
|
opts.VersionID = nullVersionID
|
||||||
|
}
|
||||||
|
stored, err := er.getObjectInfo(ctx, bucket, object, opts)
|
||||||
|
if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isErrMethodNotAllowed(err) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return verifyLockedReplica(ctx, incoming, stored, openIncoming, func() (*GetObjectReader, error) {
|
||||||
|
return er.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, opts)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (z *erasureServerPools) guardReplicaContent(ctx context.Context, bucket, object string, incoming ObjectInfo, openIncoming func() (*GetObjectReader, error)) error {
|
||||||
|
opts := ObjectOptions{VersionID: incoming.VersionID, NoLock: true, NoAuditLog: true, ReplicationRequest: true}
|
||||||
|
if opts.VersionID == "" {
|
||||||
|
opts.VersionID = nullVersionID
|
||||||
|
}
|
||||||
|
copies, err := z.objectPoolInfos(ctx, bucket, object, opts)
|
||||||
|
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, copy := range copies {
|
||||||
|
if err := verifyLockedReplica(ctx, incoming, copy.ObjInfo, openIncoming, func() (*GetObjectReader, error) {
|
||||||
|
return z.serverPools[copy.Index].GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, opts)
|
||||||
|
}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkReplicaContent reads the staged version before rename commits it. Its
|
||||||
|
// data is decoded/decrypted just like a GET (raw ciphertext for SSE-C), so this
|
||||||
|
// covers multipart, compression and destination SSE-S3/KMS re-encryption.
|
||||||
|
// disks and metadata are in erasure shard order, as passed to renameData.
|
||||||
|
func (er erasureObjects) checkReplicaContent(ctx context.Context, bucket, object, stageBucket, stageObject string, metadata []FileInfo, disks []StorageAPI, opts ObjectOptions) error {
|
||||||
|
if (!opts.ReplicaLockReconcile && !opts.ReplicationRequest) || opts.DataMovement {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var fi FileInfo
|
||||||
|
for i, disk := range disks {
|
||||||
|
if disk != nil {
|
||||||
|
fi = metadata[i]
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
incoming := fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
|
||||||
|
guard := opts.replicaContentGuard
|
||||||
|
if guard == nil {
|
||||||
|
guard = er.guardReplicaContent
|
||||||
|
}
|
||||||
|
return guard(ctx, bucket, object, incoming, func() (*GetObjectReader, error) {
|
||||||
|
// A marker-only peer write does not reconcile the stored protection.
|
||||||
|
// It therefore cannot replace a protected version, even with equal data.
|
||||||
|
if !opts.ReplicaLockReconcile {
|
||||||
|
return nil, ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: incoming.VersionID}
|
||||||
|
}
|
||||||
|
if incoming.IsCompressed() {
|
||||||
|
// A replica may assert the source's logical size. A compressed GET
|
||||||
|
// limits its output to that size, so verify it against the lengths
|
||||||
|
// recorded while consuming the uploaded parts before comparing GETs.
|
||||||
|
// Otherwise a forged zero size could hide a non-empty replacement.
|
||||||
|
var actual int64
|
||||||
|
for _, part := range incoming.Parts {
|
||||||
|
actual += part.ActualSize
|
||||||
|
}
|
||||||
|
declared, err := incoming.GetActualSize()
|
||||||
|
if err != nil || declared != actual {
|
||||||
|
return nil, ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: incoming.VersionID}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
readOpts := ObjectOptions{NoDecryption: crypto.SSEC.IsEncrypted(incoming.UserDefined)}
|
||||||
|
fn, off, length, err := NewGetObjectReader(nil, incoming, readOpts, http.Header{})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// The ordinary reader expects original drive order, not shard order.
|
||||||
|
readDisks, readMeta := make([]StorageAPI, len(disks)), make([]FileInfo, len(metadata))
|
||||||
|
for i, shard := range fi.Erasure.Distribution {
|
||||||
|
readDisks[i], readMeta[i] = disks[shard-1], metadata[shard-1]
|
||||||
|
readMeta[i].Erasure.Index = shard
|
||||||
|
}
|
||||||
|
pr, pw := xioutil.WaitPipe()
|
||||||
|
go func() {
|
||||||
|
pw.CloseWithError(er.getObjectWithFileInfo(ctx, stageBucket, stageObject, off, length, pw, fi, readMeta, readDisks))
|
||||||
|
}()
|
||||||
|
reader, err := fn(pr, http.Header{}, func() { pr.CloseWithError(nil) })
|
||||||
|
if err != nil {
|
||||||
|
pr.CloseWithError(err)
|
||||||
|
}
|
||||||
|
return reader, err
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,314 @@
|
|||||||
|
// Copyright (c) 2026 Feng Ruohang
|
||||||
|
//
|
||||||
|
// This file is part of Silo Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/xml"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"maps"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
"github.com/minio/minio/internal/crypto"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
"github.com/minio/minio/internal/kms"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAPIReplicaPreservesProtectedContent(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t, makeBucketOptions: MakeBucketOptions{LockEnabled: true},
|
||||||
|
objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
|
||||||
|
oldTLS, oldKMS := globalIsTLS, GlobalKMS
|
||||||
|
globalIsTLS, GlobalKMS = true, kms.NewStub("replica-content")
|
||||||
|
defer func() { globalIsTLS, GlobalKMS = oldTLS, oldKMS }()
|
||||||
|
restoreCompression := disableCompression()
|
||||||
|
defer restoreCompression()
|
||||||
|
request := func(method, url string, data []byte, headers map[string]string) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
|
req, err := newTestSignedRequestV4(method, url, int64(len(data)), bytes.NewReader(data), cred.AccessKey, cred.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, req)
|
||||||
|
return rec
|
||||||
|
}
|
||||||
|
for _, encryption := range []string{"plain", "sse-s3", "sse-kms", "sse-c", "plain-compressed", "sse-s3-compressed"} {
|
||||||
|
restore := func() {}
|
||||||
|
if strings.HasSuffix(encryption, "-compressed") {
|
||||||
|
restore = setCopyChecksumCompression(true)
|
||||||
|
}
|
||||||
|
for _, multipart := range []bool{false, true} {
|
||||||
|
for _, protection := range []string{"COMPLIANCE", "LEGALHOLD", "GOVERNANCE"} {
|
||||||
|
key := fmt.Sprintf("protected/%s/%v/%s.txt", encryption, multipart, protection)
|
||||||
|
enc := map[string]string{}
|
||||||
|
switch encryption {
|
||||||
|
case "sse-s3", "sse-s3-compressed":
|
||||||
|
enc[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
|
||||||
|
case "sse-kms":
|
||||||
|
enc[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
|
||||||
|
enc[xhttp.AmzServerSideEncryptionKmsID] = "replica-content"
|
||||||
|
case "sse-c":
|
||||||
|
enc = ssecKeyHeaders(bytes.Repeat([]byte{0x37}, 32), false)
|
||||||
|
}
|
||||||
|
headers := maps.Clone(enc)
|
||||||
|
maps.Copy(headers, batchLockMeta(protection))
|
||||||
|
// Both inline and disk-backed writes are exercised.
|
||||||
|
body := bytes.Repeat([]byte("immutable content "), 20000)
|
||||||
|
switch protection {
|
||||||
|
case "LEGALHOLD":
|
||||||
|
body = body[:64]
|
||||||
|
case "GOVERNANCE":
|
||||||
|
body = nil
|
||||||
|
}
|
||||||
|
if !multipart {
|
||||||
|
putCopyChecksumSource(t, router, cred, bucket, key, body, headers)
|
||||||
|
} else {
|
||||||
|
initRec := request(http.MethodPost, getNewMultipartURL("", bucket, key), nil, headers)
|
||||||
|
if initRec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("source initiate: %d %s", initRec.Code, initRec.Body.String())
|
||||||
|
}
|
||||||
|
var init InitiateMultipartUploadResponse
|
||||||
|
if err := xml.Unmarshal(initRec.Body.Bytes(), &init); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
part := request(http.MethodPut, getPutObjectPartURL("", bucket, key, init.UploadID, "1"), body, enc)
|
||||||
|
if part.Code != http.StatusOK {
|
||||||
|
t.Fatalf("source part: %d %s", part.Code, part.Body.String())
|
||||||
|
}
|
||||||
|
complete, err := xml.Marshal(CompleteMultipartUpload{Parts: []CompletePart{{PartNumber: 1, ETag: canonicalizeETag(part.Header()[xhttp.ETag][0])}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := request(http.MethodPost, getCompleteMultipartUploadURL("", bucket, key, init.UploadID), complete, enc)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("source complete: %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
gr, err := obj.GetObjectNInfo(t.Context(), bucket, key, nil, http.Header{}, ObjectOptions{ReplicationRequest: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
src := gr.ObjInfo
|
||||||
|
wire, err := io.ReadAll(gr)
|
||||||
|
gr.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, altered := range []bool{true, false} {
|
||||||
|
data := bytes.Clone(wire)
|
||||||
|
if altered {
|
||||||
|
if len(data) == 0 {
|
||||||
|
data = []byte("x")
|
||||||
|
} else {
|
||||||
|
data[len(data)-1] ^= 0x20
|
||||||
|
}
|
||||||
|
}
|
||||||
|
opts, _, err := putReplicationOpts(t.Context(), "", src)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hdrs := map[string]string{}
|
||||||
|
for k, vs := range opts.Header() {
|
||||||
|
if len(vs) > 0 {
|
||||||
|
hdrs[k] = vs[0]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
hdrs[xhttp.MinIOSourceReplicationRequest] = "true"
|
||||||
|
hdrs[xhttp.AmzBucketReplicationStatus] = "REPLICA"
|
||||||
|
// Forge the old ETag and size along with a newer tag revision:
|
||||||
|
// the body comparison must still reject different bytes.
|
||||||
|
hdrs[xhttp.MinIOSourceETag] = src.ETag
|
||||||
|
hdrs[xhttp.MinIOSourceTaggingTimestamp] = UTCNow().Add(time.Minute).Format(time.RFC3339Nano)
|
||||||
|
hdrs[xhttp.AmzObjectTagging] = "replica=updated"
|
||||||
|
var rec *httptest.ResponseRecorder
|
||||||
|
if !multipart {
|
||||||
|
rec = request(http.MethodPut, getPutObjectURL("", bucket, key)+"?versionId="+src.VersionID, data, hdrs)
|
||||||
|
} else {
|
||||||
|
initRec := request(http.MethodPost, getNewMultipartURL("", bucket, key)+"&versionId="+src.VersionID, nil, hdrs)
|
||||||
|
if initRec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s %s initiate: %d %s", backend, key, initRec.Code, initRec.Body.String())
|
||||||
|
}
|
||||||
|
var init InitiateMultipartUploadResponse
|
||||||
|
if err := xml.Unmarshal(initRec.Body.Bytes(), &init); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
partHdrs := map[string]string{xhttp.MinIOSourceReplicationRequest: "true"}
|
||||||
|
partRec := request(http.MethodPut, getPutObjectPartURL("", bucket, key, init.UploadID, "1"), data, partHdrs)
|
||||||
|
if partRec.Code != http.StatusOK && (!altered || !strings.Contains(partRec.Body.String(), "XMinioObjectTampered")) {
|
||||||
|
t.Fatalf("%s %s part: %d %s", backend, key, partRec.Code, partRec.Body.String())
|
||||||
|
}
|
||||||
|
if partRec.Code != http.StatusOK {
|
||||||
|
rec = partRec
|
||||||
|
} else {
|
||||||
|
complete, err := xml.Marshal(CompleteMultipartUpload{Parts: []CompletePart{{PartNumber: 1, ETag: canonicalizeETag(partRec.Header()[xhttp.ETag][0])}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
actual, err := src.GetActualSize()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
partHdrs[xhttp.MinIOReplicationActualObjectSize] = strconv.FormatInt(actual, 10)
|
||||||
|
rec = request(http.MethodPost, getCompleteMultipartUploadURL("", bucket, key, init.UploadID), complete, partHdrs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if altered {
|
||||||
|
if rec.Code < 400 || !strings.Contains(rec.Body.String(), "WORM") && !strings.Contains(rec.Body.String(), "XMinioObjectTampered") {
|
||||||
|
t.Fatalf("%s %s: different content not refused: %d %s", backend, key, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
} else if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s %s: same content refused: %d %s", backend, key, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
getHdrs := map[string]string{}
|
||||||
|
if encryption == "sse-c" {
|
||||||
|
getHdrs = enc
|
||||||
|
}
|
||||||
|
got := request(http.MethodGet, getGetObjectURL("", bucket, key)+"?versionId="+src.VersionID, nil, getHdrs)
|
||||||
|
if got.Code != http.StatusOK || !bytes.Equal(got.Body.Bytes(), body) {
|
||||||
|
t.Fatalf("%s %s: protected content changed or became unreadable: %d", backend, key, got.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
restore()
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPoolsReplicaProtectedContentAcrossPools(t *testing.T) {
|
||||||
|
z, bucket := consistencyPools(t)
|
||||||
|
const key = "protected-across-pools"
|
||||||
|
id := mustGetUUID()
|
||||||
|
stamp := UTCNow().Add(-time.Hour)
|
||||||
|
putConsistencyObject(t, z, bucket, key, 0, "original", ObjectOptions{Versioned: true, VersionID: id, MTime: stamp})
|
||||||
|
putConsistencyObject(t, z, bucket, key, 1, "original", ObjectOptions{Versioned: true, VersionID: id, MTime: stamp, UserDefined: batchLockMeta("COMPLIANCE")})
|
||||||
|
_, err := z.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, strings.NewReader("tampered"), 8, "", ""), ObjectOptions{Versioned: true, VersionID: id, ReplicationRequest: true, ReplicaLockReconcile: true, UserDefined: map[string]string{}})
|
||||||
|
if _, ok := err.(ObjectLocked); !ok {
|
||||||
|
t.Fatalf("protected pool copy allowed overwrite: %v", err)
|
||||||
|
}
|
||||||
|
for i, pool := range z.serverPools {
|
||||||
|
gr, err := pool.GetObjectNInfo(t.Context(), bucket, key, nil, http.Header{}, ObjectOptions{VersionID: id})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(gr)
|
||||||
|
gr.Close()
|
||||||
|
if err != nil || string(data) != "original" {
|
||||||
|
t.Fatalf("pool %d: changed protected data: %q %v", i, data, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLockedSSECReplicaRejectsChangedSeal(t *testing.T) {
|
||||||
|
stored := ObjectInfo{Bucket: "bucket", Name: "key", Size: 0, UserDefined: batchLockMeta("LEGALHOLD")}
|
||||||
|
stored.UserDefined[crypto.MetaSealedKeySSEC] = "stored-seal"
|
||||||
|
incoming := stored
|
||||||
|
incoming.UserDefined = maps.Clone(stored.UserDefined)
|
||||||
|
incoming.UserDefined[crypto.MetaSealedKeySSEC] = "changed-seal"
|
||||||
|
open := func() (*GetObjectReader, error) {
|
||||||
|
t.Fatal("changed seal must be rejected before reading")
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err := verifyLockedReplica(t.Context(), incoming, stored, open, open); err == nil {
|
||||||
|
t.Fatal("changed SSE-C seal accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPIReplicaCopyPreservesProtectedVersion(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t, makeBucketOptions: MakeBucketOptions{LockEnabled: true},
|
||||||
|
objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
|
||||||
|
const key = "copy-protected-version"
|
||||||
|
original := batchPutVersion(t, obj, bucket, key, batchLockMeta("COMPLIANCE"))
|
||||||
|
data := []byte("different source version")
|
||||||
|
source, err := obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{Versioned: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
headers := map[string]string{
|
||||||
|
xhttp.AmzCopySource: url.QueryEscape("/"+bucket+"/"+key) + "?versionId=" + source.VersionID,
|
||||||
|
xhttp.MinIOSourceReplicationRequest: "true", xhttp.AmzBucketReplicationStatus: "REPLICA",
|
||||||
|
xhttp.AmzMetadataDirective: replaceDirective,
|
||||||
|
}
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut, getCopyObjectURL("", bucket, key)+"?versionId="+original.VersionID, 0, nil, cred.AccessKey, cred.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, req)
|
||||||
|
if rec.Code < 400 || !strings.Contains(rec.Body.String(), "WORM") {
|
||||||
|
t.Fatalf("%s: copy replaced protected version: %d %s", backend, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
gr, err := obj.GetObjectNInfo(t.Context(), bucket, key, nil, http.Header{}, ObjectOptions{VersionID: original.VersionID})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := io.ReadAll(gr)
|
||||||
|
gr.Close()
|
||||||
|
if err != nil || string(got) != "batch lock "+key {
|
||||||
|
t.Fatalf("%s: protected body changed: %q %v", backend, got, err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPIReplicaCannotWeakenCompliance(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t, makeBucketOptions: MakeBucketOptions{LockEnabled: true},
|
||||||
|
objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
|
||||||
|
const key = "compliance-removal"
|
||||||
|
oi := batchPutVersion(t, obj, bucket, key, batchLockMeta("COMPLIANCE"))
|
||||||
|
until := oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)]
|
||||||
|
for _, copyRequest := range []bool{false, true} {
|
||||||
|
headers := map[string]string{xhttp.MinIOSourceReplicationRequest: "true", xhttp.AmzBucketReplicationStatus: "REPLICA", xhttp.MinIOSourceObjectRetentionTimestamp: UTCNow().Add(time.Hour).Format(time.RFC3339Nano)}
|
||||||
|
body := []byte("batch lock " + key)
|
||||||
|
if copyRequest {
|
||||||
|
headers[xhttp.AmzCopySource] = url.QueryEscape("/"+bucket+"/"+key) + "?versionId=" + oi.VersionID
|
||||||
|
headers[xhttp.AmzMetadataDirective] = replaceDirective
|
||||||
|
body = nil
|
||||||
|
}
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucket, key)+"?versionId="+oi.VersionID, int64(len(body)), bytes.NewReader(body), cred.AccessKey, cred.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s copy=%v: %d %s", backend, copyRequest, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
stored, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: oi.VersionID})
|
||||||
|
if err != nil || stored.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] != "COMPLIANCE" || stored.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] != until {
|
||||||
|
t.Fatalf("%s copy=%v: a replica cleared COMPLIANCE: %+v %v", backend, copyRequest, stored.UserDefined, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
// Copyright (c) 2026 Feng Ruohang
|
||||||
|
//
|
||||||
|
// This file is part of Silo Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestReplicationControlsRequireSignedHeaders(t *testing.T) {
|
||||||
|
for _, header := range replicationRequestHeaders {
|
||||||
|
t.Run(header, func(t *testing.T) {
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodGet, "http://localhost/bucket/key", 0, nil, "access-key", "secret-key", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Membership matters even for a present-empty internal header.
|
||||||
|
req.Header.Set(header, "")
|
||||||
|
if replicationHeadersSigned(req) {
|
||||||
|
t.Fatal("unsigned control granted replication trust")
|
||||||
|
}
|
||||||
|
if err := signRequestV4(req, "access-key", "secret-key"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !replicationHeadersSigned(req) {
|
||||||
|
t.Fatal("signed control rejected")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for _, authorization := range []string{"", "AWS access-key:signature", "Credential=access-key"} {
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/bucket/key", nil)
|
||||||
|
req.Header.Set(xhttp.Authorization, authorization)
|
||||||
|
req.Header.Set(xhttp.MinIOSourceReplicationRequest, "true")
|
||||||
|
if replicationHeadersSigned(req) {
|
||||||
|
t.Fatalf("non-SigV4 authorization granted trust: %q", authorization)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "http://localhost/bucket/key", nil)
|
||||||
|
req.Header.Set(xhttp.MinIOSourceReplicationRequest, "true")
|
||||||
|
presignBoundaryRequest(t, req, UTCNow(), []string{"host", "x-minio-source-replication-request"}, auth.Credentials{AccessKey: "access-key", SecretKey: "secret-key"})
|
||||||
|
getRequestAuthType(req)
|
||||||
|
if replicationHeadersSigned(req) {
|
||||||
|
t.Fatal("presigned request granted peer trust")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPIUnsignedReplicationControlsAreIgnored(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t, endpoints: []string{"PutObject"},
|
||||||
|
objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
|
||||||
|
for _, signedMarker := range []bool{false, true} {
|
||||||
|
headers := map[string]string{}
|
||||||
|
if signedMarker {
|
||||||
|
headers[xhttp.MinIOSourceReplicationRequest] = "true"
|
||||||
|
}
|
||||||
|
body := []byte("replication controls are authenticated")
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucket, "key"), int64(len(body)), bytes.NewReader(body), cred.AccessKey, cred.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
req.Header.Set(xhttp.MinIOSourceReplicationRequest, "true")
|
||||||
|
req.Header.Set(xhttp.MinIOSourceMTime, "2100-01-01T00:00:00Z")
|
||||||
|
req.Header.Set(xhttp.MinIOSourceETag, "forged-source-etag")
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, req)
|
||||||
|
oi, err := obj.GetObjectInfo(t.Context(), bucket, "key", ObjectOptions{})
|
||||||
|
if rec.Code != http.StatusOK || err != nil || oi.ModTime.Year() == 2100 || oi.ETag == "forged-source-etag" {
|
||||||
|
t.Fatalf("%s signedMarker=%v: unsigned replication state was accepted: HTTP=%d info=%+v err=%v", backend, signedMarker, rec.Code, oi, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -1850,6 +1850,12 @@ func testReplicaLockReconcileNullVersion(obj ObjectLayer, instanceType, bucketNa
|
|||||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||||
) {
|
) {
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
|
// A protected retransmit must preserve both the source ciphertext and
|
||||||
|
// its seal. Generate it once for the stored null version and its retry.
|
||||||
|
var ciphertext bytes.Buffer
|
||||||
|
if _, err := sio.Encrypt(&ciphertext, bytes.NewReader([]byte("data")), sio.Config{Key: bytes.Repeat([]byte{1}, 32)}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
// completeNullMPU runs an SSE-C replica multipart upload addressing the null
|
// completeNullMPU runs an SSE-C replica multipart upload addressing the null
|
||||||
// version (VersionSuspended) carrying uploadLock, through the reconcile.
|
// version (VersionSuspended) carrying uploadLock, through the reconcile.
|
||||||
@@ -1861,13 +1867,6 @@ func testReplicaLockReconcileNullVersion(obj ObjectLayer, instanceType, bucketNa
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
// PutObjectPart now validates the DARE stream length (#119). This
|
|
||||||
// object-layer fixture needs an encrypted body before it can exercise
|
|
||||||
// the completion-time null-version metadata reconciliation.
|
|
||||||
var ciphertext bytes.Buffer
|
|
||||||
if _, err := sio.Encrypt(&ciphertext, bytes.NewReader([]byte("data")), sio.Config{Key: bytes.Repeat([]byte{1}, 32)}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
part, err := obj.PutObjectPart(ctx, bucketName, object, res.UploadID, 1,
|
part, err := obj.PutObjectPart(ctx, bucketName, object, res.UploadID, 1,
|
||||||
mustGetPutObjReader(t, bytes.NewReader(ciphertext.Bytes()), int64(ciphertext.Len()), "", ""), ObjectOptions{})
|
mustGetPutObjReader(t, bytes.NewReader(ciphertext.Bytes()), int64(ciphertext.Len()), "", ""), ObjectOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1884,8 +1883,9 @@ func testReplicaLockReconcileNullVersion(obj ObjectLayer, instanceType, bucketNa
|
|||||||
object := "reconcile/null-vs-uuid"
|
object := "reconcile/null-vs-uuid"
|
||||||
// The null version holds a newer legal hold ON@10:00.
|
// The null version holds a newer legal hold ON@10:00.
|
||||||
if _, err := obj.PutObject(ctx, bucketName, object,
|
if _, err := obj.PutObject(ctx, bucketName, object,
|
||||||
mustGetPutObjReader(t, bytes.NewReader([]byte("data")), 4, "", ""),
|
mustGetPutObjReader(t, bytes.NewReader(ciphertext.Bytes()), int64(ciphertext.Len()), "", ""),
|
||||||
ObjectOptions{VersionSuspended: true, MTime: time.Date(2026, 9, 3, 10, 0, 0, 0, time.UTC), UserDefined: map[string]string{
|
ObjectOptions{VersionSuspended: true, MTime: time.Date(2026, 9, 3, 10, 0, 0, 0, time.UTC), UserDefined: map[string]string{
|
||||||
|
crypto.MetaSealedKeySSEC: "dummy-sealed-key",
|
||||||
strings.ToLower(xhttp.AmzObjectLockLegalHold): "ON",
|
strings.ToLower(xhttp.AmzObjectLockLegalHold): "ON",
|
||||||
ReservedMetadataPrefixLower + ObjectLockLegalHoldTimestamp: objectLockTestStamp1000,
|
ReservedMetadataPrefixLower + ObjectLockLegalHoldTimestamp: objectLockTestStamp1000,
|
||||||
}}); err != nil {
|
}}); err != nil {
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ package cmd
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
|
||||||
objectreplication "github.com/minio/minio/internal/bucket/replication"
|
objectreplication "github.com/minio/minio/internal/bucket/replication"
|
||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
@@ -62,7 +64,7 @@ func isReplicaTrusted(ctx context.Context) bool {
|
|||||||
// grant must not turn client-controlled internal headers into trusted state.
|
// grant must not turn client-controlled internal headers into trusted state.
|
||||||
func replicationPermissionAllowed(ctx context.Context, r *http.Request, bucket, object string, action policy.Action) bool {
|
func replicationPermissionAllowed(ctx context.Context, r *http.Request, bucket, object string, action policy.Action) bool {
|
||||||
reqInfo := logger.GetReqInfo(ctx)
|
reqInfo := logger.GetReqInfo(ctx)
|
||||||
if reqInfo == nil || reqInfo.Cred.AccessKey == "" {
|
if reqInfo == nil || reqInfo.Cred.AccessKey == "" || !replicationHeadersSigned(r) {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
reqInfo.BucketName = bucket
|
reqInfo.BucketName = bucket
|
||||||
@@ -70,6 +72,25 @@ func replicationPermissionAllowed(ctx context.Context, r *http.Request, bucket,
|
|||||||
return authorizeRequest(ctx, r, action) == ErrNone
|
return authorizeRequest(ctx, r, action) == ErrNone
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// replicationHeadersSigned binds every internal control to the authenticated
|
||||||
|
// header signature. Query grants and SigV2 requests cannot confer peer trust.
|
||||||
|
// Call only after the request's signature has been verified.
|
||||||
|
func replicationHeadersSigned(r *http.Request) bool {
|
||||||
|
if !isRequestSignatureV4(r) || isRequestPresignedSignatureV4(r) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
sig, errCode := parseSignV4(r.Header.Get(xhttp.Authorization), globalSite.Region(), serviceS3)
|
||||||
|
if errCode != ErrNone {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, name := range replicationRequestHeaders {
|
||||||
|
if _, present := r.Header[http.CanonicalHeaderKey(name)]; present && !slices.Contains(sig.SignedHeaders, strings.ToLower(name)) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
// evaluateReplicationTrust decides whether a request may carry replication
|
// evaluateReplicationTrust decides whether a request may carry replication
|
||||||
// semantics for the given action. A request that declares itself a replica
|
// semantics for the given action. A request that declares itself a replica
|
||||||
// without holding the replication permission is rejected. trusted reports that
|
// without holding the replication permission is rejected. trusted reports that
|
||||||
|
|||||||
@@ -0,0 +1,141 @@
|
|||||||
|
// Copyright (c) 2026 Feng Ruohang
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/minio/madmin-go/v3"
|
||||||
|
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||||
|
"github.com/minio/minio/internal/bucket/versioning"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestVersionRevisionRestoreFlags(t *testing.T) {
|
||||||
|
base := authorityRevision(t, [16]byte{1}, 1, 1, "")
|
||||||
|
cleared := base
|
||||||
|
cleared.header.Flags &^= xlFlagUsesDataDir
|
||||||
|
set := base
|
||||||
|
set.header.Flags |= xlFlagUsesDataDir
|
||||||
|
if versionRevision(cleared, xlMetaVersion) != versionRevision(set, xlMetaVersion) {
|
||||||
|
t.Fatal("derived flag splits one revision")
|
||||||
|
}
|
||||||
|
for _, change := range []struct {
|
||||||
|
name string
|
||||||
|
apply func(*xlMetaV2Object)
|
||||||
|
}{
|
||||||
|
{"data-dir", func(o *xlMetaV2Object) { o.DataDir[0]++ }},
|
||||||
|
{"restore", func(o *xlMetaV2Object) { o.MetaUser[xhttp.AmzRestore] = `ongoing-request="true"` }},
|
||||||
|
{"retention", func(o *xlMetaV2Object) { o.MetaUser["x-amz-object-lock-mode"] = "COMPLIANCE" }},
|
||||||
|
{"layout", func(o *xlMetaV2Object) { o.ErasureM, o.ErasureN = 8, 8 }},
|
||||||
|
} {
|
||||||
|
t.Run(change.name, func(t *testing.T) {
|
||||||
|
var v xlMetaV2Version
|
||||||
|
if _, err := v.unmarshalV(xlMetaVersion, base.meta); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
change.apply(v.ObjectV2)
|
||||||
|
meta, err := v.MarshalMsg(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
different := xlMetaV2ShallowVersion{header: cleared.header, meta: meta}
|
||||||
|
if versionRevision(different, xlMetaVersion) == versionRevision(set, xlMetaVersion) {
|
||||||
|
t.Fatal("real metadata change lost")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
// Every other header flag remains part of identity, even if metadata is equal.
|
||||||
|
for flag := xlFlags(1); flag != 0; flag <<= 1 {
|
||||||
|
if flag == xlFlagUsesDataDir {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
other := base
|
||||||
|
other.header.Flags ^= flag
|
||||||
|
if versionRevision(other, xlMetaVersion) == versionRevision(base, xlMetaVersion) {
|
||||||
|
t.Fatalf("flag %d was normalized", flag)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
malformed := base
|
||||||
|
malformed.meta = []byte{0xc1}
|
||||||
|
other := malformed
|
||||||
|
other.header.Flags ^= xlFlagUsesDataDir
|
||||||
|
if versionRevision(malformed, xlMetaVersion) == versionRevision(other, xlMetaVersion) {
|
||||||
|
t.Fatal("undecodable metadata was normalized")
|
||||||
|
}
|
||||||
|
marker := authorityMarker(t, [16]byte{2}, 2)
|
||||||
|
other = marker
|
||||||
|
other.header.Flags ^= xlFlagUsesDataDir
|
||||||
|
if versionRevision(marker, xlMetaVersion) == versionRevision(other, xlMetaVersion) {
|
||||||
|
t.Fatal("delete marker was normalized as an object")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Two drives are replaced one at a time after the restore expires. The old
|
||||||
|
// headers describe the pre-expiry state; heal writes the current derived flag.
|
||||||
|
func TestRestoreExpiryAcrossSuccessiveHeals(t *testing.T) {
|
||||||
|
r := newRestoreSafetyTest(t, 2, versioning.Enabled)
|
||||||
|
ctx := t.Context()
|
||||||
|
oi := r.put("restored", false, true)
|
||||||
|
r.restore(oi)
|
||||||
|
before := r.fileInfo(oi.VersionID)
|
||||||
|
disks := r.z.serverPools[0].getHashedSet(r.object).getDisks()
|
||||||
|
// Model the headers written before wall-clock expiry without a timing race.
|
||||||
|
for _, disk := range disks {
|
||||||
|
rewriteTierTestMeta(t, disk, r.bucket, r.object, func(xl *xlMetaV2) { xl.versions[0].header.Flags |= xlFlagUsesDataDir })
|
||||||
|
}
|
||||||
|
for _, i := range []int{3, 2} {
|
||||||
|
if err := disks[i].Delete(ctx, r.bucket, r.object, DeleteOptions{Recursive: true, Immediate: true}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := r.z.HealObject(ctx, r.bucket, r.object, oi.VersionID, madmin.HealOpts{ScanMode: madmin.HealNormalScan}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for i, disk := range disks {
|
||||||
|
raw, err := disk.ReadXL(ctx, r.bucket, r.object, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var xl xlMetaV2
|
||||||
|
if err = xl.Load(raw.Buf); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := xl.versions[0].header.Flags&xlFlagUsesDataDir != 0; got != (i < 2) {
|
||||||
|
t.Fatalf("drive %d has unexpected derived flag %v", i, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, _, err := r.z.lifecycleEventsAllPools(ctx, r.bucket, r.object); err != nil {
|
||||||
|
t.Fatalf("history with split flags: %v", err)
|
||||||
|
}
|
||||||
|
current, err := r.z.GetObjectInfo(ctx, r.bucket, r.object, r.opts(oi.VersionID))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err = expireTransitionedObject(ctx, r.z, ¤t, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner); err != nil {
|
||||||
|
t.Fatalf("expire restored copy: %v", err)
|
||||||
|
}
|
||||||
|
if n := r.dataDirs(before.DataDir); n != 0 {
|
||||||
|
t.Fatalf("restore cleanup left %d data directories", n)
|
||||||
|
}
|
||||||
|
if got := readCopyChecksumObject(t, r.z, r.bucket, r.object, r.opts(oi.VersionID)); string(got) != string(r.bodies["restored"]) {
|
||||||
|
t.Fatal("tiered object is no longer readable")
|
||||||
|
}
|
||||||
|
r.put("newer", false, false)
|
||||||
|
if _, _, err = r.z.lifecycleEventsAllPools(ctx, r.bucket, r.object); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err = r.z.deleteAllHistory(ctx, r.bucket, r.object); err != nil {
|
||||||
|
t.Fatalf("purge history with split flags: %v", err)
|
||||||
|
}
|
||||||
|
// The rule permits tiering, not a purge. Normalizing the header must not
|
||||||
|
// turn a readable history into permission to remove the versions.
|
||||||
|
if _, err = r.z.DeleteObject(ctx, r.bucket, r.object, ObjectOptions{DeleteAllVersions: true, Expiration: ExpirationOptions{Expire: true}}); !errors.Is(err, errLifecycleNotApplicable) {
|
||||||
|
t.Fatalf("purge without an applicable rule: %v", err)
|
||||||
|
}
|
||||||
|
if _, err = r.z.GetObjectInfo(ctx, r.bucket, r.object, r.opts(oi.VersionID)); err != nil {
|
||||||
|
t.Fatalf("refused purge removed the tiered version: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -320,11 +320,8 @@ func (er erasureObjects) stageRestore(ctx context.Context, bucket, object string
|
|||||||
|
|
||||||
// fileInfoRevisions are the revisions the entry that writing fi creates can
|
// fileInfoRevisions are the revisions the entry that writing fi creates can
|
||||||
// have, as readRestoreGeneration and the convergence identify the entries
|
// have, as readRestoreGeneration and the convergence identify the entries
|
||||||
// they read. The header flag that tells whether a transitioned version uses
|
// they read. versionRevision normalizes the time-derived UsesDataDir flag;
|
||||||
// its data directory follows whether its restored copy has expired at the
|
// the stored data directory and restore metadata still have to match.
|
||||||
// moment a drive writes the entry, so a restore whose copy expires while it
|
|
||||||
// publishes can find either value: both are this entry. Every other field of
|
|
||||||
// the header and the full metadata digest still have to match.
|
|
||||||
func fileInfoRevisions(fi FileInfo) ([]revisionKey, error) {
|
func fileInfoRevisions(fi FileInfo) ([]revisionKey, error) {
|
||||||
var x xlMetaV2
|
var x xlMetaV2
|
||||||
if err := x.AddVersion(fi); err != nil {
|
if err := x.AddVersion(fi); err != nil {
|
||||||
@@ -342,9 +339,7 @@ func fileInfoRevisions(fi FileInfo) ([]revisionKey, error) {
|
|||||||
return nil, errFileCorrupt
|
return nil, errFileCorrupt
|
||||||
}
|
}
|
||||||
ver := y.versions[0]
|
ver := y.versions[0]
|
||||||
other := ver
|
return []revisionKey{versionRevision(ver, y.metaV)}, nil
|
||||||
other.header.Flags ^= xlFlagUsesDataDir
|
|
||||||
return []revisionKey{versionRevision(ver, y.metaV), versionRevision(other, y.metaV)}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// restorePublication remembers, for a commit that published the staged copy
|
// restorePublication remembers, for a commit that published the staged copy
|
||||||
|
|||||||
@@ -731,7 +731,7 @@ func TestRestorePublishedRevisionAcrossExpiry(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
stored := func() revisionKey {
|
stored := func() (xlMetaV2VersionHeader, revisionKey) {
|
||||||
var x xlMetaV2
|
var x xlMetaV2
|
||||||
if err := x.AddVersion(published); err != nil {
|
if err := x.AddVersion(published); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -744,16 +744,19 @@ func TestRestorePublishedRevisionAcrossExpiry(t *testing.T) {
|
|||||||
if err := y.Load(buf); err != nil {
|
if err := y.Load(buf); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
return versionRevision(y.versions[0], y.metaV)
|
return y.versions[0].header, versionRevision(y.versions[0], y.metaV)
|
||||||
}
|
}
|
||||||
before := stored()
|
beforeHeader, before := stored()
|
||||||
time.Sleep(time.Until(expiry) + 100*time.Millisecond)
|
time.Sleep(time.Until(expiry) + 100*time.Millisecond)
|
||||||
after := stored()
|
afterHeader, after := stored()
|
||||||
if before == after {
|
if beforeHeader.Flags == afterHeader.Flags {
|
||||||
t.Fatal("fixture: the stored flag did not change at the expiry")
|
t.Fatal("fixture: the stored flag did not change at the expiry")
|
||||||
}
|
}
|
||||||
|
if before != after {
|
||||||
|
t.Fatal("expiry changed the identity of the published revision")
|
||||||
|
}
|
||||||
for _, got := range []revisionKey{before, after} {
|
for _, got := range []revisionKey{before, after} {
|
||||||
if got != keys[0] && got != keys[1] {
|
if got != keys[0] {
|
||||||
t.Errorf("a publication's own entry is not recognized: %v not in %v", got, keys)
|
t.Errorf("a publication's own entry is not recognized: %v not in %v", got, keys)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+86
-87
@@ -28,10 +28,10 @@ import (
|
|||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/minio/minio/internal/auth"
|
|
||||||
"github.com/minio/minio/internal/crypto"
|
"github.com/minio/minio/internal/crypto"
|
||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
xioutil "github.com/minio/minio/internal/ioutil"
|
xioutil "github.com/minio/minio/internal/ioutil"
|
||||||
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/zipindex"
|
"github.com/minio/zipindex"
|
||||||
"github.com/pgsty/silo-pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
@@ -82,35 +82,9 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
|
|||||||
|
|
||||||
getObjectInfo := objectAPI.GetObjectInfo
|
getObjectInfo := objectAPI.GetObjectInfo
|
||||||
|
|
||||||
// Check for auth type to return S3 compatible error.
|
logger.GetReqInfo(ctx).BucketName = bucket
|
||||||
// type to return the correct error (NoSuchKey vs AccessDenied)
|
logger.GetReqInfo(ctx).ObjectName = zipPath
|
||||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, bucket, zipPath); s3Error != ErrNone {
|
if s3Error := authenticateRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone {
|
||||||
if getRequestAuthType(r) == authTypeAnonymous {
|
|
||||||
// As per "Permission" section in
|
|
||||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html
|
|
||||||
// If the object you request does not exist,
|
|
||||||
// the error Amazon S3 returns depends on
|
|
||||||
// whether you also have the s3:ListBucket
|
|
||||||
// permission.
|
|
||||||
// * If you have the s3:ListBucket permission
|
|
||||||
// on the bucket, Amazon S3 will return an
|
|
||||||
// HTTP status code 404 ("no such key")
|
|
||||||
// error.
|
|
||||||
// * if you don’t have the s3:ListBucket
|
|
||||||
// permission, Amazon S3 will return an HTTP
|
|
||||||
// status code 403 ("access denied") error.`
|
|
||||||
if globalPolicySys.IsAllowed(policy.BucketPolicyArgs{
|
|
||||||
Action: policy.ListBucketAction,
|
|
||||||
BucketName: bucket,
|
|
||||||
ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
|
|
||||||
IsOwner: false,
|
|
||||||
}) {
|
|
||||||
_, err = getObjectInfo(ctx, bucket, zipPath, opts)
|
|
||||||
if toAPIError(ctx, err).Code == "NoSuchKey" {
|
|
||||||
s3Error = ErrNoSuchKey
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -128,6 +102,10 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
|
|||||||
|
|
||||||
// Validate pre-conditions if any.
|
// Validate pre-conditions if any.
|
||||||
opts.CheckPrecondFn = func(oi ObjectInfo) bool {
|
opts.CheckPrecondFn = func(oi ObjectInfo) bool {
|
||||||
|
if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, oi.UserTags); s3Error != ErrNone {
|
||||||
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||||
|
return true
|
||||||
|
}
|
||||||
if _, err := DecryptObjectInfo(&oi, r); err != nil {
|
if _, err := DecryptObjectInfo(&oi, r); err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return true
|
return true
|
||||||
@@ -138,19 +116,20 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
|
|||||||
|
|
||||||
zipObjInfo, err := getObjectInfo(ctx, bucket, zipPath, opts)
|
zipObjInfo, err := getObjectInfo(ctx, bucket, zipPath, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
err = archiveReadError(ctx, r, bucket, zipPath, opts.VersionID, err)
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if opts.CheckPrecondFn(zipObjInfo) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
zipInfo := zipObjInfo.ArchiveInfo(r.Header)
|
zipInfo := zipObjInfo.ArchiveInfo(r.Header)
|
||||||
if len(zipInfo) == 0 {
|
if len(zipInfo) == 0 {
|
||||||
opts.EncryptFn, err = zipObjInfo.metadataEncryptFn(r.Header)
|
zipInfo, _, err = readObjectZIPInfo(ctx, objectAPI, bucket, zipPath, opts)
|
||||||
if err != nil {
|
}
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
if isErrPreconditionFailed(err) {
|
||||||
return
|
return
|
||||||
}
|
|
||||||
|
|
||||||
zipInfo, err = updateObjectMetadataWithZipInfo(ctx, objectAPI, bucket, zipPath, opts)
|
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
@@ -183,6 +162,9 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
|
|||||||
end := min(file.Offset+int64(file.CompressedSize64)+64<<10, zipObjInfo.Size)
|
end := min(file.Offset+int64(file.CompressedSize64)+64<<10, zipObjInfo.Size)
|
||||||
rs := &HTTPRangeSpec{Start: file.Offset, End: end}
|
rs := &HTTPRangeSpec{Start: file.Offset, End: end}
|
||||||
gr, err := objectAPI.GetObjectNInfo(ctx, bucket, zipPath, rs, nil, opts)
|
gr, err := objectAPI.GetObjectNInfo(ctx, bucket, zipPath, rs, nil, opts)
|
||||||
|
if isErrPreconditionFailed(err) {
|
||||||
|
return
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
@@ -199,6 +181,7 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
|
|||||||
|
|
||||||
defer rc.Close()
|
defer rc.Close()
|
||||||
|
|
||||||
|
opts.Tagging = false
|
||||||
if err = setObjectHeaders(ctx, w, fileObjInfo, nil, opts); err != nil {
|
if err = setObjectHeaders(ctx, w, fileObjInfo, nil, opts); err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
@@ -230,23 +213,40 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
|
|||||||
}
|
}
|
||||||
|
|
||||||
// listObjectsV2InArchive generates S3 listing result ListObjectsV2Info from zip file, all parameters are already validated by the caller.
|
// listObjectsV2InArchive generates S3 listing result ListObjectsV2Info from zip file, all parameters are already validated by the caller.
|
||||||
func listObjectsV2InArchive(ctx context.Context, objectAPI ObjectLayer, bucket, prefix, token, delimiter string, maxKeys int, startAfter string, h http.Header) (ListObjectsV2Info, error) {
|
func listObjectsV2InArchive(ctx context.Context, objectAPI ObjectLayer, bucket, prefix, token, delimiter string, maxKeys int, startAfter string, r *http.Request) (ListObjectsV2Info, error) {
|
||||||
zipPath, _, err := splitZipExtensionPath(prefix)
|
zipPath, _, err := splitZipExtensionPath(prefix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Return empty listing
|
// Return empty listing
|
||||||
return ListObjectsV2Info{}, nil
|
return ListObjectsV2Info{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
zipObjInfo, err := objectAPI.GetObjectInfo(ctx, bucket, zipPath, ObjectOptions{})
|
// The bucket listing permission does not grant access to ZIP contents.
|
||||||
|
logger.GetReqInfo(ctx).ObjectName = zipPath
|
||||||
|
opts := ObjectOptions{}
|
||||||
|
zipObjInfo, err := objectAPI.GetObjectInfo(ctx, bucket, zipPath, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Return empty listing
|
if s3Error := authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, "", err); s3Error != ErrNone {
|
||||||
return ListObjectsV2Info{}, nil
|
return ListObjectsV2Info{}, PrefixAccessDenied{Bucket: bucket, Object: zipPath}
|
||||||
|
}
|
||||||
|
if isErrObjectNotFound(err) {
|
||||||
|
return ListObjectsV2Info{}, nil
|
||||||
|
}
|
||||||
|
return ListObjectsV2Info{}, err
|
||||||
}
|
}
|
||||||
|
if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, "", zipObjInfo.UserTags); s3Error != ErrNone {
|
||||||
zipInfo := zipObjInfo.ArchiveInfo(h)
|
return ListObjectsV2Info{}, PrefixAccessDenied{Bucket: bucket, Object: zipPath}
|
||||||
|
}
|
||||||
|
var denied bool
|
||||||
|
opts.CheckPrecondFn = func(oi ObjectInfo) bool {
|
||||||
|
denied = authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, "", oi.UserTags) != ErrNone
|
||||||
|
return denied
|
||||||
|
}
|
||||||
|
zipInfo := zipObjInfo.ArchiveInfo(r.Header)
|
||||||
if len(zipInfo) == 0 {
|
if len(zipInfo) == 0 {
|
||||||
// Always update the latest version
|
zipInfo, _, err = readObjectZIPInfo(ctx, objectAPI, bucket, zipPath, opts)
|
||||||
zipInfo, err = updateObjectMetadataWithZipInfo(ctx, objectAPI, bucket, zipPath, ObjectOptions{})
|
}
|
||||||
|
if denied {
|
||||||
|
return ListObjectsV2Info{}, PrefixAccessDenied{Bucket: bucket, Object: zipPath}
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ListObjectsV2Info{}, err
|
return ListObjectsV2Info{}, err
|
||||||
@@ -379,42 +379,19 @@ func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context,
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, bucket, zipPath); s3Error != ErrNone {
|
logger.GetReqInfo(ctx).BucketName = bucket
|
||||||
if getRequestAuthType(r) == authTypeAnonymous {
|
logger.GetReqInfo(ctx).ObjectName = zipPath
|
||||||
// As per "Permission" section in
|
if s3Error := authenticateRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone {
|
||||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html
|
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error))
|
||||||
// If the object you request does not exist,
|
|
||||||
// the error Amazon S3 returns depends on
|
|
||||||
// whether you also have the s3:ListBucket
|
|
||||||
// permission.
|
|
||||||
// * If you have the s3:ListBucket permission
|
|
||||||
// on the bucket, Amazon S3 will return an
|
|
||||||
// HTTP status code 404 ("no such key")
|
|
||||||
// error.
|
|
||||||
// * if you don’t have the s3:ListBucket
|
|
||||||
// permission, Amazon S3 will return an HTTP
|
|
||||||
// status code 403 ("access denied") error.`
|
|
||||||
if globalPolicySys.IsAllowed(policy.BucketPolicyArgs{
|
|
||||||
Action: policy.ListBucketAction,
|
|
||||||
BucketName: bucket,
|
|
||||||
ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
|
|
||||||
IsOwner: false,
|
|
||||||
}) {
|
|
||||||
_, err = getObjectInfo(ctx, bucket, zipPath, opts)
|
|
||||||
if toAPIError(ctx, err).Code == "NoSuchKey" {
|
|
||||||
s3Error = ErrNoSuchKey
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
errCode := errorCodes.ToAPIErr(s3Error)
|
|
||||||
w.Header().Set(xMinIOErrCodeHeader, errCode.Code)
|
|
||||||
w.Header().Set(xMinIOErrDescHeader, "\""+errCode.Description+"\"")
|
|
||||||
writeErrorResponseHeadersOnly(w, errCode)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate pre-conditions if any.
|
// Validate pre-conditions if any.
|
||||||
opts.CheckPrecondFn = func(oi ObjectInfo) bool {
|
opts.CheckPrecondFn = func(oi ObjectInfo) bool {
|
||||||
|
if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, oi.UserTags); s3Error != ErrNone {
|
||||||
|
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error))
|
||||||
|
return true
|
||||||
|
}
|
||||||
return checkPreconditions(ctx, w, r, oi, opts)
|
return checkPreconditions(ctx, w, r, oi, opts)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -431,18 +408,20 @@ func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context,
|
|||||||
|
|
||||||
zipObjInfo, err := getObjectInfo(ctx, bucket, zipPath, opts)
|
zipObjInfo, err := getObjectInfo(ctx, bucket, zipPath, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
err = archiveReadError(ctx, r, bucket, zipPath, opts.VersionID, err)
|
||||||
writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
|
writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if opts.CheckPrecondFn(zipObjInfo) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
zipInfo := zipObjInfo.ArchiveInfo(r.Header)
|
zipInfo := zipObjInfo.ArchiveInfo(r.Header)
|
||||||
if len(zipInfo) == 0 {
|
if len(zipInfo) == 0 {
|
||||||
opts.EncryptFn, err = zipObjInfo.metadataEncryptFn(r.Header)
|
zipInfo, _, err = readObjectZIPInfo(ctx, objectAPI, bucket, zipPath, opts)
|
||||||
if err != nil {
|
}
|
||||||
writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
|
if isErrPreconditionFailed(err) {
|
||||||
return
|
return
|
||||||
}
|
|
||||||
zipInfo, err = updateObjectMetadataWithZipInfo(ctx, objectAPI, bucket, zipPath, opts)
|
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
|
writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
|
||||||
@@ -466,6 +445,7 @@ func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context,
|
|||||||
ModTime: zipObjInfo.ModTime,
|
ModTime: zipObjInfo.ModTime,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
opts.Tagging = false
|
||||||
// Set standard object headers.
|
// Set standard object headers.
|
||||||
if err = setObjectHeaders(ctx, w, objInfo, nil, opts); err != nil {
|
if err = setObjectHeaders(ctx, w, objInfo, nil, opts); err != nil {
|
||||||
writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
|
writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
|
||||||
@@ -482,15 +462,35 @@ func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context,
|
|||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update the passed zip object metadata with the zip contents info, file name, modtime, size, etc.
|
// archiveReadError preserves the anonymous ListBucket missing-object response.
|
||||||
// The returned zip index will de decrypted.
|
// All other failures are disclosed only to callers authorized to read the ZIP.
|
||||||
func updateObjectMetadataWithZipInfo(ctx context.Context, objectAPI ObjectLayer, bucket, object string, opts ObjectOptions) ([]byte, error) {
|
func archiveReadError(ctx context.Context, r *http.Request, bucket, object, versionID string, err error) error {
|
||||||
|
if getRequestAuthType(r) == authTypeAnonymous && isErrObjectNotFound(err) &&
|
||||||
|
authorizeBucketRequest(ctx, r, policy.ListBucketAction) == ErrNone {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, versionID, err) != ErrNone {
|
||||||
|
return PrefixAccessDenied{Bucket: bucket, Object: object}
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// readObjectZIPInfo builds an index without modifying the object. A GET, HEAD
|
||||||
|
// or archive listing must remain read-only even when no index was stored.
|
||||||
|
func readObjectZIPInfo(ctx context.Context, objectAPI ObjectLayer, bucket, object string, opts ObjectOptions) ([]byte, ObjectInfo, error) {
|
||||||
files, srcInfo, err := getFilesListFromZIPObject(ctx, objectAPI, bucket, object, opts)
|
files, srcInfo, err := getFilesListFromZIPObject(ctx, objectAPI, bucket, object, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, srcInfo, err
|
||||||
}
|
}
|
||||||
files.OptimizeSize()
|
files.OptimizeSize()
|
||||||
zipInfo, err := files.Serialize()
|
zipInfo, err := files.Serialize()
|
||||||
|
return zipInfo, srcInfo, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// updateObjectMetadataWithZipInfo persists an index only for upload operations
|
||||||
|
// that explicitly request extraction. The returned index is decrypted.
|
||||||
|
func updateObjectMetadataWithZipInfo(ctx context.Context, objectAPI ObjectLayer, bucket, object string, opts ObjectOptions) ([]byte, error) {
|
||||||
|
zipInfo, srcInfo, err := readObjectZIPInfo(ctx, objectAPI, bucket, object, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -500,7 +500,6 @@ func updateObjectMetadataWithZipInfo(ctx context.Context, objectAPI ObjectLayer,
|
|||||||
at = archiveTypeEnc
|
at = archiveTypeEnc
|
||||||
zipInfoStr = string(opts.EncryptFn(archiveTypeEnc, zipInfo))
|
zipInfoStr = string(opts.EncryptFn(archiveTypeEnc, zipInfo))
|
||||||
}
|
}
|
||||||
srcInfo.UserDefined[archiveTypeMetadataKey] = at
|
|
||||||
popts := ObjectOptions{
|
popts := ObjectOptions{
|
||||||
MTime: srcInfo.ModTime,
|
MTime: srcInfo.ModTime,
|
||||||
VersionID: srcInfo.VersionID,
|
VersionID: srcInfo.VersionID,
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
// Copyright (c) 2026 Feng Ruohang
|
||||||
|
//
|
||||||
|
// This file is part of Silo Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/zip"
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"maps"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAPIArchiveReadsDoNotPersistMetadata(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t, makeBucketOptions: MakeBucketOptions{LockEnabled: true},
|
||||||
|
objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
|
||||||
|
var archive bytes.Buffer
|
||||||
|
zw := zip.NewWriter(&archive)
|
||||||
|
file, err := zw.Create("entry.txt")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err = file.Write([]byte("archive content")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err = zw.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reader := newObjectAttributesAuthzUser(t, backend, bucket, `"s3:GetObject","s3:ListBucket"`)
|
||||||
|
policy := fmt.Appendf(nil, `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":["s3:GetObject","s3:ListBucket"],"Resource":["arn:aws:s3:::%s","arn:aws:s3:::%s/*"]}]}`, bucket, bucket)
|
||||||
|
if _, err = globalBucketMetadataSys.Update(t.Context(), bucket, bucketPolicyConfig, policy); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, anonymous := range []bool{false, true} {
|
||||||
|
for _, method := range []string{http.MethodGet, http.MethodHead, "LIST"} {
|
||||||
|
key := fmt.Sprintf("archive-%v-%s.zip", anonymous, method)
|
||||||
|
before, err := obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, bytes.NewReader(archive.Bytes()), int64(archive.Len()), "", ""), ObjectOptions{Versioned: true, UserDefined: batchLockMeta("COMPLIANCE")})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
before, err = obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: before.VersionID})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
url := getGetObjectURL("", bucket, key+"/entry.txt")
|
||||||
|
verb := method
|
||||||
|
if method == "LIST" {
|
||||||
|
verb = http.MethodGet
|
||||||
|
url = "/" + bucket + "?list-type=2&prefix=" + key + "/"
|
||||||
|
}
|
||||||
|
req, err := newTestSignedRequestV4(verb, url, 0, nil, reader.AccessKey, reader.SecretKey, map[string]string{xMinIOExtract: "true"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if anonymous {
|
||||||
|
req.Header.Del("Authorization")
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s anonymous=%v %s: %d %s", backend, anonymous, method, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if method == http.MethodGet && rec.Body.String() != "archive content" {
|
||||||
|
t.Fatalf("wrong archive entry: %s", rec.Body.String())
|
||||||
|
}
|
||||||
|
if method == "LIST" && !bytes.Contains(rec.Body.Bytes(), []byte("entry.txt")) {
|
||||||
|
t.Fatal("archive listing omitted entry")
|
||||||
|
}
|
||||||
|
after, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: before.VersionID})
|
||||||
|
if err != nil || !maps.Equal(before.UserDefined, after.UserDefined) || !after.ModTime.Equal(before.ModTime) || after.ETag != before.ETag {
|
||||||
|
t.Fatalf("%s anonymous=%v %s changed stored metadata: before=%v after=%v err=%v", backend, anonymous, method, before.UserDefined, after.UserDefined, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Explicit extraction during an authorized upload still builds and
|
||||||
|
// persists the index for subsequent reads.
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucket, "indexed.zip"), int64(archive.Len()), bytes.NewReader(archive.Bytes()), cred.AccessKey, cred.SecretKey, map[string]string{xMinIOExtract: "true"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, req)
|
||||||
|
oi, err := obj.GetObjectInfo(t.Context(), bucket, "indexed.zip", ObjectOptions{})
|
||||||
|
if rec.Code != http.StatusOK || err != nil || len(oi.ArchiveInfo(nil)) == 0 {
|
||||||
|
t.Fatalf("explicit extraction lost index: %d %s %v", rec.Code, rec.Body.String(), err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -189,6 +189,16 @@ func doesPolicySignatureV4Match(formValues http.Header) (auth.Credentials, APIEr
|
|||||||
return cred, s3Err
|
return cred, s3Err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if dateStr := formValues.Get(xhttp.AmzDate); dateStr != "" {
|
||||||
|
t, err := time.Parse(iso8601Format, dateStr)
|
||||||
|
if err != nil {
|
||||||
|
return cred, ErrMalformedDate
|
||||||
|
}
|
||||||
|
if credHeader.scope.date.Format(yyyymmdd) != t.UTC().Format(yyyymmdd) {
|
||||||
|
return cred, ErrSignatureDoesNotMatch
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Get signing key.
|
// Get signing key.
|
||||||
signingKey := getSigningKey(cred.SecretKey, credHeader.scope.date, credHeader.scope.region, serviceS3)
|
signingKey := getSigningKey(cred.SecretKey, credHeader.scope.date, credHeader.scope.region, serviceS3)
|
||||||
|
|
||||||
@@ -248,6 +258,9 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
|
|||||||
|
|
||||||
// Save the date and expires.
|
// Save the date and expires.
|
||||||
t := pSignValues.Date
|
t := pSignValues.Date
|
||||||
|
if pSignValues.Credential.scope.date.Format(yyyymmdd) != t.UTC().Format(yyyymmdd) {
|
||||||
|
return ErrSignatureDoesNotMatch
|
||||||
|
}
|
||||||
expireSeconds := int(pSignValues.Expires / time.Second)
|
expireSeconds := int(pSignValues.Expires / time.Second)
|
||||||
|
|
||||||
// Construct new query.
|
// Construct new query.
|
||||||
@@ -388,6 +401,9 @@ func doesSignatureMatch(hashedPayload string, r *http.Request, region string, st
|
|||||||
if e != nil {
|
if e != nil {
|
||||||
return ErrMalformedDate
|
return ErrMalformedDate
|
||||||
}
|
}
|
||||||
|
if signV4Values.Credential.scope.date.Format(yyyymmdd) != t.UTC().Format(yyyymmdd) {
|
||||||
|
return ErrSignatureDoesNotMatch
|
||||||
|
}
|
||||||
|
|
||||||
// Query string.
|
// Query string.
|
||||||
queryStr := req.Form.Encode()
|
queryStr := req.Form.Encode()
|
||||||
|
|||||||
@@ -126,9 +126,8 @@ func latestBucketConfig(bucket, file string, info srStatusInfo) (bucketConfigSta
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *SiteReplicationSys) healBucketConfig(ctx context.Context, bucket, file string, info srStatusInfo) error {
|
func (c *SiteReplicationSys) healBucketConfig(ctx context.Context, bucket, file string, info srStatusInfo) error {
|
||||||
c.RLock()
|
state, ok := c.stateSnapshot()
|
||||||
defer c.RUnlock()
|
if !ok {
|
||||||
if !c.enabled {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
latest, found := latestBucketConfig(bucket, file, info)
|
latest, found := latestBucketConfig(bucket, file, info)
|
||||||
@@ -177,7 +176,7 @@ func (c *SiteReplicationSys) healBucketConfig(ctx context.Context, bucket, file
|
|||||||
_, err = globalBucketMetadataSys.updateAndParseMetadata(ctx, bucket, file, latest.data, false, false, &latest.at)
|
_, err = globalBucketMetadataSys.updateAndParseMetadata(ctx, bucket, file, latest.data, false, false, &latest.at)
|
||||||
} else {
|
} else {
|
||||||
var client *madmin.AdminClient
|
var client *madmin.AdminClient
|
||||||
client, err = c.getAdminClient(ctx, id)
|
client, err = state.getAdminClient(id)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
err = client.SRPeerReplicateBucketMeta(ctx, newBucketConfigReplicationEvent(bucket, file, incoming))
|
err = client.SRPeerReplicateBucketMeta(ctx, newBucketConfigReplicationEvent(bucket, file, incoming))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -622,6 +622,23 @@ func TestLocalBucketMetadataCommittedEvents(t *testing.T) {
|
|||||||
t.Fatalf("zero quota event: %+v", last)
|
t.Fatalf("zero quota event: %+v", last)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Replicated buckets must reject prefix exclusions without changing
|
||||||
|
// the stored versioning document or announcing a successful update.
|
||||||
|
beforeRejected := len(events())
|
||||||
|
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata", corsZip(t, map[string][]byte{
|
||||||
|
bucket + "/" + bucketVersioningConfig: []byte(`<VersioningConfiguration><Status>Enabled</Status><ExcludeFolders>true</ExcludeFolders></VersioningConfiguration>`),
|
||||||
|
}))
|
||||||
|
if report := corsImportReport(t, rec).Buckets[bucket]; report.Versioning.Err == "" {
|
||||||
|
t.Fatal("replicated bucket accepted excluded prefixes")
|
||||||
|
}
|
||||||
|
afterRejected, err := loadBucketMetadata(ctx, obj, bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(afterRejected.VersioningConfigXML, meta.VersioningConfigXML) || !afterRejected.VersioningConfigUpdatedAt.Equal(meta.VersioningConfigUpdatedAt) || len(events()) != beforeRejected {
|
||||||
|
t.Fatal("rejected versioning import changed metadata or emitted an event")
|
||||||
|
}
|
||||||
|
|
||||||
injectedAt := UTCNow().Add(2 * time.Hour)
|
injectedAt := UTCNow().Add(2 * time.Hour)
|
||||||
interleaving := &importInterleavingObjectLayer{ObjectLayer: obj, write: func() {
|
interleaving := &importInterleavingObjectLayer{ObjectLayer: obj, write: func() {
|
||||||
data := []byte(`{"quota":2048,"quotatype":"hard"}`)
|
data := []byte(`{"quota":2048,"quotatype":"hard"}`)
|
||||||
@@ -638,10 +655,10 @@ func TestLocalBucketMetadataCommittedEvents(t *testing.T) {
|
|||||||
bucket + "/quota.json": []byte(`{}`),
|
bucket + "/quota.json": []byte(`{}`),
|
||||||
bucket + "/" + bucketTaggingConfig: []byte(`<Tagging><TagSet><Tag><Key>imported</Key><Value>yes</Value></Tag></TagSet></Tagging>`),
|
bucket + "/" + bucketTaggingConfig: []byte(`<Tagging><TagSet><Tag><Key>imported</Key><Value>yes</Value></Tag></TagSet></Tagging>`),
|
||||||
bucket + "/" + objectLockConfig: enabledBucketObjectLockConfig,
|
bucket + "/" + objectLockConfig: enabledBucketObjectLockConfig,
|
||||||
bucket + "/" + bucketVersioningConfig: []byte(`<VersioningConfiguration><Status>Enabled</Status><ExcludeFolders>true</ExcludeFolders></VersioningConfiguration>`),
|
bucket + "/" + bucketVersioningConfig: enabledBucketVersioningConfig,
|
||||||
}))
|
}))
|
||||||
report := corsImportReport(t, rec).Buckets[bucket]
|
report := corsImportReport(t, rec).Buckets[bucket]
|
||||||
if report.Err != "" || report.Policy.Err != "" || report.Quota.Err != "" || report.Tagging.Err != "" {
|
if report.Err != "" || report.Policy.Err != "" || report.Quota.Err != "" || report.Tagging.Err != "" || report.Versioning.Err != "" {
|
||||||
t.Fatalf("import: %+v", report)
|
t.Fatalf("import: %+v", report)
|
||||||
}
|
}
|
||||||
meta, err = loadBucketMetadata(ctx, obj, bucket)
|
meta, err = loadBucketMetadata(ctx, obj, bucket)
|
||||||
|
|||||||
@@ -0,0 +1,519 @@
|
|||||||
|
// Copyright (c) 2026 Feng Ruohang
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"maps"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/madmin-go/v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Pause before the real IAM loader, which can itself read replication state.
|
||||||
|
// TryLock below proves that a configuration writer can proceed at this point
|
||||||
|
// without leaving deadlocked goroutines behind when this test regresses.
|
||||||
|
type siteStatusPausedStore struct {
|
||||||
|
IAMStorageAPI
|
||||||
|
once sync.Once
|
||||||
|
loading, proceed chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *siteStatusPausedStore) loadUsers(ctx context.Context, typ IAMUserType, users map[string]UserIdentity) error {
|
||||||
|
s.once.Do(func() { close(s.loading); <-s.proceed })
|
||||||
|
return s.IAMStorageAPI.loadUsers(ctx, typ, users)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSiteReplicationStatusReadsStorageUnlocked(t *testing.T) {
|
||||||
|
for _, entry := range []string{"metadata", "status", "heal"} {
|
||||||
|
for _, change := range []string{"disable", "topology", "cancel"} {
|
||||||
|
t.Run(entry+"/"+change, func(t *testing.T) {
|
||||||
|
site := newSvcAccSite(t)
|
||||||
|
c := &globalSiteReplicationSys
|
||||||
|
paused := &siteStatusPausedStore{IAMStorageAPI: site.sys.store.IAMStorageAPI, loading: make(chan struct{}), proceed: make(chan struct{})}
|
||||||
|
site.sys.store.IAMStorageAPI = paused
|
||||||
|
release := sync.OnceFunc(func() { close(paused.proceed) })
|
||||||
|
defer release()
|
||||||
|
c.iamMetaCache.Lock()
|
||||||
|
c.iamMetaCache.lastUpdate = time.Time{}
|
||||||
|
c.iamMetaCache.Unlock()
|
||||||
|
c.RLock()
|
||||||
|
peers := maps.Clone(c.state.Peers)
|
||||||
|
c.RUnlock()
|
||||||
|
ctx, cancel := context.WithCancel(site.ctx)
|
||||||
|
defer cancel()
|
||||||
|
var meta madmin.SRInfo
|
||||||
|
var status srStatusInfo
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
var err error
|
||||||
|
opts := madmin.SRStatusOptions{Users: true, Groups: true, Policies: true, PeerState: true}
|
||||||
|
switch entry {
|
||||||
|
case "metadata":
|
||||||
|
meta, err = c.SiteReplicationMetaInfo(ctx, site.obj, opts)
|
||||||
|
case "status":
|
||||||
|
status, err = c.siteReplicationStatus(ctx, site.obj, opts)
|
||||||
|
case "heal":
|
||||||
|
err = c.healIAMSystem(ctx, site.obj)
|
||||||
|
}
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-paused.loading:
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("did not read IAM storage: %v", err)
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("did not reach IAM loader")
|
||||||
|
}
|
||||||
|
if c.TryLock() {
|
||||||
|
switch change {
|
||||||
|
case "disable":
|
||||||
|
c.enabled = false
|
||||||
|
c.state = srState{}
|
||||||
|
case "topology":
|
||||||
|
delete(c.state.Peers, "remote")
|
||||||
|
c.state.UpdatedAt = UTCNow()
|
||||||
|
}
|
||||||
|
c.Unlock()
|
||||||
|
} else {
|
||||||
|
t.Error("replication configuration is locked during IAM storage read")
|
||||||
|
}
|
||||||
|
if change == "cancel" {
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if change != "cancel" {
|
||||||
|
mustIAM(t, err)
|
||||||
|
}
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("operation did not finish")
|
||||||
|
}
|
||||||
|
if change != "cancel" {
|
||||||
|
if entry == "metadata" && !reflect.DeepEqual(meta.State.Peers, peers) || entry == "status" && !reflect.DeepEqual(status.Sites, peers) {
|
||||||
|
t.Fatal("result topology changed after its snapshot")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSiteReplicationStatusRemoteUnlocked(t *testing.T) {
|
||||||
|
for _, fail := range []bool{false, true} {
|
||||||
|
t.Run(map[bool]string{false: "success", true: "peer-error"}[fail], func(t *testing.T) {
|
||||||
|
site := newSvcAccSite(t)
|
||||||
|
entered, proceed := make(chan struct{}), make(chan struct{})
|
||||||
|
var once sync.Once
|
||||||
|
peer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
once.Do(func() { close(entered) })
|
||||||
|
<-proceed
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
if fail {
|
||||||
|
w.WriteHeader(http.StatusForbidden)
|
||||||
|
_ = json.NewEncoder(w).Encode(madmin.ErrorResponse{Code: "AccessDenied", Message: "denied"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = json.NewEncoder(w).Encode(madmin.SRInfo{DeploymentID: "remote"})
|
||||||
|
}))
|
||||||
|
defer peer.Close()
|
||||||
|
release := sync.OnceFunc(func() { close(proceed) })
|
||||||
|
defer release()
|
||||||
|
c := &globalSiteReplicationSys
|
||||||
|
c.Lock()
|
||||||
|
c.state.Peers = site.peers(peer.URL)
|
||||||
|
c.state.UpdatedAt = UTCNow()
|
||||||
|
updated := c.state.UpdatedAt
|
||||||
|
c.Unlock()
|
||||||
|
var status srStatusInfo
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
var err error
|
||||||
|
status, err = c.siteReplicationStatus(site.ctx, site.obj, madmin.SRStatusOptions{})
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-entered:
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("did not request remote status: %v", err)
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("did not reach peer")
|
||||||
|
}
|
||||||
|
if c.TryLock() {
|
||||||
|
delete(c.state.Peers, "remote")
|
||||||
|
c.state.UpdatedAt = updated.Add(time.Second)
|
||||||
|
c.Unlock()
|
||||||
|
} else {
|
||||||
|
t.Error("replication configuration is locked during remote request")
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if fail {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "site remote (remote)") {
|
||||||
|
t.Fatalf("lost original peer name in error: %v", err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
mustIAM(t, err)
|
||||||
|
if len(status.Sites) != 2 || status.Sites["remote"].Endpoint != peer.URL || status.UpdatedAt != updated {
|
||||||
|
t.Fatal("status mixed different topology revisions")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("status did not finish")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSiteReplicationMetaInfoCachedPeerState(t *testing.T) {
|
||||||
|
site := newSvcAccSite(t)
|
||||||
|
c := &globalSiteReplicationSys
|
||||||
|
c.srCacheIAMInfo(madmin.SRInfo{DeploymentID: globalDeploymentID(), State: madmin.SRStateInfo{Name: "obsolete"}})
|
||||||
|
opts := madmin.SRStatusOptions{Users: true, Groups: true, Policies: true, PeerState: true}
|
||||||
|
got, err := c.SiteReplicationMetaInfo(site.ctx, site.obj, opts)
|
||||||
|
mustIAM(t, err)
|
||||||
|
if !reflect.DeepEqual(got.State.Peers, c.state.Peers) || got.State.Name != c.state.Name {
|
||||||
|
t.Fatal("cached IAM response returned stale peer state")
|
||||||
|
}
|
||||||
|
delete(got.State.Peers, "remote")
|
||||||
|
if len(c.state.Peers) != 2 {
|
||||||
|
t.Fatal("returned peer map aliases the replication configuration")
|
||||||
|
}
|
||||||
|
opts.PeerState = false
|
||||||
|
got, err = c.SiteReplicationMetaInfo(site.ctx, site.obj, opts)
|
||||||
|
mustIAM(t, err)
|
||||||
|
if got.State.Name != "" || got.State.Peers != nil {
|
||||||
|
t.Fatal("returned cached peer state that was not requested")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type siteBucketDeletePausedLayer struct {
|
||||||
|
ObjectLayer
|
||||||
|
once sync.Once
|
||||||
|
deleting, resume chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *siteBucketDeletePausedLayer) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
|
||||||
|
l.once.Do(func() { close(l.deleting); <-l.resume })
|
||||||
|
return l.ObjectLayer.DeleteBucket(ctx, bucket, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSiteReplicationHealAndHooksReleaseLock(t *testing.T) {
|
||||||
|
cases := []string{
|
||||||
|
"make-bucket-hook",
|
||||||
|
"delete-bucket-hook",
|
||||||
|
"bucket-meta-hook",
|
||||||
|
"iam-change-hook",
|
||||||
|
"configure-repl",
|
||||||
|
"peer-bucket-delete",
|
||||||
|
"heal-bucket-make",
|
||||||
|
"heal-bucket-delete",
|
||||||
|
"heal-bucket-repl-config",
|
||||||
|
"heal-bucket-config",
|
||||||
|
"heal-cors",
|
||||||
|
"heal-ilm-expiry-config",
|
||||||
|
"heal-bucket-ilm-expiry",
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc, func(t *testing.T) {
|
||||||
|
site := newSvcAccSite(t)
|
||||||
|
c := &globalSiteReplicationSys
|
||||||
|
const bucket = "sr-lock-test-bucket"
|
||||||
|
mustIAM(t, site.obj.MakeBucket(site.ctx, bucket, MakeBucketOptions{VersioningEnabled: true}))
|
||||||
|
_, err := globalBucketMetadataSys.Update(site.ctx, bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
|
||||||
|
mustIAM(t, err)
|
||||||
|
|
||||||
|
entered, proceed := make(chan struct{}), make(chan struct{})
|
||||||
|
var once sync.Once
|
||||||
|
pause := func() {
|
||||||
|
once.Do(func() { close(entered) })
|
||||||
|
<-proceed
|
||||||
|
}
|
||||||
|
release := sync.OnceFunc(func() { close(proceed) })
|
||||||
|
defer release()
|
||||||
|
|
||||||
|
peer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
pause()
|
||||||
|
if strings.HasSuffix(r.URL.Path, iamRevisionPeerPath) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_ = json.NewEncoder(w).Encode(iamRevisionResponse{
|
||||||
|
iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: "node", Instance: "boot", Digest: "fixture"},
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if r.Method == http.MethodGet {
|
||||||
|
if _, ok := r.URL.Query()["versioning"]; ok {
|
||||||
|
w.Header().Set("Content-Type", "application/xml")
|
||||||
|
_, _ = w.Write([]byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status></VersioningConfiguration>`))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, ok := r.URL.Query()["location"]; ok {
|
||||||
|
w.Header().Set("Content-Type", "application/xml")
|
||||||
|
_, _ = w.Write([]byte(`<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/"></LocationConstraint>`))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
defer peer.Close()
|
||||||
|
|
||||||
|
pausedObj := &siteBucketDeletePausedLayer{
|
||||||
|
ObjectLayer: site.obj,
|
||||||
|
deleting: entered,
|
||||||
|
resume: proceed,
|
||||||
|
}
|
||||||
|
if tc == "peer-bucket-delete" || tc == "heal-bucket-delete" {
|
||||||
|
setObjectLayer(pausedObj)
|
||||||
|
defer setObjectLayer(site.obj)
|
||||||
|
}
|
||||||
|
|
||||||
|
local := globalDeploymentID()
|
||||||
|
now := UTCNow().Truncate(time.Second)
|
||||||
|
peers := site.peers(peer.URL)
|
||||||
|
for id, p := range peers {
|
||||||
|
p.ReplicateILMExpiry = true
|
||||||
|
peers[id] = p
|
||||||
|
}
|
||||||
|
c.Lock()
|
||||||
|
c.enabled = true
|
||||||
|
c.state = srState{
|
||||||
|
Name: "local",
|
||||||
|
ServiceAccountAccessKey: site.parent.AccessKey,
|
||||||
|
Peers: peers,
|
||||||
|
UpdatedAt: now,
|
||||||
|
}
|
||||||
|
c.Unlock()
|
||||||
|
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
switch tc {
|
||||||
|
case "make-bucket-hook":
|
||||||
|
done <- c.MakeBucketHook(site.ctx, bucket, MakeBucketOptions{CreatedAt: now})
|
||||||
|
case "delete-bucket-hook":
|
||||||
|
done <- c.DeleteBucketHook(site.ctx, bucket, true)
|
||||||
|
case "bucket-meta-hook":
|
||||||
|
done <- c.BucketMetaHook(site.ctx, madmin.SRBucketMeta{
|
||||||
|
Type: madmin.SRBucketMetaTypeTags,
|
||||||
|
Bucket: bucket,
|
||||||
|
UpdatedAt: now,
|
||||||
|
})
|
||||||
|
case "iam-change-hook":
|
||||||
|
site.create(t, "sr-lock-hook-sa")
|
||||||
|
done <- c.IAMChangeHook(site.ctx, madmin.SRIAMItem{
|
||||||
|
Type: madmin.SRIAMItemSvcAcc,
|
||||||
|
SvcAccChange: &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{AccessKey: "sr-lock-hook-sa"}},
|
||||||
|
UpdatedAt: now,
|
||||||
|
})
|
||||||
|
case "configure-repl":
|
||||||
|
done <- c.PeerBucketConfigureReplHandler(site.ctx, bucket)
|
||||||
|
case "peer-bucket-delete":
|
||||||
|
done <- c.PeerBucketDeleteHandler(site.ctx, bucket, DeleteBucketOptions{Force: true})
|
||||||
|
case "heal-bucket-make":
|
||||||
|
info := srStatusInfo{
|
||||||
|
Sites: site.peers(peer.URL),
|
||||||
|
BucketStats: map[string]map[string]srBucketStatsSummary{
|
||||||
|
bucket: {
|
||||||
|
local: {HasBucket: true, meta: srBucketMetaInfo{SRBucketInfo: madmin.SRBucketInfo{CreatedAt: now}}},
|
||||||
|
"remote": {HasBucket: false},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
done <- c.healBucket(site.ctx, site.obj, bucket, info)
|
||||||
|
case "heal-bucket-delete":
|
||||||
|
info := srStatusInfo{
|
||||||
|
Sites: site.peers(peer.URL),
|
||||||
|
BucketStats: map[string]map[string]srBucketStatsSummary{
|
||||||
|
bucket: {
|
||||||
|
local: {HasBucket: true, meta: srBucketMetaInfo{SRBucketInfo: madmin.SRBucketInfo{CreatedAt: now.Add(-time.Minute), DeletedAt: now}}},
|
||||||
|
"remote": {HasBucket: false, BucketMarkedDeleted: true, meta: srBucketMetaInfo{SRBucketInfo: madmin.SRBucketInfo{DeletedAt: now.Add(-2 * time.Minute)}}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
done <- c.healBucket(site.ctx, pausedObj, bucket, info)
|
||||||
|
case "heal-bucket-repl-config":
|
||||||
|
info := srStatusInfo{
|
||||||
|
Sites: site.peers(peer.URL),
|
||||||
|
BucketStats: map[string]map[string]srBucketStatsSummary{
|
||||||
|
bucket: {
|
||||||
|
local: {HasBucket: true, ReplicationCfgMismatch: true},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
done <- c.healBucketReplicationConfig(site.ctx, site.obj, bucket, info, &validateReplicationDestinationOptions{CheckReady: true})
|
||||||
|
case "heal-bucket-config":
|
||||||
|
tagPayload := base64.StdEncoding.EncodeToString([]byte(`<Tagging><TagSet><Tag><Key>k</Key><Value>v</Value></Tag></TagSet></Tagging>`))
|
||||||
|
info := srStatusInfo{
|
||||||
|
Sites: site.peers(peer.URL),
|
||||||
|
BucketStats: map[string]map[string]srBucketStatsSummary{
|
||||||
|
bucket: {
|
||||||
|
local: {HasBucket: true, meta: srBucketMetaInfo{SRBucketInfo: madmin.SRBucketInfo{CreatedAt: now.Add(-time.Minute), Tags: &tagPayload, TagConfigUpdatedAt: now}}},
|
||||||
|
"remote": {HasBucket: true, meta: srBucketMetaInfo{SRBucketInfo: madmin.SRBucketInfo{CreatedAt: now.Add(-time.Minute)}}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
done <- c.healBucketConfig(site.ctx, bucket, bucketTaggingConfig, info)
|
||||||
|
case "heal-cors":
|
||||||
|
corsPayload := base64.StdEncoding.EncodeToString([]byte(`<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`))
|
||||||
|
info := srStatusInfo{
|
||||||
|
Sites: site.peers(peer.URL),
|
||||||
|
BucketStats: map[string]map[string]srBucketStatsSummary{
|
||||||
|
bucket: {
|
||||||
|
local: {HasBucket: true, meta: srBucketMetaInfo{SRBucketInfo: madmin.SRBucketInfo{CreatedAt: now.Add(-time.Minute), CorsConfig: &corsPayload, CorsConfigUpdatedAt: now}}},
|
||||||
|
"remote": {HasBucket: true, meta: srBucketMetaInfo{SRBucketInfo: madmin.SRBucketInfo{CreatedAt: now.Add(-time.Minute)}}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
done <- c.healCORSMetadata(site.ctx, site.obj, bucket, info)
|
||||||
|
case "heal-ilm-expiry-config":
|
||||||
|
stalePeers := site.peers(peer.URL)
|
||||||
|
for id, p := range stalePeers {
|
||||||
|
p.ReplicateILMExpiry = false
|
||||||
|
stalePeers[id] = p
|
||||||
|
}
|
||||||
|
info := srStatusInfo{
|
||||||
|
Sites: site.peers(peer.URL),
|
||||||
|
PeerStates: map[string]madmin.SRStateInfo{
|
||||||
|
local: {Name: "local", Peers: peers, UpdatedAt: now},
|
||||||
|
"remote": {Name: "remote", Peers: stalePeers, UpdatedAt: now.Add(-time.Minute)},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
done <- c.healILMExpiryConfig(site.ctx, site.obj, info)
|
||||||
|
case "heal-bucket-ilm-expiry":
|
||||||
|
lcXML := `<LifecycleConfiguration><Rule><ID>exp</ID><Status>Enabled</Status><Expiration><Days>1</Days></Expiration></Rule></LifecycleConfiguration>`
|
||||||
|
lcPayload := base64.StdEncoding.EncodeToString([]byte(lcXML))
|
||||||
|
info := srStatusInfo{
|
||||||
|
Sites: site.peers(peer.URL),
|
||||||
|
BucketStats: map[string]map[string]srBucketStatsSummary{
|
||||||
|
bucket: {
|
||||||
|
local: {HasBucket: true, meta: srBucketMetaInfo{SRBucketInfo: madmin.SRBucketInfo{CreatedAt: now.Add(-time.Minute), ExpiryLCConfig: &lcPayload, ExpiryLCConfigUpdatedAt: now}}},
|
||||||
|
"remote": {HasBucket: true, meta: srBucketMetaInfo{SRBucketInfo: madmin.SRBucketInfo{CreatedAt: now.Add(-time.Minute)}}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
done <- c.healBucketILMExpiry(site.ctx, site.obj, bucket, info)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-entered:
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("operation returned before reaching storage/peer pause point: %v", err)
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("timed out waiting for storage/peer pause point")
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.TryLock() {
|
||||||
|
c.state.Name = "mutated-while-in-flight"
|
||||||
|
delete(c.state.Peers, "remote")
|
||||||
|
c.state.UpdatedAt = now.Add(time.Second)
|
||||||
|
c.Unlock()
|
||||||
|
} else {
|
||||||
|
t.Error("SiteReplicationSys lock held across storage I/O or peer RPC")
|
||||||
|
}
|
||||||
|
|
||||||
|
release()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("operation deadlocked after writer mutation")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSiteReplicationConcurrentStateUpdates(t *testing.T) {
|
||||||
|
site := newSvcAccSite(t)
|
||||||
|
c := &globalSiteReplicationSys
|
||||||
|
const bucket = "sr-concurrent-state-bucket"
|
||||||
|
mustIAM(t, site.obj.MakeBucket(site.ctx, bucket, MakeBucketOptions{VersioningEnabled: true}))
|
||||||
|
_, err := globalBucketMetadataSys.Update(site.ctx, bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
|
||||||
|
mustIAM(t, err)
|
||||||
|
|
||||||
|
peer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if strings.Contains(r.URL.Path, "metainfo") {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_ = json.NewEncoder(w).Encode(madmin.SRInfo{DeploymentID: "remote"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if r.Method == http.MethodGet {
|
||||||
|
if _, ok := r.URL.Query()["versioning"]; ok {
|
||||||
|
w.Header().Set("Content-Type", "application/xml")
|
||||||
|
_, _ = w.Write([]byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status></VersioningConfiguration>`))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, ok := r.URL.Query()["location"]; ok {
|
||||||
|
w.Header().Set("Content-Type", "application/xml")
|
||||||
|
_, _ = w.Write([]byte(`<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/"></LocationConstraint>`))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
defer peer.Close()
|
||||||
|
|
||||||
|
basePeers := site.peers(peer.URL)
|
||||||
|
now := UTCNow().Truncate(time.Second)
|
||||||
|
c.Lock()
|
||||||
|
c.enabled = true
|
||||||
|
c.state = srState{
|
||||||
|
Name: "local",
|
||||||
|
ServiceAccountAccessKey: site.parent.AccessKey,
|
||||||
|
Peers: maps.Clone(basePeers),
|
||||||
|
UpdatedAt: now,
|
||||||
|
}
|
||||||
|
c.Unlock()
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
wg.Add(4)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for i := range 10 {
|
||||||
|
_ = c.PeerEditReq(site.ctx, madmin.PeerInfo{
|
||||||
|
DeploymentID: "remote",
|
||||||
|
Name: "remote",
|
||||||
|
Endpoint: peer.URL,
|
||||||
|
ReplicateILMExpiry: i%2 == 0,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for i := range 10 {
|
||||||
|
peers := site.peers(peer.URL)
|
||||||
|
p := peers["remote"]
|
||||||
|
p.ReplicateILMExpiry = i%2 == 1
|
||||||
|
peers["remote"] = p
|
||||||
|
_ = c.PeerStateEditReq(site.ctx, madmin.SRStateEditReq{
|
||||||
|
Peers: peers,
|
||||||
|
UpdatedAt: now.Add(time.Duration(i+1) * time.Second),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for range 10 {
|
||||||
|
_ = c.PeerBucketConfigureReplHandler(site.ctx, bucket)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for range 5 {
|
||||||
|
_ = c.healBuckets(site.ctx, site.obj)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
wg.Wait()
|
||||||
|
}
|
||||||
+266
-277
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,187 @@
|
|||||||
|
// Copyright (c) 2026 Feng Ruohang
|
||||||
|
//
|
||||||
|
// This file is part of Silo Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/xml"
|
||||||
|
"maps"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
"github.com/minio/minio/internal/crypto"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
"github.com/minio/minio/internal/kms"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A successful encrypted write must stay readable. Malformed context must be
|
||||||
|
// rejected before replacing an existing object or creating a multipart upload.
|
||||||
|
func TestAPISSEKMSHTTPContext(t *testing.T) {
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
// Specific PUT routes must precede the generic PutObject route.
|
||||||
|
endpoints: []string{"CopyObject", "NewMultipart", "PutObjectPart", "CompleteMultipart", "PutObject", "GetObject"},
|
||||||
|
objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
|
||||||
|
previousKMS, previousTLS := GlobalKMS, globalIsTLS
|
||||||
|
var err error
|
||||||
|
GlobalKMS, err = kms.NewBuiltin("context-key", bytes.Repeat([]byte{0x42}, 32))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
globalIsTLS = true
|
||||||
|
defer func() { GlobalKMS, globalIsTLS = previousKMS, previousTLS }()
|
||||||
|
request := func(t *testing.T, method, target string, body []byte, headers map[string]string) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
|
req, err := newTestSignedRequestV4(method, target, int64(len(body)), bytes.NewReader(body), cred.AccessKey, cred.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, req)
|
||||||
|
return rec
|
||||||
|
}
|
||||||
|
expectStatus := func(t *testing.T, rec *httptest.ResponseRecorder, want int) {
|
||||||
|
t.Helper()
|
||||||
|
if rec.Code != want {
|
||||||
|
t.Fatalf("status=%d, want %d: %s", rec.Code, want, rec.Body.String())
|
||||||
|
}
|
||||||
|
if want == http.StatusBadRequest {
|
||||||
|
var response APIErrorResponse
|
||||||
|
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil || response.Code != "InvalidArgument" {
|
||||||
|
t.Fatalf("expected InvalidArgument: %s (decode: %v)", rec.Body.String(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
encryption := func(raw string) map[string]string {
|
||||||
|
return map[string]string{
|
||||||
|
xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionKMS,
|
||||||
|
xhttp.AmzServerSideEncryptionKmsID: "context-key",
|
||||||
|
xhttp.AmzServerSideEncryptionKmsContext: base64.StdEncoding.EncodeToString([]byte(raw)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, operation := range []string{"put", "self-copy", "copy-replace", "cross-key-copy", "multipart"} {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, raw string
|
||||||
|
invalid bool
|
||||||
|
}{
|
||||||
|
{"unicode", `{"区域":"杭州","emoji":"🔑","�":"replacement"}`, false},
|
||||||
|
{"array", `[]`, true},
|
||||||
|
{"invalid-json", `not-json`, true},
|
||||||
|
{"trailing-json", `{"tenant":"ok"} {}`, true},
|
||||||
|
{"number-value", `{"tenant":1}`, true},
|
||||||
|
{"invalid-base64", "", true},
|
||||||
|
{"invalid-key", "{\"\xff\":\"value\"}", true},
|
||||||
|
{"invalid-value", "{\"tenant\":\"\xff\"}", true},
|
||||||
|
{"colliding-keys", "{\"\xff\":\"a\",\"\xfe\":\"b\"}", true},
|
||||||
|
} {
|
||||||
|
t.Run(backend+"/"+operation+"/"+tc.name, func(t *testing.T) {
|
||||||
|
key := operation + "-" + tc.name
|
||||||
|
original, replacement := []byte("original encrypted object"), []byte("updated encrypted object")
|
||||||
|
seed := request(t, http.MethodPut, getPutObjectURL("", bucket, key), original, encryption(`{"tenant":"original"}`))
|
||||||
|
expectStatus(t, seed, http.StatusOK)
|
||||||
|
before, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
headers := encryption(tc.raw)
|
||||||
|
if tc.name == "invalid-base64" {
|
||||||
|
headers[xhttp.AmzServerSideEncryptionKmsContext] = "not-base64!"
|
||||||
|
}
|
||||||
|
wantData := replacement
|
||||||
|
var rec *httptest.ResponseRecorder
|
||||||
|
switch operation {
|
||||||
|
case "put":
|
||||||
|
rec = request(t, http.MethodPut, getPutObjectURL("", bucket, key), replacement, headers)
|
||||||
|
case "self-copy", "copy-replace", "cross-key-copy":
|
||||||
|
source := key
|
||||||
|
if operation == "cross-key-copy" {
|
||||||
|
source += "-source"
|
||||||
|
seed = request(t, http.MethodPut, getPutObjectURL("", bucket, source), replacement, encryption(`{"tenant":"source"}`))
|
||||||
|
expectStatus(t, seed, http.StatusOK)
|
||||||
|
} else {
|
||||||
|
wantData = original
|
||||||
|
}
|
||||||
|
if operation == "copy-replace" {
|
||||||
|
headers[xhttp.AmzMetadataDirective] = "REPLACE"
|
||||||
|
}
|
||||||
|
headers[xhttp.AmzCopySource] = "/" + bucket + "/" + source
|
||||||
|
rec = request(t, http.MethodPut, getCopyObjectURL("", bucket, key), nil, headers)
|
||||||
|
case "multipart":
|
||||||
|
rec = request(t, http.MethodPost, getNewMultipartURL("", bucket, key), nil, headers)
|
||||||
|
if !tc.invalid {
|
||||||
|
expectStatus(t, rec, http.StatusOK)
|
||||||
|
var upload InitiateMultipartUploadResponse
|
||||||
|
if err := xml.Unmarshal(rec.Body.Bytes(), &upload); err != nil || upload.UploadID == "" {
|
||||||
|
t.Fatalf("missing upload ID: %s (decode: %v)", rec.Body.String(), err)
|
||||||
|
}
|
||||||
|
part := request(t, http.MethodPut, getPutObjectPartURL("", bucket, key, upload.UploadID, "1"), replacement, nil)
|
||||||
|
expectStatus(t, part, http.StatusOK)
|
||||||
|
// The in-process recorder retains the handler's literal
|
||||||
|
// ETag key; Header.Get would canonicalize it to Etag.
|
||||||
|
etags := part.Header()[xhttp.ETag]
|
||||||
|
if len(etags) != 1 || etags[0] == "" {
|
||||||
|
t.Fatalf("missing part ETag: %v", part.Header())
|
||||||
|
}
|
||||||
|
body, err := xml.Marshal(CompleteMultipartUpload{Parts: []CompletePart{{PartNumber: 1, ETag: strings.Trim(etags[0], "\"")}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec = request(t, http.MethodPost, getCompleteMultipartUploadURL("", bucket, key, upload.UploadID), body, nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
wantStatus := http.StatusOK
|
||||||
|
if tc.invalid {
|
||||||
|
wantStatus, wantData = http.StatusBadRequest, original
|
||||||
|
}
|
||||||
|
expectStatus(t, rec, wantStatus)
|
||||||
|
get := request(t, http.MethodGet, getGetObjectURL("", bucket, key), nil, nil)
|
||||||
|
expectStatus(t, get, http.StatusOK)
|
||||||
|
if !bytes.Equal(get.Body.Bytes(), wantData) {
|
||||||
|
t.Fatalf("GET returned %q, want %q", get.Body.Bytes(), wantData)
|
||||||
|
}
|
||||||
|
after, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if tc.invalid {
|
||||||
|
if operation == "multipart" {
|
||||||
|
uploads, err := obj.ListMultipartUploads(t.Context(), bucket, key, "", "", "", 100)
|
||||||
|
if err != nil || len(uploads.Uploads) != 0 {
|
||||||
|
t.Fatalf("rejected context created multipart upload: %+v, %v", uploads, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !maps.Equal(before.UserDefined, after.UserDefined) || !before.ModTime.Equal(after.ModTime) {
|
||||||
|
t.Fatal("rejected context changed existing object metadata")
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
_, _, _, got, err := crypto.S3KMS.ParseMetadata(after.UserDefined)
|
||||||
|
want := kms.Context{"区域": "杭州", "emoji": "🔑", "�": "replacement"}
|
||||||
|
if err != nil || !maps.Equal(got, want) {
|
||||||
|
t.Fatalf("stored context=%v error=%v, want %v", got, err, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user