mirror of
https://github.com/pgsty/minio.git
synced 2026-09-08 19:44:03 +03:00
Merge branch 'main' into feat/access-based-ilm
This commit is contained in:
@@ -1,54 +0,0 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: community, triage
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
## NOTE
|
||||
|
||||
Silo issues are handled by community maintainers on a best-effort basis. There
|
||||
is no SLA, SLO, or emergency production-support channel. Follow the local
|
||||
[Code of Conduct](../code_of_conduct.md) when participating. Report suspected
|
||||
vulnerabilities through the private process in [SECURITY.md](../SECURITY.md),
|
||||
not in a public issue.
|
||||
|
||||
<!--- Provide a general summary of the issue in the Title above -->
|
||||
|
||||
## Expected Behavior
|
||||
<!--- If you're describing a bug, tell us what should happen -->
|
||||
<!--- If you're suggesting a change/improvement, tell us how it should work -->
|
||||
|
||||
## Current Behavior
|
||||
<!--- If describing a bug, tell us what happens instead of the expected behavior -->
|
||||
<!--- If suggesting a change/improvement, explain the difference from current behavior -->
|
||||
|
||||
## Possible Solution
|
||||
<!--- Not obligatory, but suggest a fix/reason for the bug, -->
|
||||
<!--- or ideas how to implement the addition or change -->
|
||||
|
||||
## Steps to Reproduce (for bugs)
|
||||
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
|
||||
<!--- reproduce this bug. Include code to reproduce, if relevant -->
|
||||
<!--- and include relevant Silo logs with secrets and credentials removed -->
|
||||
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
4.
|
||||
|
||||
## Context
|
||||
<!--- How has this issue affected you? What are you trying to accomplish? -->
|
||||
<!--- Providing context helps us come up with a solution that is most useful in the real world -->
|
||||
|
||||
## Regression
|
||||
<!-- Is this issue a regression? (Yes / No) -->
|
||||
<!-- If Yes, optionally include the Silo version, commit id, or PR that caused this regression. -->
|
||||
|
||||
## Your Environment
|
||||
<!--- Include as many relevant details about the environment you experienced the bug in -->
|
||||
* Version used (`silo --version`):
|
||||
* Server setup and configuration:
|
||||
* Operating System and version (`uname -a`):
|
||||
@@ -7,6 +7,14 @@ assignees: ''
|
||||
|
||||
---
|
||||
|
||||
Report bugs in the PGSTY SILO server (`pgsty/silo`) here. Community maintainers
|
||||
handle reports on a best-effort basis. There is no SLA, SLO, or emergency
|
||||
production-support channel. Follow the
|
||||
[Code of Conduct](https://github.com/pgsty/silo/blob/main/code_of_conduct.md).
|
||||
Report suspected vulnerabilities privately through
|
||||
[SECURITY.md](https://github.com/pgsty/silo/blob/main/SECURITY.md).
|
||||
For patches, see the [contribution guide](https://github.com/pgsty/silo/blob/main/CONTRIBUTING.md).
|
||||
|
||||
<!--- Provide a general summary of the issue in the Title above -->
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
@@ -7,6 +7,9 @@ assignees: ''
|
||||
|
||||
---
|
||||
|
||||
Suggest improvements to the PGSTY SILO server (`pgsty/silo`) here.
|
||||
For patches, see the [contribution guide](https://github.com/pgsty/silo/blob/main/CONTRIBUTING.md).
|
||||
|
||||
**Is your feature request related to a problem? Please describe.**
|
||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
||||
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
## Contribution Licensing (no CLA, inbound=outbound, DCO required)
|
||||
|
||||
This project does not use a CLA; contributions are accepted inbound=outbound.
|
||||
This pull request contributes to PGSTY SILO (`pgsty/silo`). Code contributions
|
||||
are accepted under AGPL-3.0-or-later, the same license as the server.
|
||||
This project does not use a CLA or require a separate Apache-2.0 license grant.
|
||||
By submitting this pull request I represent that I have the right to contribute
|
||||
the changes, which are licensed under this repository's
|
||||
the code changes under this repository's
|
||||
[GNU Affero General Public License v3.0 or later](https://www.gnu.org/licenses/agpl-3.0.html)
|
||||
and remain my copyright. Every commit must carry a DCO `Signed-off-by` trailer
|
||||
and retain copyright in my original work. Existing copyright and license
|
||||
notices remain intact; separately licensed material keeps its applicable terms.
|
||||
Every commit must carry a DCO `Signed-off-by` trailer
|
||||
(`git commit -s`) certifying the
|
||||
[Developer Certificate of Origin](https://developercertificate.org/) — see
|
||||
[CONTRIBUTING.md](https://github.com/pgsty/silo/blob/main/CONTRIBUTING.md).
|
||||
|
||||
@@ -77,8 +77,10 @@ release:
|
||||
name: silo
|
||||
draft: true
|
||||
prerelease: false
|
||||
mode: append
|
||||
replace_existing_draft: true
|
||||
replace_existing_artifacts: false
|
||||
mode: replace
|
||||
# Draft replacement matches the release name; keep it identical to the tag.
|
||||
name_template: "{{ .Tag }}"
|
||||
|
||||
changelog:
|
||||
|
||||
@@ -7,6 +7,11 @@ on:
|
||||
description: "Published RELEASE.* tag to package as pgsty/silo"
|
||||
required: true
|
||||
type: string
|
||||
recovery:
|
||||
description: "Run the current main workflow against an already-published tag"
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -75,12 +80,18 @@ jobs:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Verify workflow identity matches release source
|
||||
env:
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
RECOVERY: ${{ inputs.recovery }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
CHECKED_OUT_REVISION="$(git rev-parse HEAD)"
|
||||
if [ "${CHECKED_OUT_REVISION}" != "${GITHUB_SHA}" ]; then
|
||||
echo "Checked out ${CHECKED_OUT_REVISION}, but workflow identity is ${GITHUB_SHA}. Dispatch this workflow from ${RELEASE_TAG}." >&2
|
||||
exit 1
|
||||
if [ "${RECOVERY}" != "true" ] || [ "${GITHUB_REF}" != "refs/heads/${DEFAULT_BRANCH}" ]; then
|
||||
echo "Checked out ${CHECKED_OUT_REVISION}, but workflow identity is ${GITHUB_SHA}. Dispatch from ${RELEASE_TAG}, or use recovery from ${DEFAULT_BRANCH}." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Recovery workflow ${GITHUB_SHA} is packaging published source ${CHECKED_OUT_REVISION}."
|
||||
fi
|
||||
|
||||
- name: Prepare verified Docker contexts
|
||||
@@ -133,6 +144,46 @@ jobs:
|
||||
"${context}/dockerscripts/"
|
||||
done
|
||||
|
||||
# The classic image bundles mcli. Resolve its two archive digests
|
||||
# from the immutable published release instead of trusting defaults
|
||||
# copied into an older Server tag. This also gives a recovery run a
|
||||
# narrow override when a tag selected the right mcli release but
|
||||
# accidentally retained stale archive pins.
|
||||
MC_REPO="$(awk -F= '/^ARG MC_REPO=/{print $2; exit}' Dockerfile.goreleaser)"
|
||||
MC_VERSION="$(awk -F= '/^ARG MC_VERSION=/{print $2; exit}' Dockerfile.goreleaser)"
|
||||
test -n "${MC_REPO}"
|
||||
test -n "${MC_VERSION}"
|
||||
MC_VERSION_HYPHEN="${MC_VERSION#RELEASE.}"
|
||||
MC_PKG_VERSION="$(echo "${MC_VERSION_HYPHEN}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')"
|
||||
if [ "${MC_PKG_VERSION}" = "${MC_VERSION_HYPHEN}" ]; then
|
||||
echo "Invalid bundled mcli tag: ${MC_VERSION}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$(gh release view "${MC_VERSION}" --repo "${MC_REPO}" --json isDraft --jq .isDraft)" != false ] || \
|
||||
[ "$(gh release view "${MC_VERSION}" --repo "${MC_REPO}" --json isPrerelease --jq .isPrerelease)" != false ] || \
|
||||
[ "$(gh release view "${MC_VERSION}" --repo "${MC_REPO}" --json isImmutable --jq .isImmutable)" != true ]; then
|
||||
echo "Bundled mcli ${MC_REPO}@${MC_VERSION} must be a published immutable release" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mc_checksums="mcli_${MC_PKG_VERSION}_checksums.txt"
|
||||
gh release download "${MC_VERSION}" --repo "${MC_REPO}" \
|
||||
--dir "${assets_dir}" --pattern "${mc_checksums}"
|
||||
gh attestation verify "${assets_dir}/${mc_checksums}" \
|
||||
--repo "${MC_REPO}" \
|
||||
--signer-workflow "${MC_REPO}/.github/workflows/release.yml" \
|
||||
--source-ref "refs/tags/${MC_VERSION}" >/dev/null
|
||||
|
||||
MC_AMD64_SHA256="$(awk -v name="mcli_${MC_PKG_VERSION}_linux_amd64.tar.gz" '$2 == name {print $1}' "${assets_dir}/${mc_checksums}")"
|
||||
MC_ARM64_SHA256="$(awk -v name="mcli_${MC_PKG_VERSION}_linux_arm64.tar.gz" '$2 == name {print $1}' "${assets_dir}/${mc_checksums}")"
|
||||
[[ "${MC_AMD64_SHA256}" =~ ^[0-9a-f]{64}$ ]]
|
||||
[[ "${MC_ARM64_SHA256}" =~ ^[0-9a-f]{64}$ ]]
|
||||
|
||||
{
|
||||
echo "MC_AMD64_SHA256=${MC_AMD64_SHA256}"
|
||||
echo "MC_ARM64_SHA256=${MC_ARM64_SHA256}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
echo "RELEASE_REVISION=$(git rev-parse HEAD)" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Set up QEMU
|
||||
@@ -162,6 +213,9 @@ jobs:
|
||||
file: docker-release/amd64/Dockerfile.goreleaser
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
build-args: |
|
||||
MC_AMD64_SHA256=${{ env.MC_AMD64_SHA256 }}
|
||||
MC_ARM64_SHA256=${{ env.MC_ARM64_SHA256 }}
|
||||
tags: |
|
||||
pgsty/silo:${{ env.RELEASE_TAG }}-amd64
|
||||
pgsty/silo:latest-amd64
|
||||
@@ -178,6 +232,9 @@ jobs:
|
||||
file: docker-release/arm64/Dockerfile.goreleaser
|
||||
platforms: linux/arm64
|
||||
push: true
|
||||
build-args: |
|
||||
MC_AMD64_SHA256=${{ env.MC_AMD64_SHA256 }}
|
||||
MC_ARM64_SHA256=${{ env.MC_ARM64_SHA256 }}
|
||||
tags: |
|
||||
pgsty/silo:${{ env.RELEASE_TAG }}-arm64
|
||||
pgsty/silo:latest-arm64
|
||||
|
||||
@@ -23,7 +23,7 @@ permissions:
|
||||
artifact-metadata: write
|
||||
|
||||
concurrency:
|
||||
group: finalize-release
|
||||
group: release-${{ inputs.tag }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
name: Release
|
||||
|
||||
# Retry contract: an absent or single unfinalized Draft may be rebuilt from
|
||||
# scratch; a published release or a Draft carrying finalize's GPG-derived
|
||||
# provenance marker is terminal for this lane. The per-tag lock serializes
|
||||
# workflows, but a maintainer must not publish the Draft while this job runs.
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
@@ -16,6 +21,10 @@ permissions:
|
||||
attestations: write
|
||||
artifact-metadata: write
|
||||
|
||||
concurrency:
|
||||
group: release-${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -78,6 +87,15 @@ jobs:
|
||||
echo "Invalid release tag format: ${TAG}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! TAG_COMMIT="$(git rev-parse "${TAG}^{commit}" 2>/dev/null)"; then
|
||||
echo "Release tag ${TAG} does not resolve to a commit" >&2
|
||||
exit 1
|
||||
fi
|
||||
HEAD_COMMIT="$(git rev-parse HEAD)"
|
||||
if [ "${TAG_COMMIT}" != "${HEAD_COMMIT}" ]; then
|
||||
echo "Release tag ${TAG} resolves to ${TAG_COMMIT}, checkout is ${HEAD_COMMIT}" >&2
|
||||
exit 1
|
||||
fi
|
||||
VERSION_HYPHEN="${TAG#RELEASE.}"
|
||||
PKG_VERSION="$(echo "${VERSION_HYPHEN}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')"
|
||||
VERSION_COLON="$(echo "${VERSION_HYPHEN}" | sed -E 's/T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/T\1:\2:\3Z/')"
|
||||
@@ -93,6 +111,13 @@ jobs:
|
||||
echo "Package version: ${PKG_VERSION}"
|
||||
echo "LDFLAGS: ${LDFLAGS}"
|
||||
|
||||
- name: Check existing release state
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
buildscripts/check-release-state.sh "${RELEASE_TAG}"
|
||||
|
||||
# Both installer actions are pinned to immutable commits. The explicit
|
||||
# tool versions keep the release format reproducible across workflow
|
||||
# reruns while the installers verify the downloaded executables.
|
||||
@@ -113,6 +138,7 @@ jobs:
|
||||
args: release --clean --skip=validate --config .github/goreleaser.yml
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GORELEASER_CURRENT_TAG: ${{ env.RELEASE_TAG }}
|
||||
LDFLAGS: ${{ env.LDFLAGS }}
|
||||
PKG_VERSION: ${{ env.PKG_VERSION }}
|
||||
|
||||
@@ -189,6 +215,14 @@ jobs:
|
||||
test -s "${BUNDLE_PATH}"
|
||||
cp "${BUNDLE_PATH}" "dist/silo_${PKG_VERSION}_provenance.sigstore.json"
|
||||
|
||||
- name: Confirm unfinalized Draft release state
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REQUIRE_DRAFT: "true"
|
||||
run: |
|
||||
set -euo pipefail
|
||||
buildscripts/check-release-state.sh "${RELEASE_TAG}"
|
||||
|
||||
- name: Upload nFPM packages to Draft release
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -23,6 +23,8 @@ on:
|
||||
- "buildscripts/minio-upgrade.sh"
|
||||
- "buildscripts/sign-release-rpms.sh"
|
||||
- "buildscripts/verify-build-provenance.sh"
|
||||
- "buildscripts/check-release-state.sh"
|
||||
- "buildscripts/check-release-state_test.sh"
|
||||
- "buildscripts/verify-rebrand.sh"
|
||||
- "buildscripts/verify-helm-migration.sh"
|
||||
- "buildscripts/helm-migration-guard/**"
|
||||
@@ -479,6 +481,8 @@ jobs:
|
||||
bash -n buildscripts/minio-upgrade.sh
|
||||
bash -n buildscripts/sign-release-rpms.sh
|
||||
bash -n buildscripts/verify-build-provenance.sh
|
||||
bash -n buildscripts/check-release-state.sh
|
||||
bash -n buildscripts/check-release-state_test.sh
|
||||
bash -n buildscripts/verify-rebrand.sh
|
||||
bash -n buildscripts/verify-helm-migration.sh
|
||||
sh -n buildscripts/package/postinstall.sh
|
||||
@@ -493,9 +497,12 @@ jobs:
|
||||
test -x buildscripts/package-release.sh
|
||||
test -x buildscripts/sign-release-rpms.sh
|
||||
test -x buildscripts/verify-build-provenance.sh
|
||||
test -x buildscripts/check-release-state.sh
|
||||
test -x buildscripts/check-release-state_test.sh
|
||||
test -x buildscripts/verify-rebrand.sh
|
||||
test -x buildscripts/verify-helm-migration.sh
|
||||
test -x buildscripts/package/postinstall.sh
|
||||
test -x buildscripts/package/preremove.sh
|
||||
test -x buildscripts/package/lifecycle_test.sh
|
||||
test -x dockerscripts/docker-entrypoint_test.sh
|
||||
buildscripts/check-release-state_test.sh
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
|
||||
- name: Install govulncheck
|
||||
run: |
|
||||
go install golang.org/x/vuln/cmd/govulncheck@v1.6.0
|
||||
go install golang.org/x/vuln/cmd/govulncheck@v1.7.0
|
||||
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Run govulncheck
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@ linters:
|
||||
- durationcheck
|
||||
- forcetypeassert
|
||||
- gocritic
|
||||
- gomodguard
|
||||
- gomodguard_v2
|
||||
- govet
|
||||
- ineffassign
|
||||
- misspell
|
||||
|
||||
+18
-8
@@ -44,6 +44,10 @@ Before opening a pull request:
|
||||
|
||||
- Add or update tests for changed behavior.
|
||||
- Run `make verifiers`.
|
||||
- If `make rebrand-guard` reports a changed compatibility set, review the
|
||||
listed identifiers; when the change is intended, refresh the baseline with
|
||||
`go run ./buildscripts/rebrand-guard --write` and commit
|
||||
`buildscripts/rebrand-guard/compat-baseline.json`.
|
||||
- Run the smallest relevant package tests, then `make test` when practical.
|
||||
- Run `make build` and confirm the generated executable is `silo`.
|
||||
- Explain any preserved `MINIO_*`, `minio_*`, `x-minio-*`, `/minio/*`,
|
||||
@@ -75,9 +79,10 @@ documentation is owned by the separate
|
||||
|
||||
## Licensing of Contributions
|
||||
|
||||
Silo is licensed under the [GNU AGPL v3.0 or later](LICENSE). Its core is
|
||||
Copyright (c) MinIO, Inc.; the combined work can never be relicensed, and this
|
||||
fork does not try to.
|
||||
Code contributions to PGSTY SILO (`pgsty/silo`) are accepted under the
|
||||
[GNU AGPL v3.0 or later](LICENSE), the same license as the server. Submit issues
|
||||
and pull requests to this repository's maintainers. No separate Apache-2.0
|
||||
license grant to SILO or upstream MinIO maintainers is required.
|
||||
|
||||
* **No CLA.** We do not ask you to sign a Contributor License Agreement and we
|
||||
do not take your copyright. Contributions are accepted inbound=outbound: you
|
||||
@@ -108,15 +113,20 @@ fork does not try to.
|
||||
`Signed-off-by` trailers) and add your own sign-off as the person passing it
|
||||
along. Never import code from a proprietary distribution.
|
||||
|
||||
* **File headers.** Files derived from upstream keep the original MinIO
|
||||
copyright header unchanged. New files added by this fork use the dual
|
||||
header, followed by the standard AGPL boilerplate:
|
||||
* **File headers.** Preserve existing copyright and license notices in inherited
|
||||
and third-party files. New original files name their actual copyright holders
|
||||
and use AGPL-3.0-or-later. Use a header such as the following, then append the
|
||||
standard AGPL boilerplate:
|
||||
|
||||
```
|
||||
// Copyright (c) 2015-2025 MinIO, Inc.
|
||||
// Copyright (c) 2025-2026 PGSTY
|
||||
// Copyright (c) 2026 Your Name
|
||||
```
|
||||
|
||||
* **Separately licensed material.** Documentation contributions in `docs/`
|
||||
follow its existing [CC BY 4.0 license](docs/LICENSE). Third-party components
|
||||
and earlier Apache-2.0 contributions retain their original licenses and
|
||||
attribution; this policy does not relicense earlier work.
|
||||
|
||||
* **Squash merges** must keep the `Signed-off-by:` trailers in the resulting
|
||||
commit message.
|
||||
|
||||
|
||||
+168
-63
File diff suppressed because one or more lines are too long
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.26.5-alpine AS build
|
||||
FROM golang:1.27.1-alpine AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
|
||||
@@ -6,9 +6,9 @@ ENV GOPATH=/go
|
||||
ENV CGO_ENABLED=0
|
||||
|
||||
ARG MC_REPO=pgsty/mc
|
||||
ARG MC_VERSION=RELEASE.2026-08-06T00-00-00Z
|
||||
ARG MC_AMD64_SHA256=4b488bd30af54ad4214e5b654746677c79cd93dc6cad4be3aa2d09dbb48370ff
|
||||
ARG MC_ARM64_SHA256=83f6fedb16ed9c1e8efa8aea6776203dff132bc474214543d5c0767ed2066c2f
|
||||
ARG MC_VERSION=RELEASE.2026-09-03T07-13-05Z
|
||||
ARG MC_AMD64_SHA256=cd7fcd449bb6b52e2eb727431ba6975b1e5d90df011a75869020ea9ac9e2b2a8
|
||||
ARG MC_ARM64_SHA256=7962afc37c3e60e5758b19e819cb62d2f340ee655fad7b067e2ac9bc5716c2e8
|
||||
|
||||
RUN apk add -U --no-cache \
|
||||
ca-certificates \
|
||||
|
||||
@@ -4,7 +4,7 @@ LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
|
||||
|
||||
GOOS ?= $(shell go env GOOS)
|
||||
GOARCH ?= $(shell go env GOARCH)
|
||||
GOLANGCI_VERSION ?= v2.11.3
|
||||
GOLANGCI_VERSION ?= v2.13.1
|
||||
|
||||
VERSION ?= $(shell git describe --tags)
|
||||
REPO ?= docker.io/pgsty
|
||||
@@ -48,7 +48,7 @@ credits: ## regenerate CREDITS from the licenses of Go modules linked into the b
|
||||
|
||||
check-gen: ## check for updated autogenerated files
|
||||
@go generate ./... >/dev/null
|
||||
@go mod tidy -compat=1.26
|
||||
@go mod tidy -compat=1.27
|
||||
@env bash $(PWD)/buildscripts/gen-credits.sh
|
||||
@changed=$$(git diff --name-only -- '*_gen.go' '*_gen_test.go' '*_msgp_test.go' '*_string.go' go.mod go.sum CREDITS); \
|
||||
if [ -n "$$changed" ]; then \
|
||||
|
||||
@@ -95,51 +95,52 @@ Report vulnerabilities privately as described in [`SECURITY.md`](SECURITY.md); e
|
||||
|
||||
## Contributors
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/ZouhairCharef"><img src="https://github.com/ZouhairCharef.png?size=100" width="72" alt="ZouhairCharef"><br><sub><b>@ZouhairCharef</b></sub></a><br><sub>CVE-2026-34986</sub>
|
||||
</td>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/mfredenhagen"><img src="https://github.com/mfredenhagen.png?size=100" width="72" alt="mfredenhagen"><br><sub><b>@mfredenhagen</b></sub></a><br><sub>CVE-2026-39883</sub>
|
||||
</td>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/pinginfo"><img src="https://github.com/pinginfo.png?size=100" width="72" alt="pinginfo"><br><sub><b>@pinginfo</b></sub></a><br><sub>Notification streaming</sub>
|
||||
</td>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/waterkip"><img src="https://github.com/waterkip.png?size=100" width="72" alt="waterkip"><br><sub><b>@waterkip</b></sub></a><br><sub>Documentation links</sub>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
<p>
|
||||
<a href="https://github.com/magicxor"><img src="https://github.com/magicxor.png?size=64" width="44" alt="magicxor" title="@magicxor"></a>
|
||||
<a href="https://github.com/ycjlin"><img src="https://github.com/ycjlin.png?size=64" width="44" alt="ycjlin" title="@ycjlin"></a>
|
||||
<a href="https://github.com/davinkevin"><img src="https://github.com/davinkevin.png?size=64" width="44" alt="davinkevin" title="@davinkevin"></a>
|
||||
<a href="https://github.com/lem21h"><img src="https://github.com/lem21h.png?size=64" width="44" alt="lem21h" title="@lem21h"></a>
|
||||
<a href="https://github.com/sulin37392"><img src="https://github.com/sulin37392.png?size=64" width="44" alt="sulin37392" title="@sulin37392"></a>
|
||||
<a href="https://github.com/mosesdd"><img src="https://github.com/mosesdd.png?size=64" width="44" alt="mosesdd" title="@mosesdd"></a>
|
||||
<a href="https://github.com/Xavier-777"><img src="https://github.com/Xavier-777.png?size=64" width="44" alt="Xavier-777" title="@Xavier-777"></a>
|
||||
<a href="https://github.com/jiadzh"><img src="https://github.com/jiadzh.png?size=64" width="44" alt="jiadzh" title="@jiadzh"></a>
|
||||
<a href="https://github.com/TLINDEN"><img src="https://github.com/TLINDEN.png?size=64" width="44" alt="TLINDEN" title="@TLINDEN"></a>
|
||||
<a href="https://github.com/AntonOfTheWoods"><img src="https://github.com/AntonOfTheWoods.png?size=64" width="44" alt="AntonOfTheWoods" title="@AntonOfTheWoods"></a>
|
||||
<a href="https://github.com/zylpsrs"><img src="https://github.com/zylpsrs.png?size=64" width="44" alt="zylpsrs" title="@zylpsrs"></a>
|
||||
<a href="https://github.com/nsanitate"><img src="https://github.com/nsanitate.png?size=64" width="44" alt="nsanitate" title="@nsanitate"></a>
|
||||
<a href="https://github.com/makinikm"><img src="https://github.com/makinikm.png?size=64" width="44" alt="makinikm" title="@makinikm"></a>
|
||||
<a href="https://github.com/spaceg00se-r"><img src="https://github.com/spaceg00se-r.png?size=64" width="44" alt="spaceg00se-r" title="@spaceg00se-r"></a>
|
||||
<a href="https://github.com/heroes1412"><img src="https://github.com/heroes1412.png?size=64" width="44" alt="heroes1412" title="@heroes1412"></a>
|
||||
<a href="https://github.com/vampywiz17"><img src="https://github.com/vampywiz17.png?size=64" width="44" alt="vampywiz17" title="@vampywiz17"></a>
|
||||
<a href="https://github.com/chalukyaj"><img src="https://github.com/chalukyaj.png?size=64" width="44" alt="chalukyaj" title="@chalukyaj"></a>
|
||||
<a href="https://github.com/cbornet"><img src="https://github.com/cbornet.png?size=64" width="44" alt="cbornet" title="@cbornet"></a>
|
||||
<a href="https://github.com/jvasile"><img src="https://github.com/jvasile.png?size=64" width="44" alt="jvasile" title="@jvasile"></a>
|
||||
<a href="https://github.com/Kesavaambati"><img src="https://github.com/Kesavaambati.png?size=64" width="44" alt="Kesavaambati" title="@Kesavaambati"></a>
|
||||
<a href="https://github.com/redfoxfox"><img src="https://github.com/redfoxfox.png?size=64" width="44" alt="redfoxfox" title="@redfoxfox"></a>
|
||||
<a href="https://github.com/kuldeep-link11"><img src="https://github.com/kuldeep-link11.png?size=64" width="44" alt="kuldeep-link11" title="@kuldeep-link11"></a>
|
||||
<a href="https://github.com/meesudzu"><img src="https://github.com/meesudzu.png?size=64" width="44" alt="meesudzu" title="@meesudzu"></a>
|
||||
<a href="https://github.com/pmezhuev"><img src="https://github.com/pmezhuev.png?size=64" width="44" alt="pmezhuev" title="@pmezhuev"></a>
|
||||
<a href="https://github.com/kh0mka"><img src="https://github.com/kh0mka.png?size=64" width="44" alt="kh0mka" title="@kh0mka"></a>
|
||||
**40 community contributors** build SILO, Console, mcli, shared packages, and related projects. The list includes maintainers and every human Issue or PR author, ordered by merged PRs, other PRs, then issue reports. Gold rings highlight significant contributions.
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/Vonng"><img src="https://silo.pgsty.com/images/contributors/Vonng.svg" width="60" height="60" alt="@Vonng" title="@Vonng — Maintains SILO, Console, mcli, shared packages, releases, and documentation"></a>
|
||||
<a href="https://github.com/h5vx"><img src="https://silo.pgsty.com/images/contributors/h5vx.svg" width="60" height="60" alt="@h5vx" title="@h5vx — Implemented per-bucket CORS configuration and enforcement"></a>
|
||||
<a href="https://github.com/mrjavadseydi"><img src="https://silo.pgsty.com/images/contributors/mrjavadseydi.svg" width="60" height="60" alt="@mrjavadseydi" title="@mrjavadseydi — Fixed effective bucket quota metrics; proposed access-frequency ILM"></a>
|
||||
<a href="https://github.com/Dansyuqri"><img src="https://silo.pgsty.com/images/contributors/Dansyuqri.svg" width="60" height="60" alt="@Dansyuqri" title="@Dansyuqri — Added ChecksumType to multipart completion responses"></a>
|
||||
<a href="https://github.com/ycjlin"><img src="https://silo.pgsty.com/images/contributors/ycjlin.svg" width="60" height="60" alt="@ycjlin" title="@ycjlin — Fixed missing-bucket ListObjects semantics"></a>
|
||||
<a href="https://github.com/pinginfo"><img src="https://silo.pgsty.com/images/contributors/pinginfo.svg" width="60" height="60" alt="@pinginfo" title="@pinginfo — Repaired bucket notification streaming"></a>
|
||||
<a href="https://github.com/ZouhairCharef"><img src="https://silo.pgsty.com/images/contributors/ZouhairCharef.svg" width="60" height="60" alt="@ZouhairCharef" title="@ZouhairCharef — Patched CVE-2026-34986 in go-jose"></a>
|
||||
<a href="https://github.com/mfredenhagen"><img src="https://silo.pgsty.com/images/contributors/mfredenhagen.svg" width="60" height="60" alt="@mfredenhagen" title="@mfredenhagen — Patched CVE-2026-39883 in OpenTelemetry"></a>
|
||||
<a href="https://github.com/waterkip"><img src="https://silo.pgsty.com/images/contributors/waterkip.svg" width="60" height="60" alt="@waterkip" title="@waterkip — Repointed documentation links to the SILO portal"></a>
|
||||
<a href="https://github.com/mikemikimike"><img src="https://silo.pgsty.com/images/contributors/mikemikimike.svg" width="60" height="60" alt="@mikemikimike" title="@mikemikimike — Contributed the replicated SSE-C plaintext part-size fix"></a>
|
||||
<a href="https://github.com/metaneutrons"><img src="https://silo.pgsty.com/images/contributors/metaneutrons.svg" width="60" height="60" alt="@metaneutrons" title="@metaneutrons — Reported and proposed explicit-version delete authorization"></a>
|
||||
<a href="https://github.com/magicxor"><img src="https://silo.pgsty.com/images/contributors/magicxor.svg" width="60" height="60" alt="@magicxor" title="@magicxor — Reported and proposed conditional DELETE support for If-Match"></a>
|
||||
<a href="https://github.com/davinkevin"><img src="https://silo.pgsty.com/images/contributors/davinkevin.svg" width="60" height="60" alt="@davinkevin" title="@davinkevin — Proposed the distroless container image and dependency automation"></a>
|
||||
<a href="https://github.com/lem21h"><img src="https://silo.pgsty.com/images/contributors/lem21h.svg" width="48" height="48" alt="@lem21h" title="@lem21h — Proposed robustness and goroutine improvements"></a>
|
||||
<a href="https://github.com/sulin37392"><img src="https://silo.pgsty.com/images/contributors/sulin37392.svg" width="48" height="48" alt="@sulin37392" title="@sulin37392 — Proposed dependency updates"></a>
|
||||
<a href="https://github.com/cbornet"><img src="https://silo.pgsty.com/images/contributors/cbornet.svg" width="60" height="60" alt="@cbornet" title="@cbornet — Reported multipart and streaming checksum defects and missing-bucket semantics"></a>
|
||||
<a href="https://github.com/vampywiz17"><img src="https://silo.pgsty.com/images/contributors/vampywiz17.svg" width="60" height="60" alt="@vampywiz17" title="@vampywiz17 — Reported LDAP TLS and Console login regressions"></a>
|
||||
<a href="https://github.com/mumu-lab"><img src="https://silo.pgsty.com/images/contributors/mumu-lab.svg" width="48" height="48" alt="@mumu-lab" title="@mumu-lab — Reported bucket quota metrics reading a deprecated field"></a>
|
||||
<a href="https://github.com/jvasile"><img src="https://silo.pgsty.com/images/contributors/jvasile.svg" width="48" height="48" alt="@jvasile" title="@jvasile — Reported missing user, group, and defaults in Debian packages"></a>
|
||||
<a href="https://github.com/pmezhuev"><img src="https://silo.pgsty.com/images/contributors/pmezhuev.svg" width="48" height="48" alt="@pmezhuev" title="@pmezhuev — Reported missing RPM package signatures"></a>
|
||||
<a href="https://github.com/TLINDEN"><img src="https://silo.pgsty.com/images/contributors/TLINDEN.svg" width="48" height="48" alt="@TLINDEN" title="@TLINDEN — Reported the missing client in release tarballs"></a>
|
||||
<a href="https://github.com/makinikm"><img src="https://silo.pgsty.com/images/contributors/makinikm.svg" width="48" height="48" alt="@makinikm" title="@makinikm — Reported the missing client in the container image"></a>
|
||||
<a href="https://github.com/meesudzu"><img src="https://silo.pgsty.com/images/contributors/meesudzu.svg" width="48" height="48" alt="@meesudzu" title="@meesudzu — Requested the migration guide from upstream MinIO"></a>
|
||||
<a href="https://github.com/kuldeep-link11"><img src="https://silo.pgsty.com/images/contributors/kuldeep-link11.svg" width="48" height="48" alt="@kuldeep-link11" title="@kuldeep-link11 — Reported NATS JWT credentials and target reload issues"></a>
|
||||
<a href="https://github.com/sargarass"><img src="https://silo.pgsty.com/images/contributors/sargarass.svg" width="48" height="48" alt="@sargarass" title="@sargarass — Reported ListMultipartUploads prefix and pagination semantics"></a>
|
||||
<a href="https://github.com/liuhaodongliu990-cmyk"><img src="https://silo.pgsty.com/images/contributors/liuhaodongliu990-cmyk.svg" width="48" height="48" alt="@liuhaodongliu990-cmyk" title="@liuhaodongliu990-cmyk — Reported indeterminate progress for prefix downloads"></a>
|
||||
<a href="https://github.com/Xavier-777"><img src="https://silo.pgsty.com/images/contributors/Xavier-777.svg" width="48" height="48" alt="@Xavier-777" title="@Xavier-777 — Reported Console lifecycle management and file preview gaps"></a>
|
||||
<a href="https://github.com/spaceg00se-r"><img src="https://silo.pgsty.com/images/contributors/spaceg00se-r.svg" width="48" height="48" alt="@spaceg00se-r" title="@spaceg00se-r — Requested cpuv1 support and reported a workflow token failure"></a>
|
||||
<a href="https://github.com/kh0mka"><img src="https://silo.pgsty.com/images/contributors/kh0mka.svg" width="48" height="48" alt="@kh0mka" title="@kh0mka — Reported inter-node I/O timeouts in ReadFileStreamHandler"></a>
|
||||
<a href="https://github.com/bagutzu"><img src="https://silo.pgsty.com/images/contributors/bagutzu.svg" width="48" height="48" alt="@bagutzu" title="@bagutzu — Requested KES-compatible external KMS and OpenBao support"></a>
|
||||
<a href="https://github.com/DestroyLee"><img src="https://silo.pgsty.com/images/contributors/DestroyLee.svg" width="48" height="48" alt="@DestroyLee" title="@DestroyLee — Reported the missing documentation navigation"></a>
|
||||
<a href="https://github.com/mosesdd"><img src="https://silo.pgsty.com/images/contributors/mosesdd.svg" width="48" height="48" alt="@mosesdd" title="@mosesdd — Requested a maintained Helm chart"></a>
|
||||
<a href="https://github.com/zylpsrs"><img src="https://silo.pgsty.com/images/contributors/zylpsrs.svg" width="48" height="48" alt="@zylpsrs" title="@zylpsrs — Reported missing Console tiering and site replication"></a>
|
||||
<a href="https://github.com/heroes1412"><img src="https://silo.pgsty.com/images/contributors/heroes1412.svg" width="48" height="48" alt="@heroes1412" title="@heroes1412 — Reported the unusable profiling option"></a>
|
||||
<a href="https://github.com/redfoxfox"><img src="https://silo.pgsty.com/images/contributors/redfoxfox.svg" width="48" height="48" alt="@redfoxfox" title="@redfoxfox — Reported Chinese documentation availability"></a>
|
||||
<a href="https://github.com/jiadzh"><img src="https://silo.pgsty.com/images/contributors/jiadzh.svg" width="48" height="48" alt="@jiadzh" title="@jiadzh — Requested Windows build guidance"></a>
|
||||
<a href="https://github.com/AntonOfTheWoods"><img src="https://silo.pgsty.com/images/contributors/AntonOfTheWoods.svg" width="48" height="48" alt="@AntonOfTheWoods" title="@AntonOfTheWoods — Asked for clarity on Helm chart and operator options"></a>
|
||||
<a href="https://github.com/chalukyaj"><img src="https://silo.pgsty.com/images/contributors/chalukyaj.svg" width="48" height="48" alt="@chalukyaj" title="@chalukyaj — Proposed making the SILO Operator easier to discover"></a>
|
||||
<a href="https://github.com/nsanitate"><img src="https://silo.pgsty.com/images/contributors/nsanitate.svg" width="48" height="48" alt="@nsanitate" title="@nsanitate — Proposed CNCF Sandbox governance"></a>
|
||||
<a href="https://github.com/Kesavaambati"><img src="https://silo.pgsty.com/images/contributors/Kesavaambati.svg" width="48" height="48" alt="@Kesavaambati" title="@Kesavaambati — Asked about community support and image maintenance"></a>
|
||||
</p>
|
||||
|
||||
GitHub does not generate a contributor graph for forks, so [`CONTRIBUTORS.md`](CONTRIBUTORS.md) — not the Insights page — is this project's attribution record. It names everyone alongside the change or report they contributed.
|
||||
[View the full contribution record](CONTRIBUTORS.md) for each person's proposals, fixes, and reports.
|
||||
|
||||
## Background
|
||||
|
||||
|
||||
+44
-43
@@ -95,51 +95,52 @@ S3 API、`MINIO_*` 环境变量、`minio_*` 指标、`x-minio-*` 头、`/minio/*
|
||||
|
||||
## 贡献者
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/ZouhairCharef"><img src="https://github.com/ZouhairCharef.png?size=100" width="72" alt="ZouhairCharef"><br><sub><b>@ZouhairCharef</b></sub></a><br><sub>CVE-2026-34986</sub>
|
||||
</td>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/mfredenhagen"><img src="https://github.com/mfredenhagen.png?size=100" width="72" alt="mfredenhagen"><br><sub><b>@mfredenhagen</b></sub></a><br><sub>CVE-2026-39883</sub>
|
||||
</td>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/pinginfo"><img src="https://github.com/pinginfo.png?size=100" width="72" alt="pinginfo"><br><sub><b>@pinginfo</b></sub></a><br><sub>桶通知流式输出</sub>
|
||||
</td>
|
||||
<td align="center" width="150">
|
||||
<a href="https://github.com/waterkip"><img src="https://github.com/waterkip.png?size=100" width="72" alt="waterkip"><br><sub><b>@waterkip</b></sub></a><br><sub>文档链接修正</sub>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
<p>
|
||||
<a href="https://github.com/magicxor"><img src="https://github.com/magicxor.png?size=64" width="44" alt="magicxor" title="@magicxor"></a>
|
||||
<a href="https://github.com/ycjlin"><img src="https://github.com/ycjlin.png?size=64" width="44" alt="ycjlin" title="@ycjlin"></a>
|
||||
<a href="https://github.com/davinkevin"><img src="https://github.com/davinkevin.png?size=64" width="44" alt="davinkevin" title="@davinkevin"></a>
|
||||
<a href="https://github.com/lem21h"><img src="https://github.com/lem21h.png?size=64" width="44" alt="lem21h" title="@lem21h"></a>
|
||||
<a href="https://github.com/sulin37392"><img src="https://github.com/sulin37392.png?size=64" width="44" alt="sulin37392" title="@sulin37392"></a>
|
||||
<a href="https://github.com/mosesdd"><img src="https://github.com/mosesdd.png?size=64" width="44" alt="mosesdd" title="@mosesdd"></a>
|
||||
<a href="https://github.com/Xavier-777"><img src="https://github.com/Xavier-777.png?size=64" width="44" alt="Xavier-777" title="@Xavier-777"></a>
|
||||
<a href="https://github.com/jiadzh"><img src="https://github.com/jiadzh.png?size=64" width="44" alt="jiadzh" title="@jiadzh"></a>
|
||||
<a href="https://github.com/TLINDEN"><img src="https://github.com/TLINDEN.png?size=64" width="44" alt="TLINDEN" title="@TLINDEN"></a>
|
||||
<a href="https://github.com/AntonOfTheWoods"><img src="https://github.com/AntonOfTheWoods.png?size=64" width="44" alt="AntonOfTheWoods" title="@AntonOfTheWoods"></a>
|
||||
<a href="https://github.com/zylpsrs"><img src="https://github.com/zylpsrs.png?size=64" width="44" alt="zylpsrs" title="@zylpsrs"></a>
|
||||
<a href="https://github.com/nsanitate"><img src="https://github.com/nsanitate.png?size=64" width="44" alt="nsanitate" title="@nsanitate"></a>
|
||||
<a href="https://github.com/makinikm"><img src="https://github.com/makinikm.png?size=64" width="44" alt="makinikm" title="@makinikm"></a>
|
||||
<a href="https://github.com/spaceg00se-r"><img src="https://github.com/spaceg00se-r.png?size=64" width="44" alt="spaceg00se-r" title="@spaceg00se-r"></a>
|
||||
<a href="https://github.com/heroes1412"><img src="https://github.com/heroes1412.png?size=64" width="44" alt="heroes1412" title="@heroes1412"></a>
|
||||
<a href="https://github.com/vampywiz17"><img src="https://github.com/vampywiz17.png?size=64" width="44" alt="vampywiz17" title="@vampywiz17"></a>
|
||||
<a href="https://github.com/chalukyaj"><img src="https://github.com/chalukyaj.png?size=64" width="44" alt="chalukyaj" title="@chalukyaj"></a>
|
||||
<a href="https://github.com/cbornet"><img src="https://github.com/cbornet.png?size=64" width="44" alt="cbornet" title="@cbornet"></a>
|
||||
<a href="https://github.com/jvasile"><img src="https://github.com/jvasile.png?size=64" width="44" alt="jvasile" title="@jvasile"></a>
|
||||
<a href="https://github.com/Kesavaambati"><img src="https://github.com/Kesavaambati.png?size=64" width="44" alt="Kesavaambati" title="@Kesavaambati"></a>
|
||||
<a href="https://github.com/redfoxfox"><img src="https://github.com/redfoxfox.png?size=64" width="44" alt="redfoxfox" title="@redfoxfox"></a>
|
||||
<a href="https://github.com/kuldeep-link11"><img src="https://github.com/kuldeep-link11.png?size=64" width="44" alt="kuldeep-link11" title="@kuldeep-link11"></a>
|
||||
<a href="https://github.com/meesudzu"><img src="https://github.com/meesudzu.png?size=64" width="44" alt="meesudzu" title="@meesudzu"></a>
|
||||
<a href="https://github.com/pmezhuev"><img src="https://github.com/pmezhuev.png?size=64" width="44" alt="pmezhuev" title="@pmezhuev"></a>
|
||||
<a href="https://github.com/kh0mka"><img src="https://github.com/kh0mka.png?size=64" width="44" alt="kh0mka" title="@kh0mka"></a>
|
||||
**40 位社区贡献者**共同建设 SILO、Console、mcli、公共包与相关项目。名单包含维护者,以及所有提出 Issue 或 PR 的真人作者;按已合并 PR、其他 PR、Issue 报告排序,黄圈标记显著贡献。
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/Vonng"><img src="https://silo.pgsty.com/images/contributors/Vonng.svg" width="60" height="60" alt="@Vonng" title="@Vonng — 维护 SILO、Console、mcli、公共包、发行与文档"></a>
|
||||
<a href="https://github.com/h5vx"><img src="https://silo.pgsty.com/images/contributors/h5vx.svg" width="60" height="60" alt="@h5vx" title="@h5vx — 实现单桶 CORS 配置与请求执行"></a>
|
||||
<a href="https://github.com/mrjavadseydi"><img src="https://silo.pgsty.com/images/contributors/mrjavadseydi.svg" width="60" height="60" alt="@mrjavadseydi" title="@mrjavadseydi — 修复有效桶配额指标,并提交按访问频率分层的 ILM 方案"></a>
|
||||
<a href="https://github.com/Dansyuqri"><img src="https://silo.pgsty.com/images/contributors/Dansyuqri.svg" width="60" height="60" alt="@Dansyuqri" title="@Dansyuqri — 为分片上传完成响应补充 ChecksumType"></a>
|
||||
<a href="https://github.com/ycjlin"><img src="https://silo.pgsty.com/images/contributors/ycjlin.svg" width="60" height="60" alt="@ycjlin" title="@ycjlin — 修复缺失桶的 ListObjects 语义"></a>
|
||||
<a href="https://github.com/pinginfo"><img src="https://silo.pgsty.com/images/contributors/pinginfo.svg" width="60" height="60" alt="@pinginfo" title="@pinginfo — 修复桶通知的流式输出"></a>
|
||||
<a href="https://github.com/ZouhairCharef"><img src="https://silo.pgsty.com/images/contributors/ZouhairCharef.svg" width="60" height="60" alt="@ZouhairCharef" title="@ZouhairCharef — 修复 go-jose 中的 CVE-2026-34986"></a>
|
||||
<a href="https://github.com/mfredenhagen"><img src="https://silo.pgsty.com/images/contributors/mfredenhagen.svg" width="60" height="60" alt="@mfredenhagen" title="@mfredenhagen — 修复 OpenTelemetry 中的 CVE-2026-39883"></a>
|
||||
<a href="https://github.com/waterkip"><img src="https://silo.pgsty.com/images/contributors/waterkip.svg" width="60" height="60" alt="@waterkip" title="@waterkip — 将文档链接指向 SILO 门户"></a>
|
||||
<a href="https://github.com/mikemikimike"><img src="https://silo.pgsty.com/images/contributors/mikemikimike.svg" width="60" height="60" alt="@mikemikimike" title="@mikemikimike — 提交 SSE-C 复制分片明文尺寸修复"></a>
|
||||
<a href="https://github.com/metaneutrons"><img src="https://silo.pgsty.com/images/contributors/metaneutrons.svg" width="60" height="60" alt="@metaneutrons" title="@metaneutrons — 报告并提交显式版本删除鉴权方案"></a>
|
||||
<a href="https://github.com/magicxor"><img src="https://silo.pgsty.com/images/contributors/magicxor.svg" width="60" height="60" alt="@magicxor" title="@magicxor — 报告并提交 DELETE If-Match 条件请求支持方案"></a>
|
||||
<a href="https://github.com/davinkevin"><img src="https://silo.pgsty.com/images/contributors/davinkevin.svg" width="60" height="60" alt="@davinkevin" title="@davinkevin — 提交 distroless 容器镜像与依赖自动更新方案"></a>
|
||||
<a href="https://github.com/lem21h"><img src="https://silo.pgsty.com/images/contributors/lem21h.svg" width="48" height="48" alt="@lem21h" title="@lem21h — 提交健壮性与 goroutine 改进"></a>
|
||||
<a href="https://github.com/sulin37392"><img src="https://silo.pgsty.com/images/contributors/sulin37392.svg" width="48" height="48" alt="@sulin37392" title="@sulin37392 — 提交依赖更新"></a>
|
||||
<a href="https://github.com/cbornet"><img src="https://silo.pgsty.com/images/contributors/cbornet.svg" width="60" height="60" alt="@cbornet" title="@cbornet — 报告分片与流式校验和缺陷及缺失桶语义问题"></a>
|
||||
<a href="https://github.com/vampywiz17"><img src="https://silo.pgsty.com/images/contributors/vampywiz17.svg" width="60" height="60" alt="@vampywiz17" title="@vampywiz17 — 报告 LDAP TLS 与 Console 登录回归"></a>
|
||||
<a href="https://github.com/mumu-lab"><img src="https://silo.pgsty.com/images/contributors/mumu-lab.svg" width="48" height="48" alt="@mumu-lab" title="@mumu-lab — 报告桶配额指标读取已弃用字段的问题"></a>
|
||||
<a href="https://github.com/jvasile"><img src="https://silo.pgsty.com/images/contributors/jvasile.svg" width="48" height="48" alt="@jvasile" title="@jvasile — 报告 Debian 包缺少用户、用户组与默认配置"></a>
|
||||
<a href="https://github.com/pmezhuev"><img src="https://silo.pgsty.com/images/contributors/pmezhuev.svg" width="48" height="48" alt="@pmezhuev" title="@pmezhuev — 报告 RPM 包缺少 GPG 签名"></a>
|
||||
<a href="https://github.com/TLINDEN"><img src="https://silo.pgsty.com/images/contributors/TLINDEN.svg" width="48" height="48" alt="@TLINDEN" title="@TLINDEN — 报告发布压缩包缺少客户端"></a>
|
||||
<a href="https://github.com/makinikm"><img src="https://silo.pgsty.com/images/contributors/makinikm.svg" width="48" height="48" alt="@makinikm" title="@makinikm — 报告容器镜像缺少客户端"></a>
|
||||
<a href="https://github.com/meesudzu"><img src="https://silo.pgsty.com/images/contributors/meesudzu.svg" width="48" height="48" alt="@meesudzu" title="@meesudzu — 提出从上游 MinIO 迁移的指南需求"></a>
|
||||
<a href="https://github.com/kuldeep-link11"><img src="https://silo.pgsty.com/images/contributors/kuldeep-link11.svg" width="48" height="48" alt="@kuldeep-link11" title="@kuldeep-link11 — 报告 NATS JWT 凭据与通知目标重载问题"></a>
|
||||
<a href="https://github.com/sargarass"><img src="https://silo.pgsty.com/images/contributors/sargarass.svg" width="48" height="48" alt="@sargarass" title="@sargarass — 报告 ListMultipartUploads 前缀与分页语义问题"></a>
|
||||
<a href="https://github.com/liuhaodongliu990-cmyk"><img src="https://silo.pgsty.com/images/contributors/liuhaodongliu990-cmyk.svg" width="48" height="48" alt="@liuhaodongliu990-cmyk" title="@liuhaodongliu990-cmyk — 报告前缀下载进度显示异常"></a>
|
||||
<a href="https://github.com/Xavier-777"><img src="https://silo.pgsty.com/images/contributors/Xavier-777.svg" width="48" height="48" alt="@Xavier-777" title="@Xavier-777 — 报告 Console 生命周期管理与文件预览缺失"></a>
|
||||
<a href="https://github.com/spaceg00se-r"><img src="https://silo.pgsty.com/images/contributors/spaceg00se-r.svg" width="48" height="48" alt="@spaceg00se-r" title="@spaceg00se-r — 提出 cpuv1 支持需求并报告工作流令牌错误"></a>
|
||||
<a href="https://github.com/kh0mka"><img src="https://silo.pgsty.com/images/contributors/kh0mka.svg" width="48" height="48" alt="@kh0mka" title="@kh0mka — 报告 ReadFileStreamHandler 节点间 I/O 超时"></a>
|
||||
<a href="https://github.com/bagutzu"><img src="https://silo.pgsty.com/images/contributors/bagutzu.svg" width="48" height="48" alt="@bagutzu" title="@bagutzu — 提出兼容 KES 的外部 KMS 与 OpenBao 支持需求"></a>
|
||||
<a href="https://github.com/DestroyLee"><img src="https://silo.pgsty.com/images/contributors/DestroyLee.svg" width="48" height="48" alt="@DestroyLee" title="@DestroyLee — 报告文档目录导航缺失"></a>
|
||||
<a href="https://github.com/mosesdd"><img src="https://silo.pgsty.com/images/contributors/mosesdd.svg" width="48" height="48" alt="@mosesdd" title="@mosesdd — 提出维护 Helm Chart 的需求"></a>
|
||||
<a href="https://github.com/zylpsrs"><img src="https://silo.pgsty.com/images/contributors/zylpsrs.svg" width="48" height="48" alt="@zylpsrs" title="@zylpsrs — 报告 Console 缺少分层与站点复制"></a>
|
||||
<a href="https://github.com/heroes1412"><img src="https://silo.pgsty.com/images/contributors/heroes1412.svg" width="48" height="48" alt="@heroes1412" title="@heroes1412 — 报告性能分析选项不可用"></a>
|
||||
<a href="https://github.com/redfoxfox"><img src="https://silo.pgsty.com/images/contributors/redfoxfox.svg" width="48" height="48" alt="@redfoxfox" title="@redfoxfox — 报告中文文档站点不可用"></a>
|
||||
<a href="https://github.com/jiadzh"><img src="https://silo.pgsty.com/images/contributors/jiadzh.svg" width="48" height="48" alt="@jiadzh" title="@jiadzh — 提出 Windows 构建指导需求"></a>
|
||||
<a href="https://github.com/AntonOfTheWoods"><img src="https://silo.pgsty.com/images/contributors/AntonOfTheWoods.svg" width="48" height="48" alt="@AntonOfTheWoods" title="@AntonOfTheWoods — 提出明确 Helm Chart 与 Operator 选项的需求"></a>
|
||||
<a href="https://github.com/chalukyaj"><img src="https://silo.pgsty.com/images/contributors/chalukyaj.svg" width="48" height="48" alt="@chalukyaj" title="@chalukyaj — 提出改善 SILO Operator 可发现性的建议"></a>
|
||||
<a href="https://github.com/nsanitate"><img src="https://silo.pgsty.com/images/contributors/nsanitate.svg" width="48" height="48" alt="@nsanitate" title="@nsanitate — 提出加入 CNCF Sandbox 的治理建议"></a>
|
||||
<a href="https://github.com/Kesavaambati"><img src="https://silo.pgsty.com/images/contributors/Kesavaambati.svg" width="48" height="48" alt="@Kesavaambati" title="@Kesavaambati — 提出社区支持与容器镜像维护问题"></a>
|
||||
</p>
|
||||
|
||||
GitHub 不为 fork 仓库生成贡献者图表,因此 [`CONTRIBUTORS.md`](CONTRIBUTORS.md)(而非 Insights 页面)才是本项目的署名记录,其中逐一记录了每个人对应的改动或报告。
|
||||
[查看完整贡献记录](CONTRIBUTORS.md),了解每位贡献者的提案、修复与问题报告。
|
||||
|
||||
## 背景
|
||||
|
||||
|
||||
+19
@@ -10,6 +10,25 @@ Security fixes are tracked on the active development branch and summarized in
|
||||
[docs/security/advisories.md](docs/security/advisories.md). Only the current
|
||||
Silo release line is supported unless an advisory says otherwise.
|
||||
|
||||
## Inherited Fix Evidence
|
||||
|
||||
The canonical ledger also records security fixes inherited from upstream when
|
||||
they are part of the Silo release baseline. Source and fork commits are linked
|
||||
separately even when the fork preserves the original commit object and SHA.
|
||||
|
||||
- [CVE-2025-62506](https://github.com/advisories/GHSA-jjjj-jwhf-8rgr):
|
||||
upstream [PR #21642](https://github.com/minio/minio/pull/21642) merged as
|
||||
[`minio/minio@c1a49490`](https://github.com/minio/minio/commit/c1a49490c78e9c3ebcad86ba0662319138ace190),
|
||||
inherited unchanged as
|
||||
[`pgsty/silo@c1a49490`](https://github.com/pgsty/silo/commit/c1a49490c78e9c3ebcad86ba0662319138ace190),
|
||||
and is present in every Silo community release beginning with
|
||||
[`RELEASE.2025-12-03T12-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2025-12-03T12-00-00Z).
|
||||
The inherited [service-account](https://github.com/pgsty/silo/blob/c1a49490c78e9c3ebcad86ba0662319138ace190/cmd/admin-handlers-users_test.go#L211-L212)
|
||||
and [STS](https://github.com/pgsty/silo/blob/c1a49490c78e9c3ebcad86ba0662319138ace190/cmd/sts-handlers_test.go#L45-L46)
|
||||
regression groups remain part of `go test ./cmd`; see the
|
||||
[canonical ledger](docs/security/advisories.md#inherited-upstream-advisory-baseline)
|
||||
for the operator-facing record.
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
For vulnerabilities in this fork:
|
||||
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Fail closed before a release job can replace published or finalized assets.
|
||||
# An ordinary Draft is retry state; a finalized Draft contains GPG-derived
|
||||
# materials and must never be replaced by the build lane.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
release_tag="${1:-}"
|
||||
fixture="${2:-}"
|
||||
repository="${GITHUB_REPOSITORY:-pgsty/silo}"
|
||||
require_draft="${REQUIRE_DRAFT:-false}"
|
||||
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
echo "jq is required to inspect GitHub release state" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! "${release_tag}" =~ ^RELEASE\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}Z$ ]]; then
|
||||
echo "Invalid release tag format: ${release_tag:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -n "${fixture}" ]; then
|
||||
release_json="$(<"${fixture}")"
|
||||
else
|
||||
error_file="$(mktemp)"
|
||||
trap 'rm -f "${error_file}"' EXIT
|
||||
if ! release_json="$(
|
||||
gh api --paginate "repos/${repository}/releases?per_page=100" --jq '.[]' 2>"${error_file}" |
|
||||
jq --arg tag "${release_tag}" -s '[.[] | select(.tag_name == $tag)]'
|
||||
)"; then
|
||||
cat "${error_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! jq -e 'type == "array" and all(.[]; type == "object" and (.tag_name | type == "string") and (.draft | type == "boolean"))' \
|
||||
<<<"${release_json}" >/dev/null 2>&1; then
|
||||
echo "Invalid release state response for ${release_tag}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! jq -e --arg tag "${release_tag}" 'all(.[]; .tag_name == $tag)' \
|
||||
<<<"${release_json}" >/dev/null 2>&1; then
|
||||
echo "Release state returned a tag other than ${release_tag}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
release_count="$(jq 'length' <<<"${release_json}")"
|
||||
if [ "${release_count}" -eq 0 ]; then
|
||||
if [ "${require_draft}" = "true" ]; then
|
||||
echo "Expected one Draft release for ${release_tag}, found none" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "No existing release for ${release_tag}."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "${release_count}" -ne 1 ]; then
|
||||
echo "Refusing to choose among ${release_count} releases for ${release_tag}; clean duplicate Drafts first" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$(jq -r '.[0].draft' <<<"${release_json}")" != "true" ]; then
|
||||
echo "Refusing to overwrite published release ${release_tag}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
finalize_markers="$(jq '[.[0].assets[]? | select(.name | endswith("_packages_provenance.sigstore.json"))] | length' <<<"${release_json}")"
|
||||
if [ "${finalize_markers}" -ne 0 ]; then
|
||||
echo "Refusing to replace finalized Draft ${release_tag}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Existing unfinalized Draft ${release_tag} will be replaced from scratch."
|
||||
Executable
+68
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
checker="${script_dir}/check-release-state.sh"
|
||||
tag="RELEASE.2026-08-29T00-00-00Z"
|
||||
fixture="$(mktemp)"
|
||||
stdout_file="$(mktemp)"
|
||||
stderr_file="$(mktemp)"
|
||||
trap 'rm -f "${fixture}" "${stdout_file}" "${stderr_file}"' EXIT
|
||||
|
||||
expect_success() {
|
||||
if ! "${checker}" "$@" >"${stdout_file}" 2>"${stderr_file}"; then
|
||||
cat "${stderr_file}" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
expect_failure() {
|
||||
if "${checker}" "$@" >"${stdout_file}" 2>"${stderr_file}"; then
|
||||
echo "Expected release-state check to fail: $*" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
printf '[]\n' >"${fixture}"
|
||||
expect_success "${tag}" "${fixture}"
|
||||
grep -qF "No existing release for ${tag}." "${stdout_file}"
|
||||
|
||||
if REQUIRE_DRAFT=true "${checker}" "${tag}" "${fixture}" >"${stdout_file}" 2>"${stderr_file}"; then
|
||||
echo "Expected required-Draft check to fail when no release exists" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -qF "Expected one Draft release for ${tag}, found none" "${stderr_file}"
|
||||
|
||||
printf '[{"tag_name":"%s","draft":true,"assets":[]}]\n' "${tag}" >"${fixture}"
|
||||
expect_success "${tag}" "${fixture}"
|
||||
grep -qF "Existing unfinalized Draft ${tag} will be replaced from scratch." "${stdout_file}"
|
||||
if ! REQUIRE_DRAFT=true "${checker}" "${tag}" "${fixture}" >"${stdout_file}" 2>"${stderr_file}"; then
|
||||
cat "${stderr_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf '[{"tag_name":"%s","draft":true,"assets":[{"name":"silo_20260829000000.0.0_packages_provenance.sigstore.json"}]}]\n' "${tag}" >"${fixture}"
|
||||
expect_failure "${tag}" "${fixture}"
|
||||
grep -qF "Refusing to replace finalized Draft ${tag}" "${stderr_file}"
|
||||
|
||||
printf '[{"tag_name":"%s","draft":false}]\n' "${tag}" >"${fixture}"
|
||||
expect_failure "${tag}" "${fixture}"
|
||||
grep -qF "Refusing to overwrite published release ${tag}" "${stderr_file}"
|
||||
|
||||
printf '[{"tag_name":"%s","draft":true},{"tag_name":"%s","draft":true}]\n' "${tag}" "${tag}" >"${fixture}"
|
||||
expect_failure "${tag}" "${fixture}"
|
||||
grep -qF "Refusing to choose among 2 releases" "${stderr_file}"
|
||||
|
||||
printf '[{"tag_name":"RELEASE.2026-08-28T00-00-00Z","draft":true}]\n' >"${fixture}"
|
||||
expect_failure "${tag}" "${fixture}"
|
||||
grep -qF "other than ${tag}" "${stderr_file}"
|
||||
|
||||
printf '{not-json}\n' >"${fixture}"
|
||||
expect_failure "${tag}" "${fixture}"
|
||||
grep -qF "Invalid release state response for ${tag}" "${stderr_file}"
|
||||
|
||||
expect_failure "not-a-release-tag" "${fixture}"
|
||||
grep -qF "Invalid release tag format" "${stderr_file}"
|
||||
|
||||
echo "release-state decision tests passed"
|
||||
@@ -7,7 +7,7 @@ _init() {
|
||||
|
||||
## Minimum required versions for build dependencies
|
||||
GIT_VERSION="1.0"
|
||||
GO_VERSION="1.16"
|
||||
GO_VERSION="1.27.1"
|
||||
OSX_VERSION="10.8"
|
||||
KNAME=$(uname -s)
|
||||
ARCH=$(uname -m)
|
||||
|
||||
@@ -22,8 +22,8 @@ function start_silo_4drive() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MC_HOST_silo="http://silo:silo123@127.0.0.1:${start_port}/"
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
export MC_HOST_silo="http://silo:silo1234@127.0.0.1:${start_port}/"
|
||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ if [ -n "${MCLI_BIN:-}" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
release=${MCLI_RELEASE:-RELEASE.2026-08-06T00-00-00Z}
|
||||
release=${MCLI_RELEASE:-RELEASE.2026-09-03T07-13-05Z}
|
||||
version_hyphen=${release#RELEASE.}
|
||||
package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')
|
||||
if [ "${package_version}" = "${version_hyphen}" ]; then
|
||||
@@ -74,7 +74,7 @@ expected=$(awk -v asset="${archive}" '
|
||||
{
|
||||
name=$2
|
||||
sub(/^\*/, "", name)
|
||||
if (name == asset && $1 ~ /^[0-9a-fA-F]{64}$/) print tolower($1)
|
||||
if (name == asset && length($1) == 64 && $1 ~ /^[0-9a-fA-F]+$/) print tolower($1)
|
||||
}
|
||||
' "${tmp_dir}/${checksums}")
|
||||
if ! printf '%s\n' "${expected}" | grep -Eq '^[0-9a-f]{64}$'; then
|
||||
|
||||
@@ -45,8 +45,8 @@ function start_silo_10drive() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MC_HOST_silo="http://silo:silo123@127.0.0.1:${start_port}/"
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
export MC_HOST_silo="http://silo:silo1234@127.0.0.1:${start_port}/"
|
||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
@@ -71,7 +71,7 @@ function start_silo_10drive() {
|
||||
"${PWD}/mc" mb --with-versioning silo/bucket
|
||||
|
||||
export AWS_ACCESS_KEY_ID=silo
|
||||
export AWS_SECRET_ACCESS_KEY=silo123
|
||||
export AWS_SECRET_ACCESS_KEY=silo1234
|
||||
aws --endpoint-url http://localhost:"$start_port" s3api create-multipart-upload --bucket bucket --key obj-1 >upload-id.json
|
||||
uploadId=$(jq -r '.UploadId' upload-id.json)
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -30,7 +30,7 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
const manifestVersion = 3
|
||||
const manifestVersion = 4
|
||||
|
||||
var (
|
||||
minioImportRE = regexp.MustCompile(`github\.com/minio/[A-Za-z0-9_./-]+`)
|
||||
@@ -44,19 +44,18 @@ var (
|
||||
)
|
||||
|
||||
type manifest struct {
|
||||
Version int `json:"version"`
|
||||
ModulePath string `json:"module_path"`
|
||||
MinioImports []string `json:"minio_imports"`
|
||||
Environment []string `json:"environment"`
|
||||
Metrics []string `json:"metrics"`
|
||||
Headers []string `json:"headers"`
|
||||
Routes []string `json:"routes"`
|
||||
RouteRoots []string `json:"route_roots"`
|
||||
GridRoutes []string `json:"grid_routes"`
|
||||
StorageMarkers []string `json:"storage_markers"`
|
||||
PolicyValues []string `json:"policy_values"`
|
||||
ExportedSymbols []string `json:"exported_symbols"`
|
||||
BrandAllowlist []string `json:"brand_allowlist"`
|
||||
Version int `json:"version"`
|
||||
ModulePath string `json:"module_path"`
|
||||
MinioImports []string `json:"minio_imports"`
|
||||
Environment []string `json:"environment"`
|
||||
Metrics []string `json:"metrics"`
|
||||
Headers []string `json:"headers"`
|
||||
Routes []string `json:"routes"`
|
||||
RouteRoots []string `json:"route_roots"`
|
||||
GridRoutes []string `json:"grid_routes"`
|
||||
StorageMarkers []string `json:"storage_markers"`
|
||||
PolicyValues []string `json:"policy_values"`
|
||||
BrandAllowlist []string `json:"brand_allowlist"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
@@ -101,17 +100,16 @@ func collect(repo string) (manifest, error) {
|
||||
}
|
||||
|
||||
sets := map[string]map[string]struct{}{
|
||||
"imports": {},
|
||||
"env": {},
|
||||
"metrics": {},
|
||||
"headers": {},
|
||||
"routes": {},
|
||||
"roots": {},
|
||||
"grid": {},
|
||||
"storage": {},
|
||||
"policy": {},
|
||||
"exported": {},
|
||||
"brand": {},
|
||||
"imports": {},
|
||||
"env": {},
|
||||
"metrics": {},
|
||||
"headers": {},
|
||||
"routes": {},
|
||||
"roots": {},
|
||||
"grid": {},
|
||||
"storage": {},
|
||||
"policy": {},
|
||||
"brand": {},
|
||||
}
|
||||
modulePath := ""
|
||||
fset := token.NewFileSet()
|
||||
@@ -163,17 +161,17 @@ func collect(repo string) (manifest, error) {
|
||||
sets["imports"][value] = struct{}{}
|
||||
}
|
||||
}
|
||||
collectStringMatches(sets["routes"], routeRE, file)
|
||||
collectNamedStringValues(sets["roots"], rel, file, "minioReservedBucket")
|
||||
if rel == "internal/grid/manager.go" {
|
||||
collectStringMatches(sets["grid"], routeRE, file)
|
||||
}
|
||||
if !strings.HasSuffix(rel, "_test.go") {
|
||||
collectExported(sets["exported"], filepath.ToSlash(filepath.Dir(rel)), file)
|
||||
// Test files hold request paths for fixtures, not served routes.
|
||||
collectStringMatches(sets["routes"], routeRE, file)
|
||||
if strings.HasPrefix(rel, "cmd/") || strings.HasPrefix(rel, "internal/") {
|
||||
collectBrandStrings(sets["brand"], rel, file)
|
||||
}
|
||||
}
|
||||
collectNamedStringValues(sets["roots"], rel, file, "minioReservedBucket")
|
||||
if rel == "internal/grid/manager.go" {
|
||||
collectStringMatches(sets["grid"], routeRE, file)
|
||||
}
|
||||
}
|
||||
}
|
||||
// This was a shell-local PID variable in the generated inspect script,
|
||||
@@ -184,19 +182,18 @@ func collect(repo string) (manifest, error) {
|
||||
return manifest{}, errors.New("go.mod module path was not found")
|
||||
}
|
||||
return manifest{
|
||||
Version: manifestVersion,
|
||||
ModulePath: modulePath,
|
||||
MinioImports: sorted(sets["imports"]),
|
||||
Environment: sorted(sets["env"]),
|
||||
Metrics: sorted(sets["metrics"]),
|
||||
Headers: sorted(sets["headers"]),
|
||||
Routes: sorted(sets["routes"]),
|
||||
RouteRoots: sorted(sets["roots"]),
|
||||
GridRoutes: sorted(sets["grid"]),
|
||||
StorageMarkers: sorted(sets["storage"]),
|
||||
PolicyValues: sorted(sets["policy"]),
|
||||
ExportedSymbols: sorted(sets["exported"]),
|
||||
BrandAllowlist: sorted(sets["brand"]),
|
||||
Version: manifestVersion,
|
||||
ModulePath: modulePath,
|
||||
MinioImports: sorted(sets["imports"]),
|
||||
Environment: sorted(sets["env"]),
|
||||
Metrics: sorted(sets["metrics"]),
|
||||
Headers: sorted(sets["headers"]),
|
||||
Routes: sorted(sets["routes"]),
|
||||
RouteRoots: sorted(sets["roots"]),
|
||||
GridRoutes: sorted(sets["grid"]),
|
||||
StorageMarkers: sorted(sets["storage"]),
|
||||
PolicyValues: sorted(sets["policy"]),
|
||||
BrandAllowlist: sorted(sets["brand"]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -262,7 +259,7 @@ func collectStringMatches(dst map[string]struct{}, re *regexp.Regexp, file *ast.
|
||||
}
|
||||
|
||||
func trackedFiles(repo string) ([]string, error) {
|
||||
cmd := exec.Command("git", "-C", repo, "ls-files", "--cached", "--others", "--exclude-standard", "-z")
|
||||
cmd := exec.Command("git", "-C", repo, "ls-files", "--cached", "-z")
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("git ls-files: %w", err)
|
||||
@@ -286,78 +283,6 @@ func addMatches(dst map[string]struct{}, re *regexp.Regexp, text string, lower b
|
||||
}
|
||||
}
|
||||
|
||||
func collectExported(dst map[string]struct{}, dir string, file *ast.File) {
|
||||
prefix := dir + ":" + file.Name.Name + ":"
|
||||
for _, decl := range file.Decls {
|
||||
switch decl := decl.(type) {
|
||||
case *ast.FuncDecl:
|
||||
if !ast.IsExported(decl.Name.Name) {
|
||||
continue
|
||||
}
|
||||
if decl.Recv == nil {
|
||||
dst[prefix+"func:"+decl.Name.Name] = struct{}{}
|
||||
continue
|
||||
}
|
||||
receiver := receiverName(decl.Recv.List[0].Type)
|
||||
dst[prefix+"method:"+receiver+"."+decl.Name.Name] = struct{}{}
|
||||
case *ast.GenDecl:
|
||||
for _, spec := range decl.Specs {
|
||||
switch spec := spec.(type) {
|
||||
case *ast.TypeSpec:
|
||||
if !ast.IsExported(spec.Name.Name) {
|
||||
continue
|
||||
}
|
||||
dst[prefix+"type:"+spec.Name.Name] = struct{}{}
|
||||
collectExportedFields(dst, prefix, spec.Name.Name, spec.Type)
|
||||
case *ast.ValueSpec:
|
||||
kind := strings.ToLower(decl.Tok.String())
|
||||
for _, name := range spec.Names {
|
||||
if ast.IsExported(name.Name) {
|
||||
dst[prefix+kind+":"+name.Name] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func collectExportedFields(dst map[string]struct{}, prefix, typeName string, expr ast.Expr) {
|
||||
var fields *ast.FieldList
|
||||
switch typed := expr.(type) {
|
||||
case *ast.StructType:
|
||||
fields = typed.Fields
|
||||
case *ast.InterfaceType:
|
||||
fields = typed.Methods
|
||||
default:
|
||||
return
|
||||
}
|
||||
for _, field := range fields.List {
|
||||
for _, name := range field.Names {
|
||||
if ast.IsExported(name.Name) {
|
||||
dst[prefix+"field:"+typeName+"."+name.Name] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func receiverName(expr ast.Expr) string {
|
||||
switch expr := expr.(type) {
|
||||
case *ast.Ident:
|
||||
return expr.Name
|
||||
case *ast.StarExpr:
|
||||
return receiverName(expr.X)
|
||||
case *ast.IndexExpr:
|
||||
return receiverName(expr.X)
|
||||
case *ast.IndexListExpr:
|
||||
return receiverName(expr.X)
|
||||
case *ast.SelectorExpr:
|
||||
return receiverName(expr.X) + "." + expr.Sel.Name
|
||||
default:
|
||||
return fmt.Sprintf("%T", expr)
|
||||
}
|
||||
}
|
||||
|
||||
func sorted(set map[string]struct{}) []string {
|
||||
values := make([]string, 0, len(set))
|
||||
for value := range set {
|
||||
@@ -409,7 +334,6 @@ func compare(want, got manifest) error {
|
||||
{"grid_routes", want.GridRoutes, got.GridRoutes},
|
||||
{"storage_markers", want.StorageMarkers, got.StorageMarkers},
|
||||
{"policy_values", want.PolicyValues, got.PolicyValues},
|
||||
{"exported_symbols", want.ExportedSymbols, got.ExportedSymbols},
|
||||
{"brand_allowlist", want.BrandAllowlist, got.BrandAllowlist},
|
||||
}
|
||||
for _, check := range checks {
|
||||
@@ -454,10 +378,10 @@ func setDiff(want, got []string) (missing, added []string) {
|
||||
}
|
||||
|
||||
func printSummary(value manifest) {
|
||||
fmt.Printf("compatibility manifest: imports=%d env=%d metrics=%d headers=%d routes=%d roots=%d grid=%d storage=%d policy=%d exported=%d brand=%d sha256=%s\n",
|
||||
fmt.Printf("compatibility manifest: imports=%d env=%d metrics=%d headers=%d routes=%d roots=%d grid=%d storage=%d policy=%d brand=%d sha256=%s\n",
|
||||
len(value.MinioImports), len(value.Environment), len(value.Metrics), len(value.Headers),
|
||||
len(value.Routes), len(value.RouteRoots), len(value.GridRoutes), len(value.StorageMarkers), len(value.PolicyValues),
|
||||
len(value.ExportedSymbols), len(value.BrandAllowlist), manifestDigest(value))
|
||||
len(value.BrandAllowlist), manifestDigest(value))
|
||||
}
|
||||
|
||||
func manifestDigest(value manifest) string {
|
||||
|
||||
@@ -18,8 +18,8 @@ function start_silo_5drive() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MC_HOST_silo="http://silo:silo123@127.0.0.1:${start_port}/"
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
export MC_HOST_silo="http://silo:silo1234@127.0.0.1:${start_port}/"
|
||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
|
||||
@@ -28,8 +28,8 @@ function verify_rewrite() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ACCESS_KEY=silo
|
||||
export MINIO_SECRET_KEY=silo123
|
||||
export MC_HOST_silo="http://silo:silo123@127.0.0.1:${start_port}/"
|
||||
export MINIO_SECRET_KEY=silo1234
|
||||
export MC_HOST_silo="http://silo:silo1234@127.0.0.1:${start_port}/"
|
||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
@@ -87,7 +87,7 @@ function verify_rewrite() {
|
||||
-debug \
|
||||
-versions \
|
||||
-access-key silo \
|
||||
-secret-key silo123 \
|
||||
-secret-key silo1234 \
|
||||
-endpoint "http://127.0.0.1:${start_port}/" 2>&1 | grep INTACT; then
|
||||
echo "server1 log:"
|
||||
cat "${WORK_DIR}/server1.log"
|
||||
@@ -105,14 +105,14 @@ function verify_rewrite() {
|
||||
exit 1
|
||||
fi
|
||||
|
||||
go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "silo" "silo123"
|
||||
go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "silo" "silo1234"
|
||||
sleep 1
|
||||
|
||||
if ! ./s3-check-md5 \
|
||||
-debug \
|
||||
-versions \
|
||||
-access-key silo \
|
||||
-secret-key silo123 \
|
||||
-secret-key silo1234 \
|
||||
-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
|
||||
echo "server1 log:"
|
||||
cat "${WORK_DIR}/server1.log"
|
||||
|
||||
@@ -74,8 +74,8 @@ function test_silo_with_timeout() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MC_HOST_silo="http://silo:silo123@127.0.0.1:${start_port}/"
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
export MC_HOST_silo="http://silo:silo1234@127.0.0.1:${start_port}/"
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
mkdir ${WORK_DIR}
|
||||
|
||||
@@ -15,10 +15,10 @@ WORK_DIR="$PWD/.verify-$RANDOM"
|
||||
export MINT_MODE=core
|
||||
export MINT_DATA_DIR="$WORK_DIR/data"
|
||||
export SERVER_ENDPOINT="127.0.0.1:9000"
|
||||
export MC_HOST_verify="http://silo:silo123@${SERVER_ENDPOINT}/"
|
||||
export MC_HOST_verify_ipv6="http://silo:silo123@[::1]:9000/"
|
||||
export MC_HOST_verify="http://silo:silo1234@${SERVER_ENDPOINT}/"
|
||||
export MC_HOST_verify_ipv6="http://silo:silo1234@[::1]:9000/"
|
||||
export ACCESS_KEY="silo"
|
||||
export SECRET_KEY="silo123"
|
||||
export SECRET_KEY="silo1234"
|
||||
export ENABLE_HTTPS=0
|
||||
export GO111MODULE=on
|
||||
export GOGC=25
|
||||
@@ -225,7 +225,7 @@ function __init__() {
|
||||
shred -n 1 -s 65M - 1>"$FILE_65_MB" 2>/dev/null
|
||||
|
||||
## version is purposefully set to '3' for minio to migrate configuration file
|
||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo123"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo1234"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||
|
||||
"$(git rev-parse --show-toplevel)/buildscripts/install-verified-fixture.sh" \
|
||||
https://raw.githubusercontent.com/pgsty/mc/4c4dcc4b55baf238cd0c81030d77945b3828f157/functional-tests.sh \
|
||||
@@ -282,7 +282,6 @@ function main() {
|
||||
purge "$WORK_DIR"
|
||||
}
|
||||
|
||||
(__init__ "$@" && main "$@")
|
||||
rv=$?
|
||||
purge "$WORK_DIR"
|
||||
exit "$rv"
|
||||
trap 'purge "$WORK_DIR"' EXIT
|
||||
__init__ "$@"
|
||||
main "$@"
|
||||
|
||||
@@ -15,7 +15,7 @@ SILO=("$PWD/silo" --config-dir "$SILO_CONFIG_DIR" server)
|
||||
|
||||
function start_silo_3_node() {
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
export MINIO_ERASURE_SET_DRIVE_COUNT=6
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
@@ -37,7 +37,7 @@ function start_silo_3_node() {
|
||||
pid3=$!
|
||||
disown $pid3
|
||||
|
||||
export MC_HOST_mysilo="http://silo:silo123@127.0.0.1:$((start_port + 1))"
|
||||
export MC_HOST_mysilo="http://silo:silo1234@127.0.0.1:$((start_port + 1))"
|
||||
|
||||
timeout 15m /tmp/mc ready mysilo || fail
|
||||
|
||||
@@ -116,7 +116,7 @@ function __init__() {
|
||||
mkdir -p "$SILO_CONFIG_DIR"
|
||||
|
||||
## version is purposefully set to '3' for minio to migrate configuration file
|
||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo123"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo1234"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||
|
||||
if [ ! -f /tmp/mc ]; then
|
||||
"$(git rev-parse --show-toplevel)/buildscripts/install-mcli.sh" /tmp/mc
|
||||
|
||||
@@ -17,7 +17,7 @@ function start_silo() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||
unset MINIO_CI_CD
|
||||
unset CI
|
||||
|
||||
@@ -20,7 +20,7 @@ function start_silo_3_node() {
|
||||
done
|
||||
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
export MINIO_ERASURE_SET_DRIVE_COUNT=6
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
@@ -46,7 +46,7 @@ function start_silo_3_node() {
|
||||
pid3=$!
|
||||
disown $pid3
|
||||
|
||||
export MC_HOST_mysilo="http://silo:silo123@127.0.0.1:$((start_port + 1))"
|
||||
export MC_HOST_mysilo="http://silo:silo1234@127.0.0.1:$((start_port + 1))"
|
||||
timeout 15m /tmp/mc ready mysilo || fail
|
||||
|
||||
[ ${first_time} -eq 0 ] && upload_objects
|
||||
@@ -117,7 +117,7 @@ function __init__() {
|
||||
mkdir -p "$SILO_CONFIG_DIR"
|
||||
|
||||
## version is purposefully set to '3' for minio to migrate configuration file
|
||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo123"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo1234"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||
|
||||
if [ ! -f /tmp/mc ]; then
|
||||
"$(git rev-parse --show-toplevel)/buildscripts/install-mcli.sh" /tmp/mc
|
||||
|
||||
@@ -113,7 +113,7 @@ helm_run template my-release "${new_chart}" \
|
||||
go run ./buildscripts/helm-migration-guard "${old_render}" "${new_render}"
|
||||
|
||||
helm_run package "${new_chart}" --destination "${output_dir}" >/dev/null
|
||||
test -s "${work_dir}/silo-7.0.1.tgz"
|
||||
test -s "${work_dir}/silo-7.0.2.tgz"
|
||||
if find "${work_dir}" -maxdepth 1 -type f -name 'minio-*.tgz' | grep -q .; then
|
||||
echo "Helm packaging emitted a legacy MinIO chart name" >&2
|
||||
exit 1
|
||||
|
||||
@@ -159,11 +159,13 @@ fi
|
||||
# The repository and its default branch are pgsty/silo and main. The invariant
|
||||
# is that the old name is never a live target, not that it is never spoken: the
|
||||
# READMEs have to name it to explain the rename and to point at the archived
|
||||
# artifacts, which is the opposite of stranding a reader on it.
|
||||
# artifacts, which is the opposite of stranding a reader on it. CONTRIBUTORS.md
|
||||
# also quotes historical issue titles.
|
||||
#
|
||||
# So two rules. First, no live URL may resolve to the old repository anywhere,
|
||||
# READMEs included.
|
||||
stale_repo_url="$(rg -n -e 'github\.com/pgsty/minio' -e 'hub\.docker\.com/r/pgsty/minio' \
|
||||
# READMEs and CONTRIBUTORS.md included.
|
||||
old_repo_pattern='pgsty/minio(\.git)?([^[:alnum:]_.-]|$)'
|
||||
stale_repo_url="$(rg -n -e "github\.com/${old_repo_pattern}" -e "hub\.docker\.com/r/${old_repo_pattern}" \
|
||||
--glob '!.git/**' --glob '!dist/**' \
|
||||
--glob '!SILO_REBRANDING_MIGRATION.md' \
|
||||
--glob '!buildscripts/rebrand-guard/compat-baseline.json' . |
|
||||
@@ -175,10 +177,10 @@ fi
|
||||
|
||||
# Second, the bare name may only appear where it is deliberate: the pinned
|
||||
# pre-rebrand image digest in the upgrade test, the two guards that refuse a
|
||||
# legacy image, and the two READMEs that document the rename and the archived
|
||||
# minio branch.
|
||||
repo_guard_allowlist='^(buildscripts/minio-upgrade\.sh|buildscripts/verify-rebrand\.sh|buildscripts/helm-migration-guard/main\.go|README\.md|README_ZH\.md):'
|
||||
stale_repo="$(rg -n 'pgsty/minio' --glob '!.git/**' --glob '!dist/**' \
|
||||
# legacy image, the two READMEs that document the rename and the archived
|
||||
# minio branch, and historical issue titles in CONTRIBUTORS.md.
|
||||
repo_guard_allowlist='^(buildscripts/minio-upgrade\.sh|buildscripts/verify-rebrand\.sh|buildscripts/helm-migration-guard/main\.go|README\.md|README_ZH\.md|CONTRIBUTORS\.md):'
|
||||
stale_repo="$(rg -n "${old_repo_pattern}" --glob '!.git/**' --glob '!dist/**' \
|
||||
--glob '!SILO_REBRANDING_MIGRATION.md' \
|
||||
--glob '!buildscripts/rebrand-guard/compat-baseline.json' . |
|
||||
sed 's#^\./##' | grep -Ev "${repo_guard_allowlist}" || true)"
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// Data types used for returning dummy access control
|
||||
|
||||
@@ -0,0 +1,348 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/mux"
|
||||
)
|
||||
|
||||
func corsAdminRequest(t *testing.T, cred auth.Credentials, method, path string, body []byte) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
router := mux.NewRouter()
|
||||
registerAdminRouter(router, true)
|
||||
req, err := newTestSignedRequestV4(method, adminPathPrefix+adminAPIVersionPrefix+path,
|
||||
int64(len(body)), bytes.NewReader(body), cred.AccessKey, cred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("admin %s: %d: %s", path, rec.Code, rec.Body.String())
|
||||
}
|
||||
return rec
|
||||
}
|
||||
|
||||
func corsImportReport(t *testing.T, rec *httptest.ResponseRecorder) madmin.BucketMetaImportErrs {
|
||||
t.Helper()
|
||||
var rpt madmin.BucketMetaImportErrs
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &rpt); err != nil {
|
||||
t.Fatalf("import report %q: %v", rec.Body.String(), err)
|
||||
}
|
||||
return rpt
|
||||
}
|
||||
|
||||
func corsZip(t *testing.T, entries map[string][]byte) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for name, data := range entries {
|
||||
w, err := zw.Create(name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = w.Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// corsCorruptedZip builds an archive holding a stored (uncompressed) cors.xml
|
||||
// whose payload is altered after the checksum is computed, plus the given
|
||||
// companion entries. The altered document stays well formed, so only the zip
|
||||
// checksum tells the two apart.
|
||||
func corsCorruptedZip(t *testing.T, name string, doc []byte, others map[string][]byte) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
w, err := zw.CreateHeader(&zip.FileHeader{Name: name, Method: zip.Store})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = w.Write(doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for other, data := range others {
|
||||
ow, err := zw.Create(other)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = ow.Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err = zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw := buf.Bytes()
|
||||
at := bytes.Index(raw, []byte("app.example.com"))
|
||||
if at < 0 {
|
||||
t.Fatalf("stored CORS payload not found in archive")
|
||||
}
|
||||
raw[at] = 'A'
|
||||
return raw
|
||||
}
|
||||
|
||||
// TestAdminBucketMetadataCORSRoundTrip covers the export/import round trip for
|
||||
// per-bucket CORS, per-file error reporting for an invalid document, and that
|
||||
// an archive without cors.xml leaves an existing configuration alone.
|
||||
func TestAdminBucketMetadataCORSRoundTrip(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instanceType, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
corsXML := []byte(testSiteReplicationCORSDoc)
|
||||
if _, err := updateLocalBucketCORSMetadata(t.Context(), obj, bucket, corsXML); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Export must carry the stored document verbatim.
|
||||
rec := corsAdminRequest(t, cred, http.MethodGet, "/export-bucket-metadata?bucket="+bucket, nil)
|
||||
archive := rec.Body.Bytes()
|
||||
zr, err := zip.NewReader(bytes.NewReader(archive), int64(len(archive)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var exported []byte
|
||||
for _, f := range zr.File {
|
||||
if f.Name != bucket+"/"+bucketCorsConfig {
|
||||
continue
|
||||
}
|
||||
r, err := f.Open()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exported, err = io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if !bytes.Equal(exported, corsXML) {
|
||||
t.Fatalf("%s: exported CORS = %q, want %q", instanceType, exported, corsXML)
|
||||
}
|
||||
|
||||
// Drop the configuration: the archive must then omit the entry.
|
||||
if _, err = updateLocalBucketCORSMetadata(t.Context(), obj, bucket, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err = globalBucketMetadataSys.GetCorsConfigXML(bucket); err == nil {
|
||||
t.Fatalf("%s: CORS still present before restore", instanceType)
|
||||
}
|
||||
rec = corsAdminRequest(t, cred, http.MethodGet, "/export-bucket-metadata?bucket="+bucket, nil)
|
||||
empty := rec.Body.Bytes()
|
||||
zr, err = zip.NewReader(bytes.NewReader(empty), int64(len(empty)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range zr.File {
|
||||
if f.Name == bucket+"/"+bucketCorsConfig {
|
||||
t.Fatalf("%s: export emitted %s for a bucket without CORS", instanceType, f.Name)
|
||||
}
|
||||
}
|
||||
|
||||
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata", archive)
|
||||
if st := corsImportReport(t, rec).Buckets[bucket]; !st.Cors.IsSet || st.Cors.Err != "" {
|
||||
t.Fatalf("%s: import report cors = %+v", instanceType, st.Cors)
|
||||
}
|
||||
stored, storedAt, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil || !bytes.Equal(stored, corsXML) {
|
||||
t.Fatalf("%s: restored CORS = %q, err = %v", instanceType, stored, err)
|
||||
}
|
||||
created, err := globalBucketMetadataSys.CreatedAt(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !storedAt.After(created) {
|
||||
t.Fatalf("%s: restored CORS timestamp %v is not after bucket creation %v", instanceType, storedAt, created)
|
||||
}
|
||||
|
||||
// An archive without cors.xml must not remove the configuration.
|
||||
corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||
corsZip(t, map[string][]byte{bucket + "/quota.json": []byte(`{"quota":0}`)}))
|
||||
if stored, _, err = globalBucketMetadataSys.GetCorsConfigXML(bucket); err != nil || !bytes.Equal(stored, corsXML) {
|
||||
t.Fatalf("%s: import without cors.xml changed CORS: %q, err = %v", instanceType, stored, err)
|
||||
}
|
||||
|
||||
// A bucket the import itself creates must still land above its own
|
||||
// creation time, otherwise CORS replication would drop the restore.
|
||||
fresh := "cors-import-created-bucket"
|
||||
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||
corsZip(t, map[string][]byte{fresh + "/" + bucketCorsConfig: corsXML}))
|
||||
if st := corsImportReport(t, rec).Buckets[fresh]; !st.Cors.IsSet || st.Cors.Err != "" {
|
||||
t.Fatalf("%s: fresh bucket import report cors = %+v", instanceType, st.Cors)
|
||||
}
|
||||
freshStored, freshAt, err := globalBucketMetadataSys.GetCorsConfigXML(fresh)
|
||||
if err != nil || !bytes.Equal(freshStored, corsXML) {
|
||||
t.Fatalf("%s: fresh bucket CORS = %q, err = %v", instanceType, freshStored, err)
|
||||
}
|
||||
freshCreated, err := globalBucketMetadataSys.CreatedAt(fresh)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !freshAt.After(freshCreated) {
|
||||
t.Fatalf("%s: fresh bucket CORS timestamp %v is not after creation %v", instanceType, freshAt, freshCreated)
|
||||
}
|
||||
|
||||
// An invalid document must fail loudly for that bucket and change nothing.
|
||||
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||
corsZip(t, map[string][]byte{bucket + "/" + bucketCorsConfig: []byte("<CORSConfiguration><CORSRule>")}))
|
||||
if st := corsImportReport(t, rec).Buckets[bucket]; st.Cors.Err == "" {
|
||||
t.Fatalf("%s: invalid CORS import reported no error: %+v", instanceType, st)
|
||||
}
|
||||
if stored, _, err = globalBucketMetadataSys.GetCorsConfigXML(bucket); err != nil || !bytes.Equal(stored, corsXML) {
|
||||
t.Fatalf("%s: invalid CORS import changed stored config: %q, err = %v", instanceType, stored, err)
|
||||
}
|
||||
|
||||
// A well formed document carried by a corrupt zip entry must be
|
||||
// rejected too, leaving the stored document and its timestamp alone
|
||||
// while the other configs in the same archive still apply.
|
||||
_, corsAt, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||
corsCorruptedZip(t, bucket+"/"+bucketCorsConfig, corsXML,
|
||||
map[string][]byte{bucket + "/quota.json": []byte(`{"quota":4096,"quotatype":"hard"}`)}))
|
||||
st := corsImportReport(t, rec).Buckets[bucket]
|
||||
if st.Cors.Err == "" {
|
||||
t.Fatalf("%s: corrupt CORS entry reported no error: %+v", instanceType, st)
|
||||
}
|
||||
if !st.Quota.IsSet || st.Quota.Err != "" {
|
||||
t.Fatalf("%s: corrupt CORS entry blocked the neighboring quota: %+v", instanceType, st.Quota)
|
||||
}
|
||||
stored, storedAt, err = globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil || !bytes.Equal(stored, corsXML) || !storedAt.Equal(corsAt) {
|
||||
t.Fatalf("%s: corrupt CORS entry changed stored config: %q at %v (was %v), err = %v", instanceType, stored, storedAt, corsAt, err)
|
||||
}
|
||||
quota, _, err := globalBucketMetadataSys.GetQuotaConfig(t.Context(), bucket)
|
||||
if err != nil || quota == nil || quota.Quota != 4096 {
|
||||
t.Fatalf("%s: neighboring quota not applied: %+v, err = %v", instanceType, quota, err)
|
||||
}
|
||||
}})
|
||||
}
|
||||
|
||||
// corsPeerStub is a stand-in site-replication peer. It records every
|
||||
// SRBucketMeta it is asked to apply and answers with status.
|
||||
func corsPeerStub(t *testing.T, applied chan<- madmin.SRBucketMeta, status int) *httptest.Server {
|
||||
t.Helper()
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodPut && applied != nil {
|
||||
var item madmin.SRBucketMeta
|
||||
if err := json.NewDecoder(r.Body).Decode(&item); err != nil {
|
||||
t.Errorf("decode peer apply: %v", err)
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
applied <- item
|
||||
}
|
||||
w.WriteHeader(status)
|
||||
}))
|
||||
}
|
||||
|
||||
// TestAdminBucketMetadataCORSImportReplicatesPastPeerFailure pins that an
|
||||
// imported CORS document reaches the reachable peers even when the shared
|
||||
// bucket metadata hook failed against an unreachable one, and that both
|
||||
// failures are still reported for the bucket.
|
||||
func TestAdminBucketMetadataCORSImportReplicatesPastPeerFailure(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instanceType, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
ctx := t.Context()
|
||||
corsXML := []byte(testSiteReplicationCORSDoc)
|
||||
|
||||
healthyApplies := make(chan madmin.SRBucketMeta, 4)
|
||||
healthy := corsPeerStub(t, healthyApplies, http.StatusOK)
|
||||
defer healthy.Close()
|
||||
broken := corsPeerStub(t, nil, http.StatusBadRequest)
|
||||
defer broken.Close()
|
||||
|
||||
// With site replication on, admin requests resolve their token signing
|
||||
// key through the site replicator account, so it has to exist.
|
||||
serviceCred, err := auth.CreateCredentials(siteReplicatorSvcAcc, "cors-import-service-secret")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serviceCred.ParentUser = cred.AccessKey
|
||||
if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
|
||||
globalSiteReplicatorCred.Set(serviceCred.SecretKey)
|
||||
defer globalSiteReplicatorCred.Set("")
|
||||
|
||||
globalSiteReplicationSys.Lock()
|
||||
oldEnabled, oldState := globalSiteReplicationSys.enabled, globalSiteReplicationSys.state
|
||||
globalSiteReplicationSys.enabled = true
|
||||
globalSiteReplicationSys.state = srState{
|
||||
Name: "cors-import-test",
|
||||
ServiceAccountAccessKey: serviceCred.AccessKey,
|
||||
Peers: map[string]madmin.PeerInfo{
|
||||
globalDeploymentID(): {Name: "local", DeploymentID: globalDeploymentID()},
|
||||
"peer-healthy": {Name: "healthy", DeploymentID: "peer-healthy", Endpoint: healthy.URL},
|
||||
"peer-broken": {Name: "broken", DeploymentID: "peer-broken", Endpoint: broken.URL},
|
||||
},
|
||||
}
|
||||
globalSiteReplicationSys.Unlock()
|
||||
defer func() {
|
||||
globalSiteReplicationSys.Lock()
|
||||
globalSiteReplicationSys.enabled, globalSiteReplicationSys.state = oldEnabled, oldState
|
||||
globalSiteReplicationSys.Unlock()
|
||||
}()
|
||||
|
||||
rec := corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||
corsZip(t, map[string][]byte{
|
||||
bucket + "/" + bucketCorsConfig: corsXML,
|
||||
bucket + "/quota.json": []byte(`{"quota":8192,"quotatype":"hard"}`),
|
||||
}))
|
||||
st := corsImportReport(t, rec).Buckets[bucket]
|
||||
if !st.Cors.IsSet || st.Cors.Err != "" {
|
||||
t.Fatalf("%s: import report cors = %+v", instanceType, st.Cors)
|
||||
}
|
||||
stored, storedAt, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil || !bytes.Equal(stored, corsXML) {
|
||||
t.Fatalf("%s: stored CORS = %q, err = %v", instanceType, stored, err)
|
||||
}
|
||||
|
||||
// The reachable peer must have been told about the CORS document,
|
||||
// carrying exactly the timestamp that was saved locally.
|
||||
var corsSeen, sharedSeen bool
|
||||
for range 2 {
|
||||
select {
|
||||
case item := <-healthyApplies:
|
||||
if item.Type != madmin.SRBucketMetaTypeCorsConfig {
|
||||
sharedSeen = item.Bucket == bucket && item.Quota != nil
|
||||
continue
|
||||
}
|
||||
if item.Bucket != bucket || item.Cors == nil || !item.UpdatedAt.Equal(storedAt) {
|
||||
t.Fatalf("%s: peer CORS event = %#v, want %s at %v", instanceType, item, bucket, storedAt)
|
||||
}
|
||||
payload, decErr := base64.StdEncoding.Strict().DecodeString(*item.Cors)
|
||||
if decErr != nil || !bytes.Equal(payload, corsXML) {
|
||||
t.Fatalf("%s: peer CORS payload = %q, err = %v", instanceType, payload, decErr)
|
||||
}
|
||||
corsSeen = true
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("%s: healthy peer received no further events (shared=%v cors=%v)", instanceType, sharedSeen, corsSeen)
|
||||
}
|
||||
}
|
||||
if !sharedSeen || !corsSeen {
|
||||
t.Fatalf("%s: healthy peer events shared=%v cors=%v, want both", instanceType, sharedSeen, corsSeen)
|
||||
}
|
||||
|
||||
// Both hook failures against the unreachable peer stay reported.
|
||||
if got := strings.Count(st.Err, "->broken:"); got != 2 {
|
||||
t.Fatalf("%s: bucket error mentions the broken peer %d times, want 2: %q", instanceType, got, st.Err)
|
||||
}
|
||||
}})
|
||||
}
|
||||
+175
-13
@@ -41,7 +41,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -417,6 +417,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
bucketLifecycleConfig,
|
||||
bucketSSEConfig,
|
||||
bucketTaggingConfig,
|
||||
bucketCorsConfig,
|
||||
bucketQuotaConfigFile,
|
||||
objectLockConfig,
|
||||
bucketVersioningConfig,
|
||||
@@ -517,6 +518,19 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case bucketCorsConfig:
|
||||
// Export the stored document verbatim: GetBucketCors returns
|
||||
// the bytes exactly as they were PUT, so the archive must
|
||||
// round-trip them unchanged.
|
||||
configData, _, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, errConfigNotFound) {
|
||||
continue
|
||||
}
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case objectLockConfig:
|
||||
config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
|
||||
if err != nil {
|
||||
@@ -589,6 +603,50 @@ type importMetaReport struct {
|
||||
madmin.BucketMetaImportErrs
|
||||
}
|
||||
|
||||
type importMetadataFields map[string]struct{}
|
||||
|
||||
func (f importMetadataFields) add(configFile string) {
|
||||
f[configFile] = struct{}{}
|
||||
}
|
||||
|
||||
func applyImportedBucketMetadata(dst *BucketMetadata, src BucketMetadata, fields importMetadataFields) {
|
||||
for configFile := range fields {
|
||||
switch configFile {
|
||||
case bucketPolicyConfig:
|
||||
dst.PolicyConfigJSON = bytes.Clone(src.PolicyConfigJSON)
|
||||
dst.PolicyConfigUpdatedAt = src.PolicyConfigUpdatedAt
|
||||
case bucketNotificationConfig:
|
||||
dst.NotificationConfigXML = bytes.Clone(src.NotificationConfigXML)
|
||||
dst.NotificationConfigUpdatedAt = src.NotificationConfigUpdatedAt
|
||||
case bucketLifecycleConfig:
|
||||
dst.LifecycleConfigXML = bytes.Clone(src.LifecycleConfigXML)
|
||||
dst.LifecycleConfigUpdatedAt = src.LifecycleConfigUpdatedAt
|
||||
case bucketSSEConfig:
|
||||
dst.EncryptionConfigXML = bytes.Clone(src.EncryptionConfigXML)
|
||||
dst.EncryptionConfigUpdatedAt = src.EncryptionConfigUpdatedAt
|
||||
case bucketTaggingConfig:
|
||||
dst.TaggingConfigXML = bytes.Clone(src.TaggingConfigXML)
|
||||
dst.TaggingConfigUpdatedAt = src.TaggingConfigUpdatedAt
|
||||
case bucketQuotaConfigFile:
|
||||
dst.QuotaConfigJSON = bytes.Clone(src.QuotaConfigJSON)
|
||||
dst.QuotaConfigUpdatedAt = src.QuotaConfigUpdatedAt
|
||||
case bucketCorsConfig:
|
||||
// The import stamps its fields before creating any missing bucket,
|
||||
// and a CORS event stamped before bucket creation is discarded as
|
||||
// belonging to an older incarnation, so the imported document takes
|
||||
// the same monotonic timestamp a local PutBucketCors would assign.
|
||||
dst.CorsConfigUpdatedAt = localCORSUpdatedAt(*dst, src.CorsConfigUpdatedAt)
|
||||
dst.CorsConfigXML = bytes.Clone(src.CorsConfigXML)
|
||||
case objectLockConfig:
|
||||
dst.ObjectLockConfigXML = bytes.Clone(src.ObjectLockConfigXML)
|
||||
dst.ObjectLockConfigUpdatedAt = src.ObjectLockConfigUpdatedAt
|
||||
case bucketVersioningConfig:
|
||||
dst.VersioningConfigXML = bytes.Clone(src.VersioningConfigXML)
|
||||
dst.VersioningConfigUpdatedAt = src.VersioningConfigUpdatedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (i *importMetaReport) SetStatus(bucket, fname string, err error) {
|
||||
st := i.Buckets[bucket]
|
||||
var errMsg string
|
||||
@@ -608,6 +666,8 @@ func (i *importMetaReport) SetStatus(bucket, fname string, err error) {
|
||||
st.Tagging = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case bucketQuotaConfigFile:
|
||||
st.Quota = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case bucketCorsConfig:
|
||||
st.Cors = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case objectLockConfig:
|
||||
st.ObjectLock = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case bucketVersioningConfig:
|
||||
@@ -649,6 +709,16 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
}
|
||||
|
||||
bucketMap := make(map[string]*BucketMetadata, len(zr.File))
|
||||
importedFields := make(map[string]importMetadataFields, len(zr.File))
|
||||
blockedBuckets := make(map[string]struct{})
|
||||
markImported := func(bucket, configFile string) {
|
||||
fields := importedFields[bucket]
|
||||
if fields == nil {
|
||||
fields = make(importMetadataFields)
|
||||
importedFields[bucket] = fields
|
||||
}
|
||||
fields.add(configFile)
|
||||
}
|
||||
|
||||
updatedAt := UTCNow()
|
||||
|
||||
@@ -664,6 +734,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
bucketMap[bucket] = &meta
|
||||
} else if err != errConfigNotFound {
|
||||
rpt.SetStatus(bucket, "", err)
|
||||
blockedBuckets[bucket] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -675,6 +746,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
bucket, fileName := slc[0], slc[1]
|
||||
if _, blocked := blockedBuckets[bucket]; blocked {
|
||||
continue
|
||||
}
|
||||
if fileName == objectLockConfig {
|
||||
reader, err := file.Open()
|
||||
if err != nil {
|
||||
@@ -708,6 +782,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].ObjectLockConfigXML = configData
|
||||
bucketMap[bucket].ObjectLockConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
}
|
||||
}
|
||||
@@ -720,6 +795,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
bucket, fileName := slc[0], slc[1]
|
||||
if _, blocked := blockedBuckets[bucket]; blocked {
|
||||
continue
|
||||
}
|
||||
if fileName == bucketVersioningConfig {
|
||||
reader, err := file.Open()
|
||||
if err != nil {
|
||||
@@ -764,6 +842,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].VersioningConfigXML = configData
|
||||
bucketMap[bucket].VersioningConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
}
|
||||
}
|
||||
@@ -781,6 +860,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
bucket, fileName := slc[0], slc[1]
|
||||
if _, blocked := blockedBuckets[bucket]; blocked {
|
||||
continue
|
||||
}
|
||||
|
||||
// create bucket if it does not exist yet.
|
||||
if _, ok := bucketMap[bucket]; !ok {
|
||||
@@ -813,6 +895,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].NotificationConfigXML = configData
|
||||
bucketMap[bucket].NotificationConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketPolicyConfig:
|
||||
// Error out if Content-Length is beyond allowed size.
|
||||
@@ -847,6 +930,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].PolicyConfigJSON = configData
|
||||
bucketMap[bucket].PolicyConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketLifecycleConfig:
|
||||
bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(reader, sz))
|
||||
@@ -879,6 +963,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].LifecycleConfigXML = configData
|
||||
bucketMap[bucket].LifecycleConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketSSEConfig:
|
||||
// Parse bucket encryption xml
|
||||
@@ -917,6 +1002,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].EncryptionConfigXML = configData
|
||||
bucketMap[bucket].EncryptionConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketTaggingConfig:
|
||||
tags, err := tags.ParseBucketXML(io.LimitReader(reader, sz))
|
||||
@@ -933,6 +1019,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].TaggingConfigXML = configData
|
||||
bucketMap[bucket].TaggingConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketQuotaConfigFile:
|
||||
data, err := io.ReadAll(reader)
|
||||
@@ -949,6 +1036,33 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].QuotaConfigJSON = data
|
||||
bucketMap[bucket].QuotaConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketCorsConfig:
|
||||
if sz > maxBucketCorsSize {
|
||||
rpt.SetStatus(bucket, fileName, errors.New(ErrEntityTooLarge.String()))
|
||||
continue
|
||||
}
|
||||
|
||||
// Read one byte past the declared size: stopping exactly at sz
|
||||
// leaves archive/zip short of EOF, so it never verifies the entry
|
||||
// checksum and a corrupt entry carrying well formed XML would be
|
||||
// stored as a valid document. The extra byte also lets the reader
|
||||
// reject an entry longer than it declares.
|
||||
corsData, err := io.ReadAll(io.LimitReader(reader, sz+1))
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if err = validateCORSReplicationPayload(corsData); err != nil {
|
||||
rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
|
||||
continue
|
||||
}
|
||||
|
||||
bucketMap[bucket].CorsConfigXML = corsData
|
||||
bucketMap[bucket].CorsConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
}
|
||||
}
|
||||
@@ -962,22 +1076,70 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
}
|
||||
|
||||
for bucket, meta := range bucketMap {
|
||||
err := globalBucketMetadataSys.save(ctx, *meta)
|
||||
fields := importedFields[bucket]
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
var merged BucketMetadata
|
||||
err := func() error {
|
||||
lockCtx, unlock, err := lockBucketMetadata(ctx, objectAPI, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer unlock()
|
||||
merged, err = loadBucketMetadataParse(lockCtx, objectAPI, bucket, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
applyImportedBucketMetadata(&merged, *meta, fields)
|
||||
return globalBucketMetadataSys.saveMetadata(lockCtx, objectAPI, merged)
|
||||
}()
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, "", err)
|
||||
continue
|
||||
}
|
||||
// Call site replication hook.
|
||||
if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Bucket: bucket,
|
||||
Quota: meta.QuotaConfigJSON,
|
||||
Policy: meta.PolicyConfigJSON,
|
||||
Versioning: enc(meta.VersioningConfigXML),
|
||||
Tags: enc(meta.TaggingConfigXML),
|
||||
ObjectLockConfig: enc(meta.ObjectLockConfigXML),
|
||||
SSEConfig: enc(meta.EncryptionConfigXML),
|
||||
UpdatedAt: updatedAt,
|
||||
}); err != nil {
|
||||
*meta = merged
|
||||
globalNotificationSys.LoadBucketMetadata(bgContext(ctx), bucket)
|
||||
hook := madmin.SRBucketMeta{Bucket: bucket, UpdatedAt: updatedAt}
|
||||
var hookNeeded bool
|
||||
if _, ok := fields[bucketQuotaConfigFile]; ok {
|
||||
hook.Quota = meta.QuotaConfigJSON
|
||||
hookNeeded = true
|
||||
}
|
||||
if _, ok := fields[bucketPolicyConfig]; ok {
|
||||
hook.Policy = meta.PolicyConfigJSON
|
||||
hookNeeded = true
|
||||
}
|
||||
if _, ok := fields[bucketVersioningConfig]; ok {
|
||||
hook.Versioning = enc(meta.VersioningConfigXML)
|
||||
hookNeeded = true
|
||||
}
|
||||
if _, ok := fields[bucketTaggingConfig]; ok {
|
||||
hook.Tags = enc(meta.TaggingConfigXML)
|
||||
hookNeeded = true
|
||||
}
|
||||
if _, ok := fields[objectLockConfig]; ok {
|
||||
hook.ObjectLockConfig = enc(meta.ObjectLockConfigXML)
|
||||
hookNeeded = true
|
||||
}
|
||||
if _, ok := fields[bucketSSEConfig]; ok {
|
||||
hook.SSEConfig = enc(meta.EncryptionConfigXML)
|
||||
hookNeeded = true
|
||||
}
|
||||
if hookNeeded {
|
||||
err = globalSiteReplicationSys.BucketMetaHook(ctx, hook)
|
||||
}
|
||||
if _, ok := fields[bucketCorsConfig]; ok {
|
||||
// CORS carries its own timestamp, so it replicates through the
|
||||
// dedicated event rather than the shared bucket metadata hook. It
|
||||
// is announced even when the shared hook failed: the document is
|
||||
// already committed locally, and a peer that is unreachable for
|
||||
// one config must not withhold CORS from the reachable ones.
|
||||
if corsEvent, live := newBucketCORSReplicationEvent(bucket, *meta); live {
|
||||
err = errors.Join(err, globalSiteReplicationSys.BucketMetaHook(ctx, corsEvent))
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, "", err)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// validateAdminReq will validate request against and return whether it is allowed.
|
||||
|
||||
@@ -37,7 +37,7 @@ import (
|
||||
"github.com/minio/minio/internal/config/subnet"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// DelConfigKVHandler - DELETE /minio/admin/v3/del-config-kv
|
||||
|
||||
@@ -32,8 +32,8 @@ import (
|
||||
cfgldap "github.com/minio/minio/internal/config/identity/ldap"
|
||||
"github.com/minio/minio/internal/config/identity/openid"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/ldap"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/ldap"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
|
||||
|
||||
@@ -28,8 +28,8 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/mux"
|
||||
xldap "github.com/minio/pkg/v3/ldap"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
xldap "github.com/pgsty/silo-pkg/v3/ldap"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// ListLDAPPolicyMappingEntities lists users/groups mapped to given/all policies.
|
||||
|
||||
@@ -25,7 +25,7 @@ import (
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const dummyRoleARN = "dummy-internal"
|
||||
|
||||
@@ -27,8 +27,8 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
@@ -33,7 +33,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// SiteReplicationAdd - PUT /minio/admin/v3/site-replication/add
|
||||
@@ -255,9 +255,11 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
|
||||
case madmin.SRBucketMetaTypeTags:
|
||||
err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags, item.UpdatedAt)
|
||||
case madmin.SRBucketMetaTypeObjectLockConfig:
|
||||
err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig, item.UpdatedAt)
|
||||
err = globalSiteReplicationSys.peerBucketObjectLockConfigItem(ctx, item)
|
||||
case madmin.SRBucketMetaTypeSSEConfig:
|
||||
err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig, item.UpdatedAt)
|
||||
case madmin.SRBucketMetaTypeCorsConfig:
|
||||
err = globalSiteReplicationSys.PeerBucketCorsConfigHandler(ctx, item.Bucket, item.Cors, item.UpdatedAt)
|
||||
case madmin.SRBucketMetaLCConfig:
|
||||
err = globalSiteReplicationSys.PeerBucketLCConfigHandler(ctx, item.Bucket, item.ExpiryLCConfig, item.UpdatedAt)
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ import (
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
minio "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
)
|
||||
|
||||
func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {
|
||||
|
||||
+29
-15
@@ -40,8 +40,8 @@ import (
|
||||
"github.com/minio/minio/internal/config/dns"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
xldap "github.com/minio/pkg/v3/ldap"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
xldap "github.com/pgsty/silo-pkg/v3/ldap"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/puzpuzpuz/xsync/v3"
|
||||
)
|
||||
|
||||
@@ -355,18 +355,25 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// SetGroupStatus - PUT /minio/admin/v3/set-group-status?group=mygroup1&status=enabled
|
||||
func setGroupStatusAdminAction(status string) policy.AdminAction {
|
||||
if madmin.GroupStatus(status) == madmin.GroupDisabled {
|
||||
return policy.DisableGroupAdminAction
|
||||
}
|
||||
return policy.EnableGroupAdminAction
|
||||
}
|
||||
|
||||
func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.EnableGroupAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
group := vars["group"]
|
||||
status := vars["status"]
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, setGroupStatusAdminAction(status))
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
var (
|
||||
err error
|
||||
updatedAt time.Time
|
||||
@@ -398,18 +405,25 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
|
||||
// SetUserStatus - PUT /minio/admin/v3/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
|
||||
func setUserStatusAdminAction(status string) policy.AdminAction {
|
||||
if madmin.AccountStatus(status) == madmin.AccountDisabled {
|
||||
return policy.DisableUserAdminAction
|
||||
}
|
||||
return policy.EnableUserAdminAction
|
||||
}
|
||||
|
||||
func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, creds := validateAdminReq(ctx, w, r, policy.EnableUserAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
accessKey := vars["accessKey"]
|
||||
status := vars["status"]
|
||||
|
||||
objectAPI, creds := validateAdminReq(ctx, w, r, setUserStatusAdminAction(status))
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// you cannot enable or disable yourself.
|
||||
if accessKey == creds.AccessKey {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
|
||||
@@ -859,7 +873,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
|
||||
var sp *policy.Policy
|
||||
if len(updateReq.NewPolicy) > 0 {
|
||||
sp, err = policy.ParseConfig(bytes.NewReader(updateReq.NewPolicy))
|
||||
sp, err = policy.ParseConfigStrict(bytes.NewReader(updateReq.NewPolicy))
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -1729,7 +1743,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
|
||||
return
|
||||
}
|
||||
|
||||
iamPolicy, err := policy.ParseConfig(bytes.NewReader(iamPolicyBytes))
|
||||
iamPolicy, err := policy.ParseConfigStrict(bytes.NewReader(iamPolicyBytes))
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -2981,7 +2995,7 @@ func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.
|
||||
|
||||
var sp *policy.Policy
|
||||
if len(createReq.Policy) > 0 {
|
||||
sp, err = policy.ParseConfig(bytes.NewReader(createReq.Policy))
|
||||
sp, err = policy.ParseConfigStrict(bytes.NewReader(createReq.Policy))
|
||||
if err != nil {
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err)
|
||||
}
|
||||
|
||||
@@ -40,13 +40,54 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio-go/v7/pkg/signer"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
testDefaultTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
func TestSetUserStatusAdminAction(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
status string
|
||||
want policy.AdminAction
|
||||
}{
|
||||
{name: "enable", status: string(madmin.AccountEnabled), want: policy.EnableUserAdminAction},
|
||||
{name: "disable", status: string(madmin.AccountDisabled), want: policy.DisableUserAdminAction},
|
||||
{name: "invalid preserves authenticated default", status: "invalid", want: policy.EnableUserAdminAction},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := setUserStatusAdminAction(tt.status); got != tt.want {
|
||||
t.Fatalf("setUserStatusAdminAction(%q) = %q, want %q", tt.status, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetGroupStatusAdminAction(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
status string
|
||||
want policy.AdminAction
|
||||
}{
|
||||
{name: "enable", status: string(madmin.GroupEnabled), want: policy.EnableGroupAdminAction},
|
||||
{name: "disable", status: string(madmin.GroupDisabled), want: policy.DisableGroupAdminAction},
|
||||
{name: "invalid preserves authenticated default", status: "invalid", want: policy.EnableGroupAdminAction},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := setGroupStatusAdminAction(tt.status); got != tt.want {
|
||||
t.Fatalf("setGroupStatusAdminAction(%q) = %q, want %q", tt.status, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// API suite container for IAM
|
||||
type TestSuiteIAM struct {
|
||||
TestSuiteCommon
|
||||
@@ -202,8 +243,11 @@ func TestIAMInternalIDPServerSuite(t *testing.T) {
|
||||
|
||||
suite.SetUpSuite(c)
|
||||
suite.TestUserCreate(c)
|
||||
suite.TestUserStatusActionAuthorization(c)
|
||||
suite.TestGroupStatusActionAuthorization(c)
|
||||
suite.TestUserPolicyEscalationBug(c)
|
||||
suite.TestPolicyCreate(c)
|
||||
suite.TestServiceAccountBareARNPolicyRejected(c)
|
||||
suite.TestCannedPolicies(c)
|
||||
suite.TestGroupAddRemove(c)
|
||||
suite.TestServiceAccountOpsByAdmin(c)
|
||||
@@ -312,6 +356,184 @@ func (s *TestSuiteIAM) TestUserCreate(c *check) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestUserStatusActionAuthorization(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
|
||||
var createdUsers []string
|
||||
var createdPolicies []string
|
||||
defer func() {
|
||||
for _, user := range createdUsers {
|
||||
if err := s.adm.RemoveUser(ctx, user); err != nil {
|
||||
c.Errorf("unable to remove test user %s: %v", user, err)
|
||||
}
|
||||
}
|
||||
for _, policyName := range createdPolicies {
|
||||
if err := s.adm.RemoveCannedPolicy(ctx, policyName); err != nil {
|
||||
c.Errorf("unable to remove test policy %s: %v", policyName, err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
createUser := func() (string, string) {
|
||||
accessKey, secretKey := mustGenerateCredentials(c)
|
||||
if err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled); err != nil {
|
||||
c.Fatalf("unable to create test user: %v", err)
|
||||
}
|
||||
createdUsers = append(createdUsers, accessKey)
|
||||
return accessKey, secretKey
|
||||
}
|
||||
|
||||
createStatusClient := func(action policy.AdminAction) *madmin.AdminClient {
|
||||
accessKey, secretKey := createUser()
|
||||
policyName := getRandomBucketName()
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["%s"]
|
||||
}]
|
||||
}`, action)
|
||||
if err := s.adm.AddCannedPolicy(ctx, policyName, policyBytes); err != nil {
|
||||
c.Fatalf("unable to add status policy: %v", err)
|
||||
}
|
||||
createdPolicies = append(createdPolicies, policyName)
|
||||
if _, err := s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
|
||||
Policies: []string{policyName},
|
||||
User: accessKey,
|
||||
}); err != nil {
|
||||
c.Fatalf("unable to attach status policy: %v", err)
|
||||
}
|
||||
|
||||
client, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Secure: s.secure,
|
||||
})
|
||||
if err != nil {
|
||||
c.Fatalf("unable to create status admin client: %v", err)
|
||||
}
|
||||
client.SetCustomTransport(s.TestSuiteCommon.client.Transport)
|
||||
return client
|
||||
}
|
||||
|
||||
targetAccessKey, _ := createUser()
|
||||
disableClient := createStatusClient(policy.DisableUserAdminAction)
|
||||
if err := disableClient.SetUserStatus(ctx, targetAccessKey, madmin.AccountDisabled); err != nil {
|
||||
c.Fatalf("DisableUser-only client could not disable a user: %v", err)
|
||||
}
|
||||
if err := disableClient.SetUserStatus(ctx, targetAccessKey, madmin.AccountEnabled); err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("DisableUser-only client unexpectedly enabled a user: %v", err)
|
||||
}
|
||||
|
||||
enableClient := createStatusClient(policy.EnableUserAdminAction)
|
||||
if err := enableClient.SetUserStatus(ctx, targetAccessKey, madmin.AccountEnabled); err != nil {
|
||||
c.Fatalf("EnableUser-only client could not enable a user: %v", err)
|
||||
}
|
||||
if err := enableClient.SetUserStatus(ctx, targetAccessKey, madmin.AccountDisabled); err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("EnableUser-only client unexpectedly disabled a user: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestGroupStatusActionAuthorization(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
|
||||
var createdUsers []string
|
||||
var createdPolicies []string
|
||||
group := getRandomBucketName()
|
||||
var groupCreated bool
|
||||
defer func() {
|
||||
if groupCreated {
|
||||
if err := s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{
|
||||
Group: group,
|
||||
Members: createdUsers[:1],
|
||||
IsRemove: true,
|
||||
}); err != nil {
|
||||
c.Errorf("unable to remove group member: %v", err)
|
||||
}
|
||||
if err := s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{Group: group, IsRemove: true}); err != nil {
|
||||
c.Errorf("unable to remove test group: %v", err)
|
||||
}
|
||||
}
|
||||
for _, user := range createdUsers {
|
||||
if err := s.adm.RemoveUser(ctx, user); err != nil {
|
||||
c.Errorf("unable to remove test user %s: %v", user, err)
|
||||
}
|
||||
}
|
||||
for _, policyName := range createdPolicies {
|
||||
if err := s.adm.RemoveCannedPolicy(ctx, policyName); err != nil {
|
||||
c.Errorf("unable to remove test policy %s: %v", policyName, err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
createUser := func() (string, string) {
|
||||
accessKey, secretKey := mustGenerateCredentials(c)
|
||||
if err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled); err != nil {
|
||||
c.Fatalf("unable to create test user: %v", err)
|
||||
}
|
||||
createdUsers = append(createdUsers, accessKey)
|
||||
return accessKey, secretKey
|
||||
}
|
||||
|
||||
targetAccessKey, _ := createUser()
|
||||
if err := s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{
|
||||
Group: group,
|
||||
Members: []string{targetAccessKey},
|
||||
}); err != nil {
|
||||
c.Fatalf("unable to create test group: %v", err)
|
||||
}
|
||||
groupCreated = true
|
||||
|
||||
createStatusClient := func(action policy.AdminAction) *madmin.AdminClient {
|
||||
accessKey, secretKey := createUser()
|
||||
policyName := getRandomBucketName()
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["%s"]
|
||||
}]
|
||||
}`, action)
|
||||
if err := s.adm.AddCannedPolicy(ctx, policyName, policyBytes); err != nil {
|
||||
c.Fatalf("unable to add group status policy: %v", err)
|
||||
}
|
||||
createdPolicies = append(createdPolicies, policyName)
|
||||
if _, err := s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
|
||||
Policies: []string{policyName},
|
||||
User: accessKey,
|
||||
}); err != nil {
|
||||
c.Fatalf("unable to attach group status policy: %v", err)
|
||||
}
|
||||
|
||||
client, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Secure: s.secure,
|
||||
})
|
||||
if err != nil {
|
||||
c.Fatalf("unable to create group status admin client: %v", err)
|
||||
}
|
||||
client.SetCustomTransport(s.TestSuiteCommon.client.Transport)
|
||||
return client
|
||||
}
|
||||
|
||||
disableClient := createStatusClient(policy.DisableGroupAdminAction)
|
||||
if err := disableClient.SetGroupStatus(ctx, group, madmin.GroupDisabled); err != nil {
|
||||
c.Fatalf("DisableGroup-only client could not disable a group: %v", err)
|
||||
}
|
||||
if err := disableClient.SetGroupStatus(ctx, group, madmin.GroupEnabled); err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("DisableGroup-only client unexpectedly enabled a group: %v", err)
|
||||
}
|
||||
|
||||
enableClient := createStatusClient(policy.EnableGroupAdminAction)
|
||||
if err := enableClient.SetGroupStatus(ctx, group, madmin.GroupEnabled); err != nil {
|
||||
c.Fatalf("EnableGroup-only client could not enable a group: %v", err)
|
||||
}
|
||||
if err := enableClient.SetGroupStatus(ctx, group, madmin.GroupDisabled); err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("EnableGroup-only client unexpectedly disabled a group: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestUserPolicyEscalationBug(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
@@ -600,6 +822,20 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) {
|
||||
c.Fatalf("invalid policy creation success")
|
||||
}
|
||||
|
||||
for i, resource := range []string{"arn:aws:s3:::", "*arn:aws:s3:::"} {
|
||||
barePolicyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Deny",
|
||||
"Action": ["s3:GetObject"],
|
||||
"Resource": ["%s"]
|
||||
}]
|
||||
}`, resource)
|
||||
if err = s.adm.AddCannedPolicy(ctx, fmt.Sprintf("%s-bare-%d", policy, i), barePolicyBytes); err == nil {
|
||||
c.Fatalf("bare ARN policy creation succeeded for %q", resource)
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Create a user, associate policy and verify access
|
||||
accessKey, secretKey := mustGenerateCredentials(c)
|
||||
err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
|
||||
@@ -653,6 +889,51 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestServiceAccountBareARNPolicyRejected(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
|
||||
barePolicy := []byte(`{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject"],
|
||||
"NotResource": ["arn:aws:s3:::"]
|
||||
}]
|
||||
}`)
|
||||
if _, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
|
||||
TargetUser: globalActiveCred.AccessKey,
|
||||
Policy: barePolicy,
|
||||
}); err == nil {
|
||||
c.Fatal("service account creation accepted a bare ARN policy")
|
||||
}
|
||||
|
||||
validPolicy := []byte(`{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject"],
|
||||
"Resource": ["arn:aws:s3:::*"]
|
||||
}]
|
||||
}`)
|
||||
credentials, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
|
||||
TargetUser: globalActiveCred.AccessKey,
|
||||
Policy: validPolicy,
|
||||
})
|
||||
if err != nil {
|
||||
c.Fatalf("service account creation rejected an explicit resource: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = s.adm.DeleteServiceAccount(ctx, credentials.AccessKey)
|
||||
}()
|
||||
|
||||
if err = s.adm.UpdateServiceAccount(ctx, credentials.AccessKey, madmin.UpdateServiceAccountReq{
|
||||
NewPolicy: barePolicy,
|
||||
}); err == nil {
|
||||
c.Fatal("service account update accepted a bare ARN policy")
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestCannedPolicies(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -60,8 +60,8 @@ import (
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/secure-io/sio-go"
|
||||
"github.com/zeebo/xxh3"
|
||||
)
|
||||
|
||||
@@ -30,7 +30,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
)
|
||||
|
||||
// getLocalServerProperty - returns madmin.ServerProperties for only the
|
||||
|
||||
+18
-2
@@ -48,7 +48,7 @@ import (
|
||||
levent "github.com/minio/minio/internal/config/lambda/event"
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// APIError structure
|
||||
@@ -1523,10 +1523,14 @@ var errorCodes = errorCodeMap{
|
||||
Description: "Your Host header is malformed.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
// The stored object cannot be served: a server-side data condition, not a
|
||||
// successful partial read. Upstream maps it to http.StatusPartialContent
|
||||
// (since ca6b4773e, 2017), which lets SDKs accept the XML error document
|
||||
// as object content; SILO deliberately diverges and returns 500.
|
||||
ErrObjectTampered: {
|
||||
Code: "XMinioObjectTampered",
|
||||
Description: errObjectTampered.Error(),
|
||||
HTTPStatusCode: http.StatusPartialContent,
|
||||
HTTPStatusCode: http.StatusInternalServerError,
|
||||
},
|
||||
|
||||
ErrSiteReplicationInvalidRequest: {
|
||||
@@ -2169,6 +2173,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
err = unwrapAll(err)
|
||||
|
||||
switch err {
|
||||
case errCompleteMultipartChecksumMismatch, errCompleteMultipartChecksumTypeMismatch:
|
||||
apiErr = ErrBadDigest
|
||||
case errMissingPartChecksum:
|
||||
apiErr = ErrInvalidRequest
|
||||
case errInvalidArgument:
|
||||
apiErr = ErrAdminInvalidArgument
|
||||
case errNoSuchPolicy:
|
||||
@@ -2465,6 +2473,14 @@ func toAPIError(ctx context.Context, err error) APIError {
|
||||
}
|
||||
|
||||
apiErr := errorCodes.ToAPIErr(toAPIErrorCode(ctx, err))
|
||||
switch {
|
||||
case errors.Is(err, errCompleteMultipartChecksumMismatch):
|
||||
apiErr.Description = strings.TrimPrefix(err.Error(), errCompleteMultipartChecksumMismatch.Error()+": ")
|
||||
case errors.Is(err, errCompleteMultipartChecksumTypeMismatch):
|
||||
apiErr.Description = strings.TrimPrefix(err.Error(), errCompleteMultipartChecksumTypeMismatch.Error()+": ")
|
||||
case errors.Is(err, errMissingPartChecksum):
|
||||
apiErr.Description = strings.TrimPrefix(err.Error(), errMissingPartChecksum.Error()+": ")
|
||||
}
|
||||
switch apiErr.Code {
|
||||
case "NotImplemented":
|
||||
apiErr = APIError{
|
||||
|
||||
@@ -39,6 +39,10 @@ var toAPIErrorTests = []struct {
|
||||
{err: ObjectNameInvalid{}, errCode: ErrInvalidObjectName},
|
||||
{err: InvalidUploadID{}, errCode: ErrNoSuchUpload},
|
||||
{err: InvalidPart{}, errCode: ErrInvalidPart},
|
||||
{err: errCompleteMultipartChecksumMismatch, errCode: ErrBadDigest},
|
||||
{err: errCompleteMultipartChecksumTypeMismatch, errCode: ErrBadDigest},
|
||||
{err: errMissingPartChecksum, errCode: ErrInvalidRequest},
|
||||
{err: hash.ChecksumMismatch{}, errCode: ErrContentChecksumMismatch},
|
||||
{err: InsufficientReadQuorum{}, errCode: ErrSlowDownRead},
|
||||
{err: InsufficientWriteQuorum{}, errCode: ErrSlowDownWrite},
|
||||
{err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented},
|
||||
|
||||
+4
-1
@@ -212,7 +212,10 @@ func setObjectHeaders(ctx context.Context, w http.ResponseWriter, objInfo Object
|
||||
}
|
||||
|
||||
if rs == nil && opts.PartNumber > 0 {
|
||||
rs = partNumberToRangeSpec(objInfo, opts.PartNumber)
|
||||
rs, err = partNumberToRangeSpec(objInfo, opts.PartNumber)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// For providing ranged content
|
||||
|
||||
+35
-10
@@ -27,7 +27,6 @@ import (
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/amztime"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
@@ -35,8 +34,8 @@ import (
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
xxml "github.com/minio/xxml"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -380,6 +379,13 @@ type CopyObjectResponse struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
|
||||
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
||||
ETag string // md5sum of the copied object.
|
||||
|
||||
ChecksumCRC32 string `xml:",omitempty"`
|
||||
ChecksumCRC32C string `xml:",omitempty"`
|
||||
ChecksumSHA1 string `xml:",omitempty"`
|
||||
ChecksumSHA256 string `xml:",omitempty"`
|
||||
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||
ChecksumType string `xml:",omitempty"`
|
||||
}
|
||||
|
||||
// CopyObjectPartResponse container returns ETag and LastModified of the successfully copied object
|
||||
@@ -387,6 +393,12 @@ type CopyObjectPartResponse struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"`
|
||||
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
||||
ETag string // md5sum of the copied object part.
|
||||
|
||||
ChecksumCRC32 string `xml:",omitempty"`
|
||||
ChecksumCRC32C string `xml:",omitempty"`
|
||||
ChecksumSHA1 string `xml:",omitempty"`
|
||||
ChecksumSHA256 string `xml:",omitempty"`
|
||||
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||
}
|
||||
|
||||
// Initiator inherit from Owner struct, fields are same
|
||||
@@ -416,6 +428,7 @@ type CompleteMultipartUploadResponse struct {
|
||||
Key string
|
||||
ETag string
|
||||
|
||||
ChecksumType string `xml:"ChecksumType,omitempty"`
|
||||
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
|
||||
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
|
||||
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
|
||||
@@ -763,19 +776,30 @@ func generateListObjectsV2Response(ctx context.Context, bucket, prefix, token, n
|
||||
|
||||
type metaCheckFn = func(name string, action policy.Action) (s3Err APIErrorCode)
|
||||
|
||||
// generates CopyObjectResponse from etag and lastModified time.
|
||||
func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
|
||||
// generates CopyObjectResponse from the committed object information.
|
||||
func generateCopyObjectResponse(oi ObjectInfo, cs map[string]string) CopyObjectResponse {
|
||||
return CopyObjectResponse{
|
||||
ETag: "\"" + etag + "\"",
|
||||
LastModified: amztime.ISO8601Format(lastModified.UTC()),
|
||||
ETag: "\"" + oi.ETag + "\"",
|
||||
LastModified: amztime.ISO8601Format(oi.ModTime.UTC()),
|
||||
ChecksumCRC32: cs[hash.ChecksumCRC32.String()],
|
||||
ChecksumCRC32C: cs[hash.ChecksumCRC32C.String()],
|
||||
ChecksumSHA1: cs[hash.ChecksumSHA1.String()],
|
||||
ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
|
||||
ChecksumCRC64NVME: cs[hash.ChecksumCRC64NVME.String()],
|
||||
ChecksumType: cs[xhttp.AmzChecksumType],
|
||||
}
|
||||
}
|
||||
|
||||
// generates CopyObjectPartResponse from etag and lastModified time.
|
||||
func generateCopyObjectPartResponse(etag string, lastModified time.Time) CopyObjectPartResponse {
|
||||
// generates CopyObjectPartResponse from the uploaded part information.
|
||||
func generateCopyObjectPartResponse(partInfo PartInfo) CopyObjectPartResponse {
|
||||
return CopyObjectPartResponse{
|
||||
ETag: "\"" + etag + "\"",
|
||||
LastModified: amztime.ISO8601Format(lastModified.UTC()),
|
||||
ETag: "\"" + partInfo.ETag + "\"",
|
||||
LastModified: amztime.ISO8601Format(partInfo.LastModified.UTC()),
|
||||
ChecksumCRC32: partInfo.ChecksumCRC32,
|
||||
ChecksumCRC32C: partInfo.ChecksumCRC32C,
|
||||
ChecksumSHA1: partInfo.ChecksumSHA1,
|
||||
ChecksumSHA256: partInfo.ChecksumSHA256,
|
||||
ChecksumCRC64NVME: partInfo.ChecksumCRC64NVME,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -797,6 +821,7 @@ func generateCompleteMultipartUploadResponse(bucket, key, location string, oi Ob
|
||||
Key: key,
|
||||
// AWS S3 quotes the ETag in XML, make sure we are compatible here.
|
||||
ETag: "\"" + oi.ETag + "\"",
|
||||
ChecksumType: cs[xhttp.AmzChecksumType],
|
||||
ChecksumSHA1: cs[hash.ChecksumSHA1.String()],
|
||||
ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
|
||||
ChecksumCRC32: cs[hash.ChecksumCRC32.String()],
|
||||
|
||||
+127
-10
@@ -18,16 +18,23 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
consoleapi "github.com/minio/console/api"
|
||||
bktcors "github.com/minio/minio/internal/bucket/cors"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/wildcard"
|
||||
"github.com/pgsty/silo-pkg/v3/wildcard"
|
||||
"github.com/rs/cors"
|
||||
)
|
||||
|
||||
type bucketCorsAppliedKey struct{}
|
||||
|
||||
func newHTTPServerFn() *xhttp.Server {
|
||||
globalObjLayerMutex.RLock()
|
||||
defer globalObjLayerMutex.RUnlock()
|
||||
@@ -111,11 +118,6 @@ var rejectedBucketAPIs = []rejectedAPI{
|
||||
methods: []string{http.MethodGet, http.MethodPut, http.MethodDelete},
|
||||
queries: []string{"inventory", ""},
|
||||
},
|
||||
{
|
||||
api: "cors",
|
||||
methods: []string{http.MethodPut, http.MethodDelete},
|
||||
queries: []string{"cors", ""},
|
||||
},
|
||||
{
|
||||
api: "metrics",
|
||||
methods: []string{http.MethodGet, http.MethodPut, http.MethodDelete},
|
||||
@@ -459,15 +461,15 @@ func registerAPIRouter(router *mux.Router) {
|
||||
router.Methods(http.MethodPut).
|
||||
HandlerFunc(s3APIMiddleware(api.PutBucketACLHandler)).
|
||||
Queries("acl", "")
|
||||
// GetBucketCors - this is a dummy call.
|
||||
// GetBucketCors
|
||||
router.Methods(http.MethodGet).
|
||||
HandlerFunc(s3APIMiddleware(api.GetBucketCorsHandler)).
|
||||
Queries("cors", "")
|
||||
// PutBucketCors - this is a dummy call.
|
||||
// PutBucketCors
|
||||
router.Methods(http.MethodPut).
|
||||
HandlerFunc(s3APIMiddleware(api.PutBucketCorsHandler)).
|
||||
Queries("cors", "")
|
||||
// DeleteBucketCors - this is a dummy call.
|
||||
// DeleteBucketCors
|
||||
router.Methods(http.MethodDelete).
|
||||
HandlerFunc(s3APIMiddleware(api.DeleteBucketCorsHandler)).
|
||||
Queries("cors", "")
|
||||
@@ -648,6 +650,94 @@ func registerAPIRouter(router *mux.Router) {
|
||||
apiRouter.MethodNotAllowedHandler = collectAPIStats("methodnotallowed", httpTraceAll(methodNotAllowedHandler("S3")))
|
||||
}
|
||||
|
||||
// applyBucketCors applies a bucket's CORS configuration to the request.
|
||||
// For an OPTIONS preflight it writes the full CORS response and returns true
|
||||
// (request is complete). For an actual request it adds the applicable
|
||||
// Access-Control-* response headers and returns false so the request
|
||||
// continues down the handler chain. If no rule matches a preflight it writes
|
||||
// 403 and returns true. A matched actual request is marked in its context so
|
||||
// inner legacy middleware does not rewrite an explicitly allowed null origin.
|
||||
func applyBucketCors(w http.ResponseWriter, r *http.Request, cfg *bktcors.Config) (handled bool) {
|
||||
origin := r.Header.Get("Origin")
|
||||
if origin == "" {
|
||||
return false // not a CORS request
|
||||
}
|
||||
h := w.Header()
|
||||
h.Add("Vary", "Origin")
|
||||
|
||||
isPreflight := r.Method == http.MethodOptions &&
|
||||
r.Header.Get("Access-Control-Request-Method") != ""
|
||||
|
||||
if isPreflight {
|
||||
method := r.Header.Get("Access-Control-Request-Method")
|
||||
reqHeaders := splitAndTrim(r.Header.Get("Access-Control-Request-Headers"))
|
||||
// A preflight response depends on all three request headers that
|
||||
// determine the outcome, including when the request is rejected.
|
||||
h.Add("Vary", "Access-Control-Request-Method")
|
||||
h.Add("Vary", "Access-Control-Request-Headers")
|
||||
rule, allowedOrigin, allowedHeaders, maxAgeSeconds, ok := cfg.MatchPreflight(origin, method, reqHeaders)
|
||||
if !ok {
|
||||
writeResponse(w, http.StatusForbidden, nil, mimeNone)
|
||||
return true
|
||||
}
|
||||
setBucketCorsOriginHeaders(h, allowedOrigin, origin)
|
||||
h.Set("Access-Control-Allow-Methods", strings.Join(rule.AllowedMethods, ", "))
|
||||
if len(allowedHeaders) > 0 {
|
||||
h.Set("Access-Control-Allow-Headers", strings.Join(allowedHeaders, ", "))
|
||||
}
|
||||
if len(rule.ExposeHeaders) > 0 {
|
||||
h.Set("Access-Control-Expose-Headers", strings.Join(rule.ExposeHeaders, ", "))
|
||||
}
|
||||
if maxAgeSeconds != nil {
|
||||
h.Set("Access-Control-Max-Age", strconv.Itoa(*maxAgeSeconds))
|
||||
}
|
||||
writeResponse(w, http.StatusOK, nil, mimeNone)
|
||||
return true
|
||||
}
|
||||
|
||||
// Actual request: attach headers if the origin+method match.
|
||||
rule, allowedOrigin, ok := cfg.MatchRule(origin, r.Method)
|
||||
if !ok {
|
||||
return false // no matching rule → no CORS headers, continue normally
|
||||
}
|
||||
*r = *r.WithContext(context.WithValue(r.Context(), bucketCorsAppliedKey{}, struct{}{}))
|
||||
setBucketCorsOriginHeaders(h, allowedOrigin, origin)
|
||||
if len(rule.ExposeHeaders) > 0 {
|
||||
h.Set("Access-Control-Expose-Headers", strings.Join(rule.ExposeHeaders, ", "))
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func bucketCorsWasApplied(r *http.Request) bool {
|
||||
_, ok := r.Context().Value(bucketCorsAppliedKey{}).(struct{})
|
||||
return ok
|
||||
}
|
||||
|
||||
func setBucketCorsOriginHeaders(h http.Header, allowedOrigin, requestOrigin string) {
|
||||
if allowedOrigin == "*" {
|
||||
h.Set("Access-Control-Allow-Origin", "*")
|
||||
h.Del("Access-Control-Allow-Credentials")
|
||||
return
|
||||
}
|
||||
h.Set("Access-Control-Allow-Origin", requestOrigin)
|
||||
h.Set("Access-Control-Allow-Credentials", "true")
|
||||
}
|
||||
|
||||
// splitAndTrim splits a comma-separated header list into trimmed, non-empty values.
|
||||
func splitAndTrim(s string) []string {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
parts := strings.Split(s, ",")
|
||||
out := parts[:0]
|
||||
for _, p := range parts {
|
||||
if p = strings.TrimSpace(p); p != "" {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// corsHandler handler for CORS (Cross Origin Resource Sharing)
|
||||
func corsHandler(handler http.Handler) http.Handler {
|
||||
commonS3Headers := []string{
|
||||
@@ -693,5 +783,32 @@ func corsHandler(handler http.Handler) http.Handler {
|
||||
ExposedHeaders: commonS3Headers,
|
||||
AllowCredentials: true,
|
||||
}
|
||||
return cors.New(opts).Handler(handler)
|
||||
globalCors := cors.New(opts).Handler(handler)
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Origin") != "" {
|
||||
if bucket, _ := request2BucketObjectName(r); bucket != "" && globalBucketMetadataSys != nil {
|
||||
// Resident-only lookup: this runs before authentication with a
|
||||
// client-supplied path segment as the bucket name, so it must
|
||||
// never load or cache metadata. While startup loading is still
|
||||
// running, for a real bucket whose metadata failed to load, and
|
||||
// for a bucket whose stored CORS document failed to parse, the
|
||||
// request gets no CORS headers; any other non-resident name falls
|
||||
// back to the global policy below.
|
||||
cfg, _, err := globalBucketMetadataSys.GetResidentCorsConfig(bucket)
|
||||
if err == nil && cfg != nil {
|
||||
if applyBucketCors(w, r, cfg) {
|
||||
return
|
||||
}
|
||||
handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
if err != nil && !errors.Is(err, errConfigNotFound) {
|
||||
internalLogOnceIf(r.Context(), err, "bucket-cors-metadata")
|
||||
handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
globalCors.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
+66
-31
@@ -41,7 +41,7 @@ import (
|
||||
xjwt "github.com/minio/minio/internal/jwt"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/mcontext"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// Verify if request has JWT.
|
||||
@@ -363,17 +363,6 @@ func checkRequestAuthTypeWithRequestTags(ctx context.Context, r *http.Request, a
|
||||
return authorizeRequestWithTags(ctx, r, action, "", requestTags)
|
||||
}
|
||||
|
||||
// checkRequestAuthTypeWithVID is similar to checkRequestAuthType
|
||||
// passes versionID additionally.
|
||||
func checkRequestAuthTypeWithVID(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName, versionID string) (s3Err APIErrorCode) {
|
||||
logger.GetReqInfo(ctx).BucketName = bucketName
|
||||
logger.GetReqInfo(ctx).ObjectName = objectName
|
||||
logger.GetReqInfo(ctx).VersionID = versionID
|
||||
|
||||
_, _, s3Err = checkRequestAuthTypeCredential(ctx, r, action)
|
||||
return s3Err
|
||||
}
|
||||
|
||||
func authenticateRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
|
||||
if logger.GetReqInfo(ctx) == nil {
|
||||
bugLogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.ErrorKind)
|
||||
@@ -439,6 +428,23 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
||||
return authorizeRequestWithExistingTags(ctx, r, action, "")
|
||||
}
|
||||
|
||||
func deleteObjectAction(versionID string) policy.Action {
|
||||
if versionID != "" {
|
||||
return policy.DeleteObjectVersionAction
|
||||
}
|
||||
return policy.DeleteObjectAction
|
||||
}
|
||||
|
||||
func actionUsesObjectVersion(action policy.Action) bool {
|
||||
switch action {
|
||||
case policy.DeleteObjectAction, policy.DeleteObjectVersionAction,
|
||||
policy.ReplicateDeleteAction, policy.BypassGovernanceRetentionAction:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func authorizeRequestWithExistingTags(ctx context.Context, r *http.Request, action policy.Action, existingTags string) (s3Err APIErrorCode) {
|
||||
return authorizeRequestWithTags(ctx, r, action, existingTags, nil)
|
||||
}
|
||||
@@ -457,7 +463,7 @@ func authorizeRequestWithTags(ctx context.Context, r *http.Request, action polic
|
||||
versionID := reqInfo.VersionID
|
||||
conditionValuesForAuth := func(locationConstraint string, credentials auth.Credentials) map[string][]string {
|
||||
values := getConditionValuesWithTags(r, locationConstraint, credentials, existingTags, requestTags)
|
||||
if action == policy.DeleteObjectAction {
|
||||
if actionUsesObjectVersion(action) {
|
||||
// DeleteObjects carries the effective version ID in each XML object,
|
||||
// not in the request query. Keep authorization scoped to that entry.
|
||||
if versionID == "" {
|
||||
@@ -503,21 +509,6 @@ func authorizeRequestWithTags(ctx context.Context, r *http.Request, action polic
|
||||
|
||||
return ErrAccessDenied
|
||||
}
|
||||
if action == policy.DeleteObjectAction && versionID != "" {
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.Action(policy.DeleteObjectVersionAction),
|
||||
BucketName: bucket,
|
||||
ConditionValues: conditionValuesForAuth("", cred),
|
||||
ObjectName: object,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: true,
|
||||
}) { // Request is not allowed if Deny action on DeleteObjectVersionAction
|
||||
return ErrAccessDenied
|
||||
}
|
||||
}
|
||||
if globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
@@ -553,6 +544,40 @@ func authorizeRequestWithTags(ctx context.Context, r *http.Request, action polic
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
// authorizeReplicationDelete preserves the established target-credential
|
||||
// contract for trusted replication: DeleteObject and ReplicateDelete must be
|
||||
// allowed, while an explicit DeleteObjectVersion deny still blocks a named
|
||||
// version. Ordinary S3 requests never use this compatibility path.
|
||||
func authorizeReplicationDelete(ctx context.Context, r *http.Request) APIErrorCode {
|
||||
if s3Err := authorizeRequest(ctx, r, policy.DeleteObjectAction); s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
if reqInfo == nil {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
if reqInfo.VersionID == "" {
|
||||
return ErrNone
|
||||
}
|
||||
cred := reqInfo.Cred
|
||||
values := getConditionValuesWithTags(r, "", cred, "", nil)
|
||||
values["versionid"] = []string{reqInfo.VersionID}
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.DeleteObjectVersionAction,
|
||||
BucketName: reqInfo.BucketName,
|
||||
ConditionValues: values,
|
||||
ObjectName: reqInfo.ObjectName,
|
||||
IsOwner: reqInfo.Owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: true,
|
||||
}) {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
// Check request auth type verifies the incoming http request
|
||||
// - validates the request signature
|
||||
// - validates the policy action if anonymous tests bucket policies if any,
|
||||
@@ -786,10 +811,20 @@ func isPutActionAllowedWithRequestTags(ctx context.Context, atype authType, buck
|
||||
return s3Err
|
||||
}
|
||||
|
||||
logger.GetReqInfo(ctx).Cred = cred
|
||||
logger.GetReqInfo(ctx).Owner = owner
|
||||
logger.GetReqInfo(ctx).Region = region
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
if reqInfo == nil {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
reqInfo.Lock()
|
||||
reqInfo.Cred = cred
|
||||
reqInfo.Owner = owner
|
||||
reqInfo.Region = region
|
||||
reqInfo.Unlock()
|
||||
|
||||
return isPutActionAllowedWithCred(bucketName, objectName, r, action, requestTags, cred, owner)
|
||||
}
|
||||
|
||||
func isPutActionAllowedWithCred(bucketName, objectName string, r *http.Request, action policy.Action, requestTags *string, cred auth.Credentials, owner bool) APIErrorCode {
|
||||
// Do not check for PutObjectRetentionAction permission,
|
||||
// if mode and retain until date are not set.
|
||||
// Can happen when bucket has default lock config set
|
||||
|
||||
@@ -28,7 +28,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
type nullReader struct{}
|
||||
|
||||
@@ -25,7 +25,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
)
|
||||
|
||||
// healTask represents what to heal along with options
|
||||
|
||||
@@ -34,7 +34,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
+4
-4
@@ -33,10 +33,10 @@ import (
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/minio/pkg/v3/wildcard"
|
||||
"github.com/minio/pkg/v3/workers"
|
||||
"github.com/minio/pkg/v3/xtime"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/wildcard"
|
||||
"github.com/pgsty/silo-pkg/v3/workers"
|
||||
"github.com/pgsty/silo-pkg/v3/xtime"
|
||||
"go.yaml.in/yaml/v3"
|
||||
)
|
||||
|
||||
|
||||
@@ -48,10 +48,10 @@ import (
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/pkg/v3/console"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/workers"
|
||||
"github.com/pgsty/silo-pkg/v3/console"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/workers"
|
||||
"go.yaml.in/yaml/v3"
|
||||
)
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/pkg/v3/wildcard"
|
||||
"github.com/pgsty/silo-pkg/v3/wildcard"
|
||||
"go.yaml.in/yaml/v3"
|
||||
)
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
|
||||
miniogo "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/pkg/v3/xtime"
|
||||
"github.com/pgsty/silo-pkg/v3/xtime"
|
||||
)
|
||||
|
||||
//go:generate msgp -file $GOFILE
|
||||
|
||||
+2
-2
@@ -34,8 +34,8 @@ import (
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/minio/pkg/v3/workers"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/workers"
|
||||
)
|
||||
|
||||
// keyrotate:
|
||||
|
||||
@@ -34,7 +34,7 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio/internal/grid"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
)
|
||||
|
||||
// To abstract a node over network.
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"hash/crc32"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
)
|
||||
|
||||
func TestPutBucketCorsWireValidation(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testPutBucketCorsWireValidation,
|
||||
endpoints: []string{"PutBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testPutBucketCorsWireValidation(_ ObjectLayer, _ string, bucketName string, apiRouter http.Handler,
|
||||
creds auth.Credentials, t *testing.T,
|
||||
) {
|
||||
valid := `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`
|
||||
rule := `<CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule>`
|
||||
tests := []struct {
|
||||
name string
|
||||
body string
|
||||
want int
|
||||
wantCode string
|
||||
}{
|
||||
{
|
||||
name: "second XML root",
|
||||
body: valid + `<Extra/>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "255 Unicode character ID",
|
||||
body: `<CORSConfiguration><CORSRule><ID>` + strings.Repeat("界", 255) + `</ID><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "256 Unicode character ID",
|
||||
body: `<CORSConfiguration><CORSRule><ID>` + strings.Repeat("界", 256) + `</ID><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "lowercase method",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>get</AllowedMethod></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "empty origin",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin/><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "question mark origin wildcard",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>https://?.example.com</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "question mark header wildcard",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><AllowedHeader>x-amz-?</AllowedHeader></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "unknown element",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><Unknown/></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "empty max age",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><MaxAgeSeconds/></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "zero max age",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><MaxAgeSeconds>0</MaxAgeSeconds></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "max age int32 overflow",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><MaxAgeSeconds>2147483648</MaxAgeSeconds></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "100 rules",
|
||||
body: `<CORSConfiguration>` + strings.Repeat(rule, 100) + `</CORSConfiguration>`,
|
||||
want: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "101 rules",
|
||||
body: `<CORSConfiguration>` + strings.Repeat(rule, 101) + `</CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "exactly 64 KiB",
|
||||
body: sizedCORSConfig(maxBucketCorsSize),
|
||||
want: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "over 64 KiB",
|
||||
body: sizedCORSConfig(maxBucketCorsSize + 1),
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "EntityTooLarge",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, getBucketCorsURL("", bucketName),
|
||||
int64(len(tt.body)), bytes.NewReader([]byte(tt.body)), creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != tt.want {
|
||||
t.Fatalf("expected status %d, got %d: %s", tt.want, rec.Code, rec.Body.String())
|
||||
}
|
||||
if tt.wantCode != "" && !bytes.Contains(rec.Body.Bytes(), []byte("<Code>"+tt.wantCode+"</Code>")) {
|
||||
t.Fatalf("expected error code %s, got: %s", tt.wantCode, rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func sizedCORSConfig(size int) string {
|
||||
prefix := `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod>`
|
||||
suffix := `</CORSRule></CORSConfiguration>`
|
||||
return prefix + strings.Repeat(" ", size-len(prefix)-len(suffix)) + suffix
|
||||
}
|
||||
|
||||
func TestPutBucketCorsChecksumValidation(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testPutBucketCorsChecksumValidation,
|
||||
endpoints: []string{"PutBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testPutBucketCorsChecksumValidation(_ ObjectLayer, _ string, bucketName string, apiRouter http.Handler,
|
||||
creds auth.Credentials, t *testing.T,
|
||||
) {
|
||||
body := []byte(`<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`)
|
||||
tests := []struct {
|
||||
name string
|
||||
configure func(*http.Request)
|
||||
want int
|
||||
wantCode string
|
||||
}{
|
||||
{
|
||||
name: "missing checksum",
|
||||
configure: func(req *http.Request) {
|
||||
req.Header.Del("Content-Md5")
|
||||
},
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MissingContentMD5",
|
||||
},
|
||||
{
|
||||
name: "bad content md5",
|
||||
configure: func(req *http.Request) {
|
||||
req.Header.Set("Content-Md5", getMD5HashBase64([]byte("different body")))
|
||||
},
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "BadDigest",
|
||||
},
|
||||
{
|
||||
name: "valid sdk crc32",
|
||||
configure: func(req *http.Request) {
|
||||
req.Header.Del("Content-Md5")
|
||||
req.Header.Set("X-Amz-Sdk-Checksum-Algorithm", "CRC32")
|
||||
req.Header.Set("X-Amz-Checksum-Crc32", corsCRC32Base64(body))
|
||||
},
|
||||
want: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "bad sdk crc32",
|
||||
configure: func(req *http.Request) {
|
||||
req.Header.Del("Content-Md5")
|
||||
req.Header.Set("X-Amz-Sdk-Checksum-Algorithm", "CRC32")
|
||||
req.Header.Set("X-Amz-Checksum-Crc32", corsCRC32Base64([]byte("different body")))
|
||||
},
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "BadDigest",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req, err := newTestRequest(http.MethodPut, getBucketCorsURL("", bucketName), int64(len(body)), bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tt.configure(req)
|
||||
if err = signRequestV4(req, creds.AccessKey, creds.SecretKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != tt.want || (tt.wantCode != "" && !bytes.Contains(rec.Body.Bytes(), []byte("<Code>"+tt.wantCode+"</Code>"))) {
|
||||
t.Fatalf("expected status %d and code %s, got %d: %s", tt.want, tt.wantCode, rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func corsCRC32Base64(data []byte) string {
|
||||
var checksum [4]byte
|
||||
binary.BigEndian.PutUint32(checksum[:], crc32.ChecksumIEEE(data))
|
||||
return base64.StdEncoding.EncodeToString(checksum[:])
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/bucket/cors"
|
||||
hashpkg "github.com/minio/minio/internal/hash"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// maxBucketCorsSize is the maximum allowed size of a CORS configuration document.
|
||||
const maxBucketCorsSize = 64 * humanize.KiByte
|
||||
|
||||
// PutBucketCorsHandler - PUT bucket cors.
|
||||
func (api objectAPIHandlers) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PutBucketCors")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketCorsAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if r.ContentLength <= 0 {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
|
||||
return
|
||||
}
|
||||
if r.ContentLength > maxBucketCorsSize {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// PutBucketCors requires a Content-Md5 or a supported full-header
|
||||
// checksum. validateLengthAndChecksum wraps r.Body so the supplied digest
|
||||
// is verified as the body is read below.
|
||||
if !validateLengthAndChecksum(r) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
corsBytes, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
if errors.Is(err, hashpkg.ErrInvalidChecksum) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadDigest), r.URL)
|
||||
return
|
||||
}
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
corsCfg, err := cors.ParseBucketCorsConfig(bytes.NewReader(corsBytes))
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL)
|
||||
return
|
||||
}
|
||||
if err := corsCfg.Validate(); err != nil {
|
||||
writeErrorResponse(ctx, w, APIError{
|
||||
Code: "MalformedXML",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
Description: err.Error(),
|
||||
}, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
updatedAt, err := updateLocalBucketCORSMetadata(ctx, objAPI, bucket, corsBytes)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Call site replication hook.
|
||||
//
|
||||
// We encode the xml bytes as base64 to ensure there are no encoding
|
||||
// errors.
|
||||
cfgStr := base64.StdEncoding.EncodeToString(corsBytes)
|
||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypeCorsConfig,
|
||||
Bucket: bucket,
|
||||
Cors: &cfgStr,
|
||||
UpdatedAt: updatedAt,
|
||||
}))
|
||||
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
// GetBucketCorsHandler - GET bucket cors.
|
||||
func (api objectAPIHandlers) GetBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetBucketCors")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketCorsAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
configData, _, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, errConfigNotFound) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNoSuchCORSConfiguration), r.URL)
|
||||
return
|
||||
}
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseXML(w, configData)
|
||||
}
|
||||
|
||||
// DeleteBucketCorsHandler - DELETE bucket cors.
|
||||
func (api objectAPIHandlers) DeleteBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "DeleteBucketCors")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketCorsAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
updatedAt, err := updateLocalBucketCORSMetadata(ctx, objAPI, bucket, nil)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypeCorsConfig,
|
||||
Bucket: bucket,
|
||||
Cors: nil,
|
||||
UpdatedAt: updatedAt,
|
||||
}))
|
||||
|
||||
writeSuccessNoContent(w)
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
)
|
||||
|
||||
const testCORSDoc = `<CORSConfiguration><CORSRule><AllowedOrigin>http://example.com</AllowedOrigin><AllowedMethod>GET</AllowedMethod><AllowedMethod>PUT</AllowedMethod><ExposeHeader>ETag</ExposeHeader><MaxAgeSeconds>3000</MaxAgeSeconds></CORSRule></CORSConfiguration>`
|
||||
|
||||
func TestBucketCorsHandlers(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testBucketCorsHandlers, endpoints: []string{"PutBucketCors", "GetBucketCors", "DeleteBucketCors"}})
|
||||
}
|
||||
|
||||
func testBucketCorsHandlers(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
creds auth.Credentials, t *testing.T,
|
||||
) {
|
||||
// PUT
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, getBucketCorsURL("", bucketName),
|
||||
int64(len(testCORSDoc)), bytes.NewReader([]byte(testCORSDoc)), creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("PUT cors: expected 200, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// GET returns what we stored
|
||||
req, err = newTestSignedRequestV4(http.MethodGet, getBucketCorsURL("", bucketName),
|
||||
0, nil, creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("GET cors: expected 200, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !bytes.Contains(rec.Body.Bytes(), []byte("http://example.com")) {
|
||||
t.Fatalf("GET cors: body missing origin: %s", rec.Body.String())
|
||||
}
|
||||
|
||||
// DELETE
|
||||
req, err = newTestSignedRequestV4(http.MethodDelete, getBucketCorsURL("", bucketName),
|
||||
0, nil, creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("DELETE cors: expected 204, got %d", rec.Code)
|
||||
}
|
||||
|
||||
// GET after delete → 404 NoSuchCORSConfiguration
|
||||
req, err = newTestSignedRequestV4(http.MethodGet, getBucketCorsURL("", bucketName),
|
||||
0, nil, creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("GET cors after delete: expected 404, got %d", rec.Code)
|
||||
}
|
||||
|
||||
// Malformed XML → 400
|
||||
req, err = newTestSignedRequestV4(http.MethodPut, getBucketCorsURL("", bucketName),
|
||||
int64(len("<bad>")), bytes.NewReader([]byte("<bad>")), creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("PUT malformed cors: expected 400, got %d", rec.Code)
|
||||
}
|
||||
|
||||
// Missing Content-MD5 is rejected before the body is parsed.
|
||||
req, err = newTestRequest(http.MethodPut, getBucketCorsURL("", bucketName),
|
||||
int64(len(testCORSDoc)), bytes.NewReader([]byte(testCORSDoc)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Header.Del("Content-Md5")
|
||||
if err = signRequestV4(req, creds.AccessKey, creds.SecretKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusBadRequest || !bytes.Contains(rec.Body.Bytes(), []byte("<Code>MissingContentMD5</Code>")) {
|
||||
t.Fatalf("PUT cors without Content-MD5: expected MissingContentMD5, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// A signed but incorrect Content-MD5 is rejected while reading the body.
|
||||
req, err = newTestRequest(http.MethodPut, getBucketCorsURL("", bucketName),
|
||||
int64(len(testCORSDoc)), bytes.NewReader([]byte(testCORSDoc)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Header.Set("Content-Md5", getMD5HashBase64([]byte("different body")))
|
||||
if err = signRequestV4(req, creds.AccessKey, creds.SecretKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusBadRequest || !bytes.Contains(rec.Body.Bytes(), []byte("<Code>BadDigest</Code>")) {
|
||||
t.Fatalf("PUT cors with bad Content-MD5: expected BadDigest, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// Re-PUT the config so the store→GetCorsConfig→enforce seam below has
|
||||
// something to enforce (the earlier DELETE removed it).
|
||||
req, err = newTestSignedRequestV4(http.MethodPut, getBucketCorsURL("", bucketName),
|
||||
int64(len(testCORSDoc)), bytes.NewReader([]byte(testCORSDoc)), creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("PUT cors (re-put): expected 200, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// End-to-end enforcement: drive an OPTIONS preflight through the real
|
||||
// corsHandler wrapper (not applyBucketCors in isolation), exercising the
|
||||
// full store -> globalBucketMetadataSys.GetCorsConfig -> enforce seam.
|
||||
wrapped := corsHandler(apiRouter)
|
||||
|
||||
preflightURL := getBucketCorsURL("", bucketName)
|
||||
preflightReq := httptest.NewRequest(http.MethodOptions, preflightURL, nil)
|
||||
preflightReq.Header.Set("Origin", "http://example.com")
|
||||
preflightReq.Header.Set("Access-Control-Request-Method", http.MethodGet)
|
||||
|
||||
rec = httptest.NewRecorder()
|
||||
wrapped.ServeHTTP(rec, preflightReq)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("OPTIONS preflight via corsHandler: expected 200, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "http://example.com" {
|
||||
t.Fatalf("OPTIONS preflight via corsHandler: expected Access-Control-Allow-Origin echoed, got %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,743 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/bucket/cors"
|
||||
)
|
||||
|
||||
type corsLookupCountingObjectLayer struct {
|
||||
ObjectLayer
|
||||
getObjectNInfoCalls atomic.Int64
|
||||
}
|
||||
|
||||
func (o *corsLookupCountingObjectLayer) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
|
||||
o.getObjectNInfoCalls.Add(1)
|
||||
return o.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
|
||||
}
|
||||
|
||||
func TestPerBucketCorsPreflight(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"http://example.com"},
|
||||
AllowedMethods: []string{"GET", "PUT"},
|
||||
AllowedHeaders: []string{"*"},
|
||||
ExposeHeaders: []string{"ETag"},
|
||||
MaxAgeSeconds: 3000,
|
||||
}}}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodOptions, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "http://example.com")
|
||||
req.Header.Set("Access-Control-Request-Method", "GET")
|
||||
req.Header.Set("Access-Control-Request-Headers", "X-Amz-Date")
|
||||
|
||||
handled := applyBucketCors(rec, req, cfg)
|
||||
if !handled {
|
||||
t.Fatal("expected preflight to be handled")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "http://example.com" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "true" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Methods"); got != "GET, PUT" {
|
||||
t.Fatalf("allow-methods = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Headers"); got != "X-Amz-Date" {
|
||||
t.Fatalf("allow-headers = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Expose-Headers"); got != "ETag" {
|
||||
t.Fatalf("expose-headers = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Max-Age"); got != "3000" {
|
||||
t.Fatalf("max-age = %q", got)
|
||||
}
|
||||
requireCorsVary(t, rec.Header())
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("preflight status = %d", rec.Code)
|
||||
}
|
||||
requireCorsOriginVary(t, rec.Header())
|
||||
}
|
||||
|
||||
func TestPerBucketCorsActualRequestNoMatchVariesByOrigin(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"https://allowed.example.com"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
}}}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "https://denied.example.com")
|
||||
|
||||
if handled := applyBucketCors(rec, req, cfg); handled {
|
||||
t.Fatal("actual request must continue when CORS does not match")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
requireCorsOriginVary(t, rec.Header())
|
||||
}
|
||||
|
||||
func TestPerBucketCorsPreflightNoMatch(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"http://example.com"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
}}}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodOptions, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "http://evil.com")
|
||||
req.Header.Set("Access-Control-Request-Method", "GET")
|
||||
|
||||
handled := applyBucketCors(rec, req, cfg)
|
||||
if !handled {
|
||||
t.Fatal("expected preflight to be handled (rejected)")
|
||||
}
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("expected 403 for disallowed origin, got %d", rec.Code)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
|
||||
t.Fatalf("rejected preflight returned allow-origin %q", got)
|
||||
}
|
||||
requireCorsVary(t, rec.Header())
|
||||
}
|
||||
|
||||
func TestPerBucketCorsPreflightWildcardOriginAndZeroMaxAge(t *testing.T) {
|
||||
doc := `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><AllowedMethod>HEAD</AllowedMethod><AllowedHeader>*</AllowedHeader><ExposeHeader>ETag</ExposeHeader><MaxAgeSeconds>0</MaxAgeSeconds></CORSRule></CORSConfiguration>`
|
||||
cfg, err := cors.ParseBucketCorsConfig(strings.NewReader(doc))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodOptions, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
req.Header.Set("Access-Control-Request-Method", "GET")
|
||||
req.Header.Set("Access-Control-Request-Headers", "RANGE")
|
||||
|
||||
if handled := applyBucketCors(rec, req, cfg); !handled {
|
||||
t.Fatal("expected preflight to be handled")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Methods"); got != "GET, HEAD" {
|
||||
t.Fatalf("allow-methods = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Headers"); got != "RANGE" {
|
||||
t.Fatalf("allow-headers = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Expose-Headers"); got != "ETag" {
|
||||
t.Fatalf("expose-headers = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Max-Age"); got != "0" {
|
||||
t.Fatalf("max-age = %q", got)
|
||||
}
|
||||
requireCorsVary(t, rec.Header())
|
||||
}
|
||||
|
||||
func TestPerBucketCorsPreflightUsesFirstFullyMatchingRule(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{
|
||||
{
|
||||
AllowedOrigins: []string{"https://app.example.com"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
AllowedHeaders: []string{"x-a"},
|
||||
ExposeHeaders: []string{"x-rule-a"},
|
||||
MaxAgeSeconds: 1,
|
||||
},
|
||||
{
|
||||
AllowedOrigins: []string{"https://app.example.com"},
|
||||
AllowedMethods: []string{"GET", "HEAD"},
|
||||
AllowedHeaders: []string{"*"},
|
||||
ExposeHeaders: []string{"x-rule-b"},
|
||||
MaxAgeSeconds: 2,
|
||||
},
|
||||
}}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodOptions, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
req.Header.Set("Access-Control-Request-Method", "GET")
|
||||
req.Header.Set("Access-Control-Request-Headers", "X-B")
|
||||
|
||||
if handled := applyBucketCors(rec, req, cfg); !handled {
|
||||
t.Fatal("expected preflight to be handled")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Expose-Headers"); got != "x-rule-b" {
|
||||
t.Fatalf("selected rule expose-headers = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Methods"); got != "GET, HEAD" {
|
||||
t.Fatalf("selected rule allow-methods = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Max-Age"); got != "2" {
|
||||
t.Fatalf("selected rule max-age = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPerBucketCorsActualRequest(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"*"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
ExposeHeaders: []string{"ETag"},
|
||||
}}}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "http://any.com")
|
||||
|
||||
handled := applyBucketCors(rec, req, cfg)
|
||||
if handled {
|
||||
t.Fatal("actual (non-preflight) request must not be terminated by CORS")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Expose-Headers"); got != "ETag" {
|
||||
t.Fatalf("expose-headers = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPerBucketCorsOriginPatternResponse(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"https://app.example.com", "https://*", "*"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
}}}
|
||||
|
||||
tests := []struct {
|
||||
origin string
|
||||
wantOrigin string
|
||||
wantCredentials string
|
||||
}{
|
||||
{"https://app.example.com", "https://app.example.com", "true"},
|
||||
{"https://other.example.com", "https://other.example.com", "true"},
|
||||
{"http://other.example.com", "*", ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", tt.origin)
|
||||
if handled := applyBucketCors(rec, req, cfg); handled {
|
||||
t.Fatal("actual request must not be terminated by CORS")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != tt.wantOrigin {
|
||||
t.Fatalf("origin %q: allow-origin = %q, want %q", tt.origin, got, tt.wantOrigin)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != tt.wantCredentials {
|
||||
t.Fatalf("origin %q: allow-credentials = %q, want %q", tt.origin, got, tt.wantCredentials)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketCorsMetadataErrorFailsClosed(t *testing.T) {
|
||||
oldObjectAPI := newObjectLayerFn()
|
||||
oldMetadataSys := globalBucketMetadataSys
|
||||
setObjectLayer(nil)
|
||||
globalBucketMetadataSys = NewBucketMetadataSys()
|
||||
// A resident bucket whose stored CORS document does not parse must not be
|
||||
// answered with the global policy: it has a configuration we cannot honor.
|
||||
meta := newBucketMetadata("cors-metadata-error")
|
||||
meta.corsConfigErr = fmt.Errorf("invalid bucket CORS configuration")
|
||||
globalBucketMetadataSys.Set("cors-metadata-error", meta)
|
||||
defer func() {
|
||||
setObjectLayer(oldObjectAPI)
|
||||
globalBucketMetadataSys = oldMetadataSys
|
||||
}()
|
||||
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
|
||||
for _, method := range []string{http.MethodGet, http.MethodOptions} {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(method, getGetObjectURL("", "cors-metadata-error", "object"), nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
if method == http.MethodOptions {
|
||||
req.Header.Set("Access-Control-Request-Method", http.MethodGet)
|
||||
}
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("%s status = %d, want %d", method, rec.Code, http.StatusNoContent)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
|
||||
t.Fatalf("%s metadata error fell back to global allow-origin %q", method, got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "" {
|
||||
t.Fatalf("%s metadata error fell back to global credentials %q", method, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketCorsSkipsMetadataLookupWithoutOrigin(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsSkipsMetadataLookupWithoutOrigin,
|
||||
endpoints: []string{"GetObject"},
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketCorsSkipsMetadataLookupWithoutOrigin(obj ObjectLayer, _ string, _ string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
oldObjectAPI := newObjectLayerFn()
|
||||
counting := &corsLookupCountingObjectLayer{ObjectLayer: obj}
|
||||
setObjectLayer(counting)
|
||||
defer setObjectLayer(oldObjectAPI)
|
||||
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
rec := httptest.NewRecorder()
|
||||
wrapped.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, getGetObjectURL("", "api", "v1/login"), nil))
|
||||
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusNoContent)
|
||||
}
|
||||
requireCorsOriginVary(t, rec.Header())
|
||||
if got := counting.getObjectNInfoCalls.Load(); got != 0 {
|
||||
t.Fatalf("request without Origin performed %d bucket metadata reads", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketCorsOriginlessPreflightShapeUsesGlobalHandler(t *testing.T) {
|
||||
nextCalled := false
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
nextCalled = true
|
||||
w.WriteHeader(http.StatusTeapot)
|
||||
}))
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodOptions, getGetObjectURL("", "api", "v1/login"), nil)
|
||||
req.Header.Set("Access-Control-Request-Method", http.MethodGet)
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusNoContent)
|
||||
}
|
||||
if nextCalled {
|
||||
t.Fatal("originless preflight-shaped OPTIONS reached the application handler")
|
||||
}
|
||||
requireCorsOriginVary(t, rec.Header())
|
||||
}
|
||||
|
||||
func TestBucketCorsNoConfigUsesGlobalFallback(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsNoConfigUsesGlobalFallback,
|
||||
endpoints: []string{"GetBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func TestBucketCorsMissingBucketUsesGlobalFallback(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsMissingBucketUsesGlobalFallback,
|
||||
endpoints: []string{"GetBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketCorsMissingBucketUsesGlobalFallback(obj ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
// Model a fully started server: bucket metadata loading has completed, so
|
||||
// a name that is not resident is genuinely not a CORS-bearing bucket.
|
||||
restore := markBucketMetadataInitialized(t)
|
||||
defer restore()
|
||||
|
||||
// A non-resident bucket name must not cause any bucket-metadata disk read.
|
||||
oldObjectAPI := newObjectLayerFn()
|
||||
counting := &corsLookupCountingObjectLayer{ObjectLayer: obj}
|
||||
setObjectLayer(counting)
|
||||
defer setObjectLayer(oldObjectAPI)
|
||||
|
||||
before := bucketMetadataMapLen()
|
||||
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, getGetObjectURL("", bucket+"-missing", "object"), nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusNotFound)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://app.example.com" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "true" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
// Regression guard: the pre-auth CORS lookup for a non-existent bucket must
|
||||
// neither read bucket metadata from disk nor cache a synthetic entry.
|
||||
if got := counting.getObjectNInfoCalls.Load(); got != 0 {
|
||||
t.Fatalf("missing-bucket CORS lookup performed %d bucket metadata reads", got)
|
||||
}
|
||||
if after := bucketMetadataMapLen(); after != before {
|
||||
t.Fatalf("missing-bucket CORS lookup grew metadataMap from %d to %d", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
func testBucketCorsNoConfigUsesGlobalFallback(_ ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, getGetObjectURL("", bucket, "object"), nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusNoContent)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://app.example.com" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "true" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPerBucketCorsActualPatternOriginSupportsCredentials(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"https://*.example.com"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
}}}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
|
||||
if handled := applyBucketCors(rec, req, cfg); handled {
|
||||
t.Fatal("actual request must continue")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://app.example.com" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "true" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPerBucketCorsActualNullOriginSurvivesForwardingMiddleware(t *testing.T) {
|
||||
next := setBucketForwardingMiddleware(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
|
||||
t.Run("per-bucket null origin", func(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"null"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
}}}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "null")
|
||||
|
||||
if handled := applyBucketCors(rec, req, cfg); handled {
|
||||
t.Fatal("actual request must continue")
|
||||
}
|
||||
next.ServeHTTP(rec, req)
|
||||
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "null" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "true" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("legacy unmarked null origin", func(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
rec.Header().Set("Access-Control-Allow-Origin", "null")
|
||||
req := httptest.NewRequest(http.MethodGet, "/mybucket/obj", nil)
|
||||
|
||||
next.ServeHTTP(rec, req)
|
||||
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func requireCorsVary(t *testing.T, header http.Header) {
|
||||
t.Helper()
|
||||
values := strings.Join(header.Values("Vary"), ",")
|
||||
for _, want := range []string{"Origin", "Access-Control-Request-Method", "Access-Control-Request-Headers"} {
|
||||
if !strings.Contains(values, want) {
|
||||
t.Fatalf("Vary = %q, missing %q", values, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func requireCorsOriginVary(t *testing.T, header http.Header) {
|
||||
t.Helper()
|
||||
if values := strings.Join(header.Values("Vary"), ","); !strings.Contains(values, "Origin") {
|
||||
t.Fatalf("Vary = %q, missing Origin", values)
|
||||
}
|
||||
}
|
||||
|
||||
// markBucketMetadataInitialized marks the global bucket-metadata subsystem as
|
||||
// fully loaded, modeling a running server (the API test harness sets up the
|
||||
// subsystem but does not run Init). It returns a function that restores the
|
||||
// previous state.
|
||||
func markBucketMetadataInitialized(t *testing.T) func() {
|
||||
t.Helper()
|
||||
sys := globalBucketMetadataSys
|
||||
if sys == nil {
|
||||
t.Fatal("globalBucketMetadataSys is nil")
|
||||
}
|
||||
sys.Lock()
|
||||
prev := sys.initialized
|
||||
sys.initialized = true
|
||||
sys.Unlock()
|
||||
return func() {
|
||||
sys.Lock()
|
||||
sys.initialized = prev
|
||||
sys.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// bucketMetadataMapLen returns the number of resident bucket-metadata entries.
|
||||
func bucketMetadataMapLen() int {
|
||||
sys := globalBucketMetadataSys
|
||||
if sys == nil {
|
||||
return 0
|
||||
}
|
||||
sys.RLock()
|
||||
defer sys.RUnlock()
|
||||
return len(sys.metadataMap)
|
||||
}
|
||||
|
||||
// TestBucketCorsUnknownBucketDoesNotGrowMetadata is the regression guard for
|
||||
// the pre-auth resource-exhaustion path: an unauthenticated, Origin-bearing
|
||||
// request whose first path segment is not a real bucket must fall back to the
|
||||
// global CORS policy without loading bucket metadata from disk and without
|
||||
// caching a synthetic entry. Before the resident-only lookup, each distinct
|
||||
// name grew metadataMap by one and issued an erasure metadata probe.
|
||||
func TestBucketCorsUnknownBucketDoesNotGrowMetadata(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsUnknownBucketDoesNotGrowMetadata,
|
||||
endpoints: []string{"GetBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketCorsUnknownBucketDoesNotGrowMetadata(obj ObjectLayer, _ string, _ string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
restore := markBucketMetadataInitialized(t)
|
||||
defer restore()
|
||||
|
||||
oldObjectAPI := newObjectLayerFn()
|
||||
counting := &corsLookupCountingObjectLayer{ObjectLayer: obj}
|
||||
setObjectLayer(counting)
|
||||
defer setObjectLayer(oldObjectAPI)
|
||||
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
|
||||
before := bucketMetadataMapLen()
|
||||
// Console/admin routes plus enough distinct valid names to make accidental
|
||||
// cache growth or one metadata probe per name unambiguous.
|
||||
names := []string{"minio", "api"}
|
||||
for i := 0; i < 500; i++ {
|
||||
names = append(names, fmt.Sprintf("cors-missing-%03d", i))
|
||||
}
|
||||
for _, name := range names {
|
||||
for _, method := range []string{http.MethodGet, http.MethodOptions} {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(method, getGetObjectURL("", name, "obj"), nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
if method == http.MethodOptions {
|
||||
req.Header.Set("Access-Control-Request-Method", http.MethodGet)
|
||||
}
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://app.example.com" {
|
||||
t.Fatalf("%s/%s: allow-origin = %q, want global fallback", name, method, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, path := range []string{"/../obj", "/A/obj", "/x/obj", "/minio/admin/v3/info", "/api/v1/login"} {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://app.example.com" {
|
||||
t.Fatalf("%s: allow-origin = %q, want global fallback", path, got)
|
||||
}
|
||||
}
|
||||
|
||||
if got := counting.getObjectNInfoCalls.Load(); got != 0 {
|
||||
t.Fatalf("unknown-bucket CORS lookups performed %d bucket metadata reads", got)
|
||||
}
|
||||
if after := bucketMetadataMapLen(); after != before {
|
||||
t.Fatalf("unknown-bucket CORS lookups grew metadataMap from %d to %d", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketCorsStartupMissFailsClosedWithoutIO(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsStartupMissFailsClosedWithoutIO,
|
||||
endpoints: []string{"GetBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketCorsStartupMissFailsClosedWithoutIO(obj ObjectLayer, _ string, _ string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
oldObjectAPI := newObjectLayerFn()
|
||||
oldMetadataSys := globalBucketMetadataSys
|
||||
counting := &corsLookupCountingObjectLayer{ObjectLayer: obj}
|
||||
setObjectLayer(counting)
|
||||
globalBucketMetadataSys = NewBucketMetadataSys()
|
||||
defer func() {
|
||||
setObjectLayer(oldObjectAPI)
|
||||
globalBucketMetadataSys = oldMetadataSys
|
||||
}()
|
||||
|
||||
innerCalled := false
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
innerCalled = true
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/startup-missing/object", nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if !innerCalled || rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("startup miss did not reach inner handler: called=%v status=%d", innerCalled, rec.Code)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
|
||||
t.Fatalf("startup miss used permissive global CORS: %q", got)
|
||||
}
|
||||
if got := counting.getObjectNInfoCalls.Load(); got != 0 {
|
||||
t.Fatalf("startup miss performed %d metadata reads", got)
|
||||
}
|
||||
if got := globalBucketMetadataSys.Count(); got != 0 {
|
||||
t.Fatalf("startup miss grew metadataMap to %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// markBucketMetadataLoadFailed records a bucket as one whose metadata failed to
|
||||
// load at startup while the subsystem is Initialized, modeling the degraded
|
||||
// state where a real bucket is not resident. Returns a restore function.
|
||||
func markBucketMetadataLoadFailed(t *testing.T, bucket string) func() {
|
||||
t.Helper()
|
||||
sys := globalBucketMetadataSys
|
||||
if sys == nil {
|
||||
t.Fatal("globalBucketMetadataSys is nil")
|
||||
}
|
||||
sys.Lock()
|
||||
_, had := sys.loadFailed[bucket]
|
||||
sys.loadFailed[bucket] = struct{}{}
|
||||
sys.Unlock()
|
||||
return func() {
|
||||
sys.Lock()
|
||||
if !had {
|
||||
delete(sys.loadFailed, bucket)
|
||||
}
|
||||
sys.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// TestBucketCorsLoadFailedBucketFailsClosed guards P1: a real bucket whose
|
||||
// metadata could not be loaded at startup (present in loadFailed, subsystem
|
||||
// Initialized) must NOT be answered with the permissive global CORS policy. We
|
||||
// cannot rule out a restrictive per-bucket config for it, so it must fail
|
||||
// closed — without a synchronous disk read.
|
||||
func TestBucketCorsLoadFailedBucketFailsClosed(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsLoadFailedBucketFailsClosed,
|
||||
endpoints: []string{"GetBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketCorsLoadFailedBucketFailsClosed(obj ObjectLayer, _ string, _ string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
restoreInit := markBucketMetadataInitialized(t)
|
||||
defer restoreInit()
|
||||
restoreFail := markBucketMetadataLoadFailed(t, "strict-cors-bucket")
|
||||
defer restoreFail()
|
||||
|
||||
oldObjectAPI := newObjectLayerFn()
|
||||
counting := &corsLookupCountingObjectLayer{ObjectLayer: obj}
|
||||
setObjectLayer(counting)
|
||||
defer setObjectLayer(oldObjectAPI)
|
||||
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
|
||||
for _, method := range []string{http.MethodGet, http.MethodOptions} {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(method, getGetObjectURL("", "strict-cors-bucket", "object"), nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
if method == http.MethodOptions {
|
||||
req.Header.Set("Access-Control-Request-Method", http.MethodGet)
|
||||
}
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
|
||||
t.Fatalf("%s: load-failed bucket fell back to global allow-origin %q", method, got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "" {
|
||||
t.Fatalf("%s: load-failed bucket fell back to global credentials %q", method, got)
|
||||
}
|
||||
}
|
||||
if got := counting.getObjectNInfoCalls.Load(); got != 0 {
|
||||
t.Fatalf("load-failed CORS lookup performed %d synchronous bucket metadata reads", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBucketCorsResidentConfigSurvivesRefreshFailure: a resident bucket keeps
|
||||
// its last loaded CORS configuration through a failed refresh, like every
|
||||
// other bucket configuration, and the failure set never records a resident
|
||||
// bucket. Only a bucket that was never loaded fails closed.
|
||||
func TestBucketCorsResidentConfigSurvivesRefreshFailure(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsResidentConfigSurvivesRefreshFailure,
|
||||
endpoints: []string{"GetBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketCorsResidentConfigSurvivesRefreshFailure(obj ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
if _, err := updateLocalBucketCORSMetadata(t.Context(), obj, bucket, []byte(testSiteReplicationCORSDoc)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sys := globalBucketMetadataSys
|
||||
sys.Lock()
|
||||
sys.noteLoadFailure(bucket)
|
||||
_, marked := sys.loadFailed[bucket]
|
||||
sys.Unlock()
|
||||
if marked {
|
||||
t.Fatal("a resident bucket was recorded as a load failure")
|
||||
}
|
||||
cfg, _, err := sys.GetResidentCorsConfig(bucket)
|
||||
if err != nil || cfg == nil {
|
||||
t.Fatalf("resident CORS configuration lost after a refresh failure: cfg=%v err=%v", cfg, err)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -30,7 +30,7 @@ import (
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
+18
-18
@@ -62,8 +62,8 @@ import (
|
||||
"github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -463,13 +463,20 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
// Make sure to update context to print ObjectNames for multi objects.
|
||||
ctx = updateReqContext(ctx, objects...)
|
||||
|
||||
// Call checkRequestAuthType to populate ReqInfo.AccessKey before GetBucketInfo()
|
||||
// Ignore errors here to preserve the S3 error behavior of GetBucketInfo()
|
||||
checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "")
|
||||
|
||||
deleteObjectsFn := objectAPI.DeleteObjects
|
||||
|
||||
// Return Malformed XML as S3 spec if the number of objects is empty
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
if reqInfo == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
reqInfo.BucketName = bucket
|
||||
reqInfo.ObjectName = ""
|
||||
if s3Err := authenticateRequest(ctx, r, policy.DeleteObjectAction); s3Err != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
// Return Malformed XML as S3 spec if the number of objects is empty.
|
||||
if len(deleteObjectsReq.Objects) == 0 || len(deleteObjectsReq.Objects) > maxDeleteList {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL)
|
||||
return
|
||||
@@ -499,11 +506,9 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
vc, _ := globalBucketVersioningSys.Get(bucket)
|
||||
oss := make([]*objSweeper, len(deleteObjectsReq.Objects))
|
||||
for index, object := range deleteObjectsReq.Objects {
|
||||
if apiErrCode := checkRequestAuthTypeWithVID(ctx, r, policy.DeleteObjectAction, bucket, object.ObjectName, object.VersionID); apiErrCode != ErrNone {
|
||||
if apiErrCode == ErrSignatureDoesNotMatch || apiErrCode == ErrInvalidAccessKeyID {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL)
|
||||
return
|
||||
}
|
||||
reqInfo.ObjectName = object.ObjectName
|
||||
reqInfo.VersionID = object.VersionID
|
||||
if apiErrCode := authorizeRequest(ctx, r, deleteObjectAction(object.VersionID)); apiErrCode != ErrNone {
|
||||
apiErr := errorCodes.ToAPIErr(apiErrCode)
|
||||
deleteResults[index].errInfo = DeleteError{
|
||||
Code: apiErr.Code,
|
||||
@@ -1844,12 +1849,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
|
||||
// We encode the xml bytes as base64 to ensure there are no encoding
|
||||
// errors.
|
||||
cfgStr := base64.StdEncoding.EncodeToString(configData)
|
||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypeObjectLockConfig,
|
||||
Bucket: bucket,
|
||||
ObjectLockConfig: &cfgStr,
|
||||
UpdatedAt: updatedAt,
|
||||
}))
|
||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, newSRBucketObjectLockMeta(bucket, &cfgStr, updatedAt)))
|
||||
|
||||
// Write success response.
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
|
||||
+71
-18
@@ -24,12 +24,52 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
)
|
||||
|
||||
func TestListObjectsNonExistentBucketHandler(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testListObjectsNonExistentBucketHandler})
|
||||
}
|
||||
|
||||
func testListObjectsNonExistentBucketHandler(_ ObjectLayer, instanceType, _ string, apiRouter http.Handler,
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
const bucket = "missing-bucket"
|
||||
testCases := []struct {
|
||||
name string
|
||||
query url.Values
|
||||
}{
|
||||
{name: "ListObjects", query: url.Values{"prefix": {"/"}}},
|
||||
{name: "ListObjectsV2", query: url.Values{"list-type": {"2"}, "prefix": {"/"}}},
|
||||
{name: "ListObjectVersions", query: url.Values{"versions": {""}, "prefix": {"/"}}},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
req, err := newTestSignedRequestV4(http.MethodGet, makeTestTargetURL("", bucket, "", tc.query), 0, nil,
|
||||
credentials.AccessKey, credentials.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %s: failed to create request: %v", instanceType, tc.name, err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Errorf("%s: %s: expected status %d, got %d", instanceType, tc.name, http.StatusNotFound, rec.Code)
|
||||
}
|
||||
|
||||
var apiErr APIErrorResponse
|
||||
if err = xml.Unmarshal(rec.Body.Bytes(), &apiErr); err != nil {
|
||||
t.Fatalf("%s: %s: failed to decode error response: %v", instanceType, tc.name, err)
|
||||
}
|
||||
if apiErr.Code != "NoSuchBucket" {
|
||||
t.Errorf("%s: %s: expected NoSuchBucket, got %q", instanceType, tc.name, apiErr.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Wrapper for calling RemoveBucket HTTP handler tests for both Erasure multiple disks and single node setup.
|
||||
func TestRemoveBucketHandler(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testRemoveBucketHandler, endpoints: []string{"RemoveBucket"}})
|
||||
@@ -978,14 +1018,23 @@ func testAPIDeleteMultipleObjectsVersionIDNullCondition(obj ObjectLayer, instanc
|
||||
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[{
|
||||
"Effect":"Allow",
|
||||
"Principal":"*",
|
||||
"Action":"s3:DeleteObject",
|
||||
"Resource":"arn:aws:s3:::%s/*",
|
||||
"Condition":{"Null":{"s3:versionid":"true"}}
|
||||
}]
|
||||
}`, bucketName)
|
||||
"Statement":[
|
||||
{
|
||||
"Effect":"Allow",
|
||||
"Principal":"*",
|
||||
"Action":"s3:DeleteObject",
|
||||
"Resource":"arn:aws:s3:::%s/*",
|
||||
"Condition":{"Null":{"s3:versionid":"true"}}
|
||||
},
|
||||
{
|
||||
"Effect":"Allow",
|
||||
"Principal":"*",
|
||||
"Action":"s3:DeleteObjectVersion",
|
||||
"Resource":"arn:aws:s3:::%s/*",
|
||||
"Condition":{"StringEquals":{"s3:versionid":"%s"}}
|
||||
}
|
||||
]
|
||||
}`, bucketName, bucketName, versionIDs["with-version-id"])
|
||||
policyReq, err := newTestSignedRequestV4(http.MethodPut, getPutPolicyURL("", bucketName), int64(len(policyBytes)),
|
||||
bytes.NewReader(policyBytes), credentials.AccessKey, credentials.SecretKey, nil)
|
||||
if err != nil {
|
||||
@@ -1031,29 +1080,30 @@ func testAPIDeleteMultipleObjectsVersionIDNullCondition(obj ObjectLayer, instanc
|
||||
t.Errorf("%s: %q was not a successful delete-marker creation: %+v", instanceType, objectName, response.DeletedObjects)
|
||||
}
|
||||
}
|
||||
if len(deleted) != 2 {
|
||||
if object, ok := deleted["with-version-id"]; !ok || object.VersionID != versionIDs["with-version-id"] {
|
||||
t.Errorf("%s: matching explicit version was not deleted: %+v", instanceType, response.DeletedObjects)
|
||||
}
|
||||
if len(deleted) != 3 {
|
||||
t.Errorf("%s: unexpected deleted objects: %+v", instanceType, response.DeletedObjects)
|
||||
}
|
||||
errorsByKey := make(map[string]DeleteError, len(response.Errors))
|
||||
for _, deleteErr := range response.Errors {
|
||||
errorsByKey[deleteErr.Key] = deleteErr
|
||||
}
|
||||
for objectName, versionID := range map[string]string{
|
||||
"with-version-id": versionIDs["with-version-id"],
|
||||
"with-null-version-id": nullVersionID,
|
||||
} {
|
||||
for objectName, versionID := range map[string]string{"with-null-version-id": nullVersionID} {
|
||||
deleteErr, ok := errorsByKey[objectName]
|
||||
if !ok || deleteErr.VersionID != versionID || deleteErr.Code != errorCodes[ErrAccessDenied].Code {
|
||||
t.Errorf("%s: %q did not return AccessDenied for version %q: %+v", instanceType, objectName, versionID, response.Errors)
|
||||
}
|
||||
}
|
||||
if len(errorsByKey) != 2 {
|
||||
if len(errorsByKey) != 1 {
|
||||
t.Errorf("%s: unexpected delete errors: %+v", instanceType, response.Errors)
|
||||
}
|
||||
|
||||
// A simple delete adds a marker and keeps the old version. The explicitly
|
||||
// named version must also remain because its policy condition did not match.
|
||||
for objectName, versionID := range versionIDs {
|
||||
// A simple delete adds a marker and keeps the old version. The null-version
|
||||
// delete remains denied because its per-entry condition does not match.
|
||||
for _, objectName := range []string{"without-version-id-before", "without-version-id-after", "with-null-version-id"} {
|
||||
versionID := versionIDs[objectName]
|
||||
if _, err = obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{VersionID: versionID}); err != nil {
|
||||
t.Errorf("%s: version %s of %q was not preserved: %v", instanceType, versionID, objectName, err)
|
||||
}
|
||||
@@ -1063,7 +1113,10 @@ func testAPIDeleteMultipleObjectsVersionIDNullCondition(obj ObjectLayer, instanc
|
||||
t.Errorf("%s: simple delete of %q did not hide the latest object behind a delete marker: %v", instanceType, objectName, err)
|
||||
}
|
||||
}
|
||||
for _, objectName := range []string{"with-version-id", "with-null-version-id"} {
|
||||
if _, err = obj.GetObjectInfo(t.Context(), bucketName, "with-version-id", ObjectOptions{VersionID: versionIDs["with-version-id"]}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Errorf("%s: matching explicit version still exists: %v", instanceType, err)
|
||||
}
|
||||
for _, objectName := range []string{"with-null-version-id"} {
|
||||
if info, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err != nil {
|
||||
t.Errorf("%s: denied version delete removed latest %q: %v", instanceType, objectName, err)
|
||||
} else if info.VersionID != versionIDs[objectName] {
|
||||
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -41,7 +41,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/s3select"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
"github.com/zeebo/xxh3"
|
||||
)
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ import (
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// Validate all the ListObjects query arguments, returns an APIErrorCode
|
||||
|
||||
@@ -0,0 +1,455 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
// Copyright (c) 2026 PGSTY
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
)
|
||||
|
||||
type metadataRMWWriterKey struct{}
|
||||
|
||||
type metadataRMWBarrierObjectLayer struct {
|
||||
ObjectLayer
|
||||
bucket string
|
||||
aReady chan struct{}
|
||||
aRelease chan struct{}
|
||||
bLockAttempt chan struct{}
|
||||
aReadyOnce sync.Once
|
||||
bLockOnce sync.Once
|
||||
cancelOnce sync.Once
|
||||
cancelOnPut context.CancelFunc
|
||||
reads atomic.Int64
|
||||
}
|
||||
|
||||
func (o *metadataRMWBarrierObjectLayer) metadataObject() string {
|
||||
return pathJoin(bucketMetaPrefix, o.bucket, bucketMetadataFile)
|
||||
}
|
||||
|
||||
func (o *metadataRMWBarrierObjectLayer) metadataLock() string {
|
||||
return pathJoin(bucketMetaPrefix, o.bucket, "metadata.lock")
|
||||
}
|
||||
|
||||
func (o *metadataRMWBarrierObjectLayer) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
|
||||
if bucket == minioMetaBucket && object == o.metadataObject() {
|
||||
o.reads.Add(1)
|
||||
}
|
||||
return o.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
|
||||
}
|
||||
|
||||
func (o *metadataRMWBarrierObjectLayer) PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
|
||||
if bucket == minioMetaBucket && object == o.metadataObject() && o.cancelOnPut != nil {
|
||||
o.cancelOnce.Do(o.cancelOnPut)
|
||||
}
|
||||
if bucket == minioMetaBucket && object == o.metadataObject() && ctx.Value(metadataRMWWriterKey{}) == "A" {
|
||||
o.aReadyOnce.Do(func() { close(o.aReady) })
|
||||
select {
|
||||
case <-o.aRelease:
|
||||
case <-ctx.Done():
|
||||
return ObjectInfo{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
return o.ObjectLayer.PutObject(ctx, bucket, object, data, opts)
|
||||
}
|
||||
|
||||
func (o *metadataRMWBarrierObjectLayer) NewNSLock(bucket string, objects ...string) RWLocker {
|
||||
lock := o.ObjectLayer.NewNSLock(bucket, objects...)
|
||||
if bucket != minioMetaBucket || len(objects) != 1 || objects[0] != o.metadataLock() {
|
||||
return lock
|
||||
}
|
||||
return metadataObservedRWLocker{RWLocker: lock, onLock: func(ctx context.Context) {
|
||||
if ctx.Value(metadataRMWWriterKey{}) == "B" {
|
||||
o.bLockOnce.Do(func() { close(o.bLockAttempt) })
|
||||
}
|
||||
}}
|
||||
}
|
||||
|
||||
type metadataObservedRWLocker struct {
|
||||
RWLocker
|
||||
onLock func(context.Context)
|
||||
}
|
||||
|
||||
func (l metadataObservedRWLocker) GetLock(ctx context.Context, timeout *dynamicTimeout) (LockContext, error) {
|
||||
l.onLock(ctx)
|
||||
return l.RWLocker.GetLock(ctx, timeout)
|
||||
}
|
||||
|
||||
func TestBucketMetadataLockPreservesPolicyAndCORS(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketMetadataLockPreservesPolicyAndCORS,
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketMetadataLockPreservesPolicyAndCORS(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
policyJSON := fmt.Appendf(nil, `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket)
|
||||
corsXML := []byte(testSiteReplicationCORSDoc)
|
||||
runBucketMetadataRMWConflict(t, obj, bucket,
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, policyJSON)
|
||||
return err
|
||||
},
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := updateLocalBucketCORSMetadata(ctx, objectAPI, bucket, corsXML)
|
||||
return err
|
||||
},
|
||||
func(meta BucketMetadata) bool {
|
||||
return bytes.Equal(meta.PolicyConfigJSON, policyJSON) && bytes.Equal(meta.CorsConfigXML, corsXML)
|
||||
}, instanceType+": policy+CORS")
|
||||
}
|
||||
|
||||
func TestBucketMetadataLockPreservesTaggingAndSSE(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketMetadataLockPreservesTaggingAndSSE,
|
||||
})
|
||||
}
|
||||
|
||||
func TestBucketMetadataLockPreservesPeerBulkAndLocalUpdate(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketMetadataLockPreservesPeerBulkAndLocalUpdate,
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketMetadataLockPreservesPeerBulkAndLocalUpdate(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
meta, err := readBucketMetadata(t.Context(), obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
policyJSON := fmt.Appendf(nil, `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket)
|
||||
tagXML := []byte(`<Tagging><TagSet><Tag><Key>local</Key><Value>tag</Value></Tag></TagSet></Tagging>`)
|
||||
runBucketMetadataRMWConflict(t, obj, bucket,
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
return globalSiteReplicationSys.PeerBucketMetadataUpdateHandler(ctx, madmin.SRBucketMeta{
|
||||
Bucket: bucket, Policy: policyJSON, UpdatedAt: meta.Created.Add(time.Second),
|
||||
})
|
||||
},
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, tagXML)
|
||||
return err
|
||||
},
|
||||
func(meta BucketMetadata) bool {
|
||||
return bytes.Equal(meta.PolicyConfigJSON, policyJSON) && bytes.Equal(meta.TaggingConfigXML, tagXML)
|
||||
}, instanceType+": peer bulk+local tagging")
|
||||
}
|
||||
|
||||
func TestBucketMetadataLockPreservesLifecycleDeleteAndSSE(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketMetadataLockPreservesLifecycleDeleteAndSSE,
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketMetadataLockPreservesLifecycleDeleteAndSSE(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
lifecycleXML := []byte(`<LifecycleConfiguration><Rule><ID>expire</ID><Filter><Prefix>logs/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>`)
|
||||
if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketLifecycleConfig, lifecycleXML); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sseXML := []byte(`<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`)
|
||||
runBucketMetadataRMWConflict(t, obj, bucket,
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketLifecycleConfig)
|
||||
return err
|
||||
},
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, sseXML)
|
||||
return err
|
||||
},
|
||||
func(meta BucketMetadata) bool {
|
||||
cfg, err := lifecycle.ParseLifecycleConfig(bytes.NewReader(meta.LifecycleConfigXML))
|
||||
return err == nil && cfg.ExpiryUpdatedAt != nil && len(cfg.Rules) == 0 && bytes.Equal(meta.EncryptionConfigXML, sseXML)
|
||||
}, instanceType+": lifecycle delete+SSE")
|
||||
}
|
||||
|
||||
func TestMakeBucketForceCreatePreservesMetadata(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testMakeBucketForceCreatePreservesMetadata,
|
||||
})
|
||||
}
|
||||
|
||||
func testMakeBucketForceCreatePreservesMetadata(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
ctx := t.Context()
|
||||
policyJSON := fmt.Appendf(nil, `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket)
|
||||
corsXML := []byte(testSiteReplicationCORSDoc)
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, policyJSON); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := updateLocalBucketCORSMetadata(ctx, obj, bucket, corsXML); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, err := readBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{ForceCreate: true}); err != nil {
|
||||
t.Fatalf("%s: ForceCreate existing bucket: %v", instanceType, err)
|
||||
}
|
||||
after, err := readBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !after.Created.Equal(before.Created) || !bytes.Equal(after.PolicyConfigJSON, policyJSON) || !bytes.Equal(after.CorsConfigXML, corsXML) {
|
||||
t.Fatalf("%s: ForceCreate replaced metadata: before=%+v after=%+v", instanceType, before, after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyImportedBucketMetadataPreservesUnspecifiedFields(t *testing.T) {
|
||||
policyJSON := []byte(`{"Version":"2012-10-17","Statement":[]}`)
|
||||
tagXML := []byte(`<Tagging><TagSet><Tag><Key>existing</Key><Value>tag</Value></Tag></TagSet></Tagging>`)
|
||||
src := newBucketMetadata("bucket")
|
||||
src.PolicyConfigJSON = policyJSON
|
||||
src.PolicyConfigUpdatedAt = UTCNow()
|
||||
dst := newBucketMetadata("bucket")
|
||||
dst.TaggingConfigXML = bytes.Clone(tagXML)
|
||||
|
||||
applyImportedBucketMetadata(&dst, src, importMetadataFields{bucketPolicyConfig: {}})
|
||||
if !bytes.Equal(dst.PolicyConfigJSON, policyJSON) || !bytes.Equal(dst.TaggingConfigXML, tagXML) {
|
||||
t.Fatalf("import patch overwrote unspecified metadata: %+v", dst)
|
||||
}
|
||||
src.PolicyConfigJSON[0] = '!'
|
||||
if dst.PolicyConfigJSON[0] == '!' {
|
||||
t.Fatal("import patch retained the source byte slice")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMakeBucketDoesNotAdoptGhostMetadata(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testMakeBucketDoesNotAdoptGhostMetadata,
|
||||
})
|
||||
}
|
||||
|
||||
func testMakeBucketDoesNotAdoptGhostMetadata(obj ObjectLayer, instanceType, _ string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
ctx := t.Context()
|
||||
bucket := getRandomBucketName()
|
||||
if err := obj.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
policyJSON := fmt.Appendf(nil, `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket)
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, policyJSON); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldMeta, err := readBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z, ok := obj.(*erasureServerPools)
|
||||
if !ok {
|
||||
t.Fatalf("%s: object layer is %T, want *erasureServerPools", instanceType, obj)
|
||||
}
|
||||
if err = z.s3Peer.DeleteBucket(ctx, bucket, DeleteBucketOptions{Force: true}); err != nil {
|
||||
t.Fatalf("%s: delete bucket volume only: %v", instanceType, err)
|
||||
}
|
||||
if err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatalf("%s: recreate bucket: %v", instanceType, err)
|
||||
}
|
||||
newMeta, err := readBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bytes.Equal(newMeta.PolicyConfigJSON, policyJSON) || newMeta.Created.Equal(oldMeta.Created) {
|
||||
t.Fatalf("%s: new bucket adopted ghost metadata: old=%+v new=%+v", instanceType, oldMeta, newMeta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMakeBucketForceCreateLockEnablesVersioning(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testMakeBucketForceCreateLockEnablesVersioning,
|
||||
})
|
||||
}
|
||||
|
||||
func TestPeerBucketMetadataSaveSurvivesCallerCancellation(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testPeerBucketMetadataSaveSurvivesCallerCancellation,
|
||||
})
|
||||
}
|
||||
|
||||
func testPeerBucketMetadataSaveSurvivesCallerCancellation(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
previousObjectAPI := newObjectLayerFn()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
barrier := &metadataRMWBarrierObjectLayer{
|
||||
ObjectLayer: obj,
|
||||
bucket: bucket,
|
||||
cancelOnPut: cancel,
|
||||
}
|
||||
setObjectLayer(barrier)
|
||||
defer setObjectLayer(previousObjectAPI)
|
||||
|
||||
err := globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(ctx, bucket, MakeBucketOptions{VersioningEnabled: true})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: peer metadata save failed after caller cancellation: %v", instanceType, err)
|
||||
}
|
||||
if ctx.Err() != context.Canceled {
|
||||
t.Fatalf("%s: metadata write did not trigger caller cancellation", instanceType)
|
||||
}
|
||||
meta, err := readBucketMetadata(t.Context(), obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := versioning.ParseConfig(bytes.NewReader(meta.VersioningConfigXML))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !cfg.Enabled() {
|
||||
t.Fatalf("%s: peer metadata save lost versioning after cancellation", instanceType)
|
||||
}
|
||||
}
|
||||
|
||||
func testMakeBucketForceCreateLockEnablesVersioning(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
ctx := t.Context()
|
||||
suspended := []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Suspended</Status></VersioningConfiguration>`)
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, suspended); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := obj.MakeBucket(ctx, bucket, MakeBucketOptions{ForceCreate: true, LockEnabled: true}); err != nil {
|
||||
t.Fatalf("%s: ForceCreate with object lock: %v", instanceType, err)
|
||||
}
|
||||
meta, err := readBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := versioning.ParseConfig(bytes.NewReader(meta.VersioningConfigXML))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !cfg.Enabled() || len(meta.ObjectLockConfigXML) == 0 {
|
||||
t.Fatalf("%s: object lock state lacks enabled versioning: metadata=%+v", instanceType, meta)
|
||||
}
|
||||
}
|
||||
|
||||
func testBucketMetadataLockPreservesTaggingAndSSE(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
tagXML := []byte(`<Tagging><TagSet><Tag><Key>key</Key><Value>value</Value></Tag></TagSet></Tagging>`)
|
||||
sseXML := []byte(`<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`)
|
||||
runBucketMetadataRMWConflict(t, obj, bucket,
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, tagXML)
|
||||
return err
|
||||
},
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, sseXML)
|
||||
return err
|
||||
},
|
||||
func(meta BucketMetadata) bool {
|
||||
return bytes.Equal(meta.TaggingConfigXML, tagXML) && bytes.Equal(meta.EncryptionConfigXML, sseXML)
|
||||
}, instanceType+": tagging+SSE")
|
||||
}
|
||||
|
||||
func runBucketMetadataRMWConflict(t *testing.T, obj ObjectLayer, bucket string,
|
||||
writerA, writerB func(context.Context, ObjectLayer) error,
|
||||
complete func(BucketMetadata) bool, name string,
|
||||
) {
|
||||
t.Helper()
|
||||
previousObjectAPI := newObjectLayerFn()
|
||||
barrier := &metadataRMWBarrierObjectLayer{
|
||||
ObjectLayer: obj,
|
||||
bucket: bucket,
|
||||
aReady: make(chan struct{}),
|
||||
aRelease: make(chan struct{}),
|
||||
bLockAttempt: make(chan struct{}),
|
||||
}
|
||||
setObjectLayer(barrier)
|
||||
defer setObjectLayer(previousObjectAPI)
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
aCtx := context.WithValue(ctx, metadataRMWWriterKey{}, "A")
|
||||
bCtx := context.WithValue(ctx, metadataRMWWriterKey{}, "B")
|
||||
aDone := make(chan error, 1)
|
||||
bDone := make(chan error, 1)
|
||||
go func() { aDone <- writerA(aCtx, barrier) }()
|
||||
|
||||
select {
|
||||
case <-barrier.aReady:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("%s: writer A did not reach metadata save: %v", name, ctx.Err())
|
||||
}
|
||||
go func() { bDone <- writerB(bCtx, barrier) }()
|
||||
|
||||
var (
|
||||
bErr error
|
||||
bFinished bool
|
||||
)
|
||||
select {
|
||||
case <-barrier.bLockAttempt:
|
||||
if got := barrier.reads.Load(); got != 1 {
|
||||
t.Fatalf("%s: writer B read metadata before acquiring metadata.lock: reads=%d", name, got)
|
||||
}
|
||||
case bErr = <-bDone:
|
||||
bFinished = true
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("%s: writer B neither completed nor attempted metadata.lock: %v", name, ctx.Err())
|
||||
}
|
||||
close(barrier.aRelease)
|
||||
if err := <-aDone; err != nil {
|
||||
t.Fatalf("%s: writer A failed: %v", name, err)
|
||||
}
|
||||
if !bFinished {
|
||||
select {
|
||||
case bErr = <-bDone:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("%s: writer B did not finish: %v", name, ctx.Err())
|
||||
}
|
||||
}
|
||||
if bErr != nil {
|
||||
t.Fatalf("%s: writer B failed: %v", name, bErr)
|
||||
}
|
||||
|
||||
disk, err := readBucketMetadata(ctx, barrier, bucket)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: read disk metadata: %v", name, err)
|
||||
}
|
||||
resident, err := globalBucketMetadataSys.Get(bucket)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: read resident metadata: %v", name, err)
|
||||
}
|
||||
if !complete(disk) || !complete(resident) {
|
||||
t.Fatalf("%s: concurrent updates lost a field: disk=%+v resident=%+v", name, disk, resident)
|
||||
}
|
||||
}
|
||||
+212
-88
@@ -29,6 +29,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/bucket/cors"
|
||||
bucketsse "github.com/minio/minio/internal/bucket/encryption"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
@@ -37,8 +38,8 @@ import (
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
@@ -50,8 +51,27 @@ type BucketMetadataSys struct {
|
||||
initialized bool
|
||||
group *singleflight.Group
|
||||
metadataMap map[string]BucketMetadata
|
||||
// loadFailed records real buckets whose metadata has never been loaded
|
||||
// successfully because the startup load or a refresh failed. They are
|
||||
// absent from metadataMap even though the subsystem is initialized, and
|
||||
// without this bit a resident-only lookup could not tell them apart from a
|
||||
// name that is not a bucket at all. It never holds a resident bucket, is
|
||||
// bounded by the number of failed loads, and is empty in normal operation.
|
||||
loadFailed map[string]struct{}
|
||||
}
|
||||
|
||||
// noteLoadFailure and clearLoadFailure maintain loadFailed; both expect the
|
||||
// caller to hold sys.Lock. A bucket that is resident keeps its last loaded
|
||||
// metadata through a failed refresh, exactly like every other bucket
|
||||
// configuration, so the set only ever holds non-resident buckets.
|
||||
func (sys *BucketMetadataSys) noteLoadFailure(bucket string) {
|
||||
if _, resident := sys.metadataMap[bucket]; !resident {
|
||||
sys.loadFailed[bucket] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
func (sys *BucketMetadataSys) clearLoadFailure(bucket string) { delete(sys.loadFailed, bucket) }
|
||||
|
||||
// Count returns number of bucket metadata map entries.
|
||||
func (sys *BucketMetadataSys) Count() int {
|
||||
sys.RLock()
|
||||
@@ -66,6 +86,7 @@ func (sys *BucketMetadataSys) Remove(buckets ...string) {
|
||||
for _, bucket := range buckets {
|
||||
sys.group.Forget(bucket)
|
||||
delete(sys.metadataMap, bucket)
|
||||
sys.clearLoadFailure(bucket)
|
||||
globalBucketMonitor.DeleteBucket(bucket)
|
||||
}
|
||||
sys.Unlock()
|
||||
@@ -83,6 +104,11 @@ func (sys *BucketMetadataSys) RemoveStaleBuckets(diskBuckets set.StringSet) {
|
||||
delete(sys.metadataMap, bucket)
|
||||
globalBucketMonitor.DeleteBucket(bucket)
|
||||
}
|
||||
for bucket := range sys.loadFailed {
|
||||
if !diskBuckets.Contains(bucket) {
|
||||
sys.clearLoadFailure(bucket)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Set - sets a new metadata in-memory.
|
||||
@@ -94,11 +120,12 @@ func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
|
||||
if !isMinioMetaBucketName(bucket) {
|
||||
sys.Lock()
|
||||
sys.metadataMap[bucket] = meta
|
||||
sys.clearLoadFailure(bucket)
|
||||
sys.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string, configFile string, configData []byte, parse bool) (updatedAt time.Time, err error) {
|
||||
func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string, configFile string, configData []byte, parse, lifecycleDelete bool) (updatedAt time.Time, err error) {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return updatedAt, errServerNotInitialized
|
||||
@@ -107,60 +134,78 @@ func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string,
|
||||
if isMinioMetaBucketName(bucket) {
|
||||
return updatedAt, errInvalidArgument
|
||||
}
|
||||
|
||||
meta, err := loadBucketMetadataParse(ctx, objAPI, bucket, parse)
|
||||
notifyCtx := ctx
|
||||
ctx, unlock, err := lockBucketMetadata(ctx, objAPI, bucket)
|
||||
if err != nil {
|
||||
if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
|
||||
// Only single drive mode needs this fallback.
|
||||
meta = newBucketMetadata(bucket)
|
||||
} else {
|
||||
return updatedAt, err
|
||||
}
|
||||
}
|
||||
updatedAt = UTCNow()
|
||||
switch configFile {
|
||||
case bucketPolicyConfig:
|
||||
meta.PolicyConfigJSON = configData
|
||||
meta.PolicyConfigUpdatedAt = updatedAt
|
||||
case bucketNotificationConfig:
|
||||
meta.NotificationConfigXML = configData
|
||||
meta.NotificationConfigUpdatedAt = updatedAt
|
||||
case bucketLifecycleConfig:
|
||||
meta.LifecycleConfigXML = configData
|
||||
meta.LifecycleConfigUpdatedAt = updatedAt
|
||||
case bucketSSEConfig:
|
||||
meta.EncryptionConfigXML = configData
|
||||
meta.EncryptionConfigUpdatedAt = updatedAt
|
||||
case bucketTaggingConfig:
|
||||
meta.TaggingConfigXML = configData
|
||||
meta.TaggingConfigUpdatedAt = updatedAt
|
||||
case bucketQuotaConfigFile:
|
||||
meta.QuotaConfigJSON = configData
|
||||
meta.QuotaConfigUpdatedAt = updatedAt
|
||||
case objectLockConfig:
|
||||
meta.ObjectLockConfigXML = configData
|
||||
meta.ObjectLockConfigUpdatedAt = updatedAt
|
||||
case bucketVersioningConfig:
|
||||
meta.VersioningConfigXML = configData
|
||||
meta.VersioningConfigUpdatedAt = updatedAt
|
||||
case bucketReplicationConfig:
|
||||
meta.ReplicationConfigXML = configData
|
||||
meta.ReplicationConfigUpdatedAt = updatedAt
|
||||
case bucketTargetsFile:
|
||||
meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(ctx, meta.Name, configData, kms.Context{
|
||||
bucket: meta.Name,
|
||||
bucketTargetsFile: bucketTargetsFile,
|
||||
})
|
||||
if err != nil {
|
||||
return updatedAt, fmt.Errorf("Error encrypting bucket target metadata %w", err)
|
||||
}
|
||||
meta.BucketTargetsConfigUpdatedAt = updatedAt
|
||||
meta.BucketTargetsConfigMetaUpdatedAt = updatedAt
|
||||
default:
|
||||
return updatedAt, fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
|
||||
return updatedAt, err
|
||||
}
|
||||
|
||||
return updatedAt, sys.save(ctx, meta)
|
||||
err = func() error {
|
||||
defer unlock()
|
||||
meta, err := loadBucketMetadataParse(ctx, objAPI, bucket, parse)
|
||||
if err != nil {
|
||||
if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
|
||||
// Only single drive mode needs this fallback.
|
||||
meta = newBucketMetadata(bucket)
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if lifecycleDelete {
|
||||
configData, err = lifecycleDeleteConfig(meta.LifecycleConfigXML)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
updatedAt = UTCNow()
|
||||
switch configFile {
|
||||
case bucketPolicyConfig:
|
||||
meta.PolicyConfigJSON = configData
|
||||
meta.PolicyConfigUpdatedAt = updatedAt
|
||||
case bucketNotificationConfig:
|
||||
meta.NotificationConfigXML = configData
|
||||
meta.NotificationConfigUpdatedAt = updatedAt
|
||||
case bucketLifecycleConfig:
|
||||
meta.LifecycleConfigXML = configData
|
||||
meta.LifecycleConfigUpdatedAt = updatedAt
|
||||
case bucketSSEConfig:
|
||||
meta.EncryptionConfigXML = configData
|
||||
meta.EncryptionConfigUpdatedAt = updatedAt
|
||||
case bucketTaggingConfig:
|
||||
meta.TaggingConfigXML = configData
|
||||
meta.TaggingConfigUpdatedAt = updatedAt
|
||||
case bucketQuotaConfigFile:
|
||||
meta.QuotaConfigJSON = configData
|
||||
meta.QuotaConfigUpdatedAt = updatedAt
|
||||
case objectLockConfig:
|
||||
meta.ObjectLockConfigXML = configData
|
||||
meta.ObjectLockConfigUpdatedAt = updatedAt
|
||||
case bucketVersioningConfig:
|
||||
meta.VersioningConfigXML = configData
|
||||
meta.VersioningConfigUpdatedAt = updatedAt
|
||||
case bucketReplicationConfig:
|
||||
meta.ReplicationConfigXML = configData
|
||||
meta.ReplicationConfigUpdatedAt = updatedAt
|
||||
case bucketTargetsFile:
|
||||
meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(ctx, meta.Name, configData, kms.Context{
|
||||
bucket: meta.Name,
|
||||
bucketTargetsFile: bucketTargetsFile,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error encrypting bucket target metadata %w", err)
|
||||
}
|
||||
meta.BucketTargetsConfigUpdatedAt = updatedAt
|
||||
meta.BucketTargetsConfigMetaUpdatedAt = updatedAt
|
||||
default:
|
||||
return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
|
||||
}
|
||||
return sys.saveMetadata(ctx, objAPI, meta)
|
||||
}()
|
||||
if err != nil {
|
||||
return updatedAt, err
|
||||
}
|
||||
globalNotificationSys.LoadBucketMetadata(bgContext(notifyCtx), bucket) // Do not use caller context here
|
||||
return updatedAt, nil
|
||||
}
|
||||
|
||||
func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) error {
|
||||
@@ -173,59 +218,78 @@ func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) err
|
||||
return errInvalidArgument
|
||||
}
|
||||
|
||||
if err := meta.Save(ctx, objAPI); err != nil {
|
||||
if err := sys.saveMetadata(ctx, objAPI, meta); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sys.Set(meta.Name, meta)
|
||||
globalNotificationSys.LoadBucketMetadata(bgContext(ctx), meta.Name) // Do not use caller context here
|
||||
return nil
|
||||
}
|
||||
|
||||
// saveMetadata persists and publishes metadata locally. Callers performing a
|
||||
// read-modify-write must hold metadata.lock and release it before peer fan-out.
|
||||
func (sys *BucketMetadataSys) saveMetadata(ctx context.Context, objAPI ObjectLayer, meta BucketMetadata) error {
|
||||
if err := meta.Save(ctx, objAPI); err != nil {
|
||||
return err
|
||||
}
|
||||
sys.Set(meta.Name, meta)
|
||||
return nil
|
||||
}
|
||||
|
||||
func lockBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (context.Context, func(), error) {
|
||||
return lockBucketMetadataWithTimeout(ctx, objectAPI, bucket, globalOperationTimeout)
|
||||
}
|
||||
|
||||
func lockBucketMetadataWithTimeout(ctx context.Context, objectAPI ObjectLayer, bucket string, timeout *dynamicTimeout) (context.Context, func(), error) {
|
||||
lock := objectAPI.NewNSLock(minioMetaBucket, pathJoin(bucketMetaPrefix, bucket, "metadata.lock"))
|
||||
lkctx, err := lock.GetLock(ctx, timeout)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
ctx = context.WithValue(lkctx.Context(), bucketMetadataLockContextKey{}, bucket)
|
||||
return ctx, func() { lock.Unlock(lkctx) }, nil
|
||||
}
|
||||
|
||||
type bucketMetadataLockContextKey struct{}
|
||||
|
||||
func bucketMetadataLockHeld(ctx context.Context, bucket string) bool {
|
||||
lockedBucket, _ := ctx.Value(bucketMetadataLockContextKey{}).(string)
|
||||
return lockedBucket == bucket
|
||||
}
|
||||
|
||||
// Delete delete the bucket metadata for the specified bucket.
|
||||
// must be used by all callers instead of using Update() with nil configData.
|
||||
func (sys *BucketMetadataSys) Delete(ctx context.Context, bucket string, configFile string) (updatedAt time.Time, err error) {
|
||||
if configFile == bucketLifecycleConfig {
|
||||
// Get bucket config from current site
|
||||
meta, e := globalBucketMetadataSys.GetConfigFromDisk(ctx, bucket)
|
||||
if e != nil && !errors.Is(e, errConfigNotFound) {
|
||||
return updatedAt, e
|
||||
}
|
||||
var expiryRuleRemoved bool
|
||||
if len(meta.LifecycleConfigXML) > 0 {
|
||||
var lcCfg lifecycle.Lifecycle
|
||||
if err := xml.Unmarshal(meta.LifecycleConfigXML, &lcCfg); err != nil {
|
||||
return updatedAt, err
|
||||
}
|
||||
// find a single expiry rule set the flag
|
||||
for _, rl := range lcCfg.Rules {
|
||||
if !rl.Expiration.IsNull() || !rl.NoncurrentVersionExpiration.IsNull() {
|
||||
expiryRuleRemoved = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return sys.updateAndParse(ctx, bucket, configFile, nil, false, configFile == bucketLifecycleConfig)
|
||||
}
|
||||
|
||||
// Form empty ILM details with `ExpiryUpdatedAt` field and save
|
||||
var cfgData []byte
|
||||
if expiryRuleRemoved {
|
||||
var lcCfg lifecycle.Lifecycle
|
||||
currtime := time.Now()
|
||||
lcCfg.ExpiryUpdatedAt = &currtime
|
||||
cfgData, err = xml.Marshal(lcCfg)
|
||||
if err != nil {
|
||||
return updatedAt, err
|
||||
func lifecycleDeleteConfig(current []byte) ([]byte, error) {
|
||||
var expiryRuleRemoved bool
|
||||
if len(current) > 0 {
|
||||
var lcCfg lifecycle.Lifecycle
|
||||
if err := xml.Unmarshal(current, &lcCfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, rl := range lcCfg.Rules {
|
||||
if !rl.Expiration.IsNull() || !rl.NoncurrentVersionExpiration.IsNull() {
|
||||
expiryRuleRemoved = true
|
||||
break
|
||||
}
|
||||
}
|
||||
return sys.updateAndParse(ctx, bucket, configFile, cfgData, false)
|
||||
}
|
||||
return sys.updateAndParse(ctx, bucket, configFile, nil, false)
|
||||
if !expiryRuleRemoved {
|
||||
return nil, nil
|
||||
}
|
||||
var lcCfg lifecycle.Lifecycle
|
||||
currtime := time.Now()
|
||||
lcCfg.ExpiryUpdatedAt = &currtime
|
||||
return xml.Marshal(lcCfg)
|
||||
}
|
||||
|
||||
// Update update bucket metadata for the specified bucket.
|
||||
// The configData data should not be modified after being sent here.
|
||||
func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) (updatedAt time.Time, err error) {
|
||||
return sys.updateAndParse(ctx, bucket, configFile, configData, true)
|
||||
return sys.updateAndParse(ctx, bucket, configFile, configData, true, false)
|
||||
}
|
||||
|
||||
// Get metadata for a bucket.
|
||||
@@ -359,6 +423,57 @@ func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEC
|
||||
return meta.sseConfig, meta.EncryptionConfigUpdatedAt, nil
|
||||
}
|
||||
|
||||
// GetResidentCorsConfig returns the CORS configuration of a bucket whose
|
||||
// metadata is already resident in memory. It runs before authentication for
|
||||
// every Origin-bearing request with a client-supplied path segment, so it
|
||||
// never loads or caches metadata. A non-resident name gets no CORS answer
|
||||
// (errBucketMetadataNotInitialized) while startup loading is still running,
|
||||
// and afterwards when it is a real bucket whose metadata failed to load: a
|
||||
// presigned URL is authenticated on its own, so the bucket's CORS document is
|
||||
// the only origin boundary a browser enforces for it. Any other non-resident
|
||||
// name reports errConfigNotFound and the caller applies the global CORS
|
||||
// policy exactly as releases without per-bucket CORS did.
|
||||
func (sys *BucketMetadataSys) GetResidentCorsConfig(bucket string) (*cors.Config, time.Time, error) {
|
||||
if isReservedOrInvalidBucket(bucket, true) {
|
||||
return nil, time.Time{}, errConfigNotFound
|
||||
}
|
||||
sys.RLock()
|
||||
meta, ok := sys.metadataMap[bucket]
|
||||
_, failed := sys.loadFailed[bucket]
|
||||
initialized := sys.initialized
|
||||
sys.RUnlock()
|
||||
if !ok {
|
||||
if !initialized || failed {
|
||||
return nil, time.Time{}, errBucketMetadataNotInitialized
|
||||
}
|
||||
return nil, time.Time{}, errConfigNotFound
|
||||
}
|
||||
if meta.corsConfigErr != nil {
|
||||
return nil, meta.CorsConfigUpdatedAt, meta.corsConfigErr
|
||||
}
|
||||
if meta.corsConfig == nil {
|
||||
return nil, time.Time{}, errConfigNotFound
|
||||
}
|
||||
return meta.corsConfig, meta.CorsConfigUpdatedAt, nil
|
||||
}
|
||||
|
||||
// GetCorsConfigXML returns the raw stored CORS configuration XML for the
|
||||
// given bucket, preserving the document exactly as it was PUT (including
|
||||
// the S3 xmlns and any unmodeled elements).
|
||||
func (sys *BucketMetadataSys) GetCorsConfigXML(bucket string) ([]byte, time.Time, error) {
|
||||
meta, _, err := sys.GetConfig(GlobalContext, bucket)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, err
|
||||
}
|
||||
if meta.corsConfigErr != nil {
|
||||
return nil, meta.CorsConfigUpdatedAt, meta.corsConfigErr
|
||||
}
|
||||
if len(meta.CorsConfigXML) == 0 {
|
||||
return nil, time.Time{}, errConfigNotFound
|
||||
}
|
||||
return meta.CorsConfigXML, meta.CorsConfigUpdatedAt, nil
|
||||
}
|
||||
|
||||
// CreatedAt returns the time of creation of bucket
|
||||
func (sys *BucketMetadataSys) CreatedAt(bucket string) (time.Time, error) {
|
||||
meta, _, err := sys.GetConfig(GlobalContext, bucket)
|
||||
@@ -488,6 +603,7 @@ func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (met
|
||||
}
|
||||
sys.Lock()
|
||||
sys.metadataMap[bucket] = meta
|
||||
sys.clearLoadFailure(bucket)
|
||||
sys.Unlock()
|
||||
|
||||
return meta, true, nil
|
||||
@@ -539,8 +655,10 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []stri
|
||||
sys.Lock()
|
||||
for i, meta := range bucketMetas {
|
||||
if errs[i] != nil {
|
||||
sys.noteLoadFailure(buckets[i])
|
||||
continue
|
||||
}
|
||||
sys.clearLoadFailure(buckets[i])
|
||||
sys.metadataMap[buckets[i]] = meta
|
||||
}
|
||||
sys.Unlock()
|
||||
@@ -590,6 +708,9 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
|
||||
meta, err := loadBucketMetadata(ctx, sys.objAPI, bucket)
|
||||
if err != nil {
|
||||
internalLogIf(ctx, err, logger.WarningKind)
|
||||
sys.Lock()
|
||||
sys.noteLoadFailure(bucket)
|
||||
sys.Unlock()
|
||||
wait() // wait to proceed to next entry.
|
||||
continue
|
||||
}
|
||||
@@ -600,6 +721,7 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
|
||||
updated = true
|
||||
sys.metadataMap[bucket] = meta
|
||||
}
|
||||
sys.clearLoadFailure(bucket)
|
||||
sys.Unlock()
|
||||
|
||||
if updated {
|
||||
@@ -647,6 +769,7 @@ func (sys *BucketMetadataSys) init(ctx context.Context, buckets []string) {
|
||||
func (sys *BucketMetadataSys) Reset() {
|
||||
sys.Lock()
|
||||
clear(sys.metadataMap)
|
||||
clear(sys.loadFailed)
|
||||
sys.Unlock()
|
||||
}
|
||||
|
||||
@@ -654,6 +777,7 @@ func (sys *BucketMetadataSys) Reset() {
|
||||
func NewBucketMetadataSys() *BucketMetadataSys {
|
||||
return &BucketMetadataSys{
|
||||
metadataMap: make(map[string]BucketMetadata),
|
||||
loadFailed: make(map[string]struct{}),
|
||||
group: &singleflight.Group{},
|
||||
}
|
||||
}
|
||||
|
||||
+85
-6
@@ -31,6 +31,7 @@ import (
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/bucket/cors"
|
||||
bucketsse "github.com/minio/minio/internal/bucket/encryption"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
@@ -40,8 +41,8 @@ import (
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/sio"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -58,6 +59,9 @@ var (
|
||||
enabledBucketVersioningConfig = []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status></VersioningConfiguration>`)
|
||||
)
|
||||
|
||||
// Bucket CORS configuration file.
|
||||
const bucketCorsConfig = "cors.xml"
|
||||
|
||||
//go:generate msgp -file $GOFILE
|
||||
|
||||
// BucketMetadata contains bucket metadata.
|
||||
@@ -80,6 +84,7 @@ type BucketMetadata struct {
|
||||
ReplicationConfigXML []byte
|
||||
BucketTargetsConfigJSON []byte
|
||||
BucketTargetsConfigMetaJSON []byte
|
||||
CorsConfigXML []byte
|
||||
|
||||
PolicyConfigUpdatedAt time.Time
|
||||
ObjectLockConfigUpdatedAt time.Time
|
||||
@@ -92,6 +97,7 @@ type BucketMetadata struct {
|
||||
NotificationConfigUpdatedAt time.Time
|
||||
BucketTargetsConfigUpdatedAt time.Time
|
||||
BucketTargetsConfigMetaUpdatedAt time.Time
|
||||
CorsConfigUpdatedAt time.Time
|
||||
// Add a new UpdatedAt field and update lastUpdate function
|
||||
|
||||
// Unexported fields. Must be updated atomically.
|
||||
@@ -106,6 +112,8 @@ type BucketMetadata struct {
|
||||
replicationConfig *replication.Config
|
||||
bucketTargetConfig *madmin.BucketTargets
|
||||
bucketTargetConfigMeta map[string]string
|
||||
corsConfig *cors.Config
|
||||
corsConfigErr error
|
||||
}
|
||||
|
||||
// newBucketMetadata creates BucketMetadata with the supplied name and Created to Now.
|
||||
@@ -160,6 +168,9 @@ func (b BucketMetadata) lastUpdate() (t time.Time) {
|
||||
if b.BucketTargetsConfigMetaUpdatedAt.After(t) {
|
||||
t = b.BucketTargetsConfigMetaUpdatedAt
|
||||
}
|
||||
if b.CorsConfigUpdatedAt.After(t) {
|
||||
t = b.CorsConfigUpdatedAt
|
||||
}
|
||||
|
||||
return t
|
||||
}
|
||||
@@ -238,8 +249,17 @@ func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket
|
||||
}
|
||||
|
||||
if len(configs) > 0 {
|
||||
// Old bucket without bucket metadata. Hence we migrate existing settings.
|
||||
if err = b.convertLegacyConfigs(ctx, objectAPI, configs); err != nil {
|
||||
if !bucketMetadataLockHeld(ctx, bucket) {
|
||||
migrated, lockErr := loadBucketMetadataParseUnderLock(ctx, objectAPI, bucket, parse)
|
||||
if lockErr == nil {
|
||||
return migrated, nil
|
||||
}
|
||||
if !errors.Is(lockErr, errBucketMetadataMigrationLockUnavailable) {
|
||||
return b, lockErr
|
||||
}
|
||||
internalLogOnceIf(ctx, fmt.Errorf("unable to persist bucket metadata migration for %s, using the legacy configuration in memory: %w", bucket, lockErr), "bucket-metadata-migration-lock-"+bucket)
|
||||
b.applyLegacyConfigs(configs)
|
||||
} else if err = b.convertLegacyConfigs(ctx, objectAPI, configs); err != nil {
|
||||
return b, err
|
||||
}
|
||||
}
|
||||
@@ -251,8 +271,25 @@ func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket
|
||||
return b, err
|
||||
}
|
||||
}
|
||||
if b.corsConfigErr != nil {
|
||||
// Keep the rest of the bucket metadata available so an operator can
|
||||
// replace or delete a CORS document accepted by an older, more lenient
|
||||
// build. Defer unrelated metadata migration until CORS is repaired.
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// migrate unencrypted remote targets
|
||||
if len(b.BucketTargetsConfigJSON) != 0 && GlobalKMS != nil && len(b.BucketTargetsConfigMetaJSON) == 0 && !bucketMetadataLockHeld(ctx, bucket) {
|
||||
migrated, lockErr := loadBucketMetadataParseUnderLock(ctx, objectAPI, bucket, parse)
|
||||
if lockErr == nil {
|
||||
return migrated, nil
|
||||
}
|
||||
if !errors.Is(lockErr, errBucketMetadataMigrationLockUnavailable) {
|
||||
return b, lockErr
|
||||
}
|
||||
internalLogOnceIf(ctx, fmt.Errorf("unable to persist encrypted bucket target metadata for %s, using the existing configuration in memory: %w", bucket, lockErr), "bucket-metadata-migration-lock-"+bucket)
|
||||
return b, nil
|
||||
}
|
||||
if err = b.migrateTargetConfig(ctx, objectAPI); err != nil {
|
||||
return b, err
|
||||
}
|
||||
@@ -260,6 +297,20 @@ func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func loadBucketMetadataParseUnderLock(ctx context.Context, objectAPI ObjectLayer, bucket string, parse bool) (BucketMetadata, error) {
|
||||
ctx, unlock, err := lockBucketMetadataWithTimeout(ctx, objectAPI, bucket, bucketMetadataMigrationTimeout)
|
||||
if err != nil {
|
||||
return newBucketMetadata(bucket), fmt.Errorf("%w: %v", errBucketMetadataMigrationLockUnavailable, err)
|
||||
}
|
||||
defer unlock()
|
||||
return loadBucketMetadataParse(ctx, objectAPI, bucket, parse)
|
||||
}
|
||||
|
||||
var (
|
||||
bucketMetadataMigrationTimeout = newDynamicTimeout(5*time.Second, time.Second)
|
||||
errBucketMetadataMigrationLockUnavailable = errors.New("bucket metadata migration lock unavailable")
|
||||
)
|
||||
|
||||
// loadBucketMetadata loads and migrates to bucket metadata.
|
||||
func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
|
||||
return loadBucketMetadataParse(ctx, objectAPI, bucket, true)
|
||||
@@ -310,8 +361,20 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
|
||||
b.taggingConfig = nil
|
||||
}
|
||||
|
||||
if bytes.Equal(b.ObjectLockConfigXML, enabledBucketObjectLockConfig) {
|
||||
b.VersioningConfigXML = enabledBucketVersioningConfig
|
||||
b.corsConfigErr = nil
|
||||
if len(b.CorsConfigXML) != 0 {
|
||||
cfg, corsErr := cors.ParseBucketCorsConfig(bytes.NewReader(b.CorsConfigXML))
|
||||
if corsErr == nil {
|
||||
corsErr = cfg.Validate()
|
||||
}
|
||||
if corsErr != nil {
|
||||
b.corsConfig = nil
|
||||
b.corsConfigErr = fmt.Errorf("invalid bucket CORS configuration: %w", corsErr)
|
||||
} else {
|
||||
b.corsConfig = cfg
|
||||
}
|
||||
} else {
|
||||
b.corsConfig = nil
|
||||
}
|
||||
|
||||
if len(b.ObjectLockConfigXML) != 0 {
|
||||
@@ -322,6 +385,15 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
|
||||
} else {
|
||||
b.objectLockConfig = nil
|
||||
}
|
||||
if b.objectLockConfig != nil {
|
||||
// Object Lock requires every object to be versioned. Whatever the lock
|
||||
// document contains, a suspended or prefix-excluded versioning document
|
||||
// is replaced by plain Enabled versioning; Save persists the result.
|
||||
config, versioningErr := versioning.ParseConfig(bytes.NewReader(b.VersioningConfigXML))
|
||||
if versioningErr != nil || !config.Enabled() || config.PrefixesExcluded() {
|
||||
b.VersioningConfigXML = enabledBucketVersioningConfig
|
||||
}
|
||||
}
|
||||
|
||||
if len(b.VersioningConfigXML) != 0 {
|
||||
b.versioningConfig, err = versioning.ParseConfig(bytes.NewReader(b.VersioningConfigXML))
|
||||
@@ -404,7 +476,7 @@ func (b *BucketMetadata) getAllLegacyConfigs(ctx context.Context, objectAPI Obje
|
||||
return configs, nil
|
||||
}
|
||||
|
||||
func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer, configs map[string][]byte) error {
|
||||
func (b *BucketMetadata) applyLegacyConfigs(configs map[string][]byte) {
|
||||
for legacyFile, configData := range configs {
|
||||
switch legacyFile {
|
||||
case legacyBucketObjectLockEnabledConfigFile:
|
||||
@@ -436,6 +508,10 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
|
||||
}
|
||||
}
|
||||
b.defaultTimestamps()
|
||||
}
|
||||
|
||||
func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer, configs map[string][]byte) error {
|
||||
b.applyLegacyConfigs(configs)
|
||||
|
||||
if err := b.Save(ctx, objectAPI); err != nil {
|
||||
return err
|
||||
@@ -503,6 +579,9 @@ func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
|
||||
if err := b.parseAllConfigs(ctx, api); err != nil {
|
||||
return err
|
||||
}
|
||||
if b.corsConfigErr != nil {
|
||||
return b.corsConfigErr
|
||||
}
|
||||
|
||||
data := make([]byte, 4, b.Msgsize()+4)
|
||||
|
||||
|
||||
@@ -108,6 +108,12 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
|
||||
return
|
||||
}
|
||||
case "CorsConfigXML":
|
||||
z.CorsConfigXML, err = dc.ReadBytes(z.CorsConfigXML)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CorsConfigXML")
|
||||
return
|
||||
}
|
||||
case "PolicyConfigUpdatedAt":
|
||||
z.PolicyConfigUpdatedAt, err = dc.ReadTime()
|
||||
if err != nil {
|
||||
@@ -174,6 +180,12 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaUpdatedAt")
|
||||
return
|
||||
}
|
||||
case "CorsConfigUpdatedAt":
|
||||
z.CorsConfigUpdatedAt, err = dc.ReadTime()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CorsConfigUpdatedAt")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
@@ -187,9 +199,9 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// map header, size 25
|
||||
// map header, size 27
|
||||
// write "Name"
|
||||
err = en.Append(0xde, 0x0, 0x19, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
err = en.Append(0xde, 0x0, 0x1b, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -328,6 +340,16 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
|
||||
return
|
||||
}
|
||||
// write "CorsConfigXML"
|
||||
err = en.Append(0xad, 0x43, 0x6f, 0x72, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteBytes(z.CorsConfigXML)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CorsConfigXML")
|
||||
return
|
||||
}
|
||||
// write "PolicyConfigUpdatedAt"
|
||||
err = en.Append(0xb5, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
|
||||
if err != nil {
|
||||
@@ -438,15 +460,25 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaUpdatedAt")
|
||||
return
|
||||
}
|
||||
// write "CorsConfigUpdatedAt"
|
||||
err = en.Append(0xb3, 0x43, 0x6f, 0x72, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteTime(z.CorsConfigUpdatedAt)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CorsConfigUpdatedAt")
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// map header, size 25
|
||||
// map header, size 27
|
||||
// string "Name"
|
||||
o = append(o, 0xde, 0x0, 0x19, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
o = append(o, 0xde, 0x0, 0x1b, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
o = msgp.AppendString(o, z.Name)
|
||||
// string "Created"
|
||||
o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
|
||||
@@ -487,6 +519,9 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
// string "BucketTargetsConfigMetaJSON"
|
||||
o = append(o, 0xbb, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x4a, 0x53, 0x4f, 0x4e)
|
||||
o = msgp.AppendBytes(o, z.BucketTargetsConfigMetaJSON)
|
||||
// string "CorsConfigXML"
|
||||
o = append(o, 0xad, 0x43, 0x6f, 0x72, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
|
||||
o = msgp.AppendBytes(o, z.CorsConfigXML)
|
||||
// string "PolicyConfigUpdatedAt"
|
||||
o = append(o, 0xb5, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
|
||||
o = msgp.AppendTime(o, z.PolicyConfigUpdatedAt)
|
||||
@@ -520,6 +555,9 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
// string "BucketTargetsConfigMetaUpdatedAt"
|
||||
o = append(o, 0xd9, 0x20, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
|
||||
o = msgp.AppendTime(o, z.BucketTargetsConfigMetaUpdatedAt)
|
||||
// string "CorsConfigUpdatedAt"
|
||||
o = append(o, 0xb3, 0x43, 0x6f, 0x72, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
|
||||
o = msgp.AppendTime(o, z.CorsConfigUpdatedAt)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -625,6 +663,12 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
|
||||
return
|
||||
}
|
||||
case "CorsConfigXML":
|
||||
z.CorsConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.CorsConfigXML)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CorsConfigXML")
|
||||
return
|
||||
}
|
||||
case "PolicyConfigUpdatedAt":
|
||||
z.PolicyConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
|
||||
if err != nil {
|
||||
@@ -691,6 +735,12 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaUpdatedAt")
|
||||
return
|
||||
}
|
||||
case "CorsConfigUpdatedAt":
|
||||
z.CorsConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CorsConfigUpdatedAt")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
@@ -705,6 +755,6 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *BucketMetadata) Msgsize() (s int) {
|
||||
s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize + 26 + msgp.TimeSize + 25 + msgp.TimeSize + 28 + msgp.TimeSize + 29 + msgp.TimeSize + 34 + msgp.TimeSize
|
||||
s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 14 + msgp.BytesPrefixSize + len(z.CorsConfigXML) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize + 26 + msgp.TimeSize + 25 + msgp.TimeSize + 28 + msgp.TimeSize + 29 + msgp.TimeSize + 34 + msgp.TimeSize + 20 + msgp.TimeSize
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestBucketMetadataCorsRoundTrip(t *testing.T) {
|
||||
meta := newBucketMetadata("test-cors")
|
||||
meta.CorsConfigXML = []byte(`<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`)
|
||||
meta.CorsConfigUpdatedAt = UTCNow()
|
||||
|
||||
buf, err := meta.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got BucketMetadata
|
||||
if _, err := got.UnmarshalMsg(buf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got.CorsConfigXML) != string(meta.CorsConfigXML) {
|
||||
t.Fatalf("CorsConfigXML not preserved: %q", string(got.CorsConfigXML))
|
||||
}
|
||||
if !got.CorsConfigUpdatedAt.Equal(meta.CorsConfigUpdatedAt) {
|
||||
t.Fatalf("CorsConfigUpdatedAt not preserved")
|
||||
}
|
||||
}
|
||||
@@ -26,7 +26,7 @@ import (
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
+190
-12
@@ -22,13 +22,15 @@ import (
|
||||
"errors"
|
||||
"math"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/amztime"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// BucketObjectLockSys - map of bucket and retention configuration.
|
||||
@@ -150,7 +152,11 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
|
||||
}
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
|
||||
// If you try to delete objects protected by governance mode and have s3:BypassGovernanceRetention, the operation will succeed.
|
||||
if checkRequestAuthType(ctx, r, policy.BypassGovernanceRetentionAction, bucket, object.ObjectName) != ErrNone {
|
||||
if reqInfo := logger.GetReqInfo(ctx); reqInfo != nil {
|
||||
reqInfo.BucketName = bucket
|
||||
reqInfo.ObjectName = object.ObjectName
|
||||
}
|
||||
if authorizeRequest(ctx, r, policy.BypassGovernanceRetentionAction) != ErrNone {
|
||||
return errAuthentication
|
||||
}
|
||||
}
|
||||
@@ -198,7 +204,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
|
||||
byPassSet, r, cred, owner)
|
||||
// Governance mode retention period cannot be shortened, if x-amz-bypass-governance is not set.
|
||||
if !byPassSet {
|
||||
if objRetention.Mode != objectlock.RetGovernance || objRetention.RetainUntilDate.Before((ret.RetainUntilDate.Time)) {
|
||||
if objRetention.Mode != objectlock.RetGovernance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) {
|
||||
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
||||
}
|
||||
}
|
||||
@@ -209,7 +215,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
|
||||
case objectlock.RetCompliance:
|
||||
// Compliance retention mode cannot be changed or shortened.
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
|
||||
if objRetention.Mode != objectlock.RetCompliance || objRetention.RetainUntilDate.Before((ret.RetainUntilDate.Time)) {
|
||||
if objRetention.Mode != objectlock.RetCompliance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) {
|
||||
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
||||
}
|
||||
apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
|
||||
@@ -242,7 +248,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
|
||||
// For objects in "Compliance" mode, retention date cannot be shortened, and mode cannot be altered.
|
||||
// For objects with legal hold header set, the s3:PutObjectLegalHold permission is expected to be set
|
||||
// Both legal hold and retention can be applied independently on an object
|
||||
func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, retentionPermErr, legalHoldPermErr APIErrorCode) (objectlock.RetMode, objectlock.RetentionDate, objectlock.ObjectLegalHold, APIErrorCode) {
|
||||
func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, retentionPermErr, legalHoldPermErr APIErrorCode, replicaTrusted bool) (objectlock.RetMode, objectlock.RetentionDate, objectlock.ObjectLegalHold, APIErrorCode) {
|
||||
var mode objectlock.RetMode
|
||||
var retainDate objectlock.RetentionDate
|
||||
var legalHold objectlock.ObjectLegalHold
|
||||
@@ -269,9 +275,7 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
||||
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
|
||||
}
|
||||
|
||||
replica := rq.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String()
|
||||
|
||||
if opts.VersionID != "" && !replica {
|
||||
if opts.VersionID != "" && !replicaTrusted {
|
||||
if objInfo, err := getObjectInfoFn(ctx, bucket, object, opts); err == nil {
|
||||
r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
|
||||
t, err := objectlock.UTCNowNTP()
|
||||
@@ -307,8 +311,8 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
||||
if err != nil {
|
||||
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
|
||||
}
|
||||
rMode, rDate, err := objectlock.ParseObjectLockRetentionHeaders(rq.Header)
|
||||
if err != nil && (!replica || rMode != "" || !rDate.IsZero()) {
|
||||
rMode, rDate, err := objectlock.ParseObjectLockRetentionHeaders(rq.Header, replicaTrusted)
|
||||
if err != nil && (!replicaTrusted || rMode != "" || !rDate.IsZero()) {
|
||||
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
|
||||
}
|
||||
if retentionPermErr != ErrNone {
|
||||
@@ -316,7 +320,7 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
||||
}
|
||||
return rMode, rDate, legalHold, ErrNone
|
||||
}
|
||||
if replica { // replica inherits retention metadata only from source
|
||||
if replicaTrusted { // replica inherits retention metadata only from source
|
||||
return "", objectlock.RetentionDate{}, legalHold, ErrNone
|
||||
}
|
||||
if !retentionRequested && retentionCfg.Validity > 0 {
|
||||
@@ -343,3 +347,177 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
||||
func NewBucketObjectLockSys() *BucketObjectLockSys {
|
||||
return &BucketObjectLockSys{}
|
||||
}
|
||||
|
||||
// objectLockState is the Object Lock metadata of a stored object version
|
||||
// together with the replication timestamps that order updates to it.
|
||||
type objectLockState struct {
|
||||
mode, retainUntil, retentionTimestamp string
|
||||
legalHold, legalHoldTimestamp string
|
||||
}
|
||||
|
||||
func storedObjectLockState(metadata map[string]string) objectLockState {
|
||||
return objectLockState{
|
||||
mode: metadata[strings.ToLower(xhttp.AmzObjectLockMode)],
|
||||
retainUntil: metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)],
|
||||
retentionTimestamp: metadata[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp],
|
||||
legalHold: metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)],
|
||||
legalHoldTimestamp: metadata[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp],
|
||||
}
|
||||
}
|
||||
|
||||
// olderThan reports whether a stored replication timestamp is missing,
|
||||
// unreadable, or earlier than the source timestamp, in which case the
|
||||
// replica update wins. A zero source timestamp never wins.
|
||||
func olderThan(stored string, src time.Time) bool {
|
||||
if src.IsZero() {
|
||||
return false
|
||||
}
|
||||
ondisk, err := time.Parse(time.RFC3339Nano, stored)
|
||||
return err != nil || ondisk.Before(src)
|
||||
}
|
||||
|
||||
func (s objectLockState) retentionIsOlderThan(src time.Time) bool {
|
||||
return olderThan(s.retentionTimestamp, src)
|
||||
}
|
||||
|
||||
func (s objectLockState) legalHoldIsOlderThan(src time.Time) bool {
|
||||
return olderThan(s.legalHoldTimestamp, src)
|
||||
}
|
||||
|
||||
// restoreRetention and restoreLegalHold put the stored state back into
|
||||
// metadata that was rebuilt from a request whose update was not applied.
|
||||
func (s objectLockState) restoreRetention(metadata map[string]string) {
|
||||
// The stored timestamp orders the next update and must survive even when
|
||||
// the stored value is empty, which is how a removal is recorded.
|
||||
if s.retentionTimestamp != "" {
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = s.retentionTimestamp
|
||||
}
|
||||
if s.mode == "" {
|
||||
return
|
||||
}
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = s.mode
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = s.retainUntil
|
||||
}
|
||||
|
||||
func (s objectLockState) restoreLegalHold(metadata map[string]string) {
|
||||
if s.legalHoldTimestamp != "" {
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = s.legalHoldTimestamp
|
||||
}
|
||||
if s.legalHold == "" {
|
||||
return
|
||||
}
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = s.legalHold
|
||||
}
|
||||
|
||||
// replicaStoredLock reads the Object Lock state stored on the addressed version
|
||||
// so a trusted replica write can order its update against it. A missing object
|
||||
// or version yields an empty state, which is correct for the first write of a
|
||||
// version; any other read error is returned so the caller fails the write rather
|
||||
// than ordering an incoming update against lock state it merely failed to read
|
||||
// (an older incoming value must not win over a newer stored one just because the
|
||||
// read timed out).
|
||||
func replicaStoredLock(ctx context.Context, getObjectInfo GetObjectInfoFn, bucket, object, versionID string) (objectLockState, error) {
|
||||
oi, err := getObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: versionID})
|
||||
switch {
|
||||
case err == nil:
|
||||
return storedObjectLockState(oi.UserDefined), nil
|
||||
case isErrObjectNotFound(err) || isErrVersionNotFound(err):
|
||||
return objectLockState{}, nil
|
||||
default:
|
||||
return objectLockState{}, err
|
||||
}
|
||||
}
|
||||
|
||||
// applyReplicatedObjectLock writes the retention and legal-hold decision into
|
||||
// metadata for a PUT, CopyObject, or multipart-initiation request. A request
|
||||
// that is not an actual trusted replica -- a normal user write, or a trusted
|
||||
// peer that carried the replication marker without REPLICA status -- takes
|
||||
// ordinary write semantics: a validated value is applied and stamped now, and a
|
||||
// missing value is left as is. Only an actual replica update is ordered against
|
||||
// the state already stored on the addressed version, so a stale value cannot
|
||||
// overwrite a newer one and a full retransmit cannot roll a destination back.
|
||||
// The stored argument is meaningful only for a replica; callers pass an empty
|
||||
// state otherwise. Only the two Object Lock keys and their reserved ordering
|
||||
// timestamps are touched; any encryption-metadata reconciliation stays with the
|
||||
// caller.
|
||||
func applyReplicatedObjectLock(metadata map[string]string, stored objectLockState,
|
||||
replicaTrusted bool,
|
||||
retentionMode objectlock.RetMode, retentionDate objectlock.RetentionDate,
|
||||
legalHold objectlock.ObjectLegalHold, srcRetentionTimestamp, srcLegalholdTimestamp time.Time,
|
||||
) {
|
||||
switch {
|
||||
case !replicaTrusted:
|
||||
// Ordinary write semantics: apply a validated retention and stamp it now;
|
||||
// a missing value carries no instruction, so leave the metadata as it is.
|
||||
if retentionMode.Valid() {
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC())
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
}
|
||||
case !stored.retentionIsOlderThan(srcRetentionTimestamp):
|
||||
// The stored update is at least as new as this replica's, or the replica
|
||||
// carries no ordering timestamp: keep what is stored. This is also how a
|
||||
// stale retransmit is rejected.
|
||||
stored.restoreRetention(metadata)
|
||||
default:
|
||||
// The replica update wins. A removal carries no value but still records
|
||||
// the source timestamp that orders it.
|
||||
if retentionMode.Valid() {
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC())
|
||||
}
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = srcRetentionTimestamp.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
// Legal hold has no removal in S3: an explicitly empty header is already
|
||||
// rejected as an invalid status, so the only value-less shape that gets here
|
||||
// is an absent one, which conveys no legal-hold change. Only a valid status
|
||||
// can win.
|
||||
switch {
|
||||
case !replicaTrusted:
|
||||
if legalHold.Status.Valid() {
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
}
|
||||
case legalHold.Status.Valid() && stored.legalHoldIsOlderThan(srcLegalholdTimestamp):
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = srcLegalholdTimestamp.UTC().Format(time.RFC3339Nano)
|
||||
default:
|
||||
stored.restoreLegalHold(metadata)
|
||||
}
|
||||
}
|
||||
|
||||
// reconcileStoredObjectLock re-orders the Object Lock already written into
|
||||
// metadata against the state currently stored on the destination version, both
|
||||
// compared by their reserved ordering timestamps. It runs inside the object
|
||||
// layer under the namespace write lock that guards the version replacement,
|
||||
// after the destination version is read and before the new one is committed, so
|
||||
// a replica update whose ordering was decided at handler time (or, for multipart,
|
||||
// at initiation) cannot overwrite a newer lock update that reached the version in
|
||||
// between. metadata already carries the incoming update with its source
|
||||
// timestamps; a stored value that is not older than the incoming one is put back,
|
||||
// which for a stored removal means clearing the incoming value and keeping only
|
||||
// the removal's timestamp. Only the two lock keys and their reserved timestamps
|
||||
// move; a non-replica write never sets the flag that invokes this.
|
||||
func reconcileStoredObjectLock(metadata map[string]string, stored objectLockState) {
|
||||
incoming := storedObjectLockState(metadata)
|
||||
|
||||
incomingRetentionTS, _ := time.Parse(time.RFC3339Nano, incoming.retentionTimestamp)
|
||||
if !stored.retentionIsOlderThan(incomingRetentionTS) {
|
||||
// The stored retention is at least as new as the incoming one (or the
|
||||
// incoming update is unordered): drop the incoming value and put the stored
|
||||
// state back, which may itself be a removal (value keys absent, timestamp
|
||||
// present).
|
||||
delete(metadata, strings.ToLower(xhttp.AmzObjectLockMode))
|
||||
delete(metadata, strings.ToLower(xhttp.AmzObjectLockRetainUntilDate))
|
||||
delete(metadata, ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp)
|
||||
stored.restoreRetention(metadata)
|
||||
}
|
||||
|
||||
incomingLegalHoldTS, _ := time.Parse(time.RFC3339Nano, incoming.legalHoldTimestamp)
|
||||
if incoming.legalHold == "" || !stored.legalHoldIsOlderThan(incomingLegalHoldTS) {
|
||||
delete(metadata, strings.ToLower(xhttp.AmzObjectLockLegalHold))
|
||||
delete(metadata, ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp)
|
||||
stored.restoreLegalHold(metadata)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -29,8 +29,8 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/policy/condition"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy/condition"
|
||||
)
|
||||
|
||||
func getAnonReadOnlyBucketPolicy(bucketName string) *policy.BucketPolicy {
|
||||
|
||||
@@ -33,8 +33,8 @@ import (
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/policy/condition"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy/condition"
|
||||
)
|
||||
|
||||
// PolicySys - policy subsystem.
|
||||
|
||||
@@ -29,8 +29,8 @@ import (
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/policy/condition"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy/condition"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
+12
-8
@@ -43,6 +43,17 @@ func NewBucketQuotaSys() *BucketQuotaSys {
|
||||
return &BucketQuotaSys{}
|
||||
}
|
||||
|
||||
// getBucketQuotaSize returns the effective enforced hard-quota size.
|
||||
func getBucketQuotaSize(quota *madmin.BucketQuota) uint64 {
|
||||
if quota == nil || quota.Type != madmin.HardQuota {
|
||||
return 0
|
||||
}
|
||||
if quota.Size > 0 {
|
||||
return quota.Size
|
||||
}
|
||||
return quota.Quota
|
||||
}
|
||||
|
||||
var bucketStorageCache = cachevalue.New[DataUsageInfo]()
|
||||
|
||||
// Init initialize bucket quota.
|
||||
@@ -110,14 +121,7 @@ func (sys *BucketQuotaSys) enforceQuotaHard(ctx context.Context, bucket string,
|
||||
return err
|
||||
}
|
||||
|
||||
var quotaSize uint64
|
||||
if q != nil && q.Type == madmin.HardQuota {
|
||||
if q.Size > 0 {
|
||||
quotaSize = q.Size
|
||||
} else if q.Quota > 0 {
|
||||
quotaSize = q.Quota
|
||||
}
|
||||
}
|
||||
quotaSize := getBucketQuotaSize(q)
|
||||
if quotaSize > 0 {
|
||||
if uint64(size) >= quotaSize { // check if file size already exceeds the quota
|
||||
return BucketQuotaExceeded{Bucket: bucket}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
// Copyright (c) 2015-2025 MinIO, Inc.
|
||||
// Copyright (c) 2025-2026 PGSTY
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
)
|
||||
|
||||
func TestGetBucketQuotaSize(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
quota *madmin.BucketQuota
|
||||
want uint64
|
||||
}{
|
||||
{name: "nil"},
|
||||
{name: "empty", quota: &madmin.BucketQuota{}},
|
||||
{name: "current size", quota: &madmin.BucketQuota{Type: madmin.HardQuota, Size: 1024}, want: 1024},
|
||||
{name: "legacy quota", quota: &madmin.BucketQuota{Type: madmin.HardQuota, Quota: 2048}, want: 2048},
|
||||
{name: "size takes precedence", quota: &madmin.BucketQuota{Type: madmin.HardQuota, Size: 1024, Quota: 2048}, want: 1024},
|
||||
{name: "missing type", quota: &madmin.BucketQuota{Size: 1024}},
|
||||
{name: "unsupported type", quota: &madmin.BucketQuota{Type: "fifo", Size: 1024}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := getBucketQuotaSize(tt.quota); got != tt.want {
|
||||
t.Fatalf("getBucketQuotaSize() = %d, want %d", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsBktQuotaCfgReplicated(t *testing.T) {
|
||||
hardQuota := func(size, legacy uint64) *madmin.BucketQuota {
|
||||
return &madmin.BucketQuota{Type: madmin.HardQuota, Size: size, Quota: legacy}
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
quotas []*madmin.BucketQuota
|
||||
want bool
|
||||
}{
|
||||
{name: "none configured", quotas: []*madmin.BucketQuota{nil, nil}, want: true},
|
||||
{name: "missing from one site", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), nil}},
|
||||
{name: "matching size", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), hardQuota(1024, 0)}, want: true},
|
||||
{name: "different size", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), hardQuota(2048, 0)}},
|
||||
{name: "equivalent representations", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), hardQuota(0, 1024)}, want: true},
|
||||
{name: "different typeless size", quotas: []*madmin.BucketQuota{{Size: 1024}, {Size: 2048}}},
|
||||
{name: "different type", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), {Type: "fifo", Size: 1024}}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := isBktQuotaCfgReplicated(len(tt.quotas), tt.quotas); got != tt.want {
|
||||
t.Fatalf("isBktQuotaCfgReplicated() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -34,7 +34,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// PutBucketReplicationConfigHandler - PUT Bucket replication configuration.
|
||||
@@ -617,7 +617,7 @@ func (api objectAPIHandlers) ValidateBucketReplicationCredsHandler(w http.Respon
|
||||
ReplicationValidityCheck: true, // set this to validate the replication config
|
||||
},
|
||||
}
|
||||
obj := path.Join(minioReservedBucket, globalLocalNodeNameHex, "deleteme")
|
||||
obj := replicationValidationObject(rule)
|
||||
ui, err := c.PutObject(ctx, clnt.Bucket, obj, reader, int64(len(buf)), "", "", putOpts)
|
||||
if err != nil && !isReplicationPermissionCheck(ErrorRespToObjectError(err, bucket, obj)) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, fmt.Errorf("s3:ReplicateObject permissions missing for replication user: %w", err)), r.URL)
|
||||
@@ -658,3 +658,7 @@ func (api objectAPIHandlers) ValidateBucketReplicationCredsHandler(w http.Respon
|
||||
// Write success response.
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
func replicationValidationObject(rule replication.Rule) string {
|
||||
return path.Join(rule.Prefix(), minioReservedBucket, globalLocalNodeNameHex, "deleteme")
|
||||
}
|
||||
|
||||
+341
-65
@@ -418,7 +418,12 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
|
||||
// target cluster, the object version is marked deleted on the source and hidden from listing. It is permanently
|
||||
// deleted from the source when the VersionPurgeStatus changes to "Complete", i.e after replication succeeds
|
||||
// on target.
|
||||
func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer) {
|
||||
// replicateDelete replicates a delete (delete marker or version purge) to all
|
||||
// applicable targets and returns the per-target replication outcome. Callers
|
||||
// that only trigger replication may ignore the return value; the resync path
|
||||
// uses it to classify success/failure per target rather than inferring it from
|
||||
// the mere presence or absence of the target version.
|
||||
func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer) replicatedInfos {
|
||||
var replicationStatus replication.StatusType
|
||||
bucket := dobj.Bucket
|
||||
versionID := dobj.DeleteMarkerVersionID
|
||||
@@ -453,7 +458,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
||||
Host: globalLocalNodeName,
|
||||
EventName: event.ObjectReplicationNotTracked,
|
||||
})
|
||||
return
|
||||
return replicatedInfos{}
|
||||
}
|
||||
dsc, err := parseReplicateDecision(ctx, bucket, dobj.ReplicationState.ReplicateDecisionStr)
|
||||
if err != nil {
|
||||
@@ -471,7 +476,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
||||
Host: globalLocalNodeName,
|
||||
EventName: event.ObjectReplicationNotTracked,
|
||||
})
|
||||
return
|
||||
return replicatedInfos{}
|
||||
}
|
||||
|
||||
// Lock the object name before starting replication operation.
|
||||
@@ -492,7 +497,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
||||
Host: globalLocalNodeName,
|
||||
EventName: event.ObjectReplicationNotTracked,
|
||||
})
|
||||
return
|
||||
return replicatedInfos{}
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
@@ -597,6 +602,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
||||
EventName: eventName,
|
||||
})
|
||||
}
|
||||
return rinfos
|
||||
}
|
||||
|
||||
func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationInfo, tgt *TargetClient) (rinfo replicatedTargetInfo) {
|
||||
@@ -779,6 +785,15 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (put
|
||||
meta := make(map[string]string)
|
||||
isSSEC := crypto.SSEC.IsEncrypted(objInfo.UserDefined)
|
||||
|
||||
// An SSE-C object is replicated as raw ciphertext, and the replication
|
||||
// headers carry no compression state. Sending a compressed SSE-C object
|
||||
// would land a replica that decrypts to an S2 stream instead of the
|
||||
// object, so fail loudly instead of writing a wrong replica.
|
||||
if isSSEC && objInfo.IsCompressed() {
|
||||
return putOpts, false, fmt.Errorf("replication of a compressed SSE-C object is not supported: %s/%s(%s)",
|
||||
objInfo.Bucket, objInfo.Name, objInfo.VersionID)
|
||||
}
|
||||
|
||||
for k, v := range objInfo.UserDefined {
|
||||
_, isValidSSEHeader := validSSEReplicationHeaders[k]
|
||||
// In case of SSE-C objects copy the allowed internal headers as well
|
||||
@@ -857,19 +872,29 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (put
|
||||
if cc, ok := lkMap.Lookup(xhttp.CacheControl); ok {
|
||||
putOpts.CacheControl = cc
|
||||
}
|
||||
if mode, ok := lkMap.Lookup(xhttp.AmzObjectLockMode); ok {
|
||||
rmode := minio.RetentionMode(mode)
|
||||
putOpts.Mode = rmode
|
||||
mode, hasMode := lkMap.Lookup(xhttp.AmzObjectLockMode)
|
||||
retainDateStr, hasRetainDate := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate)
|
||||
if hasMode {
|
||||
putOpts.Mode = minio.RetentionMode(mode)
|
||||
}
|
||||
if retainDateStr, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok {
|
||||
// A removed retention is stored as an empty or absent mode and date; it is
|
||||
// sent as a value-less update that still carries its ordering timestamp.
|
||||
if hasRetainDate && retainDateStr != "" {
|
||||
rdate, err := amztime.ISO8601Parse(retainDateStr)
|
||||
if err != nil {
|
||||
return putOpts, false, err
|
||||
}
|
||||
putOpts.RetainUntilDate = rdate
|
||||
// set retention timestamp in opts
|
||||
}
|
||||
retainTmstampStr, hasRetainTmstamp := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp]
|
||||
if hasMode || hasRetainDate || hasRetainTmstamp {
|
||||
// Send the ordering timestamp whenever the version carries one, even for a
|
||||
// removal whose value keys are absent (the shape a retransmit PUT leaves),
|
||||
// so the next hop can order the removal instead of keeping obsolete
|
||||
// retention.
|
||||
retTimestamp := objInfo.ModTime
|
||||
if retainTmstampStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp]; ok {
|
||||
if hasRetainTmstamp {
|
||||
var err error
|
||||
retTimestamp, err = time.Parse(time.RFC3339Nano, retainTmstampStr)
|
||||
if err != nil {
|
||||
return putOpts, false, err
|
||||
@@ -931,11 +956,17 @@ func equals(k1 string, keys ...string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// nullVersionExcludedFromResync reports the exclusion at the head of getReplicationAction, kept
|
||||
// verbatim from upstream: an existing object resync leaves a null version alone when the source
|
||||
// modification time is later than the one the target reports, without comparing anything else.
|
||||
func nullVersionExcludedFromResync(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replication.Type) bool {
|
||||
return opType == replication.ExistingObjectReplicationType &&
|
||||
oi1.ModTime.Unix() > oi2.LastModified.Unix() && oi1.VersionID == nullVersionID
|
||||
}
|
||||
|
||||
// returns replicationAction by comparing metadata between source and target
|
||||
func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replication.Type) replicationAction {
|
||||
// Avoid resyncing null versions created prior to enabling replication if target has a newer copy
|
||||
if opType == replication.ExistingObjectReplicationType &&
|
||||
oi1.ModTime.Unix() > oi2.LastModified.Unix() && oi1.VersionID == nullVersionID {
|
||||
if nullVersionExcludedFromResync(oi1, oi2, opType) {
|
||||
return replicateNone
|
||||
}
|
||||
sz, _ := oi1.GetActualSize()
|
||||
@@ -986,9 +1017,21 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
|
||||
"X-Amz-Meta-",
|
||||
}
|
||||
|
||||
// An empty object lock mode or retain-until-date records a removed retention, but
|
||||
// it is omitted from GET/HEAD response headers: setObjectHeaders() skips both keys
|
||||
// when the value is empty, and FilterObjectLockMetadata() drops them when the mode
|
||||
// is not valid. The target can therefore never report them, so treat empty and
|
||||
// absent as equal rather than as a permanent difference.
|
||||
emptyLockValue := func(k, v string) bool {
|
||||
return v == "" && equals(k, xhttp.AmzObjectLockMode, xhttp.AmzObjectLockRetainUntilDate)
|
||||
}
|
||||
|
||||
// compare metadata on both maps to see if meta is identical
|
||||
compareMeta1 := make(map[string]string)
|
||||
for k, v := range oi1.UserDefined {
|
||||
if emptyLockValue(k, v) {
|
||||
continue
|
||||
}
|
||||
var found bool
|
||||
for _, prefix := range compareKeys {
|
||||
if !stringsHasPrefixFold(k, prefix) {
|
||||
@@ -1004,6 +1047,10 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
|
||||
|
||||
compareMeta2 := make(map[string]string)
|
||||
for k, v := range oi2.Metadata {
|
||||
val := strings.Join(v, ",")
|
||||
if emptyLockValue(k, val) {
|
||||
continue
|
||||
}
|
||||
var found bool
|
||||
for _, prefix := range compareKeys {
|
||||
if !stringsHasPrefixFold(k, prefix) {
|
||||
@@ -1013,7 +1060,7 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
|
||||
break
|
||||
}
|
||||
if found {
|
||||
compareMeta2[strings.ToLower(k)] = strings.Join(v, ",")
|
||||
compareMeta2[strings.ToLower(k)] = val
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1024,9 +1071,87 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
|
||||
return replicateNone
|
||||
}
|
||||
|
||||
// objectRetentionGetter is the part of the replication target client used to confirm whether a
|
||||
// destination version still holds Object Lock retention.
|
||||
type objectRetentionGetter interface {
|
||||
GetObjectRetention(ctx context.Context, bucketName, objectName, versionID string) (*minio.RetentionMode, *time.Time, error)
|
||||
}
|
||||
|
||||
// retentionRemovedAtSource reports whether oi carries the shape a removed retention leaves behind.
|
||||
// Two representations persist. A retention removed directly on this cluster keeps the object lock
|
||||
// keys present with empty values (PutObjectRetentionHandler, cmd/object-handlers.go:3309-3316). A
|
||||
// removal that arrived by replication keeps only the retention ordering timestamp, with the mode
|
||||
// and retain-until-date keys absent, because restoreRetention and the replica update path write
|
||||
// the timestamp alone when the mode is empty (cmd/bucket-object-lock.go:388-399,
|
||||
// cmd/object-handlers.go:1782-1797). A present ordering timestamp paired with a non-empty mode is
|
||||
// a retention that was set, not removed, and must not be mistaken for one.
|
||||
func retentionRemovedAtSource(oi ObjectInfo) bool {
|
||||
lkMap := caseInsensitiveMap(oi.UserDefined)
|
||||
// Representation (1): an object lock key is present with an empty value.
|
||||
for _, k := range []string{xhttp.AmzObjectLockMode, xhttp.AmzObjectLockRetainUntilDate} {
|
||||
if v, ok := lkMap.Lookup(k); ok && v == "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// Representation (2): a recorded retention ordering timestamp with the mode value absent or
|
||||
// empty is a removal restoreRetention persisted without the empty public keys.
|
||||
if _, ok := oi.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp]; ok {
|
||||
if v, ok := lkMap.Lookup(xhttp.AmzObjectLockMode); !ok || v == "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// targetRetentionConfirmedAbsent reports whether the destination version is known to hold no
|
||||
// retention. A HEAD response omits retention both when the version has none and when the
|
||||
// replication credential lacks s3:GetObjectRetention (cmd/object-handlers.go:942-946), so the
|
||||
// comparison in getReplicationAction on its own cannot tell a removal that is already in sync from
|
||||
// one the destination still holds. Only NoSuchObjectLockConfiguration, the answer for a version
|
||||
// that carries no retention, and a response naming no retention mode count as absent. Everything
|
||||
// else is uncertainty and is treated as still present, so that the removal is resent exactly as it
|
||||
// is today: a denied or unreachable destination, a mode the SDK returned without recognizing since
|
||||
// it does not validate it, and InvalidRequest, which names a bucket with no Object Lock
|
||||
// configuration but is also what a destination answers when its own read of that configuration
|
||||
// fails (cmd/bucket-object-lock.go:39-50 returns an error with a zero Retention, discarded at
|
||||
// cmd/object-handlers.go:3275).
|
||||
func targetRetentionConfirmedAbsent(ctx context.Context, tgt objectRetentionGetter, bucket, object, versionID string) bool {
|
||||
mode, _, err := tgt.GetObjectRetention(ctx, bucket, object, versionID)
|
||||
if err != nil {
|
||||
return minio.ToErrorResponse(err).Code == "NoSuchObjectLockConfiguration"
|
||||
}
|
||||
// An absent or empty mode is no retention. A non-empty mode is retention, whether or not this
|
||||
// SDK recognizes it.
|
||||
return mode == nil || *mode == ""
|
||||
}
|
||||
|
||||
// replicationActionForTarget returns the action for a source version against a destination that
|
||||
// answered HEAD. It is getReplicationAction plus the confirmation that a removed retention which
|
||||
// compares as in sync really is: see targetRetentionConfirmedAbsent.
|
||||
func replicationActionForTarget(ctx context.Context, oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replication.Type, tgt objectRetentionGetter, bucket, object string) replicationAction {
|
||||
rAction := getReplicationAction(oi1, oi2, opType)
|
||||
if rAction != replicateNone || !retentionRemovedAtSource(oi1) {
|
||||
return rAction
|
||||
}
|
||||
// A null version the resync deliberately leaves alone is not a comparison result, so it is
|
||||
// not the confirmation's to reopen.
|
||||
if nullVersionExcludedFromResync(oi1, oi2, opType) {
|
||||
return rAction
|
||||
}
|
||||
if targetRetentionConfirmedAbsent(ctx, tgt, bucket, object, oi1.VersionID) {
|
||||
return rAction
|
||||
}
|
||||
return replicateMetadata
|
||||
}
|
||||
|
||||
// replicateObject replicates the specified version of the object to destination bucket
|
||||
// The source object is then updated to reflect the replication status.
|
||||
func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI ObjectLayer) {
|
||||
// replicateObject replicates a single object version to all applicable targets
|
||||
// and returns the per-target replication outcome. Callers that only trigger
|
||||
// replication may ignore the return value; the resync path uses it to classify
|
||||
// success/failure per target rather than inferring it from the mere existence
|
||||
// of the target version.
|
||||
func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI ObjectLayer) replicatedInfos {
|
||||
var replicationStatus replication.StatusType
|
||||
defer func() {
|
||||
if replicationStatus.Empty() {
|
||||
@@ -1059,7 +1184,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
|
||||
UserAgent: "Internal: [Replication]",
|
||||
Host: globalLocalNodeName,
|
||||
})
|
||||
return
|
||||
return replicatedInfos{}
|
||||
}
|
||||
tgtArns := cfg.FilterTargetArns(replication.ObjectOpts{
|
||||
Name: object,
|
||||
@@ -1079,7 +1204,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
|
||||
Host: globalLocalNodeName,
|
||||
})
|
||||
globalReplicationPool.Get().queueMRFSave(ri.ToMRFEntry())
|
||||
return
|
||||
return replicatedInfos{}
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
@@ -1185,6 +1310,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
|
||||
ri.RetryCount++
|
||||
globalReplicationPool.Get().queueMRFSave(ri.ToMRFEntry())
|
||||
}
|
||||
return rinfos
|
||||
}
|
||||
|
||||
// replicateObject replicates object data for specified version of the object to destination bucket
|
||||
@@ -1299,6 +1425,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
|
||||
|
||||
putOpts, isMP, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
|
||||
if err != nil {
|
||||
rinfo.Err = err
|
||||
rinfo.ReplicationStatus = replication.Failed
|
||||
replLogIf(ctx, fmt.Errorf("failure setting options for replication bucket:%s err:%w", bucket, err))
|
||||
sendEvent(eventArgs{
|
||||
EventName: event.ObjectReplicationNotTracked,
|
||||
@@ -1467,7 +1595,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
|
||||
sOpts.Set(xhttp.AmzTagDirective, "ACCESS")
|
||||
oi, cerr := tgt.StatObject(ctx, tgt.Bucket, object, sOpts)
|
||||
if cerr == nil {
|
||||
rAction = getReplicationAction(objInfo, oi, ri.OpType)
|
||||
rAction = replicationActionForTarget(ctx, objInfo, oi, ri.OpType, tgt, tgt.Bucket, object)
|
||||
rinfo.ReplicationStatus = replication.Completed
|
||||
if rAction == replicateNone {
|
||||
if ri.OpType == replication.ExistingObjectReplicationType &&
|
||||
@@ -1497,11 +1625,14 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
|
||||
return rinfo
|
||||
}
|
||||
} else {
|
||||
// SSEC objects will refuse HeadObject without the decryption key.
|
||||
// Ignore the error, since we know the object exists and versioning prevents overwriting existing versions.
|
||||
// The sender holds no customer key, so the target refuses HeadObject on
|
||||
// an SSE-C object and the replica cannot be compared. The metadata-only
|
||||
// CopyObject that a replicateMetadata action would run then fails on any
|
||||
// non-empty object, because the undecryptable source checksum makes the
|
||||
// target recompute one and rewrite the data. A full retransmit is the
|
||||
// only action that completes.
|
||||
if isSSEC && strings.Contains(cerr.Error(), errorCodes[ErrSSEEncryptedObject].Description) {
|
||||
rinfo.ReplicationStatus = replication.Completed
|
||||
rinfo.ReplicationAction = replicateNone
|
||||
rAction = replicateAll
|
||||
goto applyAction
|
||||
}
|
||||
// if target returns error other than NoSuchKey, defer replication attempt
|
||||
@@ -1586,6 +1717,11 @@ applyAction:
|
||||
} else {
|
||||
putOpts, isMP, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
|
||||
if err != nil {
|
||||
// rinfo was primed Completed above; a failure to build the write
|
||||
// options means nothing reached the target, so mark it Failed and
|
||||
// carry the error instead of reporting a phantom success.
|
||||
rinfo.ReplicationStatus = replication.Failed
|
||||
rinfo.Err = err
|
||||
replLogIf(ctx, fmt.Errorf("failed to set replicate options for object %s/%s(%s) (target %s) err:%w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), err))
|
||||
sendEvent(eventArgs{
|
||||
EventName: event.ObjectReplicationNotTracked,
|
||||
@@ -2873,6 +3009,150 @@ func (s *replicationResyncer) incStats(ts TargetReplicationResyncStatus, opts re
|
||||
s.statusMap[opts.bucket] = m
|
||||
}
|
||||
|
||||
// resyncResults consumes the per-object outcomes produced by the resync worker
|
||||
// pool and applies each to the in-memory resync status via apply. It centralizes
|
||||
// the finalization ordering so a status persisted after finish() returns always
|
||||
// reflects every result.
|
||||
type resyncResults struct {
|
||||
ch chan TargetReplicationResyncStatus
|
||||
apply func(TargetReplicationResyncStatus)
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// newResyncResults starts the result-consuming goroutine that folds each worker
|
||||
// result into the bucket's resync status.
|
||||
func (s *replicationResyncer) newResyncResults(opts resyncOpts) *resyncResults {
|
||||
return startResyncResults(func(r TargetReplicationResyncStatus) {
|
||||
s.incStats(r, opts)
|
||||
globalSiteResyncMetrics.updateMetric(r, opts.resyncID)
|
||||
})
|
||||
}
|
||||
|
||||
// startResyncResults starts a goroutine that applies every received result with
|
||||
// apply. Injecting the apply action keeps the shutdown ordering in finish()
|
||||
// testable.
|
||||
func startResyncResults(apply func(TargetReplicationResyncStatus)) *resyncResults {
|
||||
rr := &resyncResults{
|
||||
ch: make(chan TargetReplicationResyncStatus, 1),
|
||||
apply: apply,
|
||||
}
|
||||
rr.wg.Add(1)
|
||||
go func() {
|
||||
defer rr.wg.Done()
|
||||
for r := range rr.ch {
|
||||
rr.apply(r)
|
||||
}
|
||||
}()
|
||||
return rr
|
||||
}
|
||||
|
||||
// finish shuts the resync pipeline down in an order that guarantees a status
|
||||
// persisted afterwards reflects every result. It first closes the worker input
|
||||
// channels and waits for the producer workers to exit, so none can send on a
|
||||
// closed result channel (a hazard on early-return paths) and every submitted
|
||||
// result is delivered (a result a worker discards on cancellation is
|
||||
// intentionally not); only then does it close the result channel and wait for
|
||||
// the consumer to apply the last buffered result.
|
||||
func (rr *resyncResults) finish(workers []chan ReplicateObjectInfo, workerWg *sync.WaitGroup) {
|
||||
for i := range workers {
|
||||
xioutil.SafeClose(workers[i])
|
||||
}
|
||||
workerWg.Wait()
|
||||
xioutil.SafeClose(rr.ch)
|
||||
rr.wg.Wait()
|
||||
}
|
||||
|
||||
// sendResyncResult delivers a worker's computed per-object result to ch,
|
||||
// returning false if the worker must stop first. On the resync-cancel signal it
|
||||
// records the abort - the already-computed result is dropped - so
|
||||
// finalResyncStatus can downgrade a Completed run; on ctx cancellation it stops
|
||||
// without recording, since finalResyncStatus's parent-context check covers that.
|
||||
func (s *replicationResyncer) sendResyncResult(ctx context.Context, ch chan<- TargetReplicationResyncStatus, st TargetReplicationResyncStatus, workerAborted *atomic.Bool) bool {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-s.resyncCancelCh:
|
||||
workerAborted.Store(true)
|
||||
return false
|
||||
case ch <- st:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// finalResyncStatus downgrades a Completed status to Failed when the run could
|
||||
// not have observed every object: the parent context was canceled (workers then
|
||||
// return without sending their computed result) or a worker dropped a result on
|
||||
// the resync-cancel signal. Without this a persisted Completed would misrepresent
|
||||
// an incomplete resync.
|
||||
func finalResyncStatus(status ResyncStatusType, ctxErr error, workerAborted bool) ResyncStatusType {
|
||||
if status == ResyncCompleted && (ctxErr != nil || workerAborted) {
|
||||
return ResyncFailed
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
// resyncTargetSucceeded reports whether this object (or delete) actually
|
||||
// replicated to the target, from the target's own outcome. A version purge
|
||||
// reports success through VersionPurgeStatus, not ReplicationStatus. For an
|
||||
// object or delete marker, success requires a Completed status; a retained
|
||||
// error is a real failure unless it is the benign duplicate 412 the
|
||||
// destination returns when it already holds this exact ETag and version, which
|
||||
// replicateAll deliberately keeps Completed.
|
||||
func resyncTargetSucceeded(t replicatedTargetInfo, roi ReplicateObjectInfo) bool {
|
||||
if !roi.VersionPurgeStatus.Empty() {
|
||||
return t.VersionPurgeStatus == replication.VersionPurgeComplete
|
||||
}
|
||||
if t.ReplicationStatus != replication.Completed {
|
||||
return false
|
||||
}
|
||||
return t.Err == nil || minio.ToErrorResponse(t.Err).Code == "PreconditionFailed"
|
||||
}
|
||||
|
||||
// resyncResultFor derives the resync outcome for target arn from the aggregate
|
||||
// replication result of a single object (or delete). The target counts as a
|
||||
// success only when its own replication Completed without error - not when the
|
||||
// target version merely exists. A target that Failed, errored, or was not
|
||||
// attempted for this object (its arn absent from the result) counts as a
|
||||
// failure, and the failed byte count is recorded (previously always zero).
|
||||
func resyncResultFor(rinfos replicatedInfos, arn string, roi ReplicateObjectInfo) TargetReplicationResyncStatus {
|
||||
st := TargetReplicationResyncStatus{Object: roi.Name, Bucket: roi.Bucket}
|
||||
for _, t := range rinfos.Targets {
|
||||
if t.Arn != arn {
|
||||
continue
|
||||
}
|
||||
if resyncTargetSucceeded(t, roi) {
|
||||
sz := t.Size
|
||||
if sz == 0 {
|
||||
sz = roi.Size
|
||||
}
|
||||
st.ReplicatedCount++
|
||||
st.ReplicatedSize += sz
|
||||
} else {
|
||||
st.FailedCount++
|
||||
st.FailedSize += roi.Size
|
||||
}
|
||||
return st
|
||||
}
|
||||
// arn was not attempted for this object: a resync that cannot confirm the
|
||||
// object reached the target is not a success.
|
||||
st.FailedCount++
|
||||
st.FailedSize += roi.Size
|
||||
return st
|
||||
}
|
||||
|
||||
// objectNeedsResyncForARN reports whether roi must be resynced for target arn
|
||||
// specifically. The resync worker pool is scoped to a single target (opts.arn),
|
||||
// so an object that only qualifies for a different target must be skipped here:
|
||||
// admitting it would replicate it for arn's peers only, leaving arn absent from
|
||||
// the per-object result, which resyncResultFor then (correctly, but
|
||||
// misleadingly) counts as a failure for arn - an object arn was never
|
||||
// responsible for. Only opts.arn carries this resync's ResetID, and that reset
|
||||
// is already folded into its per-target decision, so the per-target check both
|
||||
// scopes dispatch and honors the reset.
|
||||
func objectNeedsResyncForARN(roi ReplicateObjectInfo, arn string) bool {
|
||||
return roi.ExistingObjResync.mustResyncTarget(arn)
|
||||
}
|
||||
|
||||
// resyncBucket resyncs all qualifying objects as per replication rules for the target
|
||||
// ARN
|
||||
func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI ObjectLayer, heal bool, opts resyncOpts) {
|
||||
@@ -2883,7 +3163,18 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
}
|
||||
|
||||
resyncStatus := ResyncFailed
|
||||
// workerAborted records that a worker dropped an already-computed result on
|
||||
// the resync-cancel signal. With a canceled parent context (which makes
|
||||
// workers return without sending their result), it means a Completed run did
|
||||
// not actually observe every object - see finalResyncStatus below.
|
||||
var workerAborted atomic.Bool
|
||||
defer func() {
|
||||
// Downgrade a Completed status whose counts are incomplete, so the
|
||||
// persisted status is not a misleading Completed. Runs after results.finish
|
||||
// drains (LIFO) and before markStatus persists - markStatus uses its own
|
||||
// background context, so a parent cancellation during the drain would
|
||||
// otherwise still record Completed.
|
||||
resyncStatus = finalResyncStatus(resyncStatus, ctx.Err(), workerAborted.Load())
|
||||
s.markStatus(resyncStatus, opts, objectAPI)
|
||||
globalSiteResyncMetrics.incBucket(opts, resyncStatus)
|
||||
s.workerCh <- struct{}{}
|
||||
@@ -2939,16 +3230,14 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
lastCheckpoint = st.Object
|
||||
}
|
||||
workers := make([]chan ReplicateObjectInfo, resyncParallelRoutines)
|
||||
resultCh := make(chan TargetReplicationResyncStatus, 1)
|
||||
defer xioutil.SafeClose(resultCh)
|
||||
go func() {
|
||||
for r := range resultCh {
|
||||
s.incStats(r, opts)
|
||||
globalSiteResyncMetrics.updateMetric(r, opts.resyncID)
|
||||
}
|
||||
}()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
// results consumes each worker's per-object outcome and folds it into the
|
||||
// in-memory status. finish() (deferred below) stops the workers and drains
|
||||
// every result before the deferred markStatus persists, so a Completed status
|
||||
// cannot race the last incStats. Registered after the markStatus finalizer, so
|
||||
// LIFO runs finish first.
|
||||
results := s.newResyncResults(opts)
|
||||
defer results.finish(workers, &wg)
|
||||
for i := range resyncParallelRoutines {
|
||||
wg.Add(1)
|
||||
workers[i] = make(chan ReplicateObjectInfo, 100)
|
||||
@@ -2963,6 +3252,7 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
default:
|
||||
}
|
||||
traceFn := s.trace(tgt.ResetID, fmt.Sprintf("%s/%s (%s)", opts.bucket, roi.Name, roi.VersionID))
|
||||
var rinfos replicatedInfos
|
||||
if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
|
||||
versionID := ""
|
||||
dmVersionID := ""
|
||||
@@ -2985,43 +3275,28 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
OpType: replication.ExistingObjectReplicationType,
|
||||
EventType: ReplicateExistingDelete,
|
||||
}
|
||||
replicateDelete(ctx, doi, objectAPI)
|
||||
rinfos = replicateDelete(ctx, doi, objectAPI)
|
||||
} else {
|
||||
roi.OpType = replication.ExistingObjectReplicationType
|
||||
roi.EventType = ReplicateExisting
|
||||
replicateObject(ctx, roi, objectAPI)
|
||||
rinfos = replicateObject(ctx, roi, objectAPI)
|
||||
}
|
||||
|
||||
st := TargetReplicationResyncStatus{
|
||||
Object: roi.Name,
|
||||
Bucket: roi.Bucket,
|
||||
}
|
||||
|
||||
_, err := tgt.StatObject(ctx, tgt.Bucket, roi.Name, minio.StatObjectOptions{
|
||||
VersionID: roi.VersionID,
|
||||
Internal: minio.AdvancedGetOptions{
|
||||
ReplicationProxyRequest: "false",
|
||||
},
|
||||
})
|
||||
sz := roi.Size
|
||||
if err != nil {
|
||||
if roi.DeleteMarker && isErrMethodNotAllowed(ErrorRespToObjectError(err, opts.bucket, roi.Name)) {
|
||||
st.ReplicatedCount++
|
||||
} else {
|
||||
st.FailedCount++
|
||||
// Classify success/failure from the actual replication outcome
|
||||
// for this target, not from whether the target version merely
|
||||
// exists (a rejected update leaves the old version in place).
|
||||
st := resyncResultFor(rinfos, opts.arn, roi)
|
||||
var traceSize int64
|
||||
var traceErr error
|
||||
for i := range rinfos.Targets {
|
||||
if rinfos.Targets[i].Arn == opts.arn {
|
||||
traceSize, traceErr = rinfos.Targets[i].Size, rinfos.Targets[i].Err
|
||||
break
|
||||
}
|
||||
sz = 0
|
||||
} else {
|
||||
st.ReplicatedCount++
|
||||
st.ReplicatedSize += roi.Size
|
||||
}
|
||||
traceFn(sz, err)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
traceFn(traceSize, traceErr)
|
||||
if !s.sendResyncResult(ctx, results.ch, st, &workerAborted) {
|
||||
return
|
||||
case <-s.resyncCancelCh:
|
||||
return
|
||||
case resultCh <- st:
|
||||
}
|
||||
}
|
||||
}(ctx, i)
|
||||
@@ -3045,7 +3320,12 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
}
|
||||
lastCheckpoint = ""
|
||||
roi := getHealReplicateObjectInfo(res.Item, rcfg)
|
||||
if !roi.ExistingObjResync.mustResync() {
|
||||
// Scope dispatch to this resync's target: the worker pool is for
|
||||
// opts.arn, so skip objects that only need resync for a different
|
||||
// target (each target has its own resync). Without this, a cross-target
|
||||
// object leaves opts.arn absent from its per-object result and is
|
||||
// miscounted as an opts.arn failure.
|
||||
if !objectNeedsResyncForARN(roi, opts.arn) {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
@@ -3058,10 +3338,6 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
workers[h%uint64(resyncParallelRoutines)] <- roi
|
||||
}
|
||||
}
|
||||
for i := range resyncParallelRoutines {
|
||||
xioutil.SafeClose(workers[i])
|
||||
}
|
||||
wg.Wait()
|
||||
resyncStatus = ResyncCompleted
|
||||
}
|
||||
|
||||
|
||||
@@ -18,12 +18,22 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
@@ -287,3 +297,808 @@ func TestReplicationResyncwrapper(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplicationValidationObjectUsesRulePrefix(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
rule replication.Rule
|
||||
want string
|
||||
}{
|
||||
{name: "empty prefix", rule: replication.Rule{}, want: path.Join(minioReservedBucket, globalLocalNodeNameHex, "deleteme")},
|
||||
{name: "filter prefix", rule: replication.Rule{Filter: replication.Filter{Prefix: "data/"}}, want: path.Join("data", minioReservedBucket, globalLocalNodeNameHex, "deleteme")},
|
||||
{name: "and prefix", rule: replication.Rule{Filter: replication.Filter{And: replication.And{Prefix: "archive/"}}}, want: path.Join("archive", minioReservedBucket, globalLocalNodeNameHex, "deleteme")},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := replicationValidationObject(test.rule); got != test.want {
|
||||
t.Fatalf("replicationValidationObject() = %q, want %q", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The resync-finalization tests below exercise the real result sink, the
|
||||
// finish() shutdown ordering, and the sendResyncResult / finalResyncStatus
|
||||
// helpers, plus (for the persistence cases) markStatus with on-disk
|
||||
// round-tripping. resyncBucket cannot be driven end to end in a unit test
|
||||
// because its workers call a live remote target (StatObject), so the helpers it
|
||||
// uses are exercised directly. The blocking-order assertions run under
|
||||
// testing/synctest so a removed wait fails deterministically, with no timing
|
||||
// windows.
|
||||
|
||||
func newTestResyncer(bucket, arn string) (*replicationResyncer, resyncOpts) {
|
||||
s := &replicationResyncer{
|
||||
statusMap: map[string]BucketReplicationResyncStatus{},
|
||||
resyncCancelCh: make(chan struct{}, resyncWorkerCnt),
|
||||
}
|
||||
brs := newBucketResyncStatus(bucket)
|
||||
brs.TargetsMap[arn] = TargetReplicationResyncStatus{ResyncStatus: ResyncStarted}
|
||||
s.statusMap[bucket] = brs
|
||||
return s, resyncOpts{bucket: bucket, arn: arn, resyncID: "reset-" + bucket}
|
||||
}
|
||||
|
||||
// TestResyncBucketFinalize round-trips the terminal status through a real
|
||||
// ObjectLayer: a clean run persists Completed with every result, while a run
|
||||
// whose parent context was canceled during the drain, or in which a worker
|
||||
// dropped a result on the cancel signal, is downgraded to Failed so a persisted
|
||||
// Completed never misrepresents an incomplete resync.
|
||||
func TestResyncBucketFinalize(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
objAPI, fsDirs, err := prepareErasure16(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare erasure backend: %v", err)
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
// persistTerminal applies resyncBucket's finalizer logic (finalResyncStatus
|
||||
// then markStatus, which persists) and reads the status back the way the
|
||||
// resync status API does.
|
||||
persistTerminal := func(t *testing.T, s *replicationResyncer, opts resyncOpts, status ResyncStatusType, ctxErr error, aborted bool) TargetReplicationResyncStatus {
|
||||
t.Helper()
|
||||
s.markStatus(finalResyncStatus(status, ctxErr, aborted), opts, objAPI)
|
||||
brs, err := loadBucketResyncMetadata(ctx, opts.bucket, objAPI)
|
||||
if err != nil {
|
||||
t.Fatalf("load persisted resync metadata: %v", err)
|
||||
}
|
||||
return brs.TargetsMap[opts.arn]
|
||||
}
|
||||
|
||||
// 1. Clean completion: every result - including the failed object - is folded
|
||||
// into the persisted status, which stays Completed.
|
||||
t.Run("persists complete counts", func(t *testing.T) {
|
||||
s, opts := newTestResyncer("finalize-counts", "arn1")
|
||||
results := s.newResyncResults(opts)
|
||||
results.ch <- TargetReplicationResyncStatus{Object: "ok-1", ReplicatedCount: 1, ReplicatedSize: 100}
|
||||
results.ch <- TargetReplicationResyncStatus{Object: "ok-2", ReplicatedCount: 1, ReplicatedSize: 200}
|
||||
results.ch <- TargetReplicationResyncStatus{Object: "bad", FailedCount: 1, FailedSize: 300}
|
||||
|
||||
var wg sync.WaitGroup // no producer workers for this case
|
||||
results.finish(nil, &wg)
|
||||
|
||||
st := persistTerminal(t, s, opts, ResyncCompleted, nil, false)
|
||||
if st.ResyncStatus != ResyncCompleted {
|
||||
t.Fatalf("persisted status = %s, want Completed", st.ResyncStatus)
|
||||
}
|
||||
if st.ReplicatedCount != 2 || st.ReplicatedSize != 300 || st.FailedCount != 1 || st.FailedSize != 300 {
|
||||
t.Fatalf("persisted counts = {replicated:%d/%d failed:%d/%d}, want {2/300 1/300}",
|
||||
st.ReplicatedCount, st.ReplicatedSize, st.FailedCount, st.FailedSize)
|
||||
}
|
||||
})
|
||||
|
||||
// 2. Parent context canceled during the drain -> Completed downgraded to
|
||||
// Failed (markStatus persists under its own context, so nothing else stops
|
||||
// a bare Completed from being recorded).
|
||||
t.Run("parent cancel during drain downgrades to failed", func(t *testing.T) {
|
||||
s, opts := newTestResyncer("finalize-parent-cancel", "arn1")
|
||||
results := s.newResyncResults(opts)
|
||||
results.ch <- TargetReplicationResyncStatus{Object: "ok-1", ReplicatedCount: 1, ReplicatedSize: 100}
|
||||
var wg sync.WaitGroup
|
||||
results.finish(nil, &wg)
|
||||
|
||||
cctx, ccancel := context.WithCancel(context.Background())
|
||||
ccancel()
|
||||
st := persistTerminal(t, s, opts, ResyncCompleted, cctx.Err(), false)
|
||||
if st.ResyncStatus != ResyncFailed {
|
||||
t.Fatalf("persisted status = %s, want Failed (parent canceled during drain)", st.ResyncStatus)
|
||||
}
|
||||
})
|
||||
|
||||
// 3. A worker dropped a computed result on the resync-cancel token (parent
|
||||
// still alive) -> sendResyncResult records the abort and Completed is
|
||||
// downgraded to Failed.
|
||||
t.Run("worker abort downgrades to failed", func(t *testing.T) {
|
||||
s, opts := newTestResyncer("finalize-worker-abort", "arn1")
|
||||
s.resyncCancelCh <- struct{}{} // cancel token waiting
|
||||
ch := make(chan TargetReplicationResyncStatus) // no reader: the send would block
|
||||
var aborted atomic.Bool
|
||||
if s.sendResyncResult(context.Background(), ch, TargetReplicationResyncStatus{Object: "dropped", ReplicatedCount: 1}, &aborted) {
|
||||
t.Fatal("sendResyncResult reported success despite the cancel token")
|
||||
}
|
||||
if !aborted.Load() {
|
||||
t.Fatal("worker abort was not recorded")
|
||||
}
|
||||
st := persistTerminal(t, s, opts, ResyncCompleted, nil, aborted.Load())
|
||||
if st.ResyncStatus != ResyncFailed {
|
||||
t.Fatalf("persisted status = %s, want Failed (worker dropped a result)", st.ResyncStatus)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestResyncFinishDrainsResults asserts finish() does not return until the
|
||||
// consumer has applied the final result (the #136 defect). A gated apply holds
|
||||
// the last result unapplied; under synctest finish() must stay durably blocked
|
||||
// until it is released - if rr.wg.Wait() is removed, finish() returns early and
|
||||
// the test fails deterministically.
|
||||
func TestResyncFinishDrainsResults(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
s, opts := newTestResyncer("drain", "arn1")
|
||||
reachedFinal := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
results := startResyncResults(func(r TargetReplicationResyncStatus) {
|
||||
if r.Object == "final" {
|
||||
close(reachedFinal)
|
||||
<-release
|
||||
}
|
||||
s.incStats(r, opts)
|
||||
})
|
||||
results.ch <- TargetReplicationResyncStatus{Object: "ok-1", ReplicatedCount: 1, ReplicatedSize: 100}
|
||||
results.ch <- TargetReplicationResyncStatus{Object: "final", FailedCount: 1, FailedSize: 200}
|
||||
<-reachedFinal // consumer received "final" but is gated before incStats(final)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
finishDone := make(chan struct{})
|
||||
go func() {
|
||||
results.finish(nil, &wg)
|
||||
close(finishDone)
|
||||
}()
|
||||
|
||||
synctest.Wait()
|
||||
select {
|
||||
case <-finishDone:
|
||||
close(release)
|
||||
synctest.Wait()
|
||||
t.Fatal("finish() returned before the final result was drained (drain wait missing)")
|
||||
default:
|
||||
// finish() is durably blocked in rr.wg.Wait() - correct.
|
||||
}
|
||||
|
||||
close(release)
|
||||
synctest.Wait()
|
||||
<-finishDone
|
||||
st := s.statusMap[opts.bucket].TargetsMap[opts.arn]
|
||||
if st.ReplicatedCount != 1 || st.FailedCount != 1 || st.FailedSize != 200 {
|
||||
t.Fatalf("status after finish = {replicated:%d failed:%d/%d}, want {1 1/200}",
|
||||
st.ReplicatedCount, st.FailedCount, st.FailedSize)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestResyncFinishWaitsForInflightWorker asserts finish() stops the producer
|
||||
// workers before it closes the result channel, so an in-flight worker (as on an
|
||||
// early-return path) never sends on a closed channel and its result is not lost.
|
||||
// A gated worker stays in flight past the shutdown request; under synctest
|
||||
// finish() must stay durably blocked until the worker is released - if
|
||||
// workerWg.Wait() is removed, finish() returns early and the test fails
|
||||
// deterministically.
|
||||
func TestResyncFinishWaitsForInflightWorker(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
s, opts := newTestResyncer("workers", "arn1")
|
||||
results := startResyncResults(func(r TargetReplicationResyncStatus) { s.incStats(r, opts) })
|
||||
|
||||
workers := []chan ReplicateObjectInfo{make(chan ReplicateObjectInfo, 1)}
|
||||
var wg sync.WaitGroup
|
||||
gotRoi := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for roi := range workers[0] {
|
||||
close(gotRoi)
|
||||
<-release
|
||||
// Mirror the real worker's send; recover so that if finish()
|
||||
// wrongly closed the result channel first, the test fails via the
|
||||
// assertion below instead of crashing on send-on-closed.
|
||||
func() {
|
||||
defer func() { _ = recover() }()
|
||||
results.ch <- TargetReplicationResyncStatus{Object: roi.Name, ReplicatedCount: 1, ReplicatedSize: 500}
|
||||
}()
|
||||
}
|
||||
}()
|
||||
workers[0] <- ReplicateObjectInfo{Name: "inflight"}
|
||||
<-gotRoi // worker holds a result in flight, not yet delivered
|
||||
|
||||
finishDone := make(chan struct{})
|
||||
go func() {
|
||||
results.finish(workers, &wg)
|
||||
close(finishDone)
|
||||
}()
|
||||
|
||||
synctest.Wait()
|
||||
select {
|
||||
case <-finishDone:
|
||||
close(release)
|
||||
synctest.Wait()
|
||||
t.Fatal("finish() closed the result channel before the in-flight worker finished (worker wait missing)")
|
||||
default:
|
||||
// finish() is durably blocked in workerWg.Wait() - correct.
|
||||
}
|
||||
|
||||
close(release)
|
||||
synctest.Wait()
|
||||
<-finishDone
|
||||
st := s.statusMap[opts.bucket].TargetsMap[opts.arn]
|
||||
if st.ReplicatedCount != 1 || st.ReplicatedSize != 500 {
|
||||
t.Fatalf("status after finish = {replicated:%d/%d}, want {1/500}", st.ReplicatedCount, st.ReplicatedSize)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestResyncResultFor asserts the resync worker classifies a target from the
|
||||
// actual replication outcome, not from whether the target version merely exists.
|
||||
// The key regression is the "failed update over an existing version" case: a
|
||||
// quota-rejected update leaves the old version in place, and counting existence
|
||||
// (the previous behavior) would score it a success. It also checks a genuine
|
||||
// success, an errored-but-Completed result, a delete failure, a delete-marker
|
||||
// success (zero bytes), and an ARN that was never attempted.
|
||||
func TestResyncResultFor(t *testing.T) {
|
||||
const arn = "arn:minio:replication::id:bucket"
|
||||
obj := ReplicateObjectInfo{Name: "obj", Bucket: "bucket", Size: 196608}
|
||||
deleteMarker := ReplicateObjectInfo{Name: "dm", Bucket: "bucket", Size: 0, DeleteMarker: true}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
roi ReplicateObjectInfo
|
||||
rinfos replicatedInfos
|
||||
wantRepl, wantReplSize, wantFail, wantFailSize int64
|
||||
}{
|
||||
{
|
||||
name: "completed update",
|
||||
roi: obj,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, ReplicationStatus: replication.Completed, Size: 196608},
|
||||
}},
|
||||
wantRepl: 1, wantReplSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "failed update over existing version",
|
||||
roi: obj,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, ReplicationStatus: replication.Failed, Err: fmt.Errorf("quota exceeded"), Size: 196608},
|
||||
}},
|
||||
wantFail: 1, wantFailSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "completed but errored is a failure",
|
||||
roi: obj,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, ReplicationStatus: replication.Completed, Err: fmt.Errorf("boom"), Size: 196608},
|
||||
}},
|
||||
wantFail: 1, wantFailSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "delete failed",
|
||||
roi: deleteMarker,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, ReplicationStatus: replication.Failed},
|
||||
}},
|
||||
wantFail: 1, wantFailSize: 0,
|
||||
},
|
||||
{
|
||||
name: "delete marker replicated counts zero bytes",
|
||||
roi: deleteMarker,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, ReplicationStatus: replication.Completed},
|
||||
}},
|
||||
wantRepl: 1, wantReplSize: 0,
|
||||
},
|
||||
{
|
||||
name: "arn not attempted is a failure",
|
||||
roi: obj,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: "arn:minio:replication::id2:bucket", ReplicationStatus: replication.Completed, Size: 196608},
|
||||
}},
|
||||
wantFail: 1, wantFailSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "completed with zero size falls back to object size",
|
||||
roi: obj,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, ReplicationStatus: replication.Completed, Size: 0},
|
||||
}},
|
||||
wantRepl: 1, wantReplSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "version purge complete is a success",
|
||||
roi: ReplicateObjectInfo{Name: "purge", Bucket: "bucket", Size: 196608, VersionPurgeStatus: replication.VersionPurgePending},
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
// a successful purge sets only VersionPurgeStatus; ReplicationStatus stays empty.
|
||||
{Arn: arn, VersionPurgeStatus: replication.VersionPurgeComplete},
|
||||
}},
|
||||
wantRepl: 1, wantReplSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "version purge failed is a failure",
|
||||
roi: ReplicateObjectInfo{Name: "purge", Bucket: "bucket", Size: 196608, VersionPurgeStatus: replication.VersionPurgePending},
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, VersionPurgeStatus: replication.VersionPurgeFailed, Err: fmt.Errorf("quota exceeded")},
|
||||
}},
|
||||
wantFail: 1, wantFailSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "benign duplicate 412 is a success",
|
||||
roi: obj,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
// the destination answers PreconditionFailed for an exact duplicate;
|
||||
// replicateAll keeps Completed but retains the error.
|
||||
{Arn: arn, ReplicationStatus: replication.Completed, Err: minio.ErrorResponse{Code: "PreconditionFailed"}, Size: 196608},
|
||||
}},
|
||||
wantRepl: 1, wantReplSize: 196608,
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
st := resyncResultFor(tc.rinfos, arn, tc.roi)
|
||||
if st.Object != tc.roi.Name || st.Bucket != tc.roi.Bucket {
|
||||
t.Fatalf("object/bucket = %s/%s, want %s/%s", st.Object, st.Bucket, tc.roi.Name, tc.roi.Bucket)
|
||||
}
|
||||
if st.ReplicatedCount != tc.wantRepl || st.ReplicatedSize != tc.wantReplSize ||
|
||||
st.FailedCount != tc.wantFail || st.FailedSize != tc.wantFailSize {
|
||||
t.Fatalf("resyncResultFor = {replicated:%d/%d failed:%d/%d}, want {%d/%d %d/%d}",
|
||||
st.ReplicatedCount, st.ReplicatedSize, st.FailedCount, st.FailedSize,
|
||||
tc.wantRepl, tc.wantReplSize, tc.wantFail, tc.wantFailSize)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestObjectNeedsResyncForARN asserts the resync dispatch is scoped to the
|
||||
// target being resynced. The worker pool runs for a single target (opts.arn),
|
||||
// so an object that only qualifies for a different target must be skipped: with
|
||||
// A/B rules and a resync of A, an object that needs replication only for B must
|
||||
// not be admitted to A's worker. Otherwise (after outcome-based classification)
|
||||
// A would be absent from that object's result and miscounted as an A failure.
|
||||
func TestObjectNeedsResyncForARN(t *testing.T) {
|
||||
const (
|
||||
arnA = "arn:minio:replication::id:bucket"
|
||||
arnB = "arn:minio:replication::id2:bucket"
|
||||
)
|
||||
tests := []struct {
|
||||
name string
|
||||
decision ResyncDecision
|
||||
arn string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "target must resync",
|
||||
decision: ResyncDecision{targets: map[string]ResyncTargetDecision{arnA: {Replicate: true}}},
|
||||
arn: arnA,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "object qualifies for B only, resyncing A",
|
||||
decision: ResyncDecision{targets: map[string]ResyncTargetDecision{arnB: {Replicate: true}}},
|
||||
arn: arnA,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "A present but not replicating, B replicating, resyncing A",
|
||||
decision: ResyncDecision{targets: map[string]ResyncTargetDecision{
|
||||
arnA: {Replicate: false},
|
||||
arnB: {Replicate: true},
|
||||
}},
|
||||
arn: arnA,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "object qualifies for both, resyncing A",
|
||||
decision: ResyncDecision{targets: map[string]ResyncTargetDecision{
|
||||
arnA: {Replicate: true},
|
||||
arnB: {Replicate: true},
|
||||
}},
|
||||
arn: arnA,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "no resync decision",
|
||||
decision: ResyncDecision{},
|
||||
arn: arnA,
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
roi := ReplicateObjectInfo{Name: "obj", Bucket: "bucket", ExistingObjResync: tc.decision}
|
||||
if got := objectNeedsResyncForARN(roi, tc.arn); got != tc.want {
|
||||
t.Fatalf("objectNeedsResyncForARN(arn=%s) = %v, want %v", tc.arn, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// newMatchingReplicationPair returns a source/target pair that getReplicationAction must
|
||||
// classify as replicateNone: same ETag, version id, size, modification time and content
|
||||
// type. Any action other than replicateNone is therefore attributable to the object lock
|
||||
// entries a caller adds on top.
|
||||
func newMatchingReplicationPair() (ObjectInfo, minio.ObjectInfo) {
|
||||
mtime := time.Date(2026, 9, 5, 10, 0, 0, 0, time.UTC)
|
||||
size := int64(7)
|
||||
src := ObjectInfo{
|
||||
Bucket: "bucket",
|
||||
Name: "object",
|
||||
ETag: "d41d8cd98f00b204e9800998ecf8427e",
|
||||
VersionID: "b0ff1d6e-0000-4000-8000-000000000001",
|
||||
Size: size,
|
||||
ActualSize: &size,
|
||||
ModTime: mtime,
|
||||
ContentType: "application/octet-stream",
|
||||
UserDefined: map[string]string{"content-type": "application/octet-stream"},
|
||||
}
|
||||
tgt := minio.ObjectInfo{
|
||||
ETag: src.ETag,
|
||||
VersionID: src.VersionID,
|
||||
Size: size,
|
||||
LastModified: mtime,
|
||||
ContentType: src.ContentType,
|
||||
Metadata: http.Header{},
|
||||
}
|
||||
return src, tgt
|
||||
}
|
||||
|
||||
// TestGetReplicationActionEmptyObjectLockValues covers the comparison of object lock entries
|
||||
// whose value is empty. Removing retention from a version stores the mode and retain-until-date
|
||||
// keys with empty values, while the target's HEAD response omits them entirely, so the two must
|
||||
// compare equal or the version can never be reported as in sync. Cases 3 and 4 are synthetic
|
||||
// comparison inputs, since a SILO target cannot return empty lock headers; cases 7 and 8 guard
|
||||
// against over-normalizing.
|
||||
func TestGetReplicationActionEmptyObjectLockValues(t *testing.T) {
|
||||
var (
|
||||
modeKey = strings.ToLower(xhttp.AmzObjectLockMode)
|
||||
dateKey = strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
|
||||
until = "2026-10-05T10:00:00.000Z"
|
||||
)
|
||||
emptyRetention := map[string]string{modeKey: "", dateKey: ""}
|
||||
realRetention := map[string]string{modeKey: "GOVERNANCE", dateKey: until}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
srcMeta map[string]string
|
||||
tgtHdr map[string]string
|
||||
want replicationAction
|
||||
}{
|
||||
{"1-both-clean-never-had-retention", nil, nil, replicateNone},
|
||||
{"2-source-present-empty-target-absent", emptyRetention, nil, replicateNone},
|
||||
{"3-source-absent-target-present-empty", nil, emptyRetention, replicateNone},
|
||||
{"4-both-present-empty", emptyRetention, emptyRetention, replicateNone},
|
||||
{"5-both-governance-equal", realRetention, realRetention, replicateNone},
|
||||
{"6-source-governance-target-absent", realRetention, nil, replicateMetadata},
|
||||
{"7-source-empty-target-real-retention", emptyRetention, realRetention, replicateMetadata},
|
||||
{"8-empty-user-metadata-is-not-normalized", map[string]string{"x-amz-meta-foo": ""}, nil, replicateMetadata},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
src, tgt := newMatchingReplicationPair()
|
||||
for k, v := range test.srcMeta {
|
||||
src.UserDefined[k] = v
|
||||
}
|
||||
for k, v := range test.tgtHdr {
|
||||
tgt.Metadata.Set(k, v)
|
||||
}
|
||||
if got := getReplicationAction(src, tgt, replication.HealReplicationType); got != test.want {
|
||||
t.Fatalf("getReplicationAction() = %q, want %q (source %v, target %v)", got, test.want, src.UserDefined, tgt.Metadata)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestEmptyRetentionValuesAreOmittedFromObjectResponseHeaders records why the target half of the
|
||||
// comparison in getReplicationAction can never report an empty object lock entry:
|
||||
// FilterObjectLockMetadata drops both keys because an empty mode is not a valid retention mode,
|
||||
// and setObjectHeaders skips them when writing response headers. Neither filter reaches the
|
||||
// replication wire: the empty entries are still carried by getCopyObjMetadata and sent by the
|
||||
// metadata CopyObject, which is why the sender's comparison is what has to tolerate them.
|
||||
// FilterObjectLockMetadata is also applied by CopyObject (cmd/object-handlers.go:1708), where it
|
||||
// strips the source's lock metadata before the destination re-derives it from the request.
|
||||
func TestEmptyRetentionValuesAreOmittedFromObjectResponseHeaders(t *testing.T) {
|
||||
modeKey := strings.ToLower(xhttp.AmzObjectLockMode)
|
||||
dateKey := strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
|
||||
meta := map[string]string{
|
||||
modeKey: "",
|
||||
dateKey: "",
|
||||
"content-type": "application/octet-stream",
|
||||
}
|
||||
|
||||
filtered := objectlock.FilterObjectLockMetadata(meta, false, false)
|
||||
if _, ok := filtered[modeKey]; ok {
|
||||
t.Errorf("FilterObjectLockMetadata() kept the empty lock mode key: %v", filtered)
|
||||
}
|
||||
if _, ok := filtered[dateKey]; ok {
|
||||
t.Errorf("FilterObjectLockMetadata() kept the empty retain-until-date key: %v", filtered)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
if err := setObjectHeaders(t.Context(), rec, ObjectInfo{UserDefined: meta, ModTime: time.Now(), Size: 7}, nil, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("setObjectHeaders() = %v", err)
|
||||
}
|
||||
if v, ok := rec.Header()[http.CanonicalHeaderKey(xhttp.AmzObjectLockMode)]; ok {
|
||||
t.Errorf("setObjectHeaders() emitted an empty lock mode header: %v", v)
|
||||
}
|
||||
if v, ok := rec.Header()[http.CanonicalHeaderKey(xhttp.AmzObjectLockRetainUntilDate)]; ok {
|
||||
t.Errorf("setObjectHeaders() emitted an empty retain-until-date header: %v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeRetentionGetter answers GetObjectRetention with a fixed result and counts its calls.
|
||||
type fakeRetentionGetter struct {
|
||||
mode *minio.RetentionMode
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeRetentionGetter) GetObjectRetention(_ context.Context, _, _, _ string) (*minio.RetentionMode, *time.Time, error) {
|
||||
f.calls++
|
||||
return f.mode, nil, f.err
|
||||
}
|
||||
|
||||
func TestRetentionRemovedAtSource(t *testing.T) {
|
||||
modeKey := strings.ToLower(xhttp.AmzObjectLockMode)
|
||||
dateKey := strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
|
||||
tsKey := ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp
|
||||
stamp := "2026-09-06T01:00:00Z"
|
||||
tests := []struct {
|
||||
name string
|
||||
meta map[string]string
|
||||
want bool
|
||||
}{
|
||||
{"no lock keys", map[string]string{"content-type": "text/plain"}, false},
|
||||
{"empty pair", map[string]string{modeKey: "", dateKey: ""}, true},
|
||||
{"empty mode only", map[string]string{modeKey: ""}, true},
|
||||
{"empty date only", map[string]string{dateKey: ""}, true},
|
||||
{"real retention", map[string]string{modeKey: "GOVERNANCE", dateKey: "2026-10-05T10:00:00.000Z"}, false},
|
||||
{"canonical case", map[string]string{xhttp.AmzObjectLockMode: ""}, true},
|
||||
{"empty user metadata", map[string]string{"x-amz-meta-foo": ""}, false},
|
||||
// Representation (2): a replicated removal persists the ordering timestamp alone,
|
||||
// with the mode and retain-until-date keys absent (restoreRetention).
|
||||
{"timestamp only, mode absent", map[string]string{tsKey: stamp}, true},
|
||||
{"timestamp with empty mode", map[string]string{tsKey: stamp, modeKey: ""}, true},
|
||||
{"timestamp with real retention is a set, not a removal", map[string]string{tsKey: stamp, modeKey: "GOVERNANCE", dateKey: "2026-10-05T10:00:00.000Z"}, false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := retentionRemovedAtSource(ObjectInfo{UserDefined: test.meta}); got != test.want {
|
||||
t.Fatalf("retentionRemovedAtSource() = %v, want %v", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestTargetRetentionConfirmedAbsent pins the rule that only an explicit answer from the
|
||||
// destination clears a removed retention. A denied or unreachable destination must read as still
|
||||
// holding retention, because HEAD hides a real retention from a credential without
|
||||
// s3:GetObjectRetention exactly as it hides one that does not exist.
|
||||
func TestTargetRetentionConfirmedAbsent(t *testing.T) {
|
||||
governance := minio.Governance
|
||||
var emptyMode minio.RetentionMode
|
||||
unknownMode := minio.RetentionMode("ARCHIVE")
|
||||
tests := []struct {
|
||||
name string
|
||||
mode *minio.RetentionMode
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{"version holds governance retention", &governance, nil, false},
|
||||
{"no retention on the version", nil, minio.ErrorResponse{Code: "NoSuchObjectLockConfiguration"}, true},
|
||||
{
|
||||
// The destination also answers this when its own read of the bucket's Object Lock
|
||||
// configuration fails, so it does not establish that Object Lock is disabled.
|
||||
"invalid request naming a missing object lock configuration",
|
||||
nil,
|
||||
minio.ErrorResponse{Code: "InvalidRequest", Message: "Bucket is missing ObjectLockConfiguration"},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"unrelated invalid request",
|
||||
nil,
|
||||
minio.ErrorResponse{Code: "InvalidRequest", Message: "Object is WORM protected and cannot be overwritten"},
|
||||
false,
|
||||
},
|
||||
{"retention read denied", nil, minio.ErrorResponse{Code: "AccessDenied"}, false},
|
||||
{"destination unreachable", nil, errors.New("dial tcp: connection refused"), false},
|
||||
{"empty mode returned", &emptyMode, nil, true},
|
||||
{"unknown non-empty mode returned", &unknownMode, nil, false},
|
||||
{"nil mode returned", nil, nil, true},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
tgt := &fakeRetentionGetter{mode: test.mode, err: test.err}
|
||||
if got := targetRetentionConfirmedAbsent(t.Context(), tgt, "bucket", "object", "v1"); got != test.want {
|
||||
t.Fatalf("targetRetentionConfirmedAbsent() = %v, want %v", got, test.want)
|
||||
}
|
||||
if tgt.calls != 1 {
|
||||
t.Fatalf("GetObjectRetention called %d times, want 1", tgt.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestReplicationActionForTargetRetentionRemoval covers the decision the replication worker makes
|
||||
// for a version whose retention was removed. The destination's HEAD never reports the empty keys,
|
||||
// so the comparison alone reads every one of these as in sync; only the confirmation separates a
|
||||
// destination that really dropped the retention from one that is hiding it.
|
||||
func TestReplicationActionForTargetRetentionRemoval(t *testing.T) {
|
||||
modeKey := strings.ToLower(xhttp.AmzObjectLockMode)
|
||||
dateKey := strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
|
||||
governance := minio.Governance
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
srcMeta map[string]string
|
||||
mode *minio.RetentionMode
|
||||
err error
|
||||
want replicationAction
|
||||
wantCalls int
|
||||
}{
|
||||
{
|
||||
name: "removal confirmed by destination",
|
||||
srcMeta: map[string]string{modeKey: "", dateKey: ""},
|
||||
err: minio.ErrorResponse{Code: "NoSuchObjectLockConfiguration"},
|
||||
want: replicateNone,
|
||||
wantCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "destination still holds the retention hidden from HEAD",
|
||||
srcMeta: map[string]string{modeKey: "", dateKey: ""},
|
||||
mode: &governance,
|
||||
want: replicateMetadata,
|
||||
wantCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "retention hidden from HEAD by permissions",
|
||||
srcMeta: map[string]string{modeKey: "", dateKey: ""},
|
||||
err: minio.ErrorResponse{Code: "AccessDenied"},
|
||||
want: replicateMetadata,
|
||||
wantCalls: 1,
|
||||
},
|
||||
{
|
||||
// A destination that names a missing Object Lock configuration answers the same way
|
||||
// when its own read of that configuration failed, so it confirms nothing.
|
||||
name: "destination reports no object lock configuration",
|
||||
srcMeta: map[string]string{modeKey: "", dateKey: ""},
|
||||
err: minio.ErrorResponse{Code: "InvalidRequest", Message: "Bucket is missing ObjectLockConfiguration"},
|
||||
want: replicateMetadata,
|
||||
wantCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "version never had retention is not confirmed",
|
||||
srcMeta: nil,
|
||||
want: replicateNone,
|
||||
wantCalls: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
src, tgtInfo := newMatchingReplicationPair()
|
||||
for k, v := range test.srcMeta {
|
||||
src.UserDefined[k] = v
|
||||
}
|
||||
tgt := &fakeRetentionGetter{mode: test.mode, err: test.err}
|
||||
got := replicationActionForTarget(t.Context(), src, tgtInfo, replication.HealReplicationType, tgt, "bucket", "object")
|
||||
if got != test.want {
|
||||
t.Fatalf("replicationActionForTarget() = %q, want %q", got, test.want)
|
||||
}
|
||||
if tgt.calls != test.wantCalls {
|
||||
t.Fatalf("GetObjectRetention called %d times, want %d", tgt.calls, test.wantCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestReplicationActionForTargetNullVersionResync pins that the confirmation does not reopen the
|
||||
// null-version exclusion at the head of getReplicationAction. An existing object resync returns
|
||||
// replicateNone for a null version whose source modification time is later than the target's,
|
||||
// before comparing anything, and that must stand even when the source carries a removed retention
|
||||
// and the destination would report retention or refuse to answer.
|
||||
func TestReplicationActionForTargetNullVersionResync(t *testing.T) {
|
||||
modeKey := strings.ToLower(xhttp.AmzObjectLockMode)
|
||||
dateKey := strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
|
||||
governance := minio.Governance
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mode *minio.RetentionMode
|
||||
err error
|
||||
}{
|
||||
{"destination holds retention", &governance, nil},
|
||||
{"retention read denied", nil, minio.ErrorResponse{Code: "AccessDenied"}},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
src, tgtInfo := newMatchingReplicationPair()
|
||||
// A null version whose source modification time is later, and whose content differs,
|
||||
// so only the exclusion can hold the action at replicateNone.
|
||||
src.VersionID = nullVersionID
|
||||
src.ModTime = tgtInfo.LastModified.Add(time.Hour)
|
||||
src.ETag = "5d41402abc4b2a76b9719d911017c592"
|
||||
src.UserDefined[modeKey] = ""
|
||||
src.UserDefined[dateKey] = ""
|
||||
tgtInfo.VersionID = nullVersionID
|
||||
|
||||
tgt := &fakeRetentionGetter{mode: test.mode, err: test.err}
|
||||
got := replicationActionForTarget(t.Context(), src, tgtInfo, replication.ExistingObjectReplicationType, tgt, "bucket", "object")
|
||||
if got != replicateNone {
|
||||
t.Fatalf("replicationActionForTarget() = %q, want %q", got, replicateNone)
|
||||
}
|
||||
if tgt.calls != 0 {
|
||||
t.Fatalf("GetObjectRetention called %d times, want 0", tgt.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestReplicationActionForTargetTimestampOnlyRemoval covers representation (2) of a removed
|
||||
// retention. A removal that arrived by replication persists only the retention ordering timestamp,
|
||||
// with the mode and retain-until-date keys absent, because restoreRetention writes the timestamp
|
||||
// alone when the mode is empty (cmd/bucket-object-lock.go). The comparison in getReplicationAction
|
||||
// reads such a source as in sync with a matching destination, so only the GetObjectRetention
|
||||
// confirmation separates a destination that dropped the retention from one hiding it behind a
|
||||
// permission-filtered HEAD. The source is built through the real restoreRetention path so the
|
||||
// fixture is the metadata a replicated removal actually leaves on disk, not a hand-rolled map.
|
||||
func TestReplicationActionForTargetTimestampOnlyRemoval(t *testing.T) {
|
||||
governance := minio.Governance
|
||||
stamp := time.Date(2026, 9, 6, 1, 0, 0, 0, time.UTC).Format(time.RFC3339Nano)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mode *minio.RetentionMode
|
||||
err error
|
||||
want replicationAction
|
||||
wantCalls int
|
||||
}{
|
||||
{
|
||||
// The reference case: a destination that denies the retention read is
|
||||
// indistinguishable from one still holding it, so the removal is resent.
|
||||
name: "retention hidden from HEAD by permissions",
|
||||
err: minio.ErrorResponse{Code: "AccessDenied"},
|
||||
want: replicateMetadata,
|
||||
wantCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "destination still holds the retention",
|
||||
mode: &governance,
|
||||
want: replicateMetadata,
|
||||
wantCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "removal confirmed by destination",
|
||||
err: minio.ErrorResponse{Code: "NoSuchObjectLockConfiguration"},
|
||||
want: replicateNone,
|
||||
wantCalls: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
src, tgtInfo := newMatchingReplicationPair()
|
||||
// Persist the timestamp-only tombstone the same way an applied replica removal does.
|
||||
objectLockState{retentionTimestamp: stamp}.restoreRetention(src.UserDefined)
|
||||
if !retentionRemovedAtSource(src) {
|
||||
t.Fatalf("restoreRetention fixture not recognized as a removal: %v", src.UserDefined)
|
||||
}
|
||||
if _, ok := src.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)]; ok {
|
||||
t.Fatalf("restoreRetention fixture wrote a mode key, fixture is not timestamp-only: %v", src.UserDefined)
|
||||
}
|
||||
|
||||
tgt := &fakeRetentionGetter{mode: test.mode, err: test.err}
|
||||
got := replicationActionForTarget(t.Context(), src, tgtInfo, replication.HealReplicationType, tgt, "bucket", "object")
|
||||
if got != test.want {
|
||||
t.Fatalf("replicationActionForTarget() = %q, want %q", got, test.want)
|
||||
}
|
||||
if tgt.calls != test.wantCalls {
|
||||
t.Fatalf("GetObjectRetention called %d times, want %d", tgt.calls, test.wantCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// These tests pin the IAM bucket/object resource boundary end to end, through
|
||||
|
||||
@@ -28,7 +28,7 @@ import (
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
+101
-19
@@ -36,6 +36,8 @@ import (
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
fcolor "github.com/fatih/color"
|
||||
@@ -57,10 +59,10 @@ import (
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/certs"
|
||||
"github.com/minio/pkg/v3/console"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
"github.com/pgsty/silo-pkg/v3/certs"
|
||||
"github.com/pgsty/silo-pkg/v3/console"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
"golang.org/x/term"
|
||||
)
|
||||
|
||||
@@ -118,15 +120,40 @@ func init() {
|
||||
|
||||
const consolePrefix = "CONSOLE_"
|
||||
|
||||
// consoleMinIOServerEnv derives the CONSOLE_MINIO_SERVER value the embedded
|
||||
// Console uses to reach the S3/STS API, and whether TLS verification of that
|
||||
// endpoint must be skipped. With no explicit endpoint configured the Console
|
||||
// reaches the API over the loopback address, whose TLS certificate is not
|
||||
// expected to carry a 127.0.0.1 SAN; because Console verifies outbound TLS by
|
||||
// default, the loopback origin has to be exempted or embedded login (local and
|
||||
// LDAP alike) fails at the STS handshake. The exemption is endpoint-scoped in
|
||||
// Console, so every other HTTPS peer stays verified. An explicitly configured
|
||||
// endpoint is always reached under its own verified name and is never exempted.
|
||||
func consoleMinIOServerEnv(endpoint string, isTLS bool, port string) (server string, skipVerify bool) {
|
||||
if endpoint != "" {
|
||||
return endpoint, false
|
||||
}
|
||||
return fmt.Sprintf("%s://127.0.0.1:%s", getURLScheme(isTLS), port), isTLS
|
||||
}
|
||||
|
||||
func minioConfigToConsoleFeatures() {
|
||||
os.Setenv("CONSOLE_PBKDF_SALT", globalDeploymentID())
|
||||
os.Setenv("CONSOLE_PBKDF_PASSPHRASE", globalDeploymentID())
|
||||
if globalMinioEndpoint != "" {
|
||||
os.Setenv("CONSOLE_MINIO_SERVER", globalMinioEndpoint)
|
||||
consoleServer, skipVerify := consoleMinIOServerEnv(globalMinioEndpoint, globalIsTLS, globalMinioPort)
|
||||
os.Setenv("CONSOLE_MINIO_SERVER", consoleServer)
|
||||
if skipVerify {
|
||||
// The embedded Console reaches the loopback S3/STS endpoint above, whose
|
||||
// certificate is not expected to carry a 127.0.0.1 SAN. Console verifies
|
||||
// outbound TLS by default (silo-console v2.3.x), so opt into the
|
||||
// endpoint-scoped compatibility switch to preserve the documented loopback
|
||||
// bypass; every other HTTPS peer (IdP, Prometheus, webhooks, ...) stays
|
||||
// verified. initConsoleServer unsets CONSOLE_* before calling this, so the
|
||||
// switch cannot be supplied by the operator on the embedded path.
|
||||
os.Setenv("CONSOLE_MINIO_SERVER_TLS_SKIP_VERIFY", "on")
|
||||
} else {
|
||||
// Explicitly set 127.0.0.1 so Console will automatically bypass TLS verification to the local S3 API.
|
||||
// This will save users from providing a certificate with IP or FQDN SAN that points to the local host.
|
||||
os.Setenv("CONSOLE_MINIO_SERVER", fmt.Sprintf("%s://127.0.0.1:%s", getURLScheme(globalIsTLS), globalMinioPort))
|
||||
// An explicitly configured endpoint is reached under its own verified name;
|
||||
// never let a loopback exemption apply to it.
|
||||
os.Unsetenv("CONSOLE_MINIO_SERVER_TLS_SKIP_VERIFY")
|
||||
}
|
||||
if value := env.Get(config.EnvMinIOLogQueryURL, ""); value != "" {
|
||||
os.Setenv("CONSOLE_LOG_QUERY_URL", value)
|
||||
@@ -230,11 +257,31 @@ func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
|
||||
return m
|
||||
}
|
||||
|
||||
func initConsoleServer() (*consoleapi.Server, error) {
|
||||
// unset all console_ environment variables.
|
||||
// resetConsoleEnvironment preserves the embedded Console's supported resource
|
||||
// settings verbatim. Server derives all other Console settings itself.
|
||||
func resetConsoleEnvironment() {
|
||||
for _, cenv := range env.List(consolePrefix) {
|
||||
switch cenv {
|
||||
case consoleapi.ConsoleWSMaxConnections,
|
||||
consoleapi.ConsoleWSMaxConnectionsPerClient,
|
||||
consoleapi.ConsoleWSMaxAnonymousConnections,
|
||||
consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient:
|
||||
continue
|
||||
}
|
||||
os.Unsetenv(cenv)
|
||||
}
|
||||
}
|
||||
|
||||
func initConsoleServer() (*consoleapi.Server, error) {
|
||||
resetConsoleEnvironment()
|
||||
// Validate explicitly: ConfigureAPI logs errors, but embedded Console logs
|
||||
// are normally silenced. Return configuration failures to Server startup.
|
||||
if err := consoleapi.ConfigureEmbeddedSourceIPTrust(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := consoleapi.ConfigureWebSocketLimits(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// enable all console environment variables
|
||||
minioConfigToConsoleFeatures()
|
||||
@@ -540,6 +587,30 @@ func (e envKV) String() string {
|
||||
return fmt.Sprintf("%s=%s", e.Key, e.Value)
|
||||
}
|
||||
|
||||
func isValidEnvName(name string) bool {
|
||||
if name == "" || !utf8.ValidString(name) {
|
||||
return false
|
||||
}
|
||||
for _, ch := range name {
|
||||
if ch == '=' || unicode.IsSpace(ch) || !unicode.IsGraphic(ch) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func trimExportPrefix(envEntry string) string {
|
||||
rest, ok := strings.CutPrefix(envEntry, "export")
|
||||
if !ok || rest == "" {
|
||||
return envEntry
|
||||
}
|
||||
trimmed := strings.TrimLeftFunc(rest, unicode.IsSpace)
|
||||
if len(trimmed) == len(rest) {
|
||||
return envEntry
|
||||
}
|
||||
return trimmed
|
||||
}
|
||||
|
||||
func parsEnvEntry(envEntry string) (envKV, error) {
|
||||
envEntry = strings.TrimSpace(envEntry)
|
||||
if envEntry == "" {
|
||||
@@ -554,13 +625,19 @@ func parsEnvEntry(envEntry string) (envKV, error) {
|
||||
Skip: true,
|
||||
}, nil
|
||||
}
|
||||
envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), config.EnvSeparator, 2)
|
||||
envTokens := strings.SplitN(trimExportPrefix(envEntry), config.EnvSeparator, 2)
|
||||
if len(envTokens) != 2 {
|
||||
return envKV{}, fmt.Errorf("envEntry malformed; %s, expected to be of form 'KEY=value'", envEntry)
|
||||
return envKV{}, errors.New("missing '='")
|
||||
}
|
||||
|
||||
key := envTokens[0]
|
||||
val := envTokens[1]
|
||||
key := strings.TrimSpace(envTokens[0])
|
||||
val := strings.TrimSpace(envTokens[1])
|
||||
if !isValidEnvName(key) {
|
||||
return envKV{}, fmt.Errorf("invalid environment variable name %q", key)
|
||||
}
|
||||
if strings.IndexByte(val, 0) >= 0 {
|
||||
return envKV{}, errors.New("environment variable value contains NUL")
|
||||
}
|
||||
|
||||
// Remove quotes from the value if found
|
||||
if len(val) >= 2 {
|
||||
@@ -587,10 +664,12 @@ func minioEnvironFromFile(envConfigFile string) ([]envKV, error) {
|
||||
defer f.Close()
|
||||
var ekvs []envKV
|
||||
scanner := bufio.NewScanner(f)
|
||||
lineNo := 0
|
||||
for scanner.Scan() {
|
||||
lineNo++
|
||||
ekv, err := parsEnvEntry(scanner.Text())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("%s:%d: %w", envConfigFile, lineNo, err)
|
||||
}
|
||||
if ekv.Skip {
|
||||
// Skips empty lines
|
||||
@@ -599,7 +678,7 @@ func minioEnvironFromFile(envConfigFile string) ([]envKV, error) {
|
||||
ekvs = append(ekvs, ekv)
|
||||
}
|
||||
if err = scanner.Err(); err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("%s: %w", envConfigFile, err)
|
||||
}
|
||||
return ekvs, nil
|
||||
}
|
||||
@@ -666,12 +745,15 @@ func loadEnvVarsFromFiles() {
|
||||
}
|
||||
|
||||
if env.IsSet(config.EnvConfigEnvFile) {
|
||||
ekvs, err := minioEnvironFromFile(env.Get(config.EnvConfigEnvFile, ""))
|
||||
envConfigFile := env.Get(config.EnvConfigEnvFile, "")
|
||||
ekvs, err := minioEnvironFromFile(envConfigFile)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
logger.Fatal(err, "Unable to read the config environment file")
|
||||
}
|
||||
for _, ekv := range ekvs {
|
||||
os.Setenv(ekv.Key, ekv.Value)
|
||||
if err := os.Setenv(ekv.Key, ekv.Value); err != nil {
|
||||
logger.Fatal(err, "Unable to set %s from config environment file %s", ekv.Key, envConfigFile)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,9 +19,15 @@ package cmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
consoleapi "github.com/minio/console/api"
|
||||
"github.com/minio/minio/internal/config"
|
||||
)
|
||||
|
||||
func Test_readFromSecret(t *testing.T) {
|
||||
@@ -181,3 +187,324 @@ MINIO_ROOT_PASSWORD=minio123`,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_minioEnvironFromFileWhitespaceAndValidation(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
content string
|
||||
want []envKV
|
||||
errLine int
|
||||
errContains string
|
||||
errExcludes string
|
||||
}{
|
||||
{
|
||||
name: "spaces and tabs around separator",
|
||||
content: "MINIO_ROOT_USER = minio\nMINIO_ROOT_PASSWORD\t=\tminio123",
|
||||
want: []envKV{
|
||||
{Key: "MINIO_ROOT_USER", Value: "minio"},
|
||||
{Key: "MINIO_ROOT_PASSWORD", Value: "minio123"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "export tab and quoted spaces",
|
||||
content: "export\tMINIO_ROOT_USER = \" minio user \"\nexport MINIO_ROOT_PASSWORD = ' minio secret '",
|
||||
want: []envKV{
|
||||
{Key: "MINIO_ROOT_USER", Value: " minio user "},
|
||||
{Key: "MINIO_ROOT_PASSWORD", Value: " minio secret "},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "export Unicode whitespace",
|
||||
content: "export\u00a0MINIO_ROOT_USER=value",
|
||||
want: []envKV{
|
||||
{Key: "MINIO_ROOT_USER", Value: "value"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "export is only a standalone prefix",
|
||||
content: "export=value\nexportFOO=bar",
|
||||
want: []envKV{
|
||||
{Key: "export", Value: "value"},
|
||||
{Key: "exportFOO", Value: "bar"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "unquoted whitespace empty value and additional separators",
|
||||
content: "UNQUOTED = value \nEMPTY =\nTOKEN = scheme://user:password@example.com?a=b",
|
||||
want: []envKV{
|
||||
{Key: "UNQUOTED", Value: "value"},
|
||||
{Key: "EMPTY", Value: ""},
|
||||
{Key: "TOKEN", Value: "scheme://user:password@example.com?a=b"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid underscore and digits",
|
||||
content: "_VALID_2=value",
|
||||
want: []envKV{
|
||||
{Key: "_VALID_2", Value: "value"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "named target punctuation and unicode",
|
||||
content: "MINIO_NOTIFY_WEBHOOK_ENABLE_my-hook=off\n" +
|
||||
"MINIO_NOTIFY_WEBHOOK_ENABLE_site.eu=off\n" +
|
||||
"MINIO_NOTIFY_WEBHOOK_ENABLE_team:blue=off\n" +
|
||||
"MINIO_NOTIFY_WEBHOOK_ENABLE_目标=off",
|
||||
want: []envKV{
|
||||
{Key: "MINIO_NOTIFY_WEBHOOK_ENABLE_my-hook", Value: "off"},
|
||||
{Key: "MINIO_NOTIFY_WEBHOOK_ENABLE_site.eu", Value: "off"},
|
||||
{Key: "MINIO_NOTIFY_WEBHOOK_ENABLE_team:blue", Value: "off"},
|
||||
{Key: "MINIO_NOTIFY_WEBHOOK_ENABLE_目标", Value: "off"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "missing separator redacts the line",
|
||||
content: "MINIO_ROOT_PASSWORD=valid\nsuper-secret-without-equals",
|
||||
errLine: 2,
|
||||
errContains: "missing '='",
|
||||
errExcludes: "super-secret-without-equals",
|
||||
},
|
||||
{
|
||||
name: "empty name",
|
||||
content: "=empty-name-secret",
|
||||
errLine: 1,
|
||||
errContains: `invalid environment variable name ""`,
|
||||
errExcludes: "empty-name-secret",
|
||||
},
|
||||
{
|
||||
name: "os compatible leading digit and punctuation",
|
||||
content: "1MINIO_ROOT_USER=digit-leading-secret\n-MINIO-ROOT-USER=hyphen-secret",
|
||||
want: []envKV{
|
||||
{Key: "1MINIO_ROOT_USER", Value: "digit-leading-secret"},
|
||||
{Key: "-MINIO-ROOT-USER", Value: "hyphen-secret"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "whitespace in name",
|
||||
content: "MINIO ROOT USER=whitespace-secret",
|
||||
errLine: 1,
|
||||
errContains: `invalid environment variable name "MINIO ROOT USER"`,
|
||||
errExcludes: "whitespace-secret",
|
||||
},
|
||||
{
|
||||
name: "NUL in name",
|
||||
content: "MINIO\x00ROOT=nul-name-secret",
|
||||
errLine: 1,
|
||||
errContains: "invalid environment variable name",
|
||||
errExcludes: "nul-name-secret",
|
||||
},
|
||||
{
|
||||
name: "format character in name",
|
||||
content: "MINIO\u200bROOT=format-secret",
|
||||
errLine: 1,
|
||||
errContains: "invalid environment variable name",
|
||||
errExcludes: "format-secret",
|
||||
},
|
||||
{
|
||||
name: "NUL in value",
|
||||
content: "MINIO_ROOT_USER=before\x00nul-value-secret",
|
||||
errLine: 1,
|
||||
errContains: "environment variable value contains NUL",
|
||||
errExcludes: "nul-value-secret",
|
||||
},
|
||||
{
|
||||
name: "diagnostic has file and line but no value",
|
||||
content: "MINIO_ROOT_USER=valid\nBAD KEY=super-secret-value",
|
||||
errLine: 2,
|
||||
errContains: `invalid environment variable name "BAD KEY"`,
|
||||
errExcludes: "super-secret-value",
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
tmpfile, err := os.CreateTemp(t.TempDir(), "testfile")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = tmpfile.WriteString(testCase.content); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = tmpfile.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := minioEnvironFromFile(tmpfile.Name())
|
||||
if testCase.errContains == "" {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, testCase.want) {
|
||||
t.Errorf("expected %v, got %v", testCase.want, got)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("expected an error")
|
||||
}
|
||||
errText := err.Error()
|
||||
location := fmt.Sprintf("%s:%d:", tmpfile.Name(), testCase.errLine)
|
||||
if !strings.Contains(errText, location) {
|
||||
t.Errorf("expected error to contain %q, got %q", location, errText)
|
||||
}
|
||||
if !strings.Contains(errText, testCase.errContains) {
|
||||
t.Errorf("expected error to contain %q, got %q", testCase.errContains, errText)
|
||||
}
|
||||
if testCase.errExcludes != "" && strings.Contains(errText, testCase.errExcludes) {
|
||||
t.Errorf("expected error to redact %q, got %q", testCase.errExcludes, errText)
|
||||
}
|
||||
if got != nil {
|
||||
t.Errorf("expected no entries on parse error, got %v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigEnvFileNamedTargetDiscovery(t *testing.T) {
|
||||
key := "MINIO_NOTIFY_WEBHOOK_ENABLE_my-hook"
|
||||
t.Setenv(key, "off")
|
||||
|
||||
targets, err := (config.Config{}).GetAvailableTargets(config.NotifyWebhookSubSys)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(targets, "my-hook") {
|
||||
t.Fatalf("named target %q not discovered from %s: %v", "my-hook", key, targets)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConsoleMinIOServerEnv locks in the loopback TLS exemption that keeps
|
||||
// embedded Console login working (issue #108) while ensuring an explicitly
|
||||
// configured endpoint is never silently exempted from TLS verification.
|
||||
func TestConsoleMinIOServerEnv(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
isTLS bool
|
||||
port string
|
||||
wantServer string
|
||||
wantSkipVerify bool
|
||||
}{
|
||||
{
|
||||
name: "loopback TLS is exempted so embedded login works",
|
||||
isTLS: true,
|
||||
port: "9000",
|
||||
wantServer: "https://127.0.0.1:9000",
|
||||
wantSkipVerify: true,
|
||||
},
|
||||
{
|
||||
name: "loopback plain HTTP needs no exemption",
|
||||
isTLS: false,
|
||||
port: "9000",
|
||||
wantServer: "http://127.0.0.1:9000",
|
||||
},
|
||||
{
|
||||
name: "explicit https endpoint stays verified",
|
||||
endpoint: "https://silo.example:9000",
|
||||
isTLS: true,
|
||||
port: "9000",
|
||||
wantServer: "https://silo.example:9000",
|
||||
},
|
||||
{
|
||||
name: "explicit http endpoint stays verified",
|
||||
endpoint: "http://silo.example:9000",
|
||||
isTLS: false,
|
||||
port: "9000",
|
||||
wantServer: "http://silo.example:9000",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
server, skipVerify := consoleMinIOServerEnv(tt.endpoint, tt.isTLS, tt.port)
|
||||
if server != tt.wantServer {
|
||||
t.Fatalf("server = %q, want %q", server, tt.wantServer)
|
||||
}
|
||||
if skipVerify != tt.wantSkipVerify {
|
||||
t.Fatalf("skipVerify = %v, want %v", skipVerify, tt.wantSkipVerify)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The startup path clears process environment, so preserve all existing Console
|
||||
// variables, including ones unrelated to this test, before exercising it.
|
||||
func preserveConsoleEnvironment(t *testing.T) {
|
||||
t.Helper()
|
||||
for _, entry := range os.Environ() {
|
||||
if strings.HasPrefix(entry, consolePrefix) {
|
||||
name, value, _ := strings.Cut(entry, "=")
|
||||
t.Setenv(name, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetConsoleEnvironment(t *testing.T) {
|
||||
preserveConsoleEnvironment(t)
|
||||
settings := map[string]string{
|
||||
consoleapi.ConsoleWSMaxConnections: "2048",
|
||||
consoleapi.ConsoleWSMaxConnectionsPerClient: "512",
|
||||
consoleapi.ConsoleWSMaxAnonymousConnections: "128",
|
||||
consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient: "16",
|
||||
}
|
||||
for key, value := range settings {
|
||||
t.Setenv(key, value)
|
||||
}
|
||||
decoys := []string{"CONSOLE_MINIO_SERVER_TLS_SKIP_VERIFY", "CONSOLE_MINIO_SERVER", "CONSOLE_PBKDF_SALT", "CONSOLE_TRUSTED_PROXIES", "CONSOLE_WS_MAX_UNKNOWN"}
|
||||
for _, key := range decoys {
|
||||
t.Setenv(key, "operator-value")
|
||||
}
|
||||
resetConsoleEnvironment()
|
||||
for key, want := range settings {
|
||||
if got, present := os.LookupEnv(key); !present || got != want {
|
||||
t.Errorf("%s = %q, present = %v; want %q", key, got, present, want)
|
||||
}
|
||||
}
|
||||
for _, key := range decoys {
|
||||
if _, present := os.LookupEnv(key); present {
|
||||
t.Errorf("unsupported override %s survived", key)
|
||||
}
|
||||
}
|
||||
for _, raw := range []string{"", " 16 ", "env://missing-limit"} {
|
||||
t.Setenv(consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient, raw)
|
||||
resetConsoleEnvironment()
|
||||
if got, present := os.LookupEnv(consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient); !present || got != raw {
|
||||
t.Fatalf("raw value %q was changed to %q (present = %v)", raw, got, present)
|
||||
}
|
||||
}
|
||||
os.Unsetenv(consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient)
|
||||
resetConsoleEnvironment()
|
||||
if _, present := os.LookupEnv(consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient); present {
|
||||
t.Fatal("unset setting became present")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitConsoleServerConfigurationErrors(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name, proxy, limit, want string
|
||||
}{
|
||||
{"proxy error precedes limit error", "proxy.internal", "bad", "MINIO_API_TRUSTED_PROXIES"},
|
||||
{"blank limit", "", "", "CONSOLE_WS_MAX_ANONYMOUS_CONNECTIONS_PER_CLIENT"},
|
||||
{"non-integer limit", "", "bad", "CONSOLE_WS_MAX_ANONYMOUS_CONNECTIONS_PER_CLIENT"},
|
||||
{"out-of-range limit", "", "0", "CONSOLE_WS_MAX_ANONYMOUS_CONNECTIONS_PER_CLIENT"},
|
||||
{"inconsistent limits", "", "256", "must be less than"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Restore the process-wide library configuration after environment cleanup.
|
||||
t.Cleanup(func() {
|
||||
_ = consoleapi.ConfigureEmbeddedSourceIPTrust()
|
||||
_ = consoleapi.ConfigureWebSocketLimits()
|
||||
})
|
||||
preserveConsoleEnvironment(t)
|
||||
t.Setenv(consoleapi.EnvMinIOTrustedProxies, tt.proxy)
|
||||
t.Setenv(consoleapi.ConsoleWSMaxConnections, "1024")
|
||||
t.Setenv(consoleapi.ConsoleWSMaxConnectionsPerClient, "256")
|
||||
t.Setenv(consoleapi.ConsoleWSMaxAnonymousConnections, "64")
|
||||
t.Setenv(consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient, tt.limit)
|
||||
server, err := initConsoleServer()
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) || server != nil {
|
||||
t.Fatalf("initConsoleServer() = %v, %v; want nil server and %q error", server, err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,738 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
// Copyright (c) 2026 PGSTY
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/s2"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
)
|
||||
|
||||
// disableCompression turns the global compression config off and returns a
|
||||
// restore func. It is the replication destination that applies no transform of
|
||||
// its own; setCopyChecksumCompression covers the enabled cases.
|
||||
func disableCompression() func() {
|
||||
globalCompressConfigMu.Lock()
|
||||
previous := globalCompressConfig
|
||||
globalCompressConfig.Enabled = false
|
||||
globalCompressConfigMu.Unlock()
|
||||
|
||||
return func() {
|
||||
globalCompressConfigMu.Lock()
|
||||
globalCompressConfig = previous
|
||||
globalCompressConfigMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// ssecTestHeaders builds the customer key headers for a key made of the given
|
||||
// repeated byte.
|
||||
func ssecTestHeaders(b byte) map[string]string {
|
||||
key := bytes.Repeat([]byte{b}, 32)
|
||||
keyMD5 := md5.Sum(key)
|
||||
return map[string]string{
|
||||
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key),
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(keyMD5[:]),
|
||||
}
|
||||
}
|
||||
|
||||
// assertStoredSSECUncompressed requires the stored object to be SSE-C sealed
|
||||
// and to carry no compression marker, so that "not compressed" is never
|
||||
// reported for an object that is not encrypted either.
|
||||
func assertStoredSSECUncompressed(t *testing.T, obj ObjectLayer, bucketName, object string) ObjectInfo {
|
||||
t.Helper()
|
||||
info, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, sealed := info.UserDefined[crypto.MetaSealedKeySSEC]; !sealed {
|
||||
t.Fatalf("%s is not SSE-C sealed, the fixture proves nothing (userDefined=%v)", object, info.UserDefined)
|
||||
}
|
||||
if marker, compressed := info.UserDefined[ReservedMetadataPrefix+"compression"]; compressed {
|
||||
t.Errorf("%s was stored as a compressed SSE-C object (compression=%q); such an object cannot be replicated",
|
||||
object, marker)
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
// assertSSECPlaintext GETs an SSE-C object with its customer key and requires
|
||||
// the body to equal the plaintext.
|
||||
func assertSSECPlaintext(t *testing.T, apiRouter http.Handler, credentials auth.Credentials,
|
||||
bucketName, object string, sseHeaders map[string]string, want []byte,
|
||||
) {
|
||||
t.Helper()
|
||||
req, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucketName, object),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("GET %s status %d: %s", object, rec.Code, rec.Body.String())
|
||||
}
|
||||
if bytes.Equal(rec.Body.Bytes(), want) {
|
||||
return
|
||||
}
|
||||
body := rec.Body.Bytes()
|
||||
head := body
|
||||
if len(head) > 16 {
|
||||
head = head[:16]
|
||||
}
|
||||
t.Errorf("GET %s returned %d bytes, want the %d byte plaintext; first bytes % x",
|
||||
object, len(body), len(want), head)
|
||||
// Name the failure mode: a body that s2-decodes to the plaintext is the raw
|
||||
// S2 stream of a compressed source shipped without its compression marker.
|
||||
if decoded, derr := io.ReadAll(s2.NewReader(bytes.NewReader(body))); derr == nil && bytes.Equal(decoded, want) {
|
||||
t.Errorf("the returned body is the raw S2 stream: s2-decoding it yields the %d byte plaintext", len(want))
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPISSECCompressionReplicaStaysReadable replicates an SSE-C object written
|
||||
// with compression enabled and allow_encryption=on, and requires the replica to
|
||||
// read back as the source plaintext.
|
||||
//
|
||||
// The source object is read the way the replication worker reads it
|
||||
// (ReplicationRequest, hence NoDecryption), its wire headers come from the
|
||||
// production option builder putReplicationOpts, and the replica is written the
|
||||
// way a destination that applies no transform of its own stores it: compression
|
||||
// off and no default encryption.
|
||||
//
|
||||
// Before the SSE-C compression exclusion the source was stored as
|
||||
// encrypt(s2(plaintext)) while putReplicationOpts dropped
|
||||
// X-Minio-Internal-compression, so the replica decrypted to an S2 stream and a
|
||||
// correct-key GET returned HTTP 200 with the wrong body.
|
||||
func TestAPISSECCompressionReplicaStaysReadable(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPISSECCompressionReplicaStaysReadable,
|
||||
})
|
||||
}
|
||||
|
||||
func testAPISSECCompressionReplicaStaysReadable(obj ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
previousTLS := globalIsTLS
|
||||
globalIsTLS = true
|
||||
defer func() { globalIsTLS = previousTLS }()
|
||||
|
||||
replicator := newObjectAttributesAuthzUser(t, instanceType, bucketName, `"s3:PutObject","s3:GetObject","s3:ReplicateObject"`)
|
||||
sseHeaders := ssecTestHeaders(0x42)
|
||||
// Highly compressible and comfortably above minCompressibleSize (4096).
|
||||
data := bytes.Repeat([]byte("silo compressed ssec replication payload "), 8192)
|
||||
|
||||
t.Run(instanceType+"/single-put", func(t *testing.T) {
|
||||
object := "replication/ssec-single.txt"
|
||||
|
||||
// --- Source side: compression ON with allow_encryption ON. ---
|
||||
restore := setCopyChecksumCompression(true)
|
||||
srcReq, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, object),
|
||||
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
srcRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(srcRec, srcReq)
|
||||
if srcRec.Code != http.StatusOK {
|
||||
restore()
|
||||
t.Fatalf("source PUT status %d: %s", srcRec.Code, srcRec.Body.String())
|
||||
}
|
||||
sourceInfo := assertStoredSSECUncompressed(t, obj, bucketName, object)
|
||||
t.Logf("source: stored size=%d compression=%q plaintext=%d", sourceInfo.Size,
|
||||
sourceInfo.UserDefined[ReservedMetadataPrefix+"compression"], len(data))
|
||||
|
||||
// The replication worker's read: raw stored bytes, no decryption.
|
||||
gr, err := obj.GetObjectNInfo(t.Context(), bucketName, object, nil, http.Header{},
|
||||
ObjectOptions{ReplicationRequest: true})
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
sourceInfo = gr.ObjInfo
|
||||
raw, err := io.ReadAll(gr)
|
||||
gr.Close()
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
replicationOpts, isMP, err := putReplicationOpts(t.Context(), "", sourceInfo)
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatalf("putReplicationOpts rejected the source: %v", err)
|
||||
}
|
||||
if isMP {
|
||||
restore()
|
||||
t.Fatal("single PUT source classified as multipart")
|
||||
}
|
||||
headers := map[string]string{}
|
||||
for name, values := range replicationOpts.Header() {
|
||||
if len(values) > 0 {
|
||||
headers[name] = values[0]
|
||||
}
|
||||
}
|
||||
restore()
|
||||
|
||||
// --- Destination side: NO compression, NO default encryption. ---
|
||||
restoreDst := disableCompression()
|
||||
defer restoreDst()
|
||||
|
||||
replReq, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, object),
|
||||
int64(len(raw)), bytes.NewReader(raw), replicator.AccessKey, replicator.SecretKey, headers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(replRec, replReq)
|
||||
if replRec.Code != http.StatusOK {
|
||||
t.Fatalf("replica PUT status %d: %s", replRec.Code, replRec.Body.String())
|
||||
}
|
||||
|
||||
info, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("replica: stored size=%d compression=%q actual-size=%q", info.Size,
|
||||
info.UserDefined[ReservedMetadataPrefix+"compression"],
|
||||
info.UserDefined[ReservedMetadataPrefix+"actual-size"])
|
||||
assertSSECPlaintext(t, apiRouter, credentials, bucketName, object, sseHeaders, data)
|
||||
})
|
||||
|
||||
t.Run(instanceType+"/multipart", func(t *testing.T) {
|
||||
object := "replication/ssec-mpu.txt"
|
||||
|
||||
restore := setCopyChecksumCompression(true)
|
||||
newReq, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
newRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(newRec, newReq)
|
||||
if newRec.Code != http.StatusOK {
|
||||
restore()
|
||||
t.Fatalf("source NewMultipart status %d: %s", newRec.Code, newRec.Body.String())
|
||||
}
|
||||
var sourceInit InitiateMultipartUploadResponse
|
||||
if err = xmlDecoder(newRec.Body, &sourceInit, int64(newRec.Body.Len())); err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
partReq, err := newTestSignedRequestV4(http.MethodPut,
|
||||
getPutObjectPartURL("", bucketName, object, sourceInit.UploadID, "1"),
|
||||
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
partRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(partRec, partReq)
|
||||
if partRec.Code != http.StatusOK {
|
||||
restore()
|
||||
t.Fatalf("source PutPart status %d: %s", partRec.Code, partRec.Body.String())
|
||||
}
|
||||
completeBody, err := xml.Marshal(CompleteMultipartUpload{Parts: []CompletePart{
|
||||
{PartNumber: 1, ETag: canonicalizeETag(partRec.Header()[xhttp.ETag][0])},
|
||||
}})
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
completeReq, err := newTestSignedRequestV4(http.MethodPost,
|
||||
getCompleteMultipartUploadURL("", bucketName, object, sourceInit.UploadID),
|
||||
int64(len(completeBody)), bytes.NewReader(completeBody), credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
completeRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(completeRec, completeReq)
|
||||
if completeRec.Code != http.StatusOK {
|
||||
restore()
|
||||
t.Fatalf("source Complete status %d: %s", completeRec.Code, completeRec.Body.String())
|
||||
}
|
||||
assertStoredSSECUncompressed(t, obj, bucketName, object)
|
||||
|
||||
gr, err := obj.GetObjectNInfo(t.Context(), bucketName, object, nil, http.Header{},
|
||||
ObjectOptions{ReplicationRequest: true})
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
sourceInfo := gr.ObjInfo
|
||||
rawPart, err := io.ReadAll(gr)
|
||||
gr.Close()
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
actualSize, err := sourceInfo.GetActualSize()
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("source mpu: stored size=%d actual-size=%d rawRead=%d plaintext=%d compression=%q",
|
||||
sourceInfo.Size, actualSize, len(rawPart), len(data),
|
||||
sourceInfo.UserDefined[ReservedMetadataPrefix+"compression"])
|
||||
|
||||
replicationOpts, isMP, err := putReplicationOpts(t.Context(), "", sourceInfo)
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatalf("putReplicationOpts rejected the source: %v", err)
|
||||
}
|
||||
if !isMP {
|
||||
restore()
|
||||
t.Fatal("SSE-C multipart source not recognized as multipart")
|
||||
}
|
||||
replicationOpts.Internal.SourceMTime = time.Time{}
|
||||
headers := map[string]string{}
|
||||
for name, values := range replicationOpts.Header() {
|
||||
if len(values) > 0 {
|
||||
headers[name] = values[0]
|
||||
}
|
||||
}
|
||||
restore()
|
||||
|
||||
// --- Destination: no compression, no default encryption. ---
|
||||
restoreDst := disableCompression()
|
||||
defer restoreDst()
|
||||
|
||||
replNewReq, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
|
||||
0, nil, replicator.AccessKey, replicator.SecretKey, headers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replNewRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(replNewRec, replNewReq)
|
||||
if replNewRec.Code != http.StatusOK {
|
||||
t.Fatalf("replica NewMultipart status %d: %s", replNewRec.Code, replNewRec.Body.String())
|
||||
}
|
||||
var replicaInit InitiateMultipartUploadResponse
|
||||
if err = xmlDecoder(replNewRec.Body, &replicaInit, int64(replNewRec.Body.Len())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replPartReq, err := newTestSignedRequestV4(http.MethodPut,
|
||||
getPutObjectPartURL("", bucketName, object, replicaInit.UploadID, "1"),
|
||||
int64(len(rawPart)), bytes.NewReader(rawPart), replicator.AccessKey, replicator.SecretKey,
|
||||
map[string]string{xhttp.MinIOSourceReplicationRequest: "true"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replPartRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(replPartRec, replPartReq)
|
||||
if replPartRec.Code != http.StatusOK {
|
||||
t.Fatalf("replica PutPart status %d: %s", replPartRec.Code, replPartRec.Body.String())
|
||||
}
|
||||
replCompleteBody, err := xml.Marshal(CompleteMultipartUpload{Parts: []CompletePart{
|
||||
{PartNumber: 1, ETag: canonicalizeETag(replPartRec.Header()[xhttp.ETag][0])},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replCompleteReq, err := newTestSignedRequestV4(http.MethodPost,
|
||||
getCompleteMultipartUploadURL("", bucketName, object, replicaInit.UploadID),
|
||||
int64(len(replCompleteBody)), bytes.NewReader(replCompleteBody), replicator.AccessKey, replicator.SecretKey,
|
||||
map[string]string{
|
||||
xhttp.MinIOSourceReplicationRequest: "true",
|
||||
xhttp.MinIOSourceMTime: sourceInfo.ModTime.Format(time.RFC3339Nano),
|
||||
xhttp.MinIOSourceETag: sourceInfo.ETag,
|
||||
xhttp.MinIOReplicationActualObjectSize: strconv.FormatInt(actualSize, 10),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replCompleteRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(replCompleteRec, replCompleteReq)
|
||||
if replCompleteRec.Code != http.StatusOK {
|
||||
t.Fatalf("replica Complete status %d: %s", replCompleteRec.Code, replCompleteRec.Body.String())
|
||||
}
|
||||
|
||||
info, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportedActual, aerr := info.GetActualSize()
|
||||
t.Logf("replica mpu: stored size=%d compression=%q actual-size=%q GetActualSize=%d(err=%v)",
|
||||
info.Size, info.UserDefined[ReservedMetadataPrefix+"compression"],
|
||||
info.UserDefined[ReservedMetadataPrefix+"actual-size"], reportedActual, aerr)
|
||||
assertSSECPlaintext(t, apiRouter, credentials, bucketName, object, sseHeaders, data)
|
||||
})
|
||||
}
|
||||
|
||||
// TestAPISSECCompressionProducerMatrix pins the scope of the exclusion across
|
||||
// the PutObject and NewMultipartUpload producers: SSE-C is never compressed,
|
||||
// while plaintext, SSE-S3 and SSE-KMS keep following allow_encryption.
|
||||
func TestAPISSECCompressionProducerMatrix(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPISSECCompressionProducerMatrix,
|
||||
})
|
||||
}
|
||||
|
||||
func testAPISSECCompressionProducerMatrix(obj ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
previousTLS := globalIsTLS
|
||||
globalIsTLS = true
|
||||
defer func() { globalIsTLS = previousTLS }()
|
||||
|
||||
ssecHeaders := ssecTestHeaders(0x5a)
|
||||
sseS3Headers := map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES}
|
||||
sseKMSHeaders := map[string]string{
|
||||
xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionKMS,
|
||||
xhttp.AmzServerSideEncryptionKmsID: "compressed-ssec-producer-matrix",
|
||||
}
|
||||
previousKMS := GlobalKMS
|
||||
GlobalKMS = kms.NewStub("compressed-ssec-producer-matrix")
|
||||
defer func() { GlobalKMS = previousKMS }()
|
||||
|
||||
big := bytes.Repeat([]byte("silo producer matrix payload "), 8192)
|
||||
small := bytes.Repeat([]byte("s"), 1024) // below minCompressibleSize
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
allowEncrypted bool
|
||||
headers map[string]string
|
||||
body []byte
|
||||
wantCompressed bool
|
||||
}{
|
||||
// SSE-C is excluded from compression in both configurations, because the
|
||||
// replication wire cannot carry the compression state.
|
||||
{"ssec+allow_encryption-on+large", true, ssecHeaders, big, false},
|
||||
{"ssec+allow_encryption-on+small", true, ssecHeaders, small, false},
|
||||
{"ssec+allow_encryption-off+large", false, ssecHeaders, big, false},
|
||||
// Plaintext still compresses in both configurations.
|
||||
{"plain+allow_encryption-on+large", true, nil, big, true},
|
||||
{"plain+allow_encryption-off+large", false, nil, big, true},
|
||||
// SSE-S3 and SSE-KMS are the reason allow_encryption exists: the server
|
||||
// owns the key, so the source decompresses before replicating.
|
||||
{"sse-s3+allow_encryption-on+large", true, sseS3Headers, big, true},
|
||||
{"sse-s3+allow_encryption-off+large", false, sseS3Headers, big, false},
|
||||
{"sse-kms+allow_encryption-on+large", true, sseKMSHeaders, big, true},
|
||||
{"sse-kms+allow_encryption-off+large", false, sseKMSHeaders, big, false},
|
||||
} {
|
||||
t.Run(instanceType+"/put/"+tc.name, func(t *testing.T) {
|
||||
restore := setCopyChecksumCompression(tc.allowEncrypted)
|
||||
defer restore()
|
||||
object := "producer/" + tc.name + ".txt"
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, object),
|
||||
int64(len(tc.body)), bytes.NewReader(tc.body), credentials.AccessKey, credentials.SecretKey, tc.headers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("PUT status %d, want %d: %s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
info, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, compressed := info.UserDefined[ReservedMetadataPrefix+"compression"]
|
||||
if compressed != tc.wantCompressed {
|
||||
t.Errorf("compressed=%v, want %v (userDefined=%v)", compressed, tc.wantCompressed, info.UserDefined)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// NewMultipartUpload has no size gate, so the exclusion turns on the SSE-C
|
||||
// and allow_encryption combination alone.
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
allowEncrypted bool
|
||||
headers map[string]string
|
||||
wantCompressed bool
|
||||
}{
|
||||
{"ssec+allow_encryption-on", true, ssecHeaders, false},
|
||||
{"ssec+allow_encryption-off", false, ssecHeaders, false},
|
||||
{"plain+allow_encryption-on", true, nil, true},
|
||||
} {
|
||||
t.Run(instanceType+"/mpu/"+tc.name, func(t *testing.T) {
|
||||
restore := setCopyChecksumCompression(tc.allowEncrypted)
|
||||
defer restore()
|
||||
object := "producer/mpu-" + tc.name + ".txt"
|
||||
req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, tc.headers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("NewMultipartUpload status %d, want %d: %s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
var init InitiateMultipartUploadResponse
|
||||
if err = xmlDecoder(rec.Body, &init, int64(rec.Body.Len())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mi, err := obj.GetMultipartInfo(t.Context(), bucketName, object, init.UploadID, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, compressed := mi.UserDefined[ReservedMetadataPrefix+"compression"]
|
||||
if compressed != tc.wantCompressed {
|
||||
t.Errorf("upload compressed=%v, want %v", compressed, tc.wantCompressed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPISSECCompressionSkippedOnCopyObject covers the third producer:
|
||||
// CopyObjectHandler decides compression before it encrypts, so a copy with a
|
||||
// destination customer key and allow_encryption=on used to store a compressed
|
||||
// SSE-C object from a plaintext source. It also covers the reverse direction,
|
||||
// where only copy-source customer headers are present and compression must
|
||||
// still apply.
|
||||
func TestAPISSECCompressionSkippedOnCopyObject(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPISSECCompressionSkippedOnCopyObject,
|
||||
endpoints: []string{"CopyObject", "PutObject", "GetObject"},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPISSECCompressionSkippedOnCopyObject(obj ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
previousTLS := globalIsTLS
|
||||
globalIsTLS = true
|
||||
defer func() { globalIsTLS = previousTLS }()
|
||||
|
||||
ssecHeaders := ssecTestHeaders(0x7c)
|
||||
data := bytes.Repeat([]byte("copy object compressed ssec payload "), 8192)
|
||||
|
||||
restore := setCopyChecksumCompression(true)
|
||||
defer restore()
|
||||
|
||||
// An unencrypted source, stored compressed because compression is on. Only
|
||||
// the copy adds encryption, so only the copy can change the decision.
|
||||
src := "copysrc/plain.txt"
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, src),
|
||||
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("source PUT status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
dst := "copydst/ssec.txt"
|
||||
copyHeaders := map[string]string{"X-Amz-Copy-Source": SlashSeparator + bucketName + SlashSeparator + src}
|
||||
for k, v := range ssecHeaders {
|
||||
copyHeaders[k] = v
|
||||
}
|
||||
copyReq, err := newTestSignedRequestV4(http.MethodPut, getCopyObjectURL("", bucketName, dst),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, copyHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
copyRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(copyRec, copyReq)
|
||||
if copyRec.Code != http.StatusOK {
|
||||
t.Fatalf("CopyObject status %d: %s", copyRec.Code, copyRec.Body.String())
|
||||
}
|
||||
|
||||
assertStoredSSECUncompressed(t, obj, bucketName, dst)
|
||||
assertSSECPlaintext(t, apiRouter, credentials, bucketName, dst, ssecHeaders, data)
|
||||
|
||||
// The plaintext source is untouched by the copy and stays compressed.
|
||||
srcInfo, err := obj.GetObjectInfo(t.Context(), bucketName, src, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, compressed := srcInfo.UserDefined[ReservedMetadataPrefix+"compression"]; !compressed {
|
||||
t.Errorf("the plaintext copy source lost compression (userDefined=%v)", srcInfo.UserDefined)
|
||||
}
|
||||
|
||||
// The reverse direction: a copy-source customer key is not a destination
|
||||
// key, so copying the SSE-C object on to a plaintext destination still
|
||||
// compresses. crypto.SSEC.IsRequested ignores the copy-source headers.
|
||||
plain := "copydst/decrypted.txt"
|
||||
decryptHeaders := map[string]string{
|
||||
"X-Amz-Copy-Source": SlashSeparator + bucketName + SlashSeparator + dst,
|
||||
xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||
xhttp.AmzServerSideEncryptionCopyCustomerKey: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKey],
|
||||
xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKeyMD5],
|
||||
}
|
||||
decryptReq, err := newTestSignedRequestV4(http.MethodPut, getCopyObjectURL("", bucketName, plain),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, decryptHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
decryptRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(decryptRec, decryptReq)
|
||||
if decryptRec.Code != http.StatusOK {
|
||||
t.Fatalf("CopyObject to a plaintext destination status %d: %s", decryptRec.Code, decryptRec.Body.String())
|
||||
}
|
||||
plainInfo, err := obj.GetObjectInfo(t.Context(), bucketName, plain, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, sealed := plainInfo.UserDefined[crypto.MetaSealedKeySSEC]; sealed {
|
||||
t.Fatalf("%s is still SSE-C sealed, the fixture proves nothing", plain)
|
||||
}
|
||||
if _, compressed := plainInfo.UserDefined[ReservedMetadataPrefix+"compression"]; !compressed {
|
||||
t.Errorf("a copy carrying only copy-source SSE-C headers was not compressed (userDefined=%v)", plainInfo.UserDefined)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPISSECCompressionSkippedOnSnowballExtract covers the fourth producer:
|
||||
// PutObjectExtractHandler decides compression per entry before it encrypts, so
|
||||
// a tar extract carrying customer key headers used to store every entry as a
|
||||
// compressed SSE-C object.
|
||||
func TestAPISSECCompressionSkippedOnSnowballExtract(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPISSECCompressionSkippedOnSnowballExtract,
|
||||
})
|
||||
}
|
||||
|
||||
func testAPISSECCompressionSkippedOnSnowballExtract(obj ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
previousTLS := globalIsTLS
|
||||
globalIsTLS = true
|
||||
defer func() { globalIsTLS = previousTLS }()
|
||||
|
||||
entry := "extracted/entry.txt"
|
||||
payload := bytes.Repeat([]byte("snowball compressed ssec entry "), 4096)
|
||||
|
||||
var body bytes.Buffer
|
||||
tw := tar.NewWriter(&body)
|
||||
if err := tw.WriteHeader(&tar.Header{Name: entry, Mode: 0o600, Size: int64(len(payload))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tw.Write(payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
restore := setCopyChecksumCompression(true)
|
||||
defer restore()
|
||||
|
||||
ssecHeaders := ssecTestHeaders(0x2d)
|
||||
headers := map[string]string{xhttp.AmzSnowballExtract: "true"}
|
||||
for k, v := range ssecHeaders {
|
||||
headers[k] = v
|
||||
}
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, "snowball.tar"),
|
||||
int64(body.Len()), bytes.NewReader(body.Bytes()), credentials.AccessKey, credentials.SecretKey, headers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("snowball extract status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
assertStoredSSECUncompressed(t, obj, bucketName, entry)
|
||||
assertSSECPlaintext(t, apiRouter, credentials, bucketName, entry, ssecHeaders, payload)
|
||||
}
|
||||
|
||||
// TestSSECBatchReplicationCannotRead is the control for the corruption path:
|
||||
// batch replication reads without ReplicationRequest, so NoDecryption is never
|
||||
// set and a non-empty SSE-C source fails at read time. Batch replication
|
||||
// therefore cannot reach the replica shape in
|
||||
// TestAPISSECCompressionReplicaStaysReadable; it cannot replicate a non-empty
|
||||
// SSE-C object at all, compressed or not. A zero-byte object takes the reader
|
||||
// shortcut, whose key check passes without a customer key.
|
||||
func TestSSECBatchReplicationCannotRead(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testSSECBatchReplicationCannotRead,
|
||||
})
|
||||
}
|
||||
|
||||
func testSSECBatchReplicationCannotRead(obj ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
previousTLS := globalIsTLS
|
||||
globalIsTLS = true
|
||||
defer func() { globalIsTLS = previousTLS }()
|
||||
|
||||
ssecHeaders := ssecTestHeaders(0x6b)
|
||||
data := bytes.Repeat([]byte("batch ssec payload "), 8192)
|
||||
object := "batch/ssec-plain.txt"
|
||||
|
||||
restore := disableCompression()
|
||||
defer restore()
|
||||
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, object),
|
||||
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, ssecHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("source PUT status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// The read shape used by BatchJobReplicateV1.ReplicateToTarget and
|
||||
// writeAsArchive: no ReplicationRequest, so NoDecryption is never set.
|
||||
gr, err := obj.GetObjectNInfo(t.Context(), bucketName, object, nil, http.Header{}, ObjectOptions{})
|
||||
if err == nil {
|
||||
gr.Close()
|
||||
t.Fatal("batch-shaped read of an SSE-C object unexpectedly succeeded")
|
||||
}
|
||||
t.Logf("batch-shaped read of an SSE-C object fails as expected: %v", err)
|
||||
|
||||
// Control: the replication worker's read shape succeeds and yields ciphertext.
|
||||
gr2, err := obj.GetObjectNInfo(t.Context(), bucketName, object, nil, http.Header{},
|
||||
ObjectOptions{ReplicationRequest: true})
|
||||
if err != nil {
|
||||
t.Fatalf("replication-shaped read failed: %v", err)
|
||||
}
|
||||
raw, err := io.ReadAll(gr2)
|
||||
gr2.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bytes.Equal(raw, data) {
|
||||
t.Fatal("replication-shaped read returned plaintext")
|
||||
}
|
||||
t.Logf("replication-shaped read returns %d bytes of ciphertext (plaintext %d)", len(raw), len(data))
|
||||
}
|
||||
@@ -52,7 +52,7 @@ import (
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
)
|
||||
|
||||
func initHelp() {
|
||||
|
||||
@@ -33,8 +33,8 @@ import (
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
"github.com/minio/minio/internal/event/target"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
"github.com/minio/pkg/v3/quick"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
"github.com/pgsty/silo-pkg/v3/quick"
|
||||
)
|
||||
|
||||
// Save config file to corresponding backend
|
||||
@@ -167,7 +167,9 @@ func readConfigWithoutMigrate(ctx context.Context, objAPI ObjectLayer) (config.C
|
||||
notify.SetNotifyMQTT(newCfg, k, args)
|
||||
}
|
||||
for k, args := range cfg.Notify.MySQL {
|
||||
notify.SetNotifyMySQL(newCfg, k, args)
|
||||
if err := notify.SetNotifyMySQL(newCfg, k, args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
for k, args := range cfg.Notify.NATS {
|
||||
notify.SetNotifyNATS(newCfg, k, args)
|
||||
@@ -176,7 +178,9 @@ func readConfigWithoutMigrate(ctx context.Context, objAPI ObjectLayer) (config.C
|
||||
notify.SetNotifyNSQ(newCfg, k, args)
|
||||
}
|
||||
for k, args := range cfg.Notify.PostgreSQL {
|
||||
notify.SetNotifyPostgres(newCfg, k, args)
|
||||
if err := notify.SetNotifyPostgres(newCfg, k, args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
for k, args := range cfg.Notify.Redis {
|
||||
notify.SetNotifyRedis(newCfg, k, args)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user