mirror of
https://github.com/pgsty/minio.git
synced 2026-10-01 23:35:59 +03:00
Compare commits
319 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e791640dac | |||
| 9b4ae82a29 | |||
| 4620be394b | |||
| 5e7d603083 | |||
| fb7c406ddc | |||
| 416826f61f | |||
| a2e2f3ee82 | |||
| 70c7ec4a9f | |||
| 2b722b7e92 | |||
| 40220bd836 | |||
| df0dfa0a34 | |||
| 80684fed59 | |||
| 055030ea53 | |||
| aea3882c95 | |||
| 0c61128d23 | |||
| 680eac66e4 | |||
| 9f3037e941 | |||
| 5f00f6762f | |||
| af2b1794d3 | |||
| d371f77dcc | |||
| 022722a7a7 | |||
| 03027727d1 | |||
| 4fcdf37ce6 | |||
| 9ebe81c1b3 | |||
| e5f5c9e7f6 | |||
| 7b4cacc392 | |||
| a0dd7dae9b | |||
| 709d50a916 | |||
| dff81f293b | |||
| f653a6ea03 | |||
| 47d239f84f | |||
| e069fe9d92 | |||
| d848fb52b5 | |||
| 3ce8319251 | |||
| 9df0f4abaf | |||
| 4d0693cb8c | |||
| bf59e3f222 | |||
| 41aa846097 | |||
| 4093fa0d78 | |||
| 13bf126ebd | |||
| 1cf529ce8a | |||
| 9b76a21675 | |||
| 89637554d6 | |||
| 2dd1e00da4 | |||
| 5d955b5b74 | |||
| f7808a172c | |||
| acc9b514f5 | |||
| 31ba01c5d5 | |||
| 48ec10312f | |||
| 114dc10529 | |||
| 461e9a7210 | |||
| fcbb93e895 | |||
| 62cf066ff5 | |||
| 1ee64a8d89 | |||
| 01aaef2b50 | |||
| bcc62afe3d | |||
| 5c57658163 | |||
| f175e98c34 | |||
| 12f631b502 | |||
| ccb676e60c | |||
| 51d41345f7 | |||
| e59a3d938e | |||
| b32f2d9dd0 | |||
| d63c92e393 | |||
| 68127c5a63 | |||
| c4b5e1cb45 | |||
| 9a303f5096 | |||
| 87d8b5967f | |||
| f760046c44 | |||
| 93e7ef4bcc | |||
| a4229b366f | |||
| 420340bc14 | |||
| e7e87402ed | |||
| a164e1dda1 | |||
| 2f61325d4a | |||
| a6145e1d2e | |||
| 5232546690 | |||
| 0c46cb641b | |||
| 1233254309 | |||
| 8a2fe9b7a0 | |||
| f26ee6bf0a | |||
| d69c4ccfe4 | |||
| 086e619505 | |||
| 48e1846525 | |||
| bcc8871b1d | |||
| 25cb3511c9 | |||
| cfefc049c1 | |||
| af56d17630 | |||
| d1105bbb3d | |||
| 66fe61ff65 | |||
| a1141a43f2 | |||
| 702f113f51 | |||
| 63aace4099 | |||
| 9d7094b770 | |||
| 450dcb8484 | |||
| 4074d00b96 | |||
| 75ba0ce402 | |||
| da142327f2 | |||
| a3df317ae0 | |||
| 2c50d11f72 | |||
| 5ac33e1583 | |||
| d57c4e8407 | |||
| 374de0fa32 | |||
| 80e23dc9f2 | |||
| 2cc0e3c6ed | |||
| f9da3b919d | |||
| 1309853f57 | |||
| 39b8e6c30a | |||
| 9a6e1477f4 | |||
| 49375ed2d3 | |||
| f817b5261c | |||
| 885bd2c20a | |||
| 89b75e7913 | |||
| 079ebb1926 | |||
| 04c29aac11 | |||
| 95e7a190b3 | |||
| 8e2392e48f | |||
| 3abe0d95a5 | |||
| 9c6c9805de | |||
| 5cb900bfad | |||
| 0af5d22286 | |||
| f1687f402b | |||
| ce606df2c4 | |||
| fd44dc4e9b | |||
| 479745e764 | |||
| cc1c54475f | |||
| e7654d470c | |||
| 4b25f7e819 | |||
| 711b092f86 | |||
| 2bc103b80c | |||
| ad873c7357 | |||
| 0af0907eff | |||
| e27ba2bc14 | |||
| 236e163c0b | |||
| 7220210e8d | |||
| 34cbca97ea | |||
| 109d824e5f | |||
| 87746913fc | |||
| 7935c84f9a | |||
| c185635b43 | |||
| c201148738 | |||
| 53adb21c52 | |||
| 58b0ee36ca | |||
| 8a1f594add | |||
| 5b9959617e | |||
| da19d91b64 | |||
| 40bee4b7ba | |||
| 6a9b5d6763 | |||
| 0720ed477e | |||
| 46e82eb54d | |||
| f8ca4a8656 | |||
| e7d0e62f16 | |||
| aee290fc34 | |||
| 62cce2b152 | |||
| 65d4806a7b | |||
| 765757473a | |||
| 425bd7fff1 | |||
| e12e739a53 | |||
| 8b736dee34 | |||
| 32e75c27bf | |||
| 885ca604a1 | |||
| d10382d0dc | |||
| 0db4bf3b00 | |||
| 87c621965d | |||
| b2dca43fda | |||
| 33a91d972f | |||
| 2cbd48a3c3 | |||
| b5409ca112 | |||
| 35bd75948a | |||
| 0c8d74205b | |||
| fcc4d77895 | |||
| c52acc1a5d | |||
| 5703426b3c | |||
| f0bd164b92 | |||
| 9936a69d89 | |||
| ce2326c946 | |||
| 4c164907f5 | |||
| 9b11dc9469 | |||
| 1eccc6908b | |||
| 202371afbd | |||
| 41ef4411d9 | |||
| 035aa6c201 | |||
| edf36bcbfa | |||
| 3598c4305d | |||
| 94fbb6df6d | |||
| f4c1286c9d | |||
| 6e112d1856 | |||
| e62dc6e023 | |||
| 84e1580a47 | |||
| ebac0ca73b | |||
| 3f9c79e919 | |||
| ec2979ca48 | |||
| 21646eebd2 | |||
| 5711996231 | |||
| 0079723d35 | |||
| bc3b35f975 | |||
| 00d864ed0e | |||
| 8f2a30d9af | |||
| d9766d7378 | |||
| 5594d284fc | |||
| 632eb4729a | |||
| 3b5de82f5a | |||
| 76195f1c68 | |||
| 6586fbfd0d | |||
| 53b09f4e25 | |||
| 59812446ad | |||
| 1c2f59604d | |||
| b3a6d5dbf8 | |||
| 11de51ec26 | |||
| 10f3a8590b | |||
| 55c22abec1 | |||
| 32a1b81e4c | |||
| 62d8c649fe | |||
| 312397739e | |||
| 7ade0c045b | |||
| f9f9fa6c90 | |||
| 0a9c777795 | |||
| d2d47a41fb | |||
| dee2c3a02b | |||
| c4140609b7 | |||
| 75a6734e49 | |||
| f8b598f1d3 | |||
| fe9dd90b22 | |||
| 04b097fd9f | |||
| ab3ae99ca3 | |||
| 5db7be4ee4 | |||
| ff44527a3c | |||
| c9ad746732 | |||
| f3438b2602 | |||
| 938603458d | |||
| d5e763b072 | |||
| 74c97d005d | |||
| 1be684ba29 | |||
| 1367364c40 | |||
| 5f44fe369c | |||
| c6cc136833 | |||
| 97e31af331 | |||
| 0507c3d56a | |||
| 87b9895bc0 | |||
| ca1f11b708 | |||
| 514a041f4a | |||
| f8106cd693 | |||
| b1ba685ac5 | |||
| bc7658f035 | |||
| f377b5c95b | |||
| 45381222b7 | |||
| 4d6e1ea8ea | |||
| 20aaefa89c | |||
| 82509ddc0a | |||
| f2ba439022 | |||
| 2ba1f38503 | |||
| 8d58343263 | |||
| 2b2e0d2a5d | |||
| e407dc58eb | |||
| 32b2aa49f1 | |||
| 8448512a1f | |||
| 150e7b5f9e | |||
| 21870fa2e7 | |||
| d4c8da162b | |||
| fc7bf7b295 | |||
| d28885d0e5 | |||
| c4fd97d0bf | |||
| b73581b05d | |||
| 474cd5801e | |||
| ee9252a608 | |||
| fb406fdc94 | |||
| dd3bdb8086 | |||
| 7c103389f5 | |||
| 3861f33cba | |||
| 07d92db523 | |||
| 5732930102 | |||
| 2aea7fe9c4 | |||
| ffb70eb373 | |||
| e73436c99d | |||
| 0b0ae2423a | |||
| 7e079ff05c | |||
| 229fe2b3c3 | |||
| 38ed9d1e1f | |||
| 47cd7807d3 | |||
| 04d3d316d2 | |||
| 4bb8c813ac | |||
| b6ef7e430c | |||
| 91d9091758 | |||
| 0eebc928f7 | |||
| 724f8703d8 | |||
| e4e3007da6 | |||
| 13e6458d90 | |||
| 590aeaa7d1 | |||
| 5d152416de | |||
| edc8be6ed1 | |||
| 49c8aeac40 | |||
| e9c5340be9 | |||
| 2e2377d1c6 | |||
| 58735ee382 | |||
| a96116b128 | |||
| c4b9d38d8a | |||
| 8d76a255c4 | |||
| c5bc57b7a3 | |||
| 3814818537 | |||
| 7a49a7a3da | |||
| ff3395d3c6 | |||
| ce4525632f | |||
| 1c9a2431fe | |||
| f2520f3346 | |||
| 05df6e70d7 | |||
| c0e7159771 | |||
| 56c67dacf1 | |||
| eee05a17c3 | |||
| 45eb2e423d | |||
| 6b0998157c | |||
| 7fea6d5a5f | |||
| 68eeb002f6 | |||
| c565987b9c | |||
| f1ba683582 | |||
| 43f4bb7ed4 | |||
| 7a060cab1e | |||
| d014a12cff | |||
| 100e2e57a7 | |||
| 6e20e74774 |
@@ -1,54 +0,0 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: community, triage
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
## NOTE
|
||||
|
||||
Silo issues are handled by community maintainers on a best-effort basis. There
|
||||
is no SLA, SLO, or emergency production-support channel. Follow the local
|
||||
[Code of Conduct](../code_of_conduct.md) when participating. Report suspected
|
||||
vulnerabilities through the private process in [SECURITY.md](../SECURITY.md),
|
||||
not in a public issue.
|
||||
|
||||
<!--- Provide a general summary of the issue in the Title above -->
|
||||
|
||||
## Expected Behavior
|
||||
<!--- If you're describing a bug, tell us what should happen -->
|
||||
<!--- If you're suggesting a change/improvement, tell us how it should work -->
|
||||
|
||||
## Current Behavior
|
||||
<!--- If describing a bug, tell us what happens instead of the expected behavior -->
|
||||
<!--- If suggesting a change/improvement, explain the difference from current behavior -->
|
||||
|
||||
## Possible Solution
|
||||
<!--- Not obligatory, but suggest a fix/reason for the bug, -->
|
||||
<!--- or ideas how to implement the addition or change -->
|
||||
|
||||
## Steps to Reproduce (for bugs)
|
||||
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
|
||||
<!--- reproduce this bug. Include code to reproduce, if relevant -->
|
||||
<!--- and include relevant Silo logs with secrets and credentials removed -->
|
||||
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
4.
|
||||
|
||||
## Context
|
||||
<!--- How has this issue affected you? What are you trying to accomplish? -->
|
||||
<!--- Providing context helps us come up with a solution that is most useful in the real world -->
|
||||
|
||||
## Regression
|
||||
<!-- Is this issue a regression? (Yes / No) -->
|
||||
<!-- If Yes, optionally include the Silo version, commit id, or PR that caused this regression. -->
|
||||
|
||||
## Your Environment
|
||||
<!--- Include as many relevant details about the environment you experienced the bug in -->
|
||||
* Version used (`silo --version`):
|
||||
* Server setup and configuration:
|
||||
* Operating System and version (`uname -a`):
|
||||
@@ -7,6 +7,14 @@ assignees: ''
|
||||
|
||||
---
|
||||
|
||||
Report bugs in the PGSTY SILO server (`pgsty/silo`) here. Community maintainers
|
||||
handle reports on a best-effort basis. There is no SLA, SLO, or emergency
|
||||
production-support channel. Follow the
|
||||
[Code of Conduct](https://github.com/pgsty/silo/blob/main/code_of_conduct.md).
|
||||
Report suspected vulnerabilities privately through
|
||||
[SECURITY.md](https://github.com/pgsty/silo/blob/main/SECURITY.md).
|
||||
For patches, see the [contribution guide](https://github.com/pgsty/silo/blob/main/CONTRIBUTING.md).
|
||||
|
||||
<!--- Provide a general summary of the issue in the Title above -->
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
@@ -7,6 +7,9 @@ assignees: ''
|
||||
|
||||
---
|
||||
|
||||
Suggest improvements to the PGSTY SILO server (`pgsty/silo`) here.
|
||||
For patches, see the [contribution guide](https://github.com/pgsty/silo/blob/main/CONTRIBUTING.md).
|
||||
|
||||
**Is your feature request related to a problem? Please describe.**
|
||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
||||
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
## Contribution Licensing (no CLA, inbound=outbound, DCO required)
|
||||
|
||||
This project does not use a CLA; contributions are accepted inbound=outbound.
|
||||
This pull request contributes to PGSTY SILO (`pgsty/silo`). Code contributions
|
||||
are accepted under AGPL-3.0-or-later, the same license as the server.
|
||||
This project does not use a CLA or require a separate Apache-2.0 license grant.
|
||||
By submitting this pull request I represent that I have the right to contribute
|
||||
the changes, which are licensed under this repository's
|
||||
the code changes under this repository's
|
||||
[GNU Affero General Public License v3.0 or later](https://www.gnu.org/licenses/agpl-3.0.html)
|
||||
and remain my copyright. Every commit must carry a DCO `Signed-off-by` trailer
|
||||
and retain copyright in my original work. Existing copyright and license
|
||||
notices remain intact; separately licensed material keeps its applicable terms.
|
||||
Every commit must carry a DCO `Signed-off-by` trailer
|
||||
(`git commit -s`) certifying the
|
||||
[Developer Certificate of Origin](https://developercertificate.org/) — see
|
||||
[CONTRIBUTING.md](https://github.com/pgsty/silo/blob/main/CONTRIBUTING.md).
|
||||
|
||||
@@ -77,8 +77,10 @@ release:
|
||||
name: silo
|
||||
draft: true
|
||||
prerelease: false
|
||||
mode: append
|
||||
replace_existing_draft: true
|
||||
replace_existing_artifacts: false
|
||||
mode: replace
|
||||
# Draft replacement matches the release name; keep it identical to the tag.
|
||||
name_template: "{{ .Tag }}"
|
||||
|
||||
changelog:
|
||||
|
||||
@@ -7,6 +7,11 @@ on:
|
||||
description: "Published RELEASE.* tag to package as pgsty/silo"
|
||||
required: true
|
||||
type: string
|
||||
recovery:
|
||||
description: "Run the current main workflow against an already-published tag"
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -75,12 +80,18 @@ jobs:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Verify workflow identity matches release source
|
||||
env:
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
RECOVERY: ${{ inputs.recovery }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
CHECKED_OUT_REVISION="$(git rev-parse HEAD)"
|
||||
if [ "${CHECKED_OUT_REVISION}" != "${GITHUB_SHA}" ]; then
|
||||
echo "Checked out ${CHECKED_OUT_REVISION}, but workflow identity is ${GITHUB_SHA}. Dispatch this workflow from ${RELEASE_TAG}." >&2
|
||||
exit 1
|
||||
if [ "${RECOVERY}" != "true" ] || [ "${GITHUB_REF}" != "refs/heads/${DEFAULT_BRANCH}" ]; then
|
||||
echo "Checked out ${CHECKED_OUT_REVISION}, but workflow identity is ${GITHUB_SHA}. Dispatch from ${RELEASE_TAG}, or use recovery from ${DEFAULT_BRANCH}." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Recovery workflow ${GITHUB_SHA} is packaging published source ${CHECKED_OUT_REVISION}."
|
||||
fi
|
||||
|
||||
- name: Prepare verified Docker contexts
|
||||
@@ -129,10 +140,50 @@ jobs:
|
||||
mkdir -p "${context}/dockerscripts"
|
||||
tar -xzf "${archive}" -C "${context}" silo
|
||||
cp Dockerfile.goreleaser Dockerfile.distroless LICENSE NOTICE CREDITS "${context}/"
|
||||
cp dockerscripts/docker-entrypoint.sh dockerscripts/download-static-curl.sh \
|
||||
cp dockerscripts/docker-entrypoint.sh dockerscripts/build-static-curl.sh \
|
||||
"${context}/dockerscripts/"
|
||||
done
|
||||
|
||||
# The classic image bundles mcli. Resolve its two archive digests
|
||||
# from the immutable published release instead of trusting defaults
|
||||
# copied into an older Server tag. This also gives a recovery run a
|
||||
# narrow override when a tag selected the right mcli release but
|
||||
# accidentally retained stale archive pins.
|
||||
MC_REPO="$(awk -F= '/^ARG MC_REPO=/{print $2; exit}' Dockerfile.goreleaser)"
|
||||
MC_VERSION="$(awk -F= '/^ARG MC_VERSION=/{print $2; exit}' Dockerfile.goreleaser)"
|
||||
test -n "${MC_REPO}"
|
||||
test -n "${MC_VERSION}"
|
||||
MC_VERSION_HYPHEN="${MC_VERSION#RELEASE.}"
|
||||
MC_PKG_VERSION="$(echo "${MC_VERSION_HYPHEN}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')"
|
||||
if [ "${MC_PKG_VERSION}" = "${MC_VERSION_HYPHEN}" ]; then
|
||||
echo "Invalid bundled mcli tag: ${MC_VERSION}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$(gh release view "${MC_VERSION}" --repo "${MC_REPO}" --json isDraft --jq .isDraft)" != false ] || \
|
||||
[ "$(gh release view "${MC_VERSION}" --repo "${MC_REPO}" --json isPrerelease --jq .isPrerelease)" != false ] || \
|
||||
[ "$(gh release view "${MC_VERSION}" --repo "${MC_REPO}" --json isImmutable --jq .isImmutable)" != true ]; then
|
||||
echo "Bundled mcli ${MC_REPO}@${MC_VERSION} must be a published immutable release" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mc_checksums="mcli_${MC_PKG_VERSION}_checksums.txt"
|
||||
gh release download "${MC_VERSION}" --repo "${MC_REPO}" \
|
||||
--dir "${assets_dir}" --pattern "${mc_checksums}"
|
||||
gh attestation verify "${assets_dir}/${mc_checksums}" \
|
||||
--repo "${MC_REPO}" \
|
||||
--signer-workflow "${MC_REPO}/.github/workflows/release.yml" \
|
||||
--source-ref "refs/tags/${MC_VERSION}" >/dev/null
|
||||
|
||||
MC_AMD64_SHA256="$(awk -v name="mcli_${MC_PKG_VERSION}_linux_amd64.tar.gz" '$2 == name {print $1}' "${assets_dir}/${mc_checksums}")"
|
||||
MC_ARM64_SHA256="$(awk -v name="mcli_${MC_PKG_VERSION}_linux_arm64.tar.gz" '$2 == name {print $1}' "${assets_dir}/${mc_checksums}")"
|
||||
[[ "${MC_AMD64_SHA256}" =~ ^[0-9a-f]{64}$ ]]
|
||||
[[ "${MC_ARM64_SHA256}" =~ ^[0-9a-f]{64}$ ]]
|
||||
|
||||
{
|
||||
echo "MC_AMD64_SHA256=${MC_AMD64_SHA256}"
|
||||
echo "MC_ARM64_SHA256=${MC_ARM64_SHA256}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
echo "RELEASE_REVISION=$(git rev-parse HEAD)" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Set up QEMU
|
||||
@@ -162,6 +213,9 @@ jobs:
|
||||
file: docker-release/amd64/Dockerfile.goreleaser
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
build-args: |
|
||||
MC_AMD64_SHA256=${{ env.MC_AMD64_SHA256 }}
|
||||
MC_ARM64_SHA256=${{ env.MC_ARM64_SHA256 }}
|
||||
tags: |
|
||||
pgsty/silo:${{ env.RELEASE_TAG }}-amd64
|
||||
pgsty/silo:latest-amd64
|
||||
@@ -178,6 +232,9 @@ jobs:
|
||||
file: docker-release/arm64/Dockerfile.goreleaser
|
||||
platforms: linux/arm64
|
||||
push: true
|
||||
build-args: |
|
||||
MC_AMD64_SHA256=${{ env.MC_AMD64_SHA256 }}
|
||||
MC_ARM64_SHA256=${{ env.MC_ARM64_SHA256 }}
|
||||
tags: |
|
||||
pgsty/silo:${{ env.RELEASE_TAG }}-arm64
|
||||
pgsty/silo:latest-arm64
|
||||
|
||||
@@ -23,7 +23,7 @@ permissions:
|
||||
artifact-metadata: write
|
||||
|
||||
concurrency:
|
||||
group: finalize-release
|
||||
group: release-${{ inputs.tag }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
|
||||
@@ -90,6 +90,9 @@ jobs:
|
||||
- name: Run S3 Select tests under race detector
|
||||
run: go test -race ./internal/s3select/... -count=1
|
||||
|
||||
- name: Run conditional PUT tests under race detector
|
||||
run: go test -race ./cmd -run '^Test(PoolsConditionalPut|SinglePoolConditionalPutHTTP)' -count=1 -timeout=5m
|
||||
|
||||
crosscompile:
|
||||
name: Cross Compile
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
name: Release
|
||||
|
||||
# Retry contract: an absent or single unfinalized Draft may be rebuilt from
|
||||
# scratch; a published release or a Draft carrying finalize's GPG-derived
|
||||
# provenance marker is terminal for this lane. The per-tag lock serializes
|
||||
# workflows, but a maintainer must not publish the Draft while this job runs.
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
@@ -16,6 +21,10 @@ permissions:
|
||||
attestations: write
|
||||
artifact-metadata: write
|
||||
|
||||
concurrency:
|
||||
group: release-${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -78,6 +87,15 @@ jobs:
|
||||
echo "Invalid release tag format: ${TAG}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! TAG_COMMIT="$(git rev-parse "${TAG}^{commit}" 2>/dev/null)"; then
|
||||
echo "Release tag ${TAG} does not resolve to a commit" >&2
|
||||
exit 1
|
||||
fi
|
||||
HEAD_COMMIT="$(git rev-parse HEAD)"
|
||||
if [ "${TAG_COMMIT}" != "${HEAD_COMMIT}" ]; then
|
||||
echo "Release tag ${TAG} resolves to ${TAG_COMMIT}, checkout is ${HEAD_COMMIT}" >&2
|
||||
exit 1
|
||||
fi
|
||||
VERSION_HYPHEN="${TAG#RELEASE.}"
|
||||
PKG_VERSION="$(echo "${VERSION_HYPHEN}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')"
|
||||
VERSION_COLON="$(echo "${VERSION_HYPHEN}" | sed -E 's/T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/T\1:\2:\3Z/')"
|
||||
@@ -93,6 +111,13 @@ jobs:
|
||||
echo "Package version: ${PKG_VERSION}"
|
||||
echo "LDFLAGS: ${LDFLAGS}"
|
||||
|
||||
- name: Check existing release state
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
buildscripts/check-release-state.sh "${RELEASE_TAG}"
|
||||
|
||||
# Both installer actions are pinned to immutable commits. The explicit
|
||||
# tool versions keep the release format reproducible across workflow
|
||||
# reruns while the installers verify the downloaded executables.
|
||||
@@ -113,6 +138,7 @@ jobs:
|
||||
args: release --clean --skip=validate --config .github/goreleaser.yml
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GORELEASER_CURRENT_TAG: ${{ env.RELEASE_TAG }}
|
||||
LDFLAGS: ${{ env.LDFLAGS }}
|
||||
PKG_VERSION: ${{ env.PKG_VERSION }}
|
||||
|
||||
@@ -189,6 +215,14 @@ jobs:
|
||||
test -s "${BUNDLE_PATH}"
|
||||
cp "${BUNDLE_PATH}" "dist/silo_${PKG_VERSION}_provenance.sigstore.json"
|
||||
|
||||
- name: Confirm unfinalized Draft release state
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REQUIRE_DRAFT: "true"
|
||||
run: |
|
||||
set -euo pipefail
|
||||
buildscripts/check-release-state.sh "${RELEASE_TAG}"
|
||||
|
||||
- name: Upload nFPM packages to Draft release
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -10,7 +10,7 @@ on:
|
||||
- "Dockerfile.distroless"
|
||||
- "cmd/healthcheck-main.go"
|
||||
- "cmd/main.go"
|
||||
- "dockerscripts/download-static-curl.sh"
|
||||
- "dockerscripts/build-static-curl.sh"
|
||||
- "dockerscripts/docker-entrypoint.sh"
|
||||
- "dockerscripts/docker-entrypoint_test.sh"
|
||||
- "silo.service"
|
||||
@@ -23,6 +23,8 @@ on:
|
||||
- "buildscripts/minio-upgrade.sh"
|
||||
- "buildscripts/sign-release-rpms.sh"
|
||||
- "buildscripts/verify-build-provenance.sh"
|
||||
- "buildscripts/check-release-state.sh"
|
||||
- "buildscripts/check-release-state_test.sh"
|
||||
- "buildscripts/verify-rebrand.sh"
|
||||
- "buildscripts/verify-helm-migration.sh"
|
||||
- "buildscripts/helm-migration-guard/**"
|
||||
@@ -39,6 +41,28 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
curl:
|
||||
name: Static curl (${{ matrix.arch }})
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- arch: amd64
|
||||
runner: ubuntu-latest
|
||||
- arch: arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- name: Build and exercise curl in an empty runtime
|
||||
run: |
|
||||
docker build --build-arg TARGETARCH=${{ matrix.arch }} \
|
||||
--target curl-runtime -f Dockerfile.goreleaser -t silo-curl-test .
|
||||
docker run --rm silo-curl-test --version | tee curl-version.txt
|
||||
grep -F 'curl 8.22.0 ' curl-version.txt
|
||||
grep -F 'HTTP2' curl-version.txt
|
||||
docker run --rm silo-curl-test --fail --silent --show-error \
|
||||
--connect-timeout 15 --max-time 60 https://curl.se/robots.txt
|
||||
|
||||
validate:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
@@ -479,6 +503,8 @@ jobs:
|
||||
bash -n buildscripts/minio-upgrade.sh
|
||||
bash -n buildscripts/sign-release-rpms.sh
|
||||
bash -n buildscripts/verify-build-provenance.sh
|
||||
bash -n buildscripts/check-release-state.sh
|
||||
bash -n buildscripts/check-release-state_test.sh
|
||||
bash -n buildscripts/verify-rebrand.sh
|
||||
bash -n buildscripts/verify-helm-migration.sh
|
||||
sh -n buildscripts/package/postinstall.sh
|
||||
@@ -486,16 +512,19 @@ jobs:
|
||||
bash -n buildscripts/package/lifecycle_test.sh
|
||||
buildscripts/package/lifecycle_test.sh
|
||||
bash -n dockerscripts/docker-entrypoint_test.sh
|
||||
bash -n dockerscripts/download-static-curl.sh
|
||||
bash -n dockerscripts/build-static-curl.sh
|
||||
dockerscripts/docker-entrypoint_test.sh
|
||||
go run ./buildscripts/rebrand-guard
|
||||
buildscripts/verify-rebrand.sh
|
||||
test -x buildscripts/package-release.sh
|
||||
test -x buildscripts/sign-release-rpms.sh
|
||||
test -x buildscripts/verify-build-provenance.sh
|
||||
test -x buildscripts/check-release-state.sh
|
||||
test -x buildscripts/check-release-state_test.sh
|
||||
test -x buildscripts/verify-rebrand.sh
|
||||
test -x buildscripts/verify-helm-migration.sh
|
||||
test -x buildscripts/package/postinstall.sh
|
||||
test -x buildscripts/package/preremove.sh
|
||||
test -x buildscripts/package/lifecycle_test.sh
|
||||
test -x dockerscripts/docker-entrypoint_test.sh
|
||||
buildscripts/check-release-state_test.sh
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
|
||||
- name: Install govulncheck
|
||||
run: |
|
||||
go install golang.org/x/vuln/cmd/govulncheck@v1.6.0
|
||||
go install golang.org/x/vuln/cmd/govulncheck@v1.8.0
|
||||
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Run govulncheck
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@ linters:
|
||||
- durationcheck
|
||||
- forcetypeassert
|
||||
- gocritic
|
||||
- gomodguard
|
||||
- gomodguard_v2
|
||||
- govet
|
||||
- ineffassign
|
||||
- misspell
|
||||
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
# Changelog
|
||||
|
||||
## Unreleased
|
||||
|
||||
The entries below describe source changes on main since the latest published Server.
|
||||
**The latest published Server remains 20260903.** These changes are not in its
|
||||
binaries, packages or images. See the [component matrix](https://silo.pgsty.com/compatibility/versions/)
|
||||
and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-09-03T13-18-01Z...main).
|
||||
|
||||
### Authorization and security
|
||||
|
||||
- Persist IAM deletion revisions and parent revocation boundaries so stale site
|
||||
events cannot restore deleted identities, policies or their older grants
|
||||
(#191, #192). Peer deletion notifications reload committed storage; deliberate
|
||||
recreation requires a newer revision, and credentials issued before the
|
||||
parent's revocation remain invalid.
|
||||
**Coordinated upgrade required:** upgrade every participating node and site.
|
||||
Mixed old/new nodes sharing an IAM backend and rolling downgrade are
|
||||
unsupported. Back up complete IAM storage and encryption material; an admin
|
||||
export of live records omits deletion history. Reissue credentials for
|
||||
recreated parents and explicitly reconcile pre-upgrade revocations whose
|
||||
history is already lost. Restoring an older backup can lose later revocations;
|
||||
keep affected sites isolated until reconciliation/rekeying is complete. See
|
||||
[the operator runbook](https://github.com/pgsty/silo.pgsty.com/blob/7bd2d57c2ce5aaa804d0b1a2fe0e5eed69d15235/content/operations/replication/iam-upgrade.md).
|
||||
- Enforce an absolute HTTP/1 request-header deadline through the connection
|
||||
wrapper (#196). Repeated small reads no longer extend that deadline, and
|
||||
`--read-header-timeout` / `MINIO_READ_HEADER_TIMEOUT` now reaches the HTTP
|
||||
server. HTTP/1 request bodies retain the rolling idle timeout; this does not
|
||||
impose a total upload/download duration. A shorter setting also constrains
|
||||
TLS handshake reads. The wrapper's strict header mode is not applied to HTTP/2.
|
||||
- Reject unsigned `x-amz-*` request headers that could turn a signed PUT into a
|
||||
copy of another object accessible to the signer (SN-2026-011). The latest
|
||||
public Server is affected; the fix is on main. See [the advisory ledger](docs/security/advisories.md).
|
||||
- Align signed request fields with policy conditions and enforce header-only
|
||||
presigned payload checksums. See [the signed-header review](https://silo.pgsty.com/blog/design/signed-header-coverage/).
|
||||
- **Breaking policy semantics:** separate self-service `admin:ChangeMyPassword`
|
||||
from `admin:CreateUser`. Built-in read-only policies follow the split. Preserve
|
||||
both denies if the previous combined restriction must survive upgrades or
|
||||
rollback. Saved policies are not rewritten. Deploy with the matching Console
|
||||
and pkg; see [the migration guide](docs/iam/password-permissions.md).
|
||||
|
||||
### Object storage and replication
|
||||
|
||||
- Preserve object tags during multi-pool metadata reconciliation by reading the
|
||||
resolved tag field together with its revision (#189). Previously, reconciliation
|
||||
could replace existing tags with an empty value.
|
||||
- Preserve the tag revision on SSE-KMS metadata replication (#193), and advance
|
||||
tag revisions monotonically on local PUT/DELETE tagging (#196). Empty tags
|
||||
participate in reconciliation as an ordered deletion, preventing older
|
||||
events from restoring removed tags. SSE-C key rotation also retains the tag
|
||||
revision. Malformed historical revisions can fail and retry; their missing
|
||||
history is not reconstructed by the upgrade.
|
||||
- Complete delete-marker version purges and preserve their identity and retry
|
||||
state through MRF recovery (#196). Recovery accepts a 405 marker response only
|
||||
when its version, bucket, object name and modification time match the task.
|
||||
Purge audit status is normalized from `COMPLETE` to `COMPLETED`.
|
||||
Thanks to Julien Laurenceau (@julienlau) for the investigation and proposed
|
||||
fix in #184 that helped shape this follow-up.
|
||||
- Restore only the six replication-specific metadata fields after ordinary
|
||||
request metadata extraction (#194). This prevents transport-only `aws-chunked`
|
||||
from being stored as Content-Encoding while preserving the signed-header
|
||||
protections. Trusted Snowball entries no longer inherit the outer archive's
|
||||
ordinary metadata. Thanks to Mikhail Khadarenka (@chodorenko) for the fix in #187.
|
||||
**Existing data:** these repairs prevent new errors; they do not scan or rewrite
|
||||
historical object metadata, recover lost tags or prove that old purge work has
|
||||
converged. Follow the [read-only audit procedure](https://github.com/pgsty/silo.pgsty.com/blob/7bd2d57c2ce5aaa804d0b1a2fe0e5eed69d15235/content/operations/replication/replica-metadata-audit.md)
|
||||
before planning any repair of stored state.
|
||||
|
||||
- Evaluate conditional multipart completion against the logical current object
|
||||
across all pools while holding the existing object lock. A stale `If-Match`
|
||||
can no longer replace newer data in another pool, and the current ETag is no
|
||||
longer rejected because the upload resides next to an older copy. Conditions
|
||||
are evaluated once; a current delete marker counts as an absent object.
|
||||
**Availability change:** if any pool's metadata cannot be read, conditional
|
||||
completion fails even when another pool can still serve GET/HEAD. This also
|
||||
applies when the unreadable pool may not hold the object: absence cannot be
|
||||
verified. Retry after the pool recovers. Unconditional completion and the
|
||||
single-pool path retain their existing behavior.
|
||||
|
||||
- Evaluate ordinary multi-pool conditional PUT against the logical current
|
||||
object across all pools, including draining pools, under the existing object
|
||||
lock (#207). A stale destination copy no longer accepts a stale ETag or rejects
|
||||
the current one; a current delete marker is treated as absence.
|
||||
**Availability change:** if any pool's object metadata cannot be verified,
|
||||
the condition fails even when GET can use another pool; read-quorum failures
|
||||
return 503. Restore readability or heal before retrying. Unconditional PUT,
|
||||
single-pool conditions and internal replication retain their existing behavior.
|
||||
A public condition with a destination `versionId` compares the current object
|
||||
while preserving the requested write version. This change does not retire
|
||||
stale copies in other pools, undo historical accepted overwrites or provide
|
||||
a new global clock-ordering guarantee. The multipart-completion repair in #190
|
||||
neither introduced nor repaired this separate PUT defect.
|
||||
|
||||
- Reconcile ordinary single-object version DELETE across all pools, including
|
||||
null versions, delete markers and unqualified directory-marker DELETE. This
|
||||
applies the deletion to every resolved pool copy under existing quorum
|
||||
rules. Pending outbound delete replication retains versions until the
|
||||
existing replication worker completes their purge; a successful response
|
||||
does not imply immediate physical removal from every drive. Unreadable
|
||||
pools now consistently return 503 instead of depending on pool traversal
|
||||
order; insufficient read quorum returns `SlowDownRead`. This extends the
|
||||
existing failure surface. Retry after recovery.
|
||||
Cleanup failures also return an error. Batch deletion already fans out across
|
||||
pools; replication and scanner cleanup keep their existing contracts. See
|
||||
[scope and limitations](docs/bucket/lifecycle/access-tiering-removal.md#version-deletion-scope).
|
||||
|
||||
- Remove the opt-in GET-frequency pool-tiering feature from PR #60, including
|
||||
its tracker, mover, scanner hooks, configuration, XML actions and metrics.
|
||||
Accept and ignore retired configuration/XML and preserve ordinary statistics
|
||||
when reading v9 caches. See [migration notes](docs/bucket/lifecycle/access-tiering-removal.md).
|
||||
The [decision record](docs/investigations/access-tiering-revert.md) preserves
|
||||
the feature's introduction, subsequent fixes, rollback scope and review history.
|
||||
- Preserve the independent multi-pool write, metadata, healing and conditional
|
||||
deletion fixes from PR #178, including shared remote-tier reference protection.
|
||||
- Enforce `If-Match` on DELETE, preserve retention and independently ordered
|
||||
Object Lock/tag updates, and correctly retransmit encrypted replicas.
|
||||
- Preserve plaintext part sizes and raw SSE-C replicas; prevent SSE-C
|
||||
compression, honor key-rotation checksums, and complete attributes pagination.
|
||||
- Repair federated CopyObject checksums, destination timestamps, reserved
|
||||
metadata, encrypted-object forwarding, legal hold and KMS context.
|
||||
- Make resync counters, target selection, cancellation and worker lifetimes
|
||||
reflect actual work, and report bounded MRF drops.
|
||||
- Converge bucket metadata with deterministic source state, deletion tombstones,
|
||||
creation time recovery and diagnostics. The mixed-version export gate requires
|
||||
coordinated upgrades before tombstones are exported. See [the #77 record](docs/investigations/issue-77-current.md).
|
||||
- Include per-bucket CORS in metadata export/import, close metadata publication
|
||||
and logger races, and report effective bucket quotas in metrics.
|
||||
|
||||
### Console, dependencies and delivery
|
||||
|
||||
- Restore embedded Console login over loopback TLS, trusted-proxy handling and
|
||||
all four WebSocket connection limits. Preserve Go TLS defaults across transports.
|
||||
- Directly require `github.com/pgsty/silo-pkg/v3` v3.14.0; select Console
|
||||
`v0.0.0-20260913015128-417559bb2c97` and MC
|
||||
`v0.0.0-20260913012246-4f609a4da3bb` with explicit PGSTY replacements.
|
||||
- Pin upstream minio-go `v7.3.1-0.20260910142817-60bd07042d49`; refresh Go x/*
|
||||
modules and security fixes including bounded AMQP frame handling. Keep Go
|
||||
1.27.1 and go-systemd v22.6.0's NetBSD compatibility replacement.
|
||||
- Refresh container base digests and build static curl 8.22.0 from verified
|
||||
source for both Linux architectures. Pin the actual mcli 20260913 archives and
|
||||
hashes. Helm's client image follows that release; its Server image still names
|
||||
the latest published Server 20260903.
|
||||
|
||||
The dependency update passed the final candidate's Go, vulnerability and Test
|
||||
Release workflows; native curl builds passed on both architectures. A local
|
||||
ARM64 image passed startup, health, S3 transfer and embedded Console checks.
|
||||
These checks do not publish a Server tag or production image and do not replace
|
||||
cluster upgrade/rollback acceptance for the next release. Dated investigations
|
||||
retain the exact source and runtime boundaries they tested.
|
||||
|
||||
## RELEASE.2026-09-03T13-18-01Z
|
||||
|
||||
Published source: `9b11dc9469e650815b775cb47b039610644f5da4`.
|
||||
[Complete release notes](https://silo.pgsty.com/blog/release/silo-20260903/) ·
|
||||
[GitHub release](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z)
|
||||
|
||||
This release ships Go 1.27.1, silo-pkg v3.13.2, upstream minio-go `0e78d3f18efe`,
|
||||
mcli 20260903 and embedded Console source `464a59d73ada` (v2.3.0 version identity).
|
||||
Installing the newer standalone mcli or Console does not replace components
|
||||
inside this existing Server binary or image.
|
||||
|
||||
Earlier releases: [release archive](https://github.com/pgsty/silo/releases).
|
||||
+18
-8
@@ -44,6 +44,10 @@ Before opening a pull request:
|
||||
|
||||
- Add or update tests for changed behavior.
|
||||
- Run `make verifiers`.
|
||||
- If `make rebrand-guard` reports a changed compatibility set, review the
|
||||
listed identifiers; when the change is intended, refresh the baseline with
|
||||
`go run ./buildscripts/rebrand-guard --write` and commit
|
||||
`buildscripts/rebrand-guard/compat-baseline.json`.
|
||||
- Run the smallest relevant package tests, then `make test` when practical.
|
||||
- Run `make build` and confirm the generated executable is `silo`.
|
||||
- Explain any preserved `MINIO_*`, `minio_*`, `x-minio-*`, `/minio/*`,
|
||||
@@ -75,9 +79,10 @@ documentation is owned by the separate
|
||||
|
||||
## Licensing of Contributions
|
||||
|
||||
Silo is licensed under the [GNU AGPL v3.0 or later](LICENSE). Its core is
|
||||
Copyright (c) MinIO, Inc.; the combined work can never be relicensed, and this
|
||||
fork does not try to.
|
||||
Code contributions to PGSTY SILO (`pgsty/silo`) are accepted under the
|
||||
[GNU AGPL v3.0 or later](LICENSE), the same license as the server. Submit issues
|
||||
and pull requests to this repository's maintainers. No separate Apache-2.0
|
||||
license grant to SILO or upstream MinIO maintainers is required.
|
||||
|
||||
* **No CLA.** We do not ask you to sign a Contributor License Agreement and we
|
||||
do not take your copyright. Contributions are accepted inbound=outbound: you
|
||||
@@ -108,15 +113,20 @@ fork does not try to.
|
||||
`Signed-off-by` trailers) and add your own sign-off as the person passing it
|
||||
along. Never import code from a proprietary distribution.
|
||||
|
||||
* **File headers.** Files derived from upstream keep the original MinIO
|
||||
copyright header unchanged. New files added by this fork use the dual
|
||||
header, followed by the standard AGPL boilerplate:
|
||||
* **File headers.** Preserve existing copyright and license notices in inherited
|
||||
and third-party files. New original files name their actual copyright holders
|
||||
and use AGPL-3.0-or-later. Use a header such as the following, then append the
|
||||
standard AGPL boilerplate:
|
||||
|
||||
```
|
||||
// Copyright (c) 2015-2025 MinIO, Inc.
|
||||
// Copyright (c) 2025-2026 PGSTY
|
||||
// Copyright (c) 2026 Your Name
|
||||
```
|
||||
|
||||
* **Separately licensed material.** Documentation contributions in `docs/`
|
||||
follow its existing [CC BY 4.0 license](docs/LICENSE). Third-party components
|
||||
and earlier Apache-2.0 contributions retain their original licenses and
|
||||
attribution; this policy does not relicense earlier work.
|
||||
|
||||
* **Squash merges** must keep the `Signed-off-by:` trailers in the resulting
|
||||
commit message.
|
||||
|
||||
|
||||
+184
File diff suppressed because one or more lines are too long
+19
-11
@@ -1,4 +1,15 @@
|
||||
FROM golang:1.26.5-alpine AS build
|
||||
FROM golang:1.27.1-alpine@sha256:cf6fca6641884b8433441b2b0652976f975e1d0fdd26d177eaaf8596087f3125 AS curl-build
|
||||
ARG TARGETARCH
|
||||
COPY dockerscripts/build-static-curl.sh /build/build-static-curl
|
||||
RUN /bin/sh /build/build-static-curl
|
||||
|
||||
# Exercise the exact shipped curl without a dynamic loader or shared libraries.
|
||||
FROM scratch AS curl-runtime
|
||||
COPY --from=curl-build /go/bin/curl /curl
|
||||
COPY --from=curl-build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
ENTRYPOINT ["/curl"]
|
||||
|
||||
FROM golang:1.27.1-alpine@sha256:cf6fca6641884b8433441b2b0652976f975e1d0fdd26d177eaaf8596087f3125 AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
|
||||
@@ -6,9 +17,9 @@ ENV GOPATH=/go
|
||||
ENV CGO_ENABLED=0
|
||||
|
||||
ARG MC_REPO=pgsty/mc
|
||||
ARG MC_VERSION=RELEASE.2026-08-06T00-00-00Z
|
||||
ARG MC_AMD64_SHA256=4b488bd30af54ad4214e5b654746677c79cd93dc6cad4be3aa2d09dbb48370ff
|
||||
ARG MC_ARM64_SHA256=83f6fedb16ed9c1e8efa8aea6776203dff132bc474214543d5c0767ed2066c2f
|
||||
ARG MC_VERSION=RELEASE.2026-09-13T00-00-00Z
|
||||
ARG MC_AMD64_SHA256=9d2a92de9c7b887d9b944fe9ddce68d23f1b6df3415092e737594e56593f5e2b
|
||||
ARG MC_ARM64_SHA256=3d82e9ea6c601c4cb44fe5dd5f2ad1b7d7d64369378110f9ada9c524688a452a
|
||||
|
||||
RUN apk add -U --no-cache \
|
||||
ca-certificates \
|
||||
@@ -56,18 +67,14 @@ RUN apk add -U --no-cache \
|
||||
chmod +x /go/bin/mcli && \
|
||||
ln -sf mcli /go/bin/mc
|
||||
|
||||
COPY dockerscripts/download-static-curl.sh /build/download-static-curl
|
||||
RUN chmod +x /build/download-static-curl && \
|
||||
/build/download-static-curl
|
||||
|
||||
FROM registry.access.redhat.com/ubi9/ubi:latest AS certs
|
||||
FROM registry.access.redhat.com/ubi9/ubi:latest@sha256:206b65b8ee0f04b992818c9a51b29081b14974630d4850bc358097d0c44ea156 AS certs
|
||||
RUN dnf -y install ca-certificates && \
|
||||
update-ca-trust && \
|
||||
cp /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /tmp/ca-certificates.crt && \
|
||||
dnf clean all && \
|
||||
rm -rf /var/cache/dnf
|
||||
|
||||
FROM registry.access.redhat.com/ubi9/ubi-micro:latest
|
||||
FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:f332c99eb8f798a8486821c91937f10ad64ee83d7e739303be2df051040918f6
|
||||
|
||||
LABEL org.opencontainers.image.title="Silo" \
|
||||
org.opencontainers.image.description="S3-Interface Libre Object Storage" \
|
||||
@@ -88,7 +95,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
COPY --from=certs /tmp/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
COPY silo /usr/bin/silo
|
||||
COPY --from=build /go/bin/mcli /usr/bin/mcli
|
||||
COPY --from=build /go/bin/curl* /usr/bin/
|
||||
COPY --from=curl-build /go/bin/curl /usr/bin/curl
|
||||
COPY --from=curl-build /go/share/curl /licenses/curl
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
COPY LICENSE /licenses/LICENSE
|
||||
COPY NOTICE /licenses/NOTICE
|
||||
|
||||
@@ -4,7 +4,7 @@ LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
|
||||
|
||||
GOOS ?= $(shell go env GOOS)
|
||||
GOARCH ?= $(shell go env GOARCH)
|
||||
GOLANGCI_VERSION ?= v2.11.3
|
||||
GOLANGCI_VERSION ?= v2.13.1
|
||||
|
||||
VERSION ?= $(shell git describe --tags)
|
||||
REPO ?= docker.io/pgsty
|
||||
@@ -48,7 +48,7 @@ credits: ## regenerate CREDITS from the licenses of Go modules linked into the b
|
||||
|
||||
check-gen: ## check for updated autogenerated files
|
||||
@go generate ./... >/dev/null
|
||||
@go mod tidy -compat=1.26
|
||||
@go mod tidy -compat=1.27
|
||||
@env bash $(PWD)/buildscripts/gen-credits.sh
|
||||
@changed=$$(git diff --name-only -- '*_gen.go' '*_gen_test.go' '*_msgp_test.go' '*_string.go' go.mod go.sum CREDITS); \
|
||||
if [ -n "$$changed" ]; then \
|
||||
@@ -216,7 +216,7 @@ docker: checks build-debugging ## builds the local Linux Silo container image
|
||||
--ldflags "$(LDFLAGS)" -o "$$context/silo"; \
|
||||
mkdir -p "$$context/dockerscripts"; \
|
||||
cp Dockerfile.goreleaser LICENSE NOTICE CREDITS "$$context/"; \
|
||||
cp dockerscripts/docker-entrypoint.sh dockerscripts/download-static-curl.sh \
|
||||
cp dockerscripts/docker-entrypoint.sh dockerscripts/build-static-curl.sh \
|
||||
"$$context/dockerscripts/"; \
|
||||
docker build -q --no-cache --platform linux/$(GOARCH) -t $(TAG) --build-arg TARGETARCH=$(GOARCH) \
|
||||
-f "$$context/Dockerfile.goreleaser" "$$context"
|
||||
|
||||
@@ -6,15 +6,17 @@
|
||||
|
||||
|
||||
<p align="center">
|
||||
<strong>S3-compatible object storage — a MinIO fork maintained by PIGSTY</strong>
|
||||
<strong>S3-compatible object storage — a MinIO fork maintained by PGSTY</strong>
|
||||
</p>
|
||||
|
||||
|
||||
<p align="center">
|
||||
<a href="https://silo.pgsty.com/">Website</a> ·
|
||||
<a href="https://silo.pgsty.com/docs/">Documentation</a> ·
|
||||
<a href="https://silo.pgsty.com/download/">Download</a> ·
|
||||
<a href="https://silo.pgsty.com/tags/silo/">Release Notes</a> ·
|
||||
<a href="https://silo.pgsty.com/compatibility/server/">Compatibility</a> ·
|
||||
<a href="https://silo.pgsty.com/about/manifesto/">Manifesto</a> ·
|
||||
<a href="SECURITY.md">Security</a> ·
|
||||
<a href="README_ZH.md">中文</a>
|
||||
</p>
|
||||
@@ -28,239 +30,162 @@
|
||||
</p>
|
||||
|
||||
> [!IMPORTANT]
|
||||
> Silo is an independent, community-maintained fork of the open-source MinIO server, published by [Pigsty](https://pigsty.io) from [`pgsty/silo`](https://github.com/pgsty/silo). It is not affiliated with, endorsed by, or sponsored by MinIO, Inc. “MinIO” is used only to identify the upstream project and compatibility lineage.
|
||||
> **PGSTY Silo** (hereinafter “Silo”) is an independent, community-maintained fork of the open-source MinIO server, published by [Pigsty](https://pigsty.io) from [`pgsty/silo`](https://github.com/pgsty/silo). It is not affiliated with, endorsed by, or sponsored by MinIO, Inc. “MinIO” is used only to identify the upstream project and compatibility lineage.
|
||||
|
||||
> [!NOTE]
|
||||
> This repository was renamed from `pgsty/minio` to `pgsty/silo`, and its default branch from `master` to `main`, on 2026-08-06. If you need the artifacts maintained under the original MinIO identity, they are on the archived [`minio`](https://github.com/pgsty/silo/tree/minio) branch and in the releases up to [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z); those assets and the `docker.io/pgsty/minio` image stay published and unmodified. The rename changed product and artifact names only — `MINIO_*` variables, `minio_*` metrics, `x-minio-*` headers, `/minio/*` routes, and the on-disk format are unchanged.
|
||||
> Renamed from `pgsty/minio` to `pgsty/silo`, default branch `master` → `main`, on 2026-08-06. Artifacts under the original MinIO identity stay published on the archived [`minio`](https://github.com/pgsty/silo/tree/minio) branch and in releases up to [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z).
|
||||
|
||||
## Current release and main branch
|
||||
|
||||
The latest published Server is [20260903](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z).
|
||||
As of 2026-09-13, the main branch has newer security, storage, Console and
|
||||
shared-package changes that have not shipped in a Server release. See
|
||||
[CHANGELOG.md](CHANGELOG.md) and the [component version matrix](https://silo.pgsty.com/compatibility/versions/)
|
||||
for the exact release/source boundary, including SN-2026-011 and password-policy migration.
|
||||
|
||||
## Overview
|
||||
|
||||
Silo maintains one downstream release line derived from the open-source MinIO server. It provides maintained builds and release artifacts for existing MinIO-compatible deployments after upstream community distribution ended. Pigsty uses Silo for object storage, including as an optional PostgreSQL backup repository.
|
||||
PGSTY SILO keeps one maintained release line of the open-source MinIO server alive after upstream ended community distribution: builds, packages, multi-arch images, security fixes, and the full web console. Pigsty runs it in production as its PostgreSQL backup repository.
|
||||
|
||||
The fork follows one rule: **the product and its delivery surfaces are renamed; the protocol and your data are not.** The `silo` executable, packages, service, and container image carry the new name, while the S3 API, `MINIO_*` variables, `minio_*` metrics, `x-minio-*` headers, reserved routes, and the on-disk format stay unchanged. The differences are recorded in the [compatibility notes](https://silo.pgsty.com/compatibility/server/).
|
||||
It follows one rule — **the product and its delivery surfaces are renamed; the protocol and your data are not.** Everything else lives on [silo.pgsty.com](https://silo.pgsty.com/).
|
||||
|
||||
The official project portal is [silo.pgsty.com](https://silo.pgsty.com/). It brings documentation, downloads, release and security notes, and project background together.
|
||||
**Related:** [`pgsty/mc`](https://github.com/pgsty/mc) client (shipped as `mcli`) · [`pgsty/silo-console`](https://github.com/pgsty/silo-console) · [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) · [`pgsty/pigsty`](https://github.com/pgsty/pigsty)
|
||||
|
||||
## Find the Right Resource
|
||||
|
||||
| Looking for | Canonical location |
|
||||
| :-- | :-- |
|
||||
| Project overview and navigation | [Silo Website](https://silo.pgsty.com/) |
|
||||
| Installation methods and downloads | [Download & Install](https://silo.pgsty.com/download/) |
|
||||
| Operations, administration, development, and reference | [Documentation](https://silo.pgsty.com/docs/) |
|
||||
| Release notes for this server | [Silo release notes](https://silo.pgsty.com/tags/silo/) |
|
||||
| Differences from upstream MinIO | [Silo vs. MinIO compatibility notes](https://silo.pgsty.com/compatibility/server/) |
|
||||
| Project news and security advisories | [Blog](https://silo.pgsty.com/blog/) · [release](https://silo.pgsty.com/blog/release/) and [security](https://silo.pgsty.com/blog/security/) notes |
|
||||
| Versioned binaries, checksums, and source archives | [GitHub Releases](https://github.com/pgsty/silo/releases) |
|
||||
| Bug reports and feature discussions | [GitHub Issues](https://github.com/pgsty/silo/issues) |
|
||||
| Vulnerability reporting | [`SECURITY.md`](SECURITY.md) and [`VULNERABILITY_REPORT.md`](VULNERABILITY_REPORT.md) |
|
||||
| License, attribution, and trademark information | [`LICENSE`](LICENSE) · [`NOTICE`](NOTICE) · [`CREDITS`](CREDITS) · portal [license](https://silo.pgsty.com/about/license/), [attribution](https://silo.pgsty.com/about/attribution/), and [trademark](https://silo.pgsty.com/about/trademark/) pages |
|
||||
|
||||
## Related Projects
|
||||
|
||||
| Repository | Description |
|
||||
| :-- | :-- |
|
||||
| [`pgsty/silo`](https://github.com/pgsty/silo) | This repository — the Silo object storage server, an S3-compatible MinIO fork |
|
||||
| [`pgsty/mc`](https://github.com/pgsty/mc) | The Silo command-line client, shipped as `mcli` with the `mc` command name |
|
||||
| [`pgsty/silo-console`](https://github.com/pgsty/silo-console) | Admin web console, embedded in the server build |
|
||||
| [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) | Shared Go packages maintained for the Silo forks |
|
||||
| [`pgsty/pigsty`](https://github.com/pgsty/pigsty) | Pigsty — the PostgreSQL distribution that ships Silo as its object storage |
|
||||
|
||||
## Maintenance Policy
|
||||
|
||||
The active release line covers:
|
||||
|
||||
- build and dependency maintenance;
|
||||
- applicable security fixes and advisories;
|
||||
- focused fixes for reproducible defects;
|
||||
- versioned binaries, packages, checksums, and multi-architecture images;
|
||||
- the web console, client, documentation, and Pigsty integration.
|
||||
|
||||
Changes are kept narrow and tested where practical. Maintenance is best effort; no response, remediation, or release schedule is guaranteed.
|
||||
|
||||
### Out of scope
|
||||
|
||||
- a separate product roadmap, new storage engine, or speculative S3 features;
|
||||
- broad rewrites or changes that materially expand the downstream delta;
|
||||
- historical releases or multiple support branches;
|
||||
- commercial support, SLAs, 24×7 coverage, or SUBNET access;
|
||||
- deployment design, access control, monitoring, backup, or recovery.
|
||||
|
||||
## Governance
|
||||
|
||||
The server, client, console, and shared packages are maintained under one release process: DCO-signed commits, reviewed pull requests, and versioned `RELEASE.YYYY-MM-DDTHH-MM-SSZ` tags whose artifacts ship with checksums, SPDX SBOMs, Sigstore-signed manifests, and GitHub build attestations. Each release is announced with a [release note](https://silo.pgsty.com/tags/silo/) on the portal; security handling follows [`SECURITY.md`](SECURITY.md) and the [advisory log](docs/security/advisories.md). Upstream copyright, license, and third-party notices are preserved in [`LICENSE`](LICENSE), [`NOTICE`](NOTICE), and [`CREDITS`](CREDITS).
|
||||
|
||||
## Compatibility
|
||||
|
||||
Silo aims to preserve:
|
||||
|
||||
- the `github.com/minio/minio` module path and `github.com/minio/*` import paths;
|
||||
- MinIO-compatible S3 APIs, wire behavior, `MINIO_*` environment variables, metrics, protocol headers, reserved routes, and storage metadata;
|
||||
- `RELEASE.YYYY-MM-DDTHH-MM-SSZ` tags and legacy `minio …` container argv translation.
|
||||
|
||||
Silo-owned delivery surfaces use the `silo` executable, package, service, Helm chart, and `pgsty/silo` container image. Native artifacts intentionally do not install a `minio` binary alias.
|
||||
|
||||
Compatibility is the default constraint. Silo preserves existing wire, client, configuration, and operational behavior whenever doing so remains safe. Compatibility is broken only when necessary to close a major security issue, and the release notes must identify the affected behavior and migration path. Treat each release as a downstream upgrade: pin versions, review the [release notes](https://silo.pgsty.com/tags/silo/), [compatibility notes](https://silo.pgsty.com/compatibility/server/), and [security advisories](docs/security/advisories.md), keep a rollback path, and test before production use.
|
||||
|
||||
## Downloads and Release Artifacts
|
||||
|
||||
Use [Download & Install](https://silo.pgsty.com/download/) to choose an installation method. GitHub Releases remains the source for versioned server binaries, checksums, and source archives.
|
||||
|
||||
| Artifact | Location |
|
||||
| :-- | :-- |
|
||||
| Source | [`github.com/pgsty/silo`](https://github.com/pgsty/silo) |
|
||||
| Container image | [`pgsty/silo`](https://hub.docker.com/r/pgsty/silo), multi-arch for `linux/amd64` and `linux/arm64` |
|
||||
| Server binaries and checksums | [GitHub Releases](https://github.com/pgsty/silo/releases) for Linux, macOS, and Windows on `amd64` and `arm64` |
|
||||
| Linux packages | RPM, DEB, and APK artifacts, also distributed through the [Pigsty repository](https://pigsty.io/docs/repo/) |
|
||||
| Client | [`pgsty/mc`](https://github.com/pgsty/mc), bundled in the container as `mcli` with an `mc` compatibility alias |
|
||||
| Console | [`pgsty/silo-console`](https://github.com/pgsty/silo-console), embedded through the compatibility import path `github.com/minio/console` |
|
||||
| Shared library | [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) v3.11.0, consumed through a `replace` directive while preserving the `github.com/minio/pkg/v3` import path |
|
||||
|
||||
Each new release publishes per-archive and per-package SPDX JSON SBOMs. The archive and package checksum manifests have detached keyless Sigstore bundles, while GitHub artifact attestations record signed provenance for every downloadable artifact and the multi-architecture container image.
|
||||
|
||||
After downloading an archive and its release files, verify integrity, the
|
||||
published SBOM, the signed manifest, and build provenance independently:
|
||||
|
||||
```bash
|
||||
# Integrity: choose the line for the artifact you downloaded.
|
||||
grep -F ' silo_<version>_linux_amd64.tar.gz' \
|
||||
silo_<version>_checksums.txt | sha256sum --check
|
||||
|
||||
# The archive SBOM is a separate checksummed release artifact.
|
||||
grep -F ' silo_<version>_linux_amd64.tar.gz.sbom.json' \
|
||||
silo_<version>_checksums.txt | sha256sum --check
|
||||
|
||||
# Signature over the archive/SBOM checksum manifest.
|
||||
cosign verify-blob \
|
||||
--bundle silo_<version>_checksums.txt.sigstore.json \
|
||||
--certificate-identity-regexp \
|
||||
'^https://github.com/pgsty/(minio|silo)/\.github/workflows/release\.yml@refs/(tags/RELEASE\..+|heads/(master|main))$' \
|
||||
--certificate-oidc-issuer https://token.actions.githubusercontent.com \
|
||||
silo_<version>_checksums.txt
|
||||
|
||||
# Signed build provenance (online verification against this repository).
|
||||
gh attestation verify silo_<version>_linux_amd64.tar.gz \
|
||||
--repo pgsty/silo
|
||||
```
|
||||
|
||||
For packages, use `silo_<version>_packages_checksums.txt` and its adjacent
|
||||
Sigstore bundle with the same identity and issuer constraints. Inspect the
|
||||
verified SPDX JSON SBOM with your preferred SPDX tooling. Verify the
|
||||
multi-architecture container provenance by digest:
|
||||
|
||||
```bash
|
||||
gh attestation verify \
|
||||
oci://index.docker.io/pgsty/silo@sha256:<manifest-digest> \
|
||||
--repo pgsty/silo
|
||||
```
|
||||
|
||||
The platform SBOM attestations are attached to the `amd64` and `arm64` image
|
||||
digests rather than the multi-architecture manifest. Verify each one explicitly:
|
||||
|
||||
```bash
|
||||
gh attestation verify \
|
||||
oci://index.docker.io/pgsty/silo@sha256:<platform-digest> \
|
||||
--repo pgsty/silo \
|
||||
--predicate-type https://spdx.dev/Document/v2.3
|
||||
```
|
||||
|
||||
Verification by digest avoids trusting a mutable image tag.
|
||||
|
||||
### Native package migration
|
||||
|
||||
The `silo` RPM, DEB, and APK do not declare `Provides`, `Obsoletes`,
|
||||
`Replaces`, or package-level `Conflicts` against `minio`. They can therefore be
|
||||
installed beside an existing MinIO package without silently replacing it. The
|
||||
two systemd units conflict at runtime, so switch them explicitly rather than
|
||||
starting both.
|
||||
|
||||
Before switching, record the old unit's enabled/active state and `User`/`Group`,
|
||||
and back up `/etc/default/minio`. Silo reads that legacy defaults file first and
|
||||
then `/etc/default/silo`; administrator-set values in the latter take
|
||||
precedence. If the existing data must continue to run under its original
|
||||
UID/GID, create `/etc/systemd/system/silo.service.d/10-legacy-user.conf`:
|
||||
|
||||
```ini
|
||||
[Service]
|
||||
User=<legacy-user>
|
||||
Group=<legacy-group>
|
||||
```
|
||||
|
||||
Run `systemctl daemon-reload`, then disable and stop `minio.service` before
|
||||
enabling and starting `silo.service`. Verify health, S3, Admin API, metrics, and
|
||||
logs before masking or uninstalling the old service. Do not recursively change
|
||||
data ownership as part of the package migration; keep the old package and unit
|
||||
available during the rollback window.
|
||||
<p align="center">
|
||||
<img src="https://silo.pgsty.com/images/silo-console/console-metrics-simple.webp" alt="Silo Console">
|
||||
</p>
|
||||
|
||||
## Quick Start
|
||||
|
||||
For local evaluation:
|
||||
|
||||
```bash
|
||||
mkdir -p data
|
||||
|
||||
export MINIO_ROOT_USER=minioadmin
|
||||
export MINIO_ROOT_PASSWORD=change-me-long-password
|
||||
|
||||
docker run -d --name silo \
|
||||
-p 9000:9000 \
|
||||
-p 9001:9001 \
|
||||
-e MINIO_ROOT_USER \
|
||||
-e MINIO_ROOT_PASSWORD \
|
||||
docker run -d --name silo -p 9000:9000 -p 9001:9001 \
|
||||
-e MINIO_ROOT_USER=minioadmin \
|
||||
-e MINIO_ROOT_PASSWORD=change-me-long-password \
|
||||
-v "$PWD/data:/data" \
|
||||
docker.io/pgsty/silo:latest server /data --console-address ":9001"
|
||||
```
|
||||
|
||||
Open the console at <http://localhost:9001>; the S3 API listens on <http://localhost:9000>.
|
||||
<p align="center">
|
||||
<img src="https://silo.pgsty.com/images/silo-console/console-login.webp" alt="Silo Console">
|
||||
</p>
|
||||
|
||||
The image includes the compatible client as `mcli`:
|
||||
Console on <http://localhost:9001>, S3 API on <http://localhost:9000>. The image bundles the client as `mcli`:
|
||||
|
||||
```bash
|
||||
docker exec silo mcli alias set local http://127.0.0.1:9000 \
|
||||
"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
|
||||
docker exec silo mcli mb local/demo
|
||||
docker exec silo mcli ls local
|
||||
docker exec silo mcli alias set local http://127.0.0.1:9000 minioadmin change-me-long-password
|
||||
docker exec silo mcli mb local/demo && docker exec silo mcli ls local
|
||||
```
|
||||
|
||||
> [!WARNING]
|
||||
> For production, pin a release, use unique credentials and TLS, monitor the service, keep independent backups, and test recovery.
|
||||
> For production, pin a release, use unique credentials and TLS, monitor the service, keep independent backups, and test recovery. Start from the [documentation](https://silo.pgsty.com/docs/).
|
||||
|
||||
Build the server from source:
|
||||
## Install
|
||||
|
||||
```bash
|
||||
go build -o silo .
|
||||
./silo --version
|
||||
```
|
||||
| Method | Where |
|
||||
| :-- | :-- |
|
||||
| Container | [`pgsty/silo`](https://hub.docker.com/r/pgsty/silo), multi-arch for `linux/amd64` and `linux/arm64` |
|
||||
| Binaries | [GitHub Releases](https://github.com/pgsty/silo/releases) — Linux, macOS, Windows on `amd64` and `arm64` |
|
||||
| Packages | RPM, DEB, and APK, also via the [Pigsty repository](https://pigsty.io/docs/repo/) |
|
||||
| Kubernetes | Helm chart, see [Download & Install](https://silo.pgsty.com/download/) |
|
||||
| Source | `go build -o silo . && ./silo --version` |
|
||||
|
||||
For other installation paths—including native packages, binaries, Podman, Kubernetes, source, and Pigsty Ansible—use [Download & Install](https://silo.pgsty.com/download/). For production deployment and administration, start with the [Silo documentation](https://silo.pgsty.com/docs/). Pigsty users can also use the [Pigsty MinIO module](https://pigsty.io/docs/minio/).
|
||||
Every release ships checksums, SPDX SBOMs, Sigstore-signed manifests, and GitHub build attestations. Installation methods and verification commands are documented at [Download & Install](https://silo.pgsty.com/download/); migrating from upstream MinIO — taking over an existing `minio.service` and its `/etc/default/minio`, and keeping data ownership stable with a `/etc/systemd/system/silo.service.d/10-legacy-user.conf` drop-in — is covered by the [migration guide](https://silo.pgsty.com/compatibility/migration/) and the [binary & service notes](https://silo.pgsty.com/compatibility/binary/).
|
||||
|
||||
## Security
|
||||
## Compatibility
|
||||
|
||||
Security fixes target the active development branch and are recorded in the [advisory log](docs/security/advisories.md) and the portal's [security notes](https://silo.pgsty.com/blog/security/). Report vulnerabilities privately as described in [`SECURITY.md`](SECURITY.md) and [`VULNERABILITY_REPORT.md`](VULNERABILITY_REPORT.md). Report issues that also affect upstream MinIO there as well.
|
||||
The S3 API, `MINIO_*` variables, `minio_*` metrics, `x-minio-*` headers, `/minio/*` routes, the `github.com/minio/*` import paths, and the on-disk format (including `.minio.sys`) are preserved and held in place by a CI compatibility check. Only Silo-owned delivery surfaces change: the `silo` executable, package, service, Helm chart, and container image — no `minio` binary alias is installed.
|
||||
|
||||
## Contributing
|
||||
Every divergence from upstream is listed in the code-verified [compatibility audit](https://silo.pgsty.com/compatibility/server/). Treat each release as a downstream upgrade: pin versions, read the [release notes](https://silo.pgsty.com/tags/silo/), and keep a rollback path.
|
||||
|
||||
Useful contributions include security and dependency updates, reproducible bug fixes, tests, release automation, packaging, and documentation.
|
||||
### TLS and Go upgrades
|
||||
|
||||
Issues and pull requests should include the affected version, reproduction steps, impact, expected behavior, tests, and compatibility notes. Discuss large changes in an issue first.
|
||||
TLS key exchange follows Go's defaults across the S3 listener, node links,
|
||||
replication, identity providers, etcd, and external HTTP services. If an endpoint
|
||||
cannot accept ML-KEM, `GODEBUG=tlsmlkem=0` disables the default hybrid exchanges
|
||||
for the process; certificate verification remains enabled. This option does not
|
||||
disable ML-DSA signatures or resolve every TLS reset. Prefer updating the
|
||||
incompatible endpoint before removing the temporary setting.
|
||||
If only the new SecP hybrids cause problems, `GODEBUG=tlssecpmlkem=0` disables
|
||||
those groups while retaining X25519MLKEM768.
|
||||
|
||||
There is no CLA: contributions are accepted inbound=outbound under the project license (AGPL-3.0-or-later) and contributors keep their copyright. Every commit must be signed off (`git commit -s`) per the [Developer Certificate of Origin](https://developercertificate.org/); see [`CONTRIBUTING.md`](CONTRIBUTING.md).
|
||||
For builds targeting Go 1.27, setting either `SSL_CERT_FILE` or `SSL_CERT_DIR`
|
||||
on macOS replaces Keychain trust with on-disk roots and Go's verifier. Stale or
|
||||
incomplete CA paths can break previously trusted connections; unset inherited
|
||||
values to restore Keychain trust. Explicit certificates in the configured `CAs`
|
||||
directory remain additive to the selected root pool.
|
||||
Go 1.27 binaries require macOS 13 or later. See the
|
||||
[Go release notes](https://go.dev/doc/go1.27) and the
|
||||
[SILO stack investigation](docs/investigations/go127-stack.md).
|
||||
|
||||
## Security & Contributing
|
||||
|
||||
Report vulnerabilities privately as described in [`SECURITY.md`](SECURITY.md); every fix ships with a public [advisory](https://silo.pgsty.com/blog/security/). Contributions are accepted inbound=outbound under AGPL-3.0-or-later with no CLA — only DCO sign-off (`git commit -s`) is required; see [`CONTRIBUTING.md`](CONTRIBUTING.md).
|
||||
|
||||
## Contributors
|
||||
|
||||
**41 community contributors** build SILO, Console, mcli, shared packages, and related projects. The list includes maintainers and every human Issue or PR author, ordered by merged PRs, other PRs, then issue reports. Gold rings highlight significant contributions.
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/Vonng"><img src="https://silo.pgsty.com/images/contributors/Vonng.svg" width="60" height="60" alt="@Vonng" title="@Vonng — Maintains SILO, Console, mcli, shared packages, releases, and documentation"></a>
|
||||
<a href="https://github.com/h5vx"><img src="https://silo.pgsty.com/images/contributors/h5vx.svg" width="60" height="60" alt="@h5vx" title="@h5vx — Implemented per-bucket CORS configuration and enforcement"></a>
|
||||
<a href="https://github.com/mrjavadseydi"><img src="https://silo.pgsty.com/images/contributors/mrjavadseydi.svg" width="60" height="60" alt="@mrjavadseydi" title="@mrjavadseydi — Fixed effective bucket quota metrics; proposed access-frequency ILM"></a>
|
||||
<a href="https://github.com/Dansyuqri"><img src="https://silo.pgsty.com/images/contributors/Dansyuqri.svg" width="60" height="60" alt="@Dansyuqri" title="@Dansyuqri — Added ChecksumType to multipart completion responses"></a>
|
||||
<a href="https://github.com/ycjlin"><img src="https://silo.pgsty.com/images/contributors/ycjlin.svg" width="60" height="60" alt="@ycjlin" title="@ycjlin — Fixed missing-bucket ListObjects semantics"></a>
|
||||
<a href="https://github.com/pinginfo"><img src="https://silo.pgsty.com/images/contributors/pinginfo.svg" width="60" height="60" alt="@pinginfo" title="@pinginfo — Repaired bucket notification streaming"></a>
|
||||
<a href="https://github.com/ZouhairCharef"><img src="https://silo.pgsty.com/images/contributors/ZouhairCharef.svg" width="60" height="60" alt="@ZouhairCharef" title="@ZouhairCharef — Patched CVE-2026-34986 in go-jose"></a>
|
||||
<a href="https://github.com/mfredenhagen"><img src="https://silo.pgsty.com/images/contributors/mfredenhagen.svg" width="60" height="60" alt="@mfredenhagen" title="@mfredenhagen — Patched CVE-2026-39883 in OpenTelemetry"></a>
|
||||
<a href="https://github.com/waterkip"><img src="https://silo.pgsty.com/images/contributors/waterkip.svg" width="60" height="60" alt="@waterkip" title="@waterkip — Repointed documentation links to the SILO portal"></a>
|
||||
<a href="https://github.com/mikemikimike"><img src="https://silo.pgsty.com/images/contributors/mikemikimike.svg" width="60" height="60" alt="@mikemikimike" title="@mikemikimike — Contributed the replicated SSE-C plaintext part-size fix"></a>
|
||||
<a href="https://github.com/metaneutrons"><img src="https://silo.pgsty.com/images/contributors/metaneutrons.svg" width="60" height="60" alt="@metaneutrons" title="@metaneutrons — Reported and proposed explicit-version delete authorization"></a>
|
||||
<a href="https://github.com/magicxor"><img src="https://silo.pgsty.com/images/contributors/magicxor.svg" width="60" height="60" alt="@magicxor" title="@magicxor — Reported and proposed conditional DELETE support for If-Match"></a>
|
||||
<a href="https://github.com/davinkevin"><img src="https://silo.pgsty.com/images/contributors/davinkevin.svg" width="60" height="60" alt="@davinkevin" title="@davinkevin — Proposed the distroless container image and dependency automation"></a>
|
||||
<a href="https://github.com/lem21h"><img src="https://silo.pgsty.com/images/contributors/lem21h.svg" width="48" height="48" alt="@lem21h" title="@lem21h — Proposed robustness and goroutine improvements"></a>
|
||||
<a href="https://github.com/sulin37392"><img src="https://silo.pgsty.com/images/contributors/sulin37392.svg" width="48" height="48" alt="@sulin37392" title="@sulin37392 — Proposed dependency updates"></a>
|
||||
<a href="https://github.com/cbornet"><img src="https://silo.pgsty.com/images/contributors/cbornet.svg" width="60" height="60" alt="@cbornet" title="@cbornet — Reported multipart and streaming checksum defects and missing-bucket semantics"></a>
|
||||
<a href="https://github.com/vampywiz17"><img src="https://silo.pgsty.com/images/contributors/vampywiz17.svg" width="60" height="60" alt="@vampywiz17" title="@vampywiz17 — Reported LDAP TLS and Console login regressions"></a>
|
||||
<a href="https://github.com/orenyomtov"><img src="https://silo.pgsty.com/images/contributors/orenyomtov.svg" width="60" height="60" alt="@orenyomtov" title="@orenyomtov — Reported the unsigned-header CopyObject cross-object read (SN-2026-011)"></a>
|
||||
<a href="https://github.com/mumu-lab"><img src="https://silo.pgsty.com/images/contributors/mumu-lab.svg" width="48" height="48" alt="@mumu-lab" title="@mumu-lab — Reported bucket quota metrics reading a deprecated field"></a>
|
||||
<a href="https://github.com/jvasile"><img src="https://silo.pgsty.com/images/contributors/jvasile.svg" width="48" height="48" alt="@jvasile" title="@jvasile — Reported missing user, group, and defaults in Debian packages"></a>
|
||||
<a href="https://github.com/pmezhuev"><img src="https://silo.pgsty.com/images/contributors/pmezhuev.svg" width="48" height="48" alt="@pmezhuev" title="@pmezhuev — Reported missing RPM package signatures"></a>
|
||||
<a href="https://github.com/TLINDEN"><img src="https://silo.pgsty.com/images/contributors/TLINDEN.svg" width="48" height="48" alt="@TLINDEN" title="@TLINDEN — Reported the missing client in release tarballs"></a>
|
||||
<a href="https://github.com/makinikm"><img src="https://silo.pgsty.com/images/contributors/makinikm.svg" width="48" height="48" alt="@makinikm" title="@makinikm — Reported the missing client in the container image"></a>
|
||||
<a href="https://github.com/meesudzu"><img src="https://silo.pgsty.com/images/contributors/meesudzu.svg" width="48" height="48" alt="@meesudzu" title="@meesudzu — Requested the migration guide from upstream MinIO"></a>
|
||||
<a href="https://github.com/kuldeep-link11"><img src="https://silo.pgsty.com/images/contributors/kuldeep-link11.svg" width="48" height="48" alt="@kuldeep-link11" title="@kuldeep-link11 — Reported NATS JWT credentials and target reload issues"></a>
|
||||
<a href="https://github.com/sargarass"><img src="https://silo.pgsty.com/images/contributors/sargarass.svg" width="48" height="48" alt="@sargarass" title="@sargarass — Reported ListMultipartUploads prefix and pagination semantics"></a>
|
||||
<a href="https://github.com/liuhaodongliu990-cmyk"><img src="https://silo.pgsty.com/images/contributors/liuhaodongliu990-cmyk.svg" width="48" height="48" alt="@liuhaodongliu990-cmyk" title="@liuhaodongliu990-cmyk — Reported indeterminate progress for prefix downloads"></a>
|
||||
<a href="https://github.com/Xavier-777"><img src="https://silo.pgsty.com/images/contributors/Xavier-777.svg" width="48" height="48" alt="@Xavier-777" title="@Xavier-777 — Reported Console lifecycle management and file preview gaps"></a>
|
||||
<a href="https://github.com/spaceg00se-r"><img src="https://silo.pgsty.com/images/contributors/spaceg00se-r.svg" width="48" height="48" alt="@spaceg00se-r" title="@spaceg00se-r — Requested cpuv1 support and reported a workflow token failure"></a>
|
||||
<a href="https://github.com/kh0mka"><img src="https://silo.pgsty.com/images/contributors/kh0mka.svg" width="48" height="48" alt="@kh0mka" title="@kh0mka — Reported inter-node I/O timeouts in ReadFileStreamHandler"></a>
|
||||
<a href="https://github.com/bagutzu"><img src="https://silo.pgsty.com/images/contributors/bagutzu.svg" width="48" height="48" alt="@bagutzu" title="@bagutzu — Requested KES-compatible external KMS and OpenBao support"></a>
|
||||
<a href="https://github.com/DestroyLee"><img src="https://silo.pgsty.com/images/contributors/DestroyLee.svg" width="48" height="48" alt="@DestroyLee" title="@DestroyLee — Reported the missing documentation navigation"></a>
|
||||
<a href="https://github.com/mosesdd"><img src="https://silo.pgsty.com/images/contributors/mosesdd.svg" width="48" height="48" alt="@mosesdd" title="@mosesdd — Requested a maintained Helm chart"></a>
|
||||
<a href="https://github.com/zylpsrs"><img src="https://silo.pgsty.com/images/contributors/zylpsrs.svg" width="48" height="48" alt="@zylpsrs" title="@zylpsrs — Reported missing Console tiering and site replication"></a>
|
||||
<a href="https://github.com/heroes1412"><img src="https://silo.pgsty.com/images/contributors/heroes1412.svg" width="48" height="48" alt="@heroes1412" title="@heroes1412 — Reported the unusable profiling option"></a>
|
||||
<a href="https://github.com/redfoxfox"><img src="https://silo.pgsty.com/images/contributors/redfoxfox.svg" width="48" height="48" alt="@redfoxfox" title="@redfoxfox — Reported Chinese documentation availability"></a>
|
||||
<a href="https://github.com/jiadzh"><img src="https://silo.pgsty.com/images/contributors/jiadzh.svg" width="48" height="48" alt="@jiadzh" title="@jiadzh — Requested Windows build guidance"></a>
|
||||
<a href="https://github.com/AntonOfTheWoods"><img src="https://silo.pgsty.com/images/contributors/AntonOfTheWoods.svg" width="48" height="48" alt="@AntonOfTheWoods" title="@AntonOfTheWoods — Asked for clarity on Helm chart and operator options"></a>
|
||||
<a href="https://github.com/chalukyaj"><img src="https://silo.pgsty.com/images/contributors/chalukyaj.svg" width="48" height="48" alt="@chalukyaj" title="@chalukyaj — Proposed making the SILO Operator easier to discover"></a>
|
||||
<a href="https://github.com/nsanitate"><img src="https://silo.pgsty.com/images/contributors/nsanitate.svg" width="48" height="48" alt="@nsanitate" title="@nsanitate — Proposed CNCF Sandbox governance"></a>
|
||||
<a href="https://github.com/Kesavaambati"><img src="https://silo.pgsty.com/images/contributors/Kesavaambati.svg" width="48" height="48" alt="@Kesavaambati" title="@Kesavaambati — Asked about community support and image maintenance"></a>
|
||||
</p>
|
||||
|
||||
[View the full contribution record](CONTRIBUTORS.md) for each person's proposals, fixes, and reports.
|
||||
|
||||
## Background
|
||||
|
||||
This project was created in response to changes in the upstream community distribution and maintenance model. The maintainer’s analysis, alternatives considered, and early maintenance record are documented below:
|
||||
Upstream wound down its community edition: the web console was cut back to a stub, prebuilt community binaries stopped, and the community repository was archived. Silo exists to keep those deployments running. The fork is a means, not an identity — if upstream restores its community edition, we will narrow our scope and offer the fixes back.
|
||||
|
||||
| Essay | Subject |
|
||||
| :-- | :-- |
|
||||
| [MinIO Is Dead](https://silo.pgsty.com/blog/post/minio-is-dead/) | Changes to the upstream project and distribution model |
|
||||
| [MinIO Is Dead, Who Takes Over?](https://silo.pgsty.com/blog/post/minio-alternative/) | Alternatives considered |
|
||||
| [MinIO Is Dead, Long Live MinIO](https://silo.pgsty.com/blog/post/minio-resurrect/) | Establishing the fork and its release pipeline |
|
||||
| [Two months into maintaining a MinIO fork](https://silo.pgsty.com/blog/post/minio-promise-kept/) | Initial security and maintenance work |
|
||||
The [**Manifesto**](https://silo.pgsty.com/about/manifesto/) is the project's public commitment in eleven articles, under one discipline: every article is either something already done with public evidence, or something explicitly refused. In short:
|
||||
|
||||
## License and Trademark
|
||||
- **Compatibility contract** — the protocol and your data do not change, and every release documents its tested rollback target and path.
|
||||
- **The license cannot change** — AGPLv3, no CLA, no copyright aggregation; nobody here, ourselves included, holds enough copyright to relicense on everyone else's behalf.
|
||||
- **The never list**, append-only — no paywalling existing features, no registration wall on downloads, no telemetry (upstream's phone-home paths are removed outright), no CLA, no license change, no trademark enforcement against normal use.
|
||||
- **Security and release discipline** — a public advisory for every fix, and a release every one to two months, at most a quarter apart. Judge both against the public record.
|
||||
|
||||
The server source is distributed under the [GNU Affero General Public License v3.0 or later](LICENSE). This fork derives from [`minio/minio`](https://github.com/minio/minio): [`NOTICE`](NOTICE) retains the upstream product notice, [`CREDITS`](CREDITS) records licenses and notices for included third-party components, and the Git history records downstream modifications.
|
||||
Essays: [MinIO Is Dead](https://silo.pgsty.com/blog/post/minio-is-dead/) · [Who Takes Over?](https://silo.pgsty.com/blog/post/minio-alternative/) · [Long Live MinIO](https://silo.pgsty.com/blog/post/minio-resurrect/) · [Promise Kept](https://silo.pgsty.com/blog/post/minio-promise-kept/)
|
||||
|
||||
MinIO is a trademark of MinIO, Inc. The name is used here only to identify the upstream project and compatibility lineage. Pigsty, Silo, and `pgsty/silo` are independent community efforts and are not affiliated with, endorsed by, or sponsored by MinIO, Inc.
|
||||
## License & Trademark
|
||||
|
||||
The portal separately publishes the project [license summary](https://silo.pgsty.com/about/license/), [documentation attribution](https://silo.pgsty.com/about/attribution/), and [trademark notice](https://silo.pgsty.com/about/trademark/).
|
||||
Silo is [AGPL-3.0-or-later](LICENSE), derived from [`minio/minio`](https://github.com/minio/minio) with upstream copyright and third-party notices preserved in [`NOTICE`](NOTICE) and [`CREDITS`](CREDITS). MinIO is a trademark of MinIO, Inc.; the name is used here only to identify the upstream project and compatibility lineage.
|
||||
|
||||
Details: [license](https://silo.pgsty.com/about/license/) · [attribution](https://silo.pgsty.com/about/attribution/) · [trademark](https://silo.pgsty.com/about/trademark/)
|
||||
|
||||
+104
-195
@@ -1,20 +1,22 @@
|
||||
<h1 align="center">
|
||||
<a href="https://silo.pgsty.com/zh/">
|
||||
<img src=".github/silo-logo.svg" alt="Silo" width="112"><br>
|
||||
<img src=".github/silo-logo.svg" alt="Silo" width="160">
|
||||
</a>
|
||||
</h1>
|
||||
|
||||
|
||||
<p align="center">
|
||||
<strong>S3 兼容对象存储 —— 由 PIGSTY 维护的 MinIO 社区分支</strong>
|
||||
<strong>S3 兼容对象存储 —— 由 PGSTY 维护的 MinIO 社区分支</strong>
|
||||
</p>
|
||||
|
||||
|
||||
<p align="center">
|
||||
<a href="https://silo.pgsty.com/zh/">官网</a> ·
|
||||
<a href="https://silo.pgsty.com/zh/docs/">文档</a> ·
|
||||
<a href="https://silo.pgsty.com/zh/download/">下载</a> ·
|
||||
<a href="https://silo.pgsty.com/zh/tags/silo/">版本说明</a> ·
|
||||
<a href="https://silo.pgsty.com/zh/compatibility/server/">兼容性</a> ·
|
||||
<a href="https://silo.pgsty.com/zh/about/manifesto/">宣言</a> ·
|
||||
<a href="SECURITY.md">安全策略</a> ·
|
||||
<a href="README.md">English</a>
|
||||
</p>
|
||||
@@ -28,234 +30,141 @@
|
||||
</p>
|
||||
|
||||
> [!IMPORTANT]
|
||||
> Silo 是由 [Pigsty](https://pigsty.cc) 独立维护、从 [`pgsty/silo`](https://github.com/pgsty/silo) 发布的开源 MinIO 社区分支。本项目与 MinIO, Inc. 不存在隶属、背书或赞助关系;文中使用 “MinIO” 仅用于说明上游项目及兼容谱系。
|
||||
> **PGSTY Silo**(以下简称 Silo)是由 [Pigsty](https://pigsty.cc) 独立维护、从 [`pgsty/silo`](https://github.com/pgsty/silo) 发布的开源 MinIO 社区分支。本项目与 MinIO, Inc. 不存在隶属、背书或赞助关系;文中使用 “MinIO” 仅用于说明上游项目及兼容谱系。
|
||||
|
||||
> [!NOTE]
|
||||
> 2026-08-06,本仓库由 `pgsty/minio` 更名为 `pgsty/silo`,默认分支由 `master` 更名为 `main`。如果你需要以原本 MinIO 形态维持的归档构件,它们位于归档的 [`minio`](https://github.com/pgsty/silo/tree/minio) 分支,以及截止到 [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z) 的历次发布中;这些资产与 `docker.io/pgsty/minio` 镜像保持已发布状态且不做改动。本次更名只改变产品与交付物名称 —— `MINIO_*` 变量、`minio_*` 指标、`x-minio-*` 头、`/minio/*` 路由与磁盘格式均保持不变。
|
||||
> 2026-08-06,本仓库由 `pgsty/minio` 更名为 `pgsty/silo`,默认分支由 `master` 更名为 `main`。以原 MinIO 形态维持的归档构件仍位于归档的 [`minio`](https://github.com/pgsty/silo/tree/minio) 分支,以及截止 [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z) 的历次发布中。
|
||||
|
||||
## 当前发行版与主分支
|
||||
|
||||
最新已发布的 Server 仍为 [20260903](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z)。
|
||||
截至 2026-09-13,主分支已合入更新的安全、存储、Console 与共享包改动,但尚未发布新 Server。
|
||||
准确的已发布/源码边界见 [CHANGELOG.md](CHANGELOG.md) 与[组件版本矩阵](https://silo.pgsty.com/zh/compatibility/versions/),
|
||||
其中包括 SN-2026-011 修复状态与密码权限迁移要求。
|
||||
|
||||
## 概述
|
||||
|
||||
Silo 维护一条源自开源 MinIO 服务端的下游版本线,为上游停止社区发行后仍在运行 MinIO 兼容部署的用户提供持续构建与发行产物。Pigsty 使用 Silo 提供对象存储,包括作为可选的 PostgreSQL 备份仓库。
|
||||
上游停止社区发行后,Silo 为开源 MinIO 服务端维护一条持续可用的版本线:构建、软件包、多架构镜像、安全修复与完整 Web 控制台。Pigsty 在生产环境中用它承载 PostgreSQL 备份存储。
|
||||
|
||||
本分支遵循一条原则:**改名的是产品与交付物,不是协议与你的数据。** `silo` 可执行文件、软件包、服务与容器镜像使用新名称,而 S3 API、`MINIO_*` 变量、`minio_*` 指标、`x-minio-*` 头、保留路由与磁盘格式保持不变。具体差异记录在[兼容性说明](https://silo.pgsty.com/zh/compatibility/server/)中。
|
||||
它只遵循一条原则:**改名的是产品与交付物,不是协议与你的数据。** 其余内容都在 [silo.pgsty.com](https://silo.pgsty.com/zh/)。
|
||||
|
||||
项目统一门户为 [silo.pgsty.com](https://silo.pgsty.com/zh/),集中提供文档、下载安装、版本与安全动态及项目背景。
|
||||
**相关项目:**[`pgsty/mc`](https://github.com/pgsty/mc) 客户端(以 `mcli` 发行) · [`pgsty/silo-console`](https://github.com/pgsty/silo-console) · [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) · [`pgsty/pigsty`](https://github.com/pgsty/pigsty)
|
||||
|
||||
## 按需求选择入口
|
||||
<p align="center">
|
||||
<img src="https://silo.pgsty.com/images/silo-console/console-metrics-simple.webp" alt="Silo 控制台">
|
||||
</p>
|
||||
|
||||
| 需求 | 权威入口 |
|
||||
| :-- | :-- |
|
||||
| 项目概览与全站导航 | [Silo 官网](https://silo.pgsty.com/zh/) |
|
||||
| 安装方式与软件下载 | [下载与安装](https://silo.pgsty.com/zh/download/) |
|
||||
| 运维、管理、开发与参考指南 | [文档](https://silo.pgsty.com/zh/docs/) |
|
||||
| 本服务端的版本说明 | [Silo 版本说明](https://silo.pgsty.com/zh/tags/silo/) |
|
||||
| 与上游 MinIO 的差异 | [Silo vs. MinIO 兼容性说明](https://silo.pgsty.com/zh/compatibility/server/) |
|
||||
| 项目动态与安全通告 | [博客](https://silo.pgsty.com/zh/blog/) · [版本发布](https://silo.pgsty.com/zh/blog/release/)与[安全通告](https://silo.pgsty.com/zh/blog/security/) |
|
||||
| 带版本的二进制、校验和与源码归档 | [GitHub Releases](https://github.com/pgsty/silo/releases) |
|
||||
| 缺陷报告与功能讨论 | [GitHub Issues](https://github.com/pgsty/silo/issues) |
|
||||
| 私密漏洞报告 | [`SECURITY.md`](SECURITY.md) 与 [`VULNERABILITY_REPORT.md`](VULNERABILITY_REPORT.md) |
|
||||
| 许可证、署名与商标信息 | [`LICENSE`](LICENSE) · [`NOTICE`](NOTICE) · [`CREDITS`](CREDITS) · 门户的[许可证](https://silo.pgsty.com/zh/about/license/)、[署名归属](https://silo.pgsty.com/zh/about/attribution/)与[商标政策](https://silo.pgsty.com/zh/about/trademark/)页面 |
|
||||
|
||||
## 相关项目
|
||||
|
||||
| 仓库 | 说明 |
|
||||
| :-- | :-- |
|
||||
| [`pgsty/silo`](https://github.com/pgsty/silo) | 本仓库 —— Silo 对象存储服务器,S3 兼容的 MinIO fork |
|
||||
| [`pgsty/mc`](https://github.com/pgsty/mc) | Silo 命令行客户端,以 `mcli` 发行、保留 `mc` 命令名 |
|
||||
| [`pgsty/silo-console`](https://github.com/pgsty/silo-console) | Web 管理控制台,内嵌于服务端构建 |
|
||||
| [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) | 为 Silo 系 fork 维护的共享 Go 包 |
|
||||
| [`pgsty/pigsty`](https://github.com/pgsty/pigsty) | Pigsty —— 内置 Silo 作为对象存储的 PostgreSQL 发行版 |
|
||||
|
||||
## 维护政策
|
||||
|
||||
活跃版本线的维护范围包括:
|
||||
|
||||
- 构建与依赖项维护;
|
||||
- 适用的安全修复与公告;
|
||||
- 针对可复现缺陷的范围明确的修复;
|
||||
- 带版本的二进制、软件包、校验和与多架构镜像;
|
||||
- Web Console、客户端、文档与 Pigsty 集成。
|
||||
|
||||
改动保持克制,并在可行时提供测试。所有维护均为尽力而为,不承诺固定的响应、修复或发布时间。
|
||||
|
||||
### 范围之外
|
||||
|
||||
- 独立产品路线图、新存储引擎或假设性的 S3 新特性;
|
||||
- 大规模重写或显著扩大下游差异的改动;
|
||||
- 历史版本或多条支持分支;
|
||||
- 商业支持、SLA、7×24 服务或 SUBNET 服务;
|
||||
- 部署设计、访问控制、监控、备份与恢复。
|
||||
|
||||
## 治理
|
||||
|
||||
服务端、客户端、控制台与共享包在同一套发布流程下共同维护:提交必须签署 DCO、Pull Request 经过评审、以 `RELEASE.YYYY-MM-DDTHH-MM-SSZ` 标签发布,产物附带校验和、SPDX SBOM、Sigstore 签名清单与 GitHub 构建证明。每个版本都会在门户发布[版本说明](https://silo.pgsty.com/zh/tags/silo/);安全问题按照 [`SECURITY.md`](SECURITY.md) 与仓库[安全公告](docs/security/advisories.md)处理。上游版权、许可证与第三方声明完整保留于 [`LICENSE`](LICENSE)、[`NOTICE`](NOTICE) 与 [`CREDITS`](CREDITS)。
|
||||
|
||||
## 兼容策略
|
||||
|
||||
Silo 尽量保留:
|
||||
|
||||
- `github.com/minio/minio` module path 与 `github.com/minio/*` 导入路径;
|
||||
- MinIO 兼容的 S3 API、线协议、`MINIO_*` 环境变量、指标、协议头、保留路由与存储元数据;
|
||||
- `RELEASE.YYYY-MM-DDTHH-MM-SSZ` 标签,以及容器入口对旧式 `minio …` 参数的转换。
|
||||
|
||||
Silo 自有交付面统一使用 `silo` 可执行文件、软件包、服务、Helm Chart 与 `pgsty/silo` 容器镜像;原生交付物不会安装 `minio` 二进制别名。
|
||||
|
||||
兼容性是默认约束。只要不会留下安全问题,Silo 就保留既有的协议、客户端、配置与运维行为;只有在修复重大安全问题确有必要时才会打破兼容,并在版本说明中明确受影响行为与迁移方式。每个版本仍应视为下游升级:锁定版本,阅读[版本说明](https://silo.pgsty.com/zh/tags/silo/)、[兼容性说明](https://silo.pgsty.com/zh/compatibility/server/)与[安全公告](docs/security/advisories.md),保留回滚路径,并在生产使用前完成测试。
|
||||
|
||||
## 下载与发行产物
|
||||
|
||||
请先在[下载与安装](https://silo.pgsty.com/zh/download/)页面选择合适的安装方式;GitHub Releases 仍是带版本服务端二进制、校验和与源码归档的获取位置。
|
||||
|
||||
| 产物 | 位置 |
|
||||
| :-- | :-- |
|
||||
| 源码 | [`github.com/pgsty/silo`](https://github.com/pgsty/silo) |
|
||||
| 容器镜像 | [`pgsty/silo`](https://hub.docker.com/r/pgsty/silo),支持 `linux/amd64` 与 `linux/arm64` 多架构清单 |
|
||||
| 服务端二进制与校验和 | [GitHub Releases](https://github.com/pgsty/silo/releases),覆盖 Linux、macOS、Windows 的 `amd64` 与 `arm64` |
|
||||
| Linux 软件包 | RPM、DEB、APK,并通过 [Pigsty 软件仓库](https://pigsty.cc/docs/repo/) 分发 |
|
||||
| 客户端 | [`pgsty/mc`](https://github.com/pgsty/mc),容器内以 `mcli` 提供,并保留 `mc` 兼容别名 |
|
||||
| 管理控制台 | [`pgsty/silo-console`](https://github.com/pgsty/silo-console),通过兼容导入路径 `github.com/minio/console` 嵌入服务端构建 |
|
||||
| 共享库 | [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) v3.11.0,通过 `replace` 指令使用,同时保留 `github.com/minio/pkg/v3` 导入路径 |
|
||||
|
||||
每个新版本都会为各平台归档和软件包发布 SPDX JSON SBOM。归档与软件包的校验和清单分别带有无长期密钥的 Sigstore 签名包;GitHub 制品证明则为全部可下载产物及多架构容器镜像记录已签名的构建来源。
|
||||
|
||||
下载归档及配套文件后,请分别验证完整性、已发布 SBOM、签名清单与构建来源:
|
||||
## 快速上手
|
||||
|
||||
```bash
|
||||
# 完整性:选择与你下载产物相符的一行。
|
||||
grep -F ' silo_<version>_linux_amd64.tar.gz' \
|
||||
silo_<version>_checksums.txt | sha256sum --check
|
||||
|
||||
# 归档 SBOM 是另一个独立校验的 Release 产物。
|
||||
grep -F ' silo_<version>_linux_amd64.tar.gz.sbom.json' \
|
||||
silo_<version>_checksums.txt | sha256sum --check
|
||||
|
||||
# 验证归档/SBOM 校验和清单的签名。
|
||||
cosign verify-blob \
|
||||
--bundle silo_<version>_checksums.txt.sigstore.json \
|
||||
--certificate-identity-regexp \
|
||||
'^https://github.com/pgsty/(minio|silo)/\.github/workflows/release\.yml@refs/(tags/RELEASE\..+|heads/(master|main))$' \
|
||||
--certificate-oidc-issuer https://token.actions.githubusercontent.com \
|
||||
silo_<version>_checksums.txt
|
||||
|
||||
# 在线验证本仓库签发的构建来源。
|
||||
gh attestation verify silo_<version>_linux_amd64.tar.gz \
|
||||
--repo pgsty/silo
|
||||
```
|
||||
|
||||
软件包使用 `silo_<version>_packages_checksums.txt` 及其相邻的 Sigstore
|
||||
签名包,并使用相同的 identity 与 issuer 约束;请用 SPDX 工具检查已验证的
|
||||
SPDX JSON SBOM。按 digest 验证多架构容器清单的构建来源:
|
||||
|
||||
```bash
|
||||
gh attestation verify \
|
||||
oci://index.docker.io/pgsty/silo@sha256:<manifest-digest> \
|
||||
--repo pgsty/silo
|
||||
```
|
||||
|
||||
分架构 SBOM 证明附在 `amd64` 与 `arm64` 平台镜像的 digest 上,而非多架构
|
||||
清单上,需要分别显式验证:
|
||||
|
||||
```bash
|
||||
gh attestation verify \
|
||||
oci://index.docker.io/pgsty/silo@sha256:<platform-digest> \
|
||||
--repo pgsty/silo \
|
||||
--predicate-type https://spdx.dev/Document/v2.3
|
||||
```
|
||||
|
||||
按 digest 验证可避免信任可变镜像标签。
|
||||
|
||||
### 原生软件包迁移
|
||||
|
||||
`silo` RPM、DEB 与 APK 不针对 `minio` 声明 `Provides`、`Obsoletes`、
|
||||
`Replaces` 或包级 `Conflicts`,因此可以与已有 MinIO 软件包并存安装,
|
||||
不会被普通升级静默替换。两个 systemd unit 在运行时互斥,应由管理员显式
|
||||
切换,不能同时启动。
|
||||
|
||||
切换前请记录旧 unit 的 enabled/active 状态与 `User`/`Group`,并备份
|
||||
`/etc/default/minio`。Silo 先读取该旧配置,再读取 `/etc/default/silo`;后者
|
||||
中由管理员设置的同名变量优先。如现有数据必须继续使用原 UID/GID,请创建
|
||||
`/etc/systemd/system/silo.service.d/10-legacy-user.conf`:
|
||||
|
||||
```ini
|
||||
[Service]
|
||||
User=<legacy-user>
|
||||
Group=<legacy-group>
|
||||
```
|
||||
|
||||
执行 `systemctl daemon-reload`,先停用并停止 `minio.service`,再启用并启动
|
||||
`silo.service`。确认健康检查、S3、Admin API、指标与日志后,才 mask 或卸载
|
||||
旧服务。软件包迁移期间不要递归修改数据属主;在回滚观察窗口内保留旧包与
|
||||
旧 unit。
|
||||
|
||||
## 快速开始
|
||||
|
||||
本地体验:
|
||||
|
||||
```bash
|
||||
mkdir -p data
|
||||
|
||||
export MINIO_ROOT_USER=minioadmin
|
||||
export MINIO_ROOT_PASSWORD=change-me-long-password
|
||||
|
||||
docker run -d --name silo \
|
||||
-p 9000:9000 \
|
||||
-p 9001:9001 \
|
||||
-e MINIO_ROOT_USER \
|
||||
-e MINIO_ROOT_PASSWORD \
|
||||
docker run -d --name silo -p 9000:9000 -p 9001:9001 \
|
||||
-e MINIO_ROOT_USER=minioadmin \
|
||||
-e MINIO_ROOT_PASSWORD=change-me-long-password \
|
||||
-v "$PWD/data:/data" \
|
||||
docker.io/pgsty/silo:latest server /data --console-address ":9001"
|
||||
```
|
||||
|
||||
管理控制台位于 <http://localhost:9001>,S3 API 位于 <http://localhost:9000>。
|
||||
<p align="center">
|
||||
<img src="https://silo.pgsty.com/images/silo-console/console-login.webp" alt="Silo 控制台">
|
||||
</p>
|
||||
|
||||
镜像内置兼容客户端 `mcli`:
|
||||
控制台位于 <http://localhost:9001>,S3 API 位于 <http://localhost:9000>。镜像内置客户端 `mcli`:
|
||||
|
||||
```bash
|
||||
docker exec silo mcli alias set local http://127.0.0.1:9000 \
|
||||
"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
|
||||
docker exec silo mcli mb local/demo
|
||||
docker exec silo mcli ls local
|
||||
docker exec silo mcli alias set local http://127.0.0.1:9000 minioadmin change-me-long-password
|
||||
docker exec silo mcli mb local/demo && docker exec silo mcli ls local
|
||||
```
|
||||
|
||||
> [!WARNING]
|
||||
> 生产环境应锁定版本,使用独立凭据与 TLS,配置监控,保留独立备份,并验证恢复流程。
|
||||
> 生产环境应锁定版本,使用独立凭据与 TLS,配置监控,保留独立备份,并验证恢复流程。请从[文档](https://silo.pgsty.com/zh/docs/)开始。
|
||||
|
||||
从源码构建服务端:
|
||||
## 安装
|
||||
|
||||
```bash
|
||||
go build -o silo .
|
||||
./silo --version
|
||||
```
|
||||
| 方式 | 位置 |
|
||||
| :-- | :-- |
|
||||
| 容器镜像 | [`pgsty/silo`](https://hub.docker.com/r/pgsty/silo),支持 `linux/amd64` 与 `linux/arm64` |
|
||||
| 二进制 | [GitHub Releases](https://github.com/pgsty/silo/releases),覆盖 Linux、macOS、Windows 的 `amd64` 与 `arm64` |
|
||||
| 软件包 | RPM、DEB、APK,也可通过 [Pigsty 软件仓库](https://pigsty.cc/docs/repo/) 安装 |
|
||||
| Kubernetes | Helm Chart,参见[下载与安装](https://silo.pgsty.com/zh/download/) |
|
||||
| 源码构建 | `go build -o silo . && ./silo --version` |
|
||||
|
||||
其他安装方式(包括原生软件包、二进制、Podman、Kubernetes、源码构建与 Pigsty Ansible)请前往[下载与安装](https://silo.pgsty.com/zh/download/);生产部署与管理请从 [Silo 文档](https://silo.pgsty.com/zh/docs/)开始。Pigsty 用户也可以直接使用 [Pigsty MinIO 模块](https://pigsty.cc/docs/minio/)。
|
||||
每个版本都附带校验和、SPDX SBOM、Sigstore 签名清单与 GitHub 构建证明。完整安装方式与验证命令见[下载与安装](https://silo.pgsty.com/zh/download/);从上游 MinIO 迁移 —— 接管既有 `minio.service` 与 `/etc/default/minio`,并用 `/etc/systemd/system/silo.service.d/10-legacy-user.conf` drop-in 保持数据属主不变 —— 见[迁移指南](https://silo.pgsty.com/zh/compatibility/migration/)与[二进制与服务说明](https://silo.pgsty.com/zh/compatibility/binary/)。
|
||||
|
||||
## 安全
|
||||
## 兼容性
|
||||
|
||||
安全修复面向当前活跃开发分支,并记录在仓库[安全公告](docs/security/advisories.md)与门户[安全通告](https://silo.pgsty.com/zh/blog/security/)中。请按照 [`SECURITY.md`](SECURITY.md) 与 [`VULNERABILITY_REPORT.md`](VULNERABILITY_REPORT.md) 私密报告漏洞;同时影响上游 MinIO 的问题也应向上游报告。
|
||||
S3 API、`MINIO_*` 环境变量、`minio_*` 指标、`x-minio-*` 头、`/minio/*` 路由、`github.com/minio/*` 导入路径与磁盘格式(含 `.minio.sys`)原样保留,并由 CI 兼容性门禁冻结。只有 Silo 自有交付面改名:`silo` 可执行文件、软件包、服务、Helm Chart 与容器镜像 —— 原生交付物不会安装 `minio` 二进制别名。
|
||||
|
||||
## 贡献
|
||||
与上游的全部分歧,以逐项核验代码的[兼容性审计](https://silo.pgsty.com/zh/compatibility/server/)形式维护。每个版本仍应视为下游升级:锁定版本,阅读[版本说明](https://silo.pgsty.com/zh/tags/silo/),并保留回滚路径。
|
||||
|
||||
欢迎安全与依赖项更新、可复现缺陷修复、测试、发布自动化、打包与文档改进。
|
||||
## 安全与贡献
|
||||
|
||||
Issue 与 Pull Request 应说明受影响版本、复现步骤、影响、预期行为、测试与兼容性说明。大型改动请先提交 Issue 讨论。
|
||||
请按照 [`SECURITY.md`](SECURITY.md) 私密报告漏洞;每项修复都会发布公开[安全公告](https://silo.pgsty.com/zh/blog/security/)。本项目不要求签署 CLA:贡献按 AGPL-3.0-or-later(inbound=outbound)接收,只需 DCO 签署(`git commit -s`),详见 [`CONTRIBUTING.md`](CONTRIBUTING.md)。
|
||||
|
||||
本项目不要求签署 CLA:贡献按项目许可证(AGPL-3.0-or-later,inbound=outbound)接收,贡献者保留自己的版权。每个提交都必须按照 [DCO](https://developercertificate.org/) 签署(`git commit -s`),详见 [`CONTRIBUTING.md`](CONTRIBUTING.md)。
|
||||
## 贡献者
|
||||
|
||||
**41 位社区贡献者**共同建设 SILO、Console、mcli、公共包与相关项目。名单包含维护者,以及所有提出 Issue 或 PR 的真人作者;按已合并 PR、其他 PR、Issue 报告排序,黄圈标记显著贡献。
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/Vonng"><img src="https://silo.pgsty.com/images/contributors/Vonng.svg" width="60" height="60" alt="@Vonng" title="@Vonng — 维护 SILO、Console、mcli、公共包、发行与文档"></a>
|
||||
<a href="https://github.com/h5vx"><img src="https://silo.pgsty.com/images/contributors/h5vx.svg" width="60" height="60" alt="@h5vx" title="@h5vx — 实现单桶 CORS 配置与请求执行"></a>
|
||||
<a href="https://github.com/mrjavadseydi"><img src="https://silo.pgsty.com/images/contributors/mrjavadseydi.svg" width="60" height="60" alt="@mrjavadseydi" title="@mrjavadseydi — 修复有效桶配额指标,并提交按访问频率分层的 ILM 方案"></a>
|
||||
<a href="https://github.com/Dansyuqri"><img src="https://silo.pgsty.com/images/contributors/Dansyuqri.svg" width="60" height="60" alt="@Dansyuqri" title="@Dansyuqri — 为分片上传完成响应补充 ChecksumType"></a>
|
||||
<a href="https://github.com/ycjlin"><img src="https://silo.pgsty.com/images/contributors/ycjlin.svg" width="60" height="60" alt="@ycjlin" title="@ycjlin — 修复缺失桶的 ListObjects 语义"></a>
|
||||
<a href="https://github.com/pinginfo"><img src="https://silo.pgsty.com/images/contributors/pinginfo.svg" width="60" height="60" alt="@pinginfo" title="@pinginfo — 修复桶通知的流式输出"></a>
|
||||
<a href="https://github.com/ZouhairCharef"><img src="https://silo.pgsty.com/images/contributors/ZouhairCharef.svg" width="60" height="60" alt="@ZouhairCharef" title="@ZouhairCharef — 修复 go-jose 中的 CVE-2026-34986"></a>
|
||||
<a href="https://github.com/mfredenhagen"><img src="https://silo.pgsty.com/images/contributors/mfredenhagen.svg" width="60" height="60" alt="@mfredenhagen" title="@mfredenhagen — 修复 OpenTelemetry 中的 CVE-2026-39883"></a>
|
||||
<a href="https://github.com/waterkip"><img src="https://silo.pgsty.com/images/contributors/waterkip.svg" width="60" height="60" alt="@waterkip" title="@waterkip — 将文档链接指向 SILO 门户"></a>
|
||||
<a href="https://github.com/mikemikimike"><img src="https://silo.pgsty.com/images/contributors/mikemikimike.svg" width="60" height="60" alt="@mikemikimike" title="@mikemikimike — 提交 SSE-C 复制分片明文尺寸修复"></a>
|
||||
<a href="https://github.com/metaneutrons"><img src="https://silo.pgsty.com/images/contributors/metaneutrons.svg" width="60" height="60" alt="@metaneutrons" title="@metaneutrons — 报告并提交显式版本删除鉴权方案"></a>
|
||||
<a href="https://github.com/magicxor"><img src="https://silo.pgsty.com/images/contributors/magicxor.svg" width="60" height="60" alt="@magicxor" title="@magicxor — 报告并提交 DELETE If-Match 条件请求支持方案"></a>
|
||||
<a href="https://github.com/davinkevin"><img src="https://silo.pgsty.com/images/contributors/davinkevin.svg" width="60" height="60" alt="@davinkevin" title="@davinkevin — 提交 distroless 容器镜像与依赖自动更新方案"></a>
|
||||
<a href="https://github.com/lem21h"><img src="https://silo.pgsty.com/images/contributors/lem21h.svg" width="48" height="48" alt="@lem21h" title="@lem21h — 提交健壮性与 goroutine 改进"></a>
|
||||
<a href="https://github.com/sulin37392"><img src="https://silo.pgsty.com/images/contributors/sulin37392.svg" width="48" height="48" alt="@sulin37392" title="@sulin37392 — 提交依赖更新"></a>
|
||||
<a href="https://github.com/cbornet"><img src="https://silo.pgsty.com/images/contributors/cbornet.svg" width="60" height="60" alt="@cbornet" title="@cbornet — 报告分片与流式校验和缺陷及缺失桶语义问题"></a>
|
||||
<a href="https://github.com/vampywiz17"><img src="https://silo.pgsty.com/images/contributors/vampywiz17.svg" width="60" height="60" alt="@vampywiz17" title="@vampywiz17 — 报告 LDAP TLS 与 Console 登录回归"></a>
|
||||
<a href="https://github.com/orenyomtov"><img src="https://silo.pgsty.com/images/contributors/orenyomtov.svg" width="60" height="60" alt="@orenyomtov" title="@orenyomtov — 报告未签名头导致的 CopyObject 跨对象读取(SN-2026-011)"></a>
|
||||
<a href="https://github.com/mumu-lab"><img src="https://silo.pgsty.com/images/contributors/mumu-lab.svg" width="48" height="48" alt="@mumu-lab" title="@mumu-lab — 报告桶配额指标读取已弃用字段的问题"></a>
|
||||
<a href="https://github.com/jvasile"><img src="https://silo.pgsty.com/images/contributors/jvasile.svg" width="48" height="48" alt="@jvasile" title="@jvasile — 报告 Debian 包缺少用户、用户组与默认配置"></a>
|
||||
<a href="https://github.com/pmezhuev"><img src="https://silo.pgsty.com/images/contributors/pmezhuev.svg" width="48" height="48" alt="@pmezhuev" title="@pmezhuev — 报告 RPM 包缺少 GPG 签名"></a>
|
||||
<a href="https://github.com/TLINDEN"><img src="https://silo.pgsty.com/images/contributors/TLINDEN.svg" width="48" height="48" alt="@TLINDEN" title="@TLINDEN — 报告发布压缩包缺少客户端"></a>
|
||||
<a href="https://github.com/makinikm"><img src="https://silo.pgsty.com/images/contributors/makinikm.svg" width="48" height="48" alt="@makinikm" title="@makinikm — 报告容器镜像缺少客户端"></a>
|
||||
<a href="https://github.com/meesudzu"><img src="https://silo.pgsty.com/images/contributors/meesudzu.svg" width="48" height="48" alt="@meesudzu" title="@meesudzu — 提出从上游 MinIO 迁移的指南需求"></a>
|
||||
<a href="https://github.com/kuldeep-link11"><img src="https://silo.pgsty.com/images/contributors/kuldeep-link11.svg" width="48" height="48" alt="@kuldeep-link11" title="@kuldeep-link11 — 报告 NATS JWT 凭据与通知目标重载问题"></a>
|
||||
<a href="https://github.com/sargarass"><img src="https://silo.pgsty.com/images/contributors/sargarass.svg" width="48" height="48" alt="@sargarass" title="@sargarass — 报告 ListMultipartUploads 前缀与分页语义问题"></a>
|
||||
<a href="https://github.com/liuhaodongliu990-cmyk"><img src="https://silo.pgsty.com/images/contributors/liuhaodongliu990-cmyk.svg" width="48" height="48" alt="@liuhaodongliu990-cmyk" title="@liuhaodongliu990-cmyk — 报告前缀下载进度显示异常"></a>
|
||||
<a href="https://github.com/Xavier-777"><img src="https://silo.pgsty.com/images/contributors/Xavier-777.svg" width="48" height="48" alt="@Xavier-777" title="@Xavier-777 — 报告 Console 生命周期管理与文件预览缺失"></a>
|
||||
<a href="https://github.com/spaceg00se-r"><img src="https://silo.pgsty.com/images/contributors/spaceg00se-r.svg" width="48" height="48" alt="@spaceg00se-r" title="@spaceg00se-r — 提出 cpuv1 支持需求并报告工作流令牌错误"></a>
|
||||
<a href="https://github.com/kh0mka"><img src="https://silo.pgsty.com/images/contributors/kh0mka.svg" width="48" height="48" alt="@kh0mka" title="@kh0mka — 报告 ReadFileStreamHandler 节点间 I/O 超时"></a>
|
||||
<a href="https://github.com/bagutzu"><img src="https://silo.pgsty.com/images/contributors/bagutzu.svg" width="48" height="48" alt="@bagutzu" title="@bagutzu — 提出兼容 KES 的外部 KMS 与 OpenBao 支持需求"></a>
|
||||
<a href="https://github.com/DestroyLee"><img src="https://silo.pgsty.com/images/contributors/DestroyLee.svg" width="48" height="48" alt="@DestroyLee" title="@DestroyLee — 报告文档目录导航缺失"></a>
|
||||
<a href="https://github.com/mosesdd"><img src="https://silo.pgsty.com/images/contributors/mosesdd.svg" width="48" height="48" alt="@mosesdd" title="@mosesdd — 提出维护 Helm Chart 的需求"></a>
|
||||
<a href="https://github.com/zylpsrs"><img src="https://silo.pgsty.com/images/contributors/zylpsrs.svg" width="48" height="48" alt="@zylpsrs" title="@zylpsrs — 报告 Console 缺少分层与站点复制"></a>
|
||||
<a href="https://github.com/heroes1412"><img src="https://silo.pgsty.com/images/contributors/heroes1412.svg" width="48" height="48" alt="@heroes1412" title="@heroes1412 — 报告性能分析选项不可用"></a>
|
||||
<a href="https://github.com/redfoxfox"><img src="https://silo.pgsty.com/images/contributors/redfoxfox.svg" width="48" height="48" alt="@redfoxfox" title="@redfoxfox — 报告中文文档站点不可用"></a>
|
||||
<a href="https://github.com/jiadzh"><img src="https://silo.pgsty.com/images/contributors/jiadzh.svg" width="48" height="48" alt="@jiadzh" title="@jiadzh — 提出 Windows 构建指导需求"></a>
|
||||
<a href="https://github.com/AntonOfTheWoods"><img src="https://silo.pgsty.com/images/contributors/AntonOfTheWoods.svg" width="48" height="48" alt="@AntonOfTheWoods" title="@AntonOfTheWoods — 提出明确 Helm Chart 与 Operator 选项的需求"></a>
|
||||
<a href="https://github.com/chalukyaj"><img src="https://silo.pgsty.com/images/contributors/chalukyaj.svg" width="48" height="48" alt="@chalukyaj" title="@chalukyaj — 提出改善 SILO Operator 可发现性的建议"></a>
|
||||
<a href="https://github.com/nsanitate"><img src="https://silo.pgsty.com/images/contributors/nsanitate.svg" width="48" height="48" alt="@nsanitate" title="@nsanitate — 提出加入 CNCF Sandbox 的治理建议"></a>
|
||||
<a href="https://github.com/Kesavaambati"><img src="https://silo.pgsty.com/images/contributors/Kesavaambati.svg" width="48" height="48" alt="@Kesavaambati" title="@Kesavaambati — 提出社区支持与容器镜像维护问题"></a>
|
||||
</p>
|
||||
|
||||
[查看完整贡献记录](CONTRIBUTORS.md),了解每位贡献者的提案、修复与问题报告。
|
||||
|
||||
## 背景
|
||||
|
||||
本项目源于上游社区发行与维护模式的变化。维护者对相关变化的分析、替代方案评估与早期维护记录见以下文章:
|
||||
本项目因上游收缩社区版而生:Web 控制台被削减为残桩、社区预编译制品停发、社区仓库被归档。Silo 的存在就是让这些部署继续跑下去。Fork 是手段,不是身份 —— 若上游恢复社区版承诺,我们乐意收缩范围,并把修复回馈上游。
|
||||
|
||||
| 文章 | 主题 |
|
||||
| :-- | :-- |
|
||||
| [MinIO已死](https://silo.pgsty.com/zh/blog/post/minio-is-dead/) | 上游项目与发行模式的变化 |
|
||||
| [MinIO已死,谁能接盘?](https://silo.pgsty.com/zh/blog/post/minio-alternative/) | 可选替代方案评估 |
|
||||
| [MinIO 已死,MinIO 复生](https://silo.pgsty.com/zh/blog/post/minio-resurrect/) | 建立分支及其发行流水线 |
|
||||
| [续命 MinIO:承诺兑现](https://silo.pgsty.com/zh/blog/post/minio-promise-kept/) | 初期安全与维护工作 |
|
||||
[**宣言**](https://silo.pgsty.com/zh/about/manifesto/)是项目的公开承诺,共十一条,通篇遵循一项纪律:**每一条,要么是已经在做且有公开证据的事实,要么是刻意拒绝的承诺。** 摘要:
|
||||
|
||||
- **兼容性合同** —— 协议与数据不改,每个版本都标注经过测试的回滚目标与路径。
|
||||
- **许可证无法变更** —— AGPLv3、无 CLA、不做版权聚合;包括我们自己在内,没有人握有足够版权代表所有贡献者重新授权。
|
||||
- **永不清单**(只增不减)—— 永不将既有功能移入付费墙、永不给下载设注册墙、永不加入遥测(上游回连路径已整体移除)、永不引入 CLA、永不变更许可证、永不以商标追究正常使用。
|
||||
- **安全与发布纪律** —— 每项安全修复配一篇公开公告;通常每一到两个月发布一版,最长不超过一个季度。请拿公开记录检验这两条。
|
||||
|
||||
延伸阅读:[MinIO已死](https://silo.pgsty.com/zh/blog/post/minio-is-dead/) · [谁能接盘?](https://silo.pgsty.com/zh/blog/post/minio-alternative/) · [MinIO 复生](https://silo.pgsty.com/zh/blog/post/minio-resurrect/) · [承诺兑现](https://silo.pgsty.com/zh/blog/post/minio-promise-kept/)
|
||||
|
||||
## 许可证与商标
|
||||
|
||||
服务端源码依据 [GNU Affero 通用公共许可证第 3 版或任何后续版本](LICENSE)分发。本分支衍生自 [`minio/minio`](https://github.com/minio/minio):[`NOTICE`](NOTICE) 保留上游产品声明,[`CREDITS`](CREDITS) 记录所含第三方组件的许可证与声明,Git 历史记录下游修改。
|
||||
Silo 采用 [AGPL-3.0-or-later](LICENSE),衍生自 [`minio/minio`](https://github.com/minio/minio),上游版权与第三方声明完整保留于 [`NOTICE`](NOTICE) 与 [`CREDITS`](CREDITS)。MinIO 是 MinIO, Inc. 的商标,此处使用仅为标识上游项目与兼容谱系。
|
||||
|
||||
MinIO 是 MinIO, Inc. 的商标;本文仅为标识上游项目与兼容谱系而使用该名称。Pigsty、Silo 与 `pgsty/silo` 均为独立社区项目,与 MinIO, Inc. 不存在隶属、背书或赞助关系。
|
||||
|
||||
门户另行发布项目的[许可证摘要](https://silo.pgsty.com/zh/about/license/)、[文档署名归属](https://silo.pgsty.com/zh/about/attribution/)与[商标声明](https://silo.pgsty.com/zh/about/trademark/)。
|
||||
详见:[许可证](https://silo.pgsty.com/zh/about/license/) · [署名归属](https://silo.pgsty.com/zh/about/attribution/) · [商标声明](https://silo.pgsty.com/zh/about/trademark/)
|
||||
|
||||
+19
@@ -10,6 +10,25 @@ Security fixes are tracked on the active development branch and summarized in
|
||||
[docs/security/advisories.md](docs/security/advisories.md). Only the current
|
||||
Silo release line is supported unless an advisory says otherwise.
|
||||
|
||||
## Inherited Fix Evidence
|
||||
|
||||
The canonical ledger also records security fixes inherited from upstream when
|
||||
they are part of the Silo release baseline. Source and fork commits are linked
|
||||
separately even when the fork preserves the original commit object and SHA.
|
||||
|
||||
- [CVE-2025-62506](https://github.com/advisories/GHSA-jjjj-jwhf-8rgr):
|
||||
upstream [PR #21642](https://github.com/minio/minio/pull/21642) merged as
|
||||
[`minio/minio@c1a49490`](https://github.com/minio/minio/commit/c1a49490c78e9c3ebcad86ba0662319138ace190),
|
||||
inherited unchanged as
|
||||
[`pgsty/silo@c1a49490`](https://github.com/pgsty/silo/commit/c1a49490c78e9c3ebcad86ba0662319138ace190),
|
||||
and is present in every Silo community release beginning with
|
||||
[`RELEASE.2025-12-03T12-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2025-12-03T12-00-00Z).
|
||||
The inherited [service-account](https://github.com/pgsty/silo/blob/c1a49490c78e9c3ebcad86ba0662319138ace190/cmd/admin-handlers-users_test.go#L211-L212)
|
||||
and [STS](https://github.com/pgsty/silo/blob/c1a49490c78e9c3ebcad86ba0662319138ace190/cmd/sts-handlers_test.go#L45-L46)
|
||||
regression groups remain part of `go test ./cmd`; see the
|
||||
[canonical ledger](docs/security/advisories.md#inherited-upstream-advisory-baseline)
|
||||
for the operator-facing record.
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
For vulnerabilities in this fork:
|
||||
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Fail closed before a release job can replace published or finalized assets.
|
||||
# An ordinary Draft is retry state; a finalized Draft contains GPG-derived
|
||||
# materials and must never be replaced by the build lane.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
release_tag="${1:-}"
|
||||
fixture="${2:-}"
|
||||
repository="${GITHUB_REPOSITORY:-pgsty/silo}"
|
||||
require_draft="${REQUIRE_DRAFT:-false}"
|
||||
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
echo "jq is required to inspect GitHub release state" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! "${release_tag}" =~ ^RELEASE\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}Z$ ]]; then
|
||||
echo "Invalid release tag format: ${release_tag:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -n "${fixture}" ]; then
|
||||
release_json="$(<"${fixture}")"
|
||||
else
|
||||
error_file="$(mktemp)"
|
||||
trap 'rm -f "${error_file}"' EXIT
|
||||
if ! release_json="$(
|
||||
gh api --paginate "repos/${repository}/releases?per_page=100" --jq '.[]' 2>"${error_file}" |
|
||||
jq --arg tag "${release_tag}" -s '[.[] | select(.tag_name == $tag)]'
|
||||
)"; then
|
||||
cat "${error_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! jq -e 'type == "array" and all(.[]; type == "object" and (.tag_name | type == "string") and (.draft | type == "boolean"))' \
|
||||
<<<"${release_json}" >/dev/null 2>&1; then
|
||||
echo "Invalid release state response for ${release_tag}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! jq -e --arg tag "${release_tag}" 'all(.[]; .tag_name == $tag)' \
|
||||
<<<"${release_json}" >/dev/null 2>&1; then
|
||||
echo "Release state returned a tag other than ${release_tag}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
release_count="$(jq 'length' <<<"${release_json}")"
|
||||
if [ "${release_count}" -eq 0 ]; then
|
||||
if [ "${require_draft}" = "true" ]; then
|
||||
echo "Expected one Draft release for ${release_tag}, found none" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "No existing release for ${release_tag}."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "${release_count}" -ne 1 ]; then
|
||||
echo "Refusing to choose among ${release_count} releases for ${release_tag}; clean duplicate Drafts first" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$(jq -r '.[0].draft' <<<"${release_json}")" != "true" ]; then
|
||||
echo "Refusing to overwrite published release ${release_tag}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
finalize_markers="$(jq '[.[0].assets[]? | select(.name | endswith("_packages_provenance.sigstore.json"))] | length' <<<"${release_json}")"
|
||||
if [ "${finalize_markers}" -ne 0 ]; then
|
||||
echo "Refusing to replace finalized Draft ${release_tag}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Existing unfinalized Draft ${release_tag} will be replaced from scratch."
|
||||
Executable
+68
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
checker="${script_dir}/check-release-state.sh"
|
||||
tag="RELEASE.2026-08-29T00-00-00Z"
|
||||
fixture="$(mktemp)"
|
||||
stdout_file="$(mktemp)"
|
||||
stderr_file="$(mktemp)"
|
||||
trap 'rm -f "${fixture}" "${stdout_file}" "${stderr_file}"' EXIT
|
||||
|
||||
expect_success() {
|
||||
if ! "${checker}" "$@" >"${stdout_file}" 2>"${stderr_file}"; then
|
||||
cat "${stderr_file}" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
expect_failure() {
|
||||
if "${checker}" "$@" >"${stdout_file}" 2>"${stderr_file}"; then
|
||||
echo "Expected release-state check to fail: $*" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
printf '[]\n' >"${fixture}"
|
||||
expect_success "${tag}" "${fixture}"
|
||||
grep -qF "No existing release for ${tag}." "${stdout_file}"
|
||||
|
||||
if REQUIRE_DRAFT=true "${checker}" "${tag}" "${fixture}" >"${stdout_file}" 2>"${stderr_file}"; then
|
||||
echo "Expected required-Draft check to fail when no release exists" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -qF "Expected one Draft release for ${tag}, found none" "${stderr_file}"
|
||||
|
||||
printf '[{"tag_name":"%s","draft":true,"assets":[]}]\n' "${tag}" >"${fixture}"
|
||||
expect_success "${tag}" "${fixture}"
|
||||
grep -qF "Existing unfinalized Draft ${tag} will be replaced from scratch." "${stdout_file}"
|
||||
if ! REQUIRE_DRAFT=true "${checker}" "${tag}" "${fixture}" >"${stdout_file}" 2>"${stderr_file}"; then
|
||||
cat "${stderr_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf '[{"tag_name":"%s","draft":true,"assets":[{"name":"silo_20260829000000.0.0_packages_provenance.sigstore.json"}]}]\n' "${tag}" >"${fixture}"
|
||||
expect_failure "${tag}" "${fixture}"
|
||||
grep -qF "Refusing to replace finalized Draft ${tag}" "${stderr_file}"
|
||||
|
||||
printf '[{"tag_name":"%s","draft":false}]\n' "${tag}" >"${fixture}"
|
||||
expect_failure "${tag}" "${fixture}"
|
||||
grep -qF "Refusing to overwrite published release ${tag}" "${stderr_file}"
|
||||
|
||||
printf '[{"tag_name":"%s","draft":true},{"tag_name":"%s","draft":true}]\n' "${tag}" "${tag}" >"${fixture}"
|
||||
expect_failure "${tag}" "${fixture}"
|
||||
grep -qF "Refusing to choose among 2 releases" "${stderr_file}"
|
||||
|
||||
printf '[{"tag_name":"RELEASE.2026-08-28T00-00-00Z","draft":true}]\n' >"${fixture}"
|
||||
expect_failure "${tag}" "${fixture}"
|
||||
grep -qF "other than ${tag}" "${stderr_file}"
|
||||
|
||||
printf '{not-json}\n' >"${fixture}"
|
||||
expect_failure "${tag}" "${fixture}"
|
||||
grep -qF "Invalid release state response for ${tag}" "${stderr_file}"
|
||||
|
||||
expect_failure "not-a-release-tag" "${fixture}"
|
||||
grep -qF "Invalid release tag format" "${stderr_file}"
|
||||
|
||||
echo "release-state decision tests passed"
|
||||
@@ -7,7 +7,7 @@ _init() {
|
||||
|
||||
## Minimum required versions for build dependencies
|
||||
GIT_VERSION="1.0"
|
||||
GO_VERSION="1.16"
|
||||
GO_VERSION="1.27.1"
|
||||
OSX_VERSION="10.8"
|
||||
KNAME=$(uname -s)
|
||||
ARCH=$(uname -m)
|
||||
|
||||
@@ -22,8 +22,8 @@ function start_silo_4drive() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MC_HOST_silo="http://silo:silo123@127.0.0.1:${start_port}/"
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
export MC_HOST_silo="http://silo:silo1234@127.0.0.1:${start_port}/"
|
||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ if [ -n "${MCLI_BIN:-}" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
release=${MCLI_RELEASE:-RELEASE.2026-08-06T00-00-00Z}
|
||||
release=${MCLI_RELEASE:-RELEASE.2026-09-13T00-00-00Z}
|
||||
version_hyphen=${release#RELEASE.}
|
||||
package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')
|
||||
if [ "${package_version}" = "${version_hyphen}" ]; then
|
||||
@@ -74,7 +74,7 @@ expected=$(awk -v asset="${archive}" '
|
||||
{
|
||||
name=$2
|
||||
sub(/^\*/, "", name)
|
||||
if (name == asset && $1 ~ /^[0-9a-fA-F]{64}$/) print tolower($1)
|
||||
if (name == asset && length($1) == 64 && $1 ~ /^[0-9a-fA-F]+$/) print tolower($1)
|
||||
}
|
||||
' "${tmp_dir}/${checksums}")
|
||||
if ! printf '%s\n' "${expected}" | grep -Eq '^[0-9a-f]{64}$'; then
|
||||
|
||||
@@ -45,8 +45,8 @@ function start_silo_10drive() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MC_HOST_silo="http://silo:silo123@127.0.0.1:${start_port}/"
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
export MC_HOST_silo="http://silo:silo1234@127.0.0.1:${start_port}/"
|
||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
@@ -71,7 +71,7 @@ function start_silo_10drive() {
|
||||
"${PWD}/mc" mb --with-versioning silo/bucket
|
||||
|
||||
export AWS_ACCESS_KEY_ID=silo
|
||||
export AWS_SECRET_ACCESS_KEY=silo123
|
||||
export AWS_SECRET_ACCESS_KEY=silo1234
|
||||
aws --endpoint-url http://localhost:"$start_port" s3api create-multipart-upload --bucket bucket --key obj-1 >upload-id.json
|
||||
uploadId=$(jq -r '.UploadId' upload-id.json)
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -30,7 +30,7 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
const manifestVersion = 3
|
||||
const manifestVersion = 4
|
||||
|
||||
var (
|
||||
minioImportRE = regexp.MustCompile(`github\.com/minio/[A-Za-z0-9_./-]+`)
|
||||
@@ -44,19 +44,18 @@ var (
|
||||
)
|
||||
|
||||
type manifest struct {
|
||||
Version int `json:"version"`
|
||||
ModulePath string `json:"module_path"`
|
||||
MinioImports []string `json:"minio_imports"`
|
||||
Environment []string `json:"environment"`
|
||||
Metrics []string `json:"metrics"`
|
||||
Headers []string `json:"headers"`
|
||||
Routes []string `json:"routes"`
|
||||
RouteRoots []string `json:"route_roots"`
|
||||
GridRoutes []string `json:"grid_routes"`
|
||||
StorageMarkers []string `json:"storage_markers"`
|
||||
PolicyValues []string `json:"policy_values"`
|
||||
ExportedSymbols []string `json:"exported_symbols"`
|
||||
BrandAllowlist []string `json:"brand_allowlist"`
|
||||
Version int `json:"version"`
|
||||
ModulePath string `json:"module_path"`
|
||||
MinioImports []string `json:"minio_imports"`
|
||||
Environment []string `json:"environment"`
|
||||
Metrics []string `json:"metrics"`
|
||||
Headers []string `json:"headers"`
|
||||
Routes []string `json:"routes"`
|
||||
RouteRoots []string `json:"route_roots"`
|
||||
GridRoutes []string `json:"grid_routes"`
|
||||
StorageMarkers []string `json:"storage_markers"`
|
||||
PolicyValues []string `json:"policy_values"`
|
||||
BrandAllowlist []string `json:"brand_allowlist"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
@@ -101,23 +100,24 @@ func collect(repo string) (manifest, error) {
|
||||
}
|
||||
|
||||
sets := map[string]map[string]struct{}{
|
||||
"imports": {},
|
||||
"env": {},
|
||||
"metrics": {},
|
||||
"headers": {},
|
||||
"routes": {},
|
||||
"roots": {},
|
||||
"grid": {},
|
||||
"storage": {},
|
||||
"policy": {},
|
||||
"exported": {},
|
||||
"brand": {},
|
||||
"imports": {},
|
||||
"env": {},
|
||||
"metrics": {},
|
||||
"headers": {},
|
||||
"routes": {},
|
||||
"roots": {},
|
||||
"grid": {},
|
||||
"storage": {},
|
||||
"policy": {},
|
||||
"brand": {},
|
||||
}
|
||||
modulePath := ""
|
||||
fset := token.NewFileSet()
|
||||
|
||||
for _, rel := range files {
|
||||
// Investigation artifacts contain synthetic routes and archived configurations.
|
||||
if rel == "SILO_REBRANDING_MIGRATION.md" ||
|
||||
strings.HasPrefix(rel, "docs/investigations/") ||
|
||||
strings.HasPrefix(rel, "buildscripts/rebrand-guard/") ||
|
||||
strings.HasPrefix(rel, "buildscripts/helm-migration-guard/") {
|
||||
continue
|
||||
@@ -163,17 +163,17 @@ func collect(repo string) (manifest, error) {
|
||||
sets["imports"][value] = struct{}{}
|
||||
}
|
||||
}
|
||||
collectStringMatches(sets["routes"], routeRE, file)
|
||||
collectNamedStringValues(sets["roots"], rel, file, "minioReservedBucket")
|
||||
if rel == "internal/grid/manager.go" {
|
||||
collectStringMatches(sets["grid"], routeRE, file)
|
||||
}
|
||||
if !strings.HasSuffix(rel, "_test.go") {
|
||||
collectExported(sets["exported"], filepath.ToSlash(filepath.Dir(rel)), file)
|
||||
// Test files hold request paths for fixtures, not served routes.
|
||||
collectStringMatches(sets["routes"], routeRE, file)
|
||||
if strings.HasPrefix(rel, "cmd/") || strings.HasPrefix(rel, "internal/") {
|
||||
collectBrandStrings(sets["brand"], rel, file)
|
||||
}
|
||||
}
|
||||
collectNamedStringValues(sets["roots"], rel, file, "minioReservedBucket")
|
||||
if rel == "internal/grid/manager.go" {
|
||||
collectStringMatches(sets["grid"], routeRE, file)
|
||||
}
|
||||
}
|
||||
}
|
||||
// This was a shell-local PID variable in the generated inspect script,
|
||||
@@ -184,19 +184,18 @@ func collect(repo string) (manifest, error) {
|
||||
return manifest{}, errors.New("go.mod module path was not found")
|
||||
}
|
||||
return manifest{
|
||||
Version: manifestVersion,
|
||||
ModulePath: modulePath,
|
||||
MinioImports: sorted(sets["imports"]),
|
||||
Environment: sorted(sets["env"]),
|
||||
Metrics: sorted(sets["metrics"]),
|
||||
Headers: sorted(sets["headers"]),
|
||||
Routes: sorted(sets["routes"]),
|
||||
RouteRoots: sorted(sets["roots"]),
|
||||
GridRoutes: sorted(sets["grid"]),
|
||||
StorageMarkers: sorted(sets["storage"]),
|
||||
PolicyValues: sorted(sets["policy"]),
|
||||
ExportedSymbols: sorted(sets["exported"]),
|
||||
BrandAllowlist: sorted(sets["brand"]),
|
||||
Version: manifestVersion,
|
||||
ModulePath: modulePath,
|
||||
MinioImports: sorted(sets["imports"]),
|
||||
Environment: sorted(sets["env"]),
|
||||
Metrics: sorted(sets["metrics"]),
|
||||
Headers: sorted(sets["headers"]),
|
||||
Routes: sorted(sets["routes"]),
|
||||
RouteRoots: sorted(sets["roots"]),
|
||||
GridRoutes: sorted(sets["grid"]),
|
||||
StorageMarkers: sorted(sets["storage"]),
|
||||
PolicyValues: sorted(sets["policy"]),
|
||||
BrandAllowlist: sorted(sets["brand"]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -262,7 +261,7 @@ func collectStringMatches(dst map[string]struct{}, re *regexp.Regexp, file *ast.
|
||||
}
|
||||
|
||||
func trackedFiles(repo string) ([]string, error) {
|
||||
cmd := exec.Command("git", "-C", repo, "ls-files", "--cached", "--others", "--exclude-standard", "-z")
|
||||
cmd := exec.Command("git", "-C", repo, "ls-files", "--cached", "-z")
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("git ls-files: %w", err)
|
||||
@@ -286,78 +285,6 @@ func addMatches(dst map[string]struct{}, re *regexp.Regexp, text string, lower b
|
||||
}
|
||||
}
|
||||
|
||||
func collectExported(dst map[string]struct{}, dir string, file *ast.File) {
|
||||
prefix := dir + ":" + file.Name.Name + ":"
|
||||
for _, decl := range file.Decls {
|
||||
switch decl := decl.(type) {
|
||||
case *ast.FuncDecl:
|
||||
if !ast.IsExported(decl.Name.Name) {
|
||||
continue
|
||||
}
|
||||
if decl.Recv == nil {
|
||||
dst[prefix+"func:"+decl.Name.Name] = struct{}{}
|
||||
continue
|
||||
}
|
||||
receiver := receiverName(decl.Recv.List[0].Type)
|
||||
dst[prefix+"method:"+receiver+"."+decl.Name.Name] = struct{}{}
|
||||
case *ast.GenDecl:
|
||||
for _, spec := range decl.Specs {
|
||||
switch spec := spec.(type) {
|
||||
case *ast.TypeSpec:
|
||||
if !ast.IsExported(spec.Name.Name) {
|
||||
continue
|
||||
}
|
||||
dst[prefix+"type:"+spec.Name.Name] = struct{}{}
|
||||
collectExportedFields(dst, prefix, spec.Name.Name, spec.Type)
|
||||
case *ast.ValueSpec:
|
||||
kind := strings.ToLower(decl.Tok.String())
|
||||
for _, name := range spec.Names {
|
||||
if ast.IsExported(name.Name) {
|
||||
dst[prefix+kind+":"+name.Name] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func collectExportedFields(dst map[string]struct{}, prefix, typeName string, expr ast.Expr) {
|
||||
var fields *ast.FieldList
|
||||
switch typed := expr.(type) {
|
||||
case *ast.StructType:
|
||||
fields = typed.Fields
|
||||
case *ast.InterfaceType:
|
||||
fields = typed.Methods
|
||||
default:
|
||||
return
|
||||
}
|
||||
for _, field := range fields.List {
|
||||
for _, name := range field.Names {
|
||||
if ast.IsExported(name.Name) {
|
||||
dst[prefix+"field:"+typeName+"."+name.Name] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func receiverName(expr ast.Expr) string {
|
||||
switch expr := expr.(type) {
|
||||
case *ast.Ident:
|
||||
return expr.Name
|
||||
case *ast.StarExpr:
|
||||
return receiverName(expr.X)
|
||||
case *ast.IndexExpr:
|
||||
return receiverName(expr.X)
|
||||
case *ast.IndexListExpr:
|
||||
return receiverName(expr.X)
|
||||
case *ast.SelectorExpr:
|
||||
return receiverName(expr.X) + "." + expr.Sel.Name
|
||||
default:
|
||||
return fmt.Sprintf("%T", expr)
|
||||
}
|
||||
}
|
||||
|
||||
func sorted(set map[string]struct{}) []string {
|
||||
values := make([]string, 0, len(set))
|
||||
for value := range set {
|
||||
@@ -409,7 +336,6 @@ func compare(want, got manifest) error {
|
||||
{"grid_routes", want.GridRoutes, got.GridRoutes},
|
||||
{"storage_markers", want.StorageMarkers, got.StorageMarkers},
|
||||
{"policy_values", want.PolicyValues, got.PolicyValues},
|
||||
{"exported_symbols", want.ExportedSymbols, got.ExportedSymbols},
|
||||
{"brand_allowlist", want.BrandAllowlist, got.BrandAllowlist},
|
||||
}
|
||||
for _, check := range checks {
|
||||
@@ -454,10 +380,10 @@ func setDiff(want, got []string) (missing, added []string) {
|
||||
}
|
||||
|
||||
func printSummary(value manifest) {
|
||||
fmt.Printf("compatibility manifest: imports=%d env=%d metrics=%d headers=%d routes=%d roots=%d grid=%d storage=%d policy=%d exported=%d brand=%d sha256=%s\n",
|
||||
fmt.Printf("compatibility manifest: imports=%d env=%d metrics=%d headers=%d routes=%d roots=%d grid=%d storage=%d policy=%d brand=%d sha256=%s\n",
|
||||
len(value.MinioImports), len(value.Environment), len(value.Metrics), len(value.Headers),
|
||||
len(value.Routes), len(value.RouteRoots), len(value.GridRoutes), len(value.StorageMarkers), len(value.PolicyValues),
|
||||
len(value.ExportedSymbols), len(value.BrandAllowlist), manifestDigest(value))
|
||||
len(value.BrandAllowlist), manifestDigest(value))
|
||||
}
|
||||
|
||||
func manifestDigest(value manifest) string {
|
||||
|
||||
@@ -18,8 +18,8 @@ function start_silo_5drive() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MC_HOST_silo="http://silo:silo123@127.0.0.1:${start_port}/"
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
export MC_HOST_silo="http://silo:silo1234@127.0.0.1:${start_port}/"
|
||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
|
||||
@@ -28,8 +28,8 @@ function verify_rewrite() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ACCESS_KEY=silo
|
||||
export MINIO_SECRET_KEY=silo123
|
||||
export MC_HOST_silo="http://silo:silo123@127.0.0.1:${start_port}/"
|
||||
export MINIO_SECRET_KEY=silo1234
|
||||
export MC_HOST_silo="http://silo:silo1234@127.0.0.1:${start_port}/"
|
||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
@@ -87,7 +87,7 @@ function verify_rewrite() {
|
||||
-debug \
|
||||
-versions \
|
||||
-access-key silo \
|
||||
-secret-key silo123 \
|
||||
-secret-key silo1234 \
|
||||
-endpoint "http://127.0.0.1:${start_port}/" 2>&1 | grep INTACT; then
|
||||
echo "server1 log:"
|
||||
cat "${WORK_DIR}/server1.log"
|
||||
@@ -105,14 +105,14 @@ function verify_rewrite() {
|
||||
exit 1
|
||||
fi
|
||||
|
||||
go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "silo" "silo123"
|
||||
go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "silo" "silo1234"
|
||||
sleep 1
|
||||
|
||||
if ! ./s3-check-md5 \
|
||||
-debug \
|
||||
-versions \
|
||||
-access-key silo \
|
||||
-secret-key silo123 \
|
||||
-secret-key silo1234 \
|
||||
-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
|
||||
echo "server1 log:"
|
||||
cat "${WORK_DIR}/server1.log"
|
||||
|
||||
@@ -74,8 +74,8 @@ function test_silo_with_timeout() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MC_HOST_silo="http://silo:silo123@127.0.0.1:${start_port}/"
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
export MC_HOST_silo="http://silo:silo1234@127.0.0.1:${start_port}/"
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
mkdir ${WORK_DIR}
|
||||
|
||||
@@ -15,10 +15,10 @@ WORK_DIR="$PWD/.verify-$RANDOM"
|
||||
export MINT_MODE=core
|
||||
export MINT_DATA_DIR="$WORK_DIR/data"
|
||||
export SERVER_ENDPOINT="127.0.0.1:9000"
|
||||
export MC_HOST_verify="http://silo:silo123@${SERVER_ENDPOINT}/"
|
||||
export MC_HOST_verify_ipv6="http://silo:silo123@[::1]:9000/"
|
||||
export MC_HOST_verify="http://silo:silo1234@${SERVER_ENDPOINT}/"
|
||||
export MC_HOST_verify_ipv6="http://silo:silo1234@[::1]:9000/"
|
||||
export ACCESS_KEY="silo"
|
||||
export SECRET_KEY="silo123"
|
||||
export SECRET_KEY="silo1234"
|
||||
export ENABLE_HTTPS=0
|
||||
export GO111MODULE=on
|
||||
export GOGC=25
|
||||
@@ -225,7 +225,7 @@ function __init__() {
|
||||
shred -n 1 -s 65M - 1>"$FILE_65_MB" 2>/dev/null
|
||||
|
||||
## version is purposefully set to '3' for minio to migrate configuration file
|
||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo123"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo1234"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||
|
||||
"$(git rev-parse --show-toplevel)/buildscripts/install-verified-fixture.sh" \
|
||||
https://raw.githubusercontent.com/pgsty/mc/4c4dcc4b55baf238cd0c81030d77945b3828f157/functional-tests.sh \
|
||||
@@ -282,7 +282,6 @@ function main() {
|
||||
purge "$WORK_DIR"
|
||||
}
|
||||
|
||||
(__init__ "$@" && main "$@")
|
||||
rv=$?
|
||||
purge "$WORK_DIR"
|
||||
exit "$rv"
|
||||
trap 'purge "$WORK_DIR"' EXIT
|
||||
__init__ "$@"
|
||||
main "$@"
|
||||
|
||||
@@ -15,7 +15,7 @@ SILO=("$PWD/silo" --config-dir "$SILO_CONFIG_DIR" server)
|
||||
|
||||
function start_silo_3_node() {
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
export MINIO_ERASURE_SET_DRIVE_COUNT=6
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
@@ -37,7 +37,7 @@ function start_silo_3_node() {
|
||||
pid3=$!
|
||||
disown $pid3
|
||||
|
||||
export MC_HOST_mysilo="http://silo:silo123@127.0.0.1:$((start_port + 1))"
|
||||
export MC_HOST_mysilo="http://silo:silo1234@127.0.0.1:$((start_port + 1))"
|
||||
|
||||
timeout 15m /tmp/mc ready mysilo || fail
|
||||
|
||||
@@ -116,7 +116,7 @@ function __init__() {
|
||||
mkdir -p "$SILO_CONFIG_DIR"
|
||||
|
||||
## version is purposefully set to '3' for minio to migrate configuration file
|
||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo123"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo1234"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||
|
||||
if [ ! -f /tmp/mc ]; then
|
||||
"$(git rev-parse --show-toplevel)/buildscripts/install-mcli.sh" /tmp/mc
|
||||
|
||||
@@ -17,7 +17,7 @@ function start_silo() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||
unset MINIO_CI_CD
|
||||
unset CI
|
||||
|
||||
@@ -20,7 +20,7 @@ function start_silo_3_node() {
|
||||
done
|
||||
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
export MINIO_ERASURE_SET_DRIVE_COUNT=6
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
@@ -46,7 +46,7 @@ function start_silo_3_node() {
|
||||
pid3=$!
|
||||
disown $pid3
|
||||
|
||||
export MC_HOST_mysilo="http://silo:silo123@127.0.0.1:$((start_port + 1))"
|
||||
export MC_HOST_mysilo="http://silo:silo1234@127.0.0.1:$((start_port + 1))"
|
||||
timeout 15m /tmp/mc ready mysilo || fail
|
||||
|
||||
[ ${first_time} -eq 0 ] && upload_objects
|
||||
@@ -117,7 +117,7 @@ function __init__() {
|
||||
mkdir -p "$SILO_CONFIG_DIR"
|
||||
|
||||
## version is purposefully set to '3' for minio to migrate configuration file
|
||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo123"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo1234"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||
|
||||
if [ ! -f /tmp/mc ]; then
|
||||
"$(git rev-parse --show-toplevel)/buildscripts/install-mcli.sh" /tmp/mc
|
||||
|
||||
@@ -113,7 +113,7 @@ helm_run template my-release "${new_chart}" \
|
||||
go run ./buildscripts/helm-migration-guard "${old_render}" "${new_render}"
|
||||
|
||||
helm_run package "${new_chart}" --destination "${output_dir}" >/dev/null
|
||||
test -s "${work_dir}/silo-7.0.1.tgz"
|
||||
test -s "${work_dir}/silo-7.0.2.tgz"
|
||||
if find "${work_dir}" -maxdepth 1 -type f -name 'minio-*.tgz' | grep -q .; then
|
||||
echo "Helm packaging emitted a legacy MinIO chart name" >&2
|
||||
exit 1
|
||||
|
||||
@@ -36,7 +36,7 @@ for file in \
|
||||
buildscripts/verify-helm-migration.sh \
|
||||
Dockerfile.goreleaser \
|
||||
Dockerfile.distroless \
|
||||
dockerscripts/download-static-curl.sh \
|
||||
dockerscripts/build-static-curl.sh \
|
||||
dockerscripts/docker-entrypoint.sh \
|
||||
helm/silo/Chart.yaml \
|
||||
helm/silo/values.yaml \
|
||||
@@ -101,7 +101,7 @@ require_text Dockerfile.goreleaser "Published checksum drift"
|
||||
require_text Dockerfile.distroless 'COPY --chmod=0755 silo /usr/bin/silo'
|
||||
require_text Dockerfile.distroless 'ENTRYPOINT ["/usr/bin/silo"]'
|
||||
require_text Dockerfile.distroless '"/usr/bin/silo", "healthcheck", "ready"'
|
||||
require_text dockerscripts/download-static-curl.sh "sha256sum -c"
|
||||
require_text dockerscripts/build-static-curl.sh "sha256sum -c"
|
||||
require_text helm/silo/Chart.yaml "name: silo"
|
||||
require_text helm/silo/values.yaml "repository: pgsty/silo"
|
||||
require_text helm/silo/templates/deployment.yaml "/usr/bin/docker-entrypoint.sh silo server"
|
||||
@@ -159,11 +159,13 @@ fi
|
||||
# The repository and its default branch are pgsty/silo and main. The invariant
|
||||
# is that the old name is never a live target, not that it is never spoken: the
|
||||
# READMEs have to name it to explain the rename and to point at the archived
|
||||
# artifacts, which is the opposite of stranding a reader on it.
|
||||
# artifacts, which is the opposite of stranding a reader on it. CONTRIBUTORS.md
|
||||
# also quotes historical issue titles.
|
||||
#
|
||||
# So two rules. First, no live URL may resolve to the old repository anywhere,
|
||||
# READMEs included.
|
||||
stale_repo_url="$(rg -n -e 'github\.com/pgsty/minio' -e 'hub\.docker\.com/r/pgsty/minio' \
|
||||
# READMEs and CONTRIBUTORS.md included.
|
||||
old_repo_pattern='pgsty/minio(\.git)?([^[:alnum:]_.-]|$)'
|
||||
stale_repo_url="$(rg -n -e "github\.com/${old_repo_pattern}" -e "hub\.docker\.com/r/${old_repo_pattern}" \
|
||||
--glob '!.git/**' --glob '!dist/**' \
|
||||
--glob '!SILO_REBRANDING_MIGRATION.md' \
|
||||
--glob '!buildscripts/rebrand-guard/compat-baseline.json' . |
|
||||
@@ -175,10 +177,10 @@ fi
|
||||
|
||||
# Second, the bare name may only appear where it is deliberate: the pinned
|
||||
# pre-rebrand image digest in the upgrade test, the two guards that refuse a
|
||||
# legacy image, and the two READMEs that document the rename and the archived
|
||||
# minio branch.
|
||||
repo_guard_allowlist='^(buildscripts/minio-upgrade\.sh|buildscripts/verify-rebrand\.sh|buildscripts/helm-migration-guard/main\.go|README\.md|README_ZH\.md):'
|
||||
stale_repo="$(rg -n 'pgsty/minio' --glob '!.git/**' --glob '!dist/**' \
|
||||
# legacy image, the two READMEs that document the rename and the archived
|
||||
# minio branch, and historical issue titles in CONTRIBUTORS.md.
|
||||
repo_guard_allowlist='^(buildscripts/minio-upgrade\.sh|buildscripts/verify-rebrand\.sh|buildscripts/helm-migration-guard/main\.go|README\.md|README_ZH\.md|CONTRIBUTORS\.md):'
|
||||
stale_repo="$(rg -n "${old_repo_pattern}" --glob '!.git/**' --glob '!dist/**' \
|
||||
--glob '!SILO_REBRANDING_MIGRATION.md' \
|
||||
--glob '!buildscripts/rebrand-guard/compat-baseline.json' . |
|
||||
sed 's#^\./##' | grep -Ev "${repo_guard_allowlist}" || true)"
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// Data types used for returning dummy access control
|
||||
|
||||
@@ -0,0 +1,348 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/mux"
|
||||
)
|
||||
|
||||
func corsAdminRequest(t *testing.T, cred auth.Credentials, method, path string, body []byte) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
router := mux.NewRouter()
|
||||
registerAdminRouter(router, true)
|
||||
req, err := newTestSignedRequestV4(method, adminPathPrefix+adminAPIVersionPrefix+path,
|
||||
int64(len(body)), bytes.NewReader(body), cred.AccessKey, cred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("admin %s: %d: %s", path, rec.Code, rec.Body.String())
|
||||
}
|
||||
return rec
|
||||
}
|
||||
|
||||
func corsImportReport(t *testing.T, rec *httptest.ResponseRecorder) madmin.BucketMetaImportErrs {
|
||||
t.Helper()
|
||||
var rpt madmin.BucketMetaImportErrs
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &rpt); err != nil {
|
||||
t.Fatalf("import report %q: %v", rec.Body.String(), err)
|
||||
}
|
||||
return rpt
|
||||
}
|
||||
|
||||
func corsZip(t *testing.T, entries map[string][]byte) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for name, data := range entries {
|
||||
w, err := zw.Create(name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = w.Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// corsCorruptedZip builds an archive holding a stored (uncompressed) cors.xml
|
||||
// whose payload is altered after the checksum is computed, plus the given
|
||||
// companion entries. The altered document stays well formed, so only the zip
|
||||
// checksum tells the two apart.
|
||||
func corsCorruptedZip(t *testing.T, name string, doc []byte, others map[string][]byte) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
w, err := zw.CreateHeader(&zip.FileHeader{Name: name, Method: zip.Store})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = w.Write(doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for other, data := range others {
|
||||
ow, err := zw.Create(other)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = ow.Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err = zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw := buf.Bytes()
|
||||
at := bytes.Index(raw, []byte("app.example.com"))
|
||||
if at < 0 {
|
||||
t.Fatalf("stored CORS payload not found in archive")
|
||||
}
|
||||
raw[at] = 'A'
|
||||
return raw
|
||||
}
|
||||
|
||||
// TestAdminBucketMetadataCORSRoundTrip covers the export/import round trip for
|
||||
// per-bucket CORS, per-file error reporting for an invalid document, and that
|
||||
// an archive without cors.xml leaves an existing configuration alone.
|
||||
func TestAdminBucketMetadataCORSRoundTrip(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instanceType, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
corsXML := []byte(testSiteReplicationCORSDoc)
|
||||
if _, err := updateLocalBucketCORSMetadata(t.Context(), obj, bucket, corsXML); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Export must carry the stored document verbatim.
|
||||
rec := corsAdminRequest(t, cred, http.MethodGet, "/export-bucket-metadata?bucket="+bucket, nil)
|
||||
archive := rec.Body.Bytes()
|
||||
zr, err := zip.NewReader(bytes.NewReader(archive), int64(len(archive)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var exported []byte
|
||||
for _, f := range zr.File {
|
||||
if f.Name != bucket+"/"+bucketCorsConfig {
|
||||
continue
|
||||
}
|
||||
r, err := f.Open()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exported, err = io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if !bytes.Equal(exported, corsXML) {
|
||||
t.Fatalf("%s: exported CORS = %q, want %q", instanceType, exported, corsXML)
|
||||
}
|
||||
|
||||
// Drop the configuration: the archive must then omit the entry.
|
||||
if _, err = updateLocalBucketCORSMetadata(t.Context(), obj, bucket, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err = globalBucketMetadataSys.GetCorsConfigXML(bucket); err == nil {
|
||||
t.Fatalf("%s: CORS still present before restore", instanceType)
|
||||
}
|
||||
rec = corsAdminRequest(t, cred, http.MethodGet, "/export-bucket-metadata?bucket="+bucket, nil)
|
||||
empty := rec.Body.Bytes()
|
||||
zr, err = zip.NewReader(bytes.NewReader(empty), int64(len(empty)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range zr.File {
|
||||
if f.Name == bucket+"/"+bucketCorsConfig {
|
||||
t.Fatalf("%s: export emitted %s for a bucket without CORS", instanceType, f.Name)
|
||||
}
|
||||
}
|
||||
|
||||
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata", archive)
|
||||
if st := corsImportReport(t, rec).Buckets[bucket]; !st.Cors.IsSet || st.Cors.Err != "" {
|
||||
t.Fatalf("%s: import report cors = %+v", instanceType, st.Cors)
|
||||
}
|
||||
stored, storedAt, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil || !bytes.Equal(stored, corsXML) {
|
||||
t.Fatalf("%s: restored CORS = %q, err = %v", instanceType, stored, err)
|
||||
}
|
||||
created, err := globalBucketMetadataSys.CreatedAt(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !storedAt.After(created) {
|
||||
t.Fatalf("%s: restored CORS timestamp %v is not after bucket creation %v", instanceType, storedAt, created)
|
||||
}
|
||||
|
||||
// An archive without cors.xml must not remove the configuration.
|
||||
corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||
corsZip(t, map[string][]byte{bucket + "/quota.json": []byte(`{"quota":0}`)}))
|
||||
if stored, _, err = globalBucketMetadataSys.GetCorsConfigXML(bucket); err != nil || !bytes.Equal(stored, corsXML) {
|
||||
t.Fatalf("%s: import without cors.xml changed CORS: %q, err = %v", instanceType, stored, err)
|
||||
}
|
||||
|
||||
// A bucket the import itself creates must still land above its own
|
||||
// creation time, otherwise CORS replication would drop the restore.
|
||||
fresh := "cors-import-created-bucket"
|
||||
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||
corsZip(t, map[string][]byte{fresh + "/" + bucketCorsConfig: corsXML}))
|
||||
if st := corsImportReport(t, rec).Buckets[fresh]; !st.Cors.IsSet || st.Cors.Err != "" {
|
||||
t.Fatalf("%s: fresh bucket import report cors = %+v", instanceType, st.Cors)
|
||||
}
|
||||
freshStored, freshAt, err := globalBucketMetadataSys.GetCorsConfigXML(fresh)
|
||||
if err != nil || !bytes.Equal(freshStored, corsXML) {
|
||||
t.Fatalf("%s: fresh bucket CORS = %q, err = %v", instanceType, freshStored, err)
|
||||
}
|
||||
freshCreated, err := globalBucketMetadataSys.CreatedAt(fresh)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !freshAt.After(freshCreated) {
|
||||
t.Fatalf("%s: fresh bucket CORS timestamp %v is not after creation %v", instanceType, freshAt, freshCreated)
|
||||
}
|
||||
|
||||
// An invalid document must fail loudly for that bucket and change nothing.
|
||||
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||
corsZip(t, map[string][]byte{bucket + "/" + bucketCorsConfig: []byte("<CORSConfiguration><CORSRule>")}))
|
||||
if st := corsImportReport(t, rec).Buckets[bucket]; st.Cors.Err == "" {
|
||||
t.Fatalf("%s: invalid CORS import reported no error: %+v", instanceType, st)
|
||||
}
|
||||
if stored, _, err = globalBucketMetadataSys.GetCorsConfigXML(bucket); err != nil || !bytes.Equal(stored, corsXML) {
|
||||
t.Fatalf("%s: invalid CORS import changed stored config: %q, err = %v", instanceType, stored, err)
|
||||
}
|
||||
|
||||
// A well formed document carried by a corrupt zip entry must be
|
||||
// rejected too, leaving the stored document and its timestamp alone
|
||||
// while the other configs in the same archive still apply.
|
||||
_, corsAt, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||
corsCorruptedZip(t, bucket+"/"+bucketCorsConfig, corsXML,
|
||||
map[string][]byte{bucket + "/quota.json": []byte(`{"quota":4096,"quotatype":"hard"}`)}))
|
||||
st := corsImportReport(t, rec).Buckets[bucket]
|
||||
if st.Cors.Err == "" {
|
||||
t.Fatalf("%s: corrupt CORS entry reported no error: %+v", instanceType, st)
|
||||
}
|
||||
if !st.Quota.IsSet || st.Quota.Err != "" {
|
||||
t.Fatalf("%s: corrupt CORS entry blocked the neighboring quota: %+v", instanceType, st.Quota)
|
||||
}
|
||||
stored, storedAt, err = globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil || !bytes.Equal(stored, corsXML) || !storedAt.Equal(corsAt) {
|
||||
t.Fatalf("%s: corrupt CORS entry changed stored config: %q at %v (was %v), err = %v", instanceType, stored, storedAt, corsAt, err)
|
||||
}
|
||||
quota, _, err := globalBucketMetadataSys.GetQuotaConfig(t.Context(), bucket)
|
||||
if err != nil || quota == nil || quota.Quota != 4096 {
|
||||
t.Fatalf("%s: neighboring quota not applied: %+v, err = %v", instanceType, quota, err)
|
||||
}
|
||||
}})
|
||||
}
|
||||
|
||||
// corsPeerStub is a stand-in site-replication peer. It records every
|
||||
// SRBucketMeta it is asked to apply and answers with status.
|
||||
func corsPeerStub(t *testing.T, applied chan<- madmin.SRBucketMeta, status int) *httptest.Server {
|
||||
t.Helper()
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodPut && applied != nil {
|
||||
var item madmin.SRBucketMeta
|
||||
if err := json.NewDecoder(r.Body).Decode(&item); err != nil {
|
||||
t.Errorf("decode peer apply: %v", err)
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
applied <- item
|
||||
}
|
||||
w.WriteHeader(status)
|
||||
}))
|
||||
}
|
||||
|
||||
// TestAdminBucketMetadataCORSImportReplicatesPastPeerFailure pins that an
|
||||
// imported CORS document reaches the reachable peers even when the shared
|
||||
// bucket metadata hook failed against an unreachable one, and that both
|
||||
// failures are still reported for the bucket.
|
||||
func TestAdminBucketMetadataCORSImportReplicatesPastPeerFailure(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instanceType, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
ctx := t.Context()
|
||||
corsXML := []byte(testSiteReplicationCORSDoc)
|
||||
|
||||
healthyApplies := make(chan madmin.SRBucketMeta, 4)
|
||||
healthy := corsPeerStub(t, healthyApplies, http.StatusOK)
|
||||
defer healthy.Close()
|
||||
broken := corsPeerStub(t, nil, http.StatusBadRequest)
|
||||
defer broken.Close()
|
||||
|
||||
// With site replication on, admin requests resolve their token signing
|
||||
// key through the site replicator account, so it has to exist.
|
||||
serviceCred, err := auth.CreateCredentials(siteReplicatorSvcAcc, "cors-import-service-secret")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serviceCred.ParentUser = cred.AccessKey
|
||||
if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
|
||||
globalSiteReplicatorCred.Set(serviceCred.SecretKey)
|
||||
defer globalSiteReplicatorCred.Set("")
|
||||
|
||||
globalSiteReplicationSys.Lock()
|
||||
oldEnabled, oldState := globalSiteReplicationSys.enabled, globalSiteReplicationSys.state
|
||||
globalSiteReplicationSys.enabled = true
|
||||
globalSiteReplicationSys.state = srState{
|
||||
Name: "cors-import-test",
|
||||
ServiceAccountAccessKey: serviceCred.AccessKey,
|
||||
Peers: map[string]madmin.PeerInfo{
|
||||
globalDeploymentID(): {Name: "local", DeploymentID: globalDeploymentID()},
|
||||
"peer-healthy": {Name: "healthy", DeploymentID: "peer-healthy", Endpoint: healthy.URL},
|
||||
"peer-broken": {Name: "broken", DeploymentID: "peer-broken", Endpoint: broken.URL},
|
||||
},
|
||||
}
|
||||
globalSiteReplicationSys.Unlock()
|
||||
defer func() {
|
||||
globalSiteReplicationSys.Lock()
|
||||
globalSiteReplicationSys.enabled, globalSiteReplicationSys.state = oldEnabled, oldState
|
||||
globalSiteReplicationSys.Unlock()
|
||||
}()
|
||||
|
||||
rec := corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||
corsZip(t, map[string][]byte{
|
||||
bucket + "/" + bucketCorsConfig: corsXML,
|
||||
bucket + "/quota.json": []byte(`{"quota":8192,"quotatype":"hard"}`),
|
||||
}))
|
||||
st := corsImportReport(t, rec).Buckets[bucket]
|
||||
if !st.Cors.IsSet || st.Cors.Err != "" {
|
||||
t.Fatalf("%s: import report cors = %+v", instanceType, st.Cors)
|
||||
}
|
||||
stored, storedAt, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil || !bytes.Equal(stored, corsXML) {
|
||||
t.Fatalf("%s: stored CORS = %q, err = %v", instanceType, stored, err)
|
||||
}
|
||||
|
||||
// The reachable peer must have been told about the CORS document,
|
||||
// carrying exactly the timestamp that was saved locally.
|
||||
var corsSeen, sharedSeen bool
|
||||
for range 2 {
|
||||
select {
|
||||
case item := <-healthyApplies:
|
||||
if item.Type != madmin.SRBucketMetaTypeCorsConfig {
|
||||
sharedSeen = item.Bucket == bucket && item.Quota != nil
|
||||
continue
|
||||
}
|
||||
if item.Bucket != bucket || item.Cors == nil || !item.UpdatedAt.Equal(storedAt) {
|
||||
t.Fatalf("%s: peer CORS event = %#v, want %s at %v", instanceType, item, bucket, storedAt)
|
||||
}
|
||||
payload, decErr := base64.StdEncoding.Strict().DecodeString(*item.Cors)
|
||||
if decErr != nil || !bytes.Equal(payload, corsXML) {
|
||||
t.Fatalf("%s: peer CORS payload = %q, err = %v", instanceType, payload, decErr)
|
||||
}
|
||||
corsSeen = true
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("%s: healthy peer received no further events (shared=%v cors=%v)", instanceType, sharedSeen, corsSeen)
|
||||
}
|
||||
}
|
||||
if !sharedSeen || !corsSeen {
|
||||
t.Fatalf("%s: healthy peer events shared=%v cors=%v, want both", instanceType, sharedSeen, corsSeen)
|
||||
}
|
||||
|
||||
// Both hook failures against the unreachable peer stay reported.
|
||||
if got := strings.Count(st.Err, "->broken:"); got != 2 {
|
||||
t.Fatalf("%s: bucket error mentions the broken peer %d times, want 2: %q", instanceType, got, st.Err)
|
||||
}
|
||||
}})
|
||||
}
|
||||
+205
-19
@@ -41,7 +41,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -76,7 +76,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
|
||||
return
|
||||
}
|
||||
|
||||
quotaConfig, err := parseBucketQuota(bucket, data)
|
||||
_, err = parseBucketQuota(bucket, data)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -94,9 +94,6 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
|
||||
Quota: data,
|
||||
UpdatedAt: updatedAt,
|
||||
}
|
||||
if quotaConfig.Size == 0 && quotaConfig.Quota == 0 {
|
||||
bucketMeta.Quota = nil
|
||||
}
|
||||
|
||||
// Call site replication hook.
|
||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta))
|
||||
@@ -417,6 +414,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
bucketLifecycleConfig,
|
||||
bucketSSEConfig,
|
||||
bucketTaggingConfig,
|
||||
bucketCorsConfig,
|
||||
bucketQuotaConfigFile,
|
||||
objectLockConfig,
|
||||
bucketVersioningConfig,
|
||||
@@ -437,7 +435,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
configData, err := json.Marshal(config)
|
||||
configData, err := canonicalBucketPolicy(config)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
@@ -517,6 +515,19 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case bucketCorsConfig:
|
||||
// Export the stored document verbatim: GetBucketCors returns
|
||||
// the bytes exactly as they were PUT, so the archive must
|
||||
// round-trip them unchanged.
|
||||
configData, _, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, errConfigNotFound) {
|
||||
continue
|
||||
}
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case objectLockConfig:
|
||||
config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
|
||||
if err != nil {
|
||||
@@ -589,6 +600,50 @@ type importMetaReport struct {
|
||||
madmin.BucketMetaImportErrs
|
||||
}
|
||||
|
||||
type importMetadataFields map[string]struct{}
|
||||
|
||||
func (f importMetadataFields) add(configFile string) {
|
||||
f[configFile] = struct{}{}
|
||||
}
|
||||
|
||||
func applyImportedBucketMetadata(dst *BucketMetadata, src BucketMetadata, fields importMetadataFields) {
|
||||
for configFile := range fields {
|
||||
switch configFile {
|
||||
case bucketPolicyConfig:
|
||||
dst.PolicyConfigJSON = bytes.Clone(src.PolicyConfigJSON)
|
||||
dst.PolicyConfigUpdatedAt = src.PolicyConfigUpdatedAt
|
||||
case bucketNotificationConfig:
|
||||
dst.NotificationConfigXML = bytes.Clone(src.NotificationConfigXML)
|
||||
dst.NotificationConfigUpdatedAt = src.NotificationConfigUpdatedAt
|
||||
case bucketLifecycleConfig:
|
||||
dst.LifecycleConfigXML = bytes.Clone(src.LifecycleConfigXML)
|
||||
dst.LifecycleConfigUpdatedAt = src.LifecycleConfigUpdatedAt
|
||||
case bucketSSEConfig:
|
||||
dst.EncryptionConfigXML = bytes.Clone(src.EncryptionConfigXML)
|
||||
dst.EncryptionConfigUpdatedAt = src.EncryptionConfigUpdatedAt
|
||||
case bucketTaggingConfig:
|
||||
dst.TaggingConfigXML = bytes.Clone(src.TaggingConfigXML)
|
||||
dst.TaggingConfigUpdatedAt = src.TaggingConfigUpdatedAt
|
||||
case bucketQuotaConfigFile:
|
||||
dst.QuotaConfigJSON = bytes.Clone(src.QuotaConfigJSON)
|
||||
dst.QuotaConfigUpdatedAt = src.QuotaConfigUpdatedAt
|
||||
case bucketCorsConfig:
|
||||
// The import stamps its fields before creating any missing bucket,
|
||||
// and a CORS event stamped before bucket creation is discarded as
|
||||
// belonging to an older incarnation, so the imported document takes
|
||||
// the same monotonic timestamp a local PutBucketCors would assign.
|
||||
dst.CorsConfigUpdatedAt = localCORSUpdatedAt(*dst, src.CorsConfigUpdatedAt)
|
||||
dst.CorsConfigXML = bytes.Clone(src.CorsConfigXML)
|
||||
case objectLockConfig:
|
||||
dst.ObjectLockConfigXML = bytes.Clone(src.ObjectLockConfigXML)
|
||||
dst.ObjectLockConfigUpdatedAt = src.ObjectLockConfigUpdatedAt
|
||||
case bucketVersioningConfig:
|
||||
dst.VersioningConfigXML = bytes.Clone(src.VersioningConfigXML)
|
||||
dst.VersioningConfigUpdatedAt = src.VersioningConfigUpdatedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (i *importMetaReport) SetStatus(bucket, fname string, err error) {
|
||||
st := i.Buckets[bucket]
|
||||
var errMsg string
|
||||
@@ -608,6 +663,8 @@ func (i *importMetaReport) SetStatus(bucket, fname string, err error) {
|
||||
st.Tagging = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case bucketQuotaConfigFile:
|
||||
st.Quota = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case bucketCorsConfig:
|
||||
st.Cors = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case objectLockConfig:
|
||||
st.ObjectLock = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case bucketVersioningConfig:
|
||||
@@ -649,6 +706,16 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
}
|
||||
|
||||
bucketMap := make(map[string]*BucketMetadata, len(zr.File))
|
||||
importedFields := make(map[string]importMetadataFields, len(zr.File))
|
||||
blockedBuckets := make(map[string]struct{})
|
||||
markImported := func(bucket, configFile string) {
|
||||
fields := importedFields[bucket]
|
||||
if fields == nil {
|
||||
fields = make(importMetadataFields)
|
||||
importedFields[bucket] = fields
|
||||
}
|
||||
fields.add(configFile)
|
||||
}
|
||||
|
||||
updatedAt := UTCNow()
|
||||
|
||||
@@ -664,6 +731,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
bucketMap[bucket] = &meta
|
||||
} else if err != errConfigNotFound {
|
||||
rpt.SetStatus(bucket, "", err)
|
||||
blockedBuckets[bucket] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -675,6 +743,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
bucket, fileName := slc[0], slc[1]
|
||||
if _, blocked := blockedBuckets[bucket]; blocked {
|
||||
continue
|
||||
}
|
||||
if fileName == objectLockConfig {
|
||||
reader, err := file.Open()
|
||||
if err != nil {
|
||||
@@ -708,6 +779,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].ObjectLockConfigXML = configData
|
||||
bucketMap[bucket].ObjectLockConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
}
|
||||
}
|
||||
@@ -720,6 +792,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
bucket, fileName := slc[0], slc[1]
|
||||
if _, blocked := blockedBuckets[bucket]; blocked {
|
||||
continue
|
||||
}
|
||||
if fileName == bucketVersioningConfig {
|
||||
reader, err := file.Open()
|
||||
if err != nil {
|
||||
@@ -764,6 +839,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].VersioningConfigXML = configData
|
||||
bucketMap[bucket].VersioningConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
}
|
||||
}
|
||||
@@ -781,6 +857,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
bucket, fileName := slc[0], slc[1]
|
||||
if _, blocked := blockedBuckets[bucket]; blocked {
|
||||
continue
|
||||
}
|
||||
|
||||
// create bucket if it does not exist yet.
|
||||
if _, ok := bucketMap[bucket]; !ok {
|
||||
@@ -813,6 +892,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].NotificationConfigXML = configData
|
||||
bucketMap[bucket].NotificationConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketPolicyConfig:
|
||||
// Error out if Content-Length is beyond allowed size.
|
||||
@@ -839,7 +919,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
|
||||
configData, err := json.Marshal(bucketPolicy)
|
||||
configData, err := canonicalBucketPolicy(bucketPolicy)
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
@@ -847,6 +927,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].PolicyConfigJSON = configData
|
||||
bucketMap[bucket].PolicyConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketLifecycleConfig:
|
||||
bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(reader, sz))
|
||||
@@ -879,6 +960,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].LifecycleConfigXML = configData
|
||||
bucketMap[bucket].LifecycleConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketSSEConfig:
|
||||
// Parse bucket encryption xml
|
||||
@@ -917,6 +999,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].EncryptionConfigXML = configData
|
||||
bucketMap[bucket].EncryptionConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketTaggingConfig:
|
||||
tags, err := tags.ParseBucketXML(io.LimitReader(reader, sz))
|
||||
@@ -933,6 +1016,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].TaggingConfigXML = configData
|
||||
bucketMap[bucket].TaggingConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketQuotaConfigFile:
|
||||
data, err := io.ReadAll(reader)
|
||||
@@ -949,6 +1033,33 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].QuotaConfigJSON = data
|
||||
bucketMap[bucket].QuotaConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketCorsConfig:
|
||||
if sz > maxBucketCorsSize {
|
||||
rpt.SetStatus(bucket, fileName, errors.New(ErrEntityTooLarge.String()))
|
||||
continue
|
||||
}
|
||||
|
||||
// Read one byte past the declared size: stopping exactly at sz
|
||||
// leaves archive/zip short of EOF, so it never verifies the entry
|
||||
// checksum and a corrupt entry carrying well formed XML would be
|
||||
// stored as a valid document. The extra byte also lets the reader
|
||||
// reject an entry longer than it declares.
|
||||
corsData, err := io.ReadAll(io.LimitReader(reader, sz+1))
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if err = validateCORSReplicationPayload(corsData); err != nil {
|
||||
rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
|
||||
continue
|
||||
}
|
||||
|
||||
bucketMap[bucket].CorsConfigXML = corsData
|
||||
bucketMap[bucket].CorsConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
}
|
||||
}
|
||||
@@ -962,22 +1073,97 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
}
|
||||
|
||||
for bucket, meta := range bucketMap {
|
||||
err := globalBucketMetadataSys.save(ctx, *meta)
|
||||
fields := importedFields[bucket]
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
var merged BucketMetadata
|
||||
var commitAt time.Time
|
||||
err := func() error {
|
||||
lockCtx, unlock, err := lockBucketMetadata(ctx, objectAPI, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer unlock()
|
||||
merged, err = loadBucketMetadataParse(lockCtx, objectAPI, bucket, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureBucketMetadataCreated(lockCtx, objectAPI, &merged); err != nil {
|
||||
return err
|
||||
}
|
||||
commitAt = UTCNow()
|
||||
for _, file := range replicatedBucketConfigs {
|
||||
if _, ok := fields[file]; ok {
|
||||
commitAt = localBucketConfigUpdatedAt(merged, file, commitAt)
|
||||
}
|
||||
}
|
||||
applyImportedBucketMetadata(&merged, *meta, fields)
|
||||
for _, file := range replicatedBucketConfigs {
|
||||
if _, ok := fields[file]; !ok {
|
||||
continue
|
||||
}
|
||||
data, at := replicatedBucketConfig(&merged, file)
|
||||
payload, _, err := bucketConfigPayload(bucket, file, *data, len(merged.ObjectLockConfigXML) != 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*data, *at = payload, commitAt
|
||||
}
|
||||
return globalBucketMetadataSys.saveMetadata(lockCtx, objectAPI, &merged)
|
||||
}()
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, "", err)
|
||||
continue
|
||||
}
|
||||
// Call site replication hook.
|
||||
if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Bucket: bucket,
|
||||
Quota: meta.QuotaConfigJSON,
|
||||
Policy: meta.PolicyConfigJSON,
|
||||
Versioning: enc(meta.VersioningConfigXML),
|
||||
Tags: enc(meta.TaggingConfigXML),
|
||||
ObjectLockConfig: enc(meta.ObjectLockConfigXML),
|
||||
SSEConfig: enc(meta.EncryptionConfigXML),
|
||||
UpdatedAt: updatedAt,
|
||||
}); err != nil {
|
||||
*meta = merged
|
||||
globalNotificationSys.LoadBucketMetadata(bgContext(ctx), bucket)
|
||||
hook := madmin.SRBucketMeta{Bucket: bucket, UpdatedAt: commitAt}
|
||||
var hookNeeded bool
|
||||
if _, ok := fields[bucketQuotaConfigFile]; ok {
|
||||
hook.Quota = meta.QuotaConfigJSON
|
||||
hookNeeded = true
|
||||
}
|
||||
if _, ok := fields[bucketPolicyConfig]; ok {
|
||||
hook.Policy = meta.PolicyConfigJSON
|
||||
hookNeeded = hookNeeded || len(hook.Policy) != 0
|
||||
}
|
||||
if _, ok := fields[bucketVersioningConfig]; ok {
|
||||
hook.Versioning = enc(meta.VersioningConfigXML)
|
||||
hookNeeded = true
|
||||
}
|
||||
if _, ok := fields[bucketTaggingConfig]; ok {
|
||||
hook.Tags = enc(meta.TaggingConfigXML)
|
||||
hookNeeded = true
|
||||
}
|
||||
if _, ok := fields[objectLockConfig]; ok {
|
||||
hook.ObjectLockConfig = enc(meta.ObjectLockConfigXML)
|
||||
hookNeeded = true
|
||||
}
|
||||
if _, ok := fields[bucketSSEConfig]; ok {
|
||||
hook.SSEConfig = enc(meta.EncryptionConfigXML)
|
||||
hookNeeded = true
|
||||
}
|
||||
if hookNeeded {
|
||||
err = globalSiteReplicationSys.BucketMetaHook(ctx, hook)
|
||||
}
|
||||
if _, ok := fields[bucketPolicyConfig]; ok && len(meta.PolicyConfigJSON) == 0 {
|
||||
// An omitted bulk Policy cannot express deletion.
|
||||
err = errors.Join(err, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypePolicy, Bucket: bucket, UpdatedAt: commitAt,
|
||||
}))
|
||||
}
|
||||
if _, ok := fields[bucketCorsConfig]; ok {
|
||||
// CORS carries its own timestamp, so it replicates through the
|
||||
// dedicated event rather than the shared bucket metadata hook. It
|
||||
// is announced even when the shared hook failed: the document is
|
||||
// already committed locally, and a peer that is unreachable for
|
||||
// one config must not withhold CORS from the reachable ones.
|
||||
if corsEvent, live := newBucketCORSReplicationEvent(bucket, *meta); live {
|
||||
err = errors.Join(err, globalSiteReplicationSys.BucketMetaHook(ctx, corsEvent))
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, "", err)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// validateAdminReq will validate request against and return whether it is allowed.
|
||||
|
||||
@@ -37,7 +37,7 @@ import (
|
||||
"github.com/minio/minio/internal/config/subnet"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// DelConfigKVHandler - DELETE /minio/admin/v3/del-config-kv
|
||||
|
||||
@@ -32,8 +32,8 @@ import (
|
||||
cfgldap "github.com/minio/minio/internal/config/identity/ldap"
|
||||
"github.com/minio/minio/internal/config/identity/openid"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/ldap"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/ldap"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
|
||||
|
||||
@@ -28,8 +28,8 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/mux"
|
||||
xldap "github.com/minio/pkg/v3/ldap"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
xldap "github.com/pgsty/silo-pkg/v3/ldap"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// ListLDAPPolicyMappingEntities lists users/groups mapped to given/all policies.
|
||||
|
||||
@@ -25,7 +25,7 @@ import (
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const dummyRoleARN = "dummy-internal"
|
||||
|
||||
@@ -27,8 +27,8 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
@@ -33,7 +33,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// SiteReplicationAdd - PUT /minio/admin/v3/site-replication/add
|
||||
@@ -255,9 +255,11 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
|
||||
case madmin.SRBucketMetaTypeTags:
|
||||
err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags, item.UpdatedAt)
|
||||
case madmin.SRBucketMetaTypeObjectLockConfig:
|
||||
err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig, item.UpdatedAt)
|
||||
err = globalSiteReplicationSys.peerBucketObjectLockConfigItem(ctx, item)
|
||||
case madmin.SRBucketMetaTypeSSEConfig:
|
||||
err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig, item.UpdatedAt)
|
||||
case madmin.SRBucketMetaTypeCorsConfig:
|
||||
err = globalSiteReplicationSys.PeerBucketCorsConfigHandler(ctx, item.Bucket, item.Cors, item.UpdatedAt)
|
||||
case madmin.SRBucketMetaLCConfig:
|
||||
err = globalSiteReplicationSys.PeerBucketLCConfigHandler(ctx, item.Bucket, item.ExpiryLCConfig, item.UpdatedAt)
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ import (
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
minio "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
)
|
||||
|
||||
func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {
|
||||
|
||||
+34
-16
@@ -40,8 +40,8 @@ import (
|
||||
"github.com/minio/minio/internal/config/dns"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
xldap "github.com/minio/pkg/v3/ldap"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
xldap "github.com/pgsty/silo-pkg/v3/ldap"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/puzpuzpuz/xsync/v3"
|
||||
)
|
||||
|
||||
@@ -355,18 +355,25 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// SetGroupStatus - PUT /minio/admin/v3/set-group-status?group=mygroup1&status=enabled
|
||||
func setGroupStatusAdminAction(status string) policy.AdminAction {
|
||||
if madmin.GroupStatus(status) == madmin.GroupDisabled {
|
||||
return policy.DisableGroupAdminAction
|
||||
}
|
||||
return policy.EnableGroupAdminAction
|
||||
}
|
||||
|
||||
func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.EnableGroupAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
group := vars["group"]
|
||||
status := vars["status"]
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, setGroupStatusAdminAction(status))
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
var (
|
||||
err error
|
||||
updatedAt time.Time
|
||||
@@ -398,18 +405,25 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
|
||||
// SetUserStatus - PUT /minio/admin/v3/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
|
||||
func setUserStatusAdminAction(status string) policy.AdminAction {
|
||||
if madmin.AccountStatus(status) == madmin.AccountDisabled {
|
||||
return policy.DisableUserAdminAction
|
||||
}
|
||||
return policy.EnableUserAdminAction
|
||||
}
|
||||
|
||||
func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, creds := validateAdminReq(ctx, w, r, policy.EnableUserAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
accessKey := vars["accessKey"]
|
||||
status := vars["status"]
|
||||
|
||||
objectAPI, creds := validateAdminReq(ctx, w, r, setUserStatusAdminAction(status))
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// you cannot enable or disable yourself.
|
||||
if accessKey == creds.AccessKey {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
|
||||
@@ -488,11 +502,15 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
checkDenyOnly := accessKey == cred.AccessKey
|
||||
action := policy.Action(policy.CreateUserAdminAction)
|
||||
if checkDenyOnly {
|
||||
action = policy.ChangeMyPasswordAdminAction
|
||||
}
|
||||
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.CreateUserAdminAction,
|
||||
Action: action,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
@@ -859,7 +877,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
|
||||
var sp *policy.Policy
|
||||
if len(updateReq.NewPolicy) > 0 {
|
||||
sp, err = policy.ParseConfig(bytes.NewReader(updateReq.NewPolicy))
|
||||
sp, err = policy.ParseConfigStrict(bytes.NewReader(updateReq.NewPolicy))
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -1729,7 +1747,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
|
||||
return
|
||||
}
|
||||
|
||||
iamPolicy, err := policy.ParseConfig(bytes.NewReader(iamPolicyBytes))
|
||||
iamPolicy, err := policy.ParseConfigStrict(bytes.NewReader(iamPolicyBytes))
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -2981,7 +2999,7 @@ func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.
|
||||
|
||||
var sp *policy.Policy
|
||||
if len(createReq.Policy) > 0 {
|
||||
sp, err = policy.ParseConfig(bytes.NewReader(createReq.Policy))
|
||||
sp, err = policy.ParseConfigStrict(bytes.NewReader(createReq.Policy))
|
||||
if err != nil {
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err)
|
||||
}
|
||||
|
||||
@@ -40,13 +40,54 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio-go/v7/pkg/signer"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
testDefaultTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
func TestSetUserStatusAdminAction(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
status string
|
||||
want policy.AdminAction
|
||||
}{
|
||||
{name: "enable", status: string(madmin.AccountEnabled), want: policy.EnableUserAdminAction},
|
||||
{name: "disable", status: string(madmin.AccountDisabled), want: policy.DisableUserAdminAction},
|
||||
{name: "invalid preserves authenticated default", status: "invalid", want: policy.EnableUserAdminAction},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := setUserStatusAdminAction(tt.status); got != tt.want {
|
||||
t.Fatalf("setUserStatusAdminAction(%q) = %q, want %q", tt.status, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetGroupStatusAdminAction(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
status string
|
||||
want policy.AdminAction
|
||||
}{
|
||||
{name: "enable", status: string(madmin.GroupEnabled), want: policy.EnableGroupAdminAction},
|
||||
{name: "disable", status: string(madmin.GroupDisabled), want: policy.DisableGroupAdminAction},
|
||||
{name: "invalid preserves authenticated default", status: "invalid", want: policy.EnableGroupAdminAction},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := setGroupStatusAdminAction(tt.status); got != tt.want {
|
||||
t.Fatalf("setGroupStatusAdminAction(%q) = %q, want %q", tt.status, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// API suite container for IAM
|
||||
type TestSuiteIAM struct {
|
||||
TestSuiteCommon
|
||||
@@ -202,8 +243,12 @@ func TestIAMInternalIDPServerSuite(t *testing.T) {
|
||||
|
||||
suite.SetUpSuite(c)
|
||||
suite.TestUserCreate(c)
|
||||
suite.TestUserPasswordActionAuthorization(c)
|
||||
suite.TestUserStatusActionAuthorization(c)
|
||||
suite.TestGroupStatusActionAuthorization(c)
|
||||
suite.TestUserPolicyEscalationBug(c)
|
||||
suite.TestPolicyCreate(c)
|
||||
suite.TestServiceAccountBareARNPolicyRejected(c)
|
||||
suite.TestCannedPolicies(c)
|
||||
suite.TestGroupAddRemove(c)
|
||||
suite.TestServiceAccountOpsByAdmin(c)
|
||||
@@ -312,6 +357,284 @@ func (s *TestSuiteIAM) TestUserCreate(c *check) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestUserPasswordActionAuthorization(c *check) {
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
statements string
|
||||
self bool
|
||||
other bool
|
||||
}{
|
||||
{"readonly", "", true, false},
|
||||
{"consolereadonly", "", true, false},
|
||||
{"password grant", `{"Effect":"Allow","Action":"admin:ChangeMyPassword"}`, true, false},
|
||||
{"legacy CreateUser deny", `{"Effect":"Deny","Action":"admin:CreateUser","Resource":"arn:aws:s3:::*"}`, true, false},
|
||||
{"password deny", `{"Effect":"Deny","Action":"admin:ChangeMyPassword"}`, false, false},
|
||||
{"user admin", `{"Effect":"Allow","Action":"admin:CreateUser"}`, true, true},
|
||||
{"user admin with password deny", `{"Effect":"Allow","Action":"admin:CreateUser"},{"Effect":"Deny","Action":"admin:ChangeMyPassword"}`, false, true},
|
||||
{"password deny overrides grant", `{"Effect":"Allow","Action":"admin:ChangeMyPassword"},{"Effect":"Deny","Action":"admin:ChangeMyPassword"}`, false, false},
|
||||
{"wildcard deny", `{"Effect":"Deny","Action":"admin:*"}`, false, false},
|
||||
} {
|
||||
c.Run(tt.name, func(t *testing.T) {
|
||||
c := &check{t, s.serverType}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
var users []string
|
||||
policyName := tt.name
|
||||
defer func() {
|
||||
for _, user := range users {
|
||||
if err := s.adm.RemoveUser(ctx, user); err != nil {
|
||||
c.Errorf("remove test user: %v", err)
|
||||
}
|
||||
}
|
||||
if tt.statements != "" {
|
||||
if err := s.adm.RemoveCannedPolicy(ctx, policyName); err != nil {
|
||||
c.Errorf("remove test policy: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
createUser := func() (string, string) {
|
||||
accessKey, secretKey := mustGenerateCredentials(c)
|
||||
if err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled); err != nil {
|
||||
c.Fatalf("create test user: %v", err)
|
||||
}
|
||||
users = append(users, accessKey)
|
||||
return accessKey, secretKey
|
||||
}
|
||||
client := func(accessKey, secretKey string) *madmin.AdminClient {
|
||||
adm, err := madmin.New(s.endpoint, accessKey, secretKey, s.secure)
|
||||
if err != nil {
|
||||
c.Fatal(err)
|
||||
}
|
||||
adm.SetCustomTransport(s.TestSuiteCommon.client.Transport)
|
||||
return adm
|
||||
}
|
||||
if tt.statements != "" {
|
||||
policyName = getRandomBucketName()
|
||||
doc := []byte(`{"Version":"2012-10-17","Statement":[` + tt.statements + `]}`)
|
||||
if err := s.adm.AddCannedPolicy(ctx, policyName, doc); err != nil {
|
||||
c.Fatalf("save test policy: %v", err)
|
||||
}
|
||||
}
|
||||
accessKey, secretKey := createUser()
|
||||
if _, err := s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
|
||||
User: accessKey, Policies: []string{policyName},
|
||||
}); err != nil {
|
||||
c.Fatalf("attach test policy: %v", err)
|
||||
}
|
||||
adm := client(accessKey, secretKey)
|
||||
_, newSecretKey := mustGenerateCredentials(c)
|
||||
err := adm.SetUser(ctx, accessKey, newSecretKey, madmin.AccountEnabled)
|
||||
if tt.self {
|
||||
if err != nil {
|
||||
c.Fatalf("change own password: %v", err)
|
||||
}
|
||||
if _, err = adm.AccountInfo(ctx, madmin.AccountOpts{}); err == nil {
|
||||
c.Fatal("old password still authenticates")
|
||||
}
|
||||
adm = client(accessKey, newSecretKey)
|
||||
} else if err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("self password change: expected AccessDenied, got %v", err)
|
||||
}
|
||||
if _, err := adm.AccountInfo(ctx, madmin.AccountOpts{}); err != nil {
|
||||
c.Fatalf("current password no longer authenticates: %v", err)
|
||||
}
|
||||
target, _ := createUser()
|
||||
newUser, newUserSecret := mustGenerateCredentials(c)
|
||||
for _, key := range []string{target, newUser} {
|
||||
err := adm.SetUser(ctx, key, newUserSecret, madmin.AccountEnabled)
|
||||
if tt.other {
|
||||
if err != nil {
|
||||
c.Fatalf("create or update another user: %v", err)
|
||||
}
|
||||
if key == newUser {
|
||||
users = append(users, newUser)
|
||||
}
|
||||
} else if err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("create or update another user: expected AccessDenied, got %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestUserStatusActionAuthorization(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
|
||||
var createdUsers []string
|
||||
var createdPolicies []string
|
||||
defer func() {
|
||||
for _, user := range createdUsers {
|
||||
if err := s.adm.RemoveUser(ctx, user); err != nil {
|
||||
c.Errorf("unable to remove test user %s: %v", user, err)
|
||||
}
|
||||
}
|
||||
for _, policyName := range createdPolicies {
|
||||
if err := s.adm.RemoveCannedPolicy(ctx, policyName); err != nil {
|
||||
c.Errorf("unable to remove test policy %s: %v", policyName, err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
createUser := func() (string, string) {
|
||||
accessKey, secretKey := mustGenerateCredentials(c)
|
||||
if err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled); err != nil {
|
||||
c.Fatalf("unable to create test user: %v", err)
|
||||
}
|
||||
createdUsers = append(createdUsers, accessKey)
|
||||
return accessKey, secretKey
|
||||
}
|
||||
|
||||
createStatusClient := func(action policy.AdminAction) *madmin.AdminClient {
|
||||
accessKey, secretKey := createUser()
|
||||
policyName := getRandomBucketName()
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["%s"]
|
||||
}]
|
||||
}`, action)
|
||||
if err := s.adm.AddCannedPolicy(ctx, policyName, policyBytes); err != nil {
|
||||
c.Fatalf("unable to add status policy: %v", err)
|
||||
}
|
||||
createdPolicies = append(createdPolicies, policyName)
|
||||
if _, err := s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
|
||||
Policies: []string{policyName},
|
||||
User: accessKey,
|
||||
}); err != nil {
|
||||
c.Fatalf("unable to attach status policy: %v", err)
|
||||
}
|
||||
|
||||
client, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Secure: s.secure,
|
||||
})
|
||||
if err != nil {
|
||||
c.Fatalf("unable to create status admin client: %v", err)
|
||||
}
|
||||
client.SetCustomTransport(s.TestSuiteCommon.client.Transport)
|
||||
return client
|
||||
}
|
||||
|
||||
targetAccessKey, _ := createUser()
|
||||
disableClient := createStatusClient(policy.DisableUserAdminAction)
|
||||
if err := disableClient.SetUserStatus(ctx, targetAccessKey, madmin.AccountDisabled); err != nil {
|
||||
c.Fatalf("DisableUser-only client could not disable a user: %v", err)
|
||||
}
|
||||
if err := disableClient.SetUserStatus(ctx, targetAccessKey, madmin.AccountEnabled); err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("DisableUser-only client unexpectedly enabled a user: %v", err)
|
||||
}
|
||||
|
||||
enableClient := createStatusClient(policy.EnableUserAdminAction)
|
||||
if err := enableClient.SetUserStatus(ctx, targetAccessKey, madmin.AccountEnabled); err != nil {
|
||||
c.Fatalf("EnableUser-only client could not enable a user: %v", err)
|
||||
}
|
||||
if err := enableClient.SetUserStatus(ctx, targetAccessKey, madmin.AccountDisabled); err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("EnableUser-only client unexpectedly disabled a user: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestGroupStatusActionAuthorization(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
|
||||
var createdUsers []string
|
||||
var createdPolicies []string
|
||||
group := getRandomBucketName()
|
||||
var groupCreated bool
|
||||
defer func() {
|
||||
if groupCreated {
|
||||
if err := s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{
|
||||
Group: group,
|
||||
Members: createdUsers[:1],
|
||||
IsRemove: true,
|
||||
}); err != nil {
|
||||
c.Errorf("unable to remove group member: %v", err)
|
||||
}
|
||||
if err := s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{Group: group, IsRemove: true}); err != nil {
|
||||
c.Errorf("unable to remove test group: %v", err)
|
||||
}
|
||||
}
|
||||
for _, user := range createdUsers {
|
||||
if err := s.adm.RemoveUser(ctx, user); err != nil {
|
||||
c.Errorf("unable to remove test user %s: %v", user, err)
|
||||
}
|
||||
}
|
||||
for _, policyName := range createdPolicies {
|
||||
if err := s.adm.RemoveCannedPolicy(ctx, policyName); err != nil {
|
||||
c.Errorf("unable to remove test policy %s: %v", policyName, err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
createUser := func() (string, string) {
|
||||
accessKey, secretKey := mustGenerateCredentials(c)
|
||||
if err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled); err != nil {
|
||||
c.Fatalf("unable to create test user: %v", err)
|
||||
}
|
||||
createdUsers = append(createdUsers, accessKey)
|
||||
return accessKey, secretKey
|
||||
}
|
||||
|
||||
targetAccessKey, _ := createUser()
|
||||
if err := s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{
|
||||
Group: group,
|
||||
Members: []string{targetAccessKey},
|
||||
}); err != nil {
|
||||
c.Fatalf("unable to create test group: %v", err)
|
||||
}
|
||||
groupCreated = true
|
||||
|
||||
createStatusClient := func(action policy.AdminAction) *madmin.AdminClient {
|
||||
accessKey, secretKey := createUser()
|
||||
policyName := getRandomBucketName()
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["%s"]
|
||||
}]
|
||||
}`, action)
|
||||
if err := s.adm.AddCannedPolicy(ctx, policyName, policyBytes); err != nil {
|
||||
c.Fatalf("unable to add group status policy: %v", err)
|
||||
}
|
||||
createdPolicies = append(createdPolicies, policyName)
|
||||
if _, err := s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
|
||||
Policies: []string{policyName},
|
||||
User: accessKey,
|
||||
}); err != nil {
|
||||
c.Fatalf("unable to attach group status policy: %v", err)
|
||||
}
|
||||
|
||||
client, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Secure: s.secure,
|
||||
})
|
||||
if err != nil {
|
||||
c.Fatalf("unable to create group status admin client: %v", err)
|
||||
}
|
||||
client.SetCustomTransport(s.TestSuiteCommon.client.Transport)
|
||||
return client
|
||||
}
|
||||
|
||||
disableClient := createStatusClient(policy.DisableGroupAdminAction)
|
||||
if err := disableClient.SetGroupStatus(ctx, group, madmin.GroupDisabled); err != nil {
|
||||
c.Fatalf("DisableGroup-only client could not disable a group: %v", err)
|
||||
}
|
||||
if err := disableClient.SetGroupStatus(ctx, group, madmin.GroupEnabled); err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("DisableGroup-only client unexpectedly enabled a group: %v", err)
|
||||
}
|
||||
|
||||
enableClient := createStatusClient(policy.EnableGroupAdminAction)
|
||||
if err := enableClient.SetGroupStatus(ctx, group, madmin.GroupEnabled); err != nil {
|
||||
c.Fatalf("EnableGroup-only client could not enable a group: %v", err)
|
||||
}
|
||||
if err := enableClient.SetGroupStatus(ctx, group, madmin.GroupDisabled); err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("EnableGroup-only client unexpectedly disabled a group: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestUserPolicyEscalationBug(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
@@ -600,6 +923,20 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) {
|
||||
c.Fatalf("invalid policy creation success")
|
||||
}
|
||||
|
||||
for i, resource := range []string{"arn:aws:s3:::", "*arn:aws:s3:::"} {
|
||||
barePolicyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Deny",
|
||||
"Action": ["s3:GetObject"],
|
||||
"Resource": ["%s"]
|
||||
}]
|
||||
}`, resource)
|
||||
if err = s.adm.AddCannedPolicy(ctx, fmt.Sprintf("%s-bare-%d", policy, i), barePolicyBytes); err == nil {
|
||||
c.Fatalf("bare ARN policy creation succeeded for %q", resource)
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Create a user, associate policy and verify access
|
||||
accessKey, secretKey := mustGenerateCredentials(c)
|
||||
err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
|
||||
@@ -653,6 +990,51 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestServiceAccountBareARNPolicyRejected(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
|
||||
barePolicy := []byte(`{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject"],
|
||||
"NotResource": ["arn:aws:s3:::"]
|
||||
}]
|
||||
}`)
|
||||
if _, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
|
||||
TargetUser: globalActiveCred.AccessKey,
|
||||
Policy: barePolicy,
|
||||
}); err == nil {
|
||||
c.Fatal("service account creation accepted a bare ARN policy")
|
||||
}
|
||||
|
||||
validPolicy := []byte(`{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject"],
|
||||
"Resource": ["arn:aws:s3:::*"]
|
||||
}]
|
||||
}`)
|
||||
credentials, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
|
||||
TargetUser: globalActiveCred.AccessKey,
|
||||
Policy: validPolicy,
|
||||
})
|
||||
if err != nil {
|
||||
c.Fatalf("service account creation rejected an explicit resource: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = s.adm.DeleteServiceAccount(ctx, credentials.AccessKey)
|
||||
}()
|
||||
|
||||
if err = s.adm.UpdateServiceAccount(ctx, credentials.AccessKey, madmin.UpdateServiceAccountReq{
|
||||
NewPolicy: barePolicy,
|
||||
}); err == nil {
|
||||
c.Fatal("service account update accepted a bare ARN policy")
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestCannedPolicies(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
@@ -665,6 +1047,7 @@ func (s *TestSuiteIAM) TestCannedPolicies(c *check) {
|
||||
defaultPolicies := []string{
|
||||
"readwrite",
|
||||
"readonly",
|
||||
"consolereadonly",
|
||||
"writeonly",
|
||||
"diagnostics",
|
||||
"consoleAdmin",
|
||||
|
||||
@@ -60,8 +60,8 @@ import (
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/secure-io/sio-go"
|
||||
"github.com/zeebo/xxh3"
|
||||
)
|
||||
|
||||
@@ -388,6 +388,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(adminMiddleware(adminAPI.SRPeerJoin))
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(adminMiddleware(adminAPI.SRPeerBucketOps)).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateIAMItem))
|
||||
adminRouter.Methods(http.MethodGet, http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-revisions").HandlerFunc(adminMiddleware(adminAPI.SRPeerIAMRevisions))
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateBucketItem))
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(adminMiddleware(adminAPI.SRPeerGetIDPSettings))
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationEdit))
|
||||
|
||||
@@ -30,7 +30,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
)
|
||||
|
||||
// getLocalServerProperty - returns madmin.ServerProperties for only the
|
||||
|
||||
+18
-2
@@ -48,7 +48,7 @@ import (
|
||||
levent "github.com/minio/minio/internal/config/lambda/event"
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// APIError structure
|
||||
@@ -1523,10 +1523,14 @@ var errorCodes = errorCodeMap{
|
||||
Description: "Your Host header is malformed.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
// The stored object cannot be served: a server-side data condition, not a
|
||||
// successful partial read. Upstream maps it to http.StatusPartialContent
|
||||
// (since ca6b4773e, 2017), which lets SDKs accept the XML error document
|
||||
// as object content; SILO deliberately diverges and returns 500.
|
||||
ErrObjectTampered: {
|
||||
Code: "XMinioObjectTampered",
|
||||
Description: errObjectTampered.Error(),
|
||||
HTTPStatusCode: http.StatusPartialContent,
|
||||
HTTPStatusCode: http.StatusInternalServerError,
|
||||
},
|
||||
|
||||
ErrSiteReplicationInvalidRequest: {
|
||||
@@ -2169,6 +2173,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
err = unwrapAll(err)
|
||||
|
||||
switch err {
|
||||
case errCompleteMultipartChecksumMismatch, errCompleteMultipartChecksumTypeMismatch:
|
||||
apiErr = ErrBadDigest
|
||||
case errMissingPartChecksum:
|
||||
apiErr = ErrInvalidRequest
|
||||
case errInvalidArgument:
|
||||
apiErr = ErrAdminInvalidArgument
|
||||
case errNoSuchPolicy:
|
||||
@@ -2465,6 +2473,14 @@ func toAPIError(ctx context.Context, err error) APIError {
|
||||
}
|
||||
|
||||
apiErr := errorCodes.ToAPIErr(toAPIErrorCode(ctx, err))
|
||||
switch {
|
||||
case errors.Is(err, errCompleteMultipartChecksumMismatch):
|
||||
apiErr.Description = strings.TrimPrefix(err.Error(), errCompleteMultipartChecksumMismatch.Error()+": ")
|
||||
case errors.Is(err, errCompleteMultipartChecksumTypeMismatch):
|
||||
apiErr.Description = strings.TrimPrefix(err.Error(), errCompleteMultipartChecksumTypeMismatch.Error()+": ")
|
||||
case errors.Is(err, errMissingPartChecksum):
|
||||
apiErr.Description = strings.TrimPrefix(err.Error(), errMissingPartChecksum.Error()+": ")
|
||||
}
|
||||
switch apiErr.Code {
|
||||
case "NotImplemented":
|
||||
apiErr = APIError{
|
||||
|
||||
@@ -39,6 +39,10 @@ var toAPIErrorTests = []struct {
|
||||
{err: ObjectNameInvalid{}, errCode: ErrInvalidObjectName},
|
||||
{err: InvalidUploadID{}, errCode: ErrNoSuchUpload},
|
||||
{err: InvalidPart{}, errCode: ErrInvalidPart},
|
||||
{err: errCompleteMultipartChecksumMismatch, errCode: ErrBadDigest},
|
||||
{err: errCompleteMultipartChecksumTypeMismatch, errCode: ErrBadDigest},
|
||||
{err: errMissingPartChecksum, errCode: ErrInvalidRequest},
|
||||
{err: hash.ChecksumMismatch{}, errCode: ErrContentChecksumMismatch},
|
||||
{err: InsufficientReadQuorum{}, errCode: ErrSlowDownRead},
|
||||
{err: InsufficientWriteQuorum{}, errCode: ErrSlowDownWrite},
|
||||
{err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented},
|
||||
|
||||
+4
-1
@@ -212,7 +212,10 @@ func setObjectHeaders(ctx context.Context, w http.ResponseWriter, objInfo Object
|
||||
}
|
||||
|
||||
if rs == nil && opts.PartNumber > 0 {
|
||||
rs = partNumberToRangeSpec(objInfo, opts.PartNumber)
|
||||
rs, err = partNumberToRangeSpec(objInfo, opts.PartNumber)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// For providing ranged content
|
||||
|
||||
+35
-10
@@ -27,7 +27,6 @@ import (
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/amztime"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
@@ -35,8 +34,8 @@ import (
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
xxml "github.com/minio/xxml"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -380,6 +379,13 @@ type CopyObjectResponse struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
|
||||
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
||||
ETag string // md5sum of the copied object.
|
||||
|
||||
ChecksumCRC32 string `xml:",omitempty"`
|
||||
ChecksumCRC32C string `xml:",omitempty"`
|
||||
ChecksumSHA1 string `xml:",omitempty"`
|
||||
ChecksumSHA256 string `xml:",omitempty"`
|
||||
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||
ChecksumType string `xml:",omitempty"`
|
||||
}
|
||||
|
||||
// CopyObjectPartResponse container returns ETag and LastModified of the successfully copied object
|
||||
@@ -387,6 +393,12 @@ type CopyObjectPartResponse struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"`
|
||||
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
||||
ETag string // md5sum of the copied object part.
|
||||
|
||||
ChecksumCRC32 string `xml:",omitempty"`
|
||||
ChecksumCRC32C string `xml:",omitempty"`
|
||||
ChecksumSHA1 string `xml:",omitempty"`
|
||||
ChecksumSHA256 string `xml:",omitempty"`
|
||||
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||
}
|
||||
|
||||
// Initiator inherit from Owner struct, fields are same
|
||||
@@ -416,6 +428,7 @@ type CompleteMultipartUploadResponse struct {
|
||||
Key string
|
||||
ETag string
|
||||
|
||||
ChecksumType string `xml:"ChecksumType,omitempty"`
|
||||
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
|
||||
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
|
||||
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
|
||||
@@ -763,19 +776,30 @@ func generateListObjectsV2Response(ctx context.Context, bucket, prefix, token, n
|
||||
|
||||
type metaCheckFn = func(name string, action policy.Action) (s3Err APIErrorCode)
|
||||
|
||||
// generates CopyObjectResponse from etag and lastModified time.
|
||||
func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
|
||||
// generates CopyObjectResponse from the committed object information.
|
||||
func generateCopyObjectResponse(oi ObjectInfo, cs map[string]string) CopyObjectResponse {
|
||||
return CopyObjectResponse{
|
||||
ETag: "\"" + etag + "\"",
|
||||
LastModified: amztime.ISO8601Format(lastModified.UTC()),
|
||||
ETag: "\"" + oi.ETag + "\"",
|
||||
LastModified: amztime.ISO8601Format(oi.ModTime.UTC()),
|
||||
ChecksumCRC32: cs[hash.ChecksumCRC32.String()],
|
||||
ChecksumCRC32C: cs[hash.ChecksumCRC32C.String()],
|
||||
ChecksumSHA1: cs[hash.ChecksumSHA1.String()],
|
||||
ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
|
||||
ChecksumCRC64NVME: cs[hash.ChecksumCRC64NVME.String()],
|
||||
ChecksumType: cs[xhttp.AmzChecksumType],
|
||||
}
|
||||
}
|
||||
|
||||
// generates CopyObjectPartResponse from etag and lastModified time.
|
||||
func generateCopyObjectPartResponse(etag string, lastModified time.Time) CopyObjectPartResponse {
|
||||
// generates CopyObjectPartResponse from the uploaded part information.
|
||||
func generateCopyObjectPartResponse(partInfo PartInfo) CopyObjectPartResponse {
|
||||
return CopyObjectPartResponse{
|
||||
ETag: "\"" + etag + "\"",
|
||||
LastModified: amztime.ISO8601Format(lastModified.UTC()),
|
||||
ETag: "\"" + partInfo.ETag + "\"",
|
||||
LastModified: amztime.ISO8601Format(partInfo.LastModified.UTC()),
|
||||
ChecksumCRC32: partInfo.ChecksumCRC32,
|
||||
ChecksumCRC32C: partInfo.ChecksumCRC32C,
|
||||
ChecksumSHA1: partInfo.ChecksumSHA1,
|
||||
ChecksumSHA256: partInfo.ChecksumSHA256,
|
||||
ChecksumCRC64NVME: partInfo.ChecksumCRC64NVME,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -797,6 +821,7 @@ func generateCompleteMultipartUploadResponse(bucket, key, location string, oi Ob
|
||||
Key: key,
|
||||
// AWS S3 quotes the ETag in XML, make sure we are compatible here.
|
||||
ETag: "\"" + oi.ETag + "\"",
|
||||
ChecksumType: cs[xhttp.AmzChecksumType],
|
||||
ChecksumSHA1: cs[hash.ChecksumSHA1.String()],
|
||||
ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
|
||||
ChecksumCRC32: cs[hash.ChecksumCRC32.String()],
|
||||
|
||||
+127
-10
@@ -18,16 +18,23 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
consoleapi "github.com/minio/console/api"
|
||||
bktcors "github.com/minio/minio/internal/bucket/cors"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/wildcard"
|
||||
"github.com/pgsty/silo-pkg/v3/wildcard"
|
||||
"github.com/rs/cors"
|
||||
)
|
||||
|
||||
type bucketCorsAppliedKey struct{}
|
||||
|
||||
func newHTTPServerFn() *xhttp.Server {
|
||||
globalObjLayerMutex.RLock()
|
||||
defer globalObjLayerMutex.RUnlock()
|
||||
@@ -111,11 +118,6 @@ var rejectedBucketAPIs = []rejectedAPI{
|
||||
methods: []string{http.MethodGet, http.MethodPut, http.MethodDelete},
|
||||
queries: []string{"inventory", ""},
|
||||
},
|
||||
{
|
||||
api: "cors",
|
||||
methods: []string{http.MethodPut, http.MethodDelete},
|
||||
queries: []string{"cors", ""},
|
||||
},
|
||||
{
|
||||
api: "metrics",
|
||||
methods: []string{http.MethodGet, http.MethodPut, http.MethodDelete},
|
||||
@@ -459,15 +461,15 @@ func registerAPIRouter(router *mux.Router) {
|
||||
router.Methods(http.MethodPut).
|
||||
HandlerFunc(s3APIMiddleware(api.PutBucketACLHandler)).
|
||||
Queries("acl", "")
|
||||
// GetBucketCors - this is a dummy call.
|
||||
// GetBucketCors
|
||||
router.Methods(http.MethodGet).
|
||||
HandlerFunc(s3APIMiddleware(api.GetBucketCorsHandler)).
|
||||
Queries("cors", "")
|
||||
// PutBucketCors - this is a dummy call.
|
||||
// PutBucketCors
|
||||
router.Methods(http.MethodPut).
|
||||
HandlerFunc(s3APIMiddleware(api.PutBucketCorsHandler)).
|
||||
Queries("cors", "")
|
||||
// DeleteBucketCors - this is a dummy call.
|
||||
// DeleteBucketCors
|
||||
router.Methods(http.MethodDelete).
|
||||
HandlerFunc(s3APIMiddleware(api.DeleteBucketCorsHandler)).
|
||||
Queries("cors", "")
|
||||
@@ -648,6 +650,94 @@ func registerAPIRouter(router *mux.Router) {
|
||||
apiRouter.MethodNotAllowedHandler = collectAPIStats("methodnotallowed", httpTraceAll(methodNotAllowedHandler("S3")))
|
||||
}
|
||||
|
||||
// applyBucketCors applies a bucket's CORS configuration to the request.
|
||||
// For an OPTIONS preflight it writes the full CORS response and returns true
|
||||
// (request is complete). For an actual request it adds the applicable
|
||||
// Access-Control-* response headers and returns false so the request
|
||||
// continues down the handler chain. If no rule matches a preflight it writes
|
||||
// 403 and returns true. A matched actual request is marked in its context so
|
||||
// inner legacy middleware does not rewrite an explicitly allowed null origin.
|
||||
func applyBucketCors(w http.ResponseWriter, r *http.Request, cfg *bktcors.Config) (handled bool) {
|
||||
origin := r.Header.Get("Origin")
|
||||
if origin == "" {
|
||||
return false // not a CORS request
|
||||
}
|
||||
h := w.Header()
|
||||
h.Add("Vary", "Origin")
|
||||
|
||||
isPreflight := r.Method == http.MethodOptions &&
|
||||
r.Header.Get("Access-Control-Request-Method") != ""
|
||||
|
||||
if isPreflight {
|
||||
method := r.Header.Get("Access-Control-Request-Method")
|
||||
reqHeaders := splitAndTrim(r.Header.Get("Access-Control-Request-Headers"))
|
||||
// A preflight response depends on all three request headers that
|
||||
// determine the outcome, including when the request is rejected.
|
||||
h.Add("Vary", "Access-Control-Request-Method")
|
||||
h.Add("Vary", "Access-Control-Request-Headers")
|
||||
rule, allowedOrigin, allowedHeaders, maxAgeSeconds, ok := cfg.MatchPreflight(origin, method, reqHeaders)
|
||||
if !ok {
|
||||
writeResponse(w, http.StatusForbidden, nil, mimeNone)
|
||||
return true
|
||||
}
|
||||
setBucketCorsOriginHeaders(h, allowedOrigin, origin)
|
||||
h.Set("Access-Control-Allow-Methods", strings.Join(rule.AllowedMethods, ", "))
|
||||
if len(allowedHeaders) > 0 {
|
||||
h.Set("Access-Control-Allow-Headers", strings.Join(allowedHeaders, ", "))
|
||||
}
|
||||
if len(rule.ExposeHeaders) > 0 {
|
||||
h.Set("Access-Control-Expose-Headers", strings.Join(rule.ExposeHeaders, ", "))
|
||||
}
|
||||
if maxAgeSeconds != nil {
|
||||
h.Set("Access-Control-Max-Age", strconv.Itoa(*maxAgeSeconds))
|
||||
}
|
||||
writeResponse(w, http.StatusOK, nil, mimeNone)
|
||||
return true
|
||||
}
|
||||
|
||||
// Actual request: attach headers if the origin+method match.
|
||||
rule, allowedOrigin, ok := cfg.MatchRule(origin, r.Method)
|
||||
if !ok {
|
||||
return false // no matching rule → no CORS headers, continue normally
|
||||
}
|
||||
*r = *r.WithContext(context.WithValue(r.Context(), bucketCorsAppliedKey{}, struct{}{}))
|
||||
setBucketCorsOriginHeaders(h, allowedOrigin, origin)
|
||||
if len(rule.ExposeHeaders) > 0 {
|
||||
h.Set("Access-Control-Expose-Headers", strings.Join(rule.ExposeHeaders, ", "))
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func bucketCorsWasApplied(r *http.Request) bool {
|
||||
_, ok := r.Context().Value(bucketCorsAppliedKey{}).(struct{})
|
||||
return ok
|
||||
}
|
||||
|
||||
func setBucketCorsOriginHeaders(h http.Header, allowedOrigin, requestOrigin string) {
|
||||
if allowedOrigin == "*" {
|
||||
h.Set("Access-Control-Allow-Origin", "*")
|
||||
h.Del("Access-Control-Allow-Credentials")
|
||||
return
|
||||
}
|
||||
h.Set("Access-Control-Allow-Origin", requestOrigin)
|
||||
h.Set("Access-Control-Allow-Credentials", "true")
|
||||
}
|
||||
|
||||
// splitAndTrim splits a comma-separated header list into trimmed, non-empty values.
|
||||
func splitAndTrim(s string) []string {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
parts := strings.Split(s, ",")
|
||||
out := parts[:0]
|
||||
for _, p := range parts {
|
||||
if p = strings.TrimSpace(p); p != "" {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// corsHandler handler for CORS (Cross Origin Resource Sharing)
|
||||
func corsHandler(handler http.Handler) http.Handler {
|
||||
commonS3Headers := []string{
|
||||
@@ -693,5 +783,32 @@ func corsHandler(handler http.Handler) http.Handler {
|
||||
ExposedHeaders: commonS3Headers,
|
||||
AllowCredentials: true,
|
||||
}
|
||||
return cors.New(opts).Handler(handler)
|
||||
globalCors := cors.New(opts).Handler(handler)
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Origin") != "" {
|
||||
if bucket, _ := request2BucketObjectName(r); bucket != "" && globalBucketMetadataSys != nil {
|
||||
// Resident-only lookup: this runs before authentication with a
|
||||
// client-supplied path segment as the bucket name, so it must
|
||||
// never load or cache metadata. While startup loading is still
|
||||
// running, for a real bucket whose metadata failed to load, and
|
||||
// for a bucket whose stored CORS document failed to parse, the
|
||||
// request gets no CORS headers; any other non-resident name falls
|
||||
// back to the global policy below.
|
||||
cfg, _, err := globalBucketMetadataSys.GetResidentCorsConfig(bucket)
|
||||
if err == nil && cfg != nil {
|
||||
if applyBucketCors(w, r, cfg) {
|
||||
return
|
||||
}
|
||||
handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
if err != nil && !errors.Is(err, errConfigNotFound) {
|
||||
internalLogOnceIf(r.Context(), err, "bucket-cors-metadata")
|
||||
handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
globalCors.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
+70
-42
@@ -41,7 +41,7 @@ import (
|
||||
xjwt "github.com/minio/minio/internal/jwt"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/mcontext"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// Verify if request has JWT.
|
||||
@@ -363,17 +363,6 @@ func checkRequestAuthTypeWithRequestTags(ctx context.Context, r *http.Request, a
|
||||
return authorizeRequestWithTags(ctx, r, action, "", requestTags)
|
||||
}
|
||||
|
||||
// checkRequestAuthTypeWithVID is similar to checkRequestAuthType
|
||||
// passes versionID additionally.
|
||||
func checkRequestAuthTypeWithVID(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName, versionID string) (s3Err APIErrorCode) {
|
||||
logger.GetReqInfo(ctx).BucketName = bucketName
|
||||
logger.GetReqInfo(ctx).ObjectName = objectName
|
||||
logger.GetReqInfo(ctx).VersionID = versionID
|
||||
|
||||
_, _, s3Err = checkRequestAuthTypeCredential(ctx, r, action)
|
||||
return s3Err
|
||||
}
|
||||
|
||||
func authenticateRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
|
||||
if logger.GetReqInfo(ctx) == nil {
|
||||
bugLogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.ErrorKind)
|
||||
@@ -439,6 +428,23 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
||||
return authorizeRequestWithExistingTags(ctx, r, action, "")
|
||||
}
|
||||
|
||||
func deleteObjectAction(versionID string) policy.Action {
|
||||
if versionID != "" {
|
||||
return policy.DeleteObjectVersionAction
|
||||
}
|
||||
return policy.DeleteObjectAction
|
||||
}
|
||||
|
||||
func actionUsesObjectVersion(action policy.Action) bool {
|
||||
switch action {
|
||||
case policy.DeleteObjectAction, policy.DeleteObjectVersionAction,
|
||||
policy.ReplicateDeleteAction, policy.BypassGovernanceRetentionAction:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func authorizeRequestWithExistingTags(ctx context.Context, r *http.Request, action policy.Action, existingTags string) (s3Err APIErrorCode) {
|
||||
return authorizeRequestWithTags(ctx, r, action, existingTags, nil)
|
||||
}
|
||||
@@ -457,7 +463,7 @@ func authorizeRequestWithTags(ctx context.Context, r *http.Request, action polic
|
||||
versionID := reqInfo.VersionID
|
||||
conditionValuesForAuth := func(locationConstraint string, credentials auth.Credentials) map[string][]string {
|
||||
values := getConditionValuesWithTags(r, locationConstraint, credentials, existingTags, requestTags)
|
||||
if action == policy.DeleteObjectAction {
|
||||
if actionUsesObjectVersion(action) {
|
||||
// DeleteObjects carries the effective version ID in each XML object,
|
||||
// not in the request query. Keep authorization scoped to that entry.
|
||||
if versionID == "" {
|
||||
@@ -503,21 +509,6 @@ func authorizeRequestWithTags(ctx context.Context, r *http.Request, action polic
|
||||
|
||||
return ErrAccessDenied
|
||||
}
|
||||
if action == policy.DeleteObjectAction && versionID != "" {
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.Action(policy.DeleteObjectVersionAction),
|
||||
BucketName: bucket,
|
||||
ConditionValues: conditionValuesForAuth("", cred),
|
||||
ObjectName: object,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: true,
|
||||
}) { // Request is not allowed if Deny action on DeleteObjectVersionAction
|
||||
return ErrAccessDenied
|
||||
}
|
||||
}
|
||||
if globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
@@ -553,6 +544,40 @@ func authorizeRequestWithTags(ctx context.Context, r *http.Request, action polic
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
// authorizeReplicationDelete preserves the established target-credential
|
||||
// contract for trusted replication: DeleteObject and ReplicateDelete must be
|
||||
// allowed, while an explicit DeleteObjectVersion deny still blocks a named
|
||||
// version. Ordinary S3 requests never use this compatibility path.
|
||||
func authorizeReplicationDelete(ctx context.Context, r *http.Request) APIErrorCode {
|
||||
if s3Err := authorizeRequest(ctx, r, policy.DeleteObjectAction); s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
if reqInfo == nil {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
if reqInfo.VersionID == "" {
|
||||
return ErrNone
|
||||
}
|
||||
cred := reqInfo.Cred
|
||||
values := getConditionValuesWithTags(r, "", cred, "", nil)
|
||||
values["versionid"] = []string{reqInfo.VersionID}
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.DeleteObjectVersionAction,
|
||||
BucketName: reqInfo.BucketName,
|
||||
ConditionValues: values,
|
||||
ObjectName: reqInfo.ObjectName,
|
||||
IsOwner: reqInfo.Owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: true,
|
||||
}) {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
// Check request auth type verifies the incoming http request
|
||||
// - validates the request signature
|
||||
// - validates the policy action if anonymous tests bucket policies if any,
|
||||
@@ -607,18 +632,11 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
|
||||
return ErrInvalidDigest
|
||||
}
|
||||
|
||||
// Extract either 'X-Amz-Content-Sha256' header or 'X-Amz-Content-Sha256' query parameter (if V4 presigned)
|
||||
// Do not verify 'X-Amz-Content-Sha256' if skipSHA256.
|
||||
// Honor the selected header/query checksum, including the header fallback
|
||||
// for a presigned request. STS separately hashes its body for its signature.
|
||||
var contentSHA256 []byte
|
||||
if skipSHA256 := skipContentSha256Cksum(r); !skipSHA256 && isRequestPresignedSignatureV4(r) {
|
||||
if sha256Sum, ok := r.Form[xhttp.AmzContentSha256]; ok && len(sha256Sum) > 0 {
|
||||
contentSHA256, err = hex.DecodeString(sha256Sum[0])
|
||||
if err != nil {
|
||||
return ErrContentSHA256Mismatch
|
||||
}
|
||||
}
|
||||
} else if _, ok := r.Header[xhttp.AmzContentSha256]; !skipSHA256 && ok {
|
||||
contentSHA256, err = hex.DecodeString(r.Header.Get(xhttp.AmzContentSha256))
|
||||
if !skipContentSha256Cksum(r) {
|
||||
contentSHA256, err = hex.DecodeString(getContentSha256Cksum(r, serviceS3))
|
||||
if err != nil || len(contentSHA256) == 0 {
|
||||
return ErrContentSHA256Mismatch
|
||||
}
|
||||
@@ -786,10 +804,20 @@ func isPutActionAllowedWithRequestTags(ctx context.Context, atype authType, buck
|
||||
return s3Err
|
||||
}
|
||||
|
||||
logger.GetReqInfo(ctx).Cred = cred
|
||||
logger.GetReqInfo(ctx).Owner = owner
|
||||
logger.GetReqInfo(ctx).Region = region
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
if reqInfo == nil {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
reqInfo.Lock()
|
||||
reqInfo.Cred = cred
|
||||
reqInfo.Owner = owner
|
||||
reqInfo.Region = region
|
||||
reqInfo.Unlock()
|
||||
|
||||
return isPutActionAllowedWithCred(bucketName, objectName, r, action, requestTags, cred, owner)
|
||||
}
|
||||
|
||||
func isPutActionAllowedWithCred(bucketName, objectName string, r *http.Request, action policy.Action, requestTags *string, cred auth.Credentials, owner bool) APIErrorCode {
|
||||
// Do not check for PutObjectRetentionAction permission,
|
||||
// if mode and retain until date are not set.
|
||||
// Can happen when bucket has default lock config set
|
||||
|
||||
@@ -28,7 +28,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
type nullReader struct{}
|
||||
|
||||
@@ -25,7 +25,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
)
|
||||
|
||||
// healTask represents what to heal along with options
|
||||
|
||||
@@ -34,7 +34,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
+4
-4
@@ -33,10 +33,10 @@ import (
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/minio/pkg/v3/wildcard"
|
||||
"github.com/minio/pkg/v3/workers"
|
||||
"github.com/minio/pkg/v3/xtime"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/wildcard"
|
||||
"github.com/pgsty/silo-pkg/v3/workers"
|
||||
"github.com/pgsty/silo-pkg/v3/xtime"
|
||||
"go.yaml.in/yaml/v3"
|
||||
)
|
||||
|
||||
|
||||
@@ -48,10 +48,10 @@ import (
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/pkg/v3/console"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/workers"
|
||||
"github.com/pgsty/silo-pkg/v3/console"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/workers"
|
||||
"go.yaml.in/yaml/v3"
|
||||
)
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/pkg/v3/wildcard"
|
||||
"github.com/pgsty/silo-pkg/v3/wildcard"
|
||||
"go.yaml.in/yaml/v3"
|
||||
)
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
|
||||
miniogo "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/pkg/v3/xtime"
|
||||
"github.com/pgsty/silo-pkg/v3/xtime"
|
||||
)
|
||||
|
||||
//go:generate msgp -file $GOFILE
|
||||
|
||||
+2
-2
@@ -34,8 +34,8 @@ import (
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/minio/pkg/v3/workers"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/workers"
|
||||
)
|
||||
|
||||
// keyrotate:
|
||||
|
||||
@@ -34,7 +34,7 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio/internal/grid"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
)
|
||||
|
||||
// To abstract a node over network.
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"hash/crc32"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
)
|
||||
|
||||
func TestPutBucketCorsWireValidation(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testPutBucketCorsWireValidation,
|
||||
endpoints: []string{"PutBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testPutBucketCorsWireValidation(_ ObjectLayer, _ string, bucketName string, apiRouter http.Handler,
|
||||
creds auth.Credentials, t *testing.T,
|
||||
) {
|
||||
valid := `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`
|
||||
rule := `<CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule>`
|
||||
tests := []struct {
|
||||
name string
|
||||
body string
|
||||
want int
|
||||
wantCode string
|
||||
}{
|
||||
{
|
||||
name: "second XML root",
|
||||
body: valid + `<Extra/>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "255 Unicode character ID",
|
||||
body: `<CORSConfiguration><CORSRule><ID>` + strings.Repeat("界", 255) + `</ID><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "256 Unicode character ID",
|
||||
body: `<CORSConfiguration><CORSRule><ID>` + strings.Repeat("界", 256) + `</ID><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "lowercase method",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>get</AllowedMethod></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "empty origin",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin/><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "question mark origin wildcard",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>https://?.example.com</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "question mark header wildcard",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><AllowedHeader>x-amz-?</AllowedHeader></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "unknown element",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><Unknown/></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "empty max age",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><MaxAgeSeconds/></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "zero max age",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><MaxAgeSeconds>0</MaxAgeSeconds></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "max age int32 overflow",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><MaxAgeSeconds>2147483648</MaxAgeSeconds></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "100 rules",
|
||||
body: `<CORSConfiguration>` + strings.Repeat(rule, 100) + `</CORSConfiguration>`,
|
||||
want: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "101 rules",
|
||||
body: `<CORSConfiguration>` + strings.Repeat(rule, 101) + `</CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "exactly 64 KiB",
|
||||
body: sizedCORSConfig(maxBucketCorsSize),
|
||||
want: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "over 64 KiB",
|
||||
body: sizedCORSConfig(maxBucketCorsSize + 1),
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "EntityTooLarge",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, getBucketCorsURL("", bucketName),
|
||||
int64(len(tt.body)), bytes.NewReader([]byte(tt.body)), creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != tt.want {
|
||||
t.Fatalf("expected status %d, got %d: %s", tt.want, rec.Code, rec.Body.String())
|
||||
}
|
||||
if tt.wantCode != "" && !bytes.Contains(rec.Body.Bytes(), []byte("<Code>"+tt.wantCode+"</Code>")) {
|
||||
t.Fatalf("expected error code %s, got: %s", tt.wantCode, rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func sizedCORSConfig(size int) string {
|
||||
prefix := `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod>`
|
||||
suffix := `</CORSRule></CORSConfiguration>`
|
||||
return prefix + strings.Repeat(" ", size-len(prefix)-len(suffix)) + suffix
|
||||
}
|
||||
|
||||
func TestPutBucketCorsChecksumValidation(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testPutBucketCorsChecksumValidation,
|
||||
endpoints: []string{"PutBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testPutBucketCorsChecksumValidation(_ ObjectLayer, _ string, bucketName string, apiRouter http.Handler,
|
||||
creds auth.Credentials, t *testing.T,
|
||||
) {
|
||||
body := []byte(`<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`)
|
||||
tests := []struct {
|
||||
name string
|
||||
configure func(*http.Request)
|
||||
want int
|
||||
wantCode string
|
||||
}{
|
||||
{
|
||||
name: "missing checksum",
|
||||
configure: func(req *http.Request) {
|
||||
req.Header.Del("Content-Md5")
|
||||
},
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MissingContentMD5",
|
||||
},
|
||||
{
|
||||
name: "bad content md5",
|
||||
configure: func(req *http.Request) {
|
||||
req.Header.Set("Content-Md5", getMD5HashBase64([]byte("different body")))
|
||||
},
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "BadDigest",
|
||||
},
|
||||
{
|
||||
name: "valid sdk crc32",
|
||||
configure: func(req *http.Request) {
|
||||
req.Header.Del("Content-Md5")
|
||||
req.Header.Set("X-Amz-Sdk-Checksum-Algorithm", "CRC32")
|
||||
req.Header.Set("X-Amz-Checksum-Crc32", corsCRC32Base64(body))
|
||||
},
|
||||
want: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "bad sdk crc32",
|
||||
configure: func(req *http.Request) {
|
||||
req.Header.Del("Content-Md5")
|
||||
req.Header.Set("X-Amz-Sdk-Checksum-Algorithm", "CRC32")
|
||||
req.Header.Set("X-Amz-Checksum-Crc32", corsCRC32Base64([]byte("different body")))
|
||||
},
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "BadDigest",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req, err := newTestRequest(http.MethodPut, getBucketCorsURL("", bucketName), int64(len(body)), bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tt.configure(req)
|
||||
if err = signRequestV4(req, creds.AccessKey, creds.SecretKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != tt.want || (tt.wantCode != "" && !bytes.Contains(rec.Body.Bytes(), []byte("<Code>"+tt.wantCode+"</Code>"))) {
|
||||
t.Fatalf("expected status %d and code %s, got %d: %s", tt.want, tt.wantCode, rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func corsCRC32Base64(data []byte) string {
|
||||
var checksum [4]byte
|
||||
binary.BigEndian.PutUint32(checksum[:], crc32.ChecksumIEEE(data))
|
||||
return base64.StdEncoding.EncodeToString(checksum[:])
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/bucket/cors"
|
||||
hashpkg "github.com/minio/minio/internal/hash"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// maxBucketCorsSize is the maximum allowed size of a CORS configuration document.
|
||||
const maxBucketCorsSize = 64 * humanize.KiByte
|
||||
|
||||
// PutBucketCorsHandler - PUT bucket cors.
|
||||
func (api objectAPIHandlers) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PutBucketCors")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketCorsAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if r.ContentLength <= 0 {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
|
||||
return
|
||||
}
|
||||
if r.ContentLength > maxBucketCorsSize {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// PutBucketCors requires a Content-Md5 or a supported full-header
|
||||
// checksum. validateLengthAndChecksum wraps r.Body so the supplied digest
|
||||
// is verified as the body is read below.
|
||||
if !validateLengthAndChecksum(r) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
corsBytes, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
if errors.Is(err, hashpkg.ErrInvalidChecksum) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadDigest), r.URL)
|
||||
return
|
||||
}
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
corsCfg, err := cors.ParseBucketCorsConfig(bytes.NewReader(corsBytes))
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL)
|
||||
return
|
||||
}
|
||||
if err := corsCfg.Validate(); err != nil {
|
||||
writeErrorResponse(ctx, w, APIError{
|
||||
Code: "MalformedXML",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
Description: err.Error(),
|
||||
}, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
updatedAt, err := updateLocalBucketCORSMetadata(ctx, objAPI, bucket, corsBytes)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Call site replication hook.
|
||||
//
|
||||
// We encode the xml bytes as base64 to ensure there are no encoding
|
||||
// errors.
|
||||
cfgStr := base64.StdEncoding.EncodeToString(corsBytes)
|
||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypeCorsConfig,
|
||||
Bucket: bucket,
|
||||
Cors: &cfgStr,
|
||||
UpdatedAt: updatedAt,
|
||||
}))
|
||||
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
// GetBucketCorsHandler - GET bucket cors.
|
||||
func (api objectAPIHandlers) GetBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetBucketCors")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketCorsAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
configData, _, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, errConfigNotFound) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNoSuchCORSConfiguration), r.URL)
|
||||
return
|
||||
}
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseXML(w, configData)
|
||||
}
|
||||
|
||||
// DeleteBucketCorsHandler - DELETE bucket cors.
|
||||
func (api objectAPIHandlers) DeleteBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "DeleteBucketCors")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketCorsAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
updatedAt, err := updateLocalBucketCORSMetadata(ctx, objAPI, bucket, nil)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypeCorsConfig,
|
||||
Bucket: bucket,
|
||||
Cors: nil,
|
||||
UpdatedAt: updatedAt,
|
||||
}))
|
||||
|
||||
writeSuccessNoContent(w)
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
)
|
||||
|
||||
const testCORSDoc = `<CORSConfiguration><CORSRule><AllowedOrigin>http://example.com</AllowedOrigin><AllowedMethod>GET</AllowedMethod><AllowedMethod>PUT</AllowedMethod><ExposeHeader>ETag</ExposeHeader><MaxAgeSeconds>3000</MaxAgeSeconds></CORSRule></CORSConfiguration>`
|
||||
|
||||
func TestBucketCorsHandlers(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testBucketCorsHandlers, endpoints: []string{"PutBucketCors", "GetBucketCors", "DeleteBucketCors"}})
|
||||
}
|
||||
|
||||
func testBucketCorsHandlers(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
creds auth.Credentials, t *testing.T,
|
||||
) {
|
||||
// PUT
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, getBucketCorsURL("", bucketName),
|
||||
int64(len(testCORSDoc)), bytes.NewReader([]byte(testCORSDoc)), creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("PUT cors: expected 200, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// GET returns what we stored
|
||||
req, err = newTestSignedRequestV4(http.MethodGet, getBucketCorsURL("", bucketName),
|
||||
0, nil, creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("GET cors: expected 200, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !bytes.Contains(rec.Body.Bytes(), []byte("http://example.com")) {
|
||||
t.Fatalf("GET cors: body missing origin: %s", rec.Body.String())
|
||||
}
|
||||
|
||||
// DELETE
|
||||
req, err = newTestSignedRequestV4(http.MethodDelete, getBucketCorsURL("", bucketName),
|
||||
0, nil, creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("DELETE cors: expected 204, got %d", rec.Code)
|
||||
}
|
||||
|
||||
// GET after delete → 404 NoSuchCORSConfiguration
|
||||
req, err = newTestSignedRequestV4(http.MethodGet, getBucketCorsURL("", bucketName),
|
||||
0, nil, creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("GET cors after delete: expected 404, got %d", rec.Code)
|
||||
}
|
||||
|
||||
// Malformed XML → 400
|
||||
req, err = newTestSignedRequestV4(http.MethodPut, getBucketCorsURL("", bucketName),
|
||||
int64(len("<bad>")), bytes.NewReader([]byte("<bad>")), creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("PUT malformed cors: expected 400, got %d", rec.Code)
|
||||
}
|
||||
|
||||
// Missing Content-MD5 is rejected before the body is parsed.
|
||||
req, err = newTestRequest(http.MethodPut, getBucketCorsURL("", bucketName),
|
||||
int64(len(testCORSDoc)), bytes.NewReader([]byte(testCORSDoc)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Header.Del("Content-Md5")
|
||||
if err = signRequestV4(req, creds.AccessKey, creds.SecretKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusBadRequest || !bytes.Contains(rec.Body.Bytes(), []byte("<Code>MissingContentMD5</Code>")) {
|
||||
t.Fatalf("PUT cors without Content-MD5: expected MissingContentMD5, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// A signed but incorrect Content-MD5 is rejected while reading the body.
|
||||
req, err = newTestRequest(http.MethodPut, getBucketCorsURL("", bucketName),
|
||||
int64(len(testCORSDoc)), bytes.NewReader([]byte(testCORSDoc)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Header.Set("Content-Md5", getMD5HashBase64([]byte("different body")))
|
||||
if err = signRequestV4(req, creds.AccessKey, creds.SecretKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusBadRequest || !bytes.Contains(rec.Body.Bytes(), []byte("<Code>BadDigest</Code>")) {
|
||||
t.Fatalf("PUT cors with bad Content-MD5: expected BadDigest, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// Re-PUT the config so the store→GetCorsConfig→enforce seam below has
|
||||
// something to enforce (the earlier DELETE removed it).
|
||||
req, err = newTestSignedRequestV4(http.MethodPut, getBucketCorsURL("", bucketName),
|
||||
int64(len(testCORSDoc)), bytes.NewReader([]byte(testCORSDoc)), creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("PUT cors (re-put): expected 200, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// End-to-end enforcement: drive an OPTIONS preflight through the real
|
||||
// corsHandler wrapper (not applyBucketCors in isolation), exercising the
|
||||
// full store -> globalBucketMetadataSys.GetCorsConfig -> enforce seam.
|
||||
wrapped := corsHandler(apiRouter)
|
||||
|
||||
preflightURL := getBucketCorsURL("", bucketName)
|
||||
preflightReq := httptest.NewRequest(http.MethodOptions, preflightURL, nil)
|
||||
preflightReq.Header.Set("Origin", "http://example.com")
|
||||
preflightReq.Header.Set("Access-Control-Request-Method", http.MethodGet)
|
||||
|
||||
rec = httptest.NewRecorder()
|
||||
wrapped.ServeHTTP(rec, preflightReq)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("OPTIONS preflight via corsHandler: expected 200, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "http://example.com" {
|
||||
t.Fatalf("OPTIONS preflight via corsHandler: expected Access-Control-Allow-Origin echoed, got %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,743 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/bucket/cors"
|
||||
)
|
||||
|
||||
type corsLookupCountingObjectLayer struct {
|
||||
ObjectLayer
|
||||
getObjectNInfoCalls atomic.Int64
|
||||
}
|
||||
|
||||
func (o *corsLookupCountingObjectLayer) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
|
||||
o.getObjectNInfoCalls.Add(1)
|
||||
return o.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
|
||||
}
|
||||
|
||||
func TestPerBucketCorsPreflight(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"http://example.com"},
|
||||
AllowedMethods: []string{"GET", "PUT"},
|
||||
AllowedHeaders: []string{"*"},
|
||||
ExposeHeaders: []string{"ETag"},
|
||||
MaxAgeSeconds: 3000,
|
||||
}}}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodOptions, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "http://example.com")
|
||||
req.Header.Set("Access-Control-Request-Method", "GET")
|
||||
req.Header.Set("Access-Control-Request-Headers", "X-Amz-Date")
|
||||
|
||||
handled := applyBucketCors(rec, req, cfg)
|
||||
if !handled {
|
||||
t.Fatal("expected preflight to be handled")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "http://example.com" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "true" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Methods"); got != "GET, PUT" {
|
||||
t.Fatalf("allow-methods = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Headers"); got != "X-Amz-Date" {
|
||||
t.Fatalf("allow-headers = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Expose-Headers"); got != "ETag" {
|
||||
t.Fatalf("expose-headers = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Max-Age"); got != "3000" {
|
||||
t.Fatalf("max-age = %q", got)
|
||||
}
|
||||
requireCorsVary(t, rec.Header())
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("preflight status = %d", rec.Code)
|
||||
}
|
||||
requireCorsOriginVary(t, rec.Header())
|
||||
}
|
||||
|
||||
func TestPerBucketCorsActualRequestNoMatchVariesByOrigin(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"https://allowed.example.com"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
}}}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "https://denied.example.com")
|
||||
|
||||
if handled := applyBucketCors(rec, req, cfg); handled {
|
||||
t.Fatal("actual request must continue when CORS does not match")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
requireCorsOriginVary(t, rec.Header())
|
||||
}
|
||||
|
||||
func TestPerBucketCorsPreflightNoMatch(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"http://example.com"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
}}}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodOptions, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "http://evil.com")
|
||||
req.Header.Set("Access-Control-Request-Method", "GET")
|
||||
|
||||
handled := applyBucketCors(rec, req, cfg)
|
||||
if !handled {
|
||||
t.Fatal("expected preflight to be handled (rejected)")
|
||||
}
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("expected 403 for disallowed origin, got %d", rec.Code)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
|
||||
t.Fatalf("rejected preflight returned allow-origin %q", got)
|
||||
}
|
||||
requireCorsVary(t, rec.Header())
|
||||
}
|
||||
|
||||
func TestPerBucketCorsPreflightWildcardOriginAndZeroMaxAge(t *testing.T) {
|
||||
doc := `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><AllowedMethod>HEAD</AllowedMethod><AllowedHeader>*</AllowedHeader><ExposeHeader>ETag</ExposeHeader><MaxAgeSeconds>0</MaxAgeSeconds></CORSRule></CORSConfiguration>`
|
||||
cfg, err := cors.ParseBucketCorsConfig(strings.NewReader(doc))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodOptions, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
req.Header.Set("Access-Control-Request-Method", "GET")
|
||||
req.Header.Set("Access-Control-Request-Headers", "RANGE")
|
||||
|
||||
if handled := applyBucketCors(rec, req, cfg); !handled {
|
||||
t.Fatal("expected preflight to be handled")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Methods"); got != "GET, HEAD" {
|
||||
t.Fatalf("allow-methods = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Headers"); got != "RANGE" {
|
||||
t.Fatalf("allow-headers = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Expose-Headers"); got != "ETag" {
|
||||
t.Fatalf("expose-headers = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Max-Age"); got != "0" {
|
||||
t.Fatalf("max-age = %q", got)
|
||||
}
|
||||
requireCorsVary(t, rec.Header())
|
||||
}
|
||||
|
||||
func TestPerBucketCorsPreflightUsesFirstFullyMatchingRule(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{
|
||||
{
|
||||
AllowedOrigins: []string{"https://app.example.com"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
AllowedHeaders: []string{"x-a"},
|
||||
ExposeHeaders: []string{"x-rule-a"},
|
||||
MaxAgeSeconds: 1,
|
||||
},
|
||||
{
|
||||
AllowedOrigins: []string{"https://app.example.com"},
|
||||
AllowedMethods: []string{"GET", "HEAD"},
|
||||
AllowedHeaders: []string{"*"},
|
||||
ExposeHeaders: []string{"x-rule-b"},
|
||||
MaxAgeSeconds: 2,
|
||||
},
|
||||
}}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodOptions, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
req.Header.Set("Access-Control-Request-Method", "GET")
|
||||
req.Header.Set("Access-Control-Request-Headers", "X-B")
|
||||
|
||||
if handled := applyBucketCors(rec, req, cfg); !handled {
|
||||
t.Fatal("expected preflight to be handled")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Expose-Headers"); got != "x-rule-b" {
|
||||
t.Fatalf("selected rule expose-headers = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Methods"); got != "GET, HEAD" {
|
||||
t.Fatalf("selected rule allow-methods = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Max-Age"); got != "2" {
|
||||
t.Fatalf("selected rule max-age = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPerBucketCorsActualRequest(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"*"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
ExposeHeaders: []string{"ETag"},
|
||||
}}}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "http://any.com")
|
||||
|
||||
handled := applyBucketCors(rec, req, cfg)
|
||||
if handled {
|
||||
t.Fatal("actual (non-preflight) request must not be terminated by CORS")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Expose-Headers"); got != "ETag" {
|
||||
t.Fatalf("expose-headers = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPerBucketCorsOriginPatternResponse(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"https://app.example.com", "https://*", "*"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
}}}
|
||||
|
||||
tests := []struct {
|
||||
origin string
|
||||
wantOrigin string
|
||||
wantCredentials string
|
||||
}{
|
||||
{"https://app.example.com", "https://app.example.com", "true"},
|
||||
{"https://other.example.com", "https://other.example.com", "true"},
|
||||
{"http://other.example.com", "*", ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", tt.origin)
|
||||
if handled := applyBucketCors(rec, req, cfg); handled {
|
||||
t.Fatal("actual request must not be terminated by CORS")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != tt.wantOrigin {
|
||||
t.Fatalf("origin %q: allow-origin = %q, want %q", tt.origin, got, tt.wantOrigin)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != tt.wantCredentials {
|
||||
t.Fatalf("origin %q: allow-credentials = %q, want %q", tt.origin, got, tt.wantCredentials)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketCorsMetadataErrorFailsClosed(t *testing.T) {
|
||||
oldObjectAPI := newObjectLayerFn()
|
||||
oldMetadataSys := globalBucketMetadataSys
|
||||
setObjectLayer(nil)
|
||||
globalBucketMetadataSys = NewBucketMetadataSys()
|
||||
// A resident bucket whose stored CORS document does not parse must not be
|
||||
// answered with the global policy: it has a configuration we cannot honor.
|
||||
meta := newBucketMetadata("cors-metadata-error")
|
||||
meta.corsConfigErr = fmt.Errorf("invalid bucket CORS configuration")
|
||||
globalBucketMetadataSys.Set("cors-metadata-error", meta)
|
||||
defer func() {
|
||||
setObjectLayer(oldObjectAPI)
|
||||
globalBucketMetadataSys = oldMetadataSys
|
||||
}()
|
||||
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
|
||||
for _, method := range []string{http.MethodGet, http.MethodOptions} {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(method, getGetObjectURL("", "cors-metadata-error", "object"), nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
if method == http.MethodOptions {
|
||||
req.Header.Set("Access-Control-Request-Method", http.MethodGet)
|
||||
}
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("%s status = %d, want %d", method, rec.Code, http.StatusNoContent)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
|
||||
t.Fatalf("%s metadata error fell back to global allow-origin %q", method, got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "" {
|
||||
t.Fatalf("%s metadata error fell back to global credentials %q", method, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketCorsSkipsMetadataLookupWithoutOrigin(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsSkipsMetadataLookupWithoutOrigin,
|
||||
endpoints: []string{"GetObject"},
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketCorsSkipsMetadataLookupWithoutOrigin(obj ObjectLayer, _ string, _ string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
oldObjectAPI := newObjectLayerFn()
|
||||
counting := &corsLookupCountingObjectLayer{ObjectLayer: obj}
|
||||
setObjectLayer(counting)
|
||||
defer setObjectLayer(oldObjectAPI)
|
||||
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
rec := httptest.NewRecorder()
|
||||
wrapped.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, getGetObjectURL("", "api", "v1/login"), nil))
|
||||
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusNoContent)
|
||||
}
|
||||
requireCorsOriginVary(t, rec.Header())
|
||||
if got := counting.getObjectNInfoCalls.Load(); got != 0 {
|
||||
t.Fatalf("request without Origin performed %d bucket metadata reads", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketCorsOriginlessPreflightShapeUsesGlobalHandler(t *testing.T) {
|
||||
nextCalled := false
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
nextCalled = true
|
||||
w.WriteHeader(http.StatusTeapot)
|
||||
}))
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodOptions, getGetObjectURL("", "api", "v1/login"), nil)
|
||||
req.Header.Set("Access-Control-Request-Method", http.MethodGet)
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusNoContent)
|
||||
}
|
||||
if nextCalled {
|
||||
t.Fatal("originless preflight-shaped OPTIONS reached the application handler")
|
||||
}
|
||||
requireCorsOriginVary(t, rec.Header())
|
||||
}
|
||||
|
||||
func TestBucketCorsNoConfigUsesGlobalFallback(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsNoConfigUsesGlobalFallback,
|
||||
endpoints: []string{"GetBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func TestBucketCorsMissingBucketUsesGlobalFallback(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsMissingBucketUsesGlobalFallback,
|
||||
endpoints: []string{"GetBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketCorsMissingBucketUsesGlobalFallback(obj ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
// Model a fully started server: bucket metadata loading has completed, so
|
||||
// a name that is not resident is genuinely not a CORS-bearing bucket.
|
||||
restore := markBucketMetadataInitialized(t)
|
||||
defer restore()
|
||||
|
||||
// A non-resident bucket name must not cause any bucket-metadata disk read.
|
||||
oldObjectAPI := newObjectLayerFn()
|
||||
counting := &corsLookupCountingObjectLayer{ObjectLayer: obj}
|
||||
setObjectLayer(counting)
|
||||
defer setObjectLayer(oldObjectAPI)
|
||||
|
||||
before := bucketMetadataMapLen()
|
||||
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, getGetObjectURL("", bucket+"-missing", "object"), nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusNotFound)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://app.example.com" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "true" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
// Regression guard: the pre-auth CORS lookup for a non-existent bucket must
|
||||
// neither read bucket metadata from disk nor cache a synthetic entry.
|
||||
if got := counting.getObjectNInfoCalls.Load(); got != 0 {
|
||||
t.Fatalf("missing-bucket CORS lookup performed %d bucket metadata reads", got)
|
||||
}
|
||||
if after := bucketMetadataMapLen(); after != before {
|
||||
t.Fatalf("missing-bucket CORS lookup grew metadataMap from %d to %d", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
func testBucketCorsNoConfigUsesGlobalFallback(_ ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, getGetObjectURL("", bucket, "object"), nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusNoContent)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://app.example.com" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "true" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPerBucketCorsActualPatternOriginSupportsCredentials(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"https://*.example.com"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
}}}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
|
||||
if handled := applyBucketCors(rec, req, cfg); handled {
|
||||
t.Fatal("actual request must continue")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://app.example.com" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "true" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPerBucketCorsActualNullOriginSurvivesForwardingMiddleware(t *testing.T) {
|
||||
next := setBucketForwardingMiddleware(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
|
||||
t.Run("per-bucket null origin", func(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"null"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
}}}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "null")
|
||||
|
||||
if handled := applyBucketCors(rec, req, cfg); handled {
|
||||
t.Fatal("actual request must continue")
|
||||
}
|
||||
next.ServeHTTP(rec, req)
|
||||
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "null" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "true" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("legacy unmarked null origin", func(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
rec.Header().Set("Access-Control-Allow-Origin", "null")
|
||||
req := httptest.NewRequest(http.MethodGet, "/mybucket/obj", nil)
|
||||
|
||||
next.ServeHTTP(rec, req)
|
||||
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func requireCorsVary(t *testing.T, header http.Header) {
|
||||
t.Helper()
|
||||
values := strings.Join(header.Values("Vary"), ",")
|
||||
for _, want := range []string{"Origin", "Access-Control-Request-Method", "Access-Control-Request-Headers"} {
|
||||
if !strings.Contains(values, want) {
|
||||
t.Fatalf("Vary = %q, missing %q", values, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func requireCorsOriginVary(t *testing.T, header http.Header) {
|
||||
t.Helper()
|
||||
if values := strings.Join(header.Values("Vary"), ","); !strings.Contains(values, "Origin") {
|
||||
t.Fatalf("Vary = %q, missing Origin", values)
|
||||
}
|
||||
}
|
||||
|
||||
// markBucketMetadataInitialized marks the global bucket-metadata subsystem as
|
||||
// fully loaded, modeling a running server (the API test harness sets up the
|
||||
// subsystem but does not run Init). It returns a function that restores the
|
||||
// previous state.
|
||||
func markBucketMetadataInitialized(t *testing.T) func() {
|
||||
t.Helper()
|
||||
sys := globalBucketMetadataSys
|
||||
if sys == nil {
|
||||
t.Fatal("globalBucketMetadataSys is nil")
|
||||
}
|
||||
sys.Lock()
|
||||
prev := sys.initialized
|
||||
sys.initialized = true
|
||||
sys.Unlock()
|
||||
return func() {
|
||||
sys.Lock()
|
||||
sys.initialized = prev
|
||||
sys.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// bucketMetadataMapLen returns the number of resident bucket-metadata entries.
|
||||
func bucketMetadataMapLen() int {
|
||||
sys := globalBucketMetadataSys
|
||||
if sys == nil {
|
||||
return 0
|
||||
}
|
||||
sys.RLock()
|
||||
defer sys.RUnlock()
|
||||
return len(sys.metadataMap)
|
||||
}
|
||||
|
||||
// TestBucketCorsUnknownBucketDoesNotGrowMetadata is the regression guard for
|
||||
// the pre-auth resource-exhaustion path: an unauthenticated, Origin-bearing
|
||||
// request whose first path segment is not a real bucket must fall back to the
|
||||
// global CORS policy without loading bucket metadata from disk and without
|
||||
// caching a synthetic entry. Before the resident-only lookup, each distinct
|
||||
// name grew metadataMap by one and issued an erasure metadata probe.
|
||||
func TestBucketCorsUnknownBucketDoesNotGrowMetadata(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsUnknownBucketDoesNotGrowMetadata,
|
||||
endpoints: []string{"GetBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketCorsUnknownBucketDoesNotGrowMetadata(obj ObjectLayer, _ string, _ string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
restore := markBucketMetadataInitialized(t)
|
||||
defer restore()
|
||||
|
||||
oldObjectAPI := newObjectLayerFn()
|
||||
counting := &corsLookupCountingObjectLayer{ObjectLayer: obj}
|
||||
setObjectLayer(counting)
|
||||
defer setObjectLayer(oldObjectAPI)
|
||||
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
|
||||
before := bucketMetadataMapLen()
|
||||
// Console/admin routes plus enough distinct valid names to make accidental
|
||||
// cache growth or one metadata probe per name unambiguous.
|
||||
names := []string{"minio", "api"}
|
||||
for i := 0; i < 500; i++ {
|
||||
names = append(names, fmt.Sprintf("cors-missing-%03d", i))
|
||||
}
|
||||
for _, name := range names {
|
||||
for _, method := range []string{http.MethodGet, http.MethodOptions} {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(method, getGetObjectURL("", name, "obj"), nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
if method == http.MethodOptions {
|
||||
req.Header.Set("Access-Control-Request-Method", http.MethodGet)
|
||||
}
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://app.example.com" {
|
||||
t.Fatalf("%s/%s: allow-origin = %q, want global fallback", name, method, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, path := range []string{"/../obj", "/A/obj", "/x/obj", "/minio/admin/v3/info", "/api/v1/login"} {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://app.example.com" {
|
||||
t.Fatalf("%s: allow-origin = %q, want global fallback", path, got)
|
||||
}
|
||||
}
|
||||
|
||||
if got := counting.getObjectNInfoCalls.Load(); got != 0 {
|
||||
t.Fatalf("unknown-bucket CORS lookups performed %d bucket metadata reads", got)
|
||||
}
|
||||
if after := bucketMetadataMapLen(); after != before {
|
||||
t.Fatalf("unknown-bucket CORS lookups grew metadataMap from %d to %d", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketCorsStartupMissFailsClosedWithoutIO(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsStartupMissFailsClosedWithoutIO,
|
||||
endpoints: []string{"GetBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketCorsStartupMissFailsClosedWithoutIO(obj ObjectLayer, _ string, _ string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
oldObjectAPI := newObjectLayerFn()
|
||||
oldMetadataSys := globalBucketMetadataSys
|
||||
counting := &corsLookupCountingObjectLayer{ObjectLayer: obj}
|
||||
setObjectLayer(counting)
|
||||
globalBucketMetadataSys = NewBucketMetadataSys()
|
||||
defer func() {
|
||||
setObjectLayer(oldObjectAPI)
|
||||
globalBucketMetadataSys = oldMetadataSys
|
||||
}()
|
||||
|
||||
innerCalled := false
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
innerCalled = true
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/startup-missing/object", nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if !innerCalled || rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("startup miss did not reach inner handler: called=%v status=%d", innerCalled, rec.Code)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
|
||||
t.Fatalf("startup miss used permissive global CORS: %q", got)
|
||||
}
|
||||
if got := counting.getObjectNInfoCalls.Load(); got != 0 {
|
||||
t.Fatalf("startup miss performed %d metadata reads", got)
|
||||
}
|
||||
if got := globalBucketMetadataSys.Count(); got != 0 {
|
||||
t.Fatalf("startup miss grew metadataMap to %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// markBucketMetadataLoadFailed records a bucket as one whose metadata failed to
|
||||
// load at startup while the subsystem is Initialized, modeling the degraded
|
||||
// state where a real bucket is not resident. Returns a restore function.
|
||||
func markBucketMetadataLoadFailed(t *testing.T, bucket string) func() {
|
||||
t.Helper()
|
||||
sys := globalBucketMetadataSys
|
||||
if sys == nil {
|
||||
t.Fatal("globalBucketMetadataSys is nil")
|
||||
}
|
||||
sys.Lock()
|
||||
_, had := sys.loadFailed[bucket]
|
||||
sys.loadFailed[bucket] = struct{}{}
|
||||
sys.Unlock()
|
||||
return func() {
|
||||
sys.Lock()
|
||||
if !had {
|
||||
delete(sys.loadFailed, bucket)
|
||||
}
|
||||
sys.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// TestBucketCorsLoadFailedBucketFailsClosed guards P1: a real bucket whose
|
||||
// metadata could not be loaded at startup (present in loadFailed, subsystem
|
||||
// Initialized) must NOT be answered with the permissive global CORS policy. We
|
||||
// cannot rule out a restrictive per-bucket config for it, so it must fail
|
||||
// closed — without a synchronous disk read.
|
||||
func TestBucketCorsLoadFailedBucketFailsClosed(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsLoadFailedBucketFailsClosed,
|
||||
endpoints: []string{"GetBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketCorsLoadFailedBucketFailsClosed(obj ObjectLayer, _ string, _ string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
restoreInit := markBucketMetadataInitialized(t)
|
||||
defer restoreInit()
|
||||
restoreFail := markBucketMetadataLoadFailed(t, "strict-cors-bucket")
|
||||
defer restoreFail()
|
||||
|
||||
oldObjectAPI := newObjectLayerFn()
|
||||
counting := &corsLookupCountingObjectLayer{ObjectLayer: obj}
|
||||
setObjectLayer(counting)
|
||||
defer setObjectLayer(oldObjectAPI)
|
||||
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
|
||||
for _, method := range []string{http.MethodGet, http.MethodOptions} {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(method, getGetObjectURL("", "strict-cors-bucket", "object"), nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
if method == http.MethodOptions {
|
||||
req.Header.Set("Access-Control-Request-Method", http.MethodGet)
|
||||
}
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
|
||||
t.Fatalf("%s: load-failed bucket fell back to global allow-origin %q", method, got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "" {
|
||||
t.Fatalf("%s: load-failed bucket fell back to global credentials %q", method, got)
|
||||
}
|
||||
}
|
||||
if got := counting.getObjectNInfoCalls.Load(); got != 0 {
|
||||
t.Fatalf("load-failed CORS lookup performed %d synchronous bucket metadata reads", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBucketCorsResidentConfigSurvivesRefreshFailure: a resident bucket keeps
|
||||
// its last loaded CORS configuration through a failed refresh, like every
|
||||
// other bucket configuration, and the failure set never records a resident
|
||||
// bucket. Only a bucket that was never loaded fails closed.
|
||||
func TestBucketCorsResidentConfigSurvivesRefreshFailure(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsResidentConfigSurvivesRefreshFailure,
|
||||
endpoints: []string{"GetBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketCorsResidentConfigSurvivesRefreshFailure(obj ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
if _, err := updateLocalBucketCORSMetadata(t.Context(), obj, bucket, []byte(testSiteReplicationCORSDoc)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sys := globalBucketMetadataSys
|
||||
sys.Lock()
|
||||
sys.noteLoadFailure(bucket)
|
||||
_, marked := sys.loadFailed[bucket]
|
||||
sys.Unlock()
|
||||
if marked {
|
||||
t.Fatal("a resident bucket was recorded as a load failure")
|
||||
}
|
||||
cfg, _, err := sys.GetResidentCorsConfig(bucket)
|
||||
if err != nil || cfg == nil {
|
||||
t.Fatalf("resident CORS configuration lost after a refresh failure: cfg=%v err=%v", cfg, err)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -30,7 +30,7 @@ import (
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
+18
-18
@@ -62,8 +62,8 @@ import (
|
||||
"github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -463,13 +463,20 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
// Make sure to update context to print ObjectNames for multi objects.
|
||||
ctx = updateReqContext(ctx, objects...)
|
||||
|
||||
// Call checkRequestAuthType to populate ReqInfo.AccessKey before GetBucketInfo()
|
||||
// Ignore errors here to preserve the S3 error behavior of GetBucketInfo()
|
||||
checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "")
|
||||
|
||||
deleteObjectsFn := objectAPI.DeleteObjects
|
||||
|
||||
// Return Malformed XML as S3 spec if the number of objects is empty
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
if reqInfo == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
reqInfo.BucketName = bucket
|
||||
reqInfo.ObjectName = ""
|
||||
if s3Err := authenticateRequest(ctx, r, policy.DeleteObjectAction); s3Err != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
// Return Malformed XML as S3 spec if the number of objects is empty.
|
||||
if len(deleteObjectsReq.Objects) == 0 || len(deleteObjectsReq.Objects) > maxDeleteList {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL)
|
||||
return
|
||||
@@ -499,11 +506,9 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
vc, _ := globalBucketVersioningSys.Get(bucket)
|
||||
oss := make([]*objSweeper, len(deleteObjectsReq.Objects))
|
||||
for index, object := range deleteObjectsReq.Objects {
|
||||
if apiErrCode := checkRequestAuthTypeWithVID(ctx, r, policy.DeleteObjectAction, bucket, object.ObjectName, object.VersionID); apiErrCode != ErrNone {
|
||||
if apiErrCode == ErrSignatureDoesNotMatch || apiErrCode == ErrInvalidAccessKeyID {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL)
|
||||
return
|
||||
}
|
||||
reqInfo.ObjectName = object.ObjectName
|
||||
reqInfo.VersionID = object.VersionID
|
||||
if apiErrCode := authorizeRequest(ctx, r, deleteObjectAction(object.VersionID)); apiErrCode != ErrNone {
|
||||
apiErr := errorCodes.ToAPIErr(apiErrCode)
|
||||
deleteResults[index].errInfo = DeleteError{
|
||||
Code: apiErr.Code,
|
||||
@@ -1844,12 +1849,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
|
||||
// We encode the xml bytes as base64 to ensure there are no encoding
|
||||
// errors.
|
||||
cfgStr := base64.StdEncoding.EncodeToString(configData)
|
||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypeObjectLockConfig,
|
||||
Bucket: bucket,
|
||||
ObjectLockConfig: &cfgStr,
|
||||
UpdatedAt: updatedAt,
|
||||
}))
|
||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, newSRBucketObjectLockMeta(bucket, &cfgStr, updatedAt)))
|
||||
|
||||
// Write success response.
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
|
||||
+71
-18
@@ -24,12 +24,52 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
)
|
||||
|
||||
func TestListObjectsNonExistentBucketHandler(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testListObjectsNonExistentBucketHandler})
|
||||
}
|
||||
|
||||
func testListObjectsNonExistentBucketHandler(_ ObjectLayer, instanceType, _ string, apiRouter http.Handler,
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
const bucket = "missing-bucket"
|
||||
testCases := []struct {
|
||||
name string
|
||||
query url.Values
|
||||
}{
|
||||
{name: "ListObjects", query: url.Values{"prefix": {"/"}}},
|
||||
{name: "ListObjectsV2", query: url.Values{"list-type": {"2"}, "prefix": {"/"}}},
|
||||
{name: "ListObjectVersions", query: url.Values{"versions": {""}, "prefix": {"/"}}},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
req, err := newTestSignedRequestV4(http.MethodGet, makeTestTargetURL("", bucket, "", tc.query), 0, nil,
|
||||
credentials.AccessKey, credentials.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %s: failed to create request: %v", instanceType, tc.name, err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Errorf("%s: %s: expected status %d, got %d", instanceType, tc.name, http.StatusNotFound, rec.Code)
|
||||
}
|
||||
|
||||
var apiErr APIErrorResponse
|
||||
if err = xml.Unmarshal(rec.Body.Bytes(), &apiErr); err != nil {
|
||||
t.Fatalf("%s: %s: failed to decode error response: %v", instanceType, tc.name, err)
|
||||
}
|
||||
if apiErr.Code != "NoSuchBucket" {
|
||||
t.Errorf("%s: %s: expected NoSuchBucket, got %q", instanceType, tc.name, apiErr.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Wrapper for calling RemoveBucket HTTP handler tests for both Erasure multiple disks and single node setup.
|
||||
func TestRemoveBucketHandler(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testRemoveBucketHandler, endpoints: []string{"RemoveBucket"}})
|
||||
@@ -978,14 +1018,23 @@ func testAPIDeleteMultipleObjectsVersionIDNullCondition(obj ObjectLayer, instanc
|
||||
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[{
|
||||
"Effect":"Allow",
|
||||
"Principal":"*",
|
||||
"Action":"s3:DeleteObject",
|
||||
"Resource":"arn:aws:s3:::%s/*",
|
||||
"Condition":{"Null":{"s3:versionid":"true"}}
|
||||
}]
|
||||
}`, bucketName)
|
||||
"Statement":[
|
||||
{
|
||||
"Effect":"Allow",
|
||||
"Principal":"*",
|
||||
"Action":"s3:DeleteObject",
|
||||
"Resource":"arn:aws:s3:::%s/*",
|
||||
"Condition":{"Null":{"s3:versionid":"true"}}
|
||||
},
|
||||
{
|
||||
"Effect":"Allow",
|
||||
"Principal":"*",
|
||||
"Action":"s3:DeleteObjectVersion",
|
||||
"Resource":"arn:aws:s3:::%s/*",
|
||||
"Condition":{"StringEquals":{"s3:versionid":"%s"}}
|
||||
}
|
||||
]
|
||||
}`, bucketName, bucketName, versionIDs["with-version-id"])
|
||||
policyReq, err := newTestSignedRequestV4(http.MethodPut, getPutPolicyURL("", bucketName), int64(len(policyBytes)),
|
||||
bytes.NewReader(policyBytes), credentials.AccessKey, credentials.SecretKey, nil)
|
||||
if err != nil {
|
||||
@@ -1031,29 +1080,30 @@ func testAPIDeleteMultipleObjectsVersionIDNullCondition(obj ObjectLayer, instanc
|
||||
t.Errorf("%s: %q was not a successful delete-marker creation: %+v", instanceType, objectName, response.DeletedObjects)
|
||||
}
|
||||
}
|
||||
if len(deleted) != 2 {
|
||||
if object, ok := deleted["with-version-id"]; !ok || object.VersionID != versionIDs["with-version-id"] {
|
||||
t.Errorf("%s: matching explicit version was not deleted: %+v", instanceType, response.DeletedObjects)
|
||||
}
|
||||
if len(deleted) != 3 {
|
||||
t.Errorf("%s: unexpected deleted objects: %+v", instanceType, response.DeletedObjects)
|
||||
}
|
||||
errorsByKey := make(map[string]DeleteError, len(response.Errors))
|
||||
for _, deleteErr := range response.Errors {
|
||||
errorsByKey[deleteErr.Key] = deleteErr
|
||||
}
|
||||
for objectName, versionID := range map[string]string{
|
||||
"with-version-id": versionIDs["with-version-id"],
|
||||
"with-null-version-id": nullVersionID,
|
||||
} {
|
||||
for objectName, versionID := range map[string]string{"with-null-version-id": nullVersionID} {
|
||||
deleteErr, ok := errorsByKey[objectName]
|
||||
if !ok || deleteErr.VersionID != versionID || deleteErr.Code != errorCodes[ErrAccessDenied].Code {
|
||||
t.Errorf("%s: %q did not return AccessDenied for version %q: %+v", instanceType, objectName, versionID, response.Errors)
|
||||
}
|
||||
}
|
||||
if len(errorsByKey) != 2 {
|
||||
if len(errorsByKey) != 1 {
|
||||
t.Errorf("%s: unexpected delete errors: %+v", instanceType, response.Errors)
|
||||
}
|
||||
|
||||
// A simple delete adds a marker and keeps the old version. The explicitly
|
||||
// named version must also remain because its policy condition did not match.
|
||||
for objectName, versionID := range versionIDs {
|
||||
// A simple delete adds a marker and keeps the old version. The null-version
|
||||
// delete remains denied because its per-entry condition does not match.
|
||||
for _, objectName := range []string{"without-version-id-before", "without-version-id-after", "with-null-version-id"} {
|
||||
versionID := versionIDs[objectName]
|
||||
if _, err = obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{VersionID: versionID}); err != nil {
|
||||
t.Errorf("%s: version %s of %q was not preserved: %v", instanceType, versionID, objectName, err)
|
||||
}
|
||||
@@ -1063,7 +1113,10 @@ func testAPIDeleteMultipleObjectsVersionIDNullCondition(obj ObjectLayer, instanc
|
||||
t.Errorf("%s: simple delete of %q did not hide the latest object behind a delete marker: %v", instanceType, objectName, err)
|
||||
}
|
||||
}
|
||||
for _, objectName := range []string{"with-version-id", "with-null-version-id"} {
|
||||
if _, err = obj.GetObjectInfo(t.Context(), bucketName, "with-version-id", ObjectOptions{VersionID: versionIDs["with-version-id"]}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Errorf("%s: matching explicit version still exists: %v", instanceType, err)
|
||||
}
|
||||
for _, objectName := range []string{"with-null-version-id"} {
|
||||
if info, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err != nil {
|
||||
t.Errorf("%s: denied version delete removed latest %q: %v", instanceType, objectName, err)
|
||||
} else if info.VersionID != versionIDs[objectName] {
|
||||
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -41,7 +41,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/s3select"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
"github.com/zeebo/xxh3"
|
||||
)
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ import (
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// Validate all the ListObjects query arguments, returns an APIErrorCode
|
||||
|
||||
@@ -0,0 +1,455 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
// Copyright (c) 2026 PGSTY
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
)
|
||||
|
||||
type metadataRMWWriterKey struct{}
|
||||
|
||||
type metadataRMWBarrierObjectLayer struct {
|
||||
ObjectLayer
|
||||
bucket string
|
||||
aReady chan struct{}
|
||||
aRelease chan struct{}
|
||||
bLockAttempt chan struct{}
|
||||
aReadyOnce sync.Once
|
||||
bLockOnce sync.Once
|
||||
cancelOnce sync.Once
|
||||
cancelOnPut context.CancelFunc
|
||||
reads atomic.Int64
|
||||
}
|
||||
|
||||
func (o *metadataRMWBarrierObjectLayer) metadataObject() string {
|
||||
return pathJoin(bucketMetaPrefix, o.bucket, bucketMetadataFile)
|
||||
}
|
||||
|
||||
func (o *metadataRMWBarrierObjectLayer) metadataLock() string {
|
||||
return pathJoin(bucketMetaPrefix, o.bucket, "metadata.lock")
|
||||
}
|
||||
|
||||
func (o *metadataRMWBarrierObjectLayer) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
|
||||
if bucket == minioMetaBucket && object == o.metadataObject() {
|
||||
o.reads.Add(1)
|
||||
}
|
||||
return o.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
|
||||
}
|
||||
|
||||
func (o *metadataRMWBarrierObjectLayer) PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
|
||||
if bucket == minioMetaBucket && object == o.metadataObject() && o.cancelOnPut != nil {
|
||||
o.cancelOnce.Do(o.cancelOnPut)
|
||||
}
|
||||
if bucket == minioMetaBucket && object == o.metadataObject() && ctx.Value(metadataRMWWriterKey{}) == "A" {
|
||||
o.aReadyOnce.Do(func() { close(o.aReady) })
|
||||
select {
|
||||
case <-o.aRelease:
|
||||
case <-ctx.Done():
|
||||
return ObjectInfo{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
return o.ObjectLayer.PutObject(ctx, bucket, object, data, opts)
|
||||
}
|
||||
|
||||
func (o *metadataRMWBarrierObjectLayer) NewNSLock(bucket string, objects ...string) RWLocker {
|
||||
lock := o.ObjectLayer.NewNSLock(bucket, objects...)
|
||||
if bucket != minioMetaBucket || len(objects) != 1 || objects[0] != o.metadataLock() {
|
||||
return lock
|
||||
}
|
||||
return metadataObservedRWLocker{RWLocker: lock, onLock: func(ctx context.Context) {
|
||||
if ctx.Value(metadataRMWWriterKey{}) == "B" {
|
||||
o.bLockOnce.Do(func() { close(o.bLockAttempt) })
|
||||
}
|
||||
}}
|
||||
}
|
||||
|
||||
type metadataObservedRWLocker struct {
|
||||
RWLocker
|
||||
onLock func(context.Context)
|
||||
}
|
||||
|
||||
func (l metadataObservedRWLocker) GetLock(ctx context.Context, timeout *dynamicTimeout) (LockContext, error) {
|
||||
l.onLock(ctx)
|
||||
return l.RWLocker.GetLock(ctx, timeout)
|
||||
}
|
||||
|
||||
func TestBucketMetadataLockPreservesPolicyAndCORS(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketMetadataLockPreservesPolicyAndCORS,
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketMetadataLockPreservesPolicyAndCORS(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
policyJSON := fmt.Appendf(nil, `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket)
|
||||
corsXML := []byte(testSiteReplicationCORSDoc)
|
||||
runBucketMetadataRMWConflict(t, obj, bucket,
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, policyJSON)
|
||||
return err
|
||||
},
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := updateLocalBucketCORSMetadata(ctx, objectAPI, bucket, corsXML)
|
||||
return err
|
||||
},
|
||||
func(meta BucketMetadata) bool {
|
||||
return bytes.Equal(meta.PolicyConfigJSON, policyJSON) && bytes.Equal(meta.CorsConfigXML, corsXML)
|
||||
}, instanceType+": policy+CORS")
|
||||
}
|
||||
|
||||
func TestBucketMetadataLockPreservesTaggingAndSSE(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketMetadataLockPreservesTaggingAndSSE,
|
||||
})
|
||||
}
|
||||
|
||||
func TestBucketMetadataLockPreservesPeerBulkAndLocalUpdate(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketMetadataLockPreservesPeerBulkAndLocalUpdate,
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketMetadataLockPreservesPeerBulkAndLocalUpdate(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
meta, err := readBucketMetadata(t.Context(), obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
policyJSON := fmt.Appendf(nil, `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket)
|
||||
tagXML := []byte(`<Tagging><TagSet><Tag><Key>local</Key><Value>tag</Value></Tag></TagSet></Tagging>`)
|
||||
runBucketMetadataRMWConflict(t, obj, bucket,
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
return globalSiteReplicationSys.PeerBucketMetadataUpdateHandler(ctx, madmin.SRBucketMeta{
|
||||
Bucket: bucket, Policy: policyJSON, UpdatedAt: meta.Created.Add(time.Second),
|
||||
})
|
||||
},
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, tagXML)
|
||||
return err
|
||||
},
|
||||
func(meta BucketMetadata) bool {
|
||||
return bytes.Equal(meta.PolicyConfigJSON, policyJSON) && bytes.Equal(meta.TaggingConfigXML, tagXML)
|
||||
}, instanceType+": peer bulk+local tagging")
|
||||
}
|
||||
|
||||
func TestBucketMetadataLockPreservesLifecycleDeleteAndSSE(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketMetadataLockPreservesLifecycleDeleteAndSSE,
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketMetadataLockPreservesLifecycleDeleteAndSSE(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
lifecycleXML := []byte(`<LifecycleConfiguration><Rule><ID>expire</ID><Filter><Prefix>logs/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>`)
|
||||
if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketLifecycleConfig, lifecycleXML); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sseXML := []byte(`<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`)
|
||||
runBucketMetadataRMWConflict(t, obj, bucket,
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketLifecycleConfig)
|
||||
return err
|
||||
},
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, sseXML)
|
||||
return err
|
||||
},
|
||||
func(meta BucketMetadata) bool {
|
||||
cfg, err := lifecycle.ParseLifecycleConfig(bytes.NewReader(meta.LifecycleConfigXML))
|
||||
return err == nil && cfg.ExpiryUpdatedAt != nil && len(cfg.Rules) == 0 && bytes.Equal(meta.EncryptionConfigXML, sseXML)
|
||||
}, instanceType+": lifecycle delete+SSE")
|
||||
}
|
||||
|
||||
func TestMakeBucketForceCreatePreservesMetadata(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testMakeBucketForceCreatePreservesMetadata,
|
||||
})
|
||||
}
|
||||
|
||||
func testMakeBucketForceCreatePreservesMetadata(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
ctx := t.Context()
|
||||
policyJSON := fmt.Appendf(nil, `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket)
|
||||
corsXML := []byte(testSiteReplicationCORSDoc)
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, policyJSON); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := updateLocalBucketCORSMetadata(ctx, obj, bucket, corsXML); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, err := readBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{ForceCreate: true}); err != nil {
|
||||
t.Fatalf("%s: ForceCreate existing bucket: %v", instanceType, err)
|
||||
}
|
||||
after, err := readBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !after.Created.Equal(before.Created) || !bytes.Equal(after.PolicyConfigJSON, policyJSON) || !bytes.Equal(after.CorsConfigXML, corsXML) {
|
||||
t.Fatalf("%s: ForceCreate replaced metadata: before=%+v after=%+v", instanceType, before, after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyImportedBucketMetadataPreservesUnspecifiedFields(t *testing.T) {
|
||||
policyJSON := []byte(`{"Version":"2012-10-17","Statement":[]}`)
|
||||
tagXML := []byte(`<Tagging><TagSet><Tag><Key>existing</Key><Value>tag</Value></Tag></TagSet></Tagging>`)
|
||||
src := newBucketMetadata("bucket")
|
||||
src.PolicyConfigJSON = policyJSON
|
||||
src.PolicyConfigUpdatedAt = UTCNow()
|
||||
dst := newBucketMetadata("bucket")
|
||||
dst.TaggingConfigXML = bytes.Clone(tagXML)
|
||||
|
||||
applyImportedBucketMetadata(&dst, src, importMetadataFields{bucketPolicyConfig: {}})
|
||||
if !bytes.Equal(dst.PolicyConfigJSON, policyJSON) || !bytes.Equal(dst.TaggingConfigXML, tagXML) {
|
||||
t.Fatalf("import patch overwrote unspecified metadata: %+v", dst)
|
||||
}
|
||||
src.PolicyConfigJSON[0] = '!'
|
||||
if dst.PolicyConfigJSON[0] == '!' {
|
||||
t.Fatal("import patch retained the source byte slice")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMakeBucketDoesNotAdoptGhostMetadata(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testMakeBucketDoesNotAdoptGhostMetadata,
|
||||
})
|
||||
}
|
||||
|
||||
func testMakeBucketDoesNotAdoptGhostMetadata(obj ObjectLayer, instanceType, _ string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
ctx := t.Context()
|
||||
bucket := getRandomBucketName()
|
||||
if err := obj.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
policyJSON := fmt.Appendf(nil, `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket)
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, policyJSON); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldMeta, err := readBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z, ok := obj.(*erasureServerPools)
|
||||
if !ok {
|
||||
t.Fatalf("%s: object layer is %T, want *erasureServerPools", instanceType, obj)
|
||||
}
|
||||
if err = z.s3Peer.DeleteBucket(ctx, bucket, DeleteBucketOptions{Force: true}); err != nil {
|
||||
t.Fatalf("%s: delete bucket volume only: %v", instanceType, err)
|
||||
}
|
||||
if err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatalf("%s: recreate bucket: %v", instanceType, err)
|
||||
}
|
||||
newMeta, err := readBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bytes.Equal(newMeta.PolicyConfigJSON, policyJSON) || newMeta.Created.Equal(oldMeta.Created) {
|
||||
t.Fatalf("%s: new bucket adopted ghost metadata: old=%+v new=%+v", instanceType, oldMeta, newMeta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMakeBucketForceCreateLockEnablesVersioning(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testMakeBucketForceCreateLockEnablesVersioning,
|
||||
})
|
||||
}
|
||||
|
||||
func TestPeerBucketMetadataSaveSurvivesCallerCancellation(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testPeerBucketMetadataSaveSurvivesCallerCancellation,
|
||||
})
|
||||
}
|
||||
|
||||
func testPeerBucketMetadataSaveSurvivesCallerCancellation(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
previousObjectAPI := newObjectLayerFn()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
barrier := &metadataRMWBarrierObjectLayer{
|
||||
ObjectLayer: obj,
|
||||
bucket: bucket,
|
||||
cancelOnPut: cancel,
|
||||
}
|
||||
setObjectLayer(barrier)
|
||||
defer setObjectLayer(previousObjectAPI)
|
||||
|
||||
err := globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(ctx, bucket, MakeBucketOptions{VersioningEnabled: true})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: peer metadata save failed after caller cancellation: %v", instanceType, err)
|
||||
}
|
||||
if ctx.Err() != context.Canceled {
|
||||
t.Fatalf("%s: metadata write did not trigger caller cancellation", instanceType)
|
||||
}
|
||||
meta, err := readBucketMetadata(t.Context(), obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := versioning.ParseConfig(bytes.NewReader(meta.VersioningConfigXML))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !cfg.Enabled() {
|
||||
t.Fatalf("%s: peer metadata save lost versioning after cancellation", instanceType)
|
||||
}
|
||||
}
|
||||
|
||||
func testMakeBucketForceCreateLockEnablesVersioning(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
ctx := t.Context()
|
||||
suspended := []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Suspended</Status></VersioningConfiguration>`)
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, suspended); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := obj.MakeBucket(ctx, bucket, MakeBucketOptions{ForceCreate: true, LockEnabled: true}); err != nil {
|
||||
t.Fatalf("%s: ForceCreate with object lock: %v", instanceType, err)
|
||||
}
|
||||
meta, err := readBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := versioning.ParseConfig(bytes.NewReader(meta.VersioningConfigXML))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !cfg.Enabled() || len(meta.ObjectLockConfigXML) == 0 {
|
||||
t.Fatalf("%s: object lock state lacks enabled versioning: metadata=%+v", instanceType, meta)
|
||||
}
|
||||
}
|
||||
|
||||
func testBucketMetadataLockPreservesTaggingAndSSE(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
tagXML := []byte(`<Tagging><TagSet><Tag><Key>key</Key><Value>value</Value></Tag></TagSet></Tagging>`)
|
||||
sseXML := []byte(`<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`)
|
||||
runBucketMetadataRMWConflict(t, obj, bucket,
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, tagXML)
|
||||
return err
|
||||
},
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, sseXML)
|
||||
return err
|
||||
},
|
||||
func(meta BucketMetadata) bool {
|
||||
return bytes.Equal(meta.TaggingConfigXML, tagXML) && bytes.Equal(meta.EncryptionConfigXML, sseXML)
|
||||
}, instanceType+": tagging+SSE")
|
||||
}
|
||||
|
||||
func runBucketMetadataRMWConflict(t *testing.T, obj ObjectLayer, bucket string,
|
||||
writerA, writerB func(context.Context, ObjectLayer) error,
|
||||
complete func(BucketMetadata) bool, name string,
|
||||
) {
|
||||
t.Helper()
|
||||
previousObjectAPI := newObjectLayerFn()
|
||||
barrier := &metadataRMWBarrierObjectLayer{
|
||||
ObjectLayer: obj,
|
||||
bucket: bucket,
|
||||
aReady: make(chan struct{}),
|
||||
aRelease: make(chan struct{}),
|
||||
bLockAttempt: make(chan struct{}),
|
||||
}
|
||||
setObjectLayer(barrier)
|
||||
defer setObjectLayer(previousObjectAPI)
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
aCtx := context.WithValue(ctx, metadataRMWWriterKey{}, "A")
|
||||
bCtx := context.WithValue(ctx, metadataRMWWriterKey{}, "B")
|
||||
aDone := make(chan error, 1)
|
||||
bDone := make(chan error, 1)
|
||||
go func() { aDone <- writerA(aCtx, barrier) }()
|
||||
|
||||
select {
|
||||
case <-barrier.aReady:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("%s: writer A did not reach metadata save: %v", name, ctx.Err())
|
||||
}
|
||||
go func() { bDone <- writerB(bCtx, barrier) }()
|
||||
|
||||
var (
|
||||
bErr error
|
||||
bFinished bool
|
||||
)
|
||||
select {
|
||||
case <-barrier.bLockAttempt:
|
||||
if got := barrier.reads.Load(); got != 1 {
|
||||
t.Fatalf("%s: writer B read metadata before acquiring metadata.lock: reads=%d", name, got)
|
||||
}
|
||||
case bErr = <-bDone:
|
||||
bFinished = true
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("%s: writer B neither completed nor attempted metadata.lock: %v", name, ctx.Err())
|
||||
}
|
||||
close(barrier.aRelease)
|
||||
if err := <-aDone; err != nil {
|
||||
t.Fatalf("%s: writer A failed: %v", name, err)
|
||||
}
|
||||
if !bFinished {
|
||||
select {
|
||||
case bErr = <-bDone:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("%s: writer B did not finish: %v", name, ctx.Err())
|
||||
}
|
||||
}
|
||||
if bErr != nil {
|
||||
t.Fatalf("%s: writer B failed: %v", name, bErr)
|
||||
}
|
||||
|
||||
disk, err := readBucketMetadata(ctx, barrier, bucket)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: read disk metadata: %v", name, err)
|
||||
}
|
||||
resident, err := globalBucketMetadataSys.Get(bucket)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: read resident metadata: %v", name, err)
|
||||
}
|
||||
if !complete(disk) || !complete(resident) {
|
||||
t.Fatalf("%s: concurrent updates lost a field: disk=%+v resident=%+v", name, disk, resident)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
// Copyright 2026 PGSTY contributors.
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
)
|
||||
|
||||
func TestDeleteBucketMetadataLockCancellation(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testDeleteBucketMetadataLockCancellation})
|
||||
}
|
||||
|
||||
func testDeleteBucketMetadataLockCancellation(obj ObjectLayer, instanceType, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
_, unlock, err := lockBucketMetadata(t.Context(), obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release := sync.OnceFunc(unlock)
|
||||
defer release()
|
||||
|
||||
// Observe DeleteBucket's ACTUAL metadata.lock attempt. Set the hook after
|
||||
// our own acquisition above so it only trips on the delete.
|
||||
delAtLock := make(chan struct{})
|
||||
var once sync.Once
|
||||
hook := func(b string) {
|
||||
if b == bucket {
|
||||
once.Do(func() { close(delAtLock) })
|
||||
}
|
||||
}
|
||||
lockBucketMetadataAcquireHook.Store(&hook)
|
||||
defer lockBucketMetadataAcquireHook.Store(nil)
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- obj.DeleteBucket(ctx, bucket, DeleteBucketOptions{Force: true, NoLock: true}) }()
|
||||
|
||||
select {
|
||||
case <-delAtLock:
|
||||
// Fixed tree: the delete reached metadata.lock and is blocking on the
|
||||
// lock we hold. Cancel it and confirm it fails WITHOUT deleting, while we
|
||||
// still hold the lock (release stays deferred until after the checks).
|
||||
cancel()
|
||||
if err := <-done; err == nil {
|
||||
t.Errorf("%s: canceled deletion succeeded", instanceType)
|
||||
}
|
||||
if _, err := obj.GetBucketInfo(t.Context(), bucket, BucketOptions{}); err != nil {
|
||||
t.Errorf("%s: bucket disappeared while metadata.lock was held: %v", instanceType, err)
|
||||
}
|
||||
if _, err := readBucketMetadata(t.Context(), obj, bucket); err != nil {
|
||||
t.Errorf("%s: canceled deletion removed metadata: %v", instanceType, err)
|
||||
}
|
||||
case err := <-done:
|
||||
// Broken tree: the delete finished without ever taking metadata.lock,
|
||||
// i.e. it did not serialize the destructive operation behind the lock.
|
||||
t.Errorf("%s: delete bypassed metadata.lock (err=%v)", instanceType, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueuedMetadataUpdateAfterDelete(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
for _, expiry := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("expiry=%v", expiry), func(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instanceType, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
previous := newObjectLayerFn()
|
||||
barrier := &lcMergeBarrier{ObjectLayer: obj, bucket: bucket, mAtLock: make(chan struct{}), mProceed: make(chan struct{})}
|
||||
setObjectLayer(barrier)
|
||||
defer setObjectLayer(previous)
|
||||
release := sync.OnceFunc(func() { close(barrier.mProceed) })
|
||||
defer release()
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
ctx = context.WithValue(ctx, lcMergeWriterKey{}, "M")
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
if expiry {
|
||||
done <- globalBucketMetadataSys.UpdateExpiryLCConfig(ctx, bucket, nil, UTCNow())
|
||||
return
|
||||
}
|
||||
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, []byte(`<Tagging><TagSet/></Tagging>`))
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case <-barrier.mAtLock:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("writer did not reach metadata.lock")
|
||||
}
|
||||
if err := obj.DeleteBucket(t.Context(), bucket, DeleteBucketOptions{Force: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release()
|
||||
if err := <-done; !isErrBucketNotFound(err) {
|
||||
t.Errorf("%s: queued update should reject a deleted bucket, got %v", instanceType, err)
|
||||
}
|
||||
if _, err := readBucketMetadata(t.Context(), obj, bucket); !errors.Is(err, errConfigNotFound) && !isErrBucketNotFound(err) {
|
||||
t.Errorf("%s: queued update recreated metadata: %v", instanceType, err)
|
||||
}
|
||||
}})
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
// Copyright (c) 2026 PGSTY
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"net/http"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Target 1 (issue #105): a higher-level lifecycle XML merge racing another
|
||||
// bucket-metadata transition outside BucketMetadataSys.Delete.
|
||||
//
|
||||
// Before the fix, PeerBucketLCConfigHandler / healBucketILMExpiry read the
|
||||
// current lifecycle document without metadata.lock, merged the replicated expiry
|
||||
// rules with the local transition rules, and then persisted the merged blob with
|
||||
// BucketMetadataSys.Update. Update re-read the record under metadata.lock but
|
||||
// overwrote LifecycleConfigXML wholesale with the pre-computed blob, so any
|
||||
// lifecycle transition change committed between the merge read and the merge
|
||||
// write was silently lost on disk. BucketMetadataSys.UpdateExpiryLCConfig now
|
||||
// performs the read, merge, and save under a single metadata.lock. This test
|
||||
// drives PeerBucketLCConfigHandler and asserts the concurrent change survives.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type lcMergeWriterKey struct{}
|
||||
|
||||
// lcMergeBarrier pauses the merge writer (context value "M") exactly when it
|
||||
// tries to take metadata.lock for its persisting Update. By that point the
|
||||
// merge has already read the stale lifecycle document, so the test can commit a
|
||||
// concurrent transition change before releasing the merge write.
|
||||
type lcMergeBarrier struct {
|
||||
ObjectLayer
|
||||
bucket string
|
||||
mAtLock chan struct{}
|
||||
mProceed chan struct{}
|
||||
mOnce sync.Once
|
||||
}
|
||||
|
||||
func (o *lcMergeBarrier) metadataLock() string {
|
||||
return pathJoin(bucketMetaPrefix, o.bucket, "metadata.lock")
|
||||
}
|
||||
|
||||
func (o *lcMergeBarrier) NewNSLock(bucket string, objects ...string) RWLocker {
|
||||
lock := o.ObjectLayer.NewNSLock(bucket, objects...)
|
||||
if bucket != minioMetaBucket || len(objects) != 1 || objects[0] != o.metadataLock() {
|
||||
return lock
|
||||
}
|
||||
return metadataObservedRWLocker{RWLocker: lock, onLock: func(ctx context.Context) {
|
||||
if ctx.Value(lcMergeWriterKey{}) != "M" {
|
||||
return
|
||||
}
|
||||
o.mOnce.Do(func() { close(o.mAtLock) })
|
||||
select {
|
||||
case <-o.mProceed:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}}
|
||||
}
|
||||
|
||||
func TestLifecycleExpiryMergeRaceLosesConcurrentTransition(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testLifecycleExpiryMergeRaceLosesConcurrentTransition,
|
||||
})
|
||||
}
|
||||
|
||||
func testLifecycleExpiryMergeRaceLosesConcurrentTransition(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
// Revision N: a single transition-only rule.
|
||||
baseXML := []byte(`<LifecycleConfiguration><Rule><ID>keep</ID><Filter><Prefix>data/</Prefix></Filter><Status>Enabled</Status><Transition><Days>30</Days><StorageClass>WARM</StorageClass></Transition></Rule></LifecycleConfiguration>`)
|
||||
if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketLifecycleConfig, baseXML); err != nil {
|
||||
t.Fatalf("%s: seed lifecycle: %v", instanceType, err)
|
||||
}
|
||||
|
||||
// A replicated expiry-only rule arriving from a peer site.
|
||||
expXML := `<LifecycleConfiguration><Rule><ID>expire</ID><Filter><Prefix>tmp/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>7</Days></Expiration></Rule></LifecycleConfiguration>`
|
||||
expLCConfig := base64.StdEncoding.EncodeToString([]byte(expXML))
|
||||
|
||||
previousObjectAPI := newObjectLayerFn()
|
||||
barrier := &lcMergeBarrier{
|
||||
ObjectLayer: obj,
|
||||
bucket: bucket,
|
||||
mAtLock: make(chan struct{}),
|
||||
mProceed: make(chan struct{}),
|
||||
}
|
||||
setObjectLayer(barrier)
|
||||
defer setObjectLayer(previousObjectAPI)
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
mCtx := context.WithValue(ctx, lcMergeWriterKey{}, "M")
|
||||
|
||||
mDone := make(chan error, 1)
|
||||
go func() {
|
||||
mDone <- globalSiteReplicationSys.PeerBucketLCConfigHandler(mCtx, bucket, &expLCConfig, UTCNow())
|
||||
}()
|
||||
|
||||
// Wait until the merge writer has read revision N and is about to persist.
|
||||
select {
|
||||
case <-barrier.mAtLock:
|
||||
case err := <-mDone:
|
||||
t.Fatalf("%s: merge writer finished before persisting: %v", instanceType, err)
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("%s: merge writer never reached metadata.lock: %v", instanceType, ctx.Err())
|
||||
}
|
||||
|
||||
// Concurrent local lifecycle transition change commits revision N+1.
|
||||
concurrentXML := []byte(`<LifecycleConfiguration><Rule><ID>keep</ID><Filter><Prefix>data/</Prefix></Filter><Status>Enabled</Status><Transition><Days>10</Days><StorageClass>COLD</StorageClass></Transition></Rule></LifecycleConfiguration>`)
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, concurrentXML); err != nil {
|
||||
t.Fatalf("%s: concurrent transition update: %v", instanceType, err)
|
||||
}
|
||||
|
||||
// Release the merge write so it lands after the concurrent commit.
|
||||
close(barrier.mProceed)
|
||||
if err := <-mDone; err != nil {
|
||||
t.Fatalf("%s: merge writer failed: %v", instanceType, err)
|
||||
}
|
||||
|
||||
// The persisted lifecycle must contain both the replicated expiry rule and
|
||||
// the concurrent transition change.
|
||||
cfg, _, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: read merged lifecycle: %v", instanceType, err)
|
||||
}
|
||||
var keep, expire bool
|
||||
for i := range cfg.Rules {
|
||||
switch cfg.Rules[i].ID {
|
||||
case "keep":
|
||||
keep = true
|
||||
if cfg.Rules[i].Transition.Days != 10 || cfg.Rules[i].Transition.StorageClass != "COLD" {
|
||||
t.Fatalf("%s: lifecycle merge overwrote the concurrent transition change: got Days=%d StorageClass=%q, want Days=10 StorageClass=COLD",
|
||||
instanceType, cfg.Rules[i].Transition.Days, cfg.Rules[i].Transition.StorageClass)
|
||||
}
|
||||
case "expire":
|
||||
expire = true
|
||||
}
|
||||
}
|
||||
if !expire {
|
||||
t.Fatalf("%s: merged lifecycle dropped the replicated expiry rule: %+v", instanceType, cfg.Rules)
|
||||
}
|
||||
if !keep {
|
||||
t.Fatalf("%s: merged lifecycle dropped the transition rule entirely: %+v", instanceType, cfg.Rules)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Target 2 (issue #105): DeleteBucket racing an in-flight metadata writer must
|
||||
// not resurrect a ghost .metadata.bin record.
|
||||
//
|
||||
// Before the fix, erasureServerPools.DeleteBucket took only <bucket>.lck and
|
||||
// purged the metadata prefix while config writers (updateAndParse) took only
|
||||
// metadata.lock, so a writer already MID-SAVE (holding metadata.lock, past
|
||||
// saveMetadata's existence recheck) could persist .metadata.bin after the purge.
|
||||
// DeleteBucket now takes metadata.lock before deleting, so it waits for that
|
||||
// writer and then purges whatever the writer wrote.
|
||||
//
|
||||
// This test isolates the DeleteBucket-lock fix specifically: the writer holds
|
||||
// metadata.lock and is paused at the .metadata.bin PutObject, so the earlier
|
||||
// saveMetadata existence recheck cannot save it — only serializing the delete
|
||||
// behind the writer can. Removing just DeleteBucket's metadata.lock (keeping the
|
||||
// recheck) therefore makes this test fail. The delete's ACTUAL metadata.lock
|
||||
// attempt is observed with lockBucketMetadataAcquireHook (its lock is taken
|
||||
// through the erasureServerPools receiver, invisible to the object-layer
|
||||
// barrier), so the handshake is deterministic with no timing assumption.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestDeleteBucketResurrectsGhostMetadata(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testDeleteBucketResurrectsGhostMetadata,
|
||||
})
|
||||
}
|
||||
|
||||
func testDeleteBucketResurrectsGhostMetadata(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
previousObjectAPI := newObjectLayerFn()
|
||||
// Writer A holds metadata.lock and pauses at the .metadata.bin PutObject,
|
||||
// i.e. already past saveMetadata's existence recheck and mid-save.
|
||||
barrier := &metadataRMWBarrierObjectLayer{
|
||||
ObjectLayer: obj,
|
||||
bucket: bucket,
|
||||
aReady: make(chan struct{}),
|
||||
aRelease: make(chan struct{}),
|
||||
bLockAttempt: make(chan struct{}),
|
||||
}
|
||||
setObjectLayer(barrier)
|
||||
defer setObjectLayer(previousObjectAPI)
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
aCtx := context.WithValue(ctx, metadataRMWWriterKey{}, "A")
|
||||
|
||||
policyJSON := []byte(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::` + bucket + `/*"}]}`)
|
||||
aReleased := sync.OnceFunc(func() { close(barrier.aRelease) })
|
||||
defer aReleased()
|
||||
aDone := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := globalBucketMetadataSys.Update(aCtx, bucket, bucketPolicyConfig, policyJSON)
|
||||
aDone <- err
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-barrier.aReady:
|
||||
case err := <-aDone:
|
||||
t.Fatalf("%s: writer A finished before persisting: %v", instanceType, err)
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("%s: writer A never reached metadata save: %v", instanceType, ctx.Err())
|
||||
}
|
||||
|
||||
// A now holds metadata.lock mid-save. Observe DeleteBucket's ACTUAL
|
||||
// metadata.lock attempt via the acquire hook, set only now so A's earlier
|
||||
// acquisition does not trip it.
|
||||
delAtLock := make(chan struct{})
|
||||
var once sync.Once
|
||||
hook := func(b string) {
|
||||
if b == bucket {
|
||||
once.Do(func() { close(delAtLock) })
|
||||
}
|
||||
}
|
||||
lockBucketMetadataAcquireHook.Store(&hook)
|
||||
defer lockBucketMetadataAcquireHook.Store(nil)
|
||||
|
||||
delDone := make(chan error, 1)
|
||||
go func() {
|
||||
delDone <- obj.DeleteBucket(ctx, bucket, DeleteBucketOptions{Force: true})
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-delAtLock:
|
||||
// Fixed tree: DeleteBucket reached metadata.lock and blocks on A. Release
|
||||
// A so it finishes its save and unlocks; the delete then acquires the
|
||||
// lock and purges the record A wrote.
|
||||
aReleased()
|
||||
if err := <-aDone; err != nil {
|
||||
t.Fatalf("%s: writer A save failed while holding metadata.lock: %v", instanceType, err)
|
||||
}
|
||||
if err := <-delDone; err != nil {
|
||||
t.Fatalf("%s: delete bucket: %v", instanceType, err)
|
||||
}
|
||||
case err := <-delDone:
|
||||
// Broken tree: DeleteBucket purged without taking metadata.lock. Release
|
||||
// A so its mid-save PutObject recreates .metadata.bin (the ghost).
|
||||
if err != nil {
|
||||
t.Fatalf("%s: delete bucket: %v", instanceType, err)
|
||||
}
|
||||
aReleased()
|
||||
if err := <-aDone; err != nil {
|
||||
t.Fatalf("%s: writer A save failed: %v", instanceType, err)
|
||||
}
|
||||
}
|
||||
|
||||
// After DeleteBucket, no .metadata.bin record may remain on disk.
|
||||
if meta, err := readBucketMetadata(ctx, obj, bucket); err == nil {
|
||||
t.Fatalf("%s: ghost .metadata.bin resurrected after DeleteBucket: name=%q created=%s policyLen=%d",
|
||||
instanceType, meta.Name, meta.Created, len(meta.PolicyConfigJSON))
|
||||
} else if !errors.Is(err, errConfigNotFound) && !isErrBucketNotFound(err) && !errors.Is(err, errVolumeNotFound) {
|
||||
t.Fatalf("%s: unexpected error reading deleted bucket metadata: %v", instanceType, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
// Copyright 2026 PGSTY contributors.
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/grid"
|
||||
)
|
||||
|
||||
func TestPeerMetadataReloadWithEqualMaximumTimestamp(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testPeerMetadataReloadWithEqualMaximumTimestamp})
|
||||
}
|
||||
|
||||
func testPeerMetadataReloadWithEqualMaximumTimestamp(obj ObjectLayer, instanceType, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
disk, err := loadBucketMetadata(t.Context(), obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
disk.TaggingConfigXML = []byte(`<Tagging><TagSet><Tag><Key>revision</Key><Value>new</Value></Tag></TagSet></Tagging>`)
|
||||
disk.TaggingConfigUpdatedAt = UTCNow()
|
||||
// An unrelated configuration has the greatest timestamp in both records.
|
||||
disk.PolicyConfigUpdatedAt = disk.TaggingConfigUpdatedAt.Add(time.Hour)
|
||||
if err := disk.Save(t.Context(), obj); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resident := disk
|
||||
resident.TaggingConfigXML = []byte(`<Tagging><TagSet><Tag><Key>revision</Key><Value>old</Value></Tag></TagSet></Tagging>`)
|
||||
resident.TaggingConfigUpdatedAt = disk.TaggingConfigUpdatedAt.Add(-time.Minute)
|
||||
if err := resident.parseAllConfigs(t.Context(), obj); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !resident.lastUpdate().Equal(disk.lastUpdate()) {
|
||||
t.Fatal("fixture must have equal maximum timestamps")
|
||||
}
|
||||
globalBucketMetadataSys.Set(bucket, resident)
|
||||
|
||||
args := grid.MSS{peerRESTBucket: bucket}
|
||||
if _, err := (&peerRESTServer{}).LoadBucketMetadataHandler(&args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tagging, _, err := globalBucketMetadataSys.GetTaggingConfig(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := tagging.ToMap()["revision"]; got != "new" {
|
||||
t.Errorf("%s: peer reload retained tag %q despite a newer tagging configuration", instanceType, got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
bucketsse "github.com/minio/minio/internal/bucket/encryption"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// Only these fields share the site-replication source-time ordering contract.
|
||||
// Bulk apply/import process Object Lock before Versioning, whose effective
|
||||
// document depends on it. Periodic heal retains its existing type order.
|
||||
var replicatedBucketConfigs = [...]string{
|
||||
objectLockConfig, bucketVersioningConfig, bucketPolicyConfig,
|
||||
bucketTaggingConfig, bucketSSEConfig, bucketQuotaConfigFile,
|
||||
}
|
||||
|
||||
func replicatedBucketConfig(meta *BucketMetadata, file string) (*[]byte, *time.Time) {
|
||||
switch file {
|
||||
case bucketPolicyConfig:
|
||||
return &meta.PolicyConfigJSON, &meta.PolicyConfigUpdatedAt
|
||||
case bucketTaggingConfig:
|
||||
return &meta.TaggingConfigXML, &meta.TaggingConfigUpdatedAt
|
||||
case bucketSSEConfig:
|
||||
return &meta.EncryptionConfigXML, &meta.EncryptionConfigUpdatedAt
|
||||
case bucketQuotaConfigFile:
|
||||
return &meta.QuotaConfigJSON, &meta.QuotaConfigUpdatedAt
|
||||
case bucketVersioningConfig:
|
||||
return &meta.VersioningConfigXML, &meta.VersioningConfigUpdatedAt
|
||||
case objectLockConfig:
|
||||
return &meta.ObjectLockConfigXML, &meta.ObjectLockConfigUpdatedAt
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Callers that only need to know whether a file is under the contract must not
|
||||
// probe replicatedBucketConfig with a throwaway BucketMetadata.
|
||||
func isReplicatedBucketConfig(file string) bool {
|
||||
for _, replicated := range replicatedBucketConfigs {
|
||||
if replicated == file {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func bucketConfigUpdateOnly(file string) bool {
|
||||
return file == bucketVersioningConfig || file == objectLockConfig
|
||||
}
|
||||
|
||||
// Reuse the persistence rule before comparison, so an accepted Versioning
|
||||
// event and the document Save actually writes have the same comparison key.
|
||||
func effectiveBucketVersioning(data []byte, lockEnabled bool) []byte {
|
||||
if lockEnabled {
|
||||
config, err := versioning.ParseConfig(bytes.NewReader(data))
|
||||
if err != nil || !config.Enabled() || config.PrefixesExcluded() {
|
||||
return enabledBucketVersioningConfig
|
||||
}
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
// A parsed policy still contains map-backed sets with nondeterministic Marshal
|
||||
// order. Sort every set array recursively, including statements and conditions.
|
||||
// RawMessage keeps integer values intact; decoding through float64 would not.
|
||||
func canonicalBucketPolicyJSON(data json.RawMessage) (json.RawMessage, error) {
|
||||
data = bytes.TrimSpace(data)
|
||||
if len(data) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
switch data[0] {
|
||||
case '{':
|
||||
var obj map[string]json.RawMessage
|
||||
if err := json.Unmarshal(data, &obj); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for key, value := range obj {
|
||||
var err error
|
||||
obj[key], err = canonicalBucketPolicyJSON(value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return json.Marshal(obj)
|
||||
case '[':
|
||||
var arr []json.RawMessage
|
||||
if err := json.Unmarshal(data, &arr); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range arr {
|
||||
var err error
|
||||
arr[i], err = canonicalBucketPolicyJSON(arr[i])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
sort.Slice(arr, func(i, j int) bool { return bytes.Compare(arr[i], arr[j]) < 0 })
|
||||
return json.Marshal(arr)
|
||||
default:
|
||||
var compact bytes.Buffer
|
||||
if err := json.Compact(&compact, data); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return compact.Bytes(), nil
|
||||
}
|
||||
}
|
||||
|
||||
// Encode the validated policy fields explicitly: BPStatement's required
|
||||
// Action/Resource tags otherwise try to marshal empty sets for the supported
|
||||
// NotAction/NotResource alternatives. This stays within the existing schema.
|
||||
func canonicalBucketPolicy(cfg *policy.BucketPolicy) ([]byte, error) {
|
||||
if cfg.IsEmpty() {
|
||||
return nil, nil
|
||||
}
|
||||
doc := map[string]any{"Version": cfg.Version}
|
||||
if cfg.ID != "" {
|
||||
doc["ID"] = cfg.ID
|
||||
}
|
||||
statements := make([]map[string]any, 0, len(cfg.Statements))
|
||||
for _, st := range cfg.Statements {
|
||||
statement := map[string]any{"Effect": st.Effect, "Principal": st.Principal}
|
||||
if st.SID != "" {
|
||||
statement["Sid"] = st.SID
|
||||
}
|
||||
if len(st.Actions) != 0 {
|
||||
statement["Action"] = st.Actions
|
||||
}
|
||||
if len(st.NotActions) != 0 {
|
||||
statement["NotAction"] = st.NotActions
|
||||
}
|
||||
if len(st.Resources) != 0 {
|
||||
statement["Resource"] = st.Resources
|
||||
}
|
||||
if len(st.NotResources) != 0 {
|
||||
statement["NotResource"] = st.NotResources
|
||||
}
|
||||
if len(st.Conditions) != 0 {
|
||||
statement["Condition"] = st.Conditions
|
||||
}
|
||||
statements = append(statements, statement)
|
||||
}
|
||||
doc["Statement"] = statements
|
||||
data, err := json.Marshal(doc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return canonicalBucketPolicyJSON(data)
|
||||
}
|
||||
|
||||
// Validate with the same parsers as Save. Policy's established empty-policy
|
||||
// semantics are deletion; a parsed zero quota is still a live document.
|
||||
func bucketConfigPayload(bucket, file string, data []byte, lockEnabled bool) ([]byte, []byte, error) {
|
||||
if len(data) == 0 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
var err error
|
||||
key := data
|
||||
switch file {
|
||||
case bucketPolicyConfig:
|
||||
var cfg *policy.BucketPolicy
|
||||
cfg, err = policy.ParseBucketPolicyConfig(bytes.NewReader(data), bucket)
|
||||
if err == nil {
|
||||
if cfg.IsEmpty() {
|
||||
return nil, nil, nil
|
||||
}
|
||||
key, err = canonicalBucketPolicy(cfg)
|
||||
}
|
||||
case bucketQuotaConfigFile:
|
||||
cfg, parseErr := parseBucketQuota(bucket, data)
|
||||
err = parseErr
|
||||
if err == nil {
|
||||
key, err = json.Marshal(cfg)
|
||||
}
|
||||
case bucketTaggingConfig:
|
||||
_, err = tags.ParseBucketXML(bytes.NewReader(data))
|
||||
case bucketSSEConfig:
|
||||
_, err = bucketsse.ParseBucketSSEConfig(bytes.NewReader(data))
|
||||
case objectLockConfig:
|
||||
_, err = objectlock.ParseObjectLockConfig(bytes.NewReader(data))
|
||||
case bucketVersioningConfig:
|
||||
data = effectiveBucketVersioning(data, lockEnabled)
|
||||
key = data
|
||||
_, err = versioning.ParseConfig(bytes.NewReader(data))
|
||||
}
|
||||
return data, key, err
|
||||
}
|
||||
|
||||
type bucketConfigState struct {
|
||||
data, key []byte
|
||||
at time.Time
|
||||
real, valid bool
|
||||
}
|
||||
|
||||
func newBucketConfigState(bucket, file string, data []byte, at, created time.Time, lockEnabled bool) (bucketConfigState, error) {
|
||||
data, key, err := bucketConfigPayload(bucket, file, data, lockEnabled)
|
||||
if err != nil {
|
||||
return bucketConfigState{}, err
|
||||
}
|
||||
if at.IsZero() {
|
||||
at = created
|
||||
}
|
||||
valid := !created.IsZero() && !at.Before(created)
|
||||
modified := valid && at.After(created)
|
||||
if bucketConfigUpdateOnly(file) && len(data) == 0 {
|
||||
modified = false
|
||||
}
|
||||
return bucketConfigState{data: data, key: key, at: at, real: modified, valid: valid}, nil
|
||||
}
|
||||
|
||||
func (s bucketConfigState) candidate() bool {
|
||||
return s.valid && (s.real || len(s.data) != 0)
|
||||
}
|
||||
|
||||
func compareBucketConfigStates(a, b bucketConfigState) int {
|
||||
if a.valid != b.valid {
|
||||
if a.valid {
|
||||
return 1
|
||||
}
|
||||
return -1
|
||||
}
|
||||
if a.real != b.real {
|
||||
if a.real {
|
||||
return 1
|
||||
}
|
||||
return -1
|
||||
}
|
||||
if a.real {
|
||||
if n := a.at.Compare(b.at); n != 0 {
|
||||
return n
|
||||
}
|
||||
// At equal source time a real deletion wins, preventing resurrection.
|
||||
if (len(a.data) == 0) != (len(b.data) == 0) {
|
||||
if len(a.data) == 0 {
|
||||
return 1
|
||||
}
|
||||
return -1
|
||||
}
|
||||
}
|
||||
return bytes.Compare(a.key, b.key)
|
||||
}
|
||||
|
||||
func localBucketConfigUpdatedAt(meta BucketMetadata, file string, now time.Time) time.Time {
|
||||
_, at := replicatedBucketConfig(&meta, file)
|
||||
for _, lower := range []time.Time{meta.Created, *at} {
|
||||
if !now.After(lower) {
|
||||
now = lower.Add(time.Nanosecond)
|
||||
}
|
||||
}
|
||||
return now.UTC()
|
||||
}
|
||||
|
||||
func ensureBucketMetadataCreated(ctx context.Context, obj ObjectLayer, meta *BucketMetadata) error {
|
||||
if !meta.Created.IsZero() {
|
||||
return nil
|
||||
}
|
||||
info, err := obj.GetBucketInfo(ctx, meta.Name, BucketOptions{NoMetadata: true})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Created.IsZero() {
|
||||
return errors.New("bucket metadata creation time is unknown")
|
||||
}
|
||||
meta.Created = info.Created.UTC()
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyBucketConfig runs under metadata.lock, on freshly loaded metadata. It
|
||||
// changes only the selected field; the caller persists once after all checks.
|
||||
func applyBucketConfig(meta *BucketMetadata, file string, data []byte, at time.Time) (bool, error) {
|
||||
if bucketConfigUpdateOnly(file) && len(data) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
current, currentAt := replicatedBucketConfig(meta, file)
|
||||
lockEnabled := len(meta.ObjectLockConfigXML) != 0
|
||||
incoming, err := newBucketConfigState(meta.Name, file, data, at, meta.Created, lockEnabled)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !incoming.candidate() {
|
||||
return false, nil
|
||||
}
|
||||
local, err := newBucketConfigState(meta.Name, file, *current, *currentAt, meta.Created, lockEnabled)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if compareBucketConfigStates(incoming, local) <= 0 {
|
||||
return false, nil
|
||||
}
|
||||
*current, *currentAt = bytes.Clone(incoming.data), incoming.at.UTC()
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func rebaseBucketConfigDefaults(meta *BucketMetadata, oldCreated time.Time) {
|
||||
if meta.Created.Equal(oldCreated) {
|
||||
return
|
||||
}
|
||||
// These six fields alone use Created to distinguish a baseline from a
|
||||
// tombstone. Preserve actual source times when adopting an existing bucket.
|
||||
for _, file := range replicatedBucketConfigs {
|
||||
_, at := replicatedBucketConfig(meta, file)
|
||||
if at.IsZero() || at.Equal(oldCreated) {
|
||||
*at = meta.Created
|
||||
}
|
||||
}
|
||||
}
|
||||
+323
-90
@@ -24,11 +24,13 @@ import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/bucket/cors"
|
||||
bucketsse "github.com/minio/minio/internal/bucket/encryption"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
@@ -37,8 +39,8 @@ import (
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
@@ -50,8 +52,27 @@ type BucketMetadataSys struct {
|
||||
initialized bool
|
||||
group *singleflight.Group
|
||||
metadataMap map[string]BucketMetadata
|
||||
// loadFailed records real buckets whose metadata has never been loaded
|
||||
// successfully because the startup load or a refresh failed. They are
|
||||
// absent from metadataMap even though the subsystem is initialized, and
|
||||
// without this bit a resident-only lookup could not tell them apart from a
|
||||
// name that is not a bucket at all. It never holds a resident bucket, is
|
||||
// bounded by the number of failed loads, and is empty in normal operation.
|
||||
loadFailed map[string]struct{}
|
||||
}
|
||||
|
||||
// noteLoadFailure and clearLoadFailure maintain loadFailed; both expect the
|
||||
// caller to hold sys.Lock. A bucket that is resident keeps its last loaded
|
||||
// metadata through a failed refresh, exactly like every other bucket
|
||||
// configuration, so the set only ever holds non-resident buckets.
|
||||
func (sys *BucketMetadataSys) noteLoadFailure(bucket string) {
|
||||
if _, resident := sys.metadataMap[bucket]; !resident {
|
||||
sys.loadFailed[bucket] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
func (sys *BucketMetadataSys) clearLoadFailure(bucket string) { delete(sys.loadFailed, bucket) }
|
||||
|
||||
// Count returns number of bucket metadata map entries.
|
||||
func (sys *BucketMetadataSys) Count() int {
|
||||
sys.RLock()
|
||||
@@ -66,6 +87,7 @@ func (sys *BucketMetadataSys) Remove(buckets ...string) {
|
||||
for _, bucket := range buckets {
|
||||
sys.group.Forget(bucket)
|
||||
delete(sys.metadataMap, bucket)
|
||||
sys.clearLoadFailure(bucket)
|
||||
globalBucketMonitor.DeleteBucket(bucket)
|
||||
}
|
||||
sys.Unlock()
|
||||
@@ -83,6 +105,11 @@ func (sys *BucketMetadataSys) RemoveStaleBuckets(diskBuckets set.StringSet) {
|
||||
delete(sys.metadataMap, bucket)
|
||||
globalBucketMonitor.DeleteBucket(bucket)
|
||||
}
|
||||
for bucket := range sys.loadFailed {
|
||||
if !diskBuckets.Contains(bucket) {
|
||||
sys.clearLoadFailure(bucket)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Set - sets a new metadata in-memory.
|
||||
@@ -94,73 +121,133 @@ func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
|
||||
if !isMinioMetaBucketName(bucket) {
|
||||
sys.Lock()
|
||||
sys.metadataMap[bucket] = meta
|
||||
sys.clearLoadFailure(bucket)
|
||||
sys.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string, configFile string, configData []byte, parse bool) (updatedAt time.Time, err error) {
|
||||
// bucketMetadataUpdate returns the committed snapshot to the caller. meta and
|
||||
// updatedAt hold the saved state only when changed is true. Local writes always
|
||||
// change state, because localBucketConfigUpdatedAt is strictly greater than the
|
||||
// current field time, so their handlers can broadcast meta without rechecking.
|
||||
type bucketMetadataUpdate struct {
|
||||
meta BucketMetadata
|
||||
updatedAt time.Time
|
||||
changed bool
|
||||
}
|
||||
|
||||
func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket, configFile string, configData []byte, parse, lifecycleDelete bool) (time.Time, error) {
|
||||
result, err := sys.updateAndParseMetadata(ctx, bucket, configFile, configData, parse, lifecycleDelete, nil)
|
||||
return result.updatedAt, err
|
||||
}
|
||||
|
||||
func (sys *BucketMetadataSys) updateAndParseMetadata(ctx context.Context, bucket string, configFile string, configData []byte, parse, lifecycleDelete bool, sourceTime *time.Time) (result bucketMetadataUpdate, err error) {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return updatedAt, errServerNotInitialized
|
||||
return result, errServerNotInitialized
|
||||
}
|
||||
|
||||
if isMinioMetaBucketName(bucket) {
|
||||
return updatedAt, errInvalidArgument
|
||||
return result, errInvalidArgument
|
||||
}
|
||||
|
||||
meta, err := loadBucketMetadataParse(ctx, objAPI, bucket, parse)
|
||||
// Load deletions without parsed caches (notably quota), and compare the
|
||||
// six replicated fields against the raw document under the same lock.
|
||||
if isReplicatedBucketConfig(configFile) {
|
||||
parse = false
|
||||
if bucketConfigUpdateOnly(configFile) && len(configData) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
notifyCtx := ctx
|
||||
ctx, unlock, err := lockBucketMetadata(ctx, objAPI, bucket)
|
||||
if err != nil {
|
||||
if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
|
||||
// Only single drive mode needs this fallback.
|
||||
meta = newBucketMetadata(bucket)
|
||||
} else {
|
||||
return updatedAt, err
|
||||
}
|
||||
}
|
||||
updatedAt = UTCNow()
|
||||
switch configFile {
|
||||
case bucketPolicyConfig:
|
||||
meta.PolicyConfigJSON = configData
|
||||
meta.PolicyConfigUpdatedAt = updatedAt
|
||||
case bucketNotificationConfig:
|
||||
meta.NotificationConfigXML = configData
|
||||
meta.NotificationConfigUpdatedAt = updatedAt
|
||||
case bucketLifecycleConfig:
|
||||
meta.LifecycleConfigXML = configData
|
||||
meta.LifecycleConfigUpdatedAt = updatedAt
|
||||
case bucketSSEConfig:
|
||||
meta.EncryptionConfigXML = configData
|
||||
meta.EncryptionConfigUpdatedAt = updatedAt
|
||||
case bucketTaggingConfig:
|
||||
meta.TaggingConfigXML = configData
|
||||
meta.TaggingConfigUpdatedAt = updatedAt
|
||||
case bucketQuotaConfigFile:
|
||||
meta.QuotaConfigJSON = configData
|
||||
meta.QuotaConfigUpdatedAt = updatedAt
|
||||
case objectLockConfig:
|
||||
meta.ObjectLockConfigXML = configData
|
||||
meta.ObjectLockConfigUpdatedAt = updatedAt
|
||||
case bucketVersioningConfig:
|
||||
meta.VersioningConfigXML = configData
|
||||
meta.VersioningConfigUpdatedAt = updatedAt
|
||||
case bucketReplicationConfig:
|
||||
meta.ReplicationConfigXML = configData
|
||||
meta.ReplicationConfigUpdatedAt = updatedAt
|
||||
case bucketTargetsFile:
|
||||
meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(ctx, meta.Name, configData, kms.Context{
|
||||
bucket: meta.Name,
|
||||
bucketTargetsFile: bucketTargetsFile,
|
||||
})
|
||||
if err != nil {
|
||||
return updatedAt, fmt.Errorf("Error encrypting bucket target metadata %w", err)
|
||||
}
|
||||
meta.BucketTargetsConfigUpdatedAt = updatedAt
|
||||
meta.BucketTargetsConfigMetaUpdatedAt = updatedAt
|
||||
default:
|
||||
return updatedAt, fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
|
||||
return result, err
|
||||
}
|
||||
|
||||
return updatedAt, sys.save(ctx, meta)
|
||||
err = func() error {
|
||||
defer unlock()
|
||||
meta, err := loadBucketMetadataParse(ctx, objAPI, bucket, parse)
|
||||
if err != nil {
|
||||
if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
|
||||
// Only single drive mode needs this fallback.
|
||||
meta = newBucketMetadata(bucket)
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if lifecycleDelete {
|
||||
configData, err = lifecycleDeleteConfig(meta.LifecycleConfigXML)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
updatedAt := UTCNow()
|
||||
if isReplicatedBucketConfig(configFile) {
|
||||
if err := ensureBucketMetadataCreated(ctx, objAPI, &meta); err != nil {
|
||||
var at time.Time
|
||||
if sourceTime != nil {
|
||||
at = *sourceTime
|
||||
}
|
||||
logBucketConfigReplication(ctx, bucket, configFile, "indeterminate", at, meta.Created, err.Error())
|
||||
return err
|
||||
}
|
||||
if sourceTime == nil || sourceTime.IsZero() {
|
||||
updatedAt = localBucketConfigUpdatedAt(meta, configFile, updatedAt)
|
||||
if sourceTime != nil {
|
||||
logBucketConfigReplication(ctx, bucket, configFile, "legacy-zero", *sourceTime, meta.Created, "assigned local source time")
|
||||
}
|
||||
} else {
|
||||
updatedAt = sourceTime.UTC()
|
||||
}
|
||||
if updatedAt.Before(meta.Created) {
|
||||
logBucketConfigReplication(ctx, bucket, configFile, "before-created", updatedAt, meta.Created, "peer event")
|
||||
return nil
|
||||
}
|
||||
changed, err := applyBucketConfig(&meta, configFile, configData, updatedAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !changed {
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
switch configFile {
|
||||
case bucketNotificationConfig:
|
||||
meta.NotificationConfigXML = configData
|
||||
meta.NotificationConfigUpdatedAt = updatedAt
|
||||
case bucketLifecycleConfig:
|
||||
meta.LifecycleConfigXML = configData
|
||||
meta.LifecycleConfigUpdatedAt = updatedAt
|
||||
case bucketReplicationConfig:
|
||||
meta.ReplicationConfigXML = configData
|
||||
meta.ReplicationConfigUpdatedAt = updatedAt
|
||||
case bucketTargetsFile:
|
||||
meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(ctx, meta.Name, configData, kms.Context{
|
||||
bucket: meta.Name,
|
||||
bucketTargetsFile: bucketTargetsFile,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error encrypting bucket target metadata %w", err)
|
||||
}
|
||||
meta.BucketTargetsConfigUpdatedAt = updatedAt
|
||||
meta.BucketTargetsConfigMetaUpdatedAt = updatedAt
|
||||
default:
|
||||
return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
|
||||
}
|
||||
}
|
||||
if err := sys.saveMetadata(ctx, objAPI, &meta); err != nil {
|
||||
return err
|
||||
}
|
||||
result = bucketMetadataUpdate{meta: meta, updatedAt: updatedAt, changed: true}
|
||||
return nil
|
||||
}()
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
if result.changed {
|
||||
globalNotificationSys.LoadBucketMetadata(bgContext(notifyCtx), bucket)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) error {
|
||||
@@ -173,59 +260,145 @@ func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) err
|
||||
return errInvalidArgument
|
||||
}
|
||||
|
||||
if err := meta.Save(ctx, objAPI); err != nil {
|
||||
if err := sys.saveMetadata(ctx, objAPI, &meta); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sys.Set(meta.Name, meta)
|
||||
globalNotificationSys.LoadBucketMetadata(bgContext(ctx), meta.Name) // Do not use caller context here
|
||||
return nil
|
||||
}
|
||||
|
||||
// saveMetadata persists and publishes metadata locally. Callers performing a
|
||||
// read-modify-write must hold metadata.lock and release it before peer fan-out.
|
||||
func (sys *BucketMetadataSys) saveMetadata(ctx context.Context, objAPI ObjectLayer, meta *BucketMetadata) error {
|
||||
// A writer may have queued for metadata.lock before DeleteBucket completed.
|
||||
// Recheck the physical bucket under that lock, before recreating metadata.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, meta.Name, BucketOptions{NoMetadata: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := meta.Save(ctx, objAPI); err != nil {
|
||||
return err
|
||||
}
|
||||
sys.Set(meta.Name, *meta)
|
||||
return nil
|
||||
}
|
||||
|
||||
func lockBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (context.Context, func(), error) {
|
||||
return lockBucketMetadataWithTimeout(ctx, objectAPI, bucket, globalOperationTimeout)
|
||||
}
|
||||
|
||||
// lockBucketMetadataAcquireHook, when set, is invoked at the start of every
|
||||
// metadata.lock acquisition, immediately before the blocking Lock() call. It is
|
||||
// nil in production (a single atomic load, no behavior change) and exists only
|
||||
// so tests can deterministically observe a caller reaching the metadata lock —
|
||||
// notably DeleteBucket, whose lock is taken through its erasureServerPools
|
||||
// receiver and is therefore invisible to an injected object layer.
|
||||
var lockBucketMetadataAcquireHook atomic.Pointer[func(bucket string)]
|
||||
|
||||
func lockBucketMetadataWithTimeout(ctx context.Context, objectAPI ObjectLayer, bucket string, timeout *dynamicTimeout) (context.Context, func(), error) {
|
||||
lock := objectAPI.NewNSLock(minioMetaBucket, pathJoin(bucketMetaPrefix, bucket, "metadata.lock"))
|
||||
if hook := lockBucketMetadataAcquireHook.Load(); hook != nil {
|
||||
(*hook)(bucket)
|
||||
}
|
||||
lkctx, err := lock.GetLock(ctx, timeout)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
ctx = context.WithValue(lkctx.Context(), bucketMetadataLockContextKey{}, bucket)
|
||||
return ctx, func() { lock.Unlock(lkctx) }, nil
|
||||
}
|
||||
|
||||
type bucketMetadataLockContextKey struct{}
|
||||
|
||||
func bucketMetadataLockHeld(ctx context.Context, bucket string) bool {
|
||||
lockedBucket, _ := ctx.Value(bucketMetadataLockContextKey{}).(string)
|
||||
return lockedBucket == bucket
|
||||
}
|
||||
|
||||
// Delete delete the bucket metadata for the specified bucket.
|
||||
// must be used by all callers instead of using Update() with nil configData.
|
||||
func (sys *BucketMetadataSys) Delete(ctx context.Context, bucket string, configFile string) (updatedAt time.Time, err error) {
|
||||
if configFile == bucketLifecycleConfig {
|
||||
// Get bucket config from current site
|
||||
meta, e := globalBucketMetadataSys.GetConfigFromDisk(ctx, bucket)
|
||||
if e != nil && !errors.Is(e, errConfigNotFound) {
|
||||
return updatedAt, e
|
||||
}
|
||||
var expiryRuleRemoved bool
|
||||
if len(meta.LifecycleConfigXML) > 0 {
|
||||
var lcCfg lifecycle.Lifecycle
|
||||
if err := xml.Unmarshal(meta.LifecycleConfigXML, &lcCfg); err != nil {
|
||||
return updatedAt, err
|
||||
}
|
||||
// find a single expiry rule set the flag
|
||||
for _, rl := range lcCfg.Rules {
|
||||
if !rl.Expiration.IsNull() || !rl.NoncurrentVersionExpiration.IsNull() {
|
||||
expiryRuleRemoved = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return sys.updateAndParse(ctx, bucket, configFile, nil, false, configFile == bucketLifecycleConfig)
|
||||
}
|
||||
|
||||
// Form empty ILM details with `ExpiryUpdatedAt` field and save
|
||||
var cfgData []byte
|
||||
if expiryRuleRemoved {
|
||||
var lcCfg lifecycle.Lifecycle
|
||||
currtime := time.Now()
|
||||
lcCfg.ExpiryUpdatedAt = &currtime
|
||||
cfgData, err = xml.Marshal(lcCfg)
|
||||
if err != nil {
|
||||
return updatedAt, err
|
||||
func lifecycleDeleteConfig(current []byte) ([]byte, error) {
|
||||
var expiryRuleRemoved bool
|
||||
if len(current) > 0 {
|
||||
var lcCfg lifecycle.Lifecycle
|
||||
if err := xml.Unmarshal(current, &lcCfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, rl := range lcCfg.Rules {
|
||||
if !rl.Expiration.IsNull() || !rl.NoncurrentVersionExpiration.IsNull() {
|
||||
expiryRuleRemoved = true
|
||||
break
|
||||
}
|
||||
}
|
||||
return sys.updateAndParse(ctx, bucket, configFile, cfgData, false)
|
||||
}
|
||||
return sys.updateAndParse(ctx, bucket, configFile, nil, false)
|
||||
if !expiryRuleRemoved {
|
||||
return nil, nil
|
||||
}
|
||||
var lcCfg lifecycle.Lifecycle
|
||||
currtime := time.Now()
|
||||
lcCfg.ExpiryUpdatedAt = &currtime
|
||||
return xml.Marshal(lcCfg)
|
||||
}
|
||||
|
||||
// Update update bucket metadata for the specified bucket.
|
||||
// The configData data should not be modified after being sent here.
|
||||
func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) (updatedAt time.Time, err error) {
|
||||
return sys.updateAndParse(ctx, bucket, configFile, configData, true)
|
||||
return sys.updateAndParse(ctx, bucket, configFile, configData, true, false)
|
||||
}
|
||||
|
||||
// UpdateExpiryLCConfig merges a replicated ILM expiry configuration with the
|
||||
// bucket's current lifecycle document and persists the merged result while
|
||||
// holding metadata.lock across the read, merge, and save. The site-replication
|
||||
// expiry heal and peer-apply paths must use this instead of computing the merge
|
||||
// from an unlocked GetConfigFromDisk read and then writing it with Update: that
|
||||
// two-step sequence drops any lifecycle transition change committed in between
|
||||
// (issue #105). Lock order stays <bucket>.lck -> metadata.lock -> .metadata.bin;
|
||||
// the merge and save run under metadata.lock and the peer fan-out runs after it
|
||||
// is released.
|
||||
func (sys *BucketMetadataSys) UpdateExpiryLCConfig(ctx context.Context, bucket string, expLCConfig *string, updatedAt time.Time) error {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
|
||||
if isMinioMetaBucketName(bucket) {
|
||||
return errInvalidArgument
|
||||
}
|
||||
|
||||
notifyCtx := ctx
|
||||
ctx, unlock, err := lockBucketMetadata(ctx, objAPI, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = func() error {
|
||||
defer unlock()
|
||||
meta, err := loadBucketMetadataParse(ctx, objAPI, bucket, true)
|
||||
if err != nil {
|
||||
if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
|
||||
// Only single drive mode needs this fallback.
|
||||
meta = newBucketMetadata(bucket)
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
configData, err := mergeExpiryWithLCConfig(bucket, meta, expLCConfig, updatedAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
meta.LifecycleConfigXML = configData
|
||||
meta.LifecycleConfigUpdatedAt = UTCNow()
|
||||
return sys.saveMetadata(ctx, objAPI, &meta)
|
||||
}()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
globalNotificationSys.LoadBucketMetadata(bgContext(notifyCtx), bucket) // Do not use caller context here
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get metadata for a bucket.
|
||||
@@ -359,6 +532,57 @@ func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEC
|
||||
return meta.sseConfig, meta.EncryptionConfigUpdatedAt, nil
|
||||
}
|
||||
|
||||
// GetResidentCorsConfig returns the CORS configuration of a bucket whose
|
||||
// metadata is already resident in memory. It runs before authentication for
|
||||
// every Origin-bearing request with a client-supplied path segment, so it
|
||||
// never loads or caches metadata. A non-resident name gets no CORS answer
|
||||
// (errBucketMetadataNotInitialized) while startup loading is still running,
|
||||
// and afterwards when it is a real bucket whose metadata failed to load: a
|
||||
// presigned URL is authenticated on its own, so the bucket's CORS document is
|
||||
// the only origin boundary a browser enforces for it. Any other non-resident
|
||||
// name reports errConfigNotFound and the caller applies the global CORS
|
||||
// policy exactly as releases without per-bucket CORS did.
|
||||
func (sys *BucketMetadataSys) GetResidentCorsConfig(bucket string) (*cors.Config, time.Time, error) {
|
||||
if isReservedOrInvalidBucket(bucket, true) {
|
||||
return nil, time.Time{}, errConfigNotFound
|
||||
}
|
||||
sys.RLock()
|
||||
meta, ok := sys.metadataMap[bucket]
|
||||
_, failed := sys.loadFailed[bucket]
|
||||
initialized := sys.initialized
|
||||
sys.RUnlock()
|
||||
if !ok {
|
||||
if !initialized || failed {
|
||||
return nil, time.Time{}, errBucketMetadataNotInitialized
|
||||
}
|
||||
return nil, time.Time{}, errConfigNotFound
|
||||
}
|
||||
if meta.corsConfigErr != nil {
|
||||
return nil, meta.CorsConfigUpdatedAt, meta.corsConfigErr
|
||||
}
|
||||
if meta.corsConfig == nil {
|
||||
return nil, time.Time{}, errConfigNotFound
|
||||
}
|
||||
return meta.corsConfig, meta.CorsConfigUpdatedAt, nil
|
||||
}
|
||||
|
||||
// GetCorsConfigXML returns the raw stored CORS configuration XML for the
|
||||
// given bucket, preserving the document exactly as it was PUT (including
|
||||
// the S3 xmlns and any unmodeled elements).
|
||||
func (sys *BucketMetadataSys) GetCorsConfigXML(bucket string) ([]byte, time.Time, error) {
|
||||
meta, _, err := sys.GetConfig(GlobalContext, bucket)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, err
|
||||
}
|
||||
if meta.corsConfigErr != nil {
|
||||
return nil, meta.CorsConfigUpdatedAt, meta.corsConfigErr
|
||||
}
|
||||
if len(meta.CorsConfigXML) == 0 {
|
||||
return nil, time.Time{}, errConfigNotFound
|
||||
}
|
||||
return meta.CorsConfigXML, meta.CorsConfigUpdatedAt, nil
|
||||
}
|
||||
|
||||
// CreatedAt returns the time of creation of bucket
|
||||
func (sys *BucketMetadataSys) CreatedAt(bucket string) (time.Time, error) {
|
||||
meta, _, err := sys.GetConfig(GlobalContext, bucket)
|
||||
@@ -488,6 +712,7 @@ func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (met
|
||||
}
|
||||
sys.Lock()
|
||||
sys.metadataMap[bucket] = meta
|
||||
sys.clearLoadFailure(bucket)
|
||||
sys.Unlock()
|
||||
|
||||
return meta, true, nil
|
||||
@@ -539,8 +764,10 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []stri
|
||||
sys.Lock()
|
||||
for i, meta := range bucketMetas {
|
||||
if errs[i] != nil {
|
||||
sys.noteLoadFailure(buckets[i])
|
||||
continue
|
||||
}
|
||||
sys.clearLoadFailure(buckets[i])
|
||||
sys.metadataMap[buckets[i]] = meta
|
||||
}
|
||||
sys.Unlock()
|
||||
@@ -590,6 +817,9 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
|
||||
meta, err := loadBucketMetadata(ctx, sys.objAPI, bucket)
|
||||
if err != nil {
|
||||
internalLogIf(ctx, err, logger.WarningKind)
|
||||
sys.Lock()
|
||||
sys.noteLoadFailure(bucket)
|
||||
sys.Unlock()
|
||||
wait() // wait to proceed to next entry.
|
||||
continue
|
||||
}
|
||||
@@ -600,6 +830,7 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
|
||||
updated = true
|
||||
sys.metadataMap[bucket] = meta
|
||||
}
|
||||
sys.clearLoadFailure(bucket)
|
||||
sys.Unlock()
|
||||
|
||||
if updated {
|
||||
@@ -647,6 +878,7 @@ func (sys *BucketMetadataSys) init(ctx context.Context, buckets []string) {
|
||||
func (sys *BucketMetadataSys) Reset() {
|
||||
sys.Lock()
|
||||
clear(sys.metadataMap)
|
||||
clear(sys.loadFailed)
|
||||
sys.Unlock()
|
||||
}
|
||||
|
||||
@@ -654,6 +886,7 @@ func (sys *BucketMetadataSys) Reset() {
|
||||
func NewBucketMetadataSys() *BucketMetadataSys {
|
||||
return &BucketMetadataSys{
|
||||
metadataMap: make(map[string]BucketMetadata),
|
||||
loadFailed: make(map[string]struct{}),
|
||||
group: &singleflight.Group{},
|
||||
}
|
||||
}
|
||||
|
||||
+80
-6
@@ -31,6 +31,7 @@ import (
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/bucket/cors"
|
||||
bucketsse "github.com/minio/minio/internal/bucket/encryption"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
@@ -40,8 +41,8 @@ import (
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/sio"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -58,6 +59,9 @@ var (
|
||||
enabledBucketVersioningConfig = []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status></VersioningConfiguration>`)
|
||||
)
|
||||
|
||||
// Bucket CORS configuration file.
|
||||
const bucketCorsConfig = "cors.xml"
|
||||
|
||||
//go:generate msgp -file $GOFILE
|
||||
|
||||
// BucketMetadata contains bucket metadata.
|
||||
@@ -80,6 +84,7 @@ type BucketMetadata struct {
|
||||
ReplicationConfigXML []byte
|
||||
BucketTargetsConfigJSON []byte
|
||||
BucketTargetsConfigMetaJSON []byte
|
||||
CorsConfigXML []byte
|
||||
|
||||
PolicyConfigUpdatedAt time.Time
|
||||
ObjectLockConfigUpdatedAt time.Time
|
||||
@@ -92,6 +97,7 @@ type BucketMetadata struct {
|
||||
NotificationConfigUpdatedAt time.Time
|
||||
BucketTargetsConfigUpdatedAt time.Time
|
||||
BucketTargetsConfigMetaUpdatedAt time.Time
|
||||
CorsConfigUpdatedAt time.Time
|
||||
// Add a new UpdatedAt field and update lastUpdate function
|
||||
|
||||
// Unexported fields. Must be updated atomically.
|
||||
@@ -106,6 +112,8 @@ type BucketMetadata struct {
|
||||
replicationConfig *replication.Config
|
||||
bucketTargetConfig *madmin.BucketTargets
|
||||
bucketTargetConfigMeta map[string]string
|
||||
corsConfig *cors.Config
|
||||
corsConfigErr error
|
||||
}
|
||||
|
||||
// newBucketMetadata creates BucketMetadata with the supplied name and Created to Now.
|
||||
@@ -160,6 +168,9 @@ func (b BucketMetadata) lastUpdate() (t time.Time) {
|
||||
if b.BucketTargetsConfigMetaUpdatedAt.After(t) {
|
||||
t = b.BucketTargetsConfigMetaUpdatedAt
|
||||
}
|
||||
if b.CorsConfigUpdatedAt.After(t) {
|
||||
t = b.CorsConfigUpdatedAt
|
||||
}
|
||||
|
||||
return t
|
||||
}
|
||||
@@ -238,8 +249,17 @@ func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket
|
||||
}
|
||||
|
||||
if len(configs) > 0 {
|
||||
// Old bucket without bucket metadata. Hence we migrate existing settings.
|
||||
if err = b.convertLegacyConfigs(ctx, objectAPI, configs); err != nil {
|
||||
if !bucketMetadataLockHeld(ctx, bucket) {
|
||||
migrated, lockErr := loadBucketMetadataParseUnderLock(ctx, objectAPI, bucket, parse)
|
||||
if lockErr == nil {
|
||||
return migrated, nil
|
||||
}
|
||||
if !errors.Is(lockErr, errBucketMetadataMigrationLockUnavailable) {
|
||||
return b, lockErr
|
||||
}
|
||||
internalLogOnceIf(ctx, fmt.Errorf("unable to persist bucket metadata migration for %s, using the legacy configuration in memory: %w", bucket, lockErr), "bucket-metadata-migration-lock-"+bucket)
|
||||
b.applyLegacyConfigs(configs)
|
||||
} else if err = b.convertLegacyConfigs(ctx, objectAPI, configs); err != nil {
|
||||
return b, err
|
||||
}
|
||||
}
|
||||
@@ -251,8 +271,25 @@ func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket
|
||||
return b, err
|
||||
}
|
||||
}
|
||||
if b.corsConfigErr != nil {
|
||||
// Keep the rest of the bucket metadata available so an operator can
|
||||
// replace or delete a CORS document accepted by an older, more lenient
|
||||
// build. Defer unrelated metadata migration until CORS is repaired.
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// migrate unencrypted remote targets
|
||||
if len(b.BucketTargetsConfigJSON) != 0 && GlobalKMS != nil && len(b.BucketTargetsConfigMetaJSON) == 0 && !bucketMetadataLockHeld(ctx, bucket) {
|
||||
migrated, lockErr := loadBucketMetadataParseUnderLock(ctx, objectAPI, bucket, parse)
|
||||
if lockErr == nil {
|
||||
return migrated, nil
|
||||
}
|
||||
if !errors.Is(lockErr, errBucketMetadataMigrationLockUnavailable) {
|
||||
return b, lockErr
|
||||
}
|
||||
internalLogOnceIf(ctx, fmt.Errorf("unable to persist encrypted bucket target metadata for %s, using the existing configuration in memory: %w", bucket, lockErr), "bucket-metadata-migration-lock-"+bucket)
|
||||
return b, nil
|
||||
}
|
||||
if err = b.migrateTargetConfig(ctx, objectAPI); err != nil {
|
||||
return b, err
|
||||
}
|
||||
@@ -260,6 +297,23 @@ func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func loadBucketMetadataParseUnderLock(ctx context.Context, objectAPI ObjectLayer, bucket string, parse bool) (BucketMetadata, error) {
|
||||
ctx, unlock, err := lockBucketMetadataWithTimeout(ctx, objectAPI, bucket, bucketMetadataMigrationTimeout)
|
||||
if err != nil {
|
||||
return newBucketMetadata(bucket), fmt.Errorf("%w: %v", errBucketMetadataMigrationLockUnavailable, err)
|
||||
}
|
||||
defer unlock()
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{NoMetadata: true}); err != nil {
|
||||
return newBucketMetadata(bucket), err
|
||||
}
|
||||
return loadBucketMetadataParse(ctx, objectAPI, bucket, parse)
|
||||
}
|
||||
|
||||
var (
|
||||
bucketMetadataMigrationTimeout = newDynamicTimeout(5*time.Second, time.Second)
|
||||
errBucketMetadataMigrationLockUnavailable = errors.New("bucket metadata migration lock unavailable")
|
||||
)
|
||||
|
||||
// loadBucketMetadata loads and migrates to bucket metadata.
|
||||
func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
|
||||
return loadBucketMetadataParse(ctx, objectAPI, bucket, true)
|
||||
@@ -310,8 +364,20 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
|
||||
b.taggingConfig = nil
|
||||
}
|
||||
|
||||
if bytes.Equal(b.ObjectLockConfigXML, enabledBucketObjectLockConfig) {
|
||||
b.VersioningConfigXML = enabledBucketVersioningConfig
|
||||
b.corsConfigErr = nil
|
||||
if len(b.CorsConfigXML) != 0 {
|
||||
cfg, corsErr := cors.ParseBucketCorsConfig(bytes.NewReader(b.CorsConfigXML))
|
||||
if corsErr == nil {
|
||||
corsErr = cfg.Validate()
|
||||
}
|
||||
if corsErr != nil {
|
||||
b.corsConfig = nil
|
||||
b.corsConfigErr = fmt.Errorf("invalid bucket CORS configuration: %w", corsErr)
|
||||
} else {
|
||||
b.corsConfig = cfg
|
||||
}
|
||||
} else {
|
||||
b.corsConfig = nil
|
||||
}
|
||||
|
||||
if len(b.ObjectLockConfigXML) != 0 {
|
||||
@@ -322,6 +388,7 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
|
||||
} else {
|
||||
b.objectLockConfig = nil
|
||||
}
|
||||
b.VersioningConfigXML = effectiveBucketVersioning(b.VersioningConfigXML, b.objectLockConfig != nil)
|
||||
|
||||
if len(b.VersioningConfigXML) != 0 {
|
||||
b.versioningConfig, err = versioning.ParseConfig(bytes.NewReader(b.VersioningConfigXML))
|
||||
@@ -404,7 +471,7 @@ func (b *BucketMetadata) getAllLegacyConfigs(ctx context.Context, objectAPI Obje
|
||||
return configs, nil
|
||||
}
|
||||
|
||||
func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer, configs map[string][]byte) error {
|
||||
func (b *BucketMetadata) applyLegacyConfigs(configs map[string][]byte) {
|
||||
for legacyFile, configData := range configs {
|
||||
switch legacyFile {
|
||||
case legacyBucketObjectLockEnabledConfigFile:
|
||||
@@ -436,6 +503,10 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
|
||||
}
|
||||
}
|
||||
b.defaultTimestamps()
|
||||
}
|
||||
|
||||
func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer, configs map[string][]byte) error {
|
||||
b.applyLegacyConfigs(configs)
|
||||
|
||||
if err := b.Save(ctx, objectAPI); err != nil {
|
||||
return err
|
||||
@@ -503,6 +574,9 @@ func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
|
||||
if err := b.parseAllConfigs(ctx, api); err != nil {
|
||||
return err
|
||||
}
|
||||
if b.corsConfigErr != nil {
|
||||
return b.corsConfigErr
|
||||
}
|
||||
|
||||
data := make([]byte, 4, b.Msgsize()+4)
|
||||
|
||||
|
||||
@@ -108,6 +108,12 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
|
||||
return
|
||||
}
|
||||
case "CorsConfigXML":
|
||||
z.CorsConfigXML, err = dc.ReadBytes(z.CorsConfigXML)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CorsConfigXML")
|
||||
return
|
||||
}
|
||||
case "PolicyConfigUpdatedAt":
|
||||
z.PolicyConfigUpdatedAt, err = dc.ReadTime()
|
||||
if err != nil {
|
||||
@@ -174,6 +180,12 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaUpdatedAt")
|
||||
return
|
||||
}
|
||||
case "CorsConfigUpdatedAt":
|
||||
z.CorsConfigUpdatedAt, err = dc.ReadTime()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CorsConfigUpdatedAt")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
@@ -187,9 +199,9 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// map header, size 25
|
||||
// map header, size 27
|
||||
// write "Name"
|
||||
err = en.Append(0xde, 0x0, 0x19, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
err = en.Append(0xde, 0x0, 0x1b, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -328,6 +340,16 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
|
||||
return
|
||||
}
|
||||
// write "CorsConfigXML"
|
||||
err = en.Append(0xad, 0x43, 0x6f, 0x72, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteBytes(z.CorsConfigXML)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CorsConfigXML")
|
||||
return
|
||||
}
|
||||
// write "PolicyConfigUpdatedAt"
|
||||
err = en.Append(0xb5, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
|
||||
if err != nil {
|
||||
@@ -438,15 +460,25 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaUpdatedAt")
|
||||
return
|
||||
}
|
||||
// write "CorsConfigUpdatedAt"
|
||||
err = en.Append(0xb3, 0x43, 0x6f, 0x72, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteTime(z.CorsConfigUpdatedAt)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CorsConfigUpdatedAt")
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// map header, size 25
|
||||
// map header, size 27
|
||||
// string "Name"
|
||||
o = append(o, 0xde, 0x0, 0x19, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
o = append(o, 0xde, 0x0, 0x1b, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
o = msgp.AppendString(o, z.Name)
|
||||
// string "Created"
|
||||
o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
|
||||
@@ -487,6 +519,9 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
// string "BucketTargetsConfigMetaJSON"
|
||||
o = append(o, 0xbb, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x4a, 0x53, 0x4f, 0x4e)
|
||||
o = msgp.AppendBytes(o, z.BucketTargetsConfigMetaJSON)
|
||||
// string "CorsConfigXML"
|
||||
o = append(o, 0xad, 0x43, 0x6f, 0x72, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
|
||||
o = msgp.AppendBytes(o, z.CorsConfigXML)
|
||||
// string "PolicyConfigUpdatedAt"
|
||||
o = append(o, 0xb5, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
|
||||
o = msgp.AppendTime(o, z.PolicyConfigUpdatedAt)
|
||||
@@ -520,6 +555,9 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
// string "BucketTargetsConfigMetaUpdatedAt"
|
||||
o = append(o, 0xd9, 0x20, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
|
||||
o = msgp.AppendTime(o, z.BucketTargetsConfigMetaUpdatedAt)
|
||||
// string "CorsConfigUpdatedAt"
|
||||
o = append(o, 0xb3, 0x43, 0x6f, 0x72, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
|
||||
o = msgp.AppendTime(o, z.CorsConfigUpdatedAt)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -625,6 +663,12 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
|
||||
return
|
||||
}
|
||||
case "CorsConfigXML":
|
||||
z.CorsConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.CorsConfigXML)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CorsConfigXML")
|
||||
return
|
||||
}
|
||||
case "PolicyConfigUpdatedAt":
|
||||
z.PolicyConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
|
||||
if err != nil {
|
||||
@@ -691,6 +735,12 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaUpdatedAt")
|
||||
return
|
||||
}
|
||||
case "CorsConfigUpdatedAt":
|
||||
z.CorsConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CorsConfigUpdatedAt")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
@@ -705,6 +755,6 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *BucketMetadata) Msgsize() (s int) {
|
||||
s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize + 26 + msgp.TimeSize + 25 + msgp.TimeSize + 28 + msgp.TimeSize + 29 + msgp.TimeSize + 34 + msgp.TimeSize
|
||||
s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 14 + msgp.BytesPrefixSize + len(z.CorsConfigXML) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize + 26 + msgp.TimeSize + 25 + msgp.TimeSize + 28 + msgp.TimeSize + 29 + msgp.TimeSize + 34 + msgp.TimeSize + 20 + msgp.TimeSize
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestBucketMetadataCorsRoundTrip(t *testing.T) {
|
||||
meta := newBucketMetadata("test-cors")
|
||||
meta.CorsConfigXML = []byte(`<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`)
|
||||
meta.CorsConfigUpdatedAt = UTCNow()
|
||||
|
||||
buf, err := meta.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got BucketMetadata
|
||||
if _, err := got.UnmarshalMsg(buf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got.CorsConfigXML) != string(meta.CorsConfigXML) {
|
||||
t.Fatalf("CorsConfigXML not preserved: %q", string(got.CorsConfigXML))
|
||||
}
|
||||
if !got.CorsConfigUpdatedAt.Equal(meta.CorsConfigUpdatedAt) {
|
||||
t.Fatalf("CorsConfigUpdatedAt not preserved")
|
||||
}
|
||||
}
|
||||
|
||||
// A persisted retired extension must not prevent the whole bucket's metadata
|
||||
// from loading, including unrelated versioning and ordinary lifecycle rules.
|
||||
func TestBucketMetadataRetiredAccessTiering(t *testing.T) {
|
||||
meta := newBucketMetadata("retired-access")
|
||||
meta.LifecycleConfigXML = []byte(`<LifecycleConfiguration><AccessTierQuota>500GiB</AccessTierQuota><Rule><ID>access</ID><Status>Enabled</Status><Filter><Prefix>logs/</Prefix></Filter><AccessTransition><Window>10m</Window><PromoteAfterAccesses>10</PromoteAfterAccesses></AccessTransition></Rule><Rule><ID>ordinary</ID><Status>Enabled</Status><Filter><Prefix>expired/</Prefix></Filter><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>`)
|
||||
meta.VersioningConfigXML = []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status></VersioningConfiguration>`)
|
||||
data, err := meta.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := newBucketMetadata(meta.Name)
|
||||
if _, err := got.UnmarshalMsg(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := got.parseAllConfigs(t.Context(), nil); err != nil {
|
||||
t.Fatalf("bucket metadata failed to load: %v", err)
|
||||
}
|
||||
if got.lifecycleConfig == nil || got.lifecycleConfig.HasActiveRules("logs/") || !got.lifecycleConfig.HasActiveRules("expired/") {
|
||||
t.Fatal("unexpected lifecycle behavior")
|
||||
}
|
||||
if got.versioningConfig == nil || !got.versioningConfig.Enabled() {
|
||||
t.Fatal("unrelated versioning lost")
|
||||
}
|
||||
}
|
||||
@@ -26,7 +26,7 @@ import (
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
+194
-17
@@ -22,13 +22,15 @@ import (
|
||||
"errors"
|
||||
"math"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/amztime"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// BucketObjectLockSys - map of bucket and retention configuration.
|
||||
@@ -150,7 +152,11 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
|
||||
}
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
|
||||
// If you try to delete objects protected by governance mode and have s3:BypassGovernanceRetention, the operation will succeed.
|
||||
if checkRequestAuthType(ctx, r, policy.BypassGovernanceRetentionAction, bucket, object.ObjectName) != ErrNone {
|
||||
if reqInfo := logger.GetReqInfo(ctx); reqInfo != nil {
|
||||
reqInfo.BucketName = bucket
|
||||
reqInfo.ObjectName = object.ObjectName
|
||||
}
|
||||
if authorizeRequest(ctx, r, policy.BypassGovernanceRetentionAction) != ErrNone {
|
||||
return errAuthentication
|
||||
}
|
||||
}
|
||||
@@ -198,7 +204,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
|
||||
byPassSet, r, cred, owner)
|
||||
// Governance mode retention period cannot be shortened, if x-amz-bypass-governance is not set.
|
||||
if !byPassSet {
|
||||
if objRetention.Mode != objectlock.RetGovernance || objRetention.RetainUntilDate.Before((ret.RetainUntilDate.Time)) {
|
||||
if objRetention.Mode != objectlock.RetGovernance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) {
|
||||
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
||||
}
|
||||
}
|
||||
@@ -209,7 +215,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
|
||||
case objectlock.RetCompliance:
|
||||
// Compliance retention mode cannot be changed or shortened.
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
|
||||
if objRetention.Mode != objectlock.RetCompliance || objRetention.RetainUntilDate.Before((ret.RetainUntilDate.Time)) {
|
||||
if objRetention.Mode != objectlock.RetCompliance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) {
|
||||
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
||||
}
|
||||
apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
|
||||
@@ -242,7 +248,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
|
||||
// For objects in "Compliance" mode, retention date cannot be shortened, and mode cannot be altered.
|
||||
// For objects with legal hold header set, the s3:PutObjectLegalHold permission is expected to be set
|
||||
// Both legal hold and retention can be applied independently on an object
|
||||
func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, retentionPermErr, legalHoldPermErr APIErrorCode) (objectlock.RetMode, objectlock.RetentionDate, objectlock.ObjectLegalHold, APIErrorCode) {
|
||||
func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, retentionPermErr, legalHoldPermErr APIErrorCode, replicaTrusted bool) (objectlock.RetMode, objectlock.RetentionDate, objectlock.ObjectLegalHold, APIErrorCode) {
|
||||
var mode objectlock.RetMode
|
||||
var retainDate objectlock.RetentionDate
|
||||
var legalHold objectlock.ObjectLegalHold
|
||||
@@ -269,9 +275,7 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
||||
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
|
||||
}
|
||||
|
||||
replica := rq.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String()
|
||||
|
||||
if opts.VersionID != "" && !replica {
|
||||
if opts.VersionID != "" && !replicaTrusted {
|
||||
if objInfo, err := getObjectInfoFn(ctx, bucket, object, opts); err == nil {
|
||||
r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
|
||||
t, err := objectlock.UTCNowNTP()
|
||||
@@ -307,8 +311,8 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
||||
if err != nil {
|
||||
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
|
||||
}
|
||||
rMode, rDate, err := objectlock.ParseObjectLockRetentionHeaders(rq.Header)
|
||||
if err != nil && (!replica || rMode != "" || !rDate.IsZero()) {
|
||||
rMode, rDate, err := objectlock.ParseObjectLockRetentionHeaders(rq.Header, replicaTrusted)
|
||||
if err != nil && (!replicaTrusted || rMode != "" || !rDate.IsZero()) {
|
||||
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
|
||||
}
|
||||
if retentionPermErr != ErrNone {
|
||||
@@ -316,7 +320,7 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
||||
}
|
||||
return rMode, rDate, legalHold, ErrNone
|
||||
}
|
||||
if replica { // replica inherits retention metadata only from source
|
||||
if replicaTrusted { // replica inherits retention metadata only from source
|
||||
return "", objectlock.RetentionDate{}, legalHold, ErrNone
|
||||
}
|
||||
if !retentionRequested && retentionCfg.Validity > 0 {
|
||||
@@ -330,11 +334,10 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
||||
return mode, retainDate, legalHold, ErrObjectLocked
|
||||
}
|
||||
|
||||
if !legalHoldRequested && retentionCfg.LockEnabled {
|
||||
// inherit retention from bucket configuration
|
||||
return retentionCfg.Mode, objectlock.RetentionDate{Time: t.Add(retentionCfg.Validity)}, legalHold, ErrNone
|
||||
}
|
||||
return "", objectlock.RetentionDate{}, legalHold, ErrNone
|
||||
// Inherit retention from the bucket configuration. A legal-hold header
|
||||
// on the same request, ON or OFF, is independent of retention and must
|
||||
// not suppress the default (#165).
|
||||
return retentionCfg.Mode, objectlock.RetentionDate{Time: t.Add(retentionCfg.Validity)}, legalHold, ErrNone
|
||||
}
|
||||
return mode, retainDate, legalHold, ErrNone
|
||||
}
|
||||
@@ -343,3 +346,177 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
||||
func NewBucketObjectLockSys() *BucketObjectLockSys {
|
||||
return &BucketObjectLockSys{}
|
||||
}
|
||||
|
||||
// objectLockState is the Object Lock metadata of a stored object version
|
||||
// together with the replication timestamps that order updates to it.
|
||||
type objectLockState struct {
|
||||
mode, retainUntil, retentionTimestamp string
|
||||
legalHold, legalHoldTimestamp string
|
||||
}
|
||||
|
||||
func storedObjectLockState(metadata map[string]string) objectLockState {
|
||||
return objectLockState{
|
||||
mode: metadata[strings.ToLower(xhttp.AmzObjectLockMode)],
|
||||
retainUntil: metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)],
|
||||
retentionTimestamp: metadata[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp],
|
||||
legalHold: metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)],
|
||||
legalHoldTimestamp: metadata[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp],
|
||||
}
|
||||
}
|
||||
|
||||
// olderThan reports whether a stored replication timestamp is missing,
|
||||
// unreadable, or earlier than the source timestamp, in which case the
|
||||
// replica update wins. A zero source timestamp never wins.
|
||||
func olderThan(stored string, src time.Time) bool {
|
||||
if src.IsZero() {
|
||||
return false
|
||||
}
|
||||
ondisk, err := time.Parse(time.RFC3339Nano, stored)
|
||||
return err != nil || ondisk.Before(src)
|
||||
}
|
||||
|
||||
func (s objectLockState) retentionIsOlderThan(src time.Time) bool {
|
||||
return olderThan(s.retentionTimestamp, src)
|
||||
}
|
||||
|
||||
func (s objectLockState) legalHoldIsOlderThan(src time.Time) bool {
|
||||
return olderThan(s.legalHoldTimestamp, src)
|
||||
}
|
||||
|
||||
// restoreRetention and restoreLegalHold put the stored state back into
|
||||
// metadata that was rebuilt from a request whose update was not applied.
|
||||
func (s objectLockState) restoreRetention(metadata map[string]string) {
|
||||
// The stored timestamp orders the next update and must survive even when
|
||||
// the stored value is empty, which is how a removal is recorded.
|
||||
if s.retentionTimestamp != "" {
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = s.retentionTimestamp
|
||||
}
|
||||
if s.mode == "" {
|
||||
return
|
||||
}
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = s.mode
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = s.retainUntil
|
||||
}
|
||||
|
||||
func (s objectLockState) restoreLegalHold(metadata map[string]string) {
|
||||
if s.legalHoldTimestamp != "" {
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = s.legalHoldTimestamp
|
||||
}
|
||||
if s.legalHold == "" {
|
||||
return
|
||||
}
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = s.legalHold
|
||||
}
|
||||
|
||||
// replicaStoredLock reads the Object Lock state stored on the addressed version
|
||||
// so a trusted replica write can order its update against it. A missing object
|
||||
// or version yields an empty state, which is correct for the first write of a
|
||||
// version; any other read error is returned so the caller fails the write rather
|
||||
// than ordering an incoming update against lock state it merely failed to read
|
||||
// (an older incoming value must not win over a newer stored one just because the
|
||||
// read timed out).
|
||||
func replicaStoredLock(ctx context.Context, getObjectInfo GetObjectInfoFn, bucket, object, versionID string) (objectLockState, error) {
|
||||
oi, err := getObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: versionID})
|
||||
switch {
|
||||
case err == nil:
|
||||
return storedObjectLockState(oi.UserDefined), nil
|
||||
case isErrObjectNotFound(err) || isErrVersionNotFound(err):
|
||||
return objectLockState{}, nil
|
||||
default:
|
||||
return objectLockState{}, err
|
||||
}
|
||||
}
|
||||
|
||||
// applyReplicatedObjectLock writes the retention and legal-hold decision into
|
||||
// metadata for a PUT, CopyObject, or multipart-initiation request. A request
|
||||
// that is not an actual trusted replica -- a normal user write, or a trusted
|
||||
// peer that carried the replication marker without REPLICA status -- takes
|
||||
// ordinary write semantics: a validated value is applied and stamped now, and a
|
||||
// missing value is left as is. Only an actual replica update is ordered against
|
||||
// the state already stored on the addressed version, so a stale value cannot
|
||||
// overwrite a newer one and a full retransmit cannot roll a destination back.
|
||||
// The stored argument is meaningful only for a replica; callers pass an empty
|
||||
// state otherwise. Only the two Object Lock keys and their reserved ordering
|
||||
// timestamps are touched; any encryption-metadata reconciliation stays with the
|
||||
// caller.
|
||||
func applyReplicatedObjectLock(metadata map[string]string, stored objectLockState,
|
||||
replicaTrusted bool,
|
||||
retentionMode objectlock.RetMode, retentionDate objectlock.RetentionDate,
|
||||
legalHold objectlock.ObjectLegalHold, srcRetentionTimestamp, srcLegalholdTimestamp time.Time,
|
||||
) {
|
||||
switch {
|
||||
case !replicaTrusted:
|
||||
// Ordinary write semantics: apply a validated retention and stamp it now;
|
||||
// a missing value carries no instruction, so leave the metadata as it is.
|
||||
if retentionMode.Valid() {
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC())
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
}
|
||||
case !stored.retentionIsOlderThan(srcRetentionTimestamp):
|
||||
// The stored update is at least as new as this replica's, or the replica
|
||||
// carries no ordering timestamp: keep what is stored. This is also how a
|
||||
// stale retransmit is rejected.
|
||||
stored.restoreRetention(metadata)
|
||||
default:
|
||||
// The replica update wins. A removal carries no value but still records
|
||||
// the source timestamp that orders it.
|
||||
if retentionMode.Valid() {
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC())
|
||||
}
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = srcRetentionTimestamp.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
// Legal hold has no removal in S3: an explicitly empty header is already
|
||||
// rejected as an invalid status, so the only value-less shape that gets here
|
||||
// is an absent one, which conveys no legal-hold change. Only a valid status
|
||||
// can win.
|
||||
switch {
|
||||
case !replicaTrusted:
|
||||
if legalHold.Status.Valid() {
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
}
|
||||
case legalHold.Status.Valid() && stored.legalHoldIsOlderThan(srcLegalholdTimestamp):
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = srcLegalholdTimestamp.UTC().Format(time.RFC3339Nano)
|
||||
default:
|
||||
stored.restoreLegalHold(metadata)
|
||||
}
|
||||
}
|
||||
|
||||
// reconcileStoredObjectLock re-orders the Object Lock already written into
|
||||
// metadata against the state currently stored on the destination version, both
|
||||
// compared by their reserved ordering timestamps. It runs inside the object
|
||||
// layer under the namespace write lock that guards the version replacement,
|
||||
// after the destination version is read and before the new one is committed, so
|
||||
// a replica update whose ordering was decided at handler time (or, for multipart,
|
||||
// at initiation) cannot overwrite a newer lock update that reached the version in
|
||||
// between. metadata already carries the incoming update with its source
|
||||
// timestamps; a stored value that is not older than the incoming one is put back,
|
||||
// which for a stored removal means clearing the incoming value and keeping only
|
||||
// the removal's timestamp. Only the two lock keys and their reserved timestamps
|
||||
// move; a non-replica write never sets the flag that invokes this.
|
||||
func reconcileStoredObjectLock(metadata map[string]string, stored objectLockState) {
|
||||
incoming := storedObjectLockState(metadata)
|
||||
|
||||
incomingRetentionTS, _ := time.Parse(time.RFC3339Nano, incoming.retentionTimestamp)
|
||||
if !stored.retentionIsOlderThan(incomingRetentionTS) {
|
||||
// The stored retention is at least as new as the incoming one (or the
|
||||
// incoming update is unordered): drop the incoming value and put the stored
|
||||
// state back, which may itself be a removal (value keys absent, timestamp
|
||||
// present).
|
||||
delete(metadata, strings.ToLower(xhttp.AmzObjectLockMode))
|
||||
delete(metadata, strings.ToLower(xhttp.AmzObjectLockRetainUntilDate))
|
||||
delete(metadata, ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp)
|
||||
stored.restoreRetention(metadata)
|
||||
}
|
||||
|
||||
incomingLegalHoldTS, _ := time.Parse(time.RFC3339Nano, incoming.legalHoldTimestamp)
|
||||
if incoming.legalHold == "" || !stored.legalHoldIsOlderThan(incomingLegalHoldTS) {
|
||||
delete(metadata, strings.ToLower(xhttp.AmzObjectLockLegalHold))
|
||||
delete(metadata, ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp)
|
||||
stored.restoreLegalHold(metadata)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,6 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
@@ -27,7 +26,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -100,13 +99,13 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
return
|
||||
}
|
||||
|
||||
configData, err := json.Marshal(bucketPolicy)
|
||||
configData, err := canonicalBucketPolicy(bucketPolicy)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
|
||||
result, err := globalBucketMetadataSys.updateAndParseMetadata(ctx, bucket, bucketPolicyConfig, configData, false, false, nil)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -116,8 +115,8 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypePolicy,
|
||||
Bucket: bucket,
|
||||
Policy: bucketPolicyBytes,
|
||||
UpdatedAt: updatedAt,
|
||||
Policy: result.meta.PolicyConfigJSON,
|
||||
UpdatedAt: result.updatedAt,
|
||||
}))
|
||||
|
||||
// Success.
|
||||
@@ -200,7 +199,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
return
|
||||
}
|
||||
|
||||
configData, err := json.Marshal(config)
|
||||
configData, err := canonicalBucketPolicy(config)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
|
||||
@@ -29,8 +29,8 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/policy/condition"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy/condition"
|
||||
)
|
||||
|
||||
func getAnonReadOnlyBucketPolicy(bucketName string) *policy.BucketPolicy {
|
||||
|
||||
+28
-9
@@ -33,8 +33,8 @@ import (
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/policy/condition"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy/condition"
|
||||
)
|
||||
|
||||
// PolicySys - policy subsystem.
|
||||
@@ -255,13 +255,32 @@ func getConditionValuesWithTags(r *http.Request, lc string, cred auth.Credential
|
||||
}
|
||||
|
||||
cloneHeader := r.Header.Clone()
|
||||
signatureAge := cloneHeader.Get("x-amz-signature-age")
|
||||
cloneHeader.Del("x-amz-signature-age")
|
||||
// The presigned V4 verifier overwrites this internal scratch header after
|
||||
// validating the signature. Ignore a value supplied on every other request
|
||||
// type, where it would otherwise synthesize s3:signatureAge.
|
||||
if authType == authTypePresigned && signatureAge != "" {
|
||||
args["signatureAge"] = []string{signatureAge}
|
||||
|
||||
// s3:signatureAge is derived from the presigned X-Amz-Date rather than from
|
||||
// anything the verifier writes back: PutObject and UploadPart authorize
|
||||
// before they verify the signature, so a post-verification value is not yet
|
||||
// available on the first evaluation. The date is bound by the signature
|
||||
// (doesPresignedSignatureMatch rebuilds and compares it), so a forged date
|
||||
// only changes the authorization outcome of a request that then fails
|
||||
// verification. A date that does not parse leaves the key absent; the
|
||||
// verifier rejects the request as ErrMalformedPresignedDate.
|
||||
if authType == authTypePresigned {
|
||||
if signedDate, err := time.Parse(iso8601Format, r.Form.Get(xhttp.AmzDate)); err == nil {
|
||||
args["signatureAge"] = []string{strconv.FormatInt(currTime.Sub(signedDate).Milliseconds(), 10)}
|
||||
}
|
||||
}
|
||||
|
||||
// s3:x-amz-content-sha256 must name the payload hash the request is actually
|
||||
// verified and enforced against, and only one such value. Presence of the
|
||||
// header controls whether the key exists at all (AWS documents that the
|
||||
// query-string form does not populate it), but the value comes from the same
|
||||
// selection getContentSha256Cksum makes for verification: the presigned query
|
||||
// value takes precedence over the header, and a repeated header contributes
|
||||
// only its first value. Exposing every raw header value instead let a
|
||||
// request satisfy a policy with a value the verifier never checked.
|
||||
if _, ok := cloneHeader[xhttp.AmzContentSha256]; ok {
|
||||
args[xhttp.AmzContentSha256] = []string{getContentSha256Cksum(r, serviceS3)}
|
||||
cloneHeader.Del(xhttp.AmzContentSha256)
|
||||
}
|
||||
|
||||
userTags := cloneHeader.Get(xhttp.AmzObjectTagging)
|
||||
|
||||
@@ -23,14 +23,16 @@ import (
|
||||
"net/url"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/policy/condition"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy/condition"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -579,8 +581,20 @@ func TestGetConditionValuesRejectsAbsentInternalKeys(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// s3:signatureAge is derived from the presigned X-Amz-Date, which the signature
|
||||
// binds. A client header under the former scratch name must never supply it on
|
||||
// any auth type, and a presign whose date is missing or malformed leaves the
|
||||
// key absent (the verifier then rejects the request).
|
||||
func TestGetConditionValuesOnlyAcceptsPresignedSignatureAge(t *testing.T) {
|
||||
const signatureAgeHeader = "x-amz-signature-age"
|
||||
signedDate := UTCNow().Add(-90 * time.Second)
|
||||
presignQuery := func(date string) string {
|
||||
q := url.Values{xhttp.AmzCredential: {"access/20260803/us-east-1/s3/aws4_request"}}
|
||||
if date != "" {
|
||||
q.Set(xhttp.AmzDate, date)
|
||||
}
|
||||
return "http://minio.local/bkt/obj?" + q.Encode()
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
@@ -602,19 +616,33 @@ func TestGetConditionValuesOnlyAcceptsPresignedSignatureAge(t *testing.T) {
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "presigned verifier value",
|
||||
target: "http://minio.local/bkt/obj?" + url.Values{
|
||||
xhttp.AmzCredential: {"access/20260803/us-east-1/s3/aws4_request"},
|
||||
}.Encode(),
|
||||
name: "presigned client header without date",
|
||||
target: presignQuery(""),
|
||||
headers: map[string]string{signatureAgeHeader: "250"},
|
||||
},
|
||||
{
|
||||
name: "presigned malformed date",
|
||||
target: presignQuery("yesterday"),
|
||||
},
|
||||
{
|
||||
name: "presigned signed date",
|
||||
target: presignQuery(signedDate.Format(iso8601Format)),
|
||||
headers: map[string]string{signatureAgeHeader: "250"},
|
||||
want: true,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := condValuesForRequest(t, tc.target, tc.headers)
|
||||
_, ok := got["signatureAge"]
|
||||
v, ok := got["signatureAge"]
|
||||
if ok != tc.want {
|
||||
t.Fatalf("signatureAge presence: expected %v, got %v", tc.want, got["signatureAge"])
|
||||
t.Fatalf("signatureAge presence: expected %v, got %v", tc.want, v)
|
||||
}
|
||||
if !tc.want {
|
||||
return
|
||||
}
|
||||
age, err := strconv.ParseInt(strings.Join(v, ""), 10, 64)
|
||||
if err != nil || age < (90*time.Second).Milliseconds() || age > (2*time.Minute).Milliseconds() {
|
||||
t.Fatalf("signatureAge = %v, want about 90s derived from X-Amz-Date rather than the client header", v)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+12
-8
@@ -43,6 +43,17 @@ func NewBucketQuotaSys() *BucketQuotaSys {
|
||||
return &BucketQuotaSys{}
|
||||
}
|
||||
|
||||
// getBucketQuotaSize returns the effective enforced hard-quota size.
|
||||
func getBucketQuotaSize(quota *madmin.BucketQuota) uint64 {
|
||||
if quota == nil || quota.Type != madmin.HardQuota {
|
||||
return 0
|
||||
}
|
||||
if quota.Size > 0 {
|
||||
return quota.Size
|
||||
}
|
||||
return quota.Quota
|
||||
}
|
||||
|
||||
var bucketStorageCache = cachevalue.New[DataUsageInfo]()
|
||||
|
||||
// Init initialize bucket quota.
|
||||
@@ -110,14 +121,7 @@ func (sys *BucketQuotaSys) enforceQuotaHard(ctx context.Context, bucket string,
|
||||
return err
|
||||
}
|
||||
|
||||
var quotaSize uint64
|
||||
if q != nil && q.Type == madmin.HardQuota {
|
||||
if q.Size > 0 {
|
||||
quotaSize = q.Size
|
||||
} else if q.Quota > 0 {
|
||||
quotaSize = q.Quota
|
||||
}
|
||||
}
|
||||
quotaSize := getBucketQuotaSize(q)
|
||||
if quotaSize > 0 {
|
||||
if uint64(size) >= quotaSize { // check if file size already exceeds the quota
|
||||
return BucketQuotaExceeded{Bucket: bucket}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
// Copyright (c) 2015-2025 MinIO, Inc.
|
||||
// Copyright (c) 2025-2026 PGSTY
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
)
|
||||
|
||||
func TestGetBucketQuotaSize(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
quota *madmin.BucketQuota
|
||||
want uint64
|
||||
}{
|
||||
{name: "nil"},
|
||||
{name: "empty", quota: &madmin.BucketQuota{}},
|
||||
{name: "current size", quota: &madmin.BucketQuota{Type: madmin.HardQuota, Size: 1024}, want: 1024},
|
||||
{name: "legacy quota", quota: &madmin.BucketQuota{Type: madmin.HardQuota, Quota: 2048}, want: 2048},
|
||||
{name: "size takes precedence", quota: &madmin.BucketQuota{Type: madmin.HardQuota, Size: 1024, Quota: 2048}, want: 1024},
|
||||
{name: "missing type", quota: &madmin.BucketQuota{Size: 1024}},
|
||||
{name: "unsupported type", quota: &madmin.BucketQuota{Type: "fifo", Size: 1024}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := getBucketQuotaSize(tt.quota); got != tt.want {
|
||||
t.Fatalf("getBucketQuotaSize() = %d, want %d", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsBktQuotaCfgReplicated(t *testing.T) {
|
||||
hardQuota := func(size, legacy uint64) *madmin.BucketQuota {
|
||||
return &madmin.BucketQuota{Type: madmin.HardQuota, Size: size, Quota: legacy}
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
quotas []*madmin.BucketQuota
|
||||
want bool
|
||||
}{
|
||||
{name: "none configured", quotas: []*madmin.BucketQuota{nil, nil}, want: true},
|
||||
{name: "missing from one site", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), nil}},
|
||||
{name: "matching size", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), hardQuota(1024, 0)}, want: true},
|
||||
{name: "different size", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), hardQuota(2048, 0)}},
|
||||
{name: "equivalent representations", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), hardQuota(0, 1024)}, want: true},
|
||||
{name: "different typeless size", quotas: []*madmin.BucketQuota{{Size: 1024}, {Size: 2048}}},
|
||||
{name: "different type", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), {Type: "fifo", Size: 1024}}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := isBktQuotaCfgReplicated(len(tt.quotas), tt.quotas); got != tt.want {
|
||||
t.Fatalf("isBktQuotaCfgReplicated() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -34,7 +34,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// PutBucketReplicationConfigHandler - PUT Bucket replication configuration.
|
||||
@@ -617,7 +617,7 @@ func (api objectAPIHandlers) ValidateBucketReplicationCredsHandler(w http.Respon
|
||||
ReplicationValidityCheck: true, // set this to validate the replication config
|
||||
},
|
||||
}
|
||||
obj := path.Join(minioReservedBucket, globalLocalNodeNameHex, "deleteme")
|
||||
obj := replicationValidationObject(rule)
|
||||
ui, err := c.PutObject(ctx, clnt.Bucket, obj, reader, int64(len(buf)), "", "", putOpts)
|
||||
if err != nil && !isReplicationPermissionCheck(ErrorRespToObjectError(err, bucket, obj)) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, fmt.Errorf("s3:ReplicateObject permissions missing for replication user: %w", err)), r.URL)
|
||||
@@ -658,3 +658,7 @@ func (api objectAPIHandlers) ValidateBucketReplicationCredsHandler(w http.Respon
|
||||
// Write success response.
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
func replicationValidationObject(rule replication.Rule) string {
|
||||
return path.Join(rule.Prefix(), minioReservedBucket, globalLocalNodeNameHex, "deleteme")
|
||||
}
|
||||
|
||||
@@ -418,6 +418,9 @@ func getReplicationState(rinfos replicatedInfos, prevState ReplicationState, vID
|
||||
|
||||
for _, rinfo := range rinfos.Targets {
|
||||
if rinfo.ResyncTimestamp != "" {
|
||||
if rs.ResetStatusesMap == nil {
|
||||
rs.ResetStatusesMap = make(map[string]string)
|
||||
}
|
||||
rs.ResetStatusesMap[targetResetHeader(rinfo.Arn)] = rinfo.ResyncTimestamp
|
||||
}
|
||||
}
|
||||
@@ -640,10 +643,10 @@ type VersionPurgeStatusType = replication.VersionPurgeStatusType
|
||||
|
||||
type replicationResyncer struct {
|
||||
// map of bucket to their resync status
|
||||
statusMap map[string]BucketReplicationResyncStatus
|
||||
workerSize int
|
||||
resyncCancelCh chan struct{}
|
||||
workerCh chan struct{}
|
||||
statusMap map[string]BucketReplicationResyncStatus
|
||||
workerSize int
|
||||
cancelResyncs map[resyncOpts]context.CancelCauseFunc
|
||||
workerCh chan struct{}
|
||||
sync.RWMutex
|
||||
}
|
||||
|
||||
|
||||
+565
-176
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user