mirror of
https://github.com/pgsty/minio.git
synced 2026-11-22 23:36:41 +03:00
Compare commits
201 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5f00f6762f | |||
| af2b1794d3 | |||
| d371f77dcc | |||
| 022722a7a7 | |||
| 03027727d1 | |||
| 4fcdf37ce6 | |||
| 9ebe81c1b3 | |||
| e5f5c9e7f6 | |||
| 7b4cacc392 | |||
| a0dd7dae9b | |||
| 709d50a916 | |||
| dff81f293b | |||
| f653a6ea03 | |||
| 47d239f84f | |||
| e069fe9d92 | |||
| d848fb52b5 | |||
| 3ce8319251 | |||
| 9df0f4abaf | |||
| 4d0693cb8c | |||
| bf59e3f222 | |||
| 41aa846097 | |||
| 4093fa0d78 | |||
| 13bf126ebd | |||
| 1cf529ce8a | |||
| 9b76a21675 | |||
| 89637554d6 | |||
| 2dd1e00da4 | |||
| 5d955b5b74 | |||
| f7808a172c | |||
| acc9b514f5 | |||
| 31ba01c5d5 | |||
| 48ec10312f | |||
| 114dc10529 | |||
| 461e9a7210 | |||
| fcbb93e895 | |||
| 62cf066ff5 | |||
| 1ee64a8d89 | |||
| 01aaef2b50 | |||
| bcc62afe3d | |||
| 5c57658163 | |||
| f175e98c34 | |||
| 12f631b502 | |||
| ccb676e60c | |||
| 51d41345f7 | |||
| e59a3d938e | |||
| b32f2d9dd0 | |||
| d63c92e393 | |||
| 68127c5a63 | |||
| c4b5e1cb45 | |||
| 9a303f5096 | |||
| 87d8b5967f | |||
| f760046c44 | |||
| 93e7ef4bcc | |||
| a4229b366f | |||
| 420340bc14 | |||
| e7e87402ed | |||
| a164e1dda1 | |||
| 2f61325d4a | |||
| a6145e1d2e | |||
| 5232546690 | |||
| 0c46cb641b | |||
| 1233254309 | |||
| 8a2fe9b7a0 | |||
| f26ee6bf0a | |||
| d69c4ccfe4 | |||
| 086e619505 | |||
| 48e1846525 | |||
| bcc8871b1d | |||
| 25cb3511c9 | |||
| cfefc049c1 | |||
| af56d17630 | |||
| d1105bbb3d | |||
| 66fe61ff65 | |||
| a1141a43f2 | |||
| 702f113f51 | |||
| 63aace4099 | |||
| 9d7094b770 | |||
| 450dcb8484 | |||
| 4074d00b96 | |||
| 75ba0ce402 | |||
| da142327f2 | |||
| a3df317ae0 | |||
| 2c50d11f72 | |||
| 5ac33e1583 | |||
| d57c4e8407 | |||
| 374de0fa32 | |||
| 80e23dc9f2 | |||
| 2cc0e3c6ed | |||
| f9da3b919d | |||
| 1309853f57 | |||
| 39b8e6c30a | |||
| 9a6e1477f4 | |||
| 49375ed2d3 | |||
| f817b5261c | |||
| 885bd2c20a | |||
| 89b75e7913 | |||
| 079ebb1926 | |||
| 04c29aac11 | |||
| 95e7a190b3 | |||
| 8e2392e48f | |||
| 3abe0d95a5 | |||
| 9c6c9805de | |||
| 5cb900bfad | |||
| 0af5d22286 | |||
| f1687f402b | |||
| ce606df2c4 | |||
| fd44dc4e9b | |||
| 479745e764 | |||
| cc1c54475f | |||
| e7654d470c | |||
| 4b25f7e819 | |||
| 711b092f86 | |||
| 2bc103b80c | |||
| ad873c7357 | |||
| 0af0907eff | |||
| e27ba2bc14 | |||
| 236e163c0b | |||
| 7220210e8d | |||
| 34cbca97ea | |||
| 109d824e5f | |||
| 87746913fc | |||
| 7935c84f9a | |||
| c185635b43 | |||
| c201148738 | |||
| 53adb21c52 | |||
| 58b0ee36ca | |||
| 8a1f594add | |||
| 5b9959617e | |||
| da19d91b64 | |||
| 40bee4b7ba | |||
| 6a9b5d6763 | |||
| 0720ed477e | |||
| 46e82eb54d | |||
| f8ca4a8656 | |||
| e7d0e62f16 | |||
| aee290fc34 | |||
| 62cce2b152 | |||
| 65d4806a7b | |||
| 765757473a | |||
| 425bd7fff1 | |||
| e12e739a53 | |||
| 8b736dee34 | |||
| 32e75c27bf | |||
| 885ca604a1 | |||
| d10382d0dc | |||
| 0db4bf3b00 | |||
| 87c621965d | |||
| b2dca43fda | |||
| 33a91d972f | |||
| 2cbd48a3c3 | |||
| b5409ca112 | |||
| 35bd75948a | |||
| 0c8d74205b | |||
| fcc4d77895 | |||
| c52acc1a5d | |||
| 5703426b3c | |||
| f0bd164b92 | |||
| 9936a69d89 | |||
| ce2326c946 | |||
| 4c164907f5 | |||
| 9b11dc9469 | |||
| 1eccc6908b | |||
| 202371afbd | |||
| 41ef4411d9 | |||
| 035aa6c201 | |||
| edf36bcbfa | |||
| 3598c4305d | |||
| 94fbb6df6d | |||
| f4c1286c9d | |||
| 6e112d1856 | |||
| e62dc6e023 | |||
| 84e1580a47 | |||
| ebac0ca73b | |||
| 3f9c79e919 | |||
| ec2979ca48 | |||
| 21646eebd2 | |||
| 5711996231 | |||
| 0079723d35 | |||
| bc3b35f975 | |||
| 00d864ed0e | |||
| 8f2a30d9af | |||
| d9766d7378 | |||
| 5594d284fc | |||
| 632eb4729a | |||
| 3b5de82f5a | |||
| 76195f1c68 | |||
| 6586fbfd0d | |||
| 53b09f4e25 | |||
| 59812446ad | |||
| 1c2f59604d | |||
| b3a6d5dbf8 | |||
| 11de51ec26 | |||
| 10f3a8590b | |||
| 55c22abec1 | |||
| 32a1b81e4c | |||
| 62d8c649fe | |||
| 312397739e | |||
| 7ade0c045b | |||
| f9f9fa6c90 | |||
| 0a9c777795 | |||
| 7a060cab1e |
@@ -1,54 +0,0 @@
|
|||||||
---
|
|
||||||
name: Bug report
|
|
||||||
about: Create a report to help us improve
|
|
||||||
title: ''
|
|
||||||
labels: community, triage
|
|
||||||
assignees: ''
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## NOTE
|
|
||||||
|
|
||||||
Silo issues are handled by community maintainers on a best-effort basis. There
|
|
||||||
is no SLA, SLO, or emergency production-support channel. Follow the local
|
|
||||||
[Code of Conduct](../code_of_conduct.md) when participating. Report suspected
|
|
||||||
vulnerabilities through the private process in [SECURITY.md](../SECURITY.md),
|
|
||||||
not in a public issue.
|
|
||||||
|
|
||||||
<!--- Provide a general summary of the issue in the Title above -->
|
|
||||||
|
|
||||||
## Expected Behavior
|
|
||||||
<!--- If you're describing a bug, tell us what should happen -->
|
|
||||||
<!--- If you're suggesting a change/improvement, tell us how it should work -->
|
|
||||||
|
|
||||||
## Current Behavior
|
|
||||||
<!--- If describing a bug, tell us what happens instead of the expected behavior -->
|
|
||||||
<!--- If suggesting a change/improvement, explain the difference from current behavior -->
|
|
||||||
|
|
||||||
## Possible Solution
|
|
||||||
<!--- Not obligatory, but suggest a fix/reason for the bug, -->
|
|
||||||
<!--- or ideas how to implement the addition or change -->
|
|
||||||
|
|
||||||
## Steps to Reproduce (for bugs)
|
|
||||||
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
|
|
||||||
<!--- reproduce this bug. Include code to reproduce, if relevant -->
|
|
||||||
<!--- and include relevant Silo logs with secrets and credentials removed -->
|
|
||||||
|
|
||||||
1.
|
|
||||||
2.
|
|
||||||
3.
|
|
||||||
4.
|
|
||||||
|
|
||||||
## Context
|
|
||||||
<!--- How has this issue affected you? What are you trying to accomplish? -->
|
|
||||||
<!--- Providing context helps us come up with a solution that is most useful in the real world -->
|
|
||||||
|
|
||||||
## Regression
|
|
||||||
<!-- Is this issue a regression? (Yes / No) -->
|
|
||||||
<!-- If Yes, optionally include the Silo version, commit id, or PR that caused this regression. -->
|
|
||||||
|
|
||||||
## Your Environment
|
|
||||||
<!--- Include as many relevant details about the environment you experienced the bug in -->
|
|
||||||
* Version used (`silo --version`):
|
|
||||||
* Server setup and configuration:
|
|
||||||
* Operating System and version (`uname -a`):
|
|
||||||
@@ -7,6 +7,14 @@ assignees: ''
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
Report bugs in the PGSTY SILO server (`pgsty/silo`) here. Community maintainers
|
||||||
|
handle reports on a best-effort basis. There is no SLA, SLO, or emergency
|
||||||
|
production-support channel. Follow the
|
||||||
|
[Code of Conduct](https://github.com/pgsty/silo/blob/main/code_of_conduct.md).
|
||||||
|
Report suspected vulnerabilities privately through
|
||||||
|
[SECURITY.md](https://github.com/pgsty/silo/blob/main/SECURITY.md).
|
||||||
|
For patches, see the [contribution guide](https://github.com/pgsty/silo/blob/main/CONTRIBUTING.md).
|
||||||
|
|
||||||
<!--- Provide a general summary of the issue in the Title above -->
|
<!--- Provide a general summary of the issue in the Title above -->
|
||||||
|
|
||||||
## Expected Behavior
|
## Expected Behavior
|
||||||
|
|||||||
@@ -7,6 +7,9 @@ assignees: ''
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
Suggest improvements to the PGSTY SILO server (`pgsty/silo`) here.
|
||||||
|
For patches, see the [contribution guide](https://github.com/pgsty/silo/blob/main/CONTRIBUTING.md).
|
||||||
|
|
||||||
**Is your feature request related to a problem? Please describe.**
|
**Is your feature request related to a problem? Please describe.**
|
||||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +1,14 @@
|
|||||||
## Contribution Licensing (no CLA, inbound=outbound, DCO required)
|
## Contribution Licensing (no CLA, inbound=outbound, DCO required)
|
||||||
|
|
||||||
This project does not use a CLA; contributions are accepted inbound=outbound.
|
This pull request contributes to PGSTY SILO (`pgsty/silo`). Code contributions
|
||||||
|
are accepted under AGPL-3.0-or-later, the same license as the server.
|
||||||
|
This project does not use a CLA or require a separate Apache-2.0 license grant.
|
||||||
By submitting this pull request I represent that I have the right to contribute
|
By submitting this pull request I represent that I have the right to contribute
|
||||||
the changes, which are licensed under this repository's
|
the code changes under this repository's
|
||||||
[GNU Affero General Public License v3.0 or later](https://www.gnu.org/licenses/agpl-3.0.html)
|
[GNU Affero General Public License v3.0 or later](https://www.gnu.org/licenses/agpl-3.0.html)
|
||||||
and remain my copyright. Every commit must carry a DCO `Signed-off-by` trailer
|
and retain copyright in my original work. Existing copyright and license
|
||||||
|
notices remain intact; separately licensed material keeps its applicable terms.
|
||||||
|
Every commit must carry a DCO `Signed-off-by` trailer
|
||||||
(`git commit -s`) certifying the
|
(`git commit -s`) certifying the
|
||||||
[Developer Certificate of Origin](https://developercertificate.org/) — see
|
[Developer Certificate of Origin](https://developercertificate.org/) — see
|
||||||
[CONTRIBUTING.md](https://github.com/pgsty/silo/blob/main/CONTRIBUTING.md).
|
[CONTRIBUTING.md](https://github.com/pgsty/silo/blob/main/CONTRIBUTING.md).
|
||||||
|
|||||||
@@ -7,6 +7,11 @@ on:
|
|||||||
description: "Published RELEASE.* tag to package as pgsty/silo"
|
description: "Published RELEASE.* tag to package as pgsty/silo"
|
||||||
required: true
|
required: true
|
||||||
type: string
|
type: string
|
||||||
|
recovery:
|
||||||
|
description: "Run the current main workflow against an already-published tag"
|
||||||
|
required: false
|
||||||
|
default: false
|
||||||
|
type: boolean
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
@@ -75,12 +80,18 @@ jobs:
|
|||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
|
||||||
- name: Verify workflow identity matches release source
|
- name: Verify workflow identity matches release source
|
||||||
|
env:
|
||||||
|
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||||
|
RECOVERY: ${{ inputs.recovery }}
|
||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
CHECKED_OUT_REVISION="$(git rev-parse HEAD)"
|
CHECKED_OUT_REVISION="$(git rev-parse HEAD)"
|
||||||
if [ "${CHECKED_OUT_REVISION}" != "${GITHUB_SHA}" ]; then
|
if [ "${CHECKED_OUT_REVISION}" != "${GITHUB_SHA}" ]; then
|
||||||
echo "Checked out ${CHECKED_OUT_REVISION}, but workflow identity is ${GITHUB_SHA}. Dispatch this workflow from ${RELEASE_TAG}." >&2
|
if [ "${RECOVERY}" != "true" ] || [ "${GITHUB_REF}" != "refs/heads/${DEFAULT_BRANCH}" ]; then
|
||||||
exit 1
|
echo "Checked out ${CHECKED_OUT_REVISION}, but workflow identity is ${GITHUB_SHA}. Dispatch from ${RELEASE_TAG}, or use recovery from ${DEFAULT_BRANCH}." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "Recovery workflow ${GITHUB_SHA} is packaging published source ${CHECKED_OUT_REVISION}."
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Prepare verified Docker contexts
|
- name: Prepare verified Docker contexts
|
||||||
@@ -129,10 +140,50 @@ jobs:
|
|||||||
mkdir -p "${context}/dockerscripts"
|
mkdir -p "${context}/dockerscripts"
|
||||||
tar -xzf "${archive}" -C "${context}" silo
|
tar -xzf "${archive}" -C "${context}" silo
|
||||||
cp Dockerfile.goreleaser Dockerfile.distroless LICENSE NOTICE CREDITS "${context}/"
|
cp Dockerfile.goreleaser Dockerfile.distroless LICENSE NOTICE CREDITS "${context}/"
|
||||||
cp dockerscripts/docker-entrypoint.sh dockerscripts/download-static-curl.sh \
|
cp dockerscripts/docker-entrypoint.sh dockerscripts/build-static-curl.sh \
|
||||||
"${context}/dockerscripts/"
|
"${context}/dockerscripts/"
|
||||||
done
|
done
|
||||||
|
|
||||||
|
# The classic image bundles mcli. Resolve its two archive digests
|
||||||
|
# from the immutable published release instead of trusting defaults
|
||||||
|
# copied into an older Server tag. This also gives a recovery run a
|
||||||
|
# narrow override when a tag selected the right mcli release but
|
||||||
|
# accidentally retained stale archive pins.
|
||||||
|
MC_REPO="$(awk -F= '/^ARG MC_REPO=/{print $2; exit}' Dockerfile.goreleaser)"
|
||||||
|
MC_VERSION="$(awk -F= '/^ARG MC_VERSION=/{print $2; exit}' Dockerfile.goreleaser)"
|
||||||
|
test -n "${MC_REPO}"
|
||||||
|
test -n "${MC_VERSION}"
|
||||||
|
MC_VERSION_HYPHEN="${MC_VERSION#RELEASE.}"
|
||||||
|
MC_PKG_VERSION="$(echo "${MC_VERSION_HYPHEN}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')"
|
||||||
|
if [ "${MC_PKG_VERSION}" = "${MC_VERSION_HYPHEN}" ]; then
|
||||||
|
echo "Invalid bundled mcli tag: ${MC_VERSION}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ "$(gh release view "${MC_VERSION}" --repo "${MC_REPO}" --json isDraft --jq .isDraft)" != false ] || \
|
||||||
|
[ "$(gh release view "${MC_VERSION}" --repo "${MC_REPO}" --json isPrerelease --jq .isPrerelease)" != false ] || \
|
||||||
|
[ "$(gh release view "${MC_VERSION}" --repo "${MC_REPO}" --json isImmutable --jq .isImmutable)" != true ]; then
|
||||||
|
echo "Bundled mcli ${MC_REPO}@${MC_VERSION} must be a published immutable release" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
mc_checksums="mcli_${MC_PKG_VERSION}_checksums.txt"
|
||||||
|
gh release download "${MC_VERSION}" --repo "${MC_REPO}" \
|
||||||
|
--dir "${assets_dir}" --pattern "${mc_checksums}"
|
||||||
|
gh attestation verify "${assets_dir}/${mc_checksums}" \
|
||||||
|
--repo "${MC_REPO}" \
|
||||||
|
--signer-workflow "${MC_REPO}/.github/workflows/release.yml" \
|
||||||
|
--source-ref "refs/tags/${MC_VERSION}" >/dev/null
|
||||||
|
|
||||||
|
MC_AMD64_SHA256="$(awk -v name="mcli_${MC_PKG_VERSION}_linux_amd64.tar.gz" '$2 == name {print $1}' "${assets_dir}/${mc_checksums}")"
|
||||||
|
MC_ARM64_SHA256="$(awk -v name="mcli_${MC_PKG_VERSION}_linux_arm64.tar.gz" '$2 == name {print $1}' "${assets_dir}/${mc_checksums}")"
|
||||||
|
[[ "${MC_AMD64_SHA256}" =~ ^[0-9a-f]{64}$ ]]
|
||||||
|
[[ "${MC_ARM64_SHA256}" =~ ^[0-9a-f]{64}$ ]]
|
||||||
|
|
||||||
|
{
|
||||||
|
echo "MC_AMD64_SHA256=${MC_AMD64_SHA256}"
|
||||||
|
echo "MC_ARM64_SHA256=${MC_ARM64_SHA256}"
|
||||||
|
} >> "${GITHUB_ENV}"
|
||||||
|
|
||||||
echo "RELEASE_REVISION=$(git rev-parse HEAD)" >> "${GITHUB_ENV}"
|
echo "RELEASE_REVISION=$(git rev-parse HEAD)" >> "${GITHUB_ENV}"
|
||||||
|
|
||||||
- name: Set up QEMU
|
- name: Set up QEMU
|
||||||
@@ -162,6 +213,9 @@ jobs:
|
|||||||
file: docker-release/amd64/Dockerfile.goreleaser
|
file: docker-release/amd64/Dockerfile.goreleaser
|
||||||
platforms: linux/amd64
|
platforms: linux/amd64
|
||||||
push: true
|
push: true
|
||||||
|
build-args: |
|
||||||
|
MC_AMD64_SHA256=${{ env.MC_AMD64_SHA256 }}
|
||||||
|
MC_ARM64_SHA256=${{ env.MC_ARM64_SHA256 }}
|
||||||
tags: |
|
tags: |
|
||||||
pgsty/silo:${{ env.RELEASE_TAG }}-amd64
|
pgsty/silo:${{ env.RELEASE_TAG }}-amd64
|
||||||
pgsty/silo:latest-amd64
|
pgsty/silo:latest-amd64
|
||||||
@@ -178,6 +232,9 @@ jobs:
|
|||||||
file: docker-release/arm64/Dockerfile.goreleaser
|
file: docker-release/arm64/Dockerfile.goreleaser
|
||||||
platforms: linux/arm64
|
platforms: linux/arm64
|
||||||
push: true
|
push: true
|
||||||
|
build-args: |
|
||||||
|
MC_AMD64_SHA256=${{ env.MC_AMD64_SHA256 }}
|
||||||
|
MC_ARM64_SHA256=${{ env.MC_ARM64_SHA256 }}
|
||||||
tags: |
|
tags: |
|
||||||
pgsty/silo:${{ env.RELEASE_TAG }}-arm64
|
pgsty/silo:${{ env.RELEASE_TAG }}-arm64
|
||||||
pgsty/silo:latest-arm64
|
pgsty/silo:latest-arm64
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ on:
|
|||||||
- "Dockerfile.distroless"
|
- "Dockerfile.distroless"
|
||||||
- "cmd/healthcheck-main.go"
|
- "cmd/healthcheck-main.go"
|
||||||
- "cmd/main.go"
|
- "cmd/main.go"
|
||||||
- "dockerscripts/download-static-curl.sh"
|
- "dockerscripts/build-static-curl.sh"
|
||||||
- "dockerscripts/docker-entrypoint.sh"
|
- "dockerscripts/docker-entrypoint.sh"
|
||||||
- "dockerscripts/docker-entrypoint_test.sh"
|
- "dockerscripts/docker-entrypoint_test.sh"
|
||||||
- "silo.service"
|
- "silo.service"
|
||||||
@@ -41,6 +41,28 @@ permissions:
|
|||||||
contents: read
|
contents: read
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
|
curl:
|
||||||
|
name: Static curl (${{ matrix.arch }})
|
||||||
|
runs-on: ${{ matrix.runner }}
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
include:
|
||||||
|
- arch: amd64
|
||||||
|
runner: ubuntu-latest
|
||||||
|
- arch: arm64
|
||||||
|
runner: ubuntu-24.04-arm
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v7
|
||||||
|
- name: Build and exercise curl in an empty runtime
|
||||||
|
run: |
|
||||||
|
docker build --build-arg TARGETARCH=${{ matrix.arch }} \
|
||||||
|
--target curl-runtime -f Dockerfile.goreleaser -t silo-curl-test .
|
||||||
|
docker run --rm silo-curl-test --version | tee curl-version.txt
|
||||||
|
grep -F 'curl 8.22.0 ' curl-version.txt
|
||||||
|
grep -F 'HTTP2' curl-version.txt
|
||||||
|
docker run --rm silo-curl-test --fail --silent --show-error \
|
||||||
|
--connect-timeout 15 --max-time 60 https://curl.se/robots.txt
|
||||||
|
|
||||||
validate:
|
validate:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
@@ -490,7 +512,7 @@ jobs:
|
|||||||
bash -n buildscripts/package/lifecycle_test.sh
|
bash -n buildscripts/package/lifecycle_test.sh
|
||||||
buildscripts/package/lifecycle_test.sh
|
buildscripts/package/lifecycle_test.sh
|
||||||
bash -n dockerscripts/docker-entrypoint_test.sh
|
bash -n dockerscripts/docker-entrypoint_test.sh
|
||||||
bash -n dockerscripts/download-static-curl.sh
|
bash -n dockerscripts/build-static-curl.sh
|
||||||
dockerscripts/docker-entrypoint_test.sh
|
dockerscripts/docker-entrypoint_test.sh
|
||||||
go run ./buildscripts/rebrand-guard
|
go run ./buildscripts/rebrand-guard
|
||||||
buildscripts/verify-rebrand.sh
|
buildscripts/verify-rebrand.sh
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ jobs:
|
|||||||
|
|
||||||
- name: Install govulncheck
|
- name: Install govulncheck
|
||||||
run: |
|
run: |
|
||||||
go install golang.org/x/vuln/cmd/govulncheck@v1.7.0
|
go install golang.org/x/vuln/cmd/govulncheck@v1.8.0
|
||||||
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
|
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
|
||||||
|
|
||||||
- name: Run govulncheck
|
- name: Run govulncheck
|
||||||
|
|||||||
+1
-4
@@ -5,7 +5,7 @@ linters:
|
|||||||
- durationcheck
|
- durationcheck
|
||||||
- forcetypeassert
|
- forcetypeassert
|
||||||
- gocritic
|
- gocritic
|
||||||
- gomodguard
|
- gomodguard_v2
|
||||||
- govet
|
- govet
|
||||||
- ineffassign
|
- ineffassign
|
||||||
- misspell
|
- misspell
|
||||||
@@ -59,9 +59,6 @@ formatters:
|
|||||||
exclusions:
|
exclusions:
|
||||||
generated: lax
|
generated: lax
|
||||||
paths:
|
paths:
|
||||||
# gofumpt v0.11.0 and Go 1.27's gofmt disagree on the indentation of
|
|
||||||
# multiple composite literals returned from a single statement.
|
|
||||||
- internal/ioutil/wait_pipe\.go$
|
|
||||||
- third_party$
|
- third_party$
|
||||||
- builtin$
|
- builtin$
|
||||||
- examples$
|
- examples$
|
||||||
|
|||||||
+104
@@ -0,0 +1,104 @@
|
|||||||
|
# Changelog
|
||||||
|
|
||||||
|
## Unreleased
|
||||||
|
|
||||||
|
The entries below describe source changes on main since the latest published Server.
|
||||||
|
**The latest published Server remains 20260903.** These changes are not in its
|
||||||
|
binaries, packages or images. See the [component matrix](https://silo.pgsty.com/compatibility/versions/)
|
||||||
|
and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-09-03T13-18-01Z...main).
|
||||||
|
|
||||||
|
### Authorization and security
|
||||||
|
|
||||||
|
- Reject unsigned `x-amz-*` request headers that could turn a signed PUT into a
|
||||||
|
copy of another object accessible to the signer (SN-2026-011). The latest
|
||||||
|
public Server is affected; the fix is on main. See [the advisory ledger](docs/security/advisories.md).
|
||||||
|
- Align signed request fields with policy conditions and enforce header-only
|
||||||
|
presigned payload checksums. See [the signed-header review](https://silo.pgsty.com/blog/design/signed-header-coverage/).
|
||||||
|
- **Breaking policy semantics:** separate self-service `admin:ChangeMyPassword`
|
||||||
|
from `admin:CreateUser`. Built-in read-only policies follow the split. Preserve
|
||||||
|
both denies if the previous combined restriction must survive upgrades or
|
||||||
|
rollback. Saved policies are not rewritten. Deploy with the matching Console
|
||||||
|
and pkg; see [the migration guide](docs/iam/password-permissions.md).
|
||||||
|
|
||||||
|
### Object storage and replication
|
||||||
|
|
||||||
|
- Evaluate conditional multipart completion against the logical current object
|
||||||
|
across all pools while holding the existing object lock. A stale `If-Match`
|
||||||
|
can no longer replace newer data in another pool, and the current ETag is no
|
||||||
|
longer rejected because the upload resides next to an older copy. Conditions
|
||||||
|
are evaluated once; a current delete marker counts as an absent object.
|
||||||
|
**Availability change:** if metadata cannot be read from any pool, conditional
|
||||||
|
completion fails even when another pool can still serve GET/HEAD. This also
|
||||||
|
applies when the unreadable pool may not hold the object: absence cannot be
|
||||||
|
verified. Retry after the pool recovers. Unconditional completion and the
|
||||||
|
single-pool path retain their existing behavior.
|
||||||
|
|
||||||
|
- Reconcile ordinary single-object version DELETE across all pools, including
|
||||||
|
null versions, delete markers and unqualified directory-marker DELETE. This
|
||||||
|
applies the deletion to every resolved pool copy under existing quorum
|
||||||
|
rules. Pending outbound delete replication retains versions until the
|
||||||
|
existing replication worker completes their purge; a successful response
|
||||||
|
does not imply immediate physical removal from every drive. Unreadable
|
||||||
|
pools now consistently return 503 instead of depending on pool traversal
|
||||||
|
order; insufficient read quorum returns `SlowDownRead`. This extends the
|
||||||
|
existing failure surface. Retry after recovery.
|
||||||
|
Cleanup failures also return an error. Batch deletion already fans out across
|
||||||
|
pools; replication and scanner cleanup keep their existing contracts. See
|
||||||
|
[scope and limitations](docs/bucket/lifecycle/access-tiering-removal.md#version-deletion-scope).
|
||||||
|
|
||||||
|
- Remove the opt-in GET-frequency pool-tiering feature from PR #60, including
|
||||||
|
its tracker, mover, scanner hooks, configuration, XML actions and metrics.
|
||||||
|
Accept and ignore retired configuration/XML and preserve ordinary statistics
|
||||||
|
when reading v9 caches. See [migration notes](docs/bucket/lifecycle/access-tiering-removal.md).
|
||||||
|
The [decision record](docs/investigations/access-tiering-revert.md) preserves
|
||||||
|
the feature's introduction, subsequent fixes, rollback scope and review history.
|
||||||
|
- Preserve the independent multi-pool write, metadata, healing and conditional
|
||||||
|
deletion fixes from PR #178, including shared remote-tier reference protection.
|
||||||
|
- Enforce `If-Match` on DELETE, preserve retention and independently ordered
|
||||||
|
Object Lock/tag updates, and correctly retransmit encrypted replicas.
|
||||||
|
- Preserve plaintext part sizes and raw SSE-C replicas; prevent SSE-C
|
||||||
|
compression, honor key-rotation checksums, and complete attributes pagination.
|
||||||
|
- Repair federated CopyObject checksums, destination timestamps, reserved
|
||||||
|
metadata, encrypted-object forwarding, legal hold and KMS context.
|
||||||
|
- Make resync counters, target selection, cancellation and worker lifetimes
|
||||||
|
reflect actual work; complete delete-marker purges and report bounded MRF drops.
|
||||||
|
- Converge bucket metadata with deterministic source state, deletion tombstones,
|
||||||
|
creation time recovery and diagnostics. The mixed-version export gate requires
|
||||||
|
coordinated upgrades before tombstones are exported. See [the #77 record](docs/investigations/issue-77-current.md).
|
||||||
|
- Include per-bucket CORS in metadata export/import, close metadata publication
|
||||||
|
and logger races, and report effective bucket quotas in metrics.
|
||||||
|
|
||||||
|
### Console, dependencies and delivery
|
||||||
|
|
||||||
|
- Restore embedded Console login over loopback TLS, trusted-proxy handling and
|
||||||
|
all four WebSocket connection limits. Preserve Go TLS defaults across transports.
|
||||||
|
- Directly require `github.com/pgsty/silo-pkg/v3` v3.14.0; select Console
|
||||||
|
`v0.0.0-20260913015128-417559bb2c97` and MC
|
||||||
|
`v0.0.0-20260913012246-4f609a4da3bb` with explicit PGSTY replacements.
|
||||||
|
- Pin upstream minio-go `v7.3.1-0.20260910142817-60bd07042d49`; refresh Go x/*
|
||||||
|
modules and security fixes including bounded AMQP frame handling. Keep Go
|
||||||
|
1.27.1 and go-systemd v22.6.0's NetBSD compatibility replacement.
|
||||||
|
- Refresh container base digests and build static curl 8.22.0 from verified
|
||||||
|
source for both Linux architectures. Pin the actual mcli 20260913 archives and
|
||||||
|
hashes. Helm's client image follows that release; its Server image still names
|
||||||
|
the latest published Server 20260903.
|
||||||
|
|
||||||
|
The dependency update passed the final candidate's Go, vulnerability and Test
|
||||||
|
Release workflows; native curl builds passed on both architectures. A local
|
||||||
|
ARM64 image passed startup, health, S3 transfer and embedded Console checks.
|
||||||
|
These checks do not publish a Server tag or production image and do not replace
|
||||||
|
cluster upgrade/rollback acceptance for the next release. Dated investigations
|
||||||
|
retain the exact source and runtime boundaries they tested.
|
||||||
|
|
||||||
|
## RELEASE.2026-09-03T13-18-01Z
|
||||||
|
|
||||||
|
Published source: `9b11dc9469e650815b775cb47b039610644f5da4`.
|
||||||
|
[Complete release notes](https://silo.pgsty.com/blog/release/silo-20260903/) ·
|
||||||
|
[GitHub release](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z)
|
||||||
|
|
||||||
|
This release ships Go 1.27.1, silo-pkg v3.13.2, upstream minio-go `0e78d3f18efe`,
|
||||||
|
mcli 20260903 and embedded Console source `464a59d73ada` (v2.3.0 version identity).
|
||||||
|
Installing the newer standalone mcli or Console does not replace components
|
||||||
|
inside this existing Server binary or image.
|
||||||
|
|
||||||
|
Earlier releases: [release archive](https://github.com/pgsty/silo/releases).
|
||||||
+18
-8
@@ -44,6 +44,10 @@ Before opening a pull request:
|
|||||||
|
|
||||||
- Add or update tests for changed behavior.
|
- Add or update tests for changed behavior.
|
||||||
- Run `make verifiers`.
|
- Run `make verifiers`.
|
||||||
|
- If `make rebrand-guard` reports a changed compatibility set, review the
|
||||||
|
listed identifiers; when the change is intended, refresh the baseline with
|
||||||
|
`go run ./buildscripts/rebrand-guard --write` and commit
|
||||||
|
`buildscripts/rebrand-guard/compat-baseline.json`.
|
||||||
- Run the smallest relevant package tests, then `make test` when practical.
|
- Run the smallest relevant package tests, then `make test` when practical.
|
||||||
- Run `make build` and confirm the generated executable is `silo`.
|
- Run `make build` and confirm the generated executable is `silo`.
|
||||||
- Explain any preserved `MINIO_*`, `minio_*`, `x-minio-*`, `/minio/*`,
|
- Explain any preserved `MINIO_*`, `minio_*`, `x-minio-*`, `/minio/*`,
|
||||||
@@ -75,9 +79,10 @@ documentation is owned by the separate
|
|||||||
|
|
||||||
## Licensing of Contributions
|
## Licensing of Contributions
|
||||||
|
|
||||||
Silo is licensed under the [GNU AGPL v3.0 or later](LICENSE). Its core is
|
Code contributions to PGSTY SILO (`pgsty/silo`) are accepted under the
|
||||||
Copyright (c) MinIO, Inc.; the combined work can never be relicensed, and this
|
[GNU AGPL v3.0 or later](LICENSE), the same license as the server. Submit issues
|
||||||
fork does not try to.
|
and pull requests to this repository's maintainers. No separate Apache-2.0
|
||||||
|
license grant to SILO or upstream MinIO maintainers is required.
|
||||||
|
|
||||||
* **No CLA.** We do not ask you to sign a Contributor License Agreement and we
|
* **No CLA.** We do not ask you to sign a Contributor License Agreement and we
|
||||||
do not take your copyright. Contributions are accepted inbound=outbound: you
|
do not take your copyright. Contributions are accepted inbound=outbound: you
|
||||||
@@ -108,15 +113,20 @@ fork does not try to.
|
|||||||
`Signed-off-by` trailers) and add your own sign-off as the person passing it
|
`Signed-off-by` trailers) and add your own sign-off as the person passing it
|
||||||
along. Never import code from a proprietary distribution.
|
along. Never import code from a proprietary distribution.
|
||||||
|
|
||||||
* **File headers.** Files derived from upstream keep the original MinIO
|
* **File headers.** Preserve existing copyright and license notices in inherited
|
||||||
copyright header unchanged. New files added by this fork use the dual
|
and third-party files. New original files name their actual copyright holders
|
||||||
header, followed by the standard AGPL boilerplate:
|
and use AGPL-3.0-or-later. Use a header such as the following, then append the
|
||||||
|
standard AGPL boilerplate:
|
||||||
|
|
||||||
```
|
```
|
||||||
// Copyright (c) 2015-2025 MinIO, Inc.
|
// Copyright (c) 2026 Your Name
|
||||||
// Copyright (c) 2025-2026 PGSTY
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
* **Separately licensed material.** Documentation contributions in `docs/`
|
||||||
|
follow its existing [CC BY 4.0 license](docs/LICENSE). Third-party components
|
||||||
|
and earlier Apache-2.0 contributions retain their original licenses and
|
||||||
|
attribution; this policy does not relicense earlier work.
|
||||||
|
|
||||||
* **Squash merges** must keep the `Signed-off-by:` trailers in the resulting
|
* **Squash merges** must keep the `Signed-off-by:` trailers in the resulting
|
||||||
commit message.
|
commit message.
|
||||||
|
|
||||||
|
|||||||
+170
-63
File diff suppressed because one or more lines are too long
+19
-11
@@ -1,4 +1,15 @@
|
|||||||
FROM golang:1.27.0-alpine AS build
|
FROM golang:1.27.1-alpine@sha256:cf6fca6641884b8433441b2b0652976f975e1d0fdd26d177eaaf8596087f3125 AS curl-build
|
||||||
|
ARG TARGETARCH
|
||||||
|
COPY dockerscripts/build-static-curl.sh /build/build-static-curl
|
||||||
|
RUN /bin/sh /build/build-static-curl
|
||||||
|
|
||||||
|
# Exercise the exact shipped curl without a dynamic loader or shared libraries.
|
||||||
|
FROM scratch AS curl-runtime
|
||||||
|
COPY --from=curl-build /go/bin/curl /curl
|
||||||
|
COPY --from=curl-build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||||
|
ENTRYPOINT ["/curl"]
|
||||||
|
|
||||||
|
FROM golang:1.27.1-alpine@sha256:cf6fca6641884b8433441b2b0652976f975e1d0fdd26d177eaaf8596087f3125 AS build
|
||||||
|
|
||||||
ARG TARGETARCH
|
ARG TARGETARCH
|
||||||
|
|
||||||
@@ -6,9 +17,9 @@ ENV GOPATH=/go
|
|||||||
ENV CGO_ENABLED=0
|
ENV CGO_ENABLED=0
|
||||||
|
|
||||||
ARG MC_REPO=pgsty/mc
|
ARG MC_REPO=pgsty/mc
|
||||||
ARG MC_VERSION=RELEASE.2026-08-06T00-00-00Z
|
ARG MC_VERSION=RELEASE.2026-09-13T00-00-00Z
|
||||||
ARG MC_AMD64_SHA256=4b488bd30af54ad4214e5b654746677c79cd93dc6cad4be3aa2d09dbb48370ff
|
ARG MC_AMD64_SHA256=9d2a92de9c7b887d9b944fe9ddce68d23f1b6df3415092e737594e56593f5e2b
|
||||||
ARG MC_ARM64_SHA256=83f6fedb16ed9c1e8efa8aea6776203dff132bc474214543d5c0767ed2066c2f
|
ARG MC_ARM64_SHA256=3d82e9ea6c601c4cb44fe5dd5f2ad1b7d7d64369378110f9ada9c524688a452a
|
||||||
|
|
||||||
RUN apk add -U --no-cache \
|
RUN apk add -U --no-cache \
|
||||||
ca-certificates \
|
ca-certificates \
|
||||||
@@ -56,18 +67,14 @@ RUN apk add -U --no-cache \
|
|||||||
chmod +x /go/bin/mcli && \
|
chmod +x /go/bin/mcli && \
|
||||||
ln -sf mcli /go/bin/mc
|
ln -sf mcli /go/bin/mc
|
||||||
|
|
||||||
COPY dockerscripts/download-static-curl.sh /build/download-static-curl
|
FROM registry.access.redhat.com/ubi9/ubi:latest@sha256:206b65b8ee0f04b992818c9a51b29081b14974630d4850bc358097d0c44ea156 AS certs
|
||||||
RUN chmod +x /build/download-static-curl && \
|
|
||||||
/build/download-static-curl
|
|
||||||
|
|
||||||
FROM registry.access.redhat.com/ubi9/ubi:latest AS certs
|
|
||||||
RUN dnf -y install ca-certificates && \
|
RUN dnf -y install ca-certificates && \
|
||||||
update-ca-trust && \
|
update-ca-trust && \
|
||||||
cp /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /tmp/ca-certificates.crt && \
|
cp /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /tmp/ca-certificates.crt && \
|
||||||
dnf clean all && \
|
dnf clean all && \
|
||||||
rm -rf /var/cache/dnf
|
rm -rf /var/cache/dnf
|
||||||
|
|
||||||
FROM registry.access.redhat.com/ubi9/ubi-micro:latest
|
FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:f332c99eb8f798a8486821c91937f10ad64ee83d7e739303be2df051040918f6
|
||||||
|
|
||||||
LABEL org.opencontainers.image.title="Silo" \
|
LABEL org.opencontainers.image.title="Silo" \
|
||||||
org.opencontainers.image.description="S3-Interface Libre Object Storage" \
|
org.opencontainers.image.description="S3-Interface Libre Object Storage" \
|
||||||
@@ -88,7 +95,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
|||||||
COPY --from=certs /tmp/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
COPY --from=certs /tmp/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||||
COPY silo /usr/bin/silo
|
COPY silo /usr/bin/silo
|
||||||
COPY --from=build /go/bin/mcli /usr/bin/mcli
|
COPY --from=build /go/bin/mcli /usr/bin/mcli
|
||||||
COPY --from=build /go/bin/curl* /usr/bin/
|
COPY --from=curl-build /go/bin/curl /usr/bin/curl
|
||||||
|
COPY --from=curl-build /go/share/curl /licenses/curl
|
||||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||||
COPY LICENSE /licenses/LICENSE
|
COPY LICENSE /licenses/LICENSE
|
||||||
COPY NOTICE /licenses/NOTICE
|
COPY NOTICE /licenses/NOTICE
|
||||||
|
|||||||
@@ -216,7 +216,7 @@ docker: checks build-debugging ## builds the local Linux Silo container image
|
|||||||
--ldflags "$(LDFLAGS)" -o "$$context/silo"; \
|
--ldflags "$(LDFLAGS)" -o "$$context/silo"; \
|
||||||
mkdir -p "$$context/dockerscripts"; \
|
mkdir -p "$$context/dockerscripts"; \
|
||||||
cp Dockerfile.goreleaser LICENSE NOTICE CREDITS "$$context/"; \
|
cp Dockerfile.goreleaser LICENSE NOTICE CREDITS "$$context/"; \
|
||||||
cp dockerscripts/docker-entrypoint.sh dockerscripts/download-static-curl.sh \
|
cp dockerscripts/docker-entrypoint.sh dockerscripts/build-static-curl.sh \
|
||||||
"$$context/dockerscripts/"; \
|
"$$context/dockerscripts/"; \
|
||||||
docker build -q --no-cache --platform linux/$(GOARCH) -t $(TAG) --build-arg TARGETARCH=$(GOARCH) \
|
docker build -q --no-cache --platform linux/$(GOARCH) -t $(TAG) --build-arg TARGETARCH=$(GOARCH) \
|
||||||
-f "$$context/Dockerfile.goreleaser" "$$context"
|
-f "$$context/Dockerfile.goreleaser" "$$context"
|
||||||
|
|||||||
@@ -35,6 +35,14 @@
|
|||||||
> [!NOTE]
|
> [!NOTE]
|
||||||
> Renamed from `pgsty/minio` to `pgsty/silo`, default branch `master` → `main`, on 2026-08-06. Artifacts under the original MinIO identity stay published on the archived [`minio`](https://github.com/pgsty/silo/tree/minio) branch and in releases up to [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z).
|
> Renamed from `pgsty/minio` to `pgsty/silo`, default branch `master` → `main`, on 2026-08-06. Artifacts under the original MinIO identity stay published on the archived [`minio`](https://github.com/pgsty/silo/tree/minio) branch and in releases up to [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z).
|
||||||
|
|
||||||
|
## Current release and main branch
|
||||||
|
|
||||||
|
The latest published Server is [20260903](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z).
|
||||||
|
As of 2026-09-13, the main branch has newer security, storage, Console and
|
||||||
|
shared-package changes that have not shipped in a Server release. See
|
||||||
|
[CHANGELOG.md](CHANGELOG.md) and the [component version matrix](https://silo.pgsty.com/compatibility/versions/)
|
||||||
|
for the exact release/source boundary, including SN-2026-011 and password-policy migration.
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
PGSTY SILO keeps one maintained release line of the open-source MinIO server alive after upstream ended community distribution: builds, packages, multi-arch images, security fixes, and the full web console. Pigsty runs it in production as its PostgreSQL backup repository.
|
PGSTY SILO keeps one maintained release line of the open-source MinIO server alive after upstream ended community distribution: builds, packages, multi-arch images, security fixes, and the full web console. Pigsty runs it in production as its PostgreSQL backup repository.
|
||||||
@@ -89,57 +97,79 @@ The S3 API, `MINIO_*` variables, `minio_*` metrics, `x-minio-*` headers, `/minio
|
|||||||
|
|
||||||
Every divergence from upstream is listed in the code-verified [compatibility audit](https://silo.pgsty.com/compatibility/server/). Treat each release as a downstream upgrade: pin versions, read the [release notes](https://silo.pgsty.com/tags/silo/), and keep a rollback path.
|
Every divergence from upstream is listed in the code-verified [compatibility audit](https://silo.pgsty.com/compatibility/server/). Treat each release as a downstream upgrade: pin versions, read the [release notes](https://silo.pgsty.com/tags/silo/), and keep a rollback path.
|
||||||
|
|
||||||
|
### TLS and Go upgrades
|
||||||
|
|
||||||
|
TLS key exchange follows Go's defaults across the S3 listener, node links,
|
||||||
|
replication, identity providers, etcd, and external HTTP services. If an endpoint
|
||||||
|
cannot accept ML-KEM, `GODEBUG=tlsmlkem=0` disables the default hybrid exchanges
|
||||||
|
for the process; certificate verification remains enabled. This option does not
|
||||||
|
disable ML-DSA signatures or resolve every TLS reset. Prefer updating the
|
||||||
|
incompatible endpoint before removing the temporary setting.
|
||||||
|
If only the new SecP hybrids cause problems, `GODEBUG=tlssecpmlkem=0` disables
|
||||||
|
those groups while retaining X25519MLKEM768.
|
||||||
|
|
||||||
|
For builds targeting Go 1.27, setting either `SSL_CERT_FILE` or `SSL_CERT_DIR`
|
||||||
|
on macOS replaces Keychain trust with on-disk roots and Go's verifier. Stale or
|
||||||
|
incomplete CA paths can break previously trusted connections; unset inherited
|
||||||
|
values to restore Keychain trust. Explicit certificates in the configured `CAs`
|
||||||
|
directory remain additive to the selected root pool.
|
||||||
|
Go 1.27 binaries require macOS 13 or later. See the
|
||||||
|
[Go release notes](https://go.dev/doc/go1.27) and the
|
||||||
|
[SILO stack investigation](docs/investigations/go127-stack.md).
|
||||||
|
|
||||||
## Security & Contributing
|
## Security & Contributing
|
||||||
|
|
||||||
Report vulnerabilities privately as described in [`SECURITY.md`](SECURITY.md); every fix ships with a public [advisory](https://silo.pgsty.com/blog/security/). Contributions are accepted inbound=outbound under AGPL-3.0-or-later with no CLA — only DCO sign-off (`git commit -s`) is required; see [`CONTRIBUTING.md`](CONTRIBUTING.md).
|
Report vulnerabilities privately as described in [`SECURITY.md`](SECURITY.md); every fix ships with a public [advisory](https://silo.pgsty.com/blog/security/). Contributions are accepted inbound=outbound under AGPL-3.0-or-later with no CLA — only DCO sign-off (`git commit -s`) is required; see [`CONTRIBUTING.md`](CONTRIBUTING.md).
|
||||||
|
|
||||||
## Contributors
|
## Contributors
|
||||||
|
|
||||||
<table>
|
**41 community contributors** build SILO, Console, mcli, shared packages, and related projects. The list includes maintainers and every human Issue or PR author, ordered by merged PRs, other PRs, then issue reports. Gold rings highlight significant contributions.
|
||||||
<tr>
|
|
||||||
<td align="center" width="150">
|
<p align="center">
|
||||||
<a href="https://github.com/ZouhairCharef"><img src="https://github.com/ZouhairCharef.png?size=100" width="72" alt="ZouhairCharef"><br><sub><b>@ZouhairCharef</b></sub></a><br><sub>CVE-2026-34986</sub>
|
<a href="https://github.com/Vonng"><img src="https://silo.pgsty.com/images/contributors/Vonng.svg" width="60" height="60" alt="@Vonng" title="@Vonng — Maintains SILO, Console, mcli, shared packages, releases, and documentation"></a>
|
||||||
</td>
|
<a href="https://github.com/h5vx"><img src="https://silo.pgsty.com/images/contributors/h5vx.svg" width="60" height="60" alt="@h5vx" title="@h5vx — Implemented per-bucket CORS configuration and enforcement"></a>
|
||||||
<td align="center" width="150">
|
<a href="https://github.com/mrjavadseydi"><img src="https://silo.pgsty.com/images/contributors/mrjavadseydi.svg" width="60" height="60" alt="@mrjavadseydi" title="@mrjavadseydi — Fixed effective bucket quota metrics; proposed access-frequency ILM"></a>
|
||||||
<a href="https://github.com/mfredenhagen"><img src="https://github.com/mfredenhagen.png?size=100" width="72" alt="mfredenhagen"><br><sub><b>@mfredenhagen</b></sub></a><br><sub>CVE-2026-39883</sub>
|
<a href="https://github.com/Dansyuqri"><img src="https://silo.pgsty.com/images/contributors/Dansyuqri.svg" width="60" height="60" alt="@Dansyuqri" title="@Dansyuqri — Added ChecksumType to multipart completion responses"></a>
|
||||||
</td>
|
<a href="https://github.com/ycjlin"><img src="https://silo.pgsty.com/images/contributors/ycjlin.svg" width="60" height="60" alt="@ycjlin" title="@ycjlin — Fixed missing-bucket ListObjects semantics"></a>
|
||||||
<td align="center" width="150">
|
<a href="https://github.com/pinginfo"><img src="https://silo.pgsty.com/images/contributors/pinginfo.svg" width="60" height="60" alt="@pinginfo" title="@pinginfo — Repaired bucket notification streaming"></a>
|
||||||
<a href="https://github.com/pinginfo"><img src="https://github.com/pinginfo.png?size=100" width="72" alt="pinginfo"><br><sub><b>@pinginfo</b></sub></a><br><sub>Notification streaming</sub>
|
<a href="https://github.com/ZouhairCharef"><img src="https://silo.pgsty.com/images/contributors/ZouhairCharef.svg" width="60" height="60" alt="@ZouhairCharef" title="@ZouhairCharef — Patched CVE-2026-34986 in go-jose"></a>
|
||||||
</td>
|
<a href="https://github.com/mfredenhagen"><img src="https://silo.pgsty.com/images/contributors/mfredenhagen.svg" width="60" height="60" alt="@mfredenhagen" title="@mfredenhagen — Patched CVE-2026-39883 in OpenTelemetry"></a>
|
||||||
<td align="center" width="150">
|
<a href="https://github.com/waterkip"><img src="https://silo.pgsty.com/images/contributors/waterkip.svg" width="60" height="60" alt="@waterkip" title="@waterkip — Repointed documentation links to the SILO portal"></a>
|
||||||
<a href="https://github.com/waterkip"><img src="https://github.com/waterkip.png?size=100" width="72" alt="waterkip"><br><sub><b>@waterkip</b></sub></a><br><sub>Documentation links</sub>
|
<a href="https://github.com/mikemikimike"><img src="https://silo.pgsty.com/images/contributors/mikemikimike.svg" width="60" height="60" alt="@mikemikimike" title="@mikemikimike — Contributed the replicated SSE-C plaintext part-size fix"></a>
|
||||||
</td>
|
<a href="https://github.com/metaneutrons"><img src="https://silo.pgsty.com/images/contributors/metaneutrons.svg" width="60" height="60" alt="@metaneutrons" title="@metaneutrons — Reported and proposed explicit-version delete authorization"></a>
|
||||||
</tr>
|
<a href="https://github.com/magicxor"><img src="https://silo.pgsty.com/images/contributors/magicxor.svg" width="60" height="60" alt="@magicxor" title="@magicxor — Reported and proposed conditional DELETE support for If-Match"></a>
|
||||||
</table>
|
<a href="https://github.com/davinkevin"><img src="https://silo.pgsty.com/images/contributors/davinkevin.svg" width="60" height="60" alt="@davinkevin" title="@davinkevin — Proposed the distroless container image and dependency automation"></a>
|
||||||
<p>
|
<a href="https://github.com/lem21h"><img src="https://silo.pgsty.com/images/contributors/lem21h.svg" width="48" height="48" alt="@lem21h" title="@lem21h — Proposed robustness and goroutine improvements"></a>
|
||||||
<a href="https://github.com/magicxor"><img src="https://github.com/magicxor.png?size=64" width="44" alt="magicxor" title="@magicxor"></a>
|
<a href="https://github.com/sulin37392"><img src="https://silo.pgsty.com/images/contributors/sulin37392.svg" width="48" height="48" alt="@sulin37392" title="@sulin37392 — Proposed dependency updates"></a>
|
||||||
<a href="https://github.com/ycjlin"><img src="https://github.com/ycjlin.png?size=64" width="44" alt="ycjlin" title="@ycjlin"></a>
|
<a href="https://github.com/cbornet"><img src="https://silo.pgsty.com/images/contributors/cbornet.svg" width="60" height="60" alt="@cbornet" title="@cbornet — Reported multipart and streaming checksum defects and missing-bucket semantics"></a>
|
||||||
<a href="https://github.com/davinkevin"><img src="https://github.com/davinkevin.png?size=64" width="44" alt="davinkevin" title="@davinkevin"></a>
|
<a href="https://github.com/vampywiz17"><img src="https://silo.pgsty.com/images/contributors/vampywiz17.svg" width="60" height="60" alt="@vampywiz17" title="@vampywiz17 — Reported LDAP TLS and Console login regressions"></a>
|
||||||
<a href="https://github.com/lem21h"><img src="https://github.com/lem21h.png?size=64" width="44" alt="lem21h" title="@lem21h"></a>
|
<a href="https://github.com/orenyomtov"><img src="https://silo.pgsty.com/images/contributors/orenyomtov.svg" width="60" height="60" alt="@orenyomtov" title="@orenyomtov — Reported the unsigned-header CopyObject cross-object read (SN-2026-011)"></a>
|
||||||
<a href="https://github.com/sulin37392"><img src="https://github.com/sulin37392.png?size=64" width="44" alt="sulin37392" title="@sulin37392"></a>
|
<a href="https://github.com/mumu-lab"><img src="https://silo.pgsty.com/images/contributors/mumu-lab.svg" width="48" height="48" alt="@mumu-lab" title="@mumu-lab — Reported bucket quota metrics reading a deprecated field"></a>
|
||||||
<a href="https://github.com/mosesdd"><img src="https://github.com/mosesdd.png?size=64" width="44" alt="mosesdd" title="@mosesdd"></a>
|
<a href="https://github.com/jvasile"><img src="https://silo.pgsty.com/images/contributors/jvasile.svg" width="48" height="48" alt="@jvasile" title="@jvasile — Reported missing user, group, and defaults in Debian packages"></a>
|
||||||
<a href="https://github.com/Xavier-777"><img src="https://github.com/Xavier-777.png?size=64" width="44" alt="Xavier-777" title="@Xavier-777"></a>
|
<a href="https://github.com/pmezhuev"><img src="https://silo.pgsty.com/images/contributors/pmezhuev.svg" width="48" height="48" alt="@pmezhuev" title="@pmezhuev — Reported missing RPM package signatures"></a>
|
||||||
<a href="https://github.com/jiadzh"><img src="https://github.com/jiadzh.png?size=64" width="44" alt="jiadzh" title="@jiadzh"></a>
|
<a href="https://github.com/TLINDEN"><img src="https://silo.pgsty.com/images/contributors/TLINDEN.svg" width="48" height="48" alt="@TLINDEN" title="@TLINDEN — Reported the missing client in release tarballs"></a>
|
||||||
<a href="https://github.com/TLINDEN"><img src="https://github.com/TLINDEN.png?size=64" width="44" alt="TLINDEN" title="@TLINDEN"></a>
|
<a href="https://github.com/makinikm"><img src="https://silo.pgsty.com/images/contributors/makinikm.svg" width="48" height="48" alt="@makinikm" title="@makinikm — Reported the missing client in the container image"></a>
|
||||||
<a href="https://github.com/AntonOfTheWoods"><img src="https://github.com/AntonOfTheWoods.png?size=64" width="44" alt="AntonOfTheWoods" title="@AntonOfTheWoods"></a>
|
<a href="https://github.com/meesudzu"><img src="https://silo.pgsty.com/images/contributors/meesudzu.svg" width="48" height="48" alt="@meesudzu" title="@meesudzu — Requested the migration guide from upstream MinIO"></a>
|
||||||
<a href="https://github.com/zylpsrs"><img src="https://github.com/zylpsrs.png?size=64" width="44" alt="zylpsrs" title="@zylpsrs"></a>
|
<a href="https://github.com/kuldeep-link11"><img src="https://silo.pgsty.com/images/contributors/kuldeep-link11.svg" width="48" height="48" alt="@kuldeep-link11" title="@kuldeep-link11 — Reported NATS JWT credentials and target reload issues"></a>
|
||||||
<a href="https://github.com/nsanitate"><img src="https://github.com/nsanitate.png?size=64" width="44" alt="nsanitate" title="@nsanitate"></a>
|
<a href="https://github.com/sargarass"><img src="https://silo.pgsty.com/images/contributors/sargarass.svg" width="48" height="48" alt="@sargarass" title="@sargarass — Reported ListMultipartUploads prefix and pagination semantics"></a>
|
||||||
<a href="https://github.com/makinikm"><img src="https://github.com/makinikm.png?size=64" width="44" alt="makinikm" title="@makinikm"></a>
|
<a href="https://github.com/liuhaodongliu990-cmyk"><img src="https://silo.pgsty.com/images/contributors/liuhaodongliu990-cmyk.svg" width="48" height="48" alt="@liuhaodongliu990-cmyk" title="@liuhaodongliu990-cmyk — Reported indeterminate progress for prefix downloads"></a>
|
||||||
<a href="https://github.com/spaceg00se-r"><img src="https://github.com/spaceg00se-r.png?size=64" width="44" alt="spaceg00se-r" title="@spaceg00se-r"></a>
|
<a href="https://github.com/Xavier-777"><img src="https://silo.pgsty.com/images/contributors/Xavier-777.svg" width="48" height="48" alt="@Xavier-777" title="@Xavier-777 — Reported Console lifecycle management and file preview gaps"></a>
|
||||||
<a href="https://github.com/heroes1412"><img src="https://github.com/heroes1412.png?size=64" width="44" alt="heroes1412" title="@heroes1412"></a>
|
<a href="https://github.com/spaceg00se-r"><img src="https://silo.pgsty.com/images/contributors/spaceg00se-r.svg" width="48" height="48" alt="@spaceg00se-r" title="@spaceg00se-r — Requested cpuv1 support and reported a workflow token failure"></a>
|
||||||
<a href="https://github.com/vampywiz17"><img src="https://github.com/vampywiz17.png?size=64" width="44" alt="vampywiz17" title="@vampywiz17"></a>
|
<a href="https://github.com/kh0mka"><img src="https://silo.pgsty.com/images/contributors/kh0mka.svg" width="48" height="48" alt="@kh0mka" title="@kh0mka — Reported inter-node I/O timeouts in ReadFileStreamHandler"></a>
|
||||||
<a href="https://github.com/chalukyaj"><img src="https://github.com/chalukyaj.png?size=64" width="44" alt="chalukyaj" title="@chalukyaj"></a>
|
<a href="https://github.com/bagutzu"><img src="https://silo.pgsty.com/images/contributors/bagutzu.svg" width="48" height="48" alt="@bagutzu" title="@bagutzu — Requested KES-compatible external KMS and OpenBao support"></a>
|
||||||
<a href="https://github.com/cbornet"><img src="https://github.com/cbornet.png?size=64" width="44" alt="cbornet" title="@cbornet"></a>
|
<a href="https://github.com/DestroyLee"><img src="https://silo.pgsty.com/images/contributors/DestroyLee.svg" width="48" height="48" alt="@DestroyLee" title="@DestroyLee — Reported the missing documentation navigation"></a>
|
||||||
<a href="https://github.com/jvasile"><img src="https://github.com/jvasile.png?size=64" width="44" alt="jvasile" title="@jvasile"></a>
|
<a href="https://github.com/mosesdd"><img src="https://silo.pgsty.com/images/contributors/mosesdd.svg" width="48" height="48" alt="@mosesdd" title="@mosesdd — Requested a maintained Helm chart"></a>
|
||||||
<a href="https://github.com/Kesavaambati"><img src="https://github.com/Kesavaambati.png?size=64" width="44" alt="Kesavaambati" title="@Kesavaambati"></a>
|
<a href="https://github.com/zylpsrs"><img src="https://silo.pgsty.com/images/contributors/zylpsrs.svg" width="48" height="48" alt="@zylpsrs" title="@zylpsrs — Reported missing Console tiering and site replication"></a>
|
||||||
<a href="https://github.com/redfoxfox"><img src="https://github.com/redfoxfox.png?size=64" width="44" alt="redfoxfox" title="@redfoxfox"></a>
|
<a href="https://github.com/heroes1412"><img src="https://silo.pgsty.com/images/contributors/heroes1412.svg" width="48" height="48" alt="@heroes1412" title="@heroes1412 — Reported the unusable profiling option"></a>
|
||||||
<a href="https://github.com/kuldeep-link11"><img src="https://github.com/kuldeep-link11.png?size=64" width="44" alt="kuldeep-link11" title="@kuldeep-link11"></a>
|
<a href="https://github.com/redfoxfox"><img src="https://silo.pgsty.com/images/contributors/redfoxfox.svg" width="48" height="48" alt="@redfoxfox" title="@redfoxfox — Reported Chinese documentation availability"></a>
|
||||||
<a href="https://github.com/meesudzu"><img src="https://github.com/meesudzu.png?size=64" width="44" alt="meesudzu" title="@meesudzu"></a>
|
<a href="https://github.com/jiadzh"><img src="https://silo.pgsty.com/images/contributors/jiadzh.svg" width="48" height="48" alt="@jiadzh" title="@jiadzh — Requested Windows build guidance"></a>
|
||||||
<a href="https://github.com/pmezhuev"><img src="https://github.com/pmezhuev.png?size=64" width="44" alt="pmezhuev" title="@pmezhuev"></a>
|
<a href="https://github.com/AntonOfTheWoods"><img src="https://silo.pgsty.com/images/contributors/AntonOfTheWoods.svg" width="48" height="48" alt="@AntonOfTheWoods" title="@AntonOfTheWoods — Asked for clarity on Helm chart and operator options"></a>
|
||||||
<a href="https://github.com/kh0mka"><img src="https://github.com/kh0mka.png?size=64" width="44" alt="kh0mka" title="@kh0mka"></a>
|
<a href="https://github.com/chalukyaj"><img src="https://silo.pgsty.com/images/contributors/chalukyaj.svg" width="48" height="48" alt="@chalukyaj" title="@chalukyaj — Proposed making the SILO Operator easier to discover"></a>
|
||||||
|
<a href="https://github.com/nsanitate"><img src="https://silo.pgsty.com/images/contributors/nsanitate.svg" width="48" height="48" alt="@nsanitate" title="@nsanitate — Proposed CNCF Sandbox governance"></a>
|
||||||
|
<a href="https://github.com/Kesavaambati"><img src="https://silo.pgsty.com/images/contributors/Kesavaambati.svg" width="48" height="48" alt="@Kesavaambati" title="@Kesavaambati — Asked about community support and image maintenance"></a>
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
GitHub does not generate a contributor graph for forks, so [`CONTRIBUTORS.md`](CONTRIBUTORS.md) — not the Insights page — is this project's attribution record. It names everyone alongside the change or report they contributed.
|
[View the full contribution record](CONTRIBUTORS.md) for each person's proposals, fixes, and reports.
|
||||||
|
|
||||||
## Background
|
## Background
|
||||||
|
|
||||||
|
|||||||
+52
-43
@@ -35,6 +35,13 @@
|
|||||||
> [!NOTE]
|
> [!NOTE]
|
||||||
> 2026-08-06,本仓库由 `pgsty/minio` 更名为 `pgsty/silo`,默认分支由 `master` 更名为 `main`。以原 MinIO 形态维持的归档构件仍位于归档的 [`minio`](https://github.com/pgsty/silo/tree/minio) 分支,以及截止 [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z) 的历次发布中。
|
> 2026-08-06,本仓库由 `pgsty/minio` 更名为 `pgsty/silo`,默认分支由 `master` 更名为 `main`。以原 MinIO 形态维持的归档构件仍位于归档的 [`minio`](https://github.com/pgsty/silo/tree/minio) 分支,以及截止 [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z) 的历次发布中。
|
||||||
|
|
||||||
|
## 当前发行版与主分支
|
||||||
|
|
||||||
|
最新已发布的 Server 仍为 [20260903](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z)。
|
||||||
|
截至 2026-09-13,主分支已合入更新的安全、存储、Console 与共享包改动,但尚未发布新 Server。
|
||||||
|
准确的已发布/源码边界见 [CHANGELOG.md](CHANGELOG.md) 与[组件版本矩阵](https://silo.pgsty.com/zh/compatibility/versions/),
|
||||||
|
其中包括 SN-2026-011 修复状态与密码权限迁移要求。
|
||||||
|
|
||||||
## 概述
|
## 概述
|
||||||
|
|
||||||
上游停止社区发行后,Silo 为开源 MinIO 服务端维护一条持续可用的版本线:构建、软件包、多架构镜像、安全修复与完整 Web 控制台。Pigsty 在生产环境中用它承载 PostgreSQL 备份存储。
|
上游停止社区发行后,Silo 为开源 MinIO 服务端维护一条持续可用的版本线:构建、软件包、多架构镜像、安全修复与完整 Web 控制台。Pigsty 在生产环境中用它承载 PostgreSQL 备份存储。
|
||||||
@@ -95,51 +102,53 @@ S3 API、`MINIO_*` 环境变量、`minio_*` 指标、`x-minio-*` 头、`/minio/*
|
|||||||
|
|
||||||
## 贡献者
|
## 贡献者
|
||||||
|
|
||||||
<table>
|
**41 位社区贡献者**共同建设 SILO、Console、mcli、公共包与相关项目。名单包含维护者,以及所有提出 Issue 或 PR 的真人作者;按已合并 PR、其他 PR、Issue 报告排序,黄圈标记显著贡献。
|
||||||
<tr>
|
|
||||||
<td align="center" width="150">
|
<p align="center">
|
||||||
<a href="https://github.com/ZouhairCharef"><img src="https://github.com/ZouhairCharef.png?size=100" width="72" alt="ZouhairCharef"><br><sub><b>@ZouhairCharef</b></sub></a><br><sub>CVE-2026-34986</sub>
|
<a href="https://github.com/Vonng"><img src="https://silo.pgsty.com/images/contributors/Vonng.svg" width="60" height="60" alt="@Vonng" title="@Vonng — 维护 SILO、Console、mcli、公共包、发行与文档"></a>
|
||||||
</td>
|
<a href="https://github.com/h5vx"><img src="https://silo.pgsty.com/images/contributors/h5vx.svg" width="60" height="60" alt="@h5vx" title="@h5vx — 实现单桶 CORS 配置与请求执行"></a>
|
||||||
<td align="center" width="150">
|
<a href="https://github.com/mrjavadseydi"><img src="https://silo.pgsty.com/images/contributors/mrjavadseydi.svg" width="60" height="60" alt="@mrjavadseydi" title="@mrjavadseydi — 修复有效桶配额指标,并提交按访问频率分层的 ILM 方案"></a>
|
||||||
<a href="https://github.com/mfredenhagen"><img src="https://github.com/mfredenhagen.png?size=100" width="72" alt="mfredenhagen"><br><sub><b>@mfredenhagen</b></sub></a><br><sub>CVE-2026-39883</sub>
|
<a href="https://github.com/Dansyuqri"><img src="https://silo.pgsty.com/images/contributors/Dansyuqri.svg" width="60" height="60" alt="@Dansyuqri" title="@Dansyuqri — 为分片上传完成响应补充 ChecksumType"></a>
|
||||||
</td>
|
<a href="https://github.com/ycjlin"><img src="https://silo.pgsty.com/images/contributors/ycjlin.svg" width="60" height="60" alt="@ycjlin" title="@ycjlin — 修复缺失桶的 ListObjects 语义"></a>
|
||||||
<td align="center" width="150">
|
<a href="https://github.com/pinginfo"><img src="https://silo.pgsty.com/images/contributors/pinginfo.svg" width="60" height="60" alt="@pinginfo" title="@pinginfo — 修复桶通知的流式输出"></a>
|
||||||
<a href="https://github.com/pinginfo"><img src="https://github.com/pinginfo.png?size=100" width="72" alt="pinginfo"><br><sub><b>@pinginfo</b></sub></a><br><sub>桶通知流式输出</sub>
|
<a href="https://github.com/ZouhairCharef"><img src="https://silo.pgsty.com/images/contributors/ZouhairCharef.svg" width="60" height="60" alt="@ZouhairCharef" title="@ZouhairCharef — 修复 go-jose 中的 CVE-2026-34986"></a>
|
||||||
</td>
|
<a href="https://github.com/mfredenhagen"><img src="https://silo.pgsty.com/images/contributors/mfredenhagen.svg" width="60" height="60" alt="@mfredenhagen" title="@mfredenhagen — 修复 OpenTelemetry 中的 CVE-2026-39883"></a>
|
||||||
<td align="center" width="150">
|
<a href="https://github.com/waterkip"><img src="https://silo.pgsty.com/images/contributors/waterkip.svg" width="60" height="60" alt="@waterkip" title="@waterkip — 将文档链接指向 SILO 门户"></a>
|
||||||
<a href="https://github.com/waterkip"><img src="https://github.com/waterkip.png?size=100" width="72" alt="waterkip"><br><sub><b>@waterkip</b></sub></a><br><sub>文档链接修正</sub>
|
<a href="https://github.com/mikemikimike"><img src="https://silo.pgsty.com/images/contributors/mikemikimike.svg" width="60" height="60" alt="@mikemikimike" title="@mikemikimike — 提交 SSE-C 复制分片明文尺寸修复"></a>
|
||||||
</td>
|
<a href="https://github.com/metaneutrons"><img src="https://silo.pgsty.com/images/contributors/metaneutrons.svg" width="60" height="60" alt="@metaneutrons" title="@metaneutrons — 报告并提交显式版本删除鉴权方案"></a>
|
||||||
</tr>
|
<a href="https://github.com/magicxor"><img src="https://silo.pgsty.com/images/contributors/magicxor.svg" width="60" height="60" alt="@magicxor" title="@magicxor — 报告并提交 DELETE If-Match 条件请求支持方案"></a>
|
||||||
</table>
|
<a href="https://github.com/davinkevin"><img src="https://silo.pgsty.com/images/contributors/davinkevin.svg" width="60" height="60" alt="@davinkevin" title="@davinkevin — 提交 distroless 容器镜像与依赖自动更新方案"></a>
|
||||||
<p>
|
<a href="https://github.com/lem21h"><img src="https://silo.pgsty.com/images/contributors/lem21h.svg" width="48" height="48" alt="@lem21h" title="@lem21h — 提交健壮性与 goroutine 改进"></a>
|
||||||
<a href="https://github.com/magicxor"><img src="https://github.com/magicxor.png?size=64" width="44" alt="magicxor" title="@magicxor"></a>
|
<a href="https://github.com/sulin37392"><img src="https://silo.pgsty.com/images/contributors/sulin37392.svg" width="48" height="48" alt="@sulin37392" title="@sulin37392 — 提交依赖更新"></a>
|
||||||
<a href="https://github.com/ycjlin"><img src="https://github.com/ycjlin.png?size=64" width="44" alt="ycjlin" title="@ycjlin"></a>
|
<a href="https://github.com/cbornet"><img src="https://silo.pgsty.com/images/contributors/cbornet.svg" width="60" height="60" alt="@cbornet" title="@cbornet — 报告分片与流式校验和缺陷及缺失桶语义问题"></a>
|
||||||
<a href="https://github.com/davinkevin"><img src="https://github.com/davinkevin.png?size=64" width="44" alt="davinkevin" title="@davinkevin"></a>
|
<a href="https://github.com/vampywiz17"><img src="https://silo.pgsty.com/images/contributors/vampywiz17.svg" width="60" height="60" alt="@vampywiz17" title="@vampywiz17 — 报告 LDAP TLS 与 Console 登录回归"></a>
|
||||||
<a href="https://github.com/lem21h"><img src="https://github.com/lem21h.png?size=64" width="44" alt="lem21h" title="@lem21h"></a>
|
<a href="https://github.com/orenyomtov"><img src="https://silo.pgsty.com/images/contributors/orenyomtov.svg" width="60" height="60" alt="@orenyomtov" title="@orenyomtov — 报告未签名头导致的 CopyObject 跨对象读取(SN-2026-011)"></a>
|
||||||
<a href="https://github.com/sulin37392"><img src="https://github.com/sulin37392.png?size=64" width="44" alt="sulin37392" title="@sulin37392"></a>
|
<a href="https://github.com/mumu-lab"><img src="https://silo.pgsty.com/images/contributors/mumu-lab.svg" width="48" height="48" alt="@mumu-lab" title="@mumu-lab — 报告桶配额指标读取已弃用字段的问题"></a>
|
||||||
<a href="https://github.com/mosesdd"><img src="https://github.com/mosesdd.png?size=64" width="44" alt="mosesdd" title="@mosesdd"></a>
|
<a href="https://github.com/jvasile"><img src="https://silo.pgsty.com/images/contributors/jvasile.svg" width="48" height="48" alt="@jvasile" title="@jvasile — 报告 Debian 包缺少用户、用户组与默认配置"></a>
|
||||||
<a href="https://github.com/Xavier-777"><img src="https://github.com/Xavier-777.png?size=64" width="44" alt="Xavier-777" title="@Xavier-777"></a>
|
<a href="https://github.com/pmezhuev"><img src="https://silo.pgsty.com/images/contributors/pmezhuev.svg" width="48" height="48" alt="@pmezhuev" title="@pmezhuev — 报告 RPM 包缺少 GPG 签名"></a>
|
||||||
<a href="https://github.com/jiadzh"><img src="https://github.com/jiadzh.png?size=64" width="44" alt="jiadzh" title="@jiadzh"></a>
|
<a href="https://github.com/TLINDEN"><img src="https://silo.pgsty.com/images/contributors/TLINDEN.svg" width="48" height="48" alt="@TLINDEN" title="@TLINDEN — 报告发布压缩包缺少客户端"></a>
|
||||||
<a href="https://github.com/TLINDEN"><img src="https://github.com/TLINDEN.png?size=64" width="44" alt="TLINDEN" title="@TLINDEN"></a>
|
<a href="https://github.com/makinikm"><img src="https://silo.pgsty.com/images/contributors/makinikm.svg" width="48" height="48" alt="@makinikm" title="@makinikm — 报告容器镜像缺少客户端"></a>
|
||||||
<a href="https://github.com/AntonOfTheWoods"><img src="https://github.com/AntonOfTheWoods.png?size=64" width="44" alt="AntonOfTheWoods" title="@AntonOfTheWoods"></a>
|
<a href="https://github.com/meesudzu"><img src="https://silo.pgsty.com/images/contributors/meesudzu.svg" width="48" height="48" alt="@meesudzu" title="@meesudzu — 提出从上游 MinIO 迁移的指南需求"></a>
|
||||||
<a href="https://github.com/zylpsrs"><img src="https://github.com/zylpsrs.png?size=64" width="44" alt="zylpsrs" title="@zylpsrs"></a>
|
<a href="https://github.com/kuldeep-link11"><img src="https://silo.pgsty.com/images/contributors/kuldeep-link11.svg" width="48" height="48" alt="@kuldeep-link11" title="@kuldeep-link11 — 报告 NATS JWT 凭据与通知目标重载问题"></a>
|
||||||
<a href="https://github.com/nsanitate"><img src="https://github.com/nsanitate.png?size=64" width="44" alt="nsanitate" title="@nsanitate"></a>
|
<a href="https://github.com/sargarass"><img src="https://silo.pgsty.com/images/contributors/sargarass.svg" width="48" height="48" alt="@sargarass" title="@sargarass — 报告 ListMultipartUploads 前缀与分页语义问题"></a>
|
||||||
<a href="https://github.com/makinikm"><img src="https://github.com/makinikm.png?size=64" width="44" alt="makinikm" title="@makinikm"></a>
|
<a href="https://github.com/liuhaodongliu990-cmyk"><img src="https://silo.pgsty.com/images/contributors/liuhaodongliu990-cmyk.svg" width="48" height="48" alt="@liuhaodongliu990-cmyk" title="@liuhaodongliu990-cmyk — 报告前缀下载进度显示异常"></a>
|
||||||
<a href="https://github.com/spaceg00se-r"><img src="https://github.com/spaceg00se-r.png?size=64" width="44" alt="spaceg00se-r" title="@spaceg00se-r"></a>
|
<a href="https://github.com/Xavier-777"><img src="https://silo.pgsty.com/images/contributors/Xavier-777.svg" width="48" height="48" alt="@Xavier-777" title="@Xavier-777 — 报告 Console 生命周期管理与文件预览缺失"></a>
|
||||||
<a href="https://github.com/heroes1412"><img src="https://github.com/heroes1412.png?size=64" width="44" alt="heroes1412" title="@heroes1412"></a>
|
<a href="https://github.com/spaceg00se-r"><img src="https://silo.pgsty.com/images/contributors/spaceg00se-r.svg" width="48" height="48" alt="@spaceg00se-r" title="@spaceg00se-r — 提出 cpuv1 支持需求并报告工作流令牌错误"></a>
|
||||||
<a href="https://github.com/vampywiz17"><img src="https://github.com/vampywiz17.png?size=64" width="44" alt="vampywiz17" title="@vampywiz17"></a>
|
<a href="https://github.com/kh0mka"><img src="https://silo.pgsty.com/images/contributors/kh0mka.svg" width="48" height="48" alt="@kh0mka" title="@kh0mka — 报告 ReadFileStreamHandler 节点间 I/O 超时"></a>
|
||||||
<a href="https://github.com/chalukyaj"><img src="https://github.com/chalukyaj.png?size=64" width="44" alt="chalukyaj" title="@chalukyaj"></a>
|
<a href="https://github.com/bagutzu"><img src="https://silo.pgsty.com/images/contributors/bagutzu.svg" width="48" height="48" alt="@bagutzu" title="@bagutzu — 提出兼容 KES 的外部 KMS 与 OpenBao 支持需求"></a>
|
||||||
<a href="https://github.com/cbornet"><img src="https://github.com/cbornet.png?size=64" width="44" alt="cbornet" title="@cbornet"></a>
|
<a href="https://github.com/DestroyLee"><img src="https://silo.pgsty.com/images/contributors/DestroyLee.svg" width="48" height="48" alt="@DestroyLee" title="@DestroyLee — 报告文档目录导航缺失"></a>
|
||||||
<a href="https://github.com/jvasile"><img src="https://github.com/jvasile.png?size=64" width="44" alt="jvasile" title="@jvasile"></a>
|
<a href="https://github.com/mosesdd"><img src="https://silo.pgsty.com/images/contributors/mosesdd.svg" width="48" height="48" alt="@mosesdd" title="@mosesdd — 提出维护 Helm Chart 的需求"></a>
|
||||||
<a href="https://github.com/Kesavaambati"><img src="https://github.com/Kesavaambati.png?size=64" width="44" alt="Kesavaambati" title="@Kesavaambati"></a>
|
<a href="https://github.com/zylpsrs"><img src="https://silo.pgsty.com/images/contributors/zylpsrs.svg" width="48" height="48" alt="@zylpsrs" title="@zylpsrs — 报告 Console 缺少分层与站点复制"></a>
|
||||||
<a href="https://github.com/redfoxfox"><img src="https://github.com/redfoxfox.png?size=64" width="44" alt="redfoxfox" title="@redfoxfox"></a>
|
<a href="https://github.com/heroes1412"><img src="https://silo.pgsty.com/images/contributors/heroes1412.svg" width="48" height="48" alt="@heroes1412" title="@heroes1412 — 报告性能分析选项不可用"></a>
|
||||||
<a href="https://github.com/kuldeep-link11"><img src="https://github.com/kuldeep-link11.png?size=64" width="44" alt="kuldeep-link11" title="@kuldeep-link11"></a>
|
<a href="https://github.com/redfoxfox"><img src="https://silo.pgsty.com/images/contributors/redfoxfox.svg" width="48" height="48" alt="@redfoxfox" title="@redfoxfox — 报告中文文档站点不可用"></a>
|
||||||
<a href="https://github.com/meesudzu"><img src="https://github.com/meesudzu.png?size=64" width="44" alt="meesudzu" title="@meesudzu"></a>
|
<a href="https://github.com/jiadzh"><img src="https://silo.pgsty.com/images/contributors/jiadzh.svg" width="48" height="48" alt="@jiadzh" title="@jiadzh — 提出 Windows 构建指导需求"></a>
|
||||||
<a href="https://github.com/pmezhuev"><img src="https://github.com/pmezhuev.png?size=64" width="44" alt="pmezhuev" title="@pmezhuev"></a>
|
<a href="https://github.com/AntonOfTheWoods"><img src="https://silo.pgsty.com/images/contributors/AntonOfTheWoods.svg" width="48" height="48" alt="@AntonOfTheWoods" title="@AntonOfTheWoods — 提出明确 Helm Chart 与 Operator 选项的需求"></a>
|
||||||
<a href="https://github.com/kh0mka"><img src="https://github.com/kh0mka.png?size=64" width="44" alt="kh0mka" title="@kh0mka"></a>
|
<a href="https://github.com/chalukyaj"><img src="https://silo.pgsty.com/images/contributors/chalukyaj.svg" width="48" height="48" alt="@chalukyaj" title="@chalukyaj — 提出改善 SILO Operator 可发现性的建议"></a>
|
||||||
|
<a href="https://github.com/nsanitate"><img src="https://silo.pgsty.com/images/contributors/nsanitate.svg" width="48" height="48" alt="@nsanitate" title="@nsanitate — 提出加入 CNCF Sandbox 的治理建议"></a>
|
||||||
|
<a href="https://github.com/Kesavaambati"><img src="https://silo.pgsty.com/images/contributors/Kesavaambati.svg" width="48" height="48" alt="@Kesavaambati" title="@Kesavaambati — 提出社区支持与容器镜像维护问题"></a>
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
GitHub 不为 fork 仓库生成贡献者图表,因此 [`CONTRIBUTORS.md`](CONTRIBUTORS.md)(而非 Insights 页面)才是本项目的署名记录,其中逐一记录了每个人对应的改动或报告。
|
[查看完整贡献记录](CONTRIBUTORS.md),了解每位贡献者的提案、修复与问题报告。
|
||||||
|
|
||||||
## 背景
|
## 背景
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ _init() {
|
|||||||
|
|
||||||
## Minimum required versions for build dependencies
|
## Minimum required versions for build dependencies
|
||||||
GIT_VERSION="1.0"
|
GIT_VERSION="1.0"
|
||||||
GO_VERSION="1.27.0"
|
GO_VERSION="1.27.1"
|
||||||
OSX_VERSION="10.8"
|
OSX_VERSION="10.8"
|
||||||
KNAME=$(uname -s)
|
KNAME=$(uname -s)
|
||||||
ARCH=$(uname -m)
|
ARCH=$(uname -m)
|
||||||
|
|||||||
@@ -22,8 +22,8 @@ function start_silo_4drive() {
|
|||||||
start_port=$1
|
start_port=$1
|
||||||
|
|
||||||
export MINIO_ROOT_USER=silo
|
export MINIO_ROOT_USER=silo
|
||||||
export MINIO_ROOT_PASSWORD=silo123
|
export MINIO_ROOT_PASSWORD=silo1234
|
||||||
export MC_HOST_silo="http://silo:silo123@127.0.0.1:${start_port}/"
|
export MC_HOST_silo="http://silo:silo1234@127.0.0.1:${start_port}/"
|
||||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||||
export MINIO_CI_CD=1
|
export MINIO_CI_CD=1
|
||||||
|
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ if [ -n "${MCLI_BIN:-}" ]; then
|
|||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
release=${MCLI_RELEASE:-RELEASE.2026-08-06T00-00-00Z}
|
release=${MCLI_RELEASE:-RELEASE.2026-09-13T00-00-00Z}
|
||||||
version_hyphen=${release#RELEASE.}
|
version_hyphen=${release#RELEASE.}
|
||||||
package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')
|
package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')
|
||||||
if [ "${package_version}" = "${version_hyphen}" ]; then
|
if [ "${package_version}" = "${version_hyphen}" ]; then
|
||||||
@@ -74,7 +74,7 @@ expected=$(awk -v asset="${archive}" '
|
|||||||
{
|
{
|
||||||
name=$2
|
name=$2
|
||||||
sub(/^\*/, "", name)
|
sub(/^\*/, "", name)
|
||||||
if (name == asset && $1 ~ /^[0-9a-fA-F]{64}$/) print tolower($1)
|
if (name == asset && length($1) == 64 && $1 ~ /^[0-9a-fA-F]+$/) print tolower($1)
|
||||||
}
|
}
|
||||||
' "${tmp_dir}/${checksums}")
|
' "${tmp_dir}/${checksums}")
|
||||||
if ! printf '%s\n' "${expected}" | grep -Eq '^[0-9a-f]{64}$'; then
|
if ! printf '%s\n' "${expected}" | grep -Eq '^[0-9a-f]{64}$'; then
|
||||||
|
|||||||
@@ -45,8 +45,8 @@ function start_silo_10drive() {
|
|||||||
start_port=$1
|
start_port=$1
|
||||||
|
|
||||||
export MINIO_ROOT_USER=silo
|
export MINIO_ROOT_USER=silo
|
||||||
export MINIO_ROOT_PASSWORD=silo123
|
export MINIO_ROOT_PASSWORD=silo1234
|
||||||
export MC_HOST_silo="http://silo:silo123@127.0.0.1:${start_port}/"
|
export MC_HOST_silo="http://silo:silo1234@127.0.0.1:${start_port}/"
|
||||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||||
export MINIO_CI_CD=1
|
export MINIO_CI_CD=1
|
||||||
|
|
||||||
@@ -71,7 +71,7 @@ function start_silo_10drive() {
|
|||||||
"${PWD}/mc" mb --with-versioning silo/bucket
|
"${PWD}/mc" mb --with-versioning silo/bucket
|
||||||
|
|
||||||
export AWS_ACCESS_KEY_ID=silo
|
export AWS_ACCESS_KEY_ID=silo
|
||||||
export AWS_SECRET_ACCESS_KEY=silo123
|
export AWS_SECRET_ACCESS_KEY=silo1234
|
||||||
aws --endpoint-url http://localhost:"$start_port" s3api create-multipart-upload --bucket bucket --key obj-1 >upload-id.json
|
aws --endpoint-url http://localhost:"$start_port" s3api create-multipart-upload --bucket bucket --key obj-1 >upload-id.json
|
||||||
uploadId=$(jq -r '.UploadId' upload-id.json)
|
uploadId=$(jq -r '.UploadId' upload-id.json)
|
||||||
|
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -30,7 +30,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
const manifestVersion = 3
|
const manifestVersion = 4
|
||||||
|
|
||||||
var (
|
var (
|
||||||
minioImportRE = regexp.MustCompile(`github\.com/minio/[A-Za-z0-9_./-]+`)
|
minioImportRE = regexp.MustCompile(`github\.com/minio/[A-Za-z0-9_./-]+`)
|
||||||
@@ -44,19 +44,18 @@ var (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type manifest struct {
|
type manifest struct {
|
||||||
Version int `json:"version"`
|
Version int `json:"version"`
|
||||||
ModulePath string `json:"module_path"`
|
ModulePath string `json:"module_path"`
|
||||||
MinioImports []string `json:"minio_imports"`
|
MinioImports []string `json:"minio_imports"`
|
||||||
Environment []string `json:"environment"`
|
Environment []string `json:"environment"`
|
||||||
Metrics []string `json:"metrics"`
|
Metrics []string `json:"metrics"`
|
||||||
Headers []string `json:"headers"`
|
Headers []string `json:"headers"`
|
||||||
Routes []string `json:"routes"`
|
Routes []string `json:"routes"`
|
||||||
RouteRoots []string `json:"route_roots"`
|
RouteRoots []string `json:"route_roots"`
|
||||||
GridRoutes []string `json:"grid_routes"`
|
GridRoutes []string `json:"grid_routes"`
|
||||||
StorageMarkers []string `json:"storage_markers"`
|
StorageMarkers []string `json:"storage_markers"`
|
||||||
PolicyValues []string `json:"policy_values"`
|
PolicyValues []string `json:"policy_values"`
|
||||||
ExportedSymbols []string `json:"exported_symbols"`
|
BrandAllowlist []string `json:"brand_allowlist"`
|
||||||
BrandAllowlist []string `json:"brand_allowlist"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@@ -101,23 +100,24 @@ func collect(repo string) (manifest, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
sets := map[string]map[string]struct{}{
|
sets := map[string]map[string]struct{}{
|
||||||
"imports": {},
|
"imports": {},
|
||||||
"env": {},
|
"env": {},
|
||||||
"metrics": {},
|
"metrics": {},
|
||||||
"headers": {},
|
"headers": {},
|
||||||
"routes": {},
|
"routes": {},
|
||||||
"roots": {},
|
"roots": {},
|
||||||
"grid": {},
|
"grid": {},
|
||||||
"storage": {},
|
"storage": {},
|
||||||
"policy": {},
|
"policy": {},
|
||||||
"exported": {},
|
"brand": {},
|
||||||
"brand": {},
|
|
||||||
}
|
}
|
||||||
modulePath := ""
|
modulePath := ""
|
||||||
fset := token.NewFileSet()
|
fset := token.NewFileSet()
|
||||||
|
|
||||||
for _, rel := range files {
|
for _, rel := range files {
|
||||||
|
// Investigation artifacts contain synthetic routes and archived configurations.
|
||||||
if rel == "SILO_REBRANDING_MIGRATION.md" ||
|
if rel == "SILO_REBRANDING_MIGRATION.md" ||
|
||||||
|
strings.HasPrefix(rel, "docs/investigations/") ||
|
||||||
strings.HasPrefix(rel, "buildscripts/rebrand-guard/") ||
|
strings.HasPrefix(rel, "buildscripts/rebrand-guard/") ||
|
||||||
strings.HasPrefix(rel, "buildscripts/helm-migration-guard/") {
|
strings.HasPrefix(rel, "buildscripts/helm-migration-guard/") {
|
||||||
continue
|
continue
|
||||||
@@ -163,17 +163,17 @@ func collect(repo string) (manifest, error) {
|
|||||||
sets["imports"][value] = struct{}{}
|
sets["imports"][value] = struct{}{}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
collectStringMatches(sets["routes"], routeRE, file)
|
|
||||||
collectNamedStringValues(sets["roots"], rel, file, "minioReservedBucket")
|
|
||||||
if rel == "internal/grid/manager.go" {
|
|
||||||
collectStringMatches(sets["grid"], routeRE, file)
|
|
||||||
}
|
|
||||||
if !strings.HasSuffix(rel, "_test.go") {
|
if !strings.HasSuffix(rel, "_test.go") {
|
||||||
collectExported(sets["exported"], filepath.ToSlash(filepath.Dir(rel)), file)
|
// Test files hold request paths for fixtures, not served routes.
|
||||||
|
collectStringMatches(sets["routes"], routeRE, file)
|
||||||
if strings.HasPrefix(rel, "cmd/") || strings.HasPrefix(rel, "internal/") {
|
if strings.HasPrefix(rel, "cmd/") || strings.HasPrefix(rel, "internal/") {
|
||||||
collectBrandStrings(sets["brand"], rel, file)
|
collectBrandStrings(sets["brand"], rel, file)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
collectNamedStringValues(sets["roots"], rel, file, "minioReservedBucket")
|
||||||
|
if rel == "internal/grid/manager.go" {
|
||||||
|
collectStringMatches(sets["grid"], routeRE, file)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// This was a shell-local PID variable in the generated inspect script,
|
// This was a shell-local PID variable in the generated inspect script,
|
||||||
@@ -184,19 +184,18 @@ func collect(repo string) (manifest, error) {
|
|||||||
return manifest{}, errors.New("go.mod module path was not found")
|
return manifest{}, errors.New("go.mod module path was not found")
|
||||||
}
|
}
|
||||||
return manifest{
|
return manifest{
|
||||||
Version: manifestVersion,
|
Version: manifestVersion,
|
||||||
ModulePath: modulePath,
|
ModulePath: modulePath,
|
||||||
MinioImports: sorted(sets["imports"]),
|
MinioImports: sorted(sets["imports"]),
|
||||||
Environment: sorted(sets["env"]),
|
Environment: sorted(sets["env"]),
|
||||||
Metrics: sorted(sets["metrics"]),
|
Metrics: sorted(sets["metrics"]),
|
||||||
Headers: sorted(sets["headers"]),
|
Headers: sorted(sets["headers"]),
|
||||||
Routes: sorted(sets["routes"]),
|
Routes: sorted(sets["routes"]),
|
||||||
RouteRoots: sorted(sets["roots"]),
|
RouteRoots: sorted(sets["roots"]),
|
||||||
GridRoutes: sorted(sets["grid"]),
|
GridRoutes: sorted(sets["grid"]),
|
||||||
StorageMarkers: sorted(sets["storage"]),
|
StorageMarkers: sorted(sets["storage"]),
|
||||||
PolicyValues: sorted(sets["policy"]),
|
PolicyValues: sorted(sets["policy"]),
|
||||||
ExportedSymbols: sorted(sets["exported"]),
|
BrandAllowlist: sorted(sets["brand"]),
|
||||||
BrandAllowlist: sorted(sets["brand"]),
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -262,7 +261,7 @@ func collectStringMatches(dst map[string]struct{}, re *regexp.Regexp, file *ast.
|
|||||||
}
|
}
|
||||||
|
|
||||||
func trackedFiles(repo string) ([]string, error) {
|
func trackedFiles(repo string) ([]string, error) {
|
||||||
cmd := exec.Command("git", "-C", repo, "ls-files", "--cached", "--others", "--exclude-standard", "-z")
|
cmd := exec.Command("git", "-C", repo, "ls-files", "--cached", "-z")
|
||||||
out, err := cmd.Output()
|
out, err := cmd.Output()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("git ls-files: %w", err)
|
return nil, fmt.Errorf("git ls-files: %w", err)
|
||||||
@@ -286,78 +285,6 @@ func addMatches(dst map[string]struct{}, re *regexp.Regexp, text string, lower b
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func collectExported(dst map[string]struct{}, dir string, file *ast.File) {
|
|
||||||
prefix := dir + ":" + file.Name.Name + ":"
|
|
||||||
for _, decl := range file.Decls {
|
|
||||||
switch decl := decl.(type) {
|
|
||||||
case *ast.FuncDecl:
|
|
||||||
if !ast.IsExported(decl.Name.Name) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if decl.Recv == nil {
|
|
||||||
dst[prefix+"func:"+decl.Name.Name] = struct{}{}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
receiver := receiverName(decl.Recv.List[0].Type)
|
|
||||||
dst[prefix+"method:"+receiver+"."+decl.Name.Name] = struct{}{}
|
|
||||||
case *ast.GenDecl:
|
|
||||||
for _, spec := range decl.Specs {
|
|
||||||
switch spec := spec.(type) {
|
|
||||||
case *ast.TypeSpec:
|
|
||||||
if !ast.IsExported(spec.Name.Name) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
dst[prefix+"type:"+spec.Name.Name] = struct{}{}
|
|
||||||
collectExportedFields(dst, prefix, spec.Name.Name, spec.Type)
|
|
||||||
case *ast.ValueSpec:
|
|
||||||
kind := strings.ToLower(decl.Tok.String())
|
|
||||||
for _, name := range spec.Names {
|
|
||||||
if ast.IsExported(name.Name) {
|
|
||||||
dst[prefix+kind+":"+name.Name] = struct{}{}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func collectExportedFields(dst map[string]struct{}, prefix, typeName string, expr ast.Expr) {
|
|
||||||
var fields *ast.FieldList
|
|
||||||
switch typed := expr.(type) {
|
|
||||||
case *ast.StructType:
|
|
||||||
fields = typed.Fields
|
|
||||||
case *ast.InterfaceType:
|
|
||||||
fields = typed.Methods
|
|
||||||
default:
|
|
||||||
return
|
|
||||||
}
|
|
||||||
for _, field := range fields.List {
|
|
||||||
for _, name := range field.Names {
|
|
||||||
if ast.IsExported(name.Name) {
|
|
||||||
dst[prefix+"field:"+typeName+"."+name.Name] = struct{}{}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func receiverName(expr ast.Expr) string {
|
|
||||||
switch expr := expr.(type) {
|
|
||||||
case *ast.Ident:
|
|
||||||
return expr.Name
|
|
||||||
case *ast.StarExpr:
|
|
||||||
return receiverName(expr.X)
|
|
||||||
case *ast.IndexExpr:
|
|
||||||
return receiverName(expr.X)
|
|
||||||
case *ast.IndexListExpr:
|
|
||||||
return receiverName(expr.X)
|
|
||||||
case *ast.SelectorExpr:
|
|
||||||
return receiverName(expr.X) + "." + expr.Sel.Name
|
|
||||||
default:
|
|
||||||
return fmt.Sprintf("%T", expr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func sorted(set map[string]struct{}) []string {
|
func sorted(set map[string]struct{}) []string {
|
||||||
values := make([]string, 0, len(set))
|
values := make([]string, 0, len(set))
|
||||||
for value := range set {
|
for value := range set {
|
||||||
@@ -409,7 +336,6 @@ func compare(want, got manifest) error {
|
|||||||
{"grid_routes", want.GridRoutes, got.GridRoutes},
|
{"grid_routes", want.GridRoutes, got.GridRoutes},
|
||||||
{"storage_markers", want.StorageMarkers, got.StorageMarkers},
|
{"storage_markers", want.StorageMarkers, got.StorageMarkers},
|
||||||
{"policy_values", want.PolicyValues, got.PolicyValues},
|
{"policy_values", want.PolicyValues, got.PolicyValues},
|
||||||
{"exported_symbols", want.ExportedSymbols, got.ExportedSymbols},
|
|
||||||
{"brand_allowlist", want.BrandAllowlist, got.BrandAllowlist},
|
{"brand_allowlist", want.BrandAllowlist, got.BrandAllowlist},
|
||||||
}
|
}
|
||||||
for _, check := range checks {
|
for _, check := range checks {
|
||||||
@@ -454,10 +380,10 @@ func setDiff(want, got []string) (missing, added []string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func printSummary(value manifest) {
|
func printSummary(value manifest) {
|
||||||
fmt.Printf("compatibility manifest: imports=%d env=%d metrics=%d headers=%d routes=%d roots=%d grid=%d storage=%d policy=%d exported=%d brand=%d sha256=%s\n",
|
fmt.Printf("compatibility manifest: imports=%d env=%d metrics=%d headers=%d routes=%d roots=%d grid=%d storage=%d policy=%d brand=%d sha256=%s\n",
|
||||||
len(value.MinioImports), len(value.Environment), len(value.Metrics), len(value.Headers),
|
len(value.MinioImports), len(value.Environment), len(value.Metrics), len(value.Headers),
|
||||||
len(value.Routes), len(value.RouteRoots), len(value.GridRoutes), len(value.StorageMarkers), len(value.PolicyValues),
|
len(value.Routes), len(value.RouteRoots), len(value.GridRoutes), len(value.StorageMarkers), len(value.PolicyValues),
|
||||||
len(value.ExportedSymbols), len(value.BrandAllowlist), manifestDigest(value))
|
len(value.BrandAllowlist), manifestDigest(value))
|
||||||
}
|
}
|
||||||
|
|
||||||
func manifestDigest(value manifest) string {
|
func manifestDigest(value manifest) string {
|
||||||
|
|||||||
@@ -18,8 +18,8 @@ function start_silo_5drive() {
|
|||||||
start_port=$1
|
start_port=$1
|
||||||
|
|
||||||
export MINIO_ROOT_USER=silo
|
export MINIO_ROOT_USER=silo
|
||||||
export MINIO_ROOT_PASSWORD=silo123
|
export MINIO_ROOT_PASSWORD=silo1234
|
||||||
export MC_HOST_silo="http://silo:silo123@127.0.0.1:${start_port}/"
|
export MC_HOST_silo="http://silo:silo1234@127.0.0.1:${start_port}/"
|
||||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||||
export MINIO_CI_CD=1
|
export MINIO_CI_CD=1
|
||||||
|
|
||||||
|
|||||||
@@ -28,8 +28,8 @@ function verify_rewrite() {
|
|||||||
start_port=$1
|
start_port=$1
|
||||||
|
|
||||||
export MINIO_ACCESS_KEY=silo
|
export MINIO_ACCESS_KEY=silo
|
||||||
export MINIO_SECRET_KEY=silo123
|
export MINIO_SECRET_KEY=silo1234
|
||||||
export MC_HOST_silo="http://silo:silo123@127.0.0.1:${start_port}/"
|
export MC_HOST_silo="http://silo:silo1234@127.0.0.1:${start_port}/"
|
||||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||||
export MINIO_CI_CD=1
|
export MINIO_CI_CD=1
|
||||||
|
|
||||||
@@ -87,7 +87,7 @@ function verify_rewrite() {
|
|||||||
-debug \
|
-debug \
|
||||||
-versions \
|
-versions \
|
||||||
-access-key silo \
|
-access-key silo \
|
||||||
-secret-key silo123 \
|
-secret-key silo1234 \
|
||||||
-endpoint "http://127.0.0.1:${start_port}/" 2>&1 | grep INTACT; then
|
-endpoint "http://127.0.0.1:${start_port}/" 2>&1 | grep INTACT; then
|
||||||
echo "server1 log:"
|
echo "server1 log:"
|
||||||
cat "${WORK_DIR}/server1.log"
|
cat "${WORK_DIR}/server1.log"
|
||||||
@@ -105,14 +105,14 @@ function verify_rewrite() {
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "silo" "silo123"
|
go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "silo" "silo1234"
|
||||||
sleep 1
|
sleep 1
|
||||||
|
|
||||||
if ! ./s3-check-md5 \
|
if ! ./s3-check-md5 \
|
||||||
-debug \
|
-debug \
|
||||||
-versions \
|
-versions \
|
||||||
-access-key silo \
|
-access-key silo \
|
||||||
-secret-key silo123 \
|
-secret-key silo1234 \
|
||||||
-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
|
-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
|
||||||
echo "server1 log:"
|
echo "server1 log:"
|
||||||
cat "${WORK_DIR}/server1.log"
|
cat "${WORK_DIR}/server1.log"
|
||||||
|
|||||||
@@ -74,8 +74,8 @@ function test_silo_with_timeout() {
|
|||||||
start_port=$1
|
start_port=$1
|
||||||
|
|
||||||
export MINIO_ROOT_USER=silo
|
export MINIO_ROOT_USER=silo
|
||||||
export MINIO_ROOT_PASSWORD=silo123
|
export MINIO_ROOT_PASSWORD=silo1234
|
||||||
export MC_HOST_silo="http://silo:silo123@127.0.0.1:${start_port}/"
|
export MC_HOST_silo="http://silo:silo1234@127.0.0.1:${start_port}/"
|
||||||
export MINIO_CI_CD=1
|
export MINIO_CI_CD=1
|
||||||
|
|
||||||
mkdir ${WORK_DIR}
|
mkdir ${WORK_DIR}
|
||||||
|
|||||||
@@ -15,10 +15,10 @@ WORK_DIR="$PWD/.verify-$RANDOM"
|
|||||||
export MINT_MODE=core
|
export MINT_MODE=core
|
||||||
export MINT_DATA_DIR="$WORK_DIR/data"
|
export MINT_DATA_DIR="$WORK_DIR/data"
|
||||||
export SERVER_ENDPOINT="127.0.0.1:9000"
|
export SERVER_ENDPOINT="127.0.0.1:9000"
|
||||||
export MC_HOST_verify="http://silo:silo123@${SERVER_ENDPOINT}/"
|
export MC_HOST_verify="http://silo:silo1234@${SERVER_ENDPOINT}/"
|
||||||
export MC_HOST_verify_ipv6="http://silo:silo123@[::1]:9000/"
|
export MC_HOST_verify_ipv6="http://silo:silo1234@[::1]:9000/"
|
||||||
export ACCESS_KEY="silo"
|
export ACCESS_KEY="silo"
|
||||||
export SECRET_KEY="silo123"
|
export SECRET_KEY="silo1234"
|
||||||
export ENABLE_HTTPS=0
|
export ENABLE_HTTPS=0
|
||||||
export GO111MODULE=on
|
export GO111MODULE=on
|
||||||
export GOGC=25
|
export GOGC=25
|
||||||
@@ -225,7 +225,7 @@ function __init__() {
|
|||||||
shred -n 1 -s 65M - 1>"$FILE_65_MB" 2>/dev/null
|
shred -n 1 -s 65M - 1>"$FILE_65_MB" 2>/dev/null
|
||||||
|
|
||||||
## version is purposefully set to '3' for minio to migrate configuration file
|
## version is purposefully set to '3' for minio to migrate configuration file
|
||||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo123"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo1234"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||||
|
|
||||||
"$(git rev-parse --show-toplevel)/buildscripts/install-verified-fixture.sh" \
|
"$(git rev-parse --show-toplevel)/buildscripts/install-verified-fixture.sh" \
|
||||||
https://raw.githubusercontent.com/pgsty/mc/4c4dcc4b55baf238cd0c81030d77945b3828f157/functional-tests.sh \
|
https://raw.githubusercontent.com/pgsty/mc/4c4dcc4b55baf238cd0c81030d77945b3828f157/functional-tests.sh \
|
||||||
@@ -282,7 +282,6 @@ function main() {
|
|||||||
purge "$WORK_DIR"
|
purge "$WORK_DIR"
|
||||||
}
|
}
|
||||||
|
|
||||||
(__init__ "$@" && main "$@")
|
trap 'purge "$WORK_DIR"' EXIT
|
||||||
rv=$?
|
__init__ "$@"
|
||||||
purge "$WORK_DIR"
|
main "$@"
|
||||||
exit "$rv"
|
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ SILO=("$PWD/silo" --config-dir "$SILO_CONFIG_DIR" server)
|
|||||||
|
|
||||||
function start_silo_3_node() {
|
function start_silo_3_node() {
|
||||||
export MINIO_ROOT_USER=silo
|
export MINIO_ROOT_USER=silo
|
||||||
export MINIO_ROOT_PASSWORD=silo123
|
export MINIO_ROOT_PASSWORD=silo1234
|
||||||
export MINIO_ERASURE_SET_DRIVE_COUNT=6
|
export MINIO_ERASURE_SET_DRIVE_COUNT=6
|
||||||
export MINIO_CI_CD=1
|
export MINIO_CI_CD=1
|
||||||
|
|
||||||
@@ -37,7 +37,7 @@ function start_silo_3_node() {
|
|||||||
pid3=$!
|
pid3=$!
|
||||||
disown $pid3
|
disown $pid3
|
||||||
|
|
||||||
export MC_HOST_mysilo="http://silo:silo123@127.0.0.1:$((start_port + 1))"
|
export MC_HOST_mysilo="http://silo:silo1234@127.0.0.1:$((start_port + 1))"
|
||||||
|
|
||||||
timeout 15m /tmp/mc ready mysilo || fail
|
timeout 15m /tmp/mc ready mysilo || fail
|
||||||
|
|
||||||
@@ -116,7 +116,7 @@ function __init__() {
|
|||||||
mkdir -p "$SILO_CONFIG_DIR"
|
mkdir -p "$SILO_CONFIG_DIR"
|
||||||
|
|
||||||
## version is purposefully set to '3' for minio to migrate configuration file
|
## version is purposefully set to '3' for minio to migrate configuration file
|
||||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo123"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo1234"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||||
|
|
||||||
if [ ! -f /tmp/mc ]; then
|
if [ ! -f /tmp/mc ]; then
|
||||||
"$(git rev-parse --show-toplevel)/buildscripts/install-mcli.sh" /tmp/mc
|
"$(git rev-parse --show-toplevel)/buildscripts/install-mcli.sh" /tmp/mc
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ function start_silo() {
|
|||||||
start_port=$1
|
start_port=$1
|
||||||
|
|
||||||
export MINIO_ROOT_USER=silo
|
export MINIO_ROOT_USER=silo
|
||||||
export MINIO_ROOT_PASSWORD=silo123
|
export MINIO_ROOT_PASSWORD=silo1234
|
||||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||||
unset MINIO_CI_CD
|
unset MINIO_CI_CD
|
||||||
unset CI
|
unset CI
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ function start_silo_3_node() {
|
|||||||
done
|
done
|
||||||
|
|
||||||
export MINIO_ROOT_USER=silo
|
export MINIO_ROOT_USER=silo
|
||||||
export MINIO_ROOT_PASSWORD=silo123
|
export MINIO_ROOT_PASSWORD=silo1234
|
||||||
export MINIO_ERASURE_SET_DRIVE_COUNT=6
|
export MINIO_ERASURE_SET_DRIVE_COUNT=6
|
||||||
export MINIO_CI_CD=1
|
export MINIO_CI_CD=1
|
||||||
|
|
||||||
@@ -46,7 +46,7 @@ function start_silo_3_node() {
|
|||||||
pid3=$!
|
pid3=$!
|
||||||
disown $pid3
|
disown $pid3
|
||||||
|
|
||||||
export MC_HOST_mysilo="http://silo:silo123@127.0.0.1:$((start_port + 1))"
|
export MC_HOST_mysilo="http://silo:silo1234@127.0.0.1:$((start_port + 1))"
|
||||||
timeout 15m /tmp/mc ready mysilo || fail
|
timeout 15m /tmp/mc ready mysilo || fail
|
||||||
|
|
||||||
[ ${first_time} -eq 0 ] && upload_objects
|
[ ${first_time} -eq 0 ] && upload_objects
|
||||||
@@ -117,7 +117,7 @@ function __init__() {
|
|||||||
mkdir -p "$SILO_CONFIG_DIR"
|
mkdir -p "$SILO_CONFIG_DIR"
|
||||||
|
|
||||||
## version is purposefully set to '3' for minio to migrate configuration file
|
## version is purposefully set to '3' for minio to migrate configuration file
|
||||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo123"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo1234"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||||
|
|
||||||
if [ ! -f /tmp/mc ]; then
|
if [ ! -f /tmp/mc ]; then
|
||||||
"$(git rev-parse --show-toplevel)/buildscripts/install-mcli.sh" /tmp/mc
|
"$(git rev-parse --show-toplevel)/buildscripts/install-mcli.sh" /tmp/mc
|
||||||
|
|||||||
@@ -113,7 +113,7 @@ helm_run template my-release "${new_chart}" \
|
|||||||
go run ./buildscripts/helm-migration-guard "${old_render}" "${new_render}"
|
go run ./buildscripts/helm-migration-guard "${old_render}" "${new_render}"
|
||||||
|
|
||||||
helm_run package "${new_chart}" --destination "${output_dir}" >/dev/null
|
helm_run package "${new_chart}" --destination "${output_dir}" >/dev/null
|
||||||
test -s "${work_dir}/silo-7.0.1.tgz"
|
test -s "${work_dir}/silo-7.0.2.tgz"
|
||||||
if find "${work_dir}" -maxdepth 1 -type f -name 'minio-*.tgz' | grep -q .; then
|
if find "${work_dir}" -maxdepth 1 -type f -name 'minio-*.tgz' | grep -q .; then
|
||||||
echo "Helm packaging emitted a legacy MinIO chart name" >&2
|
echo "Helm packaging emitted a legacy MinIO chart name" >&2
|
||||||
exit 1
|
exit 1
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ for file in \
|
|||||||
buildscripts/verify-helm-migration.sh \
|
buildscripts/verify-helm-migration.sh \
|
||||||
Dockerfile.goreleaser \
|
Dockerfile.goreleaser \
|
||||||
Dockerfile.distroless \
|
Dockerfile.distroless \
|
||||||
dockerscripts/download-static-curl.sh \
|
dockerscripts/build-static-curl.sh \
|
||||||
dockerscripts/docker-entrypoint.sh \
|
dockerscripts/docker-entrypoint.sh \
|
||||||
helm/silo/Chart.yaml \
|
helm/silo/Chart.yaml \
|
||||||
helm/silo/values.yaml \
|
helm/silo/values.yaml \
|
||||||
@@ -101,7 +101,7 @@ require_text Dockerfile.goreleaser "Published checksum drift"
|
|||||||
require_text Dockerfile.distroless 'COPY --chmod=0755 silo /usr/bin/silo'
|
require_text Dockerfile.distroless 'COPY --chmod=0755 silo /usr/bin/silo'
|
||||||
require_text Dockerfile.distroless 'ENTRYPOINT ["/usr/bin/silo"]'
|
require_text Dockerfile.distroless 'ENTRYPOINT ["/usr/bin/silo"]'
|
||||||
require_text Dockerfile.distroless '"/usr/bin/silo", "healthcheck", "ready"'
|
require_text Dockerfile.distroless '"/usr/bin/silo", "healthcheck", "ready"'
|
||||||
require_text dockerscripts/download-static-curl.sh "sha256sum -c"
|
require_text dockerscripts/build-static-curl.sh "sha256sum -c"
|
||||||
require_text helm/silo/Chart.yaml "name: silo"
|
require_text helm/silo/Chart.yaml "name: silo"
|
||||||
require_text helm/silo/values.yaml "repository: pgsty/silo"
|
require_text helm/silo/values.yaml "repository: pgsty/silo"
|
||||||
require_text helm/silo/templates/deployment.yaml "/usr/bin/docker-entrypoint.sh silo server"
|
require_text helm/silo/templates/deployment.yaml "/usr/bin/docker-entrypoint.sh silo server"
|
||||||
@@ -159,11 +159,13 @@ fi
|
|||||||
# The repository and its default branch are pgsty/silo and main. The invariant
|
# The repository and its default branch are pgsty/silo and main. The invariant
|
||||||
# is that the old name is never a live target, not that it is never spoken: the
|
# is that the old name is never a live target, not that it is never spoken: the
|
||||||
# READMEs have to name it to explain the rename and to point at the archived
|
# READMEs have to name it to explain the rename and to point at the archived
|
||||||
# artifacts, which is the opposite of stranding a reader on it.
|
# artifacts, which is the opposite of stranding a reader on it. CONTRIBUTORS.md
|
||||||
|
# also quotes historical issue titles.
|
||||||
#
|
#
|
||||||
# So two rules. First, no live URL may resolve to the old repository anywhere,
|
# So two rules. First, no live URL may resolve to the old repository anywhere,
|
||||||
# READMEs included.
|
# READMEs and CONTRIBUTORS.md included.
|
||||||
stale_repo_url="$(rg -n -e 'github\.com/pgsty/minio' -e 'hub\.docker\.com/r/pgsty/minio' \
|
old_repo_pattern='pgsty/minio(\.git)?([^[:alnum:]_.-]|$)'
|
||||||
|
stale_repo_url="$(rg -n -e "github\.com/${old_repo_pattern}" -e "hub\.docker\.com/r/${old_repo_pattern}" \
|
||||||
--glob '!.git/**' --glob '!dist/**' \
|
--glob '!.git/**' --glob '!dist/**' \
|
||||||
--glob '!SILO_REBRANDING_MIGRATION.md' \
|
--glob '!SILO_REBRANDING_MIGRATION.md' \
|
||||||
--glob '!buildscripts/rebrand-guard/compat-baseline.json' . |
|
--glob '!buildscripts/rebrand-guard/compat-baseline.json' . |
|
||||||
@@ -175,10 +177,10 @@ fi
|
|||||||
|
|
||||||
# Second, the bare name may only appear where it is deliberate: the pinned
|
# Second, the bare name may only appear where it is deliberate: the pinned
|
||||||
# pre-rebrand image digest in the upgrade test, the two guards that refuse a
|
# pre-rebrand image digest in the upgrade test, the two guards that refuse a
|
||||||
# legacy image, and the two READMEs that document the rename and the archived
|
# legacy image, the two READMEs that document the rename and the archived
|
||||||
# minio branch.
|
# minio branch, and historical issue titles in CONTRIBUTORS.md.
|
||||||
repo_guard_allowlist='^(buildscripts/minio-upgrade\.sh|buildscripts/verify-rebrand\.sh|buildscripts/helm-migration-guard/main\.go|README\.md|README_ZH\.md):'
|
repo_guard_allowlist='^(buildscripts/minio-upgrade\.sh|buildscripts/verify-rebrand\.sh|buildscripts/helm-migration-guard/main\.go|README\.md|README_ZH\.md|CONTRIBUTORS\.md):'
|
||||||
stale_repo="$(rg -n 'pgsty/minio' --glob '!.git/**' --glob '!dist/**' \
|
stale_repo="$(rg -n "${old_repo_pattern}" --glob '!.git/**' --glob '!dist/**' \
|
||||||
--glob '!SILO_REBRANDING_MIGRATION.md' \
|
--glob '!SILO_REBRANDING_MIGRATION.md' \
|
||||||
--glob '!buildscripts/rebrand-guard/compat-baseline.json' . |
|
--glob '!buildscripts/rebrand-guard/compat-baseline.json' . |
|
||||||
sed 's#^\./##' | grep -Ev "${repo_guard_allowlist}" || true)"
|
sed 's#^\./##' | grep -Ev "${repo_guard_allowlist}" || true)"
|
||||||
|
|||||||
+1
-1
@@ -25,7 +25,7 @@ import (
|
|||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Data types used for returning dummy access control
|
// Data types used for returning dummy access control
|
||||||
|
|||||||
@@ -0,0 +1,348 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/zip"
|
||||||
|
"bytes"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/madmin-go/v3"
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
"github.com/minio/mux"
|
||||||
|
)
|
||||||
|
|
||||||
|
func corsAdminRequest(t *testing.T, cred auth.Credentials, method, path string, body []byte) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
|
router := mux.NewRouter()
|
||||||
|
registerAdminRouter(router, true)
|
||||||
|
req, err := newTestSignedRequestV4(method, adminPathPrefix+adminAPIVersionPrefix+path,
|
||||||
|
int64(len(body)), bytes.NewReader(body), cred.AccessKey, cred.SecretKey, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("admin %s: %d: %s", path, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
return rec
|
||||||
|
}
|
||||||
|
|
||||||
|
func corsImportReport(t *testing.T, rec *httptest.ResponseRecorder) madmin.BucketMetaImportErrs {
|
||||||
|
t.Helper()
|
||||||
|
var rpt madmin.BucketMetaImportErrs
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &rpt); err != nil {
|
||||||
|
t.Fatalf("import report %q: %v", rec.Body.String(), err)
|
||||||
|
}
|
||||||
|
return rpt
|
||||||
|
}
|
||||||
|
|
||||||
|
func corsZip(t *testing.T, entries map[string][]byte) []byte {
|
||||||
|
t.Helper()
|
||||||
|
var buf bytes.Buffer
|
||||||
|
zw := zip.NewWriter(&buf)
|
||||||
|
for name, data := range entries {
|
||||||
|
w, err := zw.Create(name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err = w.Write(data); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := zw.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return buf.Bytes()
|
||||||
|
}
|
||||||
|
|
||||||
|
// corsCorruptedZip builds an archive holding a stored (uncompressed) cors.xml
|
||||||
|
// whose payload is altered after the checksum is computed, plus the given
|
||||||
|
// companion entries. The altered document stays well formed, so only the zip
|
||||||
|
// checksum tells the two apart.
|
||||||
|
func corsCorruptedZip(t *testing.T, name string, doc []byte, others map[string][]byte) []byte {
|
||||||
|
t.Helper()
|
||||||
|
var buf bytes.Buffer
|
||||||
|
zw := zip.NewWriter(&buf)
|
||||||
|
w, err := zw.CreateHeader(&zip.FileHeader{Name: name, Method: zip.Store})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err = w.Write(doc); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for other, data := range others {
|
||||||
|
ow, err := zw.Create(other)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err = ow.Write(data); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err = zw.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
raw := buf.Bytes()
|
||||||
|
at := bytes.Index(raw, []byte("app.example.com"))
|
||||||
|
if at < 0 {
|
||||||
|
t.Fatalf("stored CORS payload not found in archive")
|
||||||
|
}
|
||||||
|
raw[at] = 'A'
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAdminBucketMetadataCORSRoundTrip covers the export/import round trip for
|
||||||
|
// per-bucket CORS, per-file error reporting for an invalid document, and that
|
||||||
|
// an archive without cors.xml leaves an existing configuration alone.
|
||||||
|
func TestAdminBucketMetadataCORSRoundTrip(t *testing.T) {
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instanceType, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||||
|
corsXML := []byte(testSiteReplicationCORSDoc)
|
||||||
|
if _, err := updateLocalBucketCORSMetadata(t.Context(), obj, bucket, corsXML); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Export must carry the stored document verbatim.
|
||||||
|
rec := corsAdminRequest(t, cred, http.MethodGet, "/export-bucket-metadata?bucket="+bucket, nil)
|
||||||
|
archive := rec.Body.Bytes()
|
||||||
|
zr, err := zip.NewReader(bytes.NewReader(archive), int64(len(archive)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var exported []byte
|
||||||
|
for _, f := range zr.File {
|
||||||
|
if f.Name != bucket+"/"+bucketCorsConfig {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
r, err := f.Open()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
exported, err = io.ReadAll(r)
|
||||||
|
r.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !bytes.Equal(exported, corsXML) {
|
||||||
|
t.Fatalf("%s: exported CORS = %q, want %q", instanceType, exported, corsXML)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drop the configuration: the archive must then omit the entry.
|
||||||
|
if _, err = updateLocalBucketCORSMetadata(t.Context(), obj, bucket, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, _, err = globalBucketMetadataSys.GetCorsConfigXML(bucket); err == nil {
|
||||||
|
t.Fatalf("%s: CORS still present before restore", instanceType)
|
||||||
|
}
|
||||||
|
rec = corsAdminRequest(t, cred, http.MethodGet, "/export-bucket-metadata?bucket="+bucket, nil)
|
||||||
|
empty := rec.Body.Bytes()
|
||||||
|
zr, err = zip.NewReader(bytes.NewReader(empty), int64(len(empty)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, f := range zr.File {
|
||||||
|
if f.Name == bucket+"/"+bucketCorsConfig {
|
||||||
|
t.Fatalf("%s: export emitted %s for a bucket without CORS", instanceType, f.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata", archive)
|
||||||
|
if st := corsImportReport(t, rec).Buckets[bucket]; !st.Cors.IsSet || st.Cors.Err != "" {
|
||||||
|
t.Fatalf("%s: import report cors = %+v", instanceType, st.Cors)
|
||||||
|
}
|
||||||
|
stored, storedAt, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||||
|
if err != nil || !bytes.Equal(stored, corsXML) {
|
||||||
|
t.Fatalf("%s: restored CORS = %q, err = %v", instanceType, stored, err)
|
||||||
|
}
|
||||||
|
created, err := globalBucketMetadataSys.CreatedAt(bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !storedAt.After(created) {
|
||||||
|
t.Fatalf("%s: restored CORS timestamp %v is not after bucket creation %v", instanceType, storedAt, created)
|
||||||
|
}
|
||||||
|
|
||||||
|
// An archive without cors.xml must not remove the configuration.
|
||||||
|
corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||||
|
corsZip(t, map[string][]byte{bucket + "/quota.json": []byte(`{"quota":0}`)}))
|
||||||
|
if stored, _, err = globalBucketMetadataSys.GetCorsConfigXML(bucket); err != nil || !bytes.Equal(stored, corsXML) {
|
||||||
|
t.Fatalf("%s: import without cors.xml changed CORS: %q, err = %v", instanceType, stored, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A bucket the import itself creates must still land above its own
|
||||||
|
// creation time, otherwise CORS replication would drop the restore.
|
||||||
|
fresh := "cors-import-created-bucket"
|
||||||
|
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||||
|
corsZip(t, map[string][]byte{fresh + "/" + bucketCorsConfig: corsXML}))
|
||||||
|
if st := corsImportReport(t, rec).Buckets[fresh]; !st.Cors.IsSet || st.Cors.Err != "" {
|
||||||
|
t.Fatalf("%s: fresh bucket import report cors = %+v", instanceType, st.Cors)
|
||||||
|
}
|
||||||
|
freshStored, freshAt, err := globalBucketMetadataSys.GetCorsConfigXML(fresh)
|
||||||
|
if err != nil || !bytes.Equal(freshStored, corsXML) {
|
||||||
|
t.Fatalf("%s: fresh bucket CORS = %q, err = %v", instanceType, freshStored, err)
|
||||||
|
}
|
||||||
|
freshCreated, err := globalBucketMetadataSys.CreatedAt(fresh)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !freshAt.After(freshCreated) {
|
||||||
|
t.Fatalf("%s: fresh bucket CORS timestamp %v is not after creation %v", instanceType, freshAt, freshCreated)
|
||||||
|
}
|
||||||
|
|
||||||
|
// An invalid document must fail loudly for that bucket and change nothing.
|
||||||
|
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||||
|
corsZip(t, map[string][]byte{bucket + "/" + bucketCorsConfig: []byte("<CORSConfiguration><CORSRule>")}))
|
||||||
|
if st := corsImportReport(t, rec).Buckets[bucket]; st.Cors.Err == "" {
|
||||||
|
t.Fatalf("%s: invalid CORS import reported no error: %+v", instanceType, st)
|
||||||
|
}
|
||||||
|
if stored, _, err = globalBucketMetadataSys.GetCorsConfigXML(bucket); err != nil || !bytes.Equal(stored, corsXML) {
|
||||||
|
t.Fatalf("%s: invalid CORS import changed stored config: %q, err = %v", instanceType, stored, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A well formed document carried by a corrupt zip entry must be
|
||||||
|
// rejected too, leaving the stored document and its timestamp alone
|
||||||
|
// while the other configs in the same archive still apply.
|
||||||
|
_, corsAt, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||||
|
corsCorruptedZip(t, bucket+"/"+bucketCorsConfig, corsXML,
|
||||||
|
map[string][]byte{bucket + "/quota.json": []byte(`{"quota":4096,"quotatype":"hard"}`)}))
|
||||||
|
st := corsImportReport(t, rec).Buckets[bucket]
|
||||||
|
if st.Cors.Err == "" {
|
||||||
|
t.Fatalf("%s: corrupt CORS entry reported no error: %+v", instanceType, st)
|
||||||
|
}
|
||||||
|
if !st.Quota.IsSet || st.Quota.Err != "" {
|
||||||
|
t.Fatalf("%s: corrupt CORS entry blocked the neighboring quota: %+v", instanceType, st.Quota)
|
||||||
|
}
|
||||||
|
stored, storedAt, err = globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||||
|
if err != nil || !bytes.Equal(stored, corsXML) || !storedAt.Equal(corsAt) {
|
||||||
|
t.Fatalf("%s: corrupt CORS entry changed stored config: %q at %v (was %v), err = %v", instanceType, stored, storedAt, corsAt, err)
|
||||||
|
}
|
||||||
|
quota, _, err := globalBucketMetadataSys.GetQuotaConfig(t.Context(), bucket)
|
||||||
|
if err != nil || quota == nil || quota.Quota != 4096 {
|
||||||
|
t.Fatalf("%s: neighboring quota not applied: %+v, err = %v", instanceType, quota, err)
|
||||||
|
}
|
||||||
|
}})
|
||||||
|
}
|
||||||
|
|
||||||
|
// corsPeerStub is a stand-in site-replication peer. It records every
|
||||||
|
// SRBucketMeta it is asked to apply and answers with status.
|
||||||
|
func corsPeerStub(t *testing.T, applied chan<- madmin.SRBucketMeta, status int) *httptest.Server {
|
||||||
|
t.Helper()
|
||||||
|
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method == http.MethodPut && applied != nil {
|
||||||
|
var item madmin.SRBucketMeta
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&item); err != nil {
|
||||||
|
t.Errorf("decode peer apply: %v", err)
|
||||||
|
w.WriteHeader(http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
applied <- item
|
||||||
|
}
|
||||||
|
w.WriteHeader(status)
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAdminBucketMetadataCORSImportReplicatesPastPeerFailure pins that an
|
||||||
|
// imported CORS document reaches the reachable peers even when the shared
|
||||||
|
// bucket metadata hook failed against an unreachable one, and that both
|
||||||
|
// failures are still reported for the bucket.
|
||||||
|
func TestAdminBucketMetadataCORSImportReplicatesPastPeerFailure(t *testing.T) {
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instanceType, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||||
|
ctx := t.Context()
|
||||||
|
corsXML := []byte(testSiteReplicationCORSDoc)
|
||||||
|
|
||||||
|
healthyApplies := make(chan madmin.SRBucketMeta, 4)
|
||||||
|
healthy := corsPeerStub(t, healthyApplies, http.StatusOK)
|
||||||
|
defer healthy.Close()
|
||||||
|
broken := corsPeerStub(t, nil, http.StatusBadRequest)
|
||||||
|
defer broken.Close()
|
||||||
|
|
||||||
|
// With site replication on, admin requests resolve their token signing
|
||||||
|
// key through the site replicator account, so it has to exist.
|
||||||
|
serviceCred, err := auth.CreateCredentials(siteReplicatorSvcAcc, "cors-import-service-secret")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
serviceCred.ParentUser = cred.AccessKey
|
||||||
|
if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
|
||||||
|
globalSiteReplicatorCred.Set(serviceCred.SecretKey)
|
||||||
|
defer globalSiteReplicatorCred.Set("")
|
||||||
|
|
||||||
|
globalSiteReplicationSys.Lock()
|
||||||
|
oldEnabled, oldState := globalSiteReplicationSys.enabled, globalSiteReplicationSys.state
|
||||||
|
globalSiteReplicationSys.enabled = true
|
||||||
|
globalSiteReplicationSys.state = srState{
|
||||||
|
Name: "cors-import-test",
|
||||||
|
ServiceAccountAccessKey: serviceCred.AccessKey,
|
||||||
|
Peers: map[string]madmin.PeerInfo{
|
||||||
|
globalDeploymentID(): {Name: "local", DeploymentID: globalDeploymentID()},
|
||||||
|
"peer-healthy": {Name: "healthy", DeploymentID: "peer-healthy", Endpoint: healthy.URL},
|
||||||
|
"peer-broken": {Name: "broken", DeploymentID: "peer-broken", Endpoint: broken.URL},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
globalSiteReplicationSys.Unlock()
|
||||||
|
defer func() {
|
||||||
|
globalSiteReplicationSys.Lock()
|
||||||
|
globalSiteReplicationSys.enabled, globalSiteReplicationSys.state = oldEnabled, oldState
|
||||||
|
globalSiteReplicationSys.Unlock()
|
||||||
|
}()
|
||||||
|
|
||||||
|
rec := corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||||
|
corsZip(t, map[string][]byte{
|
||||||
|
bucket + "/" + bucketCorsConfig: corsXML,
|
||||||
|
bucket + "/quota.json": []byte(`{"quota":8192,"quotatype":"hard"}`),
|
||||||
|
}))
|
||||||
|
st := corsImportReport(t, rec).Buckets[bucket]
|
||||||
|
if !st.Cors.IsSet || st.Cors.Err != "" {
|
||||||
|
t.Fatalf("%s: import report cors = %+v", instanceType, st.Cors)
|
||||||
|
}
|
||||||
|
stored, storedAt, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||||
|
if err != nil || !bytes.Equal(stored, corsXML) {
|
||||||
|
t.Fatalf("%s: stored CORS = %q, err = %v", instanceType, stored, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The reachable peer must have been told about the CORS document,
|
||||||
|
// carrying exactly the timestamp that was saved locally.
|
||||||
|
var corsSeen, sharedSeen bool
|
||||||
|
for range 2 {
|
||||||
|
select {
|
||||||
|
case item := <-healthyApplies:
|
||||||
|
if item.Type != madmin.SRBucketMetaTypeCorsConfig {
|
||||||
|
sharedSeen = item.Bucket == bucket && item.Quota != nil
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if item.Bucket != bucket || item.Cors == nil || !item.UpdatedAt.Equal(storedAt) {
|
||||||
|
t.Fatalf("%s: peer CORS event = %#v, want %s at %v", instanceType, item, bucket, storedAt)
|
||||||
|
}
|
||||||
|
payload, decErr := base64.StdEncoding.Strict().DecodeString(*item.Cors)
|
||||||
|
if decErr != nil || !bytes.Equal(payload, corsXML) {
|
||||||
|
t.Fatalf("%s: peer CORS payload = %q, err = %v", instanceType, payload, decErr)
|
||||||
|
}
|
||||||
|
corsSeen = true
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatalf("%s: healthy peer received no further events (shared=%v cors=%v)", instanceType, sharedSeen, corsSeen)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !sharedSeen || !corsSeen {
|
||||||
|
t.Fatalf("%s: healthy peer events shared=%v cors=%v, want both", instanceType, sharedSeen, corsSeen)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Both hook failures against the unreachable peer stay reported.
|
||||||
|
if got := strings.Count(st.Err, "->broken:"); got != 2 {
|
||||||
|
t.Fatalf("%s: bucket error mentions the broken peer %d times, want 2: %q", instanceType, got, st.Err)
|
||||||
|
}
|
||||||
|
}})
|
||||||
|
}
|
||||||
+205
-19
@@ -41,7 +41,7 @@ import (
|
|||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/minio/minio/internal/kms"
|
"github.com/minio/minio/internal/kms"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -76,7 +76,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
quotaConfig, err := parseBucketQuota(bucket, data)
|
_, err = parseBucketQuota(bucket, data)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
@@ -94,9 +94,6 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
|
|||||||
Quota: data,
|
Quota: data,
|
||||||
UpdatedAt: updatedAt,
|
UpdatedAt: updatedAt,
|
||||||
}
|
}
|
||||||
if quotaConfig.Size == 0 && quotaConfig.Quota == 0 {
|
|
||||||
bucketMeta.Quota = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Call site replication hook.
|
// Call site replication hook.
|
||||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta))
|
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta))
|
||||||
@@ -417,6 +414,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
bucketLifecycleConfig,
|
bucketLifecycleConfig,
|
||||||
bucketSSEConfig,
|
bucketSSEConfig,
|
||||||
bucketTaggingConfig,
|
bucketTaggingConfig,
|
||||||
|
bucketCorsConfig,
|
||||||
bucketQuotaConfigFile,
|
bucketQuotaConfigFile,
|
||||||
objectLockConfig,
|
objectLockConfig,
|
||||||
bucketVersioningConfig,
|
bucketVersioningConfig,
|
||||||
@@ -437,7 +435,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
configData, err := json.Marshal(config)
|
configData, err := canonicalBucketPolicy(config)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||||
return
|
return
|
||||||
@@ -517,6 +515,19 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||||
|
case bucketCorsConfig:
|
||||||
|
// Export the stored document verbatim: GetBucketCors returns
|
||||||
|
// the bytes exactly as they were PUT, so the archive must
|
||||||
|
// round-trip them unchanged.
|
||||||
|
configData, _, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, errConfigNotFound) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||||
case objectLockConfig:
|
case objectLockConfig:
|
||||||
config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
|
config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -589,6 +600,50 @@ type importMetaReport struct {
|
|||||||
madmin.BucketMetaImportErrs
|
madmin.BucketMetaImportErrs
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type importMetadataFields map[string]struct{}
|
||||||
|
|
||||||
|
func (f importMetadataFields) add(configFile string) {
|
||||||
|
f[configFile] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func applyImportedBucketMetadata(dst *BucketMetadata, src BucketMetadata, fields importMetadataFields) {
|
||||||
|
for configFile := range fields {
|
||||||
|
switch configFile {
|
||||||
|
case bucketPolicyConfig:
|
||||||
|
dst.PolicyConfigJSON = bytes.Clone(src.PolicyConfigJSON)
|
||||||
|
dst.PolicyConfigUpdatedAt = src.PolicyConfigUpdatedAt
|
||||||
|
case bucketNotificationConfig:
|
||||||
|
dst.NotificationConfigXML = bytes.Clone(src.NotificationConfigXML)
|
||||||
|
dst.NotificationConfigUpdatedAt = src.NotificationConfigUpdatedAt
|
||||||
|
case bucketLifecycleConfig:
|
||||||
|
dst.LifecycleConfigXML = bytes.Clone(src.LifecycleConfigXML)
|
||||||
|
dst.LifecycleConfigUpdatedAt = src.LifecycleConfigUpdatedAt
|
||||||
|
case bucketSSEConfig:
|
||||||
|
dst.EncryptionConfigXML = bytes.Clone(src.EncryptionConfigXML)
|
||||||
|
dst.EncryptionConfigUpdatedAt = src.EncryptionConfigUpdatedAt
|
||||||
|
case bucketTaggingConfig:
|
||||||
|
dst.TaggingConfigXML = bytes.Clone(src.TaggingConfigXML)
|
||||||
|
dst.TaggingConfigUpdatedAt = src.TaggingConfigUpdatedAt
|
||||||
|
case bucketQuotaConfigFile:
|
||||||
|
dst.QuotaConfigJSON = bytes.Clone(src.QuotaConfigJSON)
|
||||||
|
dst.QuotaConfigUpdatedAt = src.QuotaConfigUpdatedAt
|
||||||
|
case bucketCorsConfig:
|
||||||
|
// The import stamps its fields before creating any missing bucket,
|
||||||
|
// and a CORS event stamped before bucket creation is discarded as
|
||||||
|
// belonging to an older incarnation, so the imported document takes
|
||||||
|
// the same monotonic timestamp a local PutBucketCors would assign.
|
||||||
|
dst.CorsConfigUpdatedAt = localCORSUpdatedAt(*dst, src.CorsConfigUpdatedAt)
|
||||||
|
dst.CorsConfigXML = bytes.Clone(src.CorsConfigXML)
|
||||||
|
case objectLockConfig:
|
||||||
|
dst.ObjectLockConfigXML = bytes.Clone(src.ObjectLockConfigXML)
|
||||||
|
dst.ObjectLockConfigUpdatedAt = src.ObjectLockConfigUpdatedAt
|
||||||
|
case bucketVersioningConfig:
|
||||||
|
dst.VersioningConfigXML = bytes.Clone(src.VersioningConfigXML)
|
||||||
|
dst.VersioningConfigUpdatedAt = src.VersioningConfigUpdatedAt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (i *importMetaReport) SetStatus(bucket, fname string, err error) {
|
func (i *importMetaReport) SetStatus(bucket, fname string, err error) {
|
||||||
st := i.Buckets[bucket]
|
st := i.Buckets[bucket]
|
||||||
var errMsg string
|
var errMsg string
|
||||||
@@ -608,6 +663,8 @@ func (i *importMetaReport) SetStatus(bucket, fname string, err error) {
|
|||||||
st.Tagging = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
st.Tagging = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||||
case bucketQuotaConfigFile:
|
case bucketQuotaConfigFile:
|
||||||
st.Quota = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
st.Quota = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||||
|
case bucketCorsConfig:
|
||||||
|
st.Cors = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||||
case objectLockConfig:
|
case objectLockConfig:
|
||||||
st.ObjectLock = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
st.ObjectLock = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||||
case bucketVersioningConfig:
|
case bucketVersioningConfig:
|
||||||
@@ -649,6 +706,16 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
}
|
}
|
||||||
|
|
||||||
bucketMap := make(map[string]*BucketMetadata, len(zr.File))
|
bucketMap := make(map[string]*BucketMetadata, len(zr.File))
|
||||||
|
importedFields := make(map[string]importMetadataFields, len(zr.File))
|
||||||
|
blockedBuckets := make(map[string]struct{})
|
||||||
|
markImported := func(bucket, configFile string) {
|
||||||
|
fields := importedFields[bucket]
|
||||||
|
if fields == nil {
|
||||||
|
fields = make(importMetadataFields)
|
||||||
|
importedFields[bucket] = fields
|
||||||
|
}
|
||||||
|
fields.add(configFile)
|
||||||
|
}
|
||||||
|
|
||||||
updatedAt := UTCNow()
|
updatedAt := UTCNow()
|
||||||
|
|
||||||
@@ -664,6 +731,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
bucketMap[bucket] = &meta
|
bucketMap[bucket] = &meta
|
||||||
} else if err != errConfigNotFound {
|
} else if err != errConfigNotFound {
|
||||||
rpt.SetStatus(bucket, "", err)
|
rpt.SetStatus(bucket, "", err)
|
||||||
|
blockedBuckets[bucket] = struct{}{}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -675,6 +743,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
bucket, fileName := slc[0], slc[1]
|
bucket, fileName := slc[0], slc[1]
|
||||||
|
if _, blocked := blockedBuckets[bucket]; blocked {
|
||||||
|
continue
|
||||||
|
}
|
||||||
if fileName == objectLockConfig {
|
if fileName == objectLockConfig {
|
||||||
reader, err := file.Open()
|
reader, err := file.Open()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -708,6 +779,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
|
|
||||||
bucketMap[bucket].ObjectLockConfigXML = configData
|
bucketMap[bucket].ObjectLockConfigXML = configData
|
||||||
bucketMap[bucket].ObjectLockConfigUpdatedAt = updatedAt
|
bucketMap[bucket].ObjectLockConfigUpdatedAt = updatedAt
|
||||||
|
markImported(bucket, fileName)
|
||||||
rpt.SetStatus(bucket, fileName, nil)
|
rpt.SetStatus(bucket, fileName, nil)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -720,6 +792,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
bucket, fileName := slc[0], slc[1]
|
bucket, fileName := slc[0], slc[1]
|
||||||
|
if _, blocked := blockedBuckets[bucket]; blocked {
|
||||||
|
continue
|
||||||
|
}
|
||||||
if fileName == bucketVersioningConfig {
|
if fileName == bucketVersioningConfig {
|
||||||
reader, err := file.Open()
|
reader, err := file.Open()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -764,6 +839,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
|
|
||||||
bucketMap[bucket].VersioningConfigXML = configData
|
bucketMap[bucket].VersioningConfigXML = configData
|
||||||
bucketMap[bucket].VersioningConfigUpdatedAt = updatedAt
|
bucketMap[bucket].VersioningConfigUpdatedAt = updatedAt
|
||||||
|
markImported(bucket, fileName)
|
||||||
rpt.SetStatus(bucket, fileName, nil)
|
rpt.SetStatus(bucket, fileName, nil)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -781,6 +857,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
bucket, fileName := slc[0], slc[1]
|
bucket, fileName := slc[0], slc[1]
|
||||||
|
if _, blocked := blockedBuckets[bucket]; blocked {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
// create bucket if it does not exist yet.
|
// create bucket if it does not exist yet.
|
||||||
if _, ok := bucketMap[bucket]; !ok {
|
if _, ok := bucketMap[bucket]; !ok {
|
||||||
@@ -813,6 +892,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
|
|
||||||
bucketMap[bucket].NotificationConfigXML = configData
|
bucketMap[bucket].NotificationConfigXML = configData
|
||||||
bucketMap[bucket].NotificationConfigUpdatedAt = updatedAt
|
bucketMap[bucket].NotificationConfigUpdatedAt = updatedAt
|
||||||
|
markImported(bucket, fileName)
|
||||||
rpt.SetStatus(bucket, fileName, nil)
|
rpt.SetStatus(bucket, fileName, nil)
|
||||||
case bucketPolicyConfig:
|
case bucketPolicyConfig:
|
||||||
// Error out if Content-Length is beyond allowed size.
|
// Error out if Content-Length is beyond allowed size.
|
||||||
@@ -839,7 +919,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
configData, err := json.Marshal(bucketPolicy)
|
configData, err := canonicalBucketPolicy(bucketPolicy)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
rpt.SetStatus(bucket, fileName, err)
|
||||||
continue
|
continue
|
||||||
@@ -847,6 +927,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
|
|
||||||
bucketMap[bucket].PolicyConfigJSON = configData
|
bucketMap[bucket].PolicyConfigJSON = configData
|
||||||
bucketMap[bucket].PolicyConfigUpdatedAt = updatedAt
|
bucketMap[bucket].PolicyConfigUpdatedAt = updatedAt
|
||||||
|
markImported(bucket, fileName)
|
||||||
rpt.SetStatus(bucket, fileName, nil)
|
rpt.SetStatus(bucket, fileName, nil)
|
||||||
case bucketLifecycleConfig:
|
case bucketLifecycleConfig:
|
||||||
bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(reader, sz))
|
bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(reader, sz))
|
||||||
@@ -879,6 +960,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
|
|
||||||
bucketMap[bucket].LifecycleConfigXML = configData
|
bucketMap[bucket].LifecycleConfigXML = configData
|
||||||
bucketMap[bucket].LifecycleConfigUpdatedAt = updatedAt
|
bucketMap[bucket].LifecycleConfigUpdatedAt = updatedAt
|
||||||
|
markImported(bucket, fileName)
|
||||||
rpt.SetStatus(bucket, fileName, nil)
|
rpt.SetStatus(bucket, fileName, nil)
|
||||||
case bucketSSEConfig:
|
case bucketSSEConfig:
|
||||||
// Parse bucket encryption xml
|
// Parse bucket encryption xml
|
||||||
@@ -917,6 +999,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
|
|
||||||
bucketMap[bucket].EncryptionConfigXML = configData
|
bucketMap[bucket].EncryptionConfigXML = configData
|
||||||
bucketMap[bucket].EncryptionConfigUpdatedAt = updatedAt
|
bucketMap[bucket].EncryptionConfigUpdatedAt = updatedAt
|
||||||
|
markImported(bucket, fileName)
|
||||||
rpt.SetStatus(bucket, fileName, nil)
|
rpt.SetStatus(bucket, fileName, nil)
|
||||||
case bucketTaggingConfig:
|
case bucketTaggingConfig:
|
||||||
tags, err := tags.ParseBucketXML(io.LimitReader(reader, sz))
|
tags, err := tags.ParseBucketXML(io.LimitReader(reader, sz))
|
||||||
@@ -933,6 +1016,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
|
|
||||||
bucketMap[bucket].TaggingConfigXML = configData
|
bucketMap[bucket].TaggingConfigXML = configData
|
||||||
bucketMap[bucket].TaggingConfigUpdatedAt = updatedAt
|
bucketMap[bucket].TaggingConfigUpdatedAt = updatedAt
|
||||||
|
markImported(bucket, fileName)
|
||||||
rpt.SetStatus(bucket, fileName, nil)
|
rpt.SetStatus(bucket, fileName, nil)
|
||||||
case bucketQuotaConfigFile:
|
case bucketQuotaConfigFile:
|
||||||
data, err := io.ReadAll(reader)
|
data, err := io.ReadAll(reader)
|
||||||
@@ -949,6 +1033,33 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
|
|
||||||
bucketMap[bucket].QuotaConfigJSON = data
|
bucketMap[bucket].QuotaConfigJSON = data
|
||||||
bucketMap[bucket].QuotaConfigUpdatedAt = updatedAt
|
bucketMap[bucket].QuotaConfigUpdatedAt = updatedAt
|
||||||
|
markImported(bucket, fileName)
|
||||||
|
rpt.SetStatus(bucket, fileName, nil)
|
||||||
|
case bucketCorsConfig:
|
||||||
|
if sz > maxBucketCorsSize {
|
||||||
|
rpt.SetStatus(bucket, fileName, errors.New(ErrEntityTooLarge.String()))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read one byte past the declared size: stopping exactly at sz
|
||||||
|
// leaves archive/zip short of EOF, so it never verifies the entry
|
||||||
|
// checksum and a corrupt entry carrying well formed XML would be
|
||||||
|
// stored as a valid document. The extra byte also lets the reader
|
||||||
|
// reject an entry longer than it declares.
|
||||||
|
corsData, err := io.ReadAll(io.LimitReader(reader, sz+1))
|
||||||
|
if err != nil {
|
||||||
|
rpt.SetStatus(bucket, fileName, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = validateCORSReplicationPayload(corsData); err != nil {
|
||||||
|
rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
bucketMap[bucket].CorsConfigXML = corsData
|
||||||
|
bucketMap[bucket].CorsConfigUpdatedAt = updatedAt
|
||||||
|
markImported(bucket, fileName)
|
||||||
rpt.SetStatus(bucket, fileName, nil)
|
rpt.SetStatus(bucket, fileName, nil)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -962,22 +1073,97 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
}
|
}
|
||||||
|
|
||||||
for bucket, meta := range bucketMap {
|
for bucket, meta := range bucketMap {
|
||||||
err := globalBucketMetadataSys.save(ctx, *meta)
|
fields := importedFields[bucket]
|
||||||
|
if len(fields) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var merged BucketMetadata
|
||||||
|
var commitAt time.Time
|
||||||
|
err := func() error {
|
||||||
|
lockCtx, unlock, err := lockBucketMetadata(ctx, objectAPI, bucket)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
|
merged, err = loadBucketMetadataParse(lockCtx, objectAPI, bucket, false)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := ensureBucketMetadataCreated(lockCtx, objectAPI, &merged); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
commitAt = UTCNow()
|
||||||
|
for _, file := range replicatedBucketConfigs {
|
||||||
|
if _, ok := fields[file]; ok {
|
||||||
|
commitAt = localBucketConfigUpdatedAt(merged, file, commitAt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
applyImportedBucketMetadata(&merged, *meta, fields)
|
||||||
|
for _, file := range replicatedBucketConfigs {
|
||||||
|
if _, ok := fields[file]; !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
data, at := replicatedBucketConfig(&merged, file)
|
||||||
|
payload, _, err := bucketConfigPayload(bucket, file, *data, len(merged.ObjectLockConfigXML) != 0)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
*data, *at = payload, commitAt
|
||||||
|
}
|
||||||
|
return globalBucketMetadataSys.saveMetadata(lockCtx, objectAPI, &merged)
|
||||||
|
}()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
rpt.SetStatus(bucket, "", err)
|
rpt.SetStatus(bucket, "", err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// Call site replication hook.
|
*meta = merged
|
||||||
if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
globalNotificationSys.LoadBucketMetadata(bgContext(ctx), bucket)
|
||||||
Bucket: bucket,
|
hook := madmin.SRBucketMeta{Bucket: bucket, UpdatedAt: commitAt}
|
||||||
Quota: meta.QuotaConfigJSON,
|
var hookNeeded bool
|
||||||
Policy: meta.PolicyConfigJSON,
|
if _, ok := fields[bucketQuotaConfigFile]; ok {
|
||||||
Versioning: enc(meta.VersioningConfigXML),
|
hook.Quota = meta.QuotaConfigJSON
|
||||||
Tags: enc(meta.TaggingConfigXML),
|
hookNeeded = true
|
||||||
ObjectLockConfig: enc(meta.ObjectLockConfigXML),
|
}
|
||||||
SSEConfig: enc(meta.EncryptionConfigXML),
|
if _, ok := fields[bucketPolicyConfig]; ok {
|
||||||
UpdatedAt: updatedAt,
|
hook.Policy = meta.PolicyConfigJSON
|
||||||
}); err != nil {
|
hookNeeded = hookNeeded || len(hook.Policy) != 0
|
||||||
|
}
|
||||||
|
if _, ok := fields[bucketVersioningConfig]; ok {
|
||||||
|
hook.Versioning = enc(meta.VersioningConfigXML)
|
||||||
|
hookNeeded = true
|
||||||
|
}
|
||||||
|
if _, ok := fields[bucketTaggingConfig]; ok {
|
||||||
|
hook.Tags = enc(meta.TaggingConfigXML)
|
||||||
|
hookNeeded = true
|
||||||
|
}
|
||||||
|
if _, ok := fields[objectLockConfig]; ok {
|
||||||
|
hook.ObjectLockConfig = enc(meta.ObjectLockConfigXML)
|
||||||
|
hookNeeded = true
|
||||||
|
}
|
||||||
|
if _, ok := fields[bucketSSEConfig]; ok {
|
||||||
|
hook.SSEConfig = enc(meta.EncryptionConfigXML)
|
||||||
|
hookNeeded = true
|
||||||
|
}
|
||||||
|
if hookNeeded {
|
||||||
|
err = globalSiteReplicationSys.BucketMetaHook(ctx, hook)
|
||||||
|
}
|
||||||
|
if _, ok := fields[bucketPolicyConfig]; ok && len(meta.PolicyConfigJSON) == 0 {
|
||||||
|
// An omitted bulk Policy cannot express deletion.
|
||||||
|
err = errors.Join(err, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||||
|
Type: madmin.SRBucketMetaTypePolicy, Bucket: bucket, UpdatedAt: commitAt,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
if _, ok := fields[bucketCorsConfig]; ok {
|
||||||
|
// CORS carries its own timestamp, so it replicates through the
|
||||||
|
// dedicated event rather than the shared bucket metadata hook. It
|
||||||
|
// is announced even when the shared hook failed: the document is
|
||||||
|
// already committed locally, and a peer that is unreachable for
|
||||||
|
// one config must not withhold CORS from the reachable ones.
|
||||||
|
if corsEvent, live := newBucketCORSReplicationEvent(bucket, *meta); live {
|
||||||
|
err = errors.Join(err, globalSiteReplicationSys.BucketMetaHook(ctx, corsEvent))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
rpt.SetStatus(bucket, "", err)
|
rpt.SetStatus(bucket, "", err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ import (
|
|||||||
"github.com/minio/madmin-go/v3"
|
"github.com/minio/madmin-go/v3"
|
||||||
"github.com/minio/minio/internal/auth"
|
"github.com/minio/minio/internal/auth"
|
||||||
"github.com/minio/minio/internal/config"
|
"github.com/minio/minio/internal/config"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
// validateAdminReq will validate request against and return whether it is allowed.
|
// validateAdminReq will validate request against and return whether it is allowed.
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ import (
|
|||||||
"github.com/minio/minio/internal/config/subnet"
|
"github.com/minio/minio/internal/config/subnet"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
// DelConfigKVHandler - DELETE /minio/admin/v3/del-config-kv
|
// DelConfigKVHandler - DELETE /minio/admin/v3/del-config-kv
|
||||||
|
|||||||
@@ -32,8 +32,8 @@ import (
|
|||||||
cfgldap "github.com/minio/minio/internal/config/identity/ldap"
|
cfgldap "github.com/minio/minio/internal/config/identity/ldap"
|
||||||
"github.com/minio/minio/internal/config/identity/openid"
|
"github.com/minio/minio/internal/config/identity/openid"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
"github.com/minio/pkg/v3/ldap"
|
"github.com/pgsty/silo-pkg/v3/ldap"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
|
func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
|
||||||
|
|||||||
@@ -28,8 +28,8 @@ import (
|
|||||||
"github.com/minio/madmin-go/v3"
|
"github.com/minio/madmin-go/v3"
|
||||||
"github.com/minio/minio/internal/auth"
|
"github.com/minio/minio/internal/auth"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
xldap "github.com/minio/pkg/v3/ldap"
|
xldap "github.com/pgsty/silo-pkg/v3/ldap"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ListLDAPPolicyMappingEntities lists users/groups mapped to given/all policies.
|
// ListLDAPPolicyMappingEntities lists users/groups mapped to given/all policies.
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ import (
|
|||||||
|
|
||||||
"github.com/minio/madmin-go/v3"
|
"github.com/minio/madmin-go/v3"
|
||||||
"github.com/minio/minio-go/v7/pkg/set"
|
"github.com/minio/minio-go/v7/pkg/set"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
const dummyRoleARN = "dummy-internal"
|
const dummyRoleARN = "dummy-internal"
|
||||||
|
|||||||
@@ -27,8 +27,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
"github.com/minio/pkg/v3/env"
|
"github.com/pgsty/silo-pkg/v3/env"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ import (
|
|||||||
"github.com/minio/madmin-go/v3"
|
"github.com/minio/madmin-go/v3"
|
||||||
xioutil "github.com/minio/minio/internal/ioutil"
|
xioutil "github.com/minio/minio/internal/ioutil"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
// SiteReplicationAdd - PUT /minio/admin/v3/site-replication/add
|
// SiteReplicationAdd - PUT /minio/admin/v3/site-replication/add
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ import (
|
|||||||
|
|
||||||
"github.com/minio/madmin-go/v3"
|
"github.com/minio/madmin-go/v3"
|
||||||
minio "github.com/minio/minio-go/v7"
|
minio "github.com/minio/minio-go/v7"
|
||||||
"github.com/minio/pkg/v3/sync/errgroup"
|
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||||
)
|
)
|
||||||
|
|
||||||
func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {
|
func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {
|
||||||
|
|||||||
@@ -40,8 +40,8 @@ import (
|
|||||||
"github.com/minio/minio/internal/config/dns"
|
"github.com/minio/minio/internal/config/dns"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
xldap "github.com/minio/pkg/v3/ldap"
|
xldap "github.com/pgsty/silo-pkg/v3/ldap"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
"github.com/puzpuzpuz/xsync/v3"
|
"github.com/puzpuzpuz/xsync/v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -502,11 +502,15 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
checkDenyOnly := accessKey == cred.AccessKey
|
checkDenyOnly := accessKey == cred.AccessKey
|
||||||
|
action := policy.Action(policy.CreateUserAdminAction)
|
||||||
|
if checkDenyOnly {
|
||||||
|
action = policy.ChangeMyPasswordAdminAction
|
||||||
|
}
|
||||||
|
|
||||||
if !globalIAMSys.IsAllowed(policy.Args{
|
if !globalIAMSys.IsAllowed(policy.Args{
|
||||||
AccountName: cred.AccessKey,
|
AccountName: cred.AccessKey,
|
||||||
Groups: cred.Groups,
|
Groups: cred.Groups,
|
||||||
Action: policy.CreateUserAdminAction,
|
Action: action,
|
||||||
ConditionValues: getConditionValues(r, "", cred),
|
ConditionValues: getConditionValues(r, "", cred),
|
||||||
IsOwner: owner,
|
IsOwner: owner,
|
||||||
Claims: cred.Claims,
|
Claims: cred.Claims,
|
||||||
|
|||||||
@@ -40,8 +40,8 @@ import (
|
|||||||
"github.com/minio/minio-go/v7/pkg/set"
|
"github.com/minio/minio-go/v7/pkg/set"
|
||||||
"github.com/minio/minio-go/v7/pkg/signer"
|
"github.com/minio/minio-go/v7/pkg/signer"
|
||||||
"github.com/minio/minio/internal/auth"
|
"github.com/minio/minio/internal/auth"
|
||||||
"github.com/minio/pkg/v3/env"
|
"github.com/pgsty/silo-pkg/v3/env"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -243,6 +243,7 @@ func TestIAMInternalIDPServerSuite(t *testing.T) {
|
|||||||
|
|
||||||
suite.SetUpSuite(c)
|
suite.SetUpSuite(c)
|
||||||
suite.TestUserCreate(c)
|
suite.TestUserCreate(c)
|
||||||
|
suite.TestUserPasswordActionAuthorization(c)
|
||||||
suite.TestUserStatusActionAuthorization(c)
|
suite.TestUserStatusActionAuthorization(c)
|
||||||
suite.TestGroupStatusActionAuthorization(c)
|
suite.TestGroupStatusActionAuthorization(c)
|
||||||
suite.TestUserPolicyEscalationBug(c)
|
suite.TestUserPolicyEscalationBug(c)
|
||||||
@@ -356,6 +357,106 @@ func (s *TestSuiteIAM) TestUserCreate(c *check) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *TestSuiteIAM) TestUserPasswordActionAuthorization(c *check) {
|
||||||
|
for _, tt := range []struct {
|
||||||
|
name string
|
||||||
|
statements string
|
||||||
|
self bool
|
||||||
|
other bool
|
||||||
|
}{
|
||||||
|
{"readonly", "", true, false},
|
||||||
|
{"consolereadonly", "", true, false},
|
||||||
|
{"password grant", `{"Effect":"Allow","Action":"admin:ChangeMyPassword"}`, true, false},
|
||||||
|
{"legacy CreateUser deny", `{"Effect":"Deny","Action":"admin:CreateUser","Resource":"arn:aws:s3:::*"}`, true, false},
|
||||||
|
{"password deny", `{"Effect":"Deny","Action":"admin:ChangeMyPassword"}`, false, false},
|
||||||
|
{"user admin", `{"Effect":"Allow","Action":"admin:CreateUser"}`, true, true},
|
||||||
|
{"user admin with password deny", `{"Effect":"Allow","Action":"admin:CreateUser"},{"Effect":"Deny","Action":"admin:ChangeMyPassword"}`, false, true},
|
||||||
|
{"password deny overrides grant", `{"Effect":"Allow","Action":"admin:ChangeMyPassword"},{"Effect":"Deny","Action":"admin:ChangeMyPassword"}`, false, false},
|
||||||
|
{"wildcard deny", `{"Effect":"Deny","Action":"admin:*"}`, false, false},
|
||||||
|
} {
|
||||||
|
c.Run(tt.name, func(t *testing.T) {
|
||||||
|
c := &check{t, s.serverType}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||||
|
defer cancel()
|
||||||
|
var users []string
|
||||||
|
policyName := tt.name
|
||||||
|
defer func() {
|
||||||
|
for _, user := range users {
|
||||||
|
if err := s.adm.RemoveUser(ctx, user); err != nil {
|
||||||
|
c.Errorf("remove test user: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if tt.statements != "" {
|
||||||
|
if err := s.adm.RemoveCannedPolicy(ctx, policyName); err != nil {
|
||||||
|
c.Errorf("remove test policy: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
createUser := func() (string, string) {
|
||||||
|
accessKey, secretKey := mustGenerateCredentials(c)
|
||||||
|
if err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled); err != nil {
|
||||||
|
c.Fatalf("create test user: %v", err)
|
||||||
|
}
|
||||||
|
users = append(users, accessKey)
|
||||||
|
return accessKey, secretKey
|
||||||
|
}
|
||||||
|
client := func(accessKey, secretKey string) *madmin.AdminClient {
|
||||||
|
adm, err := madmin.New(s.endpoint, accessKey, secretKey, s.secure)
|
||||||
|
if err != nil {
|
||||||
|
c.Fatal(err)
|
||||||
|
}
|
||||||
|
adm.SetCustomTransport(s.TestSuiteCommon.client.Transport)
|
||||||
|
return adm
|
||||||
|
}
|
||||||
|
if tt.statements != "" {
|
||||||
|
policyName = getRandomBucketName()
|
||||||
|
doc := []byte(`{"Version":"2012-10-17","Statement":[` + tt.statements + `]}`)
|
||||||
|
if err := s.adm.AddCannedPolicy(ctx, policyName, doc); err != nil {
|
||||||
|
c.Fatalf("save test policy: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
accessKey, secretKey := createUser()
|
||||||
|
if _, err := s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
|
||||||
|
User: accessKey, Policies: []string{policyName},
|
||||||
|
}); err != nil {
|
||||||
|
c.Fatalf("attach test policy: %v", err)
|
||||||
|
}
|
||||||
|
adm := client(accessKey, secretKey)
|
||||||
|
_, newSecretKey := mustGenerateCredentials(c)
|
||||||
|
err := adm.SetUser(ctx, accessKey, newSecretKey, madmin.AccountEnabled)
|
||||||
|
if tt.self {
|
||||||
|
if err != nil {
|
||||||
|
c.Fatalf("change own password: %v", err)
|
||||||
|
}
|
||||||
|
if _, err = adm.AccountInfo(ctx, madmin.AccountOpts{}); err == nil {
|
||||||
|
c.Fatal("old password still authenticates")
|
||||||
|
}
|
||||||
|
adm = client(accessKey, newSecretKey)
|
||||||
|
} else if err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||||
|
c.Fatalf("self password change: expected AccessDenied, got %v", err)
|
||||||
|
}
|
||||||
|
if _, err := adm.AccountInfo(ctx, madmin.AccountOpts{}); err != nil {
|
||||||
|
c.Fatalf("current password no longer authenticates: %v", err)
|
||||||
|
}
|
||||||
|
target, _ := createUser()
|
||||||
|
newUser, newUserSecret := mustGenerateCredentials(c)
|
||||||
|
for _, key := range []string{target, newUser} {
|
||||||
|
err := adm.SetUser(ctx, key, newUserSecret, madmin.AccountEnabled)
|
||||||
|
if tt.other {
|
||||||
|
if err != nil {
|
||||||
|
c.Fatalf("create or update another user: %v", err)
|
||||||
|
}
|
||||||
|
if key == newUser {
|
||||||
|
users = append(users, newUser)
|
||||||
|
}
|
||||||
|
} else if err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||||
|
c.Fatalf("create or update another user: expected AccessDenied, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (s *TestSuiteIAM) TestUserStatusActionAuthorization(c *check) {
|
func (s *TestSuiteIAM) TestUserStatusActionAuthorization(c *check) {
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
@@ -946,6 +1047,7 @@ func (s *TestSuiteIAM) TestCannedPolicies(c *check) {
|
|||||||
defaultPolicies := []string{
|
defaultPolicies := []string{
|
||||||
"readwrite",
|
"readwrite",
|
||||||
"readonly",
|
"readonly",
|
||||||
|
"consolereadonly",
|
||||||
"writeonly",
|
"writeonly",
|
||||||
"diagnostics",
|
"diagnostics",
|
||||||
"consoleAdmin",
|
"consoleAdmin",
|
||||||
|
|||||||
@@ -60,8 +60,8 @@ import (
|
|||||||
"github.com/minio/minio/internal/kms"
|
"github.com/minio/minio/internal/kms"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
xnet "github.com/minio/pkg/v3/net"
|
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
"github.com/secure-io/sio-go"
|
"github.com/secure-io/sio-go"
|
||||||
"github.com/zeebo/xxh3"
|
"github.com/zeebo/xxh3"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -388,6 +388,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
|||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(adminMiddleware(adminAPI.SRPeerJoin))
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(adminMiddleware(adminAPI.SRPeerJoin))
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(adminMiddleware(adminAPI.SRPeerBucketOps)).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
|
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(adminMiddleware(adminAPI.SRPeerBucketOps)).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateIAMItem))
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateIAMItem))
|
||||||
|
adminRouter.Methods(http.MethodGet, http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-revisions").HandlerFunc(adminMiddleware(adminAPI.SRPeerIAMRevisions))
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateBucketItem))
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateBucketItem))
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(adminMiddleware(adminAPI.SRPeerGetIDPSettings))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(adminMiddleware(adminAPI.SRPeerGetIDPSettings))
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationEdit))
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationEdit))
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ import (
|
|||||||
"github.com/minio/madmin-go/v3"
|
"github.com/minio/madmin-go/v3"
|
||||||
"github.com/minio/minio/internal/config"
|
"github.com/minio/minio/internal/config"
|
||||||
"github.com/minio/minio/internal/kms"
|
"github.com/minio/minio/internal/kms"
|
||||||
xnet "github.com/minio/pkg/v3/net"
|
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||||
)
|
)
|
||||||
|
|
||||||
// getLocalServerProperty - returns madmin.ServerProperties for only the
|
// getLocalServerProperty - returns madmin.ServerProperties for only the
|
||||||
|
|||||||
+6
-2
@@ -48,7 +48,7 @@ import (
|
|||||||
levent "github.com/minio/minio/internal/config/lambda/event"
|
levent "github.com/minio/minio/internal/config/lambda/event"
|
||||||
"github.com/minio/minio/internal/event"
|
"github.com/minio/minio/internal/event"
|
||||||
"github.com/minio/minio/internal/hash"
|
"github.com/minio/minio/internal/hash"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
// APIError structure
|
// APIError structure
|
||||||
@@ -1523,10 +1523,14 @@ var errorCodes = errorCodeMap{
|
|||||||
Description: "Your Host header is malformed.",
|
Description: "Your Host header is malformed.",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
|
// The stored object cannot be served: a server-side data condition, not a
|
||||||
|
// successful partial read. Upstream maps it to http.StatusPartialContent
|
||||||
|
// (since ca6b4773e, 2017), which lets SDKs accept the XML error document
|
||||||
|
// as object content; SILO deliberately diverges and returns 500.
|
||||||
ErrObjectTampered: {
|
ErrObjectTampered: {
|
||||||
Code: "XMinioObjectTampered",
|
Code: "XMinioObjectTampered",
|
||||||
Description: errObjectTampered.Error(),
|
Description: errObjectTampered.Error(),
|
||||||
HTTPStatusCode: http.StatusPartialContent,
|
HTTPStatusCode: http.StatusInternalServerError,
|
||||||
},
|
},
|
||||||
|
|
||||||
ErrSiteReplicationInvalidRequest: {
|
ErrSiteReplicationInvalidRequest: {
|
||||||
|
|||||||
+4
-1
@@ -212,7 +212,10 @@ func setObjectHeaders(ctx context.Context, w http.ResponseWriter, objInfo Object
|
|||||||
}
|
}
|
||||||
|
|
||||||
if rs == nil && opts.PartNumber > 0 {
|
if rs == nil && opts.PartNumber > 0 {
|
||||||
rs = partNumberToRangeSpec(objInfo, opts.PartNumber)
|
rs, err = partNumberToRangeSpec(objInfo, opts.PartNumber)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// For providing ranged content
|
// For providing ranged content
|
||||||
|
|||||||
+1
-1
@@ -34,8 +34,8 @@ import (
|
|||||||
"github.com/minio/minio/internal/hash"
|
"github.com/minio/minio/internal/hash"
|
||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/pkg/v3/policy"
|
|
||||||
xxml "github.com/minio/xxml"
|
xxml "github.com/minio/xxml"
|
||||||
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|||||||
+11
-13
@@ -29,7 +29,7 @@ import (
|
|||||||
bktcors "github.com/minio/minio/internal/bucket/cors"
|
bktcors "github.com/minio/minio/internal/bucket/cors"
|
||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
"github.com/minio/pkg/v3/wildcard"
|
"github.com/pgsty/silo-pkg/v3/wildcard"
|
||||||
"github.com/rs/cors"
|
"github.com/rs/cors"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -461,15 +461,15 @@ func registerAPIRouter(router *mux.Router) {
|
|||||||
router.Methods(http.MethodPut).
|
router.Methods(http.MethodPut).
|
||||||
HandlerFunc(s3APIMiddleware(api.PutBucketACLHandler)).
|
HandlerFunc(s3APIMiddleware(api.PutBucketACLHandler)).
|
||||||
Queries("acl", "")
|
Queries("acl", "")
|
||||||
// GetBucketCors - this is a dummy call.
|
// GetBucketCors
|
||||||
router.Methods(http.MethodGet).
|
router.Methods(http.MethodGet).
|
||||||
HandlerFunc(s3APIMiddleware(api.GetBucketCorsHandler)).
|
HandlerFunc(s3APIMiddleware(api.GetBucketCorsHandler)).
|
||||||
Queries("cors", "")
|
Queries("cors", "")
|
||||||
// PutBucketCors - this is a dummy call.
|
// PutBucketCors
|
||||||
router.Methods(http.MethodPut).
|
router.Methods(http.MethodPut).
|
||||||
HandlerFunc(s3APIMiddleware(api.PutBucketCorsHandler)).
|
HandlerFunc(s3APIMiddleware(api.PutBucketCorsHandler)).
|
||||||
Queries("cors", "")
|
Queries("cors", "")
|
||||||
// DeleteBucketCors - this is a dummy call.
|
// DeleteBucketCors
|
||||||
router.Methods(http.MethodDelete).
|
router.Methods(http.MethodDelete).
|
||||||
HandlerFunc(s3APIMiddleware(api.DeleteBucketCorsHandler)).
|
HandlerFunc(s3APIMiddleware(api.DeleteBucketCorsHandler)).
|
||||||
Queries("cors", "")
|
Queries("cors", "")
|
||||||
@@ -787,15 +787,13 @@ func corsHandler(handler http.Handler) http.Handler {
|
|||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.Header.Get("Origin") != "" {
|
if r.Header.Get("Origin") != "" {
|
||||||
if bucket, _ := request2BucketObjectName(r); bucket != "" && globalBucketMetadataSys != nil {
|
if bucket, _ := request2BucketObjectName(r); bucket != "" && globalBucketMetadataSys != nil {
|
||||||
// Resident-only lookup: this runs pre-auth for every
|
// Resident-only lookup: this runs before authentication with a
|
||||||
// Origin-bearing request using a client-supplied path segment as
|
// client-supplied path segment as the bucket name, so it must
|
||||||
// the bucket name. It must never load or cache metadata for
|
// never load or cache metadata. While startup loading is still
|
||||||
// arbitrary names (see GetResidentCorsConfig). GetResidentCorsConfig
|
// running, for a real bucket whose metadata failed to load, and
|
||||||
// is the single decision point: it returns errInvalidArgument for
|
// for a bucket whose stored CORS document failed to parse, the
|
||||||
// the internal .minio.sys namespace (fail closed), a config for a
|
// request gets no CORS headers; any other non-resident name falls
|
||||||
// resident bucket, errBucketMetadataNotInitialized for a real but
|
// back to the global policy below.
|
||||||
// unloaded bucket (fail closed), and errConfigNotFound otherwise
|
|
||||||
// (fall back to the global policy below).
|
|
||||||
cfg, _, err := globalBucketMetadataSys.GetResidentCorsConfig(bucket)
|
cfg, _, err := globalBucketMetadataSys.GetResidentCorsConfig(bucket)
|
||||||
if err == nil && cfg != nil {
|
if err == nil && cfg != nil {
|
||||||
if applyBucketCors(w, r, cfg) {
|
if applyBucketCors(w, r, cfg) {
|
||||||
|
|||||||
+5
-12
@@ -41,7 +41,7 @@ import (
|
|||||||
xjwt "github.com/minio/minio/internal/jwt"
|
xjwt "github.com/minio/minio/internal/jwt"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/minio/internal/mcontext"
|
"github.com/minio/minio/internal/mcontext"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Verify if request has JWT.
|
// Verify if request has JWT.
|
||||||
@@ -632,18 +632,11 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
|
|||||||
return ErrInvalidDigest
|
return ErrInvalidDigest
|
||||||
}
|
}
|
||||||
|
|
||||||
// Extract either 'X-Amz-Content-Sha256' header or 'X-Amz-Content-Sha256' query parameter (if V4 presigned)
|
// Honor the selected header/query checksum, including the header fallback
|
||||||
// Do not verify 'X-Amz-Content-Sha256' if skipSHA256.
|
// for a presigned request. STS separately hashes its body for its signature.
|
||||||
var contentSHA256 []byte
|
var contentSHA256 []byte
|
||||||
if skipSHA256 := skipContentSha256Cksum(r); !skipSHA256 && isRequestPresignedSignatureV4(r) {
|
if !skipContentSha256Cksum(r) {
|
||||||
if sha256Sum, ok := r.Form[xhttp.AmzContentSha256]; ok && len(sha256Sum) > 0 {
|
contentSHA256, err = hex.DecodeString(getContentSha256Cksum(r, serviceS3))
|
||||||
contentSHA256, err = hex.DecodeString(sha256Sum[0])
|
|
||||||
if err != nil {
|
|
||||||
return ErrContentSHA256Mismatch
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if _, ok := r.Header[xhttp.AmzContentSha256]; !skipSHA256 && ok {
|
|
||||||
contentSHA256, err = hex.DecodeString(r.Header.Get(xhttp.AmzContentSha256))
|
|
||||||
if err != nil || len(contentSHA256) == 0 {
|
if err != nil || len(contentSHA256) == 0 {
|
||||||
return ErrContentSHA256Mismatch
|
return ErrContentSHA256Mismatch
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/minio/minio/internal/auth"
|
"github.com/minio/minio/internal/auth"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
type nullReader struct{}
|
type nullReader struct{}
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/minio/madmin-go/v3"
|
"github.com/minio/madmin-go/v3"
|
||||||
"github.com/minio/pkg/v3/env"
|
"github.com/pgsty/silo-pkg/v3/env"
|
||||||
)
|
)
|
||||||
|
|
||||||
// healTask represents what to heal along with options
|
// healTask represents what to heal along with options
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ import (
|
|||||||
"github.com/minio/madmin-go/v3"
|
"github.com/minio/madmin-go/v3"
|
||||||
"github.com/minio/minio-go/v7/pkg/set"
|
"github.com/minio/minio-go/v7/pkg/set"
|
||||||
"github.com/minio/minio/internal/config"
|
"github.com/minio/minio/internal/config"
|
||||||
"github.com/minio/pkg/v3/env"
|
"github.com/pgsty/silo-pkg/v3/env"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|||||||
+4
-4
@@ -33,10 +33,10 @@ import (
|
|||||||
"github.com/minio/minio/internal/bucket/versioning"
|
"github.com/minio/minio/internal/bucket/versioning"
|
||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
xioutil "github.com/minio/minio/internal/ioutil"
|
xioutil "github.com/minio/minio/internal/ioutil"
|
||||||
"github.com/minio/pkg/v3/env"
|
"github.com/pgsty/silo-pkg/v3/env"
|
||||||
"github.com/minio/pkg/v3/wildcard"
|
"github.com/pgsty/silo-pkg/v3/wildcard"
|
||||||
"github.com/minio/pkg/v3/workers"
|
"github.com/pgsty/silo-pkg/v3/workers"
|
||||||
"github.com/minio/pkg/v3/xtime"
|
"github.com/pgsty/silo-pkg/v3/xtime"
|
||||||
"go.yaml.in/yaml/v3"
|
"go.yaml.in/yaml/v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -48,10 +48,10 @@ import (
|
|||||||
"github.com/minio/minio/internal/hash"
|
"github.com/minio/minio/internal/hash"
|
||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
xioutil "github.com/minio/minio/internal/ioutil"
|
xioutil "github.com/minio/minio/internal/ioutil"
|
||||||
"github.com/minio/pkg/v3/console"
|
"github.com/pgsty/silo-pkg/v3/console"
|
||||||
"github.com/minio/pkg/v3/env"
|
"github.com/pgsty/silo-pkg/v3/env"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
"github.com/minio/pkg/v3/workers"
|
"github.com/pgsty/silo-pkg/v3/workers"
|
||||||
"go.yaml.in/yaml/v3"
|
"go.yaml.in/yaml/v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
"github.com/minio/pkg/v3/wildcard"
|
"github.com/pgsty/silo-pkg/v3/wildcard"
|
||||||
"go.yaml.in/yaml/v3"
|
"go.yaml.in/yaml/v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ import (
|
|||||||
|
|
||||||
miniogo "github.com/minio/minio-go/v7"
|
miniogo "github.com/minio/minio-go/v7"
|
||||||
"github.com/minio/minio/internal/auth"
|
"github.com/minio/minio/internal/auth"
|
||||||
"github.com/minio/pkg/v3/xtime"
|
"github.com/pgsty/silo-pkg/v3/xtime"
|
||||||
)
|
)
|
||||||
|
|
||||||
//go:generate msgp -file $GOFILE
|
//go:generate msgp -file $GOFILE
|
||||||
|
|||||||
+2
-2
@@ -34,8 +34,8 @@ import (
|
|||||||
"github.com/minio/minio/internal/crypto"
|
"github.com/minio/minio/internal/crypto"
|
||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/minio/minio/internal/kms"
|
"github.com/minio/minio/internal/kms"
|
||||||
"github.com/minio/pkg/v3/env"
|
"github.com/pgsty/silo-pkg/v3/env"
|
||||||
"github.com/minio/pkg/v3/workers"
|
"github.com/pgsty/silo-pkg/v3/workers"
|
||||||
)
|
)
|
||||||
|
|
||||||
// keyrotate:
|
// keyrotate:
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ import (
|
|||||||
"github.com/minio/minio-go/v7/pkg/set"
|
"github.com/minio/minio-go/v7/pkg/set"
|
||||||
"github.com/minio/minio/internal/grid"
|
"github.com/minio/minio/internal/grid"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/pkg/v3/env"
|
"github.com/pgsty/silo-pkg/v3/env"
|
||||||
)
|
)
|
||||||
|
|
||||||
// To abstract a node over network.
|
// To abstract a node over network.
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ import (
|
|||||||
hashpkg "github.com/minio/minio/internal/hash"
|
hashpkg "github.com/minio/minio/internal/hash"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
// maxBucketCorsSize is the maximum allowed size of a CORS configuration document.
|
// maxBucketCorsSize is the maximum allowed size of a CORS configuration document.
|
||||||
|
|||||||
@@ -261,6 +261,11 @@ func TestBucketCorsMetadataErrorFailsClosed(t *testing.T) {
|
|||||||
oldMetadataSys := globalBucketMetadataSys
|
oldMetadataSys := globalBucketMetadataSys
|
||||||
setObjectLayer(nil)
|
setObjectLayer(nil)
|
||||||
globalBucketMetadataSys = NewBucketMetadataSys()
|
globalBucketMetadataSys = NewBucketMetadataSys()
|
||||||
|
// A resident bucket whose stored CORS document does not parse must not be
|
||||||
|
// answered with the global policy: it has a configuration we cannot honor.
|
||||||
|
meta := newBucketMetadata("cors-metadata-error")
|
||||||
|
meta.corsConfigErr = fmt.Errorf("invalid bucket CORS configuration")
|
||||||
|
globalBucketMetadataSys.Set("cors-metadata-error", meta)
|
||||||
defer func() {
|
defer func() {
|
||||||
setObjectLayer(oldObjectAPI)
|
setObjectLayer(oldObjectAPI)
|
||||||
globalBucketMetadataSys = oldMetadataSys
|
globalBucketMetadataSys = oldMetadataSys
|
||||||
@@ -707,34 +712,32 @@ func testBucketCorsLoadFailedBucketFailsClosed(obj ObjectLayer, _ string, _ stri
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestBucketCorsInternalBucketFailsClosed guards P2: an Origin-bearing request
|
// TestBucketCorsResidentConfigSurvivesRefreshFailure: a resident bucket keeps
|
||||||
// whose first path segment is the reserved .minio.sys namespace must preserve
|
// its last loaded CORS configuration through a failed refresh, like every
|
||||||
// GetConfig's errInvalidArgument semantics and fail closed, not fall back to
|
// other bucket configuration, and the failure set never records a resident
|
||||||
// the permissive global CORS policy.
|
// bucket. Only a bucket that was never loaded fails closed.
|
||||||
func TestBucketCorsInternalBucketFailsClosed(t *testing.T) {
|
func TestBucketCorsResidentConfigSurvivesRefreshFailure(t *testing.T) {
|
||||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
t: t,
|
t: t,
|
||||||
objAPITest: testBucketCorsInternalBucketFailsClosed,
|
objAPITest: testBucketCorsResidentConfigSurvivesRefreshFailure,
|
||||||
endpoints: []string{"GetBucketCors"},
|
endpoints: []string{"GetBucketCors"},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func testBucketCorsInternalBucketFailsClosed(_ ObjectLayer, _ string, _ string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
func testBucketCorsResidentConfigSurvivesRefreshFailure(obj ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||||
restoreInit := markBucketMetadataInitialized(t)
|
if _, err := updateLocalBucketCORSMetadata(t.Context(), obj, bucket, []byte(testSiteReplicationCORSDoc)); err != nil {
|
||||||
defer restoreInit()
|
t.Fatal(err)
|
||||||
|
|
||||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusNotFound)
|
|
||||||
}))
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
req := httptest.NewRequest(http.MethodGet, getGetObjectURL("", minioMetaBucket, "object"), nil)
|
|
||||||
req.Header.Set("Origin", "https://app.example.com")
|
|
||||||
wrapped.ServeHTTP(rec, req)
|
|
||||||
|
|
||||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
|
|
||||||
t.Fatalf("internal bucket fell back to global allow-origin %q", got)
|
|
||||||
}
|
}
|
||||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "" {
|
sys := globalBucketMetadataSys
|
||||||
t.Fatalf("internal bucket fell back to global credentials %q", got)
|
sys.Lock()
|
||||||
|
sys.noteLoadFailure(bucket)
|
||||||
|
_, marked := sys.loadFailed[bucket]
|
||||||
|
sys.Unlock()
|
||||||
|
if marked {
|
||||||
|
t.Fatal("a resident bucket was recorded as a load failure")
|
||||||
|
}
|
||||||
|
cfg, _, err := sys.GetResidentCorsConfig(bucket)
|
||||||
|
if err != nil || cfg == nil {
|
||||||
|
t.Fatalf("resident CORS configuration lost after a refresh failure: cfg=%v err=%v", cfg, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -85,7 +85,7 @@ func testPeerBucketCorsReplicationOrdering(_ ObjectLayer, _ string, bucket strin
|
|||||||
if !meta.CorsConfigUpdatedAt.Equal(putAt) {
|
if !meta.CorsConfigUpdatedAt.Equal(putAt) {
|
||||||
t.Fatalf("peer PUT timestamp = %v, want source time %v", meta.CorsConfigUpdatedAt, putAt)
|
t.Fatalf("peer PUT timestamp = %v, want source time %v", meta.CorsConfigUpdatedAt, putAt)
|
||||||
}
|
}
|
||||||
cfg, cfgAt, err := globalBucketMetadataSys.GetCorsConfig(bucket)
|
cfg, cfgAt, err := globalBucketMetadataSys.GetResidentCorsConfig(bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("peer PUT stored raw XML but no parsed config: %v", err)
|
t.Fatalf("peer PUT stored raw XML but no parsed config: %v", err)
|
||||||
}
|
}
|
||||||
@@ -165,10 +165,10 @@ func TestSiteReplicationMetaInfoPreservesCorsTombstone(t *testing.T) {
|
|||||||
|
|
||||||
func testSiteReplicationMetaInfoPreservesCorsTombstone(obj ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
func testSiteReplicationMetaInfoPreservesCorsTombstone(obj ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketCorsConfig, []byte(testSiteReplicationCORSDoc)); err != nil {
|
if _, err := updateLocalBucketCORSMetadata(ctx, obj, bucket, []byte(testSiteReplicationCORSDoc)); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
deleteAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketCorsConfig)
|
deleteAt, err := updateLocalBucketCORSMetadata(ctx, obj, bucket, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -467,25 +467,6 @@ func testAdversarialHealCorsPropagatesNewerEqualValueTimestamp(obj ObjectLayer,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAdversarialBucketMetadataComparisonIsBase64CaseSensitive(t *testing.T) {
|
|
||||||
upper := "QQ=="
|
|
||||||
lower := "qQ=="
|
|
||||||
upperBytes, err := base64.StdEncoding.Strict().DecodeString(upper)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
lowerBytes, err := base64.StdEncoding.Strict().DecodeString(lower)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if string(upperBytes) == string(lowerBytes) {
|
|
||||||
t.Fatal("test inputs unexpectedly decode to the same bytes")
|
|
||||||
}
|
|
||||||
if isBucketMetadataEqual(&upper, &lower) {
|
|
||||||
t.Fatal("different decoded payloads were treated as equal")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSiteReplicationStatusDetectsCorsTimestampMismatch(t *testing.T) {
|
func TestSiteReplicationStatusDetectsCorsTimestampMismatch(t *testing.T) {
|
||||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
t: t,
|
t: t,
|
||||||
@@ -918,8 +899,8 @@ func testLegacyInvalidCorsMetadataCanBeDeleted(obj ObjectLayer, _ string, bucket
|
|||||||
t.Fatalf("legacy CORS state = (%#v, %v), want fail-closed parse error", loaded.corsConfig, loaded.corsConfigErr)
|
t.Fatalf("legacy CORS state = (%#v, %v), want fail-closed parse error", loaded.corsConfig, loaded.corsConfigErr)
|
||||||
}
|
}
|
||||||
globalBucketMetadataSys.Set(bucket, loaded)
|
globalBucketMetadataSys.Set(bucket, loaded)
|
||||||
if _, gotAt, err := globalBucketMetadataSys.GetCorsConfig(bucket); err == nil || !gotAt.Equal(legacyAt) {
|
if _, gotAt, err := globalBucketMetadataSys.GetResidentCorsConfig(bucket); err == nil || !gotAt.Equal(legacyAt) {
|
||||||
t.Fatalf("GetCorsConfig = timestamp %v, error %v; want legacy timestamp and error", gotAt, err)
|
t.Fatalf("GetResidentCorsConfig = timestamp %v, error %v; want legacy timestamp and error", gotAt, err)
|
||||||
}
|
}
|
||||||
if _, gotAt, err := globalBucketMetadataSys.GetCorsConfigXML(bucket); err == nil || !gotAt.Equal(legacyAt) {
|
if _, gotAt, err := globalBucketMetadataSys.GetCorsConfigXML(bucket); err == nil || !gotAt.Equal(legacyAt) {
|
||||||
t.Fatalf("GetCorsConfigXML = timestamp %v, error %v; want legacy timestamp and error", gotAt, err)
|
t.Fatalf("GetCorsConfigXML = timestamp %v, error %v; want legacy timestamp and error", gotAt, err)
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ import (
|
|||||||
"github.com/minio/minio/internal/kms"
|
"github.com/minio/minio/internal/kms"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|||||||
@@ -62,8 +62,8 @@ import (
|
|||||||
"github.com/minio/minio/internal/ioutil"
|
"github.com/minio/minio/internal/ioutil"
|
||||||
"github.com/minio/minio/internal/kms"
|
"github.com/minio/minio/internal/kms"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
"github.com/minio/pkg/v3/sync/errgroup"
|
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -509,10 +509,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
|||||||
reqInfo.ObjectName = object.ObjectName
|
reqInfo.ObjectName = object.ObjectName
|
||||||
reqInfo.VersionID = object.VersionID
|
reqInfo.VersionID = object.VersionID
|
||||||
if apiErrCode := authorizeRequest(ctx, r, deleteObjectAction(object.VersionID)); apiErrCode != ErrNone {
|
if apiErrCode := authorizeRequest(ctx, r, deleteObjectAction(object.VersionID)); apiErrCode != ErrNone {
|
||||||
if apiErrCode == ErrSignatureDoesNotMatch || apiErrCode == ErrInvalidAccessKeyID {
|
|
||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
apiErr := errorCodes.ToAPIErr(apiErrCode)
|
apiErr := errorCodes.ToAPIErr(apiErrCode)
|
||||||
deleteResults[index].errInfo = DeleteError{
|
deleteResults[index].errInfo = DeleteError{
|
||||||
Code: apiErr.Code,
|
Code: apiErr.Code,
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ import (
|
|||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ import (
|
|||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/minio/internal/s3select"
|
"github.com/minio/minio/internal/s3select"
|
||||||
xnet "github.com/minio/pkg/v3/net"
|
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||||
"github.com/zeebo/xxh3"
|
"github.com/zeebo/xxh3"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ import (
|
|||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
|
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Validate all the ListObjects query arguments, returns an APIErrorCode
|
// Validate all the ListObjects query arguments, returns an APIErrorCode
|
||||||
|
|||||||
@@ -0,0 +1,455 @@
|
|||||||
|
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||||
|
// Copyright (c) 2026 PGSTY
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/madmin-go/v3"
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||||
|
"github.com/minio/minio/internal/bucket/versioning"
|
||||||
|
)
|
||||||
|
|
||||||
|
type metadataRMWWriterKey struct{}
|
||||||
|
|
||||||
|
type metadataRMWBarrierObjectLayer struct {
|
||||||
|
ObjectLayer
|
||||||
|
bucket string
|
||||||
|
aReady chan struct{}
|
||||||
|
aRelease chan struct{}
|
||||||
|
bLockAttempt chan struct{}
|
||||||
|
aReadyOnce sync.Once
|
||||||
|
bLockOnce sync.Once
|
||||||
|
cancelOnce sync.Once
|
||||||
|
cancelOnPut context.CancelFunc
|
||||||
|
reads atomic.Int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *metadataRMWBarrierObjectLayer) metadataObject() string {
|
||||||
|
return pathJoin(bucketMetaPrefix, o.bucket, bucketMetadataFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *metadataRMWBarrierObjectLayer) metadataLock() string {
|
||||||
|
return pathJoin(bucketMetaPrefix, o.bucket, "metadata.lock")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *metadataRMWBarrierObjectLayer) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
|
||||||
|
if bucket == minioMetaBucket && object == o.metadataObject() {
|
||||||
|
o.reads.Add(1)
|
||||||
|
}
|
||||||
|
return o.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *metadataRMWBarrierObjectLayer) PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
|
||||||
|
if bucket == minioMetaBucket && object == o.metadataObject() && o.cancelOnPut != nil {
|
||||||
|
o.cancelOnce.Do(o.cancelOnPut)
|
||||||
|
}
|
||||||
|
if bucket == minioMetaBucket && object == o.metadataObject() && ctx.Value(metadataRMWWriterKey{}) == "A" {
|
||||||
|
o.aReadyOnce.Do(func() { close(o.aReady) })
|
||||||
|
select {
|
||||||
|
case <-o.aRelease:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ObjectInfo{}, ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return o.ObjectLayer.PutObject(ctx, bucket, object, data, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *metadataRMWBarrierObjectLayer) NewNSLock(bucket string, objects ...string) RWLocker {
|
||||||
|
lock := o.ObjectLayer.NewNSLock(bucket, objects...)
|
||||||
|
if bucket != minioMetaBucket || len(objects) != 1 || objects[0] != o.metadataLock() {
|
||||||
|
return lock
|
||||||
|
}
|
||||||
|
return metadataObservedRWLocker{RWLocker: lock, onLock: func(ctx context.Context) {
|
||||||
|
if ctx.Value(metadataRMWWriterKey{}) == "B" {
|
||||||
|
o.bLockOnce.Do(func() { close(o.bLockAttempt) })
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
type metadataObservedRWLocker struct {
|
||||||
|
RWLocker
|
||||||
|
onLock func(context.Context)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l metadataObservedRWLocker) GetLock(ctx context.Context, timeout *dynamicTimeout) (LockContext, error) {
|
||||||
|
l.onLock(ctx)
|
||||||
|
return l.RWLocker.GetLock(ctx, timeout)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBucketMetadataLockPreservesPolicyAndCORS(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testBucketMetadataLockPreservesPolicyAndCORS,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testBucketMetadataLockPreservesPolicyAndCORS(obj ObjectLayer, instanceType, bucket string,
|
||||||
|
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
policyJSON := fmt.Appendf(nil, `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket)
|
||||||
|
corsXML := []byte(testSiteReplicationCORSDoc)
|
||||||
|
runBucketMetadataRMWConflict(t, obj, bucket,
|
||||||
|
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||||
|
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, policyJSON)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||||
|
_, err := updateLocalBucketCORSMetadata(ctx, objectAPI, bucket, corsXML)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
func(meta BucketMetadata) bool {
|
||||||
|
return bytes.Equal(meta.PolicyConfigJSON, policyJSON) && bytes.Equal(meta.CorsConfigXML, corsXML)
|
||||||
|
}, instanceType+": policy+CORS")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBucketMetadataLockPreservesTaggingAndSSE(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testBucketMetadataLockPreservesTaggingAndSSE,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBucketMetadataLockPreservesPeerBulkAndLocalUpdate(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testBucketMetadataLockPreservesPeerBulkAndLocalUpdate,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testBucketMetadataLockPreservesPeerBulkAndLocalUpdate(obj ObjectLayer, instanceType, bucket string,
|
||||||
|
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
meta, err := readBucketMetadata(t.Context(), obj, bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
policyJSON := fmt.Appendf(nil, `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket)
|
||||||
|
tagXML := []byte(`<Tagging><TagSet><Tag><Key>local</Key><Value>tag</Value></Tag></TagSet></Tagging>`)
|
||||||
|
runBucketMetadataRMWConflict(t, obj, bucket,
|
||||||
|
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||||
|
return globalSiteReplicationSys.PeerBucketMetadataUpdateHandler(ctx, madmin.SRBucketMeta{
|
||||||
|
Bucket: bucket, Policy: policyJSON, UpdatedAt: meta.Created.Add(time.Second),
|
||||||
|
})
|
||||||
|
},
|
||||||
|
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||||
|
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, tagXML)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
func(meta BucketMetadata) bool {
|
||||||
|
return bytes.Equal(meta.PolicyConfigJSON, policyJSON) && bytes.Equal(meta.TaggingConfigXML, tagXML)
|
||||||
|
}, instanceType+": peer bulk+local tagging")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBucketMetadataLockPreservesLifecycleDeleteAndSSE(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testBucketMetadataLockPreservesLifecycleDeleteAndSSE,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testBucketMetadataLockPreservesLifecycleDeleteAndSSE(obj ObjectLayer, instanceType, bucket string,
|
||||||
|
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
lifecycleXML := []byte(`<LifecycleConfiguration><Rule><ID>expire</ID><Filter><Prefix>logs/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>`)
|
||||||
|
if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketLifecycleConfig, lifecycleXML); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sseXML := []byte(`<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`)
|
||||||
|
runBucketMetadataRMWConflict(t, obj, bucket,
|
||||||
|
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||||
|
_, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketLifecycleConfig)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||||
|
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, sseXML)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
func(meta BucketMetadata) bool {
|
||||||
|
cfg, err := lifecycle.ParseLifecycleConfig(bytes.NewReader(meta.LifecycleConfigXML))
|
||||||
|
return err == nil && cfg.ExpiryUpdatedAt != nil && len(cfg.Rules) == 0 && bytes.Equal(meta.EncryptionConfigXML, sseXML)
|
||||||
|
}, instanceType+": lifecycle delete+SSE")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMakeBucketForceCreatePreservesMetadata(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testMakeBucketForceCreatePreservesMetadata,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testMakeBucketForceCreatePreservesMetadata(obj ObjectLayer, instanceType, bucket string,
|
||||||
|
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
ctx := t.Context()
|
||||||
|
policyJSON := fmt.Appendf(nil, `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket)
|
||||||
|
corsXML := []byte(testSiteReplicationCORSDoc)
|
||||||
|
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, policyJSON); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := updateLocalBucketCORSMetadata(ctx, obj, bucket, corsXML); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
before, err := readBucketMetadata(ctx, obj, bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{ForceCreate: true}); err != nil {
|
||||||
|
t.Fatalf("%s: ForceCreate existing bucket: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
after, err := readBucketMetadata(ctx, obj, bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !after.Created.Equal(before.Created) || !bytes.Equal(after.PolicyConfigJSON, policyJSON) || !bytes.Equal(after.CorsConfigXML, corsXML) {
|
||||||
|
t.Fatalf("%s: ForceCreate replaced metadata: before=%+v after=%+v", instanceType, before, after)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyImportedBucketMetadataPreservesUnspecifiedFields(t *testing.T) {
|
||||||
|
policyJSON := []byte(`{"Version":"2012-10-17","Statement":[]}`)
|
||||||
|
tagXML := []byte(`<Tagging><TagSet><Tag><Key>existing</Key><Value>tag</Value></Tag></TagSet></Tagging>`)
|
||||||
|
src := newBucketMetadata("bucket")
|
||||||
|
src.PolicyConfigJSON = policyJSON
|
||||||
|
src.PolicyConfigUpdatedAt = UTCNow()
|
||||||
|
dst := newBucketMetadata("bucket")
|
||||||
|
dst.TaggingConfigXML = bytes.Clone(tagXML)
|
||||||
|
|
||||||
|
applyImportedBucketMetadata(&dst, src, importMetadataFields{bucketPolicyConfig: {}})
|
||||||
|
if !bytes.Equal(dst.PolicyConfigJSON, policyJSON) || !bytes.Equal(dst.TaggingConfigXML, tagXML) {
|
||||||
|
t.Fatalf("import patch overwrote unspecified metadata: %+v", dst)
|
||||||
|
}
|
||||||
|
src.PolicyConfigJSON[0] = '!'
|
||||||
|
if dst.PolicyConfigJSON[0] == '!' {
|
||||||
|
t.Fatal("import patch retained the source byte slice")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMakeBucketDoesNotAdoptGhostMetadata(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testMakeBucketDoesNotAdoptGhostMetadata,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testMakeBucketDoesNotAdoptGhostMetadata(obj ObjectLayer, instanceType, _ string,
|
||||||
|
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
ctx := t.Context()
|
||||||
|
bucket := getRandomBucketName()
|
||||||
|
if err := obj.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
policyJSON := fmt.Appendf(nil, `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket)
|
||||||
|
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, policyJSON); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
oldMeta, err := readBucketMetadata(ctx, obj, bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
z, ok := obj.(*erasureServerPools)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("%s: object layer is %T, want *erasureServerPools", instanceType, obj)
|
||||||
|
}
|
||||||
|
if err = z.s3Peer.DeleteBucket(ctx, bucket, DeleteBucketOptions{Force: true}); err != nil {
|
||||||
|
t.Fatalf("%s: delete bucket volume only: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
if err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||||
|
t.Fatalf("%s: recreate bucket: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
newMeta, err := readBucketMetadata(ctx, obj, bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if bytes.Equal(newMeta.PolicyConfigJSON, policyJSON) || newMeta.Created.Equal(oldMeta.Created) {
|
||||||
|
t.Fatalf("%s: new bucket adopted ghost metadata: old=%+v new=%+v", instanceType, oldMeta, newMeta)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMakeBucketForceCreateLockEnablesVersioning(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testMakeBucketForceCreateLockEnablesVersioning,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPeerBucketMetadataSaveSurvivesCallerCancellation(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testPeerBucketMetadataSaveSurvivesCallerCancellation,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testPeerBucketMetadataSaveSurvivesCallerCancellation(obj ObjectLayer, instanceType, bucket string,
|
||||||
|
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
previousObjectAPI := newObjectLayerFn()
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
barrier := &metadataRMWBarrierObjectLayer{
|
||||||
|
ObjectLayer: obj,
|
||||||
|
bucket: bucket,
|
||||||
|
cancelOnPut: cancel,
|
||||||
|
}
|
||||||
|
setObjectLayer(barrier)
|
||||||
|
defer setObjectLayer(previousObjectAPI)
|
||||||
|
|
||||||
|
err := globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(ctx, bucket, MakeBucketOptions{VersioningEnabled: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: peer metadata save failed after caller cancellation: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
if ctx.Err() != context.Canceled {
|
||||||
|
t.Fatalf("%s: metadata write did not trigger caller cancellation", instanceType)
|
||||||
|
}
|
||||||
|
meta, err := readBucketMetadata(t.Context(), obj, bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cfg, err := versioning.ParseConfig(bytes.NewReader(meta.VersioningConfigXML))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !cfg.Enabled() {
|
||||||
|
t.Fatalf("%s: peer metadata save lost versioning after cancellation", instanceType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testMakeBucketForceCreateLockEnablesVersioning(obj ObjectLayer, instanceType, bucket string,
|
||||||
|
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
ctx := t.Context()
|
||||||
|
suspended := []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Suspended</Status></VersioningConfiguration>`)
|
||||||
|
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, suspended); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := obj.MakeBucket(ctx, bucket, MakeBucketOptions{ForceCreate: true, LockEnabled: true}); err != nil {
|
||||||
|
t.Fatalf("%s: ForceCreate with object lock: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
meta, err := readBucketMetadata(ctx, obj, bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cfg, err := versioning.ParseConfig(bytes.NewReader(meta.VersioningConfigXML))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !cfg.Enabled() || len(meta.ObjectLockConfigXML) == 0 {
|
||||||
|
t.Fatalf("%s: object lock state lacks enabled versioning: metadata=%+v", instanceType, meta)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testBucketMetadataLockPreservesTaggingAndSSE(obj ObjectLayer, instanceType, bucket string,
|
||||||
|
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
tagXML := []byte(`<Tagging><TagSet><Tag><Key>key</Key><Value>value</Value></Tag></TagSet></Tagging>`)
|
||||||
|
sseXML := []byte(`<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`)
|
||||||
|
runBucketMetadataRMWConflict(t, obj, bucket,
|
||||||
|
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||||
|
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, tagXML)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||||
|
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, sseXML)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
func(meta BucketMetadata) bool {
|
||||||
|
return bytes.Equal(meta.TaggingConfigXML, tagXML) && bytes.Equal(meta.EncryptionConfigXML, sseXML)
|
||||||
|
}, instanceType+": tagging+SSE")
|
||||||
|
}
|
||||||
|
|
||||||
|
func runBucketMetadataRMWConflict(t *testing.T, obj ObjectLayer, bucket string,
|
||||||
|
writerA, writerB func(context.Context, ObjectLayer) error,
|
||||||
|
complete func(BucketMetadata) bool, name string,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
previousObjectAPI := newObjectLayerFn()
|
||||||
|
barrier := &metadataRMWBarrierObjectLayer{
|
||||||
|
ObjectLayer: obj,
|
||||||
|
bucket: bucket,
|
||||||
|
aReady: make(chan struct{}),
|
||||||
|
aRelease: make(chan struct{}),
|
||||||
|
bLockAttempt: make(chan struct{}),
|
||||||
|
}
|
||||||
|
setObjectLayer(barrier)
|
||||||
|
defer setObjectLayer(previousObjectAPI)
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
aCtx := context.WithValue(ctx, metadataRMWWriterKey{}, "A")
|
||||||
|
bCtx := context.WithValue(ctx, metadataRMWWriterKey{}, "B")
|
||||||
|
aDone := make(chan error, 1)
|
||||||
|
bDone := make(chan error, 1)
|
||||||
|
go func() { aDone <- writerA(aCtx, barrier) }()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-barrier.aReady:
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatalf("%s: writer A did not reach metadata save: %v", name, ctx.Err())
|
||||||
|
}
|
||||||
|
go func() { bDone <- writerB(bCtx, barrier) }()
|
||||||
|
|
||||||
|
var (
|
||||||
|
bErr error
|
||||||
|
bFinished bool
|
||||||
|
)
|
||||||
|
select {
|
||||||
|
case <-barrier.bLockAttempt:
|
||||||
|
if got := barrier.reads.Load(); got != 1 {
|
||||||
|
t.Fatalf("%s: writer B read metadata before acquiring metadata.lock: reads=%d", name, got)
|
||||||
|
}
|
||||||
|
case bErr = <-bDone:
|
||||||
|
bFinished = true
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatalf("%s: writer B neither completed nor attempted metadata.lock: %v", name, ctx.Err())
|
||||||
|
}
|
||||||
|
close(barrier.aRelease)
|
||||||
|
if err := <-aDone; err != nil {
|
||||||
|
t.Fatalf("%s: writer A failed: %v", name, err)
|
||||||
|
}
|
||||||
|
if !bFinished {
|
||||||
|
select {
|
||||||
|
case bErr = <-bDone:
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatalf("%s: writer B did not finish: %v", name, ctx.Err())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if bErr != nil {
|
||||||
|
t.Fatalf("%s: writer B failed: %v", name, bErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
disk, err := readBucketMetadata(ctx, barrier, bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: read disk metadata: %v", name, err)
|
||||||
|
}
|
||||||
|
resident, err := globalBucketMetadataSys.Get(bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: read resident metadata: %v", name, err)
|
||||||
|
}
|
||||||
|
if !complete(disk) || !complete(resident) {
|
||||||
|
t.Fatalf("%s: concurrent updates lost a field: disk=%+v resident=%+v", name, disk, resident)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
// Copyright 2026 PGSTY contributors.
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDeleteBucketMetadataLockCancellation(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testDeleteBucketMetadataLockCancellation})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testDeleteBucketMetadataLockCancellation(obj ObjectLayer, instanceType, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||||
|
_, unlock, err := lockBucketMetadata(t.Context(), obj, bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
release := sync.OnceFunc(unlock)
|
||||||
|
defer release()
|
||||||
|
|
||||||
|
// Observe DeleteBucket's ACTUAL metadata.lock attempt. Set the hook after
|
||||||
|
// our own acquisition above so it only trips on the delete.
|
||||||
|
delAtLock := make(chan struct{})
|
||||||
|
var once sync.Once
|
||||||
|
hook := func(b string) {
|
||||||
|
if b == bucket {
|
||||||
|
once.Do(func() { close(delAtLock) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lockBucketMetadataAcquireHook.Store(&hook)
|
||||||
|
defer lockBucketMetadataAcquireHook.Store(nil)
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
defer cancel()
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- obj.DeleteBucket(ctx, bucket, DeleteBucketOptions{Force: true, NoLock: true}) }()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-delAtLock:
|
||||||
|
// Fixed tree: the delete reached metadata.lock and is blocking on the
|
||||||
|
// lock we hold. Cancel it and confirm it fails WITHOUT deleting, while we
|
||||||
|
// still hold the lock (release stays deferred until after the checks).
|
||||||
|
cancel()
|
||||||
|
if err := <-done; err == nil {
|
||||||
|
t.Errorf("%s: canceled deletion succeeded", instanceType)
|
||||||
|
}
|
||||||
|
if _, err := obj.GetBucketInfo(t.Context(), bucket, BucketOptions{}); err != nil {
|
||||||
|
t.Errorf("%s: bucket disappeared while metadata.lock was held: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
if _, err := readBucketMetadata(t.Context(), obj, bucket); err != nil {
|
||||||
|
t.Errorf("%s: canceled deletion removed metadata: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
case err := <-done:
|
||||||
|
// Broken tree: the delete finished without ever taking metadata.lock,
|
||||||
|
// i.e. it did not serialize the destructive operation behind the lock.
|
||||||
|
t.Errorf("%s: delete bypassed metadata.lock (err=%v)", instanceType, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestQueuedMetadataUpdateAfterDelete(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
for _, expiry := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprintf("expiry=%v", expiry), func(t *testing.T) {
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instanceType, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||||
|
previous := newObjectLayerFn()
|
||||||
|
barrier := &lcMergeBarrier{ObjectLayer: obj, bucket: bucket, mAtLock: make(chan struct{}), mProceed: make(chan struct{})}
|
||||||
|
setObjectLayer(barrier)
|
||||||
|
defer setObjectLayer(previous)
|
||||||
|
release := sync.OnceFunc(func() { close(barrier.mProceed) })
|
||||||
|
defer release()
|
||||||
|
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
ctx = context.WithValue(ctx, lcMergeWriterKey{}, "M")
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
if expiry {
|
||||||
|
done <- globalBucketMetadataSys.UpdateExpiryLCConfig(ctx, bucket, nil, UTCNow())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, []byte(`<Tagging><TagSet/></Tagging>`))
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-barrier.mAtLock:
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal("writer did not reach metadata.lock")
|
||||||
|
}
|
||||||
|
if err := obj.DeleteBucket(t.Context(), bucket, DeleteBucketOptions{Force: true}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
if err := <-done; !isErrBucketNotFound(err) {
|
||||||
|
t.Errorf("%s: queued update should reject a deleted bucket, got %v", instanceType, err)
|
||||||
|
}
|
||||||
|
if _, err := readBucketMetadata(t.Context(), obj, bucket); !errors.Is(err, errConfigNotFound) && !isErrBucketNotFound(err) {
|
||||||
|
t.Errorf("%s: queued update recreated metadata: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
}})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,278 @@
|
|||||||
|
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||||
|
// Copyright (c) 2026 PGSTY
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Target 1 (issue #105): a higher-level lifecycle XML merge racing another
|
||||||
|
// bucket-metadata transition outside BucketMetadataSys.Delete.
|
||||||
|
//
|
||||||
|
// Before the fix, PeerBucketLCConfigHandler / healBucketILMExpiry read the
|
||||||
|
// current lifecycle document without metadata.lock, merged the replicated expiry
|
||||||
|
// rules with the local transition rules, and then persisted the merged blob with
|
||||||
|
// BucketMetadataSys.Update. Update re-read the record under metadata.lock but
|
||||||
|
// overwrote LifecycleConfigXML wholesale with the pre-computed blob, so any
|
||||||
|
// lifecycle transition change committed between the merge read and the merge
|
||||||
|
// write was silently lost on disk. BucketMetadataSys.UpdateExpiryLCConfig now
|
||||||
|
// performs the read, merge, and save under a single metadata.lock. This test
|
||||||
|
// drives PeerBucketLCConfigHandler and asserts the concurrent change survives.
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
type lcMergeWriterKey struct{}
|
||||||
|
|
||||||
|
// lcMergeBarrier pauses the merge writer (context value "M") exactly when it
|
||||||
|
// tries to take metadata.lock for its persisting Update. By that point the
|
||||||
|
// merge has already read the stale lifecycle document, so the test can commit a
|
||||||
|
// concurrent transition change before releasing the merge write.
|
||||||
|
type lcMergeBarrier struct {
|
||||||
|
ObjectLayer
|
||||||
|
bucket string
|
||||||
|
mAtLock chan struct{}
|
||||||
|
mProceed chan struct{}
|
||||||
|
mOnce sync.Once
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *lcMergeBarrier) metadataLock() string {
|
||||||
|
return pathJoin(bucketMetaPrefix, o.bucket, "metadata.lock")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *lcMergeBarrier) NewNSLock(bucket string, objects ...string) RWLocker {
|
||||||
|
lock := o.ObjectLayer.NewNSLock(bucket, objects...)
|
||||||
|
if bucket != minioMetaBucket || len(objects) != 1 || objects[0] != o.metadataLock() {
|
||||||
|
return lock
|
||||||
|
}
|
||||||
|
return metadataObservedRWLocker{RWLocker: lock, onLock: func(ctx context.Context) {
|
||||||
|
if ctx.Value(lcMergeWriterKey{}) != "M" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
o.mOnce.Do(func() { close(o.mAtLock) })
|
||||||
|
select {
|
||||||
|
case <-o.mProceed:
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLifecycleExpiryMergeRaceLosesConcurrentTransition(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testLifecycleExpiryMergeRaceLosesConcurrentTransition,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testLifecycleExpiryMergeRaceLosesConcurrentTransition(obj ObjectLayer, instanceType, bucket string,
|
||||||
|
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
// Revision N: a single transition-only rule.
|
||||||
|
baseXML := []byte(`<LifecycleConfiguration><Rule><ID>keep</ID><Filter><Prefix>data/</Prefix></Filter><Status>Enabled</Status><Transition><Days>30</Days><StorageClass>WARM</StorageClass></Transition></Rule></LifecycleConfiguration>`)
|
||||||
|
if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketLifecycleConfig, baseXML); err != nil {
|
||||||
|
t.Fatalf("%s: seed lifecycle: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A replicated expiry-only rule arriving from a peer site.
|
||||||
|
expXML := `<LifecycleConfiguration><Rule><ID>expire</ID><Filter><Prefix>tmp/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>7</Days></Expiration></Rule></LifecycleConfiguration>`
|
||||||
|
expLCConfig := base64.StdEncoding.EncodeToString([]byte(expXML))
|
||||||
|
|
||||||
|
previousObjectAPI := newObjectLayerFn()
|
||||||
|
barrier := &lcMergeBarrier{
|
||||||
|
ObjectLayer: obj,
|
||||||
|
bucket: bucket,
|
||||||
|
mAtLock: make(chan struct{}),
|
||||||
|
mProceed: make(chan struct{}),
|
||||||
|
}
|
||||||
|
setObjectLayer(barrier)
|
||||||
|
defer setObjectLayer(previousObjectAPI)
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
mCtx := context.WithValue(ctx, lcMergeWriterKey{}, "M")
|
||||||
|
|
||||||
|
mDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
mDone <- globalSiteReplicationSys.PeerBucketLCConfigHandler(mCtx, bucket, &expLCConfig, UTCNow())
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Wait until the merge writer has read revision N and is about to persist.
|
||||||
|
select {
|
||||||
|
case <-barrier.mAtLock:
|
||||||
|
case err := <-mDone:
|
||||||
|
t.Fatalf("%s: merge writer finished before persisting: %v", instanceType, err)
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatalf("%s: merge writer never reached metadata.lock: %v", instanceType, ctx.Err())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Concurrent local lifecycle transition change commits revision N+1.
|
||||||
|
concurrentXML := []byte(`<LifecycleConfiguration><Rule><ID>keep</ID><Filter><Prefix>data/</Prefix></Filter><Status>Enabled</Status><Transition><Days>10</Days><StorageClass>COLD</StorageClass></Transition></Rule></LifecycleConfiguration>`)
|
||||||
|
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, concurrentXML); err != nil {
|
||||||
|
t.Fatalf("%s: concurrent transition update: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Release the merge write so it lands after the concurrent commit.
|
||||||
|
close(barrier.mProceed)
|
||||||
|
if err := <-mDone; err != nil {
|
||||||
|
t.Fatalf("%s: merge writer failed: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The persisted lifecycle must contain both the replicated expiry rule and
|
||||||
|
// the concurrent transition change.
|
||||||
|
cfg, _, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: read merged lifecycle: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
var keep, expire bool
|
||||||
|
for i := range cfg.Rules {
|
||||||
|
switch cfg.Rules[i].ID {
|
||||||
|
case "keep":
|
||||||
|
keep = true
|
||||||
|
if cfg.Rules[i].Transition.Days != 10 || cfg.Rules[i].Transition.StorageClass != "COLD" {
|
||||||
|
t.Fatalf("%s: lifecycle merge overwrote the concurrent transition change: got Days=%d StorageClass=%q, want Days=10 StorageClass=COLD",
|
||||||
|
instanceType, cfg.Rules[i].Transition.Days, cfg.Rules[i].Transition.StorageClass)
|
||||||
|
}
|
||||||
|
case "expire":
|
||||||
|
expire = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !expire {
|
||||||
|
t.Fatalf("%s: merged lifecycle dropped the replicated expiry rule: %+v", instanceType, cfg.Rules)
|
||||||
|
}
|
||||||
|
if !keep {
|
||||||
|
t.Fatalf("%s: merged lifecycle dropped the transition rule entirely: %+v", instanceType, cfg.Rules)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Target 2 (issue #105): DeleteBucket racing an in-flight metadata writer must
|
||||||
|
// not resurrect a ghost .metadata.bin record.
|
||||||
|
//
|
||||||
|
// Before the fix, erasureServerPools.DeleteBucket took only <bucket>.lck and
|
||||||
|
// purged the metadata prefix while config writers (updateAndParse) took only
|
||||||
|
// metadata.lock, so a writer already MID-SAVE (holding metadata.lock, past
|
||||||
|
// saveMetadata's existence recheck) could persist .metadata.bin after the purge.
|
||||||
|
// DeleteBucket now takes metadata.lock before deleting, so it waits for that
|
||||||
|
// writer and then purges whatever the writer wrote.
|
||||||
|
//
|
||||||
|
// This test isolates the DeleteBucket-lock fix specifically: the writer holds
|
||||||
|
// metadata.lock and is paused at the .metadata.bin PutObject, so the earlier
|
||||||
|
// saveMetadata existence recheck cannot save it — only serializing the delete
|
||||||
|
// behind the writer can. Removing just DeleteBucket's metadata.lock (keeping the
|
||||||
|
// recheck) therefore makes this test fail. The delete's ACTUAL metadata.lock
|
||||||
|
// attempt is observed with lockBucketMetadataAcquireHook (its lock is taken
|
||||||
|
// through the erasureServerPools receiver, invisible to the object-layer
|
||||||
|
// barrier), so the handshake is deterministic with no timing assumption.
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func TestDeleteBucketResurrectsGhostMetadata(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testDeleteBucketResurrectsGhostMetadata,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testDeleteBucketResurrectsGhostMetadata(obj ObjectLayer, instanceType, bucket string,
|
||||||
|
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
previousObjectAPI := newObjectLayerFn()
|
||||||
|
// Writer A holds metadata.lock and pauses at the .metadata.bin PutObject,
|
||||||
|
// i.e. already past saveMetadata's existence recheck and mid-save.
|
||||||
|
barrier := &metadataRMWBarrierObjectLayer{
|
||||||
|
ObjectLayer: obj,
|
||||||
|
bucket: bucket,
|
||||||
|
aReady: make(chan struct{}),
|
||||||
|
aRelease: make(chan struct{}),
|
||||||
|
bLockAttempt: make(chan struct{}),
|
||||||
|
}
|
||||||
|
setObjectLayer(barrier)
|
||||||
|
defer setObjectLayer(previousObjectAPI)
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
aCtx := context.WithValue(ctx, metadataRMWWriterKey{}, "A")
|
||||||
|
|
||||||
|
policyJSON := []byte(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::` + bucket + `/*"}]}`)
|
||||||
|
aReleased := sync.OnceFunc(func() { close(barrier.aRelease) })
|
||||||
|
defer aReleased()
|
||||||
|
aDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := globalBucketMetadataSys.Update(aCtx, bucket, bucketPolicyConfig, policyJSON)
|
||||||
|
aDone <- err
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-barrier.aReady:
|
||||||
|
case err := <-aDone:
|
||||||
|
t.Fatalf("%s: writer A finished before persisting: %v", instanceType, err)
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatalf("%s: writer A never reached metadata save: %v", instanceType, ctx.Err())
|
||||||
|
}
|
||||||
|
|
||||||
|
// A now holds metadata.lock mid-save. Observe DeleteBucket's ACTUAL
|
||||||
|
// metadata.lock attempt via the acquire hook, set only now so A's earlier
|
||||||
|
// acquisition does not trip it.
|
||||||
|
delAtLock := make(chan struct{})
|
||||||
|
var once sync.Once
|
||||||
|
hook := func(b string) {
|
||||||
|
if b == bucket {
|
||||||
|
once.Do(func() { close(delAtLock) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lockBucketMetadataAcquireHook.Store(&hook)
|
||||||
|
defer lockBucketMetadataAcquireHook.Store(nil)
|
||||||
|
|
||||||
|
delDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
delDone <- obj.DeleteBucket(ctx, bucket, DeleteBucketOptions{Force: true})
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-delAtLock:
|
||||||
|
// Fixed tree: DeleteBucket reached metadata.lock and blocks on A. Release
|
||||||
|
// A so it finishes its save and unlocks; the delete then acquires the
|
||||||
|
// lock and purges the record A wrote.
|
||||||
|
aReleased()
|
||||||
|
if err := <-aDone; err != nil {
|
||||||
|
t.Fatalf("%s: writer A save failed while holding metadata.lock: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
if err := <-delDone; err != nil {
|
||||||
|
t.Fatalf("%s: delete bucket: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
case err := <-delDone:
|
||||||
|
// Broken tree: DeleteBucket purged without taking metadata.lock. Release
|
||||||
|
// A so its mid-save PutObject recreates .metadata.bin (the ghost).
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: delete bucket: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
aReleased()
|
||||||
|
if err := <-aDone; err != nil {
|
||||||
|
t.Fatalf("%s: writer A save failed: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// After DeleteBucket, no .metadata.bin record may remain on disk.
|
||||||
|
if meta, err := readBucketMetadata(ctx, obj, bucket); err == nil {
|
||||||
|
t.Fatalf("%s: ghost .metadata.bin resurrected after DeleteBucket: name=%q created=%s policyLen=%d",
|
||||||
|
instanceType, meta.Name, meta.Created, len(meta.PolicyConfigJSON))
|
||||||
|
} else if !errors.Is(err, errConfigNotFound) && !isErrBucketNotFound(err) && !errors.Is(err, errVolumeNotFound) {
|
||||||
|
t.Fatalf("%s: unexpected error reading deleted bucket metadata: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
// Copyright 2026 PGSTY contributors.
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
"github.com/minio/minio/internal/grid"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPeerMetadataReloadWithEqualMaximumTimestamp(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testPeerMetadataReloadWithEqualMaximumTimestamp})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testPeerMetadataReloadWithEqualMaximumTimestamp(obj ObjectLayer, instanceType, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||||
|
disk, err := loadBucketMetadata(t.Context(), obj, bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
disk.TaggingConfigXML = []byte(`<Tagging><TagSet><Tag><Key>revision</Key><Value>new</Value></Tag></TagSet></Tagging>`)
|
||||||
|
disk.TaggingConfigUpdatedAt = UTCNow()
|
||||||
|
// An unrelated configuration has the greatest timestamp in both records.
|
||||||
|
disk.PolicyConfigUpdatedAt = disk.TaggingConfigUpdatedAt.Add(time.Hour)
|
||||||
|
if err := disk.Save(t.Context(), obj); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
resident := disk
|
||||||
|
resident.TaggingConfigXML = []byte(`<Tagging><TagSet><Tag><Key>revision</Key><Value>old</Value></Tag></TagSet></Tagging>`)
|
||||||
|
resident.TaggingConfigUpdatedAt = disk.TaggingConfigUpdatedAt.Add(-time.Minute)
|
||||||
|
if err := resident.parseAllConfigs(t.Context(), obj); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !resident.lastUpdate().Equal(disk.lastUpdate()) {
|
||||||
|
t.Fatal("fixture must have equal maximum timestamps")
|
||||||
|
}
|
||||||
|
globalBucketMetadataSys.Set(bucket, resident)
|
||||||
|
|
||||||
|
args := grid.MSS{peerRESTBucket: bucket}
|
||||||
|
if _, err := (&peerRESTServer{}).LoadBucketMetadataHandler(&args); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
tagging, _, err := globalBucketMetadataSys.GetTaggingConfig(bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := tagging.ToMap()["revision"]; got != "new" {
|
||||||
|
t.Errorf("%s: peer reload retained tag %q despite a newer tagging configuration", instanceType, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,330 @@
|
|||||||
|
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"sort"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio-go/v7/pkg/tags"
|
||||||
|
bucketsse "github.com/minio/minio/internal/bucket/encryption"
|
||||||
|
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||||
|
"github.com/minio/minio/internal/bucket/versioning"
|
||||||
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Only these fields share the site-replication source-time ordering contract.
|
||||||
|
// Bulk apply/import process Object Lock before Versioning, whose effective
|
||||||
|
// document depends on it. Periodic heal retains its existing type order.
|
||||||
|
var replicatedBucketConfigs = [...]string{
|
||||||
|
objectLockConfig, bucketVersioningConfig, bucketPolicyConfig,
|
||||||
|
bucketTaggingConfig, bucketSSEConfig, bucketQuotaConfigFile,
|
||||||
|
}
|
||||||
|
|
||||||
|
func replicatedBucketConfig(meta *BucketMetadata, file string) (*[]byte, *time.Time) {
|
||||||
|
switch file {
|
||||||
|
case bucketPolicyConfig:
|
||||||
|
return &meta.PolicyConfigJSON, &meta.PolicyConfigUpdatedAt
|
||||||
|
case bucketTaggingConfig:
|
||||||
|
return &meta.TaggingConfigXML, &meta.TaggingConfigUpdatedAt
|
||||||
|
case bucketSSEConfig:
|
||||||
|
return &meta.EncryptionConfigXML, &meta.EncryptionConfigUpdatedAt
|
||||||
|
case bucketQuotaConfigFile:
|
||||||
|
return &meta.QuotaConfigJSON, &meta.QuotaConfigUpdatedAt
|
||||||
|
case bucketVersioningConfig:
|
||||||
|
return &meta.VersioningConfigXML, &meta.VersioningConfigUpdatedAt
|
||||||
|
case objectLockConfig:
|
||||||
|
return &meta.ObjectLockConfigXML, &meta.ObjectLockConfigUpdatedAt
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Callers that only need to know whether a file is under the contract must not
|
||||||
|
// probe replicatedBucketConfig with a throwaway BucketMetadata.
|
||||||
|
func isReplicatedBucketConfig(file string) bool {
|
||||||
|
for _, replicated := range replicatedBucketConfigs {
|
||||||
|
if replicated == file {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func bucketConfigUpdateOnly(file string) bool {
|
||||||
|
return file == bucketVersioningConfig || file == objectLockConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reuse the persistence rule before comparison, so an accepted Versioning
|
||||||
|
// event and the document Save actually writes have the same comparison key.
|
||||||
|
func effectiveBucketVersioning(data []byte, lockEnabled bool) []byte {
|
||||||
|
if lockEnabled {
|
||||||
|
config, err := versioning.ParseConfig(bytes.NewReader(data))
|
||||||
|
if err != nil || !config.Enabled() || config.PrefixesExcluded() {
|
||||||
|
return enabledBucketVersioningConfig
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
// A parsed policy still contains map-backed sets with nondeterministic Marshal
|
||||||
|
// order. Sort every set array recursively, including statements and conditions.
|
||||||
|
// RawMessage keeps integer values intact; decoding through float64 would not.
|
||||||
|
func canonicalBucketPolicyJSON(data json.RawMessage) (json.RawMessage, error) {
|
||||||
|
data = bytes.TrimSpace(data)
|
||||||
|
if len(data) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
switch data[0] {
|
||||||
|
case '{':
|
||||||
|
var obj map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(data, &obj); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for key, value := range obj {
|
||||||
|
var err error
|
||||||
|
obj[key], err = canonicalBucketPolicyJSON(value)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return json.Marshal(obj)
|
||||||
|
case '[':
|
||||||
|
var arr []json.RawMessage
|
||||||
|
if err := json.Unmarshal(data, &arr); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for i := range arr {
|
||||||
|
var err error
|
||||||
|
arr[i], err = canonicalBucketPolicyJSON(arr[i])
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Slice(arr, func(i, j int) bool { return bytes.Compare(arr[i], arr[j]) < 0 })
|
||||||
|
return json.Marshal(arr)
|
||||||
|
default:
|
||||||
|
var compact bytes.Buffer
|
||||||
|
if err := json.Compact(&compact, data); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return compact.Bytes(), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Encode the validated policy fields explicitly: BPStatement's required
|
||||||
|
// Action/Resource tags otherwise try to marshal empty sets for the supported
|
||||||
|
// NotAction/NotResource alternatives. This stays within the existing schema.
|
||||||
|
func canonicalBucketPolicy(cfg *policy.BucketPolicy) ([]byte, error) {
|
||||||
|
if cfg.IsEmpty() {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
doc := map[string]any{"Version": cfg.Version}
|
||||||
|
if cfg.ID != "" {
|
||||||
|
doc["ID"] = cfg.ID
|
||||||
|
}
|
||||||
|
statements := make([]map[string]any, 0, len(cfg.Statements))
|
||||||
|
for _, st := range cfg.Statements {
|
||||||
|
statement := map[string]any{"Effect": st.Effect, "Principal": st.Principal}
|
||||||
|
if st.SID != "" {
|
||||||
|
statement["Sid"] = st.SID
|
||||||
|
}
|
||||||
|
if len(st.Actions) != 0 {
|
||||||
|
statement["Action"] = st.Actions
|
||||||
|
}
|
||||||
|
if len(st.NotActions) != 0 {
|
||||||
|
statement["NotAction"] = st.NotActions
|
||||||
|
}
|
||||||
|
if len(st.Resources) != 0 {
|
||||||
|
statement["Resource"] = st.Resources
|
||||||
|
}
|
||||||
|
if len(st.NotResources) != 0 {
|
||||||
|
statement["NotResource"] = st.NotResources
|
||||||
|
}
|
||||||
|
if len(st.Conditions) != 0 {
|
||||||
|
statement["Condition"] = st.Conditions
|
||||||
|
}
|
||||||
|
statements = append(statements, statement)
|
||||||
|
}
|
||||||
|
doc["Statement"] = statements
|
||||||
|
data, err := json.Marshal(doc)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return canonicalBucketPolicyJSON(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate with the same parsers as Save. Policy's established empty-policy
|
||||||
|
// semantics are deletion; a parsed zero quota is still a live document.
|
||||||
|
func bucketConfigPayload(bucket, file string, data []byte, lockEnabled bool) ([]byte, []byte, error) {
|
||||||
|
if len(data) == 0 {
|
||||||
|
return nil, nil, nil
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
key := data
|
||||||
|
switch file {
|
||||||
|
case bucketPolicyConfig:
|
||||||
|
var cfg *policy.BucketPolicy
|
||||||
|
cfg, err = policy.ParseBucketPolicyConfig(bytes.NewReader(data), bucket)
|
||||||
|
if err == nil {
|
||||||
|
if cfg.IsEmpty() {
|
||||||
|
return nil, nil, nil
|
||||||
|
}
|
||||||
|
key, err = canonicalBucketPolicy(cfg)
|
||||||
|
}
|
||||||
|
case bucketQuotaConfigFile:
|
||||||
|
cfg, parseErr := parseBucketQuota(bucket, data)
|
||||||
|
err = parseErr
|
||||||
|
if err == nil {
|
||||||
|
key, err = json.Marshal(cfg)
|
||||||
|
}
|
||||||
|
case bucketTaggingConfig:
|
||||||
|
_, err = tags.ParseBucketXML(bytes.NewReader(data))
|
||||||
|
case bucketSSEConfig:
|
||||||
|
_, err = bucketsse.ParseBucketSSEConfig(bytes.NewReader(data))
|
||||||
|
case objectLockConfig:
|
||||||
|
_, err = objectlock.ParseObjectLockConfig(bytes.NewReader(data))
|
||||||
|
case bucketVersioningConfig:
|
||||||
|
data = effectiveBucketVersioning(data, lockEnabled)
|
||||||
|
key = data
|
||||||
|
_, err = versioning.ParseConfig(bytes.NewReader(data))
|
||||||
|
}
|
||||||
|
return data, key, err
|
||||||
|
}
|
||||||
|
|
||||||
|
type bucketConfigState struct {
|
||||||
|
data, key []byte
|
||||||
|
at time.Time
|
||||||
|
real, valid bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func newBucketConfigState(bucket, file string, data []byte, at, created time.Time, lockEnabled bool) (bucketConfigState, error) {
|
||||||
|
data, key, err := bucketConfigPayload(bucket, file, data, lockEnabled)
|
||||||
|
if err != nil {
|
||||||
|
return bucketConfigState{}, err
|
||||||
|
}
|
||||||
|
if at.IsZero() {
|
||||||
|
at = created
|
||||||
|
}
|
||||||
|
valid := !created.IsZero() && !at.Before(created)
|
||||||
|
modified := valid && at.After(created)
|
||||||
|
if bucketConfigUpdateOnly(file) && len(data) == 0 {
|
||||||
|
modified = false
|
||||||
|
}
|
||||||
|
return bucketConfigState{data: data, key: key, at: at, real: modified, valid: valid}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s bucketConfigState) candidate() bool {
|
||||||
|
return s.valid && (s.real || len(s.data) != 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func compareBucketConfigStates(a, b bucketConfigState) int {
|
||||||
|
if a.valid != b.valid {
|
||||||
|
if a.valid {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
if a.real != b.real {
|
||||||
|
if a.real {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
if a.real {
|
||||||
|
if n := a.at.Compare(b.at); n != 0 {
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
// At equal source time a real deletion wins, preventing resurrection.
|
||||||
|
if (len(a.data) == 0) != (len(b.data) == 0) {
|
||||||
|
if len(a.data) == 0 {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return bytes.Compare(a.key, b.key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func localBucketConfigUpdatedAt(meta BucketMetadata, file string, now time.Time) time.Time {
|
||||||
|
_, at := replicatedBucketConfig(&meta, file)
|
||||||
|
for _, lower := range []time.Time{meta.Created, *at} {
|
||||||
|
if !now.After(lower) {
|
||||||
|
now = lower.Add(time.Nanosecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return now.UTC()
|
||||||
|
}
|
||||||
|
|
||||||
|
func ensureBucketMetadataCreated(ctx context.Context, obj ObjectLayer, meta *BucketMetadata) error {
|
||||||
|
if !meta.Created.IsZero() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
info, err := obj.GetBucketInfo(ctx, meta.Name, BucketOptions{NoMetadata: true})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info.Created.IsZero() {
|
||||||
|
return errors.New("bucket metadata creation time is unknown")
|
||||||
|
}
|
||||||
|
meta.Created = info.Created.UTC()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// applyBucketConfig runs under metadata.lock, on freshly loaded metadata. It
|
||||||
|
// changes only the selected field; the caller persists once after all checks.
|
||||||
|
func applyBucketConfig(meta *BucketMetadata, file string, data []byte, at time.Time) (bool, error) {
|
||||||
|
if bucketConfigUpdateOnly(file) && len(data) == 0 {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
current, currentAt := replicatedBucketConfig(meta, file)
|
||||||
|
lockEnabled := len(meta.ObjectLockConfigXML) != 0
|
||||||
|
incoming, err := newBucketConfigState(meta.Name, file, data, at, meta.Created, lockEnabled)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if !incoming.candidate() {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
local, err := newBucketConfigState(meta.Name, file, *current, *currentAt, meta.Created, lockEnabled)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if compareBucketConfigStates(incoming, local) <= 0 {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
*current, *currentAt = bytes.Clone(incoming.data), incoming.at.UTC()
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func rebaseBucketConfigDefaults(meta *BucketMetadata, oldCreated time.Time) {
|
||||||
|
if meta.Created.Equal(oldCreated) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// These six fields alone use Created to distinguish a baseline from a
|
||||||
|
// tombstone. Preserve actual source times when adopting an existing bucket.
|
||||||
|
for _, file := range replicatedBucketConfigs {
|
||||||
|
_, at := replicatedBucketConfig(meta, file)
|
||||||
|
if at.IsZero() || at.Equal(oldCreated) {
|
||||||
|
*at = meta.Created
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+282
-172
@@ -24,6 +24,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/minio/madmin-go/v3"
|
"github.com/minio/madmin-go/v3"
|
||||||
@@ -38,8 +39,8 @@ import (
|
|||||||
"github.com/minio/minio/internal/event"
|
"github.com/minio/minio/internal/event"
|
||||||
"github.com/minio/minio/internal/kms"
|
"github.com/minio/minio/internal/kms"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
"github.com/minio/pkg/v3/sync/errgroup"
|
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||||
"golang.org/x/sync/singleflight"
|
"golang.org/x/sync/singleflight"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -51,16 +52,27 @@ type BucketMetadataSys struct {
|
|||||||
initialized bool
|
initialized bool
|
||||||
group *singleflight.Group
|
group *singleflight.Group
|
||||||
metadataMap map[string]BucketMetadata
|
metadataMap map[string]BucketMetadata
|
||||||
// loadFailed tracks real buckets whose metadata could not be loaded at
|
// loadFailed records real buckets whose metadata has never been loaded
|
||||||
// startup (concurrentLoad) or during a refresh. Such buckets are NOT
|
// successfully because the startup load or a refresh failed. They are
|
||||||
// resident in metadataMap even though the subsystem is Initialized, so a
|
// absent from metadataMap even though the subsystem is initialized, and
|
||||||
// plain map miss cannot distinguish "not a bucket" from "known bucket whose
|
// without this bit a resident-only lookup could not tell them apart from a
|
||||||
// config we could not read". Callers that must fail closed for a real but
|
// name that is not a bucket at all. It never holds a resident bucket, is
|
||||||
// unreadable bucket (e.g. per-bucket CORS) consult this set. It is bounded
|
// bounded by the number of failed loads, and is empty in normal operation.
|
||||||
// by the number of load failures and is empty in normal operation.
|
|
||||||
loadFailed map[string]struct{}
|
loadFailed map[string]struct{}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// noteLoadFailure and clearLoadFailure maintain loadFailed; both expect the
|
||||||
|
// caller to hold sys.Lock. A bucket that is resident keeps its last loaded
|
||||||
|
// metadata through a failed refresh, exactly like every other bucket
|
||||||
|
// configuration, so the set only ever holds non-resident buckets.
|
||||||
|
func (sys *BucketMetadataSys) noteLoadFailure(bucket string) {
|
||||||
|
if _, resident := sys.metadataMap[bucket]; !resident {
|
||||||
|
sys.loadFailed[bucket] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sys *BucketMetadataSys) clearLoadFailure(bucket string) { delete(sys.loadFailed, bucket) }
|
||||||
|
|
||||||
// Count returns number of bucket metadata map entries.
|
// Count returns number of bucket metadata map entries.
|
||||||
func (sys *BucketMetadataSys) Count() int {
|
func (sys *BucketMetadataSys) Count() int {
|
||||||
sys.RLock()
|
sys.RLock()
|
||||||
@@ -75,7 +87,7 @@ func (sys *BucketMetadataSys) Remove(buckets ...string) {
|
|||||||
for _, bucket := range buckets {
|
for _, bucket := range buckets {
|
||||||
sys.group.Forget(bucket)
|
sys.group.Forget(bucket)
|
||||||
delete(sys.metadataMap, bucket)
|
delete(sys.metadataMap, bucket)
|
||||||
delete(sys.loadFailed, bucket)
|
sys.clearLoadFailure(bucket)
|
||||||
globalBucketMonitor.DeleteBucket(bucket)
|
globalBucketMonitor.DeleteBucket(bucket)
|
||||||
}
|
}
|
||||||
sys.Unlock()
|
sys.Unlock()
|
||||||
@@ -95,7 +107,7 @@ func (sys *BucketMetadataSys) RemoveStaleBuckets(diskBuckets set.StringSet) {
|
|||||||
}
|
}
|
||||||
for bucket := range sys.loadFailed {
|
for bucket := range sys.loadFailed {
|
||||||
if !diskBuckets.Contains(bucket) {
|
if !diskBuckets.Contains(bucket) {
|
||||||
delete(sys.loadFailed, bucket)
|
sys.clearLoadFailure(bucket)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -109,77 +121,133 @@ func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
|
|||||||
if !isMinioMetaBucketName(bucket) {
|
if !isMinioMetaBucketName(bucket) {
|
||||||
sys.Lock()
|
sys.Lock()
|
||||||
sys.metadataMap[bucket] = meta
|
sys.metadataMap[bucket] = meta
|
||||||
delete(sys.loadFailed, bucket)
|
sys.clearLoadFailure(bucket)
|
||||||
sys.Unlock()
|
sys.Unlock()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string, configFile string, configData []byte, parse bool) (updatedAt time.Time, err error) {
|
// bucketMetadataUpdate returns the committed snapshot to the caller. meta and
|
||||||
|
// updatedAt hold the saved state only when changed is true. Local writes always
|
||||||
|
// change state, because localBucketConfigUpdatedAt is strictly greater than the
|
||||||
|
// current field time, so their handlers can broadcast meta without rechecking.
|
||||||
|
type bucketMetadataUpdate struct {
|
||||||
|
meta BucketMetadata
|
||||||
|
updatedAt time.Time
|
||||||
|
changed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket, configFile string, configData []byte, parse, lifecycleDelete bool) (time.Time, error) {
|
||||||
|
result, err := sys.updateAndParseMetadata(ctx, bucket, configFile, configData, parse, lifecycleDelete, nil)
|
||||||
|
return result.updatedAt, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sys *BucketMetadataSys) updateAndParseMetadata(ctx context.Context, bucket string, configFile string, configData []byte, parse, lifecycleDelete bool, sourceTime *time.Time) (result bucketMetadataUpdate, err error) {
|
||||||
objAPI := newObjectLayerFn()
|
objAPI := newObjectLayerFn()
|
||||||
if objAPI == nil {
|
if objAPI == nil {
|
||||||
return updatedAt, errServerNotInitialized
|
return result, errServerNotInitialized
|
||||||
}
|
}
|
||||||
|
|
||||||
if isMinioMetaBucketName(bucket) {
|
if isMinioMetaBucketName(bucket) {
|
||||||
return updatedAt, errInvalidArgument
|
return result, errInvalidArgument
|
||||||
}
|
}
|
||||||
|
// Load deletions without parsed caches (notably quota), and compare the
|
||||||
meta, err := loadBucketMetadataParse(ctx, objAPI, bucket, parse)
|
// six replicated fields against the raw document under the same lock.
|
||||||
|
if isReplicatedBucketConfig(configFile) {
|
||||||
|
parse = false
|
||||||
|
if bucketConfigUpdateOnly(configFile) && len(configData) == 0 {
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
notifyCtx := ctx
|
||||||
|
ctx, unlock, err := lockBucketMetadata(ctx, objAPI, bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
|
return result, err
|
||||||
// Only single drive mode needs this fallback.
|
|
||||||
meta = newBucketMetadata(bucket)
|
|
||||||
} else {
|
|
||||||
return updatedAt, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
updatedAt = UTCNow()
|
|
||||||
switch configFile {
|
|
||||||
case bucketPolicyConfig:
|
|
||||||
meta.PolicyConfigJSON = configData
|
|
||||||
meta.PolicyConfigUpdatedAt = updatedAt
|
|
||||||
case bucketNotificationConfig:
|
|
||||||
meta.NotificationConfigXML = configData
|
|
||||||
meta.NotificationConfigUpdatedAt = updatedAt
|
|
||||||
case bucketLifecycleConfig:
|
|
||||||
meta.LifecycleConfigXML = configData
|
|
||||||
meta.LifecycleConfigUpdatedAt = updatedAt
|
|
||||||
case bucketSSEConfig:
|
|
||||||
meta.EncryptionConfigXML = configData
|
|
||||||
meta.EncryptionConfigUpdatedAt = updatedAt
|
|
||||||
case bucketTaggingConfig:
|
|
||||||
meta.TaggingConfigXML = configData
|
|
||||||
meta.TaggingConfigUpdatedAt = updatedAt
|
|
||||||
case bucketCorsConfig:
|
|
||||||
meta.CorsConfigXML = configData
|
|
||||||
meta.CorsConfigUpdatedAt = updatedAt
|
|
||||||
case bucketQuotaConfigFile:
|
|
||||||
meta.QuotaConfigJSON = configData
|
|
||||||
meta.QuotaConfigUpdatedAt = updatedAt
|
|
||||||
case objectLockConfig:
|
|
||||||
meta.ObjectLockConfigXML = configData
|
|
||||||
meta.ObjectLockConfigUpdatedAt = updatedAt
|
|
||||||
case bucketVersioningConfig:
|
|
||||||
meta.VersioningConfigXML = configData
|
|
||||||
meta.VersioningConfigUpdatedAt = updatedAt
|
|
||||||
case bucketReplicationConfig:
|
|
||||||
meta.ReplicationConfigXML = configData
|
|
||||||
meta.ReplicationConfigUpdatedAt = updatedAt
|
|
||||||
case bucketTargetsFile:
|
|
||||||
meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(ctx, meta.Name, configData, kms.Context{
|
|
||||||
bucket: meta.Name,
|
|
||||||
bucketTargetsFile: bucketTargetsFile,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return updatedAt, fmt.Errorf("Error encrypting bucket target metadata %w", err)
|
|
||||||
}
|
|
||||||
meta.BucketTargetsConfigUpdatedAt = updatedAt
|
|
||||||
meta.BucketTargetsConfigMetaUpdatedAt = updatedAt
|
|
||||||
default:
|
|
||||||
return updatedAt, fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return updatedAt, sys.save(ctx, meta)
|
err = func() error {
|
||||||
|
defer unlock()
|
||||||
|
meta, err := loadBucketMetadataParse(ctx, objAPI, bucket, parse)
|
||||||
|
if err != nil {
|
||||||
|
if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
|
||||||
|
// Only single drive mode needs this fallback.
|
||||||
|
meta = newBucketMetadata(bucket)
|
||||||
|
} else {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if lifecycleDelete {
|
||||||
|
configData, err = lifecycleDeleteConfig(meta.LifecycleConfigXML)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
updatedAt := UTCNow()
|
||||||
|
if isReplicatedBucketConfig(configFile) {
|
||||||
|
if err := ensureBucketMetadataCreated(ctx, objAPI, &meta); err != nil {
|
||||||
|
var at time.Time
|
||||||
|
if sourceTime != nil {
|
||||||
|
at = *sourceTime
|
||||||
|
}
|
||||||
|
logBucketConfigReplication(ctx, bucket, configFile, "indeterminate", at, meta.Created, err.Error())
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if sourceTime == nil || sourceTime.IsZero() {
|
||||||
|
updatedAt = localBucketConfigUpdatedAt(meta, configFile, updatedAt)
|
||||||
|
if sourceTime != nil {
|
||||||
|
logBucketConfigReplication(ctx, bucket, configFile, "legacy-zero", *sourceTime, meta.Created, "assigned local source time")
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
updatedAt = sourceTime.UTC()
|
||||||
|
}
|
||||||
|
if updatedAt.Before(meta.Created) {
|
||||||
|
logBucketConfigReplication(ctx, bucket, configFile, "before-created", updatedAt, meta.Created, "peer event")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
changed, err := applyBucketConfig(&meta, configFile, configData, updatedAt)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !changed {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
switch configFile {
|
||||||
|
case bucketNotificationConfig:
|
||||||
|
meta.NotificationConfigXML = configData
|
||||||
|
meta.NotificationConfigUpdatedAt = updatedAt
|
||||||
|
case bucketLifecycleConfig:
|
||||||
|
meta.LifecycleConfigXML = configData
|
||||||
|
meta.LifecycleConfigUpdatedAt = updatedAt
|
||||||
|
case bucketReplicationConfig:
|
||||||
|
meta.ReplicationConfigXML = configData
|
||||||
|
meta.ReplicationConfigUpdatedAt = updatedAt
|
||||||
|
case bucketTargetsFile:
|
||||||
|
meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(ctx, meta.Name, configData, kms.Context{
|
||||||
|
bucket: meta.Name,
|
||||||
|
bucketTargetsFile: bucketTargetsFile,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Error encrypting bucket target metadata %w", err)
|
||||||
|
}
|
||||||
|
meta.BucketTargetsConfigUpdatedAt = updatedAt
|
||||||
|
meta.BucketTargetsConfigMetaUpdatedAt = updatedAt
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := sys.saveMetadata(ctx, objAPI, &meta); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
result = bucketMetadataUpdate{meta: meta, updatedAt: updatedAt, changed: true}
|
||||||
|
return nil
|
||||||
|
}()
|
||||||
|
if err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
if result.changed {
|
||||||
|
globalNotificationSys.LoadBucketMetadata(bgContext(notifyCtx), bucket)
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) error {
|
func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) error {
|
||||||
@@ -192,59 +260,145 @@ func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) err
|
|||||||
return errInvalidArgument
|
return errInvalidArgument
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := meta.Save(ctx, objAPI); err != nil {
|
if err := sys.saveMetadata(ctx, objAPI, &meta); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
sys.Set(meta.Name, meta)
|
|
||||||
globalNotificationSys.LoadBucketMetadata(bgContext(ctx), meta.Name) // Do not use caller context here
|
globalNotificationSys.LoadBucketMetadata(bgContext(ctx), meta.Name) // Do not use caller context here
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// saveMetadata persists and publishes metadata locally. Callers performing a
|
||||||
|
// read-modify-write must hold metadata.lock and release it before peer fan-out.
|
||||||
|
func (sys *BucketMetadataSys) saveMetadata(ctx context.Context, objAPI ObjectLayer, meta *BucketMetadata) error {
|
||||||
|
// A writer may have queued for metadata.lock before DeleteBucket completed.
|
||||||
|
// Recheck the physical bucket under that lock, before recreating metadata.
|
||||||
|
if _, err := objAPI.GetBucketInfo(ctx, meta.Name, BucketOptions{NoMetadata: true}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := meta.Save(ctx, objAPI); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
sys.Set(meta.Name, *meta)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func lockBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (context.Context, func(), error) {
|
||||||
|
return lockBucketMetadataWithTimeout(ctx, objectAPI, bucket, globalOperationTimeout)
|
||||||
|
}
|
||||||
|
|
||||||
|
// lockBucketMetadataAcquireHook, when set, is invoked at the start of every
|
||||||
|
// metadata.lock acquisition, immediately before the blocking Lock() call. It is
|
||||||
|
// nil in production (a single atomic load, no behavior change) and exists only
|
||||||
|
// so tests can deterministically observe a caller reaching the metadata lock —
|
||||||
|
// notably DeleteBucket, whose lock is taken through its erasureServerPools
|
||||||
|
// receiver and is therefore invisible to an injected object layer.
|
||||||
|
var lockBucketMetadataAcquireHook atomic.Pointer[func(bucket string)]
|
||||||
|
|
||||||
|
func lockBucketMetadataWithTimeout(ctx context.Context, objectAPI ObjectLayer, bucket string, timeout *dynamicTimeout) (context.Context, func(), error) {
|
||||||
|
lock := objectAPI.NewNSLock(minioMetaBucket, pathJoin(bucketMetaPrefix, bucket, "metadata.lock"))
|
||||||
|
if hook := lockBucketMetadataAcquireHook.Load(); hook != nil {
|
||||||
|
(*hook)(bucket)
|
||||||
|
}
|
||||||
|
lkctx, err := lock.GetLock(ctx, timeout)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
ctx = context.WithValue(lkctx.Context(), bucketMetadataLockContextKey{}, bucket)
|
||||||
|
return ctx, func() { lock.Unlock(lkctx) }, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type bucketMetadataLockContextKey struct{}
|
||||||
|
|
||||||
|
func bucketMetadataLockHeld(ctx context.Context, bucket string) bool {
|
||||||
|
lockedBucket, _ := ctx.Value(bucketMetadataLockContextKey{}).(string)
|
||||||
|
return lockedBucket == bucket
|
||||||
|
}
|
||||||
|
|
||||||
// Delete delete the bucket metadata for the specified bucket.
|
// Delete delete the bucket metadata for the specified bucket.
|
||||||
// must be used by all callers instead of using Update() with nil configData.
|
// must be used by all callers instead of using Update() with nil configData.
|
||||||
func (sys *BucketMetadataSys) Delete(ctx context.Context, bucket string, configFile string) (updatedAt time.Time, err error) {
|
func (sys *BucketMetadataSys) Delete(ctx context.Context, bucket string, configFile string) (updatedAt time.Time, err error) {
|
||||||
if configFile == bucketLifecycleConfig {
|
return sys.updateAndParse(ctx, bucket, configFile, nil, false, configFile == bucketLifecycleConfig)
|
||||||
// Get bucket config from current site
|
}
|
||||||
meta, e := globalBucketMetadataSys.GetConfigFromDisk(ctx, bucket)
|
|
||||||
if e != nil && !errors.Is(e, errConfigNotFound) {
|
|
||||||
return updatedAt, e
|
|
||||||
}
|
|
||||||
var expiryRuleRemoved bool
|
|
||||||
if len(meta.LifecycleConfigXML) > 0 {
|
|
||||||
var lcCfg lifecycle.Lifecycle
|
|
||||||
if err := xml.Unmarshal(meta.LifecycleConfigXML, &lcCfg); err != nil {
|
|
||||||
return updatedAt, err
|
|
||||||
}
|
|
||||||
// find a single expiry rule set the flag
|
|
||||||
for _, rl := range lcCfg.Rules {
|
|
||||||
if !rl.Expiration.IsNull() || !rl.NoncurrentVersionExpiration.IsNull() {
|
|
||||||
expiryRuleRemoved = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Form empty ILM details with `ExpiryUpdatedAt` field and save
|
func lifecycleDeleteConfig(current []byte) ([]byte, error) {
|
||||||
var cfgData []byte
|
var expiryRuleRemoved bool
|
||||||
if expiryRuleRemoved {
|
if len(current) > 0 {
|
||||||
var lcCfg lifecycle.Lifecycle
|
var lcCfg lifecycle.Lifecycle
|
||||||
currtime := time.Now()
|
if err := xml.Unmarshal(current, &lcCfg); err != nil {
|
||||||
lcCfg.ExpiryUpdatedAt = &currtime
|
return nil, err
|
||||||
cfgData, err = xml.Marshal(lcCfg)
|
}
|
||||||
if err != nil {
|
for _, rl := range lcCfg.Rules {
|
||||||
return updatedAt, err
|
if !rl.Expiration.IsNull() || !rl.NoncurrentVersionExpiration.IsNull() {
|
||||||
|
expiryRuleRemoved = true
|
||||||
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return sys.updateAndParse(ctx, bucket, configFile, cfgData, false)
|
|
||||||
}
|
}
|
||||||
return sys.updateAndParse(ctx, bucket, configFile, nil, false)
|
if !expiryRuleRemoved {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
var lcCfg lifecycle.Lifecycle
|
||||||
|
currtime := time.Now()
|
||||||
|
lcCfg.ExpiryUpdatedAt = &currtime
|
||||||
|
return xml.Marshal(lcCfg)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update update bucket metadata for the specified bucket.
|
// Update update bucket metadata for the specified bucket.
|
||||||
// The configData data should not be modified after being sent here.
|
// The configData data should not be modified after being sent here.
|
||||||
func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) (updatedAt time.Time, err error) {
|
func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) (updatedAt time.Time, err error) {
|
||||||
return sys.updateAndParse(ctx, bucket, configFile, configData, true)
|
return sys.updateAndParse(ctx, bucket, configFile, configData, true, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdateExpiryLCConfig merges a replicated ILM expiry configuration with the
|
||||||
|
// bucket's current lifecycle document and persists the merged result while
|
||||||
|
// holding metadata.lock across the read, merge, and save. The site-replication
|
||||||
|
// expiry heal and peer-apply paths must use this instead of computing the merge
|
||||||
|
// from an unlocked GetConfigFromDisk read and then writing it with Update: that
|
||||||
|
// two-step sequence drops any lifecycle transition change committed in between
|
||||||
|
// (issue #105). Lock order stays <bucket>.lck -> metadata.lock -> .metadata.bin;
|
||||||
|
// the merge and save run under metadata.lock and the peer fan-out runs after it
|
||||||
|
// is released.
|
||||||
|
func (sys *BucketMetadataSys) UpdateExpiryLCConfig(ctx context.Context, bucket string, expLCConfig *string, updatedAt time.Time) error {
|
||||||
|
objAPI := newObjectLayerFn()
|
||||||
|
if objAPI == nil {
|
||||||
|
return errServerNotInitialized
|
||||||
|
}
|
||||||
|
|
||||||
|
if isMinioMetaBucketName(bucket) {
|
||||||
|
return errInvalidArgument
|
||||||
|
}
|
||||||
|
|
||||||
|
notifyCtx := ctx
|
||||||
|
ctx, unlock, err := lockBucketMetadata(ctx, objAPI, bucket)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = func() error {
|
||||||
|
defer unlock()
|
||||||
|
meta, err := loadBucketMetadataParse(ctx, objAPI, bucket, true)
|
||||||
|
if err != nil {
|
||||||
|
if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
|
||||||
|
// Only single drive mode needs this fallback.
|
||||||
|
meta = newBucketMetadata(bucket)
|
||||||
|
} else {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
configData, err := mergeExpiryWithLCConfig(bucket, meta, expLCConfig, updatedAt)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
meta.LifecycleConfigXML = configData
|
||||||
|
meta.LifecycleConfigUpdatedAt = UTCNow()
|
||||||
|
return sys.saveMetadata(ctx, objAPI, &meta)
|
||||||
|
}()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
globalNotificationSys.LoadBucketMetadata(bgContext(notifyCtx), bucket) // Do not use caller context here
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get metadata for a bucket.
|
// Get metadata for a bucket.
|
||||||
@@ -378,51 +532,17 @@ func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEC
|
|||||||
return meta.sseConfig, meta.EncryptionConfigUpdatedAt, nil
|
return meta.sseConfig, meta.EncryptionConfigUpdatedAt, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetCorsConfig returns the CORS configuration for the given bucket.
|
// GetResidentCorsConfig returns the CORS configuration of a bucket whose
|
||||||
// The returned object must not be modified.
|
// metadata is already resident in memory. It runs before authentication for
|
||||||
func (sys *BucketMetadataSys) GetCorsConfig(bucket string) (*cors.Config, time.Time, error) {
|
// every Origin-bearing request with a client-supplied path segment, so it
|
||||||
meta, _, err := sys.GetConfig(GlobalContext, bucket)
|
// never loads or caches metadata. A non-resident name gets no CORS answer
|
||||||
if err != nil {
|
// (errBucketMetadataNotInitialized) while startup loading is still running,
|
||||||
return nil, time.Time{}, err
|
// and afterwards when it is a real bucket whose metadata failed to load: a
|
||||||
}
|
// presigned URL is authenticated on its own, so the bucket's CORS document is
|
||||||
if meta.corsConfigErr != nil {
|
// the only origin boundary a browser enforces for it. Any other non-resident
|
||||||
return nil, meta.CorsConfigUpdatedAt, meta.corsConfigErr
|
// name reports errConfigNotFound and the caller applies the global CORS
|
||||||
}
|
// policy exactly as releases without per-bucket CORS did.
|
||||||
if meta.corsConfig == nil {
|
|
||||||
return nil, time.Time{}, errConfigNotFound
|
|
||||||
}
|
|
||||||
return meta.corsConfig, meta.CorsConfigUpdatedAt, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetResidentCorsConfig returns the CORS configuration for the given bucket
|
|
||||||
// using only bucket metadata that is already resident in memory. Unlike
|
|
||||||
// GetCorsConfig it never loads metadata from disk and never caches a new
|
|
||||||
// entry.
|
|
||||||
//
|
|
||||||
// The per-request CORS middleware runs before authentication, for every
|
|
||||||
// Origin-bearing request, using the validated first path segment as the bucket
|
|
||||||
// name.
|
|
||||||
// Routing that through GetCorsConfig (which loads and caches) let an
|
|
||||||
// unauthenticated client grow metadataMap without bound and trigger an
|
|
||||||
// erasure metadata probe for every distinct, attacker-controlled,
|
|
||||||
// non-existent name it sent with an Origin header (e.g. /minio/... , /api/... ,
|
|
||||||
// or random buckets). Every bucket that can carry a CORS document is made
|
|
||||||
// resident when the document is written (Set) and when metadata is loaded at
|
|
||||||
// startup (Init/concurrentLoad), so a resident-only read is complete for real
|
|
||||||
// buckets while costing only an in-memory map lookup for everything else.
|
|
||||||
//
|
|
||||||
// While bucket metadata is still loading (not yet Initialized) a non-resident
|
|
||||||
// bucket returns errBucketMetadataNotInitialized so the caller fails closed
|
|
||||||
// rather than answering with the permissive global policy for a bucket whose
|
|
||||||
// restrictive CORS document may simply not be loaded yet.
|
|
||||||
func (sys *BucketMetadataSys) GetResidentCorsConfig(bucket string) (*cors.Config, time.Time, error) {
|
func (sys *BucketMetadataSys) GetResidentCorsConfig(bucket string) (*cors.Config, time.Time, error) {
|
||||||
if isMinioMetaBucketName(bucket) {
|
|
||||||
// Preserve GetConfig's semantics for the internal namespace: this is
|
|
||||||
// not a real bucket, and returning a non-errConfigNotFound error makes
|
|
||||||
// the CORS middleware fail closed rather than answer for .minio.sys
|
|
||||||
// with the permissive global policy.
|
|
||||||
return nil, time.Time{}, errInvalidArgument
|
|
||||||
}
|
|
||||||
if isReservedOrInvalidBucket(bucket, true) {
|
if isReservedOrInvalidBucket(bucket, true) {
|
||||||
return nil, time.Time{}, errConfigNotFound
|
return nil, time.Time{}, errConfigNotFound
|
||||||
}
|
}
|
||||||
@@ -431,26 +551,19 @@ func (sys *BucketMetadataSys) GetResidentCorsConfig(bucket string) (*cors.Config
|
|||||||
_, failed := sys.loadFailed[bucket]
|
_, failed := sys.loadFailed[bucket]
|
||||||
initialized := sys.initialized
|
initialized := sys.initialized
|
||||||
sys.RUnlock()
|
sys.RUnlock()
|
||||||
if ok {
|
if !ok {
|
||||||
if meta.corsConfigErr != nil {
|
if !initialized || failed {
|
||||||
return nil, meta.CorsConfigUpdatedAt, meta.corsConfigErr
|
return nil, time.Time{}, errBucketMetadataNotInitialized
|
||||||
}
|
}
|
||||||
if meta.corsConfig == nil {
|
return nil, time.Time{}, errConfigNotFound
|
||||||
return nil, time.Time{}, errConfigNotFound
|
|
||||||
}
|
|
||||||
return meta.corsConfig, meta.CorsConfigUpdatedAt, nil
|
|
||||||
}
|
}
|
||||||
// Not resident. Two cases must not be conflated:
|
if meta.corsConfigErr != nil {
|
||||||
// - metadata is still loading (!initialized), or this is a real bucket
|
return nil, meta.CorsConfigUpdatedAt, meta.corsConfigErr
|
||||||
// whose metadata failed to load: we cannot rule out a restrictive CORS
|
|
||||||
// config, so fail closed rather than answer with the global policy.
|
|
||||||
// - a fully initialized subsystem with no record of the name: it is not a
|
|
||||||
// bucket that can carry CORS, so fall back to the global policy without
|
|
||||||
// loading or caching metadata for an arbitrary, client-supplied name.
|
|
||||||
if !initialized || failed {
|
|
||||||
return nil, time.Time{}, errBucketMetadataNotInitialized
|
|
||||||
}
|
}
|
||||||
return nil, time.Time{}, errConfigNotFound
|
if meta.corsConfig == nil {
|
||||||
|
return nil, time.Time{}, errConfigNotFound
|
||||||
|
}
|
||||||
|
return meta.corsConfig, meta.CorsConfigUpdatedAt, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetCorsConfigXML returns the raw stored CORS configuration XML for the
|
// GetCorsConfigXML returns the raw stored CORS configuration XML for the
|
||||||
@@ -599,6 +712,7 @@ func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (met
|
|||||||
}
|
}
|
||||||
sys.Lock()
|
sys.Lock()
|
||||||
sys.metadataMap[bucket] = meta
|
sys.metadataMap[bucket] = meta
|
||||||
|
sys.clearLoadFailure(bucket)
|
||||||
sys.Unlock()
|
sys.Unlock()
|
||||||
|
|
||||||
return meta, true, nil
|
return meta, true, nil
|
||||||
@@ -650,13 +764,10 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []stri
|
|||||||
sys.Lock()
|
sys.Lock()
|
||||||
for i, meta := range bucketMetas {
|
for i, meta := range bucketMetas {
|
||||||
if errs[i] != nil {
|
if errs[i] != nil {
|
||||||
// Real bucket whose metadata could not be loaded: record it so
|
sys.noteLoadFailure(buckets[i])
|
||||||
// consumers that must fail closed (per-bucket CORS) can tell it
|
|
||||||
// apart from a name that is not a bucket at all.
|
|
||||||
sys.loadFailed[buckets[i]] = struct{}{}
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
delete(sys.loadFailed, buckets[i])
|
sys.clearLoadFailure(buckets[i])
|
||||||
sys.metadataMap[buckets[i]] = meta
|
sys.metadataMap[buckets[i]] = meta
|
||||||
}
|
}
|
||||||
sys.Unlock()
|
sys.Unlock()
|
||||||
@@ -707,7 +818,7 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
internalLogIf(ctx, err, logger.WarningKind)
|
internalLogIf(ctx, err, logger.WarningKind)
|
||||||
sys.Lock()
|
sys.Lock()
|
||||||
sys.loadFailed[bucket] = struct{}{}
|
sys.noteLoadFailure(bucket)
|
||||||
sys.Unlock()
|
sys.Unlock()
|
||||||
wait() // wait to proceed to next entry.
|
wait() // wait to proceed to next entry.
|
||||||
continue
|
continue
|
||||||
@@ -719,8 +830,7 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
|
|||||||
updated = true
|
updated = true
|
||||||
sys.metadataMap[bucket] = meta
|
sys.metadataMap[bucket] = meta
|
||||||
}
|
}
|
||||||
// A successful (re)load clears any earlier load failure.
|
sys.clearLoadFailure(bucket)
|
||||||
delete(sys.loadFailed, bucket)
|
|
||||||
sys.Unlock()
|
sys.Unlock()
|
||||||
|
|
||||||
if updated {
|
if updated {
|
||||||
|
|||||||
+46
-11
@@ -41,8 +41,8 @@ import (
|
|||||||
"github.com/minio/minio/internal/event"
|
"github.com/minio/minio/internal/event"
|
||||||
"github.com/minio/minio/internal/kms"
|
"github.com/minio/minio/internal/kms"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/pkg/v3/policy"
|
|
||||||
"github.com/minio/sio"
|
"github.com/minio/sio"
|
||||||
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -249,8 +249,17 @@ func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket
|
|||||||
}
|
}
|
||||||
|
|
||||||
if len(configs) > 0 {
|
if len(configs) > 0 {
|
||||||
// Old bucket without bucket metadata. Hence we migrate existing settings.
|
if !bucketMetadataLockHeld(ctx, bucket) {
|
||||||
if err = b.convertLegacyConfigs(ctx, objectAPI, configs); err != nil {
|
migrated, lockErr := loadBucketMetadataParseUnderLock(ctx, objectAPI, bucket, parse)
|
||||||
|
if lockErr == nil {
|
||||||
|
return migrated, nil
|
||||||
|
}
|
||||||
|
if !errors.Is(lockErr, errBucketMetadataMigrationLockUnavailable) {
|
||||||
|
return b, lockErr
|
||||||
|
}
|
||||||
|
internalLogOnceIf(ctx, fmt.Errorf("unable to persist bucket metadata migration for %s, using the legacy configuration in memory: %w", bucket, lockErr), "bucket-metadata-migration-lock-"+bucket)
|
||||||
|
b.applyLegacyConfigs(configs)
|
||||||
|
} else if err = b.convertLegacyConfigs(ctx, objectAPI, configs); err != nil {
|
||||||
return b, err
|
return b, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -270,6 +279,17 @@ func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket
|
|||||||
}
|
}
|
||||||
|
|
||||||
// migrate unencrypted remote targets
|
// migrate unencrypted remote targets
|
||||||
|
if len(b.BucketTargetsConfigJSON) != 0 && GlobalKMS != nil && len(b.BucketTargetsConfigMetaJSON) == 0 && !bucketMetadataLockHeld(ctx, bucket) {
|
||||||
|
migrated, lockErr := loadBucketMetadataParseUnderLock(ctx, objectAPI, bucket, parse)
|
||||||
|
if lockErr == nil {
|
||||||
|
return migrated, nil
|
||||||
|
}
|
||||||
|
if !errors.Is(lockErr, errBucketMetadataMigrationLockUnavailable) {
|
||||||
|
return b, lockErr
|
||||||
|
}
|
||||||
|
internalLogOnceIf(ctx, fmt.Errorf("unable to persist encrypted bucket target metadata for %s, using the existing configuration in memory: %w", bucket, lockErr), "bucket-metadata-migration-lock-"+bucket)
|
||||||
|
return b, nil
|
||||||
|
}
|
||||||
if err = b.migrateTargetConfig(ctx, objectAPI); err != nil {
|
if err = b.migrateTargetConfig(ctx, objectAPI); err != nil {
|
||||||
return b, err
|
return b, err
|
||||||
}
|
}
|
||||||
@@ -277,6 +297,23 @@ func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket
|
|||||||
return b, nil
|
return b, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func loadBucketMetadataParseUnderLock(ctx context.Context, objectAPI ObjectLayer, bucket string, parse bool) (BucketMetadata, error) {
|
||||||
|
ctx, unlock, err := lockBucketMetadataWithTimeout(ctx, objectAPI, bucket, bucketMetadataMigrationTimeout)
|
||||||
|
if err != nil {
|
||||||
|
return newBucketMetadata(bucket), fmt.Errorf("%w: %v", errBucketMetadataMigrationLockUnavailable, err)
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
|
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{NoMetadata: true}); err != nil {
|
||||||
|
return newBucketMetadata(bucket), err
|
||||||
|
}
|
||||||
|
return loadBucketMetadataParse(ctx, objectAPI, bucket, parse)
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
bucketMetadataMigrationTimeout = newDynamicTimeout(5*time.Second, time.Second)
|
||||||
|
errBucketMetadataMigrationLockUnavailable = errors.New("bucket metadata migration lock unavailable")
|
||||||
|
)
|
||||||
|
|
||||||
// loadBucketMetadata loads and migrates to bucket metadata.
|
// loadBucketMetadata loads and migrates to bucket metadata.
|
||||||
func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
|
func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
|
||||||
return loadBucketMetadataParse(ctx, objectAPI, bucket, true)
|
return loadBucketMetadataParse(ctx, objectAPI, bucket, true)
|
||||||
@@ -343,13 +380,6 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
|
|||||||
b.corsConfig = nil
|
b.corsConfig = nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if bytes.Equal(b.ObjectLockConfigXML, enabledBucketObjectLockConfig) {
|
|
||||||
config, versioningErr := versioning.ParseConfig(bytes.NewReader(b.VersioningConfigXML))
|
|
||||||
if versioningErr != nil || !config.Enabled() {
|
|
||||||
b.VersioningConfigXML = enabledBucketVersioningConfig
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(b.ObjectLockConfigXML) != 0 {
|
if len(b.ObjectLockConfigXML) != 0 {
|
||||||
b.objectLockConfig, err = objectlock.ParseObjectLockConfig(bytes.NewReader(b.ObjectLockConfigXML))
|
b.objectLockConfig, err = objectlock.ParseObjectLockConfig(bytes.NewReader(b.ObjectLockConfigXML))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -358,6 +388,7 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
|
|||||||
} else {
|
} else {
|
||||||
b.objectLockConfig = nil
|
b.objectLockConfig = nil
|
||||||
}
|
}
|
||||||
|
b.VersioningConfigXML = effectiveBucketVersioning(b.VersioningConfigXML, b.objectLockConfig != nil)
|
||||||
|
|
||||||
if len(b.VersioningConfigXML) != 0 {
|
if len(b.VersioningConfigXML) != 0 {
|
||||||
b.versioningConfig, err = versioning.ParseConfig(bytes.NewReader(b.VersioningConfigXML))
|
b.versioningConfig, err = versioning.ParseConfig(bytes.NewReader(b.VersioningConfigXML))
|
||||||
@@ -440,7 +471,7 @@ func (b *BucketMetadata) getAllLegacyConfigs(ctx context.Context, objectAPI Obje
|
|||||||
return configs, nil
|
return configs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer, configs map[string][]byte) error {
|
func (b *BucketMetadata) applyLegacyConfigs(configs map[string][]byte) {
|
||||||
for legacyFile, configData := range configs {
|
for legacyFile, configData := range configs {
|
||||||
switch legacyFile {
|
switch legacyFile {
|
||||||
case legacyBucketObjectLockEnabledConfigFile:
|
case legacyBucketObjectLockEnabledConfigFile:
|
||||||
@@ -472,6 +503,10 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
b.defaultTimestamps()
|
b.defaultTimestamps()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer, configs map[string][]byte) error {
|
||||||
|
b.applyLegacyConfigs(configs)
|
||||||
|
|
||||||
if err := b.Save(ctx, objectAPI); err != nil {
|
if err := b.Save(ctx, objectAPI); err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -39,3 +39,28 @@ func TestBucketMetadataCorsRoundTrip(t *testing.T) {
|
|||||||
t.Fatalf("CorsConfigUpdatedAt not preserved")
|
t.Fatalf("CorsConfigUpdatedAt not preserved")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A persisted retired extension must not prevent the whole bucket's metadata
|
||||||
|
// from loading, including unrelated versioning and ordinary lifecycle rules.
|
||||||
|
func TestBucketMetadataRetiredAccessTiering(t *testing.T) {
|
||||||
|
meta := newBucketMetadata("retired-access")
|
||||||
|
meta.LifecycleConfigXML = []byte(`<LifecycleConfiguration><AccessTierQuota>500GiB</AccessTierQuota><Rule><ID>access</ID><Status>Enabled</Status><Filter><Prefix>logs/</Prefix></Filter><AccessTransition><Window>10m</Window><PromoteAfterAccesses>10</PromoteAfterAccesses></AccessTransition></Rule><Rule><ID>ordinary</ID><Status>Enabled</Status><Filter><Prefix>expired/</Prefix></Filter><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>`)
|
||||||
|
meta.VersioningConfigXML = []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status></VersioningConfiguration>`)
|
||||||
|
data, err := meta.MarshalMsg(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := newBucketMetadata(meta.Name)
|
||||||
|
if _, err := got.UnmarshalMsg(data); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := got.parseAllConfigs(t.Context(), nil); err != nil {
|
||||||
|
t.Fatalf("bucket metadata failed to load: %v", err)
|
||||||
|
}
|
||||||
|
if got.lifecycleConfig == nil || got.lifecycleConfig.HasActiveRules("logs/") || !got.lifecycleConfig.HasActiveRules("expired/") {
|
||||||
|
t.Fatal("unexpected lifecycle behavior")
|
||||||
|
}
|
||||||
|
if got.versioningConfig == nil || !got.versioningConfig.Enabled() {
|
||||||
|
t.Fatal("unrelated versioning lost")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ import (
|
|||||||
"github.com/minio/minio/internal/event"
|
"github.com/minio/minio/internal/event"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|||||||
+183
-6
@@ -22,11 +22,15 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"math"
|
"math"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/amztime"
|
||||||
"github.com/minio/minio/internal/auth"
|
"github.com/minio/minio/internal/auth"
|
||||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
// BucketObjectLockSys - map of bucket and retention configuration.
|
// BucketObjectLockSys - map of bucket and retention configuration.
|
||||||
@@ -330,11 +334,10 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
|||||||
return mode, retainDate, legalHold, ErrObjectLocked
|
return mode, retainDate, legalHold, ErrObjectLocked
|
||||||
}
|
}
|
||||||
|
|
||||||
if !legalHoldRequested && retentionCfg.LockEnabled {
|
// Inherit retention from the bucket configuration. A legal-hold header
|
||||||
// inherit retention from bucket configuration
|
// on the same request, ON or OFF, is independent of retention and must
|
||||||
return retentionCfg.Mode, objectlock.RetentionDate{Time: t.Add(retentionCfg.Validity)}, legalHold, ErrNone
|
// not suppress the default (#165).
|
||||||
}
|
return retentionCfg.Mode, objectlock.RetentionDate{Time: t.Add(retentionCfg.Validity)}, legalHold, ErrNone
|
||||||
return "", objectlock.RetentionDate{}, legalHold, ErrNone
|
|
||||||
}
|
}
|
||||||
return mode, retainDate, legalHold, ErrNone
|
return mode, retainDate, legalHold, ErrNone
|
||||||
}
|
}
|
||||||
@@ -343,3 +346,177 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
|||||||
func NewBucketObjectLockSys() *BucketObjectLockSys {
|
func NewBucketObjectLockSys() *BucketObjectLockSys {
|
||||||
return &BucketObjectLockSys{}
|
return &BucketObjectLockSys{}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// objectLockState is the Object Lock metadata of a stored object version
|
||||||
|
// together with the replication timestamps that order updates to it.
|
||||||
|
type objectLockState struct {
|
||||||
|
mode, retainUntil, retentionTimestamp string
|
||||||
|
legalHold, legalHoldTimestamp string
|
||||||
|
}
|
||||||
|
|
||||||
|
func storedObjectLockState(metadata map[string]string) objectLockState {
|
||||||
|
return objectLockState{
|
||||||
|
mode: metadata[strings.ToLower(xhttp.AmzObjectLockMode)],
|
||||||
|
retainUntil: metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)],
|
||||||
|
retentionTimestamp: metadata[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp],
|
||||||
|
legalHold: metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)],
|
||||||
|
legalHoldTimestamp: metadata[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// olderThan reports whether a stored replication timestamp is missing,
|
||||||
|
// unreadable, or earlier than the source timestamp, in which case the
|
||||||
|
// replica update wins. A zero source timestamp never wins.
|
||||||
|
func olderThan(stored string, src time.Time) bool {
|
||||||
|
if src.IsZero() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
ondisk, err := time.Parse(time.RFC3339Nano, stored)
|
||||||
|
return err != nil || ondisk.Before(src)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s objectLockState) retentionIsOlderThan(src time.Time) bool {
|
||||||
|
return olderThan(s.retentionTimestamp, src)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s objectLockState) legalHoldIsOlderThan(src time.Time) bool {
|
||||||
|
return olderThan(s.legalHoldTimestamp, src)
|
||||||
|
}
|
||||||
|
|
||||||
|
// restoreRetention and restoreLegalHold put the stored state back into
|
||||||
|
// metadata that was rebuilt from a request whose update was not applied.
|
||||||
|
func (s objectLockState) restoreRetention(metadata map[string]string) {
|
||||||
|
// The stored timestamp orders the next update and must survive even when
|
||||||
|
// the stored value is empty, which is how a removal is recorded.
|
||||||
|
if s.retentionTimestamp != "" {
|
||||||
|
metadata[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = s.retentionTimestamp
|
||||||
|
}
|
||||||
|
if s.mode == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = s.mode
|
||||||
|
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = s.retainUntil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s objectLockState) restoreLegalHold(metadata map[string]string) {
|
||||||
|
if s.legalHoldTimestamp != "" {
|
||||||
|
metadata[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = s.legalHoldTimestamp
|
||||||
|
}
|
||||||
|
if s.legalHold == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = s.legalHold
|
||||||
|
}
|
||||||
|
|
||||||
|
// replicaStoredLock reads the Object Lock state stored on the addressed version
|
||||||
|
// so a trusted replica write can order its update against it. A missing object
|
||||||
|
// or version yields an empty state, which is correct for the first write of a
|
||||||
|
// version; any other read error is returned so the caller fails the write rather
|
||||||
|
// than ordering an incoming update against lock state it merely failed to read
|
||||||
|
// (an older incoming value must not win over a newer stored one just because the
|
||||||
|
// read timed out).
|
||||||
|
func replicaStoredLock(ctx context.Context, getObjectInfo GetObjectInfoFn, bucket, object, versionID string) (objectLockState, error) {
|
||||||
|
oi, err := getObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: versionID})
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
return storedObjectLockState(oi.UserDefined), nil
|
||||||
|
case isErrObjectNotFound(err) || isErrVersionNotFound(err):
|
||||||
|
return objectLockState{}, nil
|
||||||
|
default:
|
||||||
|
return objectLockState{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// applyReplicatedObjectLock writes the retention and legal-hold decision into
|
||||||
|
// metadata for a PUT, CopyObject, or multipart-initiation request. A request
|
||||||
|
// that is not an actual trusted replica -- a normal user write, or a trusted
|
||||||
|
// peer that carried the replication marker without REPLICA status -- takes
|
||||||
|
// ordinary write semantics: a validated value is applied and stamped now, and a
|
||||||
|
// missing value is left as is. Only an actual replica update is ordered against
|
||||||
|
// the state already stored on the addressed version, so a stale value cannot
|
||||||
|
// overwrite a newer one and a full retransmit cannot roll a destination back.
|
||||||
|
// The stored argument is meaningful only for a replica; callers pass an empty
|
||||||
|
// state otherwise. Only the two Object Lock keys and their reserved ordering
|
||||||
|
// timestamps are touched; any encryption-metadata reconciliation stays with the
|
||||||
|
// caller.
|
||||||
|
func applyReplicatedObjectLock(metadata map[string]string, stored objectLockState,
|
||||||
|
replicaTrusted bool,
|
||||||
|
retentionMode objectlock.RetMode, retentionDate objectlock.RetentionDate,
|
||||||
|
legalHold objectlock.ObjectLegalHold, srcRetentionTimestamp, srcLegalholdTimestamp time.Time,
|
||||||
|
) {
|
||||||
|
switch {
|
||||||
|
case !replicaTrusted:
|
||||||
|
// Ordinary write semantics: apply a validated retention and stamp it now;
|
||||||
|
// a missing value carries no instruction, so leave the metadata as it is.
|
||||||
|
if retentionMode.Valid() {
|
||||||
|
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
|
||||||
|
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC())
|
||||||
|
metadata[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||||
|
}
|
||||||
|
case !stored.retentionIsOlderThan(srcRetentionTimestamp):
|
||||||
|
// The stored update is at least as new as this replica's, or the replica
|
||||||
|
// carries no ordering timestamp: keep what is stored. This is also how a
|
||||||
|
// stale retransmit is rejected.
|
||||||
|
stored.restoreRetention(metadata)
|
||||||
|
default:
|
||||||
|
// The replica update wins. A removal carries no value but still records
|
||||||
|
// the source timestamp that orders it.
|
||||||
|
if retentionMode.Valid() {
|
||||||
|
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
|
||||||
|
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC())
|
||||||
|
}
|
||||||
|
metadata[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = srcRetentionTimestamp.UTC().Format(time.RFC3339Nano)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Legal hold has no removal in S3: an explicitly empty header is already
|
||||||
|
// rejected as an invalid status, so the only value-less shape that gets here
|
||||||
|
// is an absent one, which conveys no legal-hold change. Only a valid status
|
||||||
|
// can win.
|
||||||
|
switch {
|
||||||
|
case !replicaTrusted:
|
||||||
|
if legalHold.Status.Valid() {
|
||||||
|
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
|
||||||
|
metadata[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||||
|
}
|
||||||
|
case legalHold.Status.Valid() && stored.legalHoldIsOlderThan(srcLegalholdTimestamp):
|
||||||
|
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
|
||||||
|
metadata[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = srcLegalholdTimestamp.UTC().Format(time.RFC3339Nano)
|
||||||
|
default:
|
||||||
|
stored.restoreLegalHold(metadata)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// reconcileStoredObjectLock re-orders the Object Lock already written into
|
||||||
|
// metadata against the state currently stored on the destination version, both
|
||||||
|
// compared by their reserved ordering timestamps. It runs inside the object
|
||||||
|
// layer under the namespace write lock that guards the version replacement,
|
||||||
|
// after the destination version is read and before the new one is committed, so
|
||||||
|
// a replica update whose ordering was decided at handler time (or, for multipart,
|
||||||
|
// at initiation) cannot overwrite a newer lock update that reached the version in
|
||||||
|
// between. metadata already carries the incoming update with its source
|
||||||
|
// timestamps; a stored value that is not older than the incoming one is put back,
|
||||||
|
// which for a stored removal means clearing the incoming value and keeping only
|
||||||
|
// the removal's timestamp. Only the two lock keys and their reserved timestamps
|
||||||
|
// move; a non-replica write never sets the flag that invokes this.
|
||||||
|
func reconcileStoredObjectLock(metadata map[string]string, stored objectLockState) {
|
||||||
|
incoming := storedObjectLockState(metadata)
|
||||||
|
|
||||||
|
incomingRetentionTS, _ := time.Parse(time.RFC3339Nano, incoming.retentionTimestamp)
|
||||||
|
if !stored.retentionIsOlderThan(incomingRetentionTS) {
|
||||||
|
// The stored retention is at least as new as the incoming one (or the
|
||||||
|
// incoming update is unordered): drop the incoming value and put the stored
|
||||||
|
// state back, which may itself be a removal (value keys absent, timestamp
|
||||||
|
// present).
|
||||||
|
delete(metadata, strings.ToLower(xhttp.AmzObjectLockMode))
|
||||||
|
delete(metadata, strings.ToLower(xhttp.AmzObjectLockRetainUntilDate))
|
||||||
|
delete(metadata, ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp)
|
||||||
|
stored.restoreRetention(metadata)
|
||||||
|
}
|
||||||
|
|
||||||
|
incomingLegalHoldTS, _ := time.Parse(time.RFC3339Nano, incoming.legalHoldTimestamp)
|
||||||
|
if incoming.legalHold == "" || !stored.legalHoldIsOlderThan(incomingLegalHoldTS) {
|
||||||
|
delete(metadata, strings.ToLower(xhttp.AmzObjectLockLegalHold))
|
||||||
|
delete(metadata, ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp)
|
||||||
|
stored.restoreLegalHold(metadata)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ package cmd
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
@@ -27,7 +26,7 @@ import (
|
|||||||
"github.com/minio/madmin-go/v3"
|
"github.com/minio/madmin-go/v3"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -100,13 +99,13 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
configData, err := json.Marshal(bucketPolicy)
|
configData, err := canonicalBucketPolicy(bucketPolicy)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
|
result, err := globalBucketMetadataSys.updateAndParseMetadata(ctx, bucket, bucketPolicyConfig, configData, false, false, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
@@ -116,8 +115,8 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
|
|||||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||||
Type: madmin.SRBucketMetaTypePolicy,
|
Type: madmin.SRBucketMetaTypePolicy,
|
||||||
Bucket: bucket,
|
Bucket: bucket,
|
||||||
Policy: bucketPolicyBytes,
|
Policy: result.meta.PolicyConfigJSON,
|
||||||
UpdatedAt: updatedAt,
|
UpdatedAt: result.updatedAt,
|
||||||
}))
|
}))
|
||||||
|
|
||||||
// Success.
|
// Success.
|
||||||
@@ -200,7 +199,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
configData, err := json.Marshal(config)
|
configData, err := canonicalBucketPolicy(config)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -29,8 +29,8 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/minio/minio/internal/auth"
|
"github.com/minio/minio/internal/auth"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
"github.com/minio/pkg/v3/policy/condition"
|
"github.com/pgsty/silo-pkg/v3/policy/condition"
|
||||||
)
|
)
|
||||||
|
|
||||||
func getAnonReadOnlyBucketPolicy(bucketName string) *policy.BucketPolicy {
|
func getAnonReadOnlyBucketPolicy(bucketName string) *policy.BucketPolicy {
|
||||||
|
|||||||
+28
-9
@@ -33,8 +33,8 @@ import (
|
|||||||
"github.com/minio/minio/internal/handlers"
|
"github.com/minio/minio/internal/handlers"
|
||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
"github.com/minio/pkg/v3/policy/condition"
|
"github.com/pgsty/silo-pkg/v3/policy/condition"
|
||||||
)
|
)
|
||||||
|
|
||||||
// PolicySys - policy subsystem.
|
// PolicySys - policy subsystem.
|
||||||
@@ -255,13 +255,32 @@ func getConditionValuesWithTags(r *http.Request, lc string, cred auth.Credential
|
|||||||
}
|
}
|
||||||
|
|
||||||
cloneHeader := r.Header.Clone()
|
cloneHeader := r.Header.Clone()
|
||||||
signatureAge := cloneHeader.Get("x-amz-signature-age")
|
|
||||||
cloneHeader.Del("x-amz-signature-age")
|
// s3:signatureAge is derived from the presigned X-Amz-Date rather than from
|
||||||
// The presigned V4 verifier overwrites this internal scratch header after
|
// anything the verifier writes back: PutObject and UploadPart authorize
|
||||||
// validating the signature. Ignore a value supplied on every other request
|
// before they verify the signature, so a post-verification value is not yet
|
||||||
// type, where it would otherwise synthesize s3:signatureAge.
|
// available on the first evaluation. The date is bound by the signature
|
||||||
if authType == authTypePresigned && signatureAge != "" {
|
// (doesPresignedSignatureMatch rebuilds and compares it), so a forged date
|
||||||
args["signatureAge"] = []string{signatureAge}
|
// only changes the authorization outcome of a request that then fails
|
||||||
|
// verification. A date that does not parse leaves the key absent; the
|
||||||
|
// verifier rejects the request as ErrMalformedPresignedDate.
|
||||||
|
if authType == authTypePresigned {
|
||||||
|
if signedDate, err := time.Parse(iso8601Format, r.Form.Get(xhttp.AmzDate)); err == nil {
|
||||||
|
args["signatureAge"] = []string{strconv.FormatInt(currTime.Sub(signedDate).Milliseconds(), 10)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// s3:x-amz-content-sha256 must name the payload hash the request is actually
|
||||||
|
// verified and enforced against, and only one such value. Presence of the
|
||||||
|
// header controls whether the key exists at all (AWS documents that the
|
||||||
|
// query-string form does not populate it), but the value comes from the same
|
||||||
|
// selection getContentSha256Cksum makes for verification: the presigned query
|
||||||
|
// value takes precedence over the header, and a repeated header contributes
|
||||||
|
// only its first value. Exposing every raw header value instead let a
|
||||||
|
// request satisfy a policy with a value the verifier never checked.
|
||||||
|
if _, ok := cloneHeader[xhttp.AmzContentSha256]; ok {
|
||||||
|
args[xhttp.AmzContentSha256] = []string{getContentSha256Cksum(r, serviceS3)}
|
||||||
|
cloneHeader.Del(xhttp.AmzContentSha256)
|
||||||
}
|
}
|
||||||
|
|
||||||
userTags := cloneHeader.Get(xhttp.AmzObjectTagging)
|
userTags := cloneHeader.Get(xhttp.AmzObjectTagging)
|
||||||
|
|||||||
@@ -23,14 +23,16 @@ import (
|
|||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"slices"
|
"slices"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/minio/minio/internal/auth"
|
"github.com/minio/minio/internal/auth"
|
||||||
"github.com/minio/minio/internal/handlers"
|
"github.com/minio/minio/internal/handlers"
|
||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
"github.com/minio/pkg/v3/policy/condition"
|
"github.com/pgsty/silo-pkg/v3/policy/condition"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -579,8 +581,20 @@ func TestGetConditionValuesRejectsAbsentInternalKeys(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// s3:signatureAge is derived from the presigned X-Amz-Date, which the signature
|
||||||
|
// binds. A client header under the former scratch name must never supply it on
|
||||||
|
// any auth type, and a presign whose date is missing or malformed leaves the
|
||||||
|
// key absent (the verifier then rejects the request).
|
||||||
func TestGetConditionValuesOnlyAcceptsPresignedSignatureAge(t *testing.T) {
|
func TestGetConditionValuesOnlyAcceptsPresignedSignatureAge(t *testing.T) {
|
||||||
const signatureAgeHeader = "x-amz-signature-age"
|
const signatureAgeHeader = "x-amz-signature-age"
|
||||||
|
signedDate := UTCNow().Add(-90 * time.Second)
|
||||||
|
presignQuery := func(date string) string {
|
||||||
|
q := url.Values{xhttp.AmzCredential: {"access/20260803/us-east-1/s3/aws4_request"}}
|
||||||
|
if date != "" {
|
||||||
|
q.Set(xhttp.AmzDate, date)
|
||||||
|
}
|
||||||
|
return "http://minio.local/bkt/obj?" + q.Encode()
|
||||||
|
}
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
@@ -602,19 +616,33 @@ func TestGetConditionValuesOnlyAcceptsPresignedSignatureAge(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "presigned verifier value",
|
name: "presigned client header without date",
|
||||||
target: "http://minio.local/bkt/obj?" + url.Values{
|
target: presignQuery(""),
|
||||||
xhttp.AmzCredential: {"access/20260803/us-east-1/s3/aws4_request"},
|
headers: map[string]string{signatureAgeHeader: "250"},
|
||||||
}.Encode(),
|
},
|
||||||
|
{
|
||||||
|
name: "presigned malformed date",
|
||||||
|
target: presignQuery("yesterday"),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "presigned signed date",
|
||||||
|
target: presignQuery(signedDate.Format(iso8601Format)),
|
||||||
headers: map[string]string{signatureAgeHeader: "250"},
|
headers: map[string]string{signatureAgeHeader: "250"},
|
||||||
want: true,
|
want: true,
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
got := condValuesForRequest(t, tc.target, tc.headers)
|
got := condValuesForRequest(t, tc.target, tc.headers)
|
||||||
_, ok := got["signatureAge"]
|
v, ok := got["signatureAge"]
|
||||||
if ok != tc.want {
|
if ok != tc.want {
|
||||||
t.Fatalf("signatureAge presence: expected %v, got %v", tc.want, got["signatureAge"])
|
t.Fatalf("signatureAge presence: expected %v, got %v", tc.want, v)
|
||||||
|
}
|
||||||
|
if !tc.want {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
age, err := strconv.ParseInt(strings.Join(v, ""), 10, 64)
|
||||||
|
if err != nil || age < (90*time.Second).Milliseconds() || age > (2*time.Minute).Milliseconds() {
|
||||||
|
t.Fatalf("signatureAge = %v, want about 90s derived from X-Amz-Date rather than the client header", v)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
+12
-8
@@ -43,6 +43,17 @@ func NewBucketQuotaSys() *BucketQuotaSys {
|
|||||||
return &BucketQuotaSys{}
|
return &BucketQuotaSys{}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getBucketQuotaSize returns the effective enforced hard-quota size.
|
||||||
|
func getBucketQuotaSize(quota *madmin.BucketQuota) uint64 {
|
||||||
|
if quota == nil || quota.Type != madmin.HardQuota {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
if quota.Size > 0 {
|
||||||
|
return quota.Size
|
||||||
|
}
|
||||||
|
return quota.Quota
|
||||||
|
}
|
||||||
|
|
||||||
var bucketStorageCache = cachevalue.New[DataUsageInfo]()
|
var bucketStorageCache = cachevalue.New[DataUsageInfo]()
|
||||||
|
|
||||||
// Init initialize bucket quota.
|
// Init initialize bucket quota.
|
||||||
@@ -110,14 +121,7 @@ func (sys *BucketQuotaSys) enforceQuotaHard(ctx context.Context, bucket string,
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
var quotaSize uint64
|
quotaSize := getBucketQuotaSize(q)
|
||||||
if q != nil && q.Type == madmin.HardQuota {
|
|
||||||
if q.Size > 0 {
|
|
||||||
quotaSize = q.Size
|
|
||||||
} else if q.Quota > 0 {
|
|
||||||
quotaSize = q.Quota
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if quotaSize > 0 {
|
if quotaSize > 0 {
|
||||||
if uint64(size) >= quotaSize { // check if file size already exceeds the quota
|
if uint64(size) >= quotaSize { // check if file size already exceeds the quota
|
||||||
return BucketQuotaExceeded{Bucket: bucket}
|
return BucketQuotaExceeded{Bucket: bucket}
|
||||||
|
|||||||
@@ -0,0 +1,77 @@
|
|||||||
|
// Copyright (c) 2015-2025 MinIO, Inc.
|
||||||
|
// Copyright (c) 2025-2026 PGSTY
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/minio/madmin-go/v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGetBucketQuotaSize(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
quota *madmin.BucketQuota
|
||||||
|
want uint64
|
||||||
|
}{
|
||||||
|
{name: "nil"},
|
||||||
|
{name: "empty", quota: &madmin.BucketQuota{}},
|
||||||
|
{name: "current size", quota: &madmin.BucketQuota{Type: madmin.HardQuota, Size: 1024}, want: 1024},
|
||||||
|
{name: "legacy quota", quota: &madmin.BucketQuota{Type: madmin.HardQuota, Quota: 2048}, want: 2048},
|
||||||
|
{name: "size takes precedence", quota: &madmin.BucketQuota{Type: madmin.HardQuota, Size: 1024, Quota: 2048}, want: 1024},
|
||||||
|
{name: "missing type", quota: &madmin.BucketQuota{Size: 1024}},
|
||||||
|
{name: "unsupported type", quota: &madmin.BucketQuota{Type: "fifo", Size: 1024}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if got := getBucketQuotaSize(tt.quota); got != tt.want {
|
||||||
|
t.Fatalf("getBucketQuotaSize() = %d, want %d", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIsBktQuotaCfgReplicated(t *testing.T) {
|
||||||
|
hardQuota := func(size, legacy uint64) *madmin.BucketQuota {
|
||||||
|
return &madmin.BucketQuota{Type: madmin.HardQuota, Size: size, Quota: legacy}
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
quotas []*madmin.BucketQuota
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{name: "none configured", quotas: []*madmin.BucketQuota{nil, nil}, want: true},
|
||||||
|
{name: "missing from one site", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), nil}},
|
||||||
|
{name: "matching size", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), hardQuota(1024, 0)}, want: true},
|
||||||
|
{name: "different size", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), hardQuota(2048, 0)}},
|
||||||
|
{name: "equivalent representations", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), hardQuota(0, 1024)}, want: true},
|
||||||
|
{name: "different typeless size", quotas: []*madmin.BucketQuota{{Size: 1024}, {Size: 2048}}},
|
||||||
|
{name: "different type", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), {Type: "fifo", Size: 1024}}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if got := isBktQuotaCfgReplicated(len(tt.quotas), tt.quotas); got != tt.want {
|
||||||
|
t.Fatalf("isBktQuotaCfgReplicated() = %v, want %v", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -34,7 +34,7 @@ import (
|
|||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
// PutBucketReplicationConfigHandler - PUT Bucket replication configuration.
|
// PutBucketReplicationConfigHandler - PUT Bucket replication configuration.
|
||||||
|
|||||||
@@ -640,10 +640,10 @@ type VersionPurgeStatusType = replication.VersionPurgeStatusType
|
|||||||
|
|
||||||
type replicationResyncer struct {
|
type replicationResyncer struct {
|
||||||
// map of bucket to their resync status
|
// map of bucket to their resync status
|
||||||
statusMap map[string]BucketReplicationResyncStatus
|
statusMap map[string]BucketReplicationResyncStatus
|
||||||
workerSize int
|
workerSize int
|
||||||
resyncCancelCh chan struct{}
|
cancelResyncs map[resyncOpts]context.CancelCauseFunc
|
||||||
workerCh chan struct{}
|
workerCh chan struct{}
|
||||||
sync.RWMutex
|
sync.RWMutex
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+454
-113
@@ -418,7 +418,12 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
|
|||||||
// target cluster, the object version is marked deleted on the source and hidden from listing. It is permanently
|
// target cluster, the object version is marked deleted on the source and hidden from listing. It is permanently
|
||||||
// deleted from the source when the VersionPurgeStatus changes to "Complete", i.e after replication succeeds
|
// deleted from the source when the VersionPurgeStatus changes to "Complete", i.e after replication succeeds
|
||||||
// on target.
|
// on target.
|
||||||
func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer) {
|
// replicateDelete replicates a delete (delete marker or version purge) to all
|
||||||
|
// applicable targets and returns the per-target replication outcome. Callers
|
||||||
|
// that only trigger replication may ignore the return value; the resync path
|
||||||
|
// uses it to classify success/failure per target rather than inferring it from
|
||||||
|
// the mere presence or absence of the target version.
|
||||||
|
func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer) replicatedInfos {
|
||||||
var replicationStatus replication.StatusType
|
var replicationStatus replication.StatusType
|
||||||
bucket := dobj.Bucket
|
bucket := dobj.Bucket
|
||||||
versionID := dobj.DeleteMarkerVersionID
|
versionID := dobj.DeleteMarkerVersionID
|
||||||
@@ -453,7 +458,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
|||||||
Host: globalLocalNodeName,
|
Host: globalLocalNodeName,
|
||||||
EventName: event.ObjectReplicationNotTracked,
|
EventName: event.ObjectReplicationNotTracked,
|
||||||
})
|
})
|
||||||
return
|
return replicatedInfos{}
|
||||||
}
|
}
|
||||||
dsc, err := parseReplicateDecision(ctx, bucket, dobj.ReplicationState.ReplicateDecisionStr)
|
dsc, err := parseReplicateDecision(ctx, bucket, dobj.ReplicationState.ReplicateDecisionStr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -471,7 +476,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
|||||||
Host: globalLocalNodeName,
|
Host: globalLocalNodeName,
|
||||||
EventName: event.ObjectReplicationNotTracked,
|
EventName: event.ObjectReplicationNotTracked,
|
||||||
})
|
})
|
||||||
return
|
return replicatedInfos{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Lock the object name before starting replication operation.
|
// Lock the object name before starting replication operation.
|
||||||
@@ -492,7 +497,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
|||||||
Host: globalLocalNodeName,
|
Host: globalLocalNodeName,
|
||||||
EventName: event.ObjectReplicationNotTracked,
|
EventName: event.ObjectReplicationNotTracked,
|
||||||
})
|
})
|
||||||
return
|
return replicatedInfos{}
|
||||||
}
|
}
|
||||||
ctx = lkctx.Context()
|
ctx = lkctx.Context()
|
||||||
defer lk.Unlock(lkctx)
|
defer lk.Unlock(lkctx)
|
||||||
@@ -597,6 +602,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
|||||||
EventName: eventName,
|
EventName: eventName,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
return rinfos
|
||||||
}
|
}
|
||||||
|
|
||||||
func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationInfo, tgt *TargetClient) (rinfo replicatedTargetInfo) {
|
func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationInfo, tgt *TargetClient) (rinfo replicatedTargetInfo) {
|
||||||
@@ -779,6 +785,15 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (put
|
|||||||
meta := make(map[string]string)
|
meta := make(map[string]string)
|
||||||
isSSEC := crypto.SSEC.IsEncrypted(objInfo.UserDefined)
|
isSSEC := crypto.SSEC.IsEncrypted(objInfo.UserDefined)
|
||||||
|
|
||||||
|
// An SSE-C object is replicated as raw ciphertext, and the replication
|
||||||
|
// headers carry no compression state. Sending a compressed SSE-C object
|
||||||
|
// would land a replica that decrypts to an S2 stream instead of the
|
||||||
|
// object, so fail loudly instead of writing a wrong replica.
|
||||||
|
if isSSEC && objInfo.IsCompressed() {
|
||||||
|
return putOpts, false, fmt.Errorf("replication of a compressed SSE-C object is not supported: %s/%s(%s)",
|
||||||
|
objInfo.Bucket, objInfo.Name, objInfo.VersionID)
|
||||||
|
}
|
||||||
|
|
||||||
for k, v := range objInfo.UserDefined {
|
for k, v := range objInfo.UserDefined {
|
||||||
_, isValidSSEHeader := validSSEReplicationHeaders[k]
|
_, isValidSSEHeader := validSSEReplicationHeaders[k]
|
||||||
// In case of SSE-C objects copy the allowed internal headers as well
|
// In case of SSE-C objects copy the allowed internal headers as well
|
||||||
@@ -857,19 +872,29 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (put
|
|||||||
if cc, ok := lkMap.Lookup(xhttp.CacheControl); ok {
|
if cc, ok := lkMap.Lookup(xhttp.CacheControl); ok {
|
||||||
putOpts.CacheControl = cc
|
putOpts.CacheControl = cc
|
||||||
}
|
}
|
||||||
if mode, ok := lkMap.Lookup(xhttp.AmzObjectLockMode); ok {
|
mode, hasMode := lkMap.Lookup(xhttp.AmzObjectLockMode)
|
||||||
rmode := minio.RetentionMode(mode)
|
retainDateStr, hasRetainDate := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate)
|
||||||
putOpts.Mode = rmode
|
if hasMode {
|
||||||
|
putOpts.Mode = minio.RetentionMode(mode)
|
||||||
}
|
}
|
||||||
if retainDateStr, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok {
|
// A removed retention is stored as an empty or absent mode and date; it is
|
||||||
|
// sent as a value-less update that still carries its ordering timestamp.
|
||||||
|
if hasRetainDate && retainDateStr != "" {
|
||||||
rdate, err := amztime.ISO8601Parse(retainDateStr)
|
rdate, err := amztime.ISO8601Parse(retainDateStr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return putOpts, false, err
|
return putOpts, false, err
|
||||||
}
|
}
|
||||||
putOpts.RetainUntilDate = rdate
|
putOpts.RetainUntilDate = rdate
|
||||||
// set retention timestamp in opts
|
}
|
||||||
|
retainTmstampStr, hasRetainTmstamp := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp]
|
||||||
|
if hasMode || hasRetainDate || hasRetainTmstamp {
|
||||||
|
// Send the ordering timestamp whenever the version carries one, even for a
|
||||||
|
// removal whose value keys are absent (the shape a retransmit PUT leaves),
|
||||||
|
// so the next hop can order the removal instead of keeping obsolete
|
||||||
|
// retention.
|
||||||
retTimestamp := objInfo.ModTime
|
retTimestamp := objInfo.ModTime
|
||||||
if retainTmstampStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp]; ok {
|
if hasRetainTmstamp {
|
||||||
|
var err error
|
||||||
retTimestamp, err = time.Parse(time.RFC3339Nano, retainTmstampStr)
|
retTimestamp, err = time.Parse(time.RFC3339Nano, retainTmstampStr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return putOpts, false, err
|
return putOpts, false, err
|
||||||
@@ -931,11 +956,17 @@ func equals(k1 string, keys ...string) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// nullVersionExcludedFromResync reports the exclusion at the head of getReplicationAction, kept
|
||||||
|
// verbatim from upstream: an existing object resync leaves a null version alone when the source
|
||||||
|
// modification time is later than the one the target reports, without comparing anything else.
|
||||||
|
func nullVersionExcludedFromResync(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replication.Type) bool {
|
||||||
|
return opType == replication.ExistingObjectReplicationType &&
|
||||||
|
oi1.ModTime.Unix() > oi2.LastModified.Unix() && oi1.VersionID == nullVersionID
|
||||||
|
}
|
||||||
|
|
||||||
// returns replicationAction by comparing metadata between source and target
|
// returns replicationAction by comparing metadata between source and target
|
||||||
func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replication.Type) replicationAction {
|
func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replication.Type) replicationAction {
|
||||||
// Avoid resyncing null versions created prior to enabling replication if target has a newer copy
|
if nullVersionExcludedFromResync(oi1, oi2, opType) {
|
||||||
if opType == replication.ExistingObjectReplicationType &&
|
|
||||||
oi1.ModTime.Unix() > oi2.LastModified.Unix() && oi1.VersionID == nullVersionID {
|
|
||||||
return replicateNone
|
return replicateNone
|
||||||
}
|
}
|
||||||
sz, _ := oi1.GetActualSize()
|
sz, _ := oi1.GetActualSize()
|
||||||
@@ -986,9 +1017,21 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
|
|||||||
"X-Amz-Meta-",
|
"X-Amz-Meta-",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// An empty object lock mode or retain-until-date records a removed retention, but
|
||||||
|
// it is omitted from GET/HEAD response headers: setObjectHeaders() skips both keys
|
||||||
|
// when the value is empty, and FilterObjectLockMetadata() drops them when the mode
|
||||||
|
// is not valid. The target can therefore never report them, so treat empty and
|
||||||
|
// absent as equal rather than as a permanent difference.
|
||||||
|
emptyLockValue := func(k, v string) bool {
|
||||||
|
return v == "" && equals(k, xhttp.AmzObjectLockMode, xhttp.AmzObjectLockRetainUntilDate)
|
||||||
|
}
|
||||||
|
|
||||||
// compare metadata on both maps to see if meta is identical
|
// compare metadata on both maps to see if meta is identical
|
||||||
compareMeta1 := make(map[string]string)
|
compareMeta1 := make(map[string]string)
|
||||||
for k, v := range oi1.UserDefined {
|
for k, v := range oi1.UserDefined {
|
||||||
|
if emptyLockValue(k, v) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
var found bool
|
var found bool
|
||||||
for _, prefix := range compareKeys {
|
for _, prefix := range compareKeys {
|
||||||
if !stringsHasPrefixFold(k, prefix) {
|
if !stringsHasPrefixFold(k, prefix) {
|
||||||
@@ -1004,6 +1047,10 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
|
|||||||
|
|
||||||
compareMeta2 := make(map[string]string)
|
compareMeta2 := make(map[string]string)
|
||||||
for k, v := range oi2.Metadata {
|
for k, v := range oi2.Metadata {
|
||||||
|
val := strings.Join(v, ",")
|
||||||
|
if emptyLockValue(k, val) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
var found bool
|
var found bool
|
||||||
for _, prefix := range compareKeys {
|
for _, prefix := range compareKeys {
|
||||||
if !stringsHasPrefixFold(k, prefix) {
|
if !stringsHasPrefixFold(k, prefix) {
|
||||||
@@ -1013,7 +1060,7 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
|
|||||||
break
|
break
|
||||||
}
|
}
|
||||||
if found {
|
if found {
|
||||||
compareMeta2[strings.ToLower(k)] = strings.Join(v, ",")
|
compareMeta2[strings.ToLower(k)] = val
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1024,9 +1071,87 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
|
|||||||
return replicateNone
|
return replicateNone
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// objectRetentionGetter is the part of the replication target client used to confirm whether a
|
||||||
|
// destination version still holds Object Lock retention.
|
||||||
|
type objectRetentionGetter interface {
|
||||||
|
GetObjectRetention(ctx context.Context, bucketName, objectName, versionID string) (*minio.RetentionMode, *time.Time, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// retentionRemovedAtSource reports whether oi carries the shape a removed retention leaves behind.
|
||||||
|
// Two representations persist. A retention removed directly on this cluster keeps the object lock
|
||||||
|
// keys present with empty values (PutObjectRetentionHandler, cmd/object-handlers.go:3309-3316). A
|
||||||
|
// removal that arrived by replication keeps only the retention ordering timestamp, with the mode
|
||||||
|
// and retain-until-date keys absent, because restoreRetention and the replica update path write
|
||||||
|
// the timestamp alone when the mode is empty (cmd/bucket-object-lock.go:388-399,
|
||||||
|
// cmd/object-handlers.go:1782-1797). A present ordering timestamp paired with a non-empty mode is
|
||||||
|
// a retention that was set, not removed, and must not be mistaken for one.
|
||||||
|
func retentionRemovedAtSource(oi ObjectInfo) bool {
|
||||||
|
lkMap := caseInsensitiveMap(oi.UserDefined)
|
||||||
|
// Representation (1): an object lock key is present with an empty value.
|
||||||
|
for _, k := range []string{xhttp.AmzObjectLockMode, xhttp.AmzObjectLockRetainUntilDate} {
|
||||||
|
if v, ok := lkMap.Lookup(k); ok && v == "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Representation (2): a recorded retention ordering timestamp with the mode value absent or
|
||||||
|
// empty is a removal restoreRetention persisted without the empty public keys.
|
||||||
|
if _, ok := oi.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp]; ok {
|
||||||
|
if v, ok := lkMap.Lookup(xhttp.AmzObjectLockMode); !ok || v == "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// targetRetentionConfirmedAbsent reports whether the destination version is known to hold no
|
||||||
|
// retention. A HEAD response omits retention both when the version has none and when the
|
||||||
|
// replication credential lacks s3:GetObjectRetention (cmd/object-handlers.go:942-946), so the
|
||||||
|
// comparison in getReplicationAction on its own cannot tell a removal that is already in sync from
|
||||||
|
// one the destination still holds. Only NoSuchObjectLockConfiguration, the answer for a version
|
||||||
|
// that carries no retention, and a response naming no retention mode count as absent. Everything
|
||||||
|
// else is uncertainty and is treated as still present, so that the removal is resent exactly as it
|
||||||
|
// is today: a denied or unreachable destination, a mode the SDK returned without recognizing since
|
||||||
|
// it does not validate it, and InvalidRequest, which names a bucket with no Object Lock
|
||||||
|
// configuration but is also what a destination answers when its own read of that configuration
|
||||||
|
// fails (cmd/bucket-object-lock.go:39-50 returns an error with a zero Retention, discarded at
|
||||||
|
// cmd/object-handlers.go:3275).
|
||||||
|
func targetRetentionConfirmedAbsent(ctx context.Context, tgt objectRetentionGetter, bucket, object, versionID string) bool {
|
||||||
|
mode, _, err := tgt.GetObjectRetention(ctx, bucket, object, versionID)
|
||||||
|
if err != nil {
|
||||||
|
return minio.ToErrorResponse(err).Code == "NoSuchObjectLockConfiguration"
|
||||||
|
}
|
||||||
|
// An absent or empty mode is no retention. A non-empty mode is retention, whether or not this
|
||||||
|
// SDK recognizes it.
|
||||||
|
return mode == nil || *mode == ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// replicationActionForTarget returns the action for a source version against a destination that
|
||||||
|
// answered HEAD. It is getReplicationAction plus the confirmation that a removed retention which
|
||||||
|
// compares as in sync really is: see targetRetentionConfirmedAbsent.
|
||||||
|
func replicationActionForTarget(ctx context.Context, oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replication.Type, tgt objectRetentionGetter, bucket, object string) replicationAction {
|
||||||
|
rAction := getReplicationAction(oi1, oi2, opType)
|
||||||
|
if rAction != replicateNone || !retentionRemovedAtSource(oi1) {
|
||||||
|
return rAction
|
||||||
|
}
|
||||||
|
// A null version the resync deliberately leaves alone is not a comparison result, so it is
|
||||||
|
// not the confirmation's to reopen.
|
||||||
|
if nullVersionExcludedFromResync(oi1, oi2, opType) {
|
||||||
|
return rAction
|
||||||
|
}
|
||||||
|
if targetRetentionConfirmedAbsent(ctx, tgt, bucket, object, oi1.VersionID) {
|
||||||
|
return rAction
|
||||||
|
}
|
||||||
|
return replicateMetadata
|
||||||
|
}
|
||||||
|
|
||||||
// replicateObject replicates the specified version of the object to destination bucket
|
// replicateObject replicates the specified version of the object to destination bucket
|
||||||
// The source object is then updated to reflect the replication status.
|
// The source object is then updated to reflect the replication status.
|
||||||
func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI ObjectLayer) {
|
// replicateObject replicates a single object version to all applicable targets
|
||||||
|
// and returns the per-target replication outcome. Callers that only trigger
|
||||||
|
// replication may ignore the return value; the resync path uses it to classify
|
||||||
|
// success/failure per target rather than inferring it from the mere existence
|
||||||
|
// of the target version.
|
||||||
|
func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI ObjectLayer) replicatedInfos {
|
||||||
var replicationStatus replication.StatusType
|
var replicationStatus replication.StatusType
|
||||||
defer func() {
|
defer func() {
|
||||||
if replicationStatus.Empty() {
|
if replicationStatus.Empty() {
|
||||||
@@ -1059,7 +1184,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
|
|||||||
UserAgent: "Internal: [Replication]",
|
UserAgent: "Internal: [Replication]",
|
||||||
Host: globalLocalNodeName,
|
Host: globalLocalNodeName,
|
||||||
})
|
})
|
||||||
return
|
return replicatedInfos{}
|
||||||
}
|
}
|
||||||
tgtArns := cfg.FilterTargetArns(replication.ObjectOpts{
|
tgtArns := cfg.FilterTargetArns(replication.ObjectOpts{
|
||||||
Name: object,
|
Name: object,
|
||||||
@@ -1079,7 +1204,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
|
|||||||
Host: globalLocalNodeName,
|
Host: globalLocalNodeName,
|
||||||
})
|
})
|
||||||
globalReplicationPool.Get().queueMRFSave(ri.ToMRFEntry())
|
globalReplicationPool.Get().queueMRFSave(ri.ToMRFEntry())
|
||||||
return
|
return replicatedInfos{}
|
||||||
}
|
}
|
||||||
ctx = lkctx.Context()
|
ctx = lkctx.Context()
|
||||||
defer lk.Unlock(lkctx)
|
defer lk.Unlock(lkctx)
|
||||||
@@ -1185,6 +1310,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
|
|||||||
ri.RetryCount++
|
ri.RetryCount++
|
||||||
globalReplicationPool.Get().queueMRFSave(ri.ToMRFEntry())
|
globalReplicationPool.Get().queueMRFSave(ri.ToMRFEntry())
|
||||||
}
|
}
|
||||||
|
return rinfos
|
||||||
}
|
}
|
||||||
|
|
||||||
// replicateObject replicates object data for specified version of the object to destination bucket
|
// replicateObject replicates object data for specified version of the object to destination bucket
|
||||||
@@ -1299,6 +1425,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
|
|||||||
|
|
||||||
putOpts, isMP, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
|
putOpts, isMP, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
rinfo.Err = err
|
||||||
|
rinfo.ReplicationStatus = replication.Failed
|
||||||
replLogIf(ctx, fmt.Errorf("failure setting options for replication bucket:%s err:%w", bucket, err))
|
replLogIf(ctx, fmt.Errorf("failure setting options for replication bucket:%s err:%w", bucket, err))
|
||||||
sendEvent(eventArgs{
|
sendEvent(eventArgs{
|
||||||
EventName: event.ObjectReplicationNotTracked,
|
EventName: event.ObjectReplicationNotTracked,
|
||||||
@@ -1467,7 +1595,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
|
|||||||
sOpts.Set(xhttp.AmzTagDirective, "ACCESS")
|
sOpts.Set(xhttp.AmzTagDirective, "ACCESS")
|
||||||
oi, cerr := tgt.StatObject(ctx, tgt.Bucket, object, sOpts)
|
oi, cerr := tgt.StatObject(ctx, tgt.Bucket, object, sOpts)
|
||||||
if cerr == nil {
|
if cerr == nil {
|
||||||
rAction = getReplicationAction(objInfo, oi, ri.OpType)
|
rAction = replicationActionForTarget(ctx, objInfo, oi, ri.OpType, tgt, tgt.Bucket, object)
|
||||||
rinfo.ReplicationStatus = replication.Completed
|
rinfo.ReplicationStatus = replication.Completed
|
||||||
if rAction == replicateNone {
|
if rAction == replicateNone {
|
||||||
if ri.OpType == replication.ExistingObjectReplicationType &&
|
if ri.OpType == replication.ExistingObjectReplicationType &&
|
||||||
@@ -1497,11 +1625,14 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
|
|||||||
return rinfo
|
return rinfo
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// SSEC objects will refuse HeadObject without the decryption key.
|
// The sender holds no customer key, so the target refuses HeadObject on
|
||||||
// Ignore the error, since we know the object exists and versioning prevents overwriting existing versions.
|
// an SSE-C object and the replica cannot be compared. The metadata-only
|
||||||
|
// CopyObject that a replicateMetadata action would run then fails on any
|
||||||
|
// non-empty object, because the undecryptable source checksum makes the
|
||||||
|
// target recompute one and rewrite the data. A full retransmit is the
|
||||||
|
// only action that completes.
|
||||||
if isSSEC && strings.Contains(cerr.Error(), errorCodes[ErrSSEEncryptedObject].Description) {
|
if isSSEC && strings.Contains(cerr.Error(), errorCodes[ErrSSEEncryptedObject].Description) {
|
||||||
rinfo.ReplicationStatus = replication.Completed
|
rAction = replicateAll
|
||||||
rinfo.ReplicationAction = replicateNone
|
|
||||||
goto applyAction
|
goto applyAction
|
||||||
}
|
}
|
||||||
// if target returns error other than NoSuchKey, defer replication attempt
|
// if target returns error other than NoSuchKey, defer replication attempt
|
||||||
@@ -1586,6 +1717,11 @@ applyAction:
|
|||||||
} else {
|
} else {
|
||||||
putOpts, isMP, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
|
putOpts, isMP, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
// rinfo was primed Completed above; a failure to build the write
|
||||||
|
// options means nothing reached the target, so mark it Failed and
|
||||||
|
// carry the error instead of reporting a phantom success.
|
||||||
|
rinfo.ReplicationStatus = replication.Failed
|
||||||
|
rinfo.Err = err
|
||||||
replLogIf(ctx, fmt.Errorf("failed to set replicate options for object %s/%s(%s) (target %s) err:%w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), err))
|
replLogIf(ctx, fmt.Errorf("failed to set replicate options for object %s/%s(%s) (target %s) err:%w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), err))
|
||||||
sendEvent(eventArgs{
|
sendEvent(eventArgs{
|
||||||
EventName: event.ObjectReplicationNotTracked,
|
EventName: event.ObjectReplicationNotTracked,
|
||||||
@@ -2218,9 +2354,9 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
|
|||||||
switch ri.OpType {
|
switch ri.OpType {
|
||||||
case replication.HealReplicationType, replication.ExistingObjectReplicationType:
|
case replication.HealReplicationType, replication.ExistingObjectReplicationType:
|
||||||
ch = p.mrfReplicaCh
|
ch = p.mrfReplicaCh
|
||||||
healCh = p.getWorkerCh(ri.Name, ri.Bucket, ri.Size)
|
healCh = p.getWorkerCh(ri.Bucket, ri.Name, ri.Size)
|
||||||
default:
|
default:
|
||||||
ch = p.getWorkerCh(ri.Name, ri.Bucket, ri.Size)
|
ch = p.getWorkerCh(ri.Bucket, ri.Name, ri.Size)
|
||||||
}
|
}
|
||||||
if ch == nil && healCh == nil {
|
if ch == nil && healCh == nil {
|
||||||
return
|
return
|
||||||
@@ -2828,10 +2964,10 @@ const (
|
|||||||
|
|
||||||
func newresyncer() *replicationResyncer {
|
func newresyncer() *replicationResyncer {
|
||||||
rs := replicationResyncer{
|
rs := replicationResyncer{
|
||||||
statusMap: make(map[string]BucketReplicationResyncStatus),
|
statusMap: make(map[string]BucketReplicationResyncStatus),
|
||||||
workerSize: resyncWorkerCnt,
|
workerSize: resyncWorkerCnt,
|
||||||
resyncCancelCh: make(chan struct{}, resyncWorkerCnt),
|
cancelResyncs: make(map[resyncOpts]context.CancelCauseFunc),
|
||||||
workerCh: make(chan struct{}, resyncWorkerCnt),
|
workerCh: make(chan struct{}, resyncWorkerCnt),
|
||||||
}
|
}
|
||||||
for i := 0; i < rs.workerSize; i++ {
|
for i := 0; i < rs.workerSize; i++ {
|
||||||
rs.workerCh <- struct{}{}
|
rs.workerCh <- struct{}{}
|
||||||
@@ -2839,13 +2975,67 @@ func newresyncer() *replicationResyncer {
|
|||||||
return &rs
|
return &rs
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var errResyncCanceled = errors.New("replication resync canceled")
|
||||||
|
|
||||||
|
// Registration and cancellation share the status lock. A queued run therefore
|
||||||
|
// cannot miss cancellation between publishing its status and taking a slot.
|
||||||
|
func (s *replicationResyncer) registerResync(parent context.Context, opts resyncOpts) (context.Context, context.CancelCauseFunc, bool) {
|
||||||
|
s.Lock()
|
||||||
|
defer s.Unlock()
|
||||||
|
st, ok := s.statusMap[opts.bucket].TargetsMap[opts.arn]
|
||||||
|
if !ok || st.ResyncID != opts.resyncID {
|
||||||
|
return nil, nil, false
|
||||||
|
}
|
||||||
|
if _, running := s.cancelResyncs[opts]; running {
|
||||||
|
return nil, nil, false
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancelCause(parent)
|
||||||
|
if s.cancelResyncs == nil {
|
||||||
|
s.cancelResyncs = make(map[resyncOpts]context.CancelCauseFunc)
|
||||||
|
}
|
||||||
|
s.cancelResyncs[opts] = cancel
|
||||||
|
if st.ResyncStatus == ResyncCanceled {
|
||||||
|
cancel(errResyncCanceled)
|
||||||
|
}
|
||||||
|
return ctx, cancel, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *replicationResyncer) cancelResyncID(resyncID string) {
|
||||||
|
s.Lock()
|
||||||
|
defer s.Unlock()
|
||||||
|
for bucket, m := range s.statusMap {
|
||||||
|
for arn, st := range m.TargetsMap {
|
||||||
|
if st.ResyncID == resyncID && (st.ResyncStatus == ResyncPending || st.ResyncStatus == ResyncStarted) {
|
||||||
|
st.ResyncStatus = ResyncCanceled
|
||||||
|
st.LastUpdate = UTCNow()
|
||||||
|
m.TargetsMap[arn] = st
|
||||||
|
m.LastUpdate = st.LastUpdate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s.statusMap[bucket] = m
|
||||||
|
}
|
||||||
|
for opts, cancel := range s.cancelResyncs {
|
||||||
|
if opts.resyncID == resyncID {
|
||||||
|
cancel(errResyncCanceled)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// mark status of replication resync on remote target for the bucket
|
// mark status of replication resync on remote target for the bucket
|
||||||
func (s *replicationResyncer) markStatus(status ResyncStatusType, opts resyncOpts, objAPI ObjectLayer) {
|
func (s *replicationResyncer) markStatus(status ResyncStatusType, opts resyncOpts, objAPI ObjectLayer) ResyncStatusType {
|
||||||
s.Lock()
|
s.Lock()
|
||||||
defer s.Unlock()
|
defer s.Unlock()
|
||||||
|
|
||||||
m := s.statusMap[opts.bucket]
|
m := s.statusMap[opts.bucket]
|
||||||
st := m.TargetsMap[opts.arn]
|
st, ok := m.TargetsMap[opts.arn]
|
||||||
|
if !ok || st.ResyncID != opts.resyncID {
|
||||||
|
return NoResync
|
||||||
|
}
|
||||||
|
// A cancel may win the lock after the finalizer checked its context.
|
||||||
|
// Persist that cancellation, never a stale Started/Completed result.
|
||||||
|
if st.ResyncStatus == ResyncCanceled {
|
||||||
|
status = ResyncCanceled
|
||||||
|
}
|
||||||
st.LastUpdate = UTCNow()
|
st.LastUpdate = UTCNow()
|
||||||
st.ResyncStatus = status
|
st.ResyncStatus = status
|
||||||
m.TargetsMap[opts.arn] = st
|
m.TargetsMap[opts.arn] = st
|
||||||
@@ -2855,14 +3045,18 @@ func (s *replicationResyncer) markStatus(status ResyncStatusType, opts resyncOpt
|
|||||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
saveResyncStatus(ctx, opts.bucket, m, objAPI)
|
saveResyncStatus(ctx, opts.bucket, m, objAPI)
|
||||||
|
return status
|
||||||
}
|
}
|
||||||
|
|
||||||
// update replication resync stats for bucket's remote target
|
// update replication resync stats for bucket's remote target
|
||||||
func (s *replicationResyncer) incStats(ts TargetReplicationResyncStatus, opts resyncOpts) {
|
func (s *replicationResyncer) incStats(ts TargetReplicationResyncStatus, opts resyncOpts) bool {
|
||||||
s.Lock()
|
s.Lock()
|
||||||
defer s.Unlock()
|
defer s.Unlock()
|
||||||
m := s.statusMap[opts.bucket]
|
m := s.statusMap[opts.bucket]
|
||||||
st := m.TargetsMap[opts.arn]
|
st, ok := m.TargetsMap[opts.arn]
|
||||||
|
if !ok || st.ResyncID != opts.resyncID || st.ResyncStatus == ResyncCanceled {
|
||||||
|
return false
|
||||||
|
}
|
||||||
st.Object = ts.Object
|
st.Object = ts.Object
|
||||||
st.ReplicatedCount += ts.ReplicatedCount
|
st.ReplicatedCount += ts.ReplicatedCount
|
||||||
st.FailedCount += ts.FailedCount
|
st.FailedCount += ts.FailedCount
|
||||||
@@ -2871,20 +3065,175 @@ func (s *replicationResyncer) incStats(ts TargetReplicationResyncStatus, opts re
|
|||||||
m.TargetsMap[opts.arn] = st
|
m.TargetsMap[opts.arn] = st
|
||||||
m.LastUpdate = UTCNow()
|
m.LastUpdate = UTCNow()
|
||||||
s.statusMap[opts.bucket] = m
|
s.statusMap[opts.bucket] = m
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// resyncResults consumes the per-object outcomes produced by the resync worker
|
||||||
|
// pool and applies each to the in-memory resync status via apply. It centralizes
|
||||||
|
// the finalization ordering so a status persisted after finish() returns always
|
||||||
|
// reflects every result.
|
||||||
|
type resyncResults struct {
|
||||||
|
ch chan TargetReplicationResyncStatus
|
||||||
|
apply func(TargetReplicationResyncStatus)
|
||||||
|
wg sync.WaitGroup
|
||||||
|
}
|
||||||
|
|
||||||
|
// newResyncResults starts the result-consuming goroutine that folds each worker
|
||||||
|
// result into the bucket's resync status.
|
||||||
|
func (s *replicationResyncer) newResyncResults(opts resyncOpts) *resyncResults {
|
||||||
|
return startResyncResults(func(r TargetReplicationResyncStatus) {
|
||||||
|
if s.incStats(r, opts) {
|
||||||
|
globalSiteResyncMetrics.updateMetric(r, opts.resyncID)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// startResyncResults starts a goroutine that applies every received result with
|
||||||
|
// apply. Injecting the apply action keeps the shutdown ordering in finish()
|
||||||
|
// testable.
|
||||||
|
func startResyncResults(apply func(TargetReplicationResyncStatus)) *resyncResults {
|
||||||
|
rr := &resyncResults{
|
||||||
|
ch: make(chan TargetReplicationResyncStatus, 1),
|
||||||
|
apply: apply,
|
||||||
|
}
|
||||||
|
rr.wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer rr.wg.Done()
|
||||||
|
for r := range rr.ch {
|
||||||
|
rr.apply(r)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return rr
|
||||||
|
}
|
||||||
|
|
||||||
|
// finish shuts the resync pipeline down in an order that guarantees a status
|
||||||
|
// persisted afterwards reflects every result. It first closes the worker input
|
||||||
|
// channels and waits for the producer workers to exit, so none can send on a
|
||||||
|
// closed result channel (a hazard on early-return paths) and every submitted
|
||||||
|
// result is delivered (a result a worker discards on cancellation is
|
||||||
|
// intentionally not); only then does it close the result channel and wait for
|
||||||
|
// the consumer to apply the last buffered result.
|
||||||
|
func (rr *resyncResults) finish(workers []chan ReplicateObjectInfo, workerWg *sync.WaitGroup) {
|
||||||
|
for i := range workers {
|
||||||
|
xioutil.SafeClose(workers[i])
|
||||||
|
}
|
||||||
|
workerWg.Wait()
|
||||||
|
xioutil.SafeClose(rr.ch)
|
||||||
|
rr.wg.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
|
// sendResyncResult delivers a worker's computed per-object result to ch,
|
||||||
|
// returning false if the run's context was canceled first.
|
||||||
|
func (s *replicationResyncer) sendResyncResult(ctx context.Context, ch chan<- TargetReplicationResyncStatus, st TargetReplicationResyncStatus) bool {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return false
|
||||||
|
case ch <- st:
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// User cancellation is terminal; interrupted completion remains retryable.
|
||||||
|
func finalResyncStatus(status ResyncStatusType, cause error) ResyncStatusType {
|
||||||
|
if errors.Is(cause, errResyncCanceled) {
|
||||||
|
return ResyncCanceled
|
||||||
|
}
|
||||||
|
if status == ResyncCompleted && cause != nil {
|
||||||
|
return ResyncFailed
|
||||||
|
}
|
||||||
|
return status
|
||||||
|
}
|
||||||
|
|
||||||
|
// resyncTargetSucceeded reports whether this object (or delete) actually
|
||||||
|
// replicated to the target, from the target's own outcome. A version purge
|
||||||
|
// reports success through VersionPurgeStatus, not ReplicationStatus. For an
|
||||||
|
// object or delete marker, success requires a Completed status; a retained
|
||||||
|
// error is a real failure unless it is the benign duplicate 412 the
|
||||||
|
// destination returns when it already holds this exact ETag and version, which
|
||||||
|
// replicateAll deliberately keeps Completed.
|
||||||
|
func resyncTargetSucceeded(t replicatedTargetInfo, roi ReplicateObjectInfo) bool {
|
||||||
|
if !roi.VersionPurgeStatus.Empty() {
|
||||||
|
return t.VersionPurgeStatus == replication.VersionPurgeComplete
|
||||||
|
}
|
||||||
|
if t.ReplicationStatus != replication.Completed {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return t.Err == nil || minio.ToErrorResponse(t.Err).Code == "PreconditionFailed"
|
||||||
|
}
|
||||||
|
|
||||||
|
// resyncResultFor derives the resync outcome for target arn from the aggregate
|
||||||
|
// replication result of a single object (or delete). The target counts as a
|
||||||
|
// success only when its own replication Completed without error - not when the
|
||||||
|
// target version merely exists. A target that Failed, errored, or was not
|
||||||
|
// attempted for this object (its arn absent from the result) counts as a
|
||||||
|
// failure, and the failed byte count is recorded (previously always zero).
|
||||||
|
func resyncResultFor(rinfos replicatedInfos, arn string, roi ReplicateObjectInfo) TargetReplicationResyncStatus {
|
||||||
|
st := TargetReplicationResyncStatus{Object: roi.Name, Bucket: roi.Bucket}
|
||||||
|
for _, t := range rinfos.Targets {
|
||||||
|
if t.Arn != arn {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if resyncTargetSucceeded(t, roi) {
|
||||||
|
sz := t.Size
|
||||||
|
if sz == 0 {
|
||||||
|
sz = roi.Size
|
||||||
|
}
|
||||||
|
st.ReplicatedCount++
|
||||||
|
st.ReplicatedSize += sz
|
||||||
|
} else {
|
||||||
|
st.FailedCount++
|
||||||
|
st.FailedSize += roi.Size
|
||||||
|
}
|
||||||
|
return st
|
||||||
|
}
|
||||||
|
// arn was not attempted for this object: a resync that cannot confirm the
|
||||||
|
// object reached the target is not a success.
|
||||||
|
st.FailedCount++
|
||||||
|
st.FailedSize += roi.Size
|
||||||
|
return st
|
||||||
|
}
|
||||||
|
|
||||||
|
// objectNeedsResyncForARN reports whether roi must be resynced for target arn
|
||||||
|
// specifically. The resync worker pool is scoped to a single target (opts.arn),
|
||||||
|
// so an object that only qualifies for a different target must be skipped here:
|
||||||
|
// admitting it would replicate it for arn's peers only, leaving arn absent from
|
||||||
|
// the per-object result, which resyncResultFor then (correctly, but
|
||||||
|
// misleadingly) counts as a failure for arn - an object arn was never
|
||||||
|
// responsible for. Only opts.arn carries this resync's ResetID, and that reset
|
||||||
|
// is already folded into its per-target decision, so the per-target check both
|
||||||
|
// scopes dispatch and honors the reset.
|
||||||
|
func objectNeedsResyncForARN(roi ReplicateObjectInfo, arn string) bool {
|
||||||
|
return roi.ExistingObjResync.mustResyncTarget(arn)
|
||||||
}
|
}
|
||||||
|
|
||||||
// resyncBucket resyncs all qualifying objects as per replication rules for the target
|
// resyncBucket resyncs all qualifying objects as per replication rules for the target
|
||||||
// ARN
|
// ARN
|
||||||
func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI ObjectLayer, heal bool, opts resyncOpts) {
|
func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI ObjectLayer, heal bool, opts resyncOpts) {
|
||||||
|
ctx, cancel, registered := s.registerResync(ctx, opts)
|
||||||
|
if !registered {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
cancel(nil)
|
||||||
|
s.Lock()
|
||||||
|
delete(s.cancelResyncs, opts)
|
||||||
|
s.Unlock()
|
||||||
|
}()
|
||||||
select {
|
select {
|
||||||
case <-s.workerCh: // block till a worker is available
|
case <-s.workerCh: // block till a worker is available
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
|
if errors.Is(context.Cause(ctx), errResyncCanceled) {
|
||||||
|
status := s.markStatus(ResyncCanceled, opts, objectAPI)
|
||||||
|
globalSiteResyncMetrics.incBucket(opts, status)
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
resyncStatus := ResyncFailed
|
resyncStatus := ResyncFailed
|
||||||
defer func() {
|
defer func() {
|
||||||
s.markStatus(resyncStatus, opts, objectAPI)
|
// Runs after workers/results drain and before our own deferred cancel.
|
||||||
|
resyncStatus = finalResyncStatus(resyncStatus, context.Cause(ctx))
|
||||||
|
resyncStatus = s.markStatus(resyncStatus, opts, objectAPI)
|
||||||
globalSiteResyncMetrics.incBucket(opts, resyncStatus)
|
globalSiteResyncMetrics.incBucket(opts, resyncStatus)
|
||||||
s.workerCh <- struct{}{}
|
s.workerCh <- struct{}{}
|
||||||
}()
|
}()
|
||||||
@@ -2919,8 +3268,8 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
// mark resync status as resync started
|
// mark resync status as resync started
|
||||||
if !heal {
|
if !heal && s.markStatus(ResyncStarted, opts, objectAPI) != ResyncStarted {
|
||||||
s.markStatus(ResyncStarted, opts, objectAPI)
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Walk through all object versions - Walk() is always in ascending order needed to ensure
|
// Walk through all object versions - Walk() is always in ascending order needed to ensure
|
||||||
@@ -2939,16 +3288,21 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
|||||||
lastCheckpoint = st.Object
|
lastCheckpoint = st.Object
|
||||||
}
|
}
|
||||||
workers := make([]chan ReplicateObjectInfo, resyncParallelRoutines)
|
workers := make([]chan ReplicateObjectInfo, resyncParallelRoutines)
|
||||||
resultCh := make(chan TargetReplicationResyncStatus, 1)
|
|
||||||
defer xioutil.SafeClose(resultCh)
|
|
||||||
go func() {
|
|
||||||
for r := range resultCh {
|
|
||||||
s.incStats(r, opts)
|
|
||||||
globalSiteResyncMetrics.updateMetric(r, opts.resyncID)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
|
// results consumes each worker's per-object outcome and folds it into the
|
||||||
|
// in-memory status. finish() (deferred below) stops the workers and drains
|
||||||
|
// every result before the deferred markStatus persists, so a Completed status
|
||||||
|
// cannot race the last incStats. Registered after the markStatus finalizer, so
|
||||||
|
// LIFO runs finish first.
|
||||||
|
results := s.newResyncResults(opts)
|
||||||
|
defer func() {
|
||||||
|
if resyncStatus != ResyncCompleted {
|
||||||
|
cancel(nil)
|
||||||
|
}
|
||||||
|
// Exactly one finish: success drains with a live context; errors stop
|
||||||
|
// blocked workers first. Both drain counts before persisting status.
|
||||||
|
results.finish(workers, &wg)
|
||||||
|
}()
|
||||||
for i := range resyncParallelRoutines {
|
for i := range resyncParallelRoutines {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
workers[i] = make(chan ReplicateObjectInfo, 100)
|
workers[i] = make(chan ReplicateObjectInfo, 100)
|
||||||
@@ -2959,10 +3313,10 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
|||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
case <-s.resyncCancelCh:
|
|
||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
traceFn := s.trace(tgt.ResetID, fmt.Sprintf("%s/%s (%s)", opts.bucket, roi.Name, roi.VersionID))
|
traceFn := s.trace(tgt.ResetID, fmt.Sprintf("%s/%s (%s)", opts.bucket, roi.Name, roi.VersionID))
|
||||||
|
var rinfos replicatedInfos
|
||||||
if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
|
if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
|
||||||
versionID := ""
|
versionID := ""
|
||||||
dmVersionID := ""
|
dmVersionID := ""
|
||||||
@@ -2985,83 +3339,69 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
|||||||
OpType: replication.ExistingObjectReplicationType,
|
OpType: replication.ExistingObjectReplicationType,
|
||||||
EventType: ReplicateExistingDelete,
|
EventType: ReplicateExistingDelete,
|
||||||
}
|
}
|
||||||
replicateDelete(ctx, doi, objectAPI)
|
rinfos = replicateDelete(ctx, doi, objectAPI)
|
||||||
} else {
|
} else {
|
||||||
roi.OpType = replication.ExistingObjectReplicationType
|
roi.OpType = replication.ExistingObjectReplicationType
|
||||||
roi.EventType = ReplicateExisting
|
roi.EventType = ReplicateExisting
|
||||||
replicateObject(ctx, roi, objectAPI)
|
rinfos = replicateObject(ctx, roi, objectAPI)
|
||||||
}
|
}
|
||||||
|
|
||||||
st := TargetReplicationResyncStatus{
|
// Classify success/failure from the actual replication outcome
|
||||||
Object: roi.Name,
|
// for this target, not from whether the target version merely
|
||||||
Bucket: roi.Bucket,
|
// exists (a rejected update leaves the old version in place).
|
||||||
}
|
st := resyncResultFor(rinfos, opts.arn, roi)
|
||||||
|
var traceSize int64
|
||||||
_, err := tgt.StatObject(ctx, tgt.Bucket, roi.Name, minio.StatObjectOptions{
|
var traceErr error
|
||||||
VersionID: roi.VersionID,
|
for i := range rinfos.Targets {
|
||||||
Internal: minio.AdvancedGetOptions{
|
if rinfos.Targets[i].Arn == opts.arn {
|
||||||
ReplicationProxyRequest: "false",
|
traceSize, traceErr = rinfos.Targets[i].Size, rinfos.Targets[i].Err
|
||||||
},
|
break
|
||||||
})
|
|
||||||
sz := roi.Size
|
|
||||||
if err != nil {
|
|
||||||
if roi.DeleteMarker && isErrMethodNotAllowed(ErrorRespToObjectError(err, opts.bucket, roi.Name)) {
|
|
||||||
st.ReplicatedCount++
|
|
||||||
} else {
|
|
||||||
st.FailedCount++
|
|
||||||
}
|
}
|
||||||
sz = 0
|
|
||||||
} else {
|
|
||||||
st.ReplicatedCount++
|
|
||||||
st.ReplicatedSize += roi.Size
|
|
||||||
}
|
}
|
||||||
traceFn(sz, err)
|
traceFn(traceSize, traceErr)
|
||||||
select {
|
if !s.sendResyncResult(ctx, results.ch, st) {
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
return
|
||||||
case <-s.resyncCancelCh:
|
|
||||||
return
|
|
||||||
case resultCh <- st:
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}(ctx, i)
|
}(ctx, i)
|
||||||
}
|
}
|
||||||
for res := range objInfoCh {
|
walkLoop:
|
||||||
|
for {
|
||||||
|
var res itemOrErr[ObjectInfo]
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case item, ok := <-objInfoCh:
|
||||||
|
if !ok {
|
||||||
|
break walkLoop
|
||||||
|
}
|
||||||
|
res = item
|
||||||
|
}
|
||||||
if res.Err != nil {
|
if res.Err != nil {
|
||||||
resyncStatus = ResyncFailed
|
resyncStatus = ResyncFailed
|
||||||
replLogIf(ctx, res.Err)
|
replLogIf(ctx, res.Err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
select {
|
|
||||||
case <-s.resyncCancelCh:
|
|
||||||
resyncStatus = ResyncCanceled
|
|
||||||
return
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
if heal && lastCheckpoint != "" && lastCheckpoint != res.Item.Name {
|
if heal && lastCheckpoint != "" && lastCheckpoint != res.Item.Name {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
lastCheckpoint = ""
|
lastCheckpoint = ""
|
||||||
roi := getHealReplicateObjectInfo(res.Item, rcfg)
|
roi := getHealReplicateObjectInfo(res.Item, rcfg)
|
||||||
if !roi.ExistingObjResync.mustResync() {
|
// Scope dispatch to this resync's target: the worker pool is for
|
||||||
|
// opts.arn, so skip objects that only need resync for a different
|
||||||
|
// target (each target has its own resync). Without this, a cross-target
|
||||||
|
// object leaves opts.arn absent from its per-object result and is
|
||||||
|
// miscounted as an opts.arn failure.
|
||||||
|
if !objectNeedsResyncForARN(roi, opts.arn) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
h := xxh3.HashString(roi.Bucket + roi.Name)
|
||||||
select {
|
select {
|
||||||
case <-s.resyncCancelCh:
|
|
||||||
return
|
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
default:
|
case workers[h%uint64(resyncParallelRoutines)] <- roi:
|
||||||
h := xxh3.HashString(roi.Bucket + roi.Name)
|
|
||||||
workers[h%uint64(resyncParallelRoutines)] <- roi
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for i := range resyncParallelRoutines {
|
|
||||||
xioutil.SafeClose(workers[i])
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
resyncStatus = ResyncCompleted
|
resyncStatus = ResyncCompleted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3087,9 +3427,9 @@ func (s *replicationResyncer) start(ctx context.Context, objAPI ObjectLayer, opt
|
|||||||
if len(tgtArns) == 0 {
|
if len(tgtArns) == 0 {
|
||||||
return fmt.Errorf("arn %s specified for resync not found in replication config", opts.arn)
|
return fmt.Errorf("arn %s specified for resync not found in replication config", opts.arn)
|
||||||
}
|
}
|
||||||
globalReplicationPool.Get().resyncer.RLock()
|
s.Lock()
|
||||||
data, ok := globalReplicationPool.Get().resyncer.statusMap[opts.bucket]
|
defer s.Unlock()
|
||||||
globalReplicationPool.Get().resyncer.RUnlock()
|
data, ok := s.statusMap[opts.bucket]
|
||||||
if !ok {
|
if !ok {
|
||||||
data, err = loadBucketResyncMetadata(ctx, opts.bucket, objAPI)
|
data, err = loadBucketResyncMetadata(ctx, opts.bucket, objAPI)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -3110,23 +3450,14 @@ func (s *replicationResyncer) start(ctx context.Context, objAPI ObjectLayer, opt
|
|||||||
ResyncStatus: ResyncPending,
|
ResyncStatus: ResyncPending,
|
||||||
Bucket: opts.bucket,
|
Bucket: opts.bucket,
|
||||||
}
|
}
|
||||||
|
data.TargetsMap = data.cloneTgtStats()
|
||||||
data.TargetsMap[opts.arn] = status
|
data.TargetsMap[opts.arn] = status
|
||||||
if err = saveResyncStatus(ctx, opts.bucket, data, objAPI); err != nil {
|
if err = saveResyncStatus(ctx, opts.bucket, data, objAPI); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
globalReplicationPool.Get().resyncer.Lock()
|
s.statusMap[opts.bucket] = data
|
||||||
defer globalReplicationPool.Get().resyncer.Unlock()
|
go s.resyncBucket(GlobalContext, objAPI, false, opts)
|
||||||
brs, ok := globalReplicationPool.Get().resyncer.statusMap[opts.bucket]
|
|
||||||
if !ok {
|
|
||||||
brs = BucketReplicationResyncStatus{
|
|
||||||
Version: resyncMetaVersion,
|
|
||||||
TargetsMap: make(map[string]TargetReplicationResyncStatus),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
brs.TargetsMap[opts.arn] = status
|
|
||||||
globalReplicationPool.Get().resyncer.statusMap[opts.bucket] = brs
|
|
||||||
go globalReplicationPool.Get().resyncer.resyncBucket(GlobalContext, objAPI, false, opts)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3200,6 +3531,9 @@ func (p *ReplicationPool) loadResync(ctx context.Context, buckets []string, objA
|
|||||||
// Make sure only one node running resync on the cluster.
|
// Make sure only one node running resync on the cluster.
|
||||||
ctx, cancel := globalLeaderLock.GetLock(ctx)
|
ctx, cancel := globalLeaderLock.GetLock(ctx)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
var workers sync.WaitGroup
|
||||||
|
// Keep the merged leader context alive until every resumed run exits.
|
||||||
|
defer workers.Wait()
|
||||||
|
|
||||||
for index := range buckets {
|
for index := range buckets {
|
||||||
bucket := buckets[index]
|
bucket := buckets[index]
|
||||||
@@ -3213,19 +3547,24 @@ func (p *ReplicationPool) loadResync(ctx context.Context, buckets []string, objA
|
|||||||
}
|
}
|
||||||
|
|
||||||
p.resyncer.Lock()
|
p.resyncer.Lock()
|
||||||
p.resyncer.statusMap[bucket] = meta
|
if current, ok := p.resyncer.statusMap[bucket]; ok {
|
||||||
p.resyncer.Unlock()
|
// A concurrent start/cancel is newer than the disk snapshot.
|
||||||
|
meta = current
|
||||||
|
} else {
|
||||||
|
p.resyncer.statusMap[bucket] = meta
|
||||||
|
}
|
||||||
tgts := meta.cloneTgtStats()
|
tgts := meta.cloneTgtStats()
|
||||||
|
p.resyncer.Unlock()
|
||||||
for arn, st := range tgts {
|
for arn, st := range tgts {
|
||||||
switch st.ResyncStatus {
|
switch st.ResyncStatus {
|
||||||
case ResyncFailed, ResyncStarted, ResyncPending:
|
case ResyncFailed, ResyncStarted, ResyncPending:
|
||||||
go p.resyncer.resyncBucket(ctx, objAPI, true, resyncOpts{
|
opts := resyncOpts{
|
||||||
bucket: bucket,
|
bucket: bucket,
|
||||||
arn: arn,
|
arn: arn,
|
||||||
resyncID: st.ResyncID,
|
resyncID: st.ResyncID,
|
||||||
resyncBefore: st.ResyncBeforeDate,
|
resyncBefore: st.ResyncBeforeDate,
|
||||||
})
|
}
|
||||||
|
workers.Go(func() { p.resyncer.resyncBucket(ctx, objAPI, true, opts) })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3544,6 +3883,7 @@ func (p *ReplicationPool) queueMRFSave(entry MRFReplicateEntry) {
|
|||||||
if entry.RetryCount > mrfRetryLimit { // let scanner catch up if retry count exceeded
|
if entry.RetryCount > mrfRetryLimit { // let scanner catch up if retry count exceeded
|
||||||
atomic.AddUint64(&p.stats.mrfStats.TotalDroppedCount, 1)
|
atomic.AddUint64(&p.stats.mrfStats.TotalDroppedCount, 1)
|
||||||
atomic.AddUint64(&p.stats.mrfStats.TotalDroppedBytes, uint64(entry.sz))
|
atomic.AddUint64(&p.stats.mrfStats.TotalDroppedBytes, uint64(entry.sz))
|
||||||
|
replLogOnceIf(GlobalContext, errors.New("Replication MRF retry limit reached; further repair is deferred to the scanner"), "replication-mrf-retry-limit", logger.WarningKind)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3558,6 +3898,7 @@ func (p *ReplicationPool) queueMRFSave(entry MRFReplicateEntry) {
|
|||||||
default:
|
default:
|
||||||
atomic.AddUint64(&p.stats.mrfStats.TotalDroppedCount, 1)
|
atomic.AddUint64(&p.stats.mrfStats.TotalDroppedCount, 1)
|
||||||
atomic.AddUint64(&p.stats.mrfStats.TotalDroppedBytes, uint64(entry.sz))
|
atomic.AddUint64(&p.stats.mrfStats.TotalDroppedBytes, uint64(entry.sz))
|
||||||
|
replLogOnceIf(GlobalContext, errors.New("Replication MRF queue is full; dropped entries will need scanner repair"), "replication-mrf-queue-full", logger.WarningKind)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,13 +18,21 @@
|
|||||||
package cmd
|
package cmd
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
"path"
|
"path"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/minio/madmin-go/v3"
|
"github.com/minio/madmin-go/v3"
|
||||||
|
"github.com/minio/minio-go/v7"
|
||||||
|
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||||
"github.com/minio/minio/internal/bucket/replication"
|
"github.com/minio/minio/internal/bucket/replication"
|
||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
)
|
)
|
||||||
@@ -307,3 +315,783 @@ func TestReplicationValidationObjectUsesRulePrefix(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The resync-finalization tests below exercise the real result sink, the
|
||||||
|
// finish() shutdown ordering, and the sendResyncResult / finalResyncStatus
|
||||||
|
// helpers, plus (for the persistence cases) markStatus with on-disk
|
||||||
|
// round-tripping. resyncBucket cannot be driven end to end in a unit test
|
||||||
|
// because its workers call a live remote target (StatObject), so the helpers it
|
||||||
|
// uses are exercised directly. The blocking-order assertions run under
|
||||||
|
// testing/synctest so a removed wait fails deterministically, with no timing
|
||||||
|
// windows.
|
||||||
|
|
||||||
|
func newTestResyncer(bucket, arn string) (*replicationResyncer, resyncOpts) {
|
||||||
|
s := &replicationResyncer{
|
||||||
|
statusMap: map[string]BucketReplicationResyncStatus{},
|
||||||
|
}
|
||||||
|
brs := newBucketResyncStatus(bucket)
|
||||||
|
brs.TargetsMap[arn] = TargetReplicationResyncStatus{ResyncStatus: ResyncStarted, ResyncID: "reset-" + bucket}
|
||||||
|
s.statusMap[bucket] = brs
|
||||||
|
return s, resyncOpts{bucket: bucket, arn: arn, resyncID: "reset-" + bucket}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestResyncBucketFinalize round-trips the terminal status through a real
|
||||||
|
// ObjectLayer: a clean run persists Completed with every result, while a run
|
||||||
|
// whose parent context was canceled during the drain is downgraded to Failed;
|
||||||
|
// a user-canceled run persists Canceled. Completed never misrepresents an
|
||||||
|
// incomplete resync.
|
||||||
|
func TestResyncBucketFinalize(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
objAPI, fsDirs, err := prepareErasure16(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare erasure backend: %v", err)
|
||||||
|
}
|
||||||
|
defer removeRoots(fsDirs)
|
||||||
|
|
||||||
|
// persistTerminal applies resyncBucket's finalizer logic (finalResyncStatus
|
||||||
|
// then markStatus, which persists) and reads the status back the way the
|
||||||
|
// resync status API does.
|
||||||
|
persistTerminal := func(t *testing.T, s *replicationResyncer, opts resyncOpts, status ResyncStatusType, cause error) TargetReplicationResyncStatus {
|
||||||
|
t.Helper()
|
||||||
|
s.markStatus(finalResyncStatus(status, cause), opts, objAPI)
|
||||||
|
brs, err := loadBucketResyncMetadata(ctx, opts.bucket, objAPI)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load persisted resync metadata: %v", err)
|
||||||
|
}
|
||||||
|
return brs.TargetsMap[opts.arn]
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1. Clean completion: every result - including the failed object - is folded
|
||||||
|
// into the persisted status, which stays Completed.
|
||||||
|
t.Run("persists complete counts", func(t *testing.T) {
|
||||||
|
s, opts := newTestResyncer("finalize-counts", "arn1")
|
||||||
|
results := s.newResyncResults(opts)
|
||||||
|
results.ch <- TargetReplicationResyncStatus{Object: "ok-1", ReplicatedCount: 1, ReplicatedSize: 100}
|
||||||
|
results.ch <- TargetReplicationResyncStatus{Object: "ok-2", ReplicatedCount: 1, ReplicatedSize: 200}
|
||||||
|
results.ch <- TargetReplicationResyncStatus{Object: "bad", FailedCount: 1, FailedSize: 300}
|
||||||
|
|
||||||
|
var wg sync.WaitGroup // no producer workers for this case
|
||||||
|
results.finish(nil, &wg)
|
||||||
|
|
||||||
|
st := persistTerminal(t, s, opts, ResyncCompleted, nil)
|
||||||
|
if st.ResyncStatus != ResyncCompleted {
|
||||||
|
t.Fatalf("persisted status = %s, want Completed", st.ResyncStatus)
|
||||||
|
}
|
||||||
|
if st.ReplicatedCount != 2 || st.ReplicatedSize != 300 || st.FailedCount != 1 || st.FailedSize != 300 {
|
||||||
|
t.Fatalf("persisted counts = {replicated:%d/%d failed:%d/%d}, want {2/300 1/300}",
|
||||||
|
st.ReplicatedCount, st.ReplicatedSize, st.FailedCount, st.FailedSize)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// 2. Parent context canceled during the drain -> Completed downgraded to
|
||||||
|
// Failed (markStatus persists under its own context, so nothing else stops
|
||||||
|
// a bare Completed from being recorded).
|
||||||
|
t.Run("parent cancel during drain downgrades to failed", func(t *testing.T) {
|
||||||
|
s, opts := newTestResyncer("finalize-parent-cancel", "arn1")
|
||||||
|
results := s.newResyncResults(opts)
|
||||||
|
results.ch <- TargetReplicationResyncStatus{Object: "ok-1", ReplicatedCount: 1, ReplicatedSize: 100}
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
results.finish(nil, &wg)
|
||||||
|
|
||||||
|
cctx, ccancel := context.WithCancel(context.Background())
|
||||||
|
ccancel()
|
||||||
|
st := persistTerminal(t, s, opts, ResyncCompleted, context.Cause(cctx))
|
||||||
|
if st.ResyncStatus != ResyncFailed {
|
||||||
|
t.Fatalf("persisted status = %s, want Failed (parent canceled during drain)", st.ResyncStatus)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// 3. A user-canceled worker cannot report a completed resync.
|
||||||
|
t.Run("user cancel persists canceled", func(t *testing.T) {
|
||||||
|
s, opts := newTestResyncer("finalize-worker-abort", "arn1")
|
||||||
|
ctx, cancel := context.WithCancelCause(context.Background())
|
||||||
|
cancel(errResyncCanceled)
|
||||||
|
ch := make(chan TargetReplicationResyncStatus)
|
||||||
|
if s.sendResyncResult(ctx, ch, TargetReplicationResyncStatus{Object: "dropped", ReplicatedCount: 1}) {
|
||||||
|
t.Fatal("sendResyncResult reported success after cancellation")
|
||||||
|
}
|
||||||
|
st := persistTerminal(t, s, opts, ResyncCompleted, context.Cause(ctx))
|
||||||
|
if st.ResyncStatus != ResyncCanceled {
|
||||||
|
t.Fatalf("persisted status = %s, want Canceled", st.ResyncStatus)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestResyncFinishDrainsResults asserts finish() does not return until the
|
||||||
|
// consumer has applied the final result (the #136 defect). A gated apply holds
|
||||||
|
// the last result unapplied; under synctest finish() must stay durably blocked
|
||||||
|
// until it is released - if rr.wg.Wait() is removed, finish() returns early and
|
||||||
|
// the test fails deterministically.
|
||||||
|
func TestResyncFinishDrainsResults(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
s, opts := newTestResyncer("drain", "arn1")
|
||||||
|
reachedFinal := make(chan struct{})
|
||||||
|
release := make(chan struct{})
|
||||||
|
results := startResyncResults(func(r TargetReplicationResyncStatus) {
|
||||||
|
if r.Object == "final" {
|
||||||
|
close(reachedFinal)
|
||||||
|
<-release
|
||||||
|
}
|
||||||
|
s.incStats(r, opts)
|
||||||
|
})
|
||||||
|
results.ch <- TargetReplicationResyncStatus{Object: "ok-1", ReplicatedCount: 1, ReplicatedSize: 100}
|
||||||
|
results.ch <- TargetReplicationResyncStatus{Object: "final", FailedCount: 1, FailedSize: 200}
|
||||||
|
<-reachedFinal // consumer received "final" but is gated before incStats(final)
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
finishDone := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
results.finish(nil, &wg)
|
||||||
|
close(finishDone)
|
||||||
|
}()
|
||||||
|
|
||||||
|
synctest.Wait()
|
||||||
|
select {
|
||||||
|
case <-finishDone:
|
||||||
|
close(release)
|
||||||
|
synctest.Wait()
|
||||||
|
t.Fatal("finish() returned before the final result was drained (drain wait missing)")
|
||||||
|
default:
|
||||||
|
// finish() is durably blocked in rr.wg.Wait() - correct.
|
||||||
|
}
|
||||||
|
|
||||||
|
close(release)
|
||||||
|
synctest.Wait()
|
||||||
|
<-finishDone
|
||||||
|
st := s.statusMap[opts.bucket].TargetsMap[opts.arn]
|
||||||
|
if st.ReplicatedCount != 1 || st.FailedCount != 1 || st.FailedSize != 200 {
|
||||||
|
t.Fatalf("status after finish = {replicated:%d failed:%d/%d}, want {1 1/200}",
|
||||||
|
st.ReplicatedCount, st.FailedCount, st.FailedSize)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestResyncFinishWaitsForInflightWorker asserts finish() stops the producer
|
||||||
|
// workers before it closes the result channel, so an in-flight worker (as on an
|
||||||
|
// early-return path) never sends on a closed channel and its result is not lost.
|
||||||
|
// A gated worker stays in flight past the shutdown request; under synctest
|
||||||
|
// finish() must stay durably blocked until the worker is released - if
|
||||||
|
// workerWg.Wait() is removed, finish() returns early and the test fails
|
||||||
|
// deterministically.
|
||||||
|
func TestResyncFinishWaitsForInflightWorker(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
s, opts := newTestResyncer("workers", "arn1")
|
||||||
|
results := startResyncResults(func(r TargetReplicationResyncStatus) { s.incStats(r, opts) })
|
||||||
|
|
||||||
|
workers := []chan ReplicateObjectInfo{make(chan ReplicateObjectInfo, 1)}
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
gotRoi := make(chan struct{})
|
||||||
|
release := make(chan struct{})
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for roi := range workers[0] {
|
||||||
|
close(gotRoi)
|
||||||
|
<-release
|
||||||
|
// Mirror the real worker's send; recover so that if finish()
|
||||||
|
// wrongly closed the result channel first, the test fails via the
|
||||||
|
// assertion below instead of crashing on send-on-closed.
|
||||||
|
func() {
|
||||||
|
defer func() { _ = recover() }()
|
||||||
|
results.ch <- TargetReplicationResyncStatus{Object: roi.Name, ReplicatedCount: 1, ReplicatedSize: 500}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
workers[0] <- ReplicateObjectInfo{Name: "inflight"}
|
||||||
|
<-gotRoi // worker holds a result in flight, not yet delivered
|
||||||
|
|
||||||
|
finishDone := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
results.finish(workers, &wg)
|
||||||
|
close(finishDone)
|
||||||
|
}()
|
||||||
|
|
||||||
|
synctest.Wait()
|
||||||
|
select {
|
||||||
|
case <-finishDone:
|
||||||
|
close(release)
|
||||||
|
synctest.Wait()
|
||||||
|
t.Fatal("finish() closed the result channel before the in-flight worker finished (worker wait missing)")
|
||||||
|
default:
|
||||||
|
// finish() is durably blocked in workerWg.Wait() - correct.
|
||||||
|
}
|
||||||
|
|
||||||
|
close(release)
|
||||||
|
synctest.Wait()
|
||||||
|
<-finishDone
|
||||||
|
st := s.statusMap[opts.bucket].TargetsMap[opts.arn]
|
||||||
|
if st.ReplicatedCount != 1 || st.ReplicatedSize != 500 {
|
||||||
|
t.Fatalf("status after finish = {replicated:%d/%d}, want {1/500}", st.ReplicatedCount, st.ReplicatedSize)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestResyncResultFor asserts the resync worker classifies a target from the
|
||||||
|
// actual replication outcome, not from whether the target version merely exists.
|
||||||
|
// The key regression is the "failed update over an existing version" case: a
|
||||||
|
// quota-rejected update leaves the old version in place, and counting existence
|
||||||
|
// (the previous behavior) would score it a success. It also checks a genuine
|
||||||
|
// success, an errored-but-Completed result, a delete failure, a delete-marker
|
||||||
|
// success (zero bytes), and an ARN that was never attempted.
|
||||||
|
func TestResyncResultFor(t *testing.T) {
|
||||||
|
const arn = "arn:minio:replication::id:bucket"
|
||||||
|
obj := ReplicateObjectInfo{Name: "obj", Bucket: "bucket", Size: 196608}
|
||||||
|
deleteMarker := ReplicateObjectInfo{Name: "dm", Bucket: "bucket", Size: 0, DeleteMarker: true}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
roi ReplicateObjectInfo
|
||||||
|
rinfos replicatedInfos
|
||||||
|
wantRepl, wantReplSize, wantFail, wantFailSize int64
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "completed update",
|
||||||
|
roi: obj,
|
||||||
|
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||||
|
{Arn: arn, ReplicationStatus: replication.Completed, Size: 196608},
|
||||||
|
}},
|
||||||
|
wantRepl: 1, wantReplSize: 196608,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "failed update over existing version",
|
||||||
|
roi: obj,
|
||||||
|
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||||
|
{Arn: arn, ReplicationStatus: replication.Failed, Err: fmt.Errorf("quota exceeded"), Size: 196608},
|
||||||
|
}},
|
||||||
|
wantFail: 1, wantFailSize: 196608,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "completed but errored is a failure",
|
||||||
|
roi: obj,
|
||||||
|
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||||
|
{Arn: arn, ReplicationStatus: replication.Completed, Err: fmt.Errorf("boom"), Size: 196608},
|
||||||
|
}},
|
||||||
|
wantFail: 1, wantFailSize: 196608,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "delete failed",
|
||||||
|
roi: deleteMarker,
|
||||||
|
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||||
|
{Arn: arn, ReplicationStatus: replication.Failed},
|
||||||
|
}},
|
||||||
|
wantFail: 1, wantFailSize: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "delete marker replicated counts zero bytes",
|
||||||
|
roi: deleteMarker,
|
||||||
|
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||||
|
{Arn: arn, ReplicationStatus: replication.Completed},
|
||||||
|
}},
|
||||||
|
wantRepl: 1, wantReplSize: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "arn not attempted is a failure",
|
||||||
|
roi: obj,
|
||||||
|
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||||
|
{Arn: "arn:minio:replication::id2:bucket", ReplicationStatus: replication.Completed, Size: 196608},
|
||||||
|
}},
|
||||||
|
wantFail: 1, wantFailSize: 196608,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "completed with zero size falls back to object size",
|
||||||
|
roi: obj,
|
||||||
|
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||||
|
{Arn: arn, ReplicationStatus: replication.Completed, Size: 0},
|
||||||
|
}},
|
||||||
|
wantRepl: 1, wantReplSize: 196608,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "version purge complete is a success",
|
||||||
|
roi: ReplicateObjectInfo{Name: "purge", Bucket: "bucket", Size: 196608, VersionPurgeStatus: replication.VersionPurgePending},
|
||||||
|
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||||
|
// a successful purge sets only VersionPurgeStatus; ReplicationStatus stays empty.
|
||||||
|
{Arn: arn, VersionPurgeStatus: replication.VersionPurgeComplete},
|
||||||
|
}},
|
||||||
|
wantRepl: 1, wantReplSize: 196608,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "version purge failed is a failure",
|
||||||
|
roi: ReplicateObjectInfo{Name: "purge", Bucket: "bucket", Size: 196608, VersionPurgeStatus: replication.VersionPurgePending},
|
||||||
|
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||||
|
{Arn: arn, VersionPurgeStatus: replication.VersionPurgeFailed, Err: fmt.Errorf("quota exceeded")},
|
||||||
|
}},
|
||||||
|
wantFail: 1, wantFailSize: 196608,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "benign duplicate 412 is a success",
|
||||||
|
roi: obj,
|
||||||
|
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||||
|
// the destination answers PreconditionFailed for an exact duplicate;
|
||||||
|
// replicateAll keeps Completed but retains the error.
|
||||||
|
{Arn: arn, ReplicationStatus: replication.Completed, Err: minio.ErrorResponse{Code: "PreconditionFailed"}, Size: 196608},
|
||||||
|
}},
|
||||||
|
wantRepl: 1, wantReplSize: 196608,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
st := resyncResultFor(tc.rinfos, arn, tc.roi)
|
||||||
|
if st.Object != tc.roi.Name || st.Bucket != tc.roi.Bucket {
|
||||||
|
t.Fatalf("object/bucket = %s/%s, want %s/%s", st.Object, st.Bucket, tc.roi.Name, tc.roi.Bucket)
|
||||||
|
}
|
||||||
|
if st.ReplicatedCount != tc.wantRepl || st.ReplicatedSize != tc.wantReplSize ||
|
||||||
|
st.FailedCount != tc.wantFail || st.FailedSize != tc.wantFailSize {
|
||||||
|
t.Fatalf("resyncResultFor = {replicated:%d/%d failed:%d/%d}, want {%d/%d %d/%d}",
|
||||||
|
st.ReplicatedCount, st.ReplicatedSize, st.FailedCount, st.FailedSize,
|
||||||
|
tc.wantRepl, tc.wantReplSize, tc.wantFail, tc.wantFailSize)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestObjectNeedsResyncForARN asserts the resync dispatch is scoped to the
|
||||||
|
// target being resynced. The worker pool runs for a single target (opts.arn),
|
||||||
|
// so an object that only qualifies for a different target must be skipped: with
|
||||||
|
// A/B rules and a resync of A, an object that needs replication only for B must
|
||||||
|
// not be admitted to A's worker. Otherwise (after outcome-based classification)
|
||||||
|
// A would be absent from that object's result and miscounted as an A failure.
|
||||||
|
func TestObjectNeedsResyncForARN(t *testing.T) {
|
||||||
|
const (
|
||||||
|
arnA = "arn:minio:replication::id:bucket"
|
||||||
|
arnB = "arn:minio:replication::id2:bucket"
|
||||||
|
)
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
decision ResyncDecision
|
||||||
|
arn string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "target must resync",
|
||||||
|
decision: ResyncDecision{targets: map[string]ResyncTargetDecision{arnA: {Replicate: true}}},
|
||||||
|
arn: arnA,
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "object qualifies for B only, resyncing A",
|
||||||
|
decision: ResyncDecision{targets: map[string]ResyncTargetDecision{arnB: {Replicate: true}}},
|
||||||
|
arn: arnA,
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "A present but not replicating, B replicating, resyncing A",
|
||||||
|
decision: ResyncDecision{targets: map[string]ResyncTargetDecision{
|
||||||
|
arnA: {Replicate: false},
|
||||||
|
arnB: {Replicate: true},
|
||||||
|
}},
|
||||||
|
arn: arnA,
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "object qualifies for both, resyncing A",
|
||||||
|
decision: ResyncDecision{targets: map[string]ResyncTargetDecision{
|
||||||
|
arnA: {Replicate: true},
|
||||||
|
arnB: {Replicate: true},
|
||||||
|
}},
|
||||||
|
arn: arnA,
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no resync decision",
|
||||||
|
decision: ResyncDecision{},
|
||||||
|
arn: arnA,
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
roi := ReplicateObjectInfo{Name: "obj", Bucket: "bucket", ExistingObjResync: tc.decision}
|
||||||
|
if got := objectNeedsResyncForARN(roi, tc.arn); got != tc.want {
|
||||||
|
t.Fatalf("objectNeedsResyncForARN(arn=%s) = %v, want %v", tc.arn, got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// newMatchingReplicationPair returns a source/target pair that getReplicationAction must
|
||||||
|
// classify as replicateNone: same ETag, version id, size, modification time and content
|
||||||
|
// type. Any action other than replicateNone is therefore attributable to the object lock
|
||||||
|
// entries a caller adds on top.
|
||||||
|
func newMatchingReplicationPair() (ObjectInfo, minio.ObjectInfo) {
|
||||||
|
mtime := time.Date(2026, 9, 5, 10, 0, 0, 0, time.UTC)
|
||||||
|
size := int64(7)
|
||||||
|
src := ObjectInfo{
|
||||||
|
Bucket: "bucket",
|
||||||
|
Name: "object",
|
||||||
|
ETag: "d41d8cd98f00b204e9800998ecf8427e",
|
||||||
|
VersionID: "b0ff1d6e-0000-4000-8000-000000000001",
|
||||||
|
Size: size,
|
||||||
|
ActualSize: &size,
|
||||||
|
ModTime: mtime,
|
||||||
|
ContentType: "application/octet-stream",
|
||||||
|
UserDefined: map[string]string{"content-type": "application/octet-stream"},
|
||||||
|
}
|
||||||
|
tgt := minio.ObjectInfo{
|
||||||
|
ETag: src.ETag,
|
||||||
|
VersionID: src.VersionID,
|
||||||
|
Size: size,
|
||||||
|
LastModified: mtime,
|
||||||
|
ContentType: src.ContentType,
|
||||||
|
Metadata: http.Header{},
|
||||||
|
}
|
||||||
|
return src, tgt
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGetReplicationActionEmptyObjectLockValues covers the comparison of object lock entries
|
||||||
|
// whose value is empty. Removing retention from a version stores the mode and retain-until-date
|
||||||
|
// keys with empty values, while the target's HEAD response omits them entirely, so the two must
|
||||||
|
// compare equal or the version can never be reported as in sync. Cases 3 and 4 are synthetic
|
||||||
|
// comparison inputs, since a SILO target cannot return empty lock headers; cases 7 and 8 guard
|
||||||
|
// against over-normalizing.
|
||||||
|
func TestGetReplicationActionEmptyObjectLockValues(t *testing.T) {
|
||||||
|
var (
|
||||||
|
modeKey = strings.ToLower(xhttp.AmzObjectLockMode)
|
||||||
|
dateKey = strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
|
||||||
|
until = "2026-10-05T10:00:00.000Z"
|
||||||
|
)
|
||||||
|
emptyRetention := map[string]string{modeKey: "", dateKey: ""}
|
||||||
|
realRetention := map[string]string{modeKey: "GOVERNANCE", dateKey: until}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
srcMeta map[string]string
|
||||||
|
tgtHdr map[string]string
|
||||||
|
want replicationAction
|
||||||
|
}{
|
||||||
|
{"1-both-clean-never-had-retention", nil, nil, replicateNone},
|
||||||
|
{"2-source-present-empty-target-absent", emptyRetention, nil, replicateNone},
|
||||||
|
{"3-source-absent-target-present-empty", nil, emptyRetention, replicateNone},
|
||||||
|
{"4-both-present-empty", emptyRetention, emptyRetention, replicateNone},
|
||||||
|
{"5-both-governance-equal", realRetention, realRetention, replicateNone},
|
||||||
|
{"6-source-governance-target-absent", realRetention, nil, replicateMetadata},
|
||||||
|
{"7-source-empty-target-real-retention", emptyRetention, realRetention, replicateMetadata},
|
||||||
|
{"8-empty-user-metadata-is-not-normalized", map[string]string{"x-amz-meta-foo": ""}, nil, replicateMetadata},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
src, tgt := newMatchingReplicationPair()
|
||||||
|
for k, v := range test.srcMeta {
|
||||||
|
src.UserDefined[k] = v
|
||||||
|
}
|
||||||
|
for k, v := range test.tgtHdr {
|
||||||
|
tgt.Metadata.Set(k, v)
|
||||||
|
}
|
||||||
|
if got := getReplicationAction(src, tgt, replication.HealReplicationType); got != test.want {
|
||||||
|
t.Fatalf("getReplicationAction() = %q, want %q (source %v, target %v)", got, test.want, src.UserDefined, tgt.Metadata)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestEmptyRetentionValuesAreOmittedFromObjectResponseHeaders records why the target half of the
|
||||||
|
// comparison in getReplicationAction can never report an empty object lock entry:
|
||||||
|
// FilterObjectLockMetadata drops both keys because an empty mode is not a valid retention mode,
|
||||||
|
// and setObjectHeaders skips them when writing response headers. Neither filter reaches the
|
||||||
|
// replication wire: the empty entries are still carried by getCopyObjMetadata and sent by the
|
||||||
|
// metadata CopyObject, which is why the sender's comparison is what has to tolerate them.
|
||||||
|
// FilterObjectLockMetadata is also applied by CopyObject (cmd/object-handlers.go:1708), where it
|
||||||
|
// strips the source's lock metadata before the destination re-derives it from the request.
|
||||||
|
func TestEmptyRetentionValuesAreOmittedFromObjectResponseHeaders(t *testing.T) {
|
||||||
|
modeKey := strings.ToLower(xhttp.AmzObjectLockMode)
|
||||||
|
dateKey := strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
|
||||||
|
meta := map[string]string{
|
||||||
|
modeKey: "",
|
||||||
|
dateKey: "",
|
||||||
|
"content-type": "application/octet-stream",
|
||||||
|
}
|
||||||
|
|
||||||
|
filtered := objectlock.FilterObjectLockMetadata(meta, false, false)
|
||||||
|
if _, ok := filtered[modeKey]; ok {
|
||||||
|
t.Errorf("FilterObjectLockMetadata() kept the empty lock mode key: %v", filtered)
|
||||||
|
}
|
||||||
|
if _, ok := filtered[dateKey]; ok {
|
||||||
|
t.Errorf("FilterObjectLockMetadata() kept the empty retain-until-date key: %v", filtered)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
if err := setObjectHeaders(t.Context(), rec, ObjectInfo{UserDefined: meta, ModTime: time.Now(), Size: 7}, nil, ObjectOptions{}); err != nil {
|
||||||
|
t.Fatalf("setObjectHeaders() = %v", err)
|
||||||
|
}
|
||||||
|
if v, ok := rec.Header()[http.CanonicalHeaderKey(xhttp.AmzObjectLockMode)]; ok {
|
||||||
|
t.Errorf("setObjectHeaders() emitted an empty lock mode header: %v", v)
|
||||||
|
}
|
||||||
|
if v, ok := rec.Header()[http.CanonicalHeaderKey(xhttp.AmzObjectLockRetainUntilDate)]; ok {
|
||||||
|
t.Errorf("setObjectHeaders() emitted an empty retain-until-date header: %v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fakeRetentionGetter answers GetObjectRetention with a fixed result and counts its calls.
|
||||||
|
type fakeRetentionGetter struct {
|
||||||
|
mode *minio.RetentionMode
|
||||||
|
err error
|
||||||
|
calls int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeRetentionGetter) GetObjectRetention(_ context.Context, _, _, _ string) (*minio.RetentionMode, *time.Time, error) {
|
||||||
|
f.calls++
|
||||||
|
return f.mode, nil, f.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRetentionRemovedAtSource(t *testing.T) {
|
||||||
|
modeKey := strings.ToLower(xhttp.AmzObjectLockMode)
|
||||||
|
dateKey := strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
|
||||||
|
tsKey := ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp
|
||||||
|
stamp := "2026-09-06T01:00:00Z"
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
meta map[string]string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"no lock keys", map[string]string{"content-type": "text/plain"}, false},
|
||||||
|
{"empty pair", map[string]string{modeKey: "", dateKey: ""}, true},
|
||||||
|
{"empty mode only", map[string]string{modeKey: ""}, true},
|
||||||
|
{"empty date only", map[string]string{dateKey: ""}, true},
|
||||||
|
{"real retention", map[string]string{modeKey: "GOVERNANCE", dateKey: "2026-10-05T10:00:00.000Z"}, false},
|
||||||
|
{"canonical case", map[string]string{xhttp.AmzObjectLockMode: ""}, true},
|
||||||
|
{"empty user metadata", map[string]string{"x-amz-meta-foo": ""}, false},
|
||||||
|
// Representation (2): a replicated removal persists the ordering timestamp alone,
|
||||||
|
// with the mode and retain-until-date keys absent (restoreRetention).
|
||||||
|
{"timestamp only, mode absent", map[string]string{tsKey: stamp}, true},
|
||||||
|
{"timestamp with empty mode", map[string]string{tsKey: stamp, modeKey: ""}, true},
|
||||||
|
{"timestamp with real retention is a set, not a removal", map[string]string{tsKey: stamp, modeKey: "GOVERNANCE", dateKey: "2026-10-05T10:00:00.000Z"}, false},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
if got := retentionRemovedAtSource(ObjectInfo{UserDefined: test.meta}); got != test.want {
|
||||||
|
t.Fatalf("retentionRemovedAtSource() = %v, want %v", got, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTargetRetentionConfirmedAbsent pins the rule that only an explicit answer from the
|
||||||
|
// destination clears a removed retention. A denied or unreachable destination must read as still
|
||||||
|
// holding retention, because HEAD hides a real retention from a credential without
|
||||||
|
// s3:GetObjectRetention exactly as it hides one that does not exist.
|
||||||
|
func TestTargetRetentionConfirmedAbsent(t *testing.T) {
|
||||||
|
governance := minio.Governance
|
||||||
|
var emptyMode minio.RetentionMode
|
||||||
|
unknownMode := minio.RetentionMode("ARCHIVE")
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mode *minio.RetentionMode
|
||||||
|
err error
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"version holds governance retention", &governance, nil, false},
|
||||||
|
{"no retention on the version", nil, minio.ErrorResponse{Code: "NoSuchObjectLockConfiguration"}, true},
|
||||||
|
{
|
||||||
|
// The destination also answers this when its own read of the bucket's Object Lock
|
||||||
|
// configuration fails, so it does not establish that Object Lock is disabled.
|
||||||
|
"invalid request naming a missing object lock configuration",
|
||||||
|
nil,
|
||||||
|
minio.ErrorResponse{Code: "InvalidRequest", Message: "Bucket is missing ObjectLockConfiguration"},
|
||||||
|
false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"unrelated invalid request",
|
||||||
|
nil,
|
||||||
|
minio.ErrorResponse{Code: "InvalidRequest", Message: "Object is WORM protected and cannot be overwritten"},
|
||||||
|
false,
|
||||||
|
},
|
||||||
|
{"retention read denied", nil, minio.ErrorResponse{Code: "AccessDenied"}, false},
|
||||||
|
{"destination unreachable", nil, errors.New("dial tcp: connection refused"), false},
|
||||||
|
{"empty mode returned", &emptyMode, nil, true},
|
||||||
|
{"unknown non-empty mode returned", &unknownMode, nil, false},
|
||||||
|
{"nil mode returned", nil, nil, true},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
tgt := &fakeRetentionGetter{mode: test.mode, err: test.err}
|
||||||
|
if got := targetRetentionConfirmedAbsent(t.Context(), tgt, "bucket", "object", "v1"); got != test.want {
|
||||||
|
t.Fatalf("targetRetentionConfirmedAbsent() = %v, want %v", got, test.want)
|
||||||
|
}
|
||||||
|
if tgt.calls != 1 {
|
||||||
|
t.Fatalf("GetObjectRetention called %d times, want 1", tgt.calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestReplicationActionForTargetRetentionRemoval covers the decision the replication worker makes
|
||||||
|
// for a version whose retention was removed. The destination's HEAD never reports the empty keys,
|
||||||
|
// so the comparison alone reads every one of these as in sync; only the confirmation separates a
|
||||||
|
// destination that really dropped the retention from one that is hiding it.
|
||||||
|
func TestReplicationActionForTargetRetentionRemoval(t *testing.T) {
|
||||||
|
modeKey := strings.ToLower(xhttp.AmzObjectLockMode)
|
||||||
|
dateKey := strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
|
||||||
|
governance := minio.Governance
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
srcMeta map[string]string
|
||||||
|
mode *minio.RetentionMode
|
||||||
|
err error
|
||||||
|
want replicationAction
|
||||||
|
wantCalls int
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "removal confirmed by destination",
|
||||||
|
srcMeta: map[string]string{modeKey: "", dateKey: ""},
|
||||||
|
err: minio.ErrorResponse{Code: "NoSuchObjectLockConfiguration"},
|
||||||
|
want: replicateNone,
|
||||||
|
wantCalls: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "destination still holds the retention hidden from HEAD",
|
||||||
|
srcMeta: map[string]string{modeKey: "", dateKey: ""},
|
||||||
|
mode: &governance,
|
||||||
|
want: replicateMetadata,
|
||||||
|
wantCalls: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "retention hidden from HEAD by permissions",
|
||||||
|
srcMeta: map[string]string{modeKey: "", dateKey: ""},
|
||||||
|
err: minio.ErrorResponse{Code: "AccessDenied"},
|
||||||
|
want: replicateMetadata,
|
||||||
|
wantCalls: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// A destination that names a missing Object Lock configuration answers the same way
|
||||||
|
// when its own read of that configuration failed, so it confirms nothing.
|
||||||
|
name: "destination reports no object lock configuration",
|
||||||
|
srcMeta: map[string]string{modeKey: "", dateKey: ""},
|
||||||
|
err: minio.ErrorResponse{Code: "InvalidRequest", Message: "Bucket is missing ObjectLockConfiguration"},
|
||||||
|
want: replicateMetadata,
|
||||||
|
wantCalls: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "version never had retention is not confirmed",
|
||||||
|
srcMeta: nil,
|
||||||
|
want: replicateNone,
|
||||||
|
wantCalls: 0,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
src, tgtInfo := newMatchingReplicationPair()
|
||||||
|
for k, v := range test.srcMeta {
|
||||||
|
src.UserDefined[k] = v
|
||||||
|
}
|
||||||
|
tgt := &fakeRetentionGetter{mode: test.mode, err: test.err}
|
||||||
|
got := replicationActionForTarget(t.Context(), src, tgtInfo, replication.HealReplicationType, tgt, "bucket", "object")
|
||||||
|
if got != test.want {
|
||||||
|
t.Fatalf("replicationActionForTarget() = %q, want %q", got, test.want)
|
||||||
|
}
|
||||||
|
if tgt.calls != test.wantCalls {
|
||||||
|
t.Fatalf("GetObjectRetention called %d times, want %d", tgt.calls, test.wantCalls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestReplicationActionForTargetNullVersionResync pins that the confirmation does not reopen the
|
||||||
|
// null-version exclusion at the head of getReplicationAction. An existing object resync returns
|
||||||
|
// replicateNone for a null version whose source modification time is later than the target's,
|
||||||
|
// before comparing anything, and that must stand even when the source carries a removed retention
|
||||||
|
// and the destination would report retention or refuse to answer.
|
||||||
|
func TestReplicationActionForTargetNullVersionResync(t *testing.T) {
|
||||||
|
modeKey := strings.ToLower(xhttp.AmzObjectLockMode)
|
||||||
|
dateKey := strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
|
||||||
|
governance := minio.Governance
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mode *minio.RetentionMode
|
||||||
|
err error
|
||||||
|
}{
|
||||||
|
{"destination holds retention", &governance, nil},
|
||||||
|
{"retention read denied", nil, minio.ErrorResponse{Code: "AccessDenied"}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
src, tgtInfo := newMatchingReplicationPair()
|
||||||
|
// A null version whose source modification time is later, and whose content differs,
|
||||||
|
// so only the exclusion can hold the action at replicateNone.
|
||||||
|
src.VersionID = nullVersionID
|
||||||
|
src.ModTime = tgtInfo.LastModified.Add(time.Hour)
|
||||||
|
src.ETag = "5d41402abc4b2a76b9719d911017c592"
|
||||||
|
src.UserDefined[modeKey] = ""
|
||||||
|
src.UserDefined[dateKey] = ""
|
||||||
|
tgtInfo.VersionID = nullVersionID
|
||||||
|
|
||||||
|
tgt := &fakeRetentionGetter{mode: test.mode, err: test.err}
|
||||||
|
got := replicationActionForTarget(t.Context(), src, tgtInfo, replication.ExistingObjectReplicationType, tgt, "bucket", "object")
|
||||||
|
if got != replicateNone {
|
||||||
|
t.Fatalf("replicationActionForTarget() = %q, want %q", got, replicateNone)
|
||||||
|
}
|
||||||
|
if tgt.calls != 0 {
|
||||||
|
t.Fatalf("GetObjectRetention called %d times, want 0", tgt.calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestReplicationActionForTargetTimestampOnlyRemoval covers representation (2) of a removed
|
||||||
|
// retention. A removal that arrived by replication persists only the retention ordering timestamp,
|
||||||
|
// with the mode and retain-until-date keys absent, because restoreRetention writes the timestamp
|
||||||
|
// alone when the mode is empty (cmd/bucket-object-lock.go). The comparison in getReplicationAction
|
||||||
|
// reads such a source as in sync with a matching destination, so only the GetObjectRetention
|
||||||
|
// confirmation separates a destination that dropped the retention from one hiding it behind a
|
||||||
|
// permission-filtered HEAD. The source is built through the real restoreRetention path so the
|
||||||
|
// fixture is the metadata a replicated removal actually leaves on disk, not a hand-rolled map.
|
||||||
|
func TestReplicationActionForTargetTimestampOnlyRemoval(t *testing.T) {
|
||||||
|
governance := minio.Governance
|
||||||
|
stamp := time.Date(2026, 9, 6, 1, 0, 0, 0, time.UTC).Format(time.RFC3339Nano)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mode *minio.RetentionMode
|
||||||
|
err error
|
||||||
|
want replicationAction
|
||||||
|
wantCalls int
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
// The reference case: a destination that denies the retention read is
|
||||||
|
// indistinguishable from one still holding it, so the removal is resent.
|
||||||
|
name: "retention hidden from HEAD by permissions",
|
||||||
|
err: minio.ErrorResponse{Code: "AccessDenied"},
|
||||||
|
want: replicateMetadata,
|
||||||
|
wantCalls: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "destination still holds the retention",
|
||||||
|
mode: &governance,
|
||||||
|
want: replicateMetadata,
|
||||||
|
wantCalls: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "removal confirmed by destination",
|
||||||
|
err: minio.ErrorResponse{Code: "NoSuchObjectLockConfiguration"},
|
||||||
|
want: replicateNone,
|
||||||
|
wantCalls: 1,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
src, tgtInfo := newMatchingReplicationPair()
|
||||||
|
// Persist the timestamp-only tombstone the same way an applied replica removal does.
|
||||||
|
objectLockState{retentionTimestamp: stamp}.restoreRetention(src.UserDefined)
|
||||||
|
if !retentionRemovedAtSource(src) {
|
||||||
|
t.Fatalf("restoreRetention fixture not recognized as a removal: %v", src.UserDefined)
|
||||||
|
}
|
||||||
|
if _, ok := src.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)]; ok {
|
||||||
|
t.Fatalf("restoreRetention fixture wrote a mode key, fixture is not timestamp-only: %v", src.UserDefined)
|
||||||
|
}
|
||||||
|
|
||||||
|
tgt := &fakeRetentionGetter{mode: test.mode, err: test.err}
|
||||||
|
got := replicationActionForTarget(t.Context(), src, tgtInfo, replication.HealReplicationType, tgt, "bucket", "object")
|
||||||
|
if got != test.want {
|
||||||
|
t.Fatalf("replicationActionForTarget() = %q, want %q", got, test.want)
|
||||||
|
}
|
||||||
|
if tgt.calls != test.wantCalls {
|
||||||
|
t.Fatalf("GetObjectRetention called %d times, want %d", tgt.calls, test.wantCalls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ import (
|
|||||||
"github.com/minio/madmin-go/v3"
|
"github.com/minio/madmin-go/v3"
|
||||||
"github.com/minio/minio-go/v7"
|
"github.com/minio/minio-go/v7"
|
||||||
"github.com/minio/minio/internal/auth"
|
"github.com/minio/minio/internal/auth"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
// These tests pin the IAM bucket/object resource boundary end to end, through
|
// These tests pin the IAM bucket/object resource boundary end to end, through
|
||||||
|
|||||||
+6
-2
@@ -319,7 +319,9 @@ func (r *ReplicationStats) getNodeQueueStats(bucket string) (qs ReplQNodeStats)
|
|||||||
qs.QStats = r.qCache.getBucketStats(bucket)
|
qs.QStats = r.qCache.getBucketStats(bucket)
|
||||||
qs.TgtXferStats = make(map[string]map[RMetricName]XferStats)
|
qs.TgtXferStats = make(map[string]map[RMetricName]XferStats)
|
||||||
qs.MRFStats = ReplicationMRFStats{
|
qs.MRFStats = ReplicationMRFStats{
|
||||||
LastFailedCount: atomic.LoadUint64(&r.mrfStats.LastFailedCount),
|
LastFailedCount: atomic.LoadUint64(&r.mrfStats.LastFailedCount),
|
||||||
|
TotalDroppedCount: atomic.LoadUint64(&r.mrfStats.TotalDroppedCount),
|
||||||
|
TotalDroppedBytes: atomic.LoadUint64(&r.mrfStats.TotalDroppedBytes),
|
||||||
}
|
}
|
||||||
|
|
||||||
r.RLock()
|
r.RLock()
|
||||||
@@ -410,7 +412,9 @@ func (r *ReplicationStats) getNodeQueueStatsSummary() (qs ReplQNodeStats) {
|
|||||||
qs.XferStats = make(map[RMetricName]XferStats)
|
qs.XferStats = make(map[RMetricName]XferStats)
|
||||||
qs.QStats = r.qCache.getSiteStats()
|
qs.QStats = r.qCache.getSiteStats()
|
||||||
qs.MRFStats = ReplicationMRFStats{
|
qs.MRFStats = ReplicationMRFStats{
|
||||||
LastFailedCount: atomic.LoadUint64(&r.mrfStats.LastFailedCount),
|
LastFailedCount: atomic.LoadUint64(&r.mrfStats.LastFailedCount),
|
||||||
|
TotalDroppedCount: atomic.LoadUint64(&r.mrfStats.TotalDroppedCount),
|
||||||
|
TotalDroppedBytes: atomic.LoadUint64(&r.mrfStats.TotalDroppedBytes),
|
||||||
}
|
}
|
||||||
r.RLock()
|
r.RLock()
|
||||||
defer r.RUnlock()
|
defer r.RUnlock()
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ import (
|
|||||||
"github.com/minio/minio/internal/bucket/versioning"
|
"github.com/minio/minio/internal/bucket/versioning"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/pgsty/silo-pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -97,7 +97,7 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData)
|
result, err := globalBucketMetadataSys.updateAndParseMetadata(ctx, bucket, bucketVersioningConfig, configData, false, false, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
@@ -107,12 +107,12 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
|
|||||||
//
|
//
|
||||||
// We encode the xml bytes as base64 to ensure there are no encoding
|
// We encode the xml bytes as base64 to ensure there are no encoding
|
||||||
// errors.
|
// errors.
|
||||||
cfgStr := base64.StdEncoding.EncodeToString(configData)
|
cfgStr := base64.StdEncoding.EncodeToString(result.meta.VersioningConfigXML)
|
||||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||||
Type: madmin.SRBucketMetaTypeVersionConfig,
|
Type: madmin.SRBucketMetaTypeVersionConfig,
|
||||||
Bucket: bucket,
|
Bucket: bucket,
|
||||||
Versioning: &cfgStr,
|
Versioning: &cfgStr,
|
||||||
UpdatedAt: updatedAt,
|
UpdatedAt: result.updatedAt,
|
||||||
}))
|
}))
|
||||||
|
|
||||||
writeSuccessResponseHeadersOnly(w)
|
writeSuccessResponseHeadersOnly(w)
|
||||||
|
|||||||
+57
-11
@@ -59,10 +59,10 @@ import (
|
|||||||
"github.com/minio/minio/internal/handlers"
|
"github.com/minio/minio/internal/handlers"
|
||||||
"github.com/minio/minio/internal/kms"
|
"github.com/minio/minio/internal/kms"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/pkg/v3/certs"
|
"github.com/pgsty/silo-pkg/v3/certs"
|
||||||
"github.com/minio/pkg/v3/console"
|
"github.com/pgsty/silo-pkg/v3/console"
|
||||||
"github.com/minio/pkg/v3/env"
|
"github.com/pgsty/silo-pkg/v3/env"
|
||||||
xnet "github.com/minio/pkg/v3/net"
|
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||||
"golang.org/x/term"
|
"golang.org/x/term"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -120,15 +120,40 @@ func init() {
|
|||||||
|
|
||||||
const consolePrefix = "CONSOLE_"
|
const consolePrefix = "CONSOLE_"
|
||||||
|
|
||||||
|
// consoleMinIOServerEnv derives the CONSOLE_MINIO_SERVER value the embedded
|
||||||
|
// Console uses to reach the S3/STS API, and whether TLS verification of that
|
||||||
|
// endpoint must be skipped. With no explicit endpoint configured the Console
|
||||||
|
// reaches the API over the loopback address, whose TLS certificate is not
|
||||||
|
// expected to carry a 127.0.0.1 SAN; because Console verifies outbound TLS by
|
||||||
|
// default, the loopback origin has to be exempted or embedded login (local and
|
||||||
|
// LDAP alike) fails at the STS handshake. The exemption is endpoint-scoped in
|
||||||
|
// Console, so every other HTTPS peer stays verified. An explicitly configured
|
||||||
|
// endpoint is always reached under its own verified name and is never exempted.
|
||||||
|
func consoleMinIOServerEnv(endpoint string, isTLS bool, port string) (server string, skipVerify bool) {
|
||||||
|
if endpoint != "" {
|
||||||
|
return endpoint, false
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s://127.0.0.1:%s", getURLScheme(isTLS), port), isTLS
|
||||||
|
}
|
||||||
|
|
||||||
func minioConfigToConsoleFeatures() {
|
func minioConfigToConsoleFeatures() {
|
||||||
os.Setenv("CONSOLE_PBKDF_SALT", globalDeploymentID())
|
os.Setenv("CONSOLE_PBKDF_SALT", globalDeploymentID())
|
||||||
os.Setenv("CONSOLE_PBKDF_PASSPHRASE", globalDeploymentID())
|
os.Setenv("CONSOLE_PBKDF_PASSPHRASE", globalDeploymentID())
|
||||||
if globalMinioEndpoint != "" {
|
consoleServer, skipVerify := consoleMinIOServerEnv(globalMinioEndpoint, globalIsTLS, globalMinioPort)
|
||||||
os.Setenv("CONSOLE_MINIO_SERVER", globalMinioEndpoint)
|
os.Setenv("CONSOLE_MINIO_SERVER", consoleServer)
|
||||||
|
if skipVerify {
|
||||||
|
// The embedded Console reaches the loopback S3/STS endpoint above, whose
|
||||||
|
// certificate is not expected to carry a 127.0.0.1 SAN. Console verifies
|
||||||
|
// outbound TLS by default (silo-console v2.3.x), so opt into the
|
||||||
|
// endpoint-scoped compatibility switch to preserve the documented loopback
|
||||||
|
// bypass; every other HTTPS peer (IdP, Prometheus, webhooks, ...) stays
|
||||||
|
// verified. initConsoleServer unsets CONSOLE_* before calling this, so the
|
||||||
|
// switch cannot be supplied by the operator on the embedded path.
|
||||||
|
os.Setenv("CONSOLE_MINIO_SERVER_TLS_SKIP_VERIFY", "on")
|
||||||
} else {
|
} else {
|
||||||
// Explicitly set 127.0.0.1 so Console will automatically bypass TLS verification to the local S3 API.
|
// An explicitly configured endpoint is reached under its own verified name;
|
||||||
// This will save users from providing a certificate with IP or FQDN SAN that points to the local host.
|
// never let a loopback exemption apply to it.
|
||||||
os.Setenv("CONSOLE_MINIO_SERVER", fmt.Sprintf("%s://127.0.0.1:%s", getURLScheme(globalIsTLS), globalMinioPort))
|
os.Unsetenv("CONSOLE_MINIO_SERVER_TLS_SKIP_VERIFY")
|
||||||
}
|
}
|
||||||
if value := env.Get(config.EnvMinIOLogQueryURL, ""); value != "" {
|
if value := env.Get(config.EnvMinIOLogQueryURL, ""); value != "" {
|
||||||
os.Setenv("CONSOLE_LOG_QUERY_URL", value)
|
os.Setenv("CONSOLE_LOG_QUERY_URL", value)
|
||||||
@@ -232,11 +257,31 @@ func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
|
|||||||
return m
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
func initConsoleServer() (*consoleapi.Server, error) {
|
// resetConsoleEnvironment preserves the embedded Console's supported resource
|
||||||
// unset all console_ environment variables.
|
// settings verbatim. Server derives all other Console settings itself.
|
||||||
|
func resetConsoleEnvironment() {
|
||||||
for _, cenv := range env.List(consolePrefix) {
|
for _, cenv := range env.List(consolePrefix) {
|
||||||
|
switch cenv {
|
||||||
|
case consoleapi.ConsoleWSMaxConnections,
|
||||||
|
consoleapi.ConsoleWSMaxConnectionsPerClient,
|
||||||
|
consoleapi.ConsoleWSMaxAnonymousConnections,
|
||||||
|
consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient:
|
||||||
|
continue
|
||||||
|
}
|
||||||
os.Unsetenv(cenv)
|
os.Unsetenv(cenv)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func initConsoleServer() (*consoleapi.Server, error) {
|
||||||
|
resetConsoleEnvironment()
|
||||||
|
// Validate explicitly: ConfigureAPI logs errors, but embedded Console logs
|
||||||
|
// are normally silenced. Return configuration failures to Server startup.
|
||||||
|
if err := consoleapi.ConfigureEmbeddedSourceIPTrust(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := consoleapi.ConfigureWebSocketLimits(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
// enable all console environment variables
|
// enable all console environment variables
|
||||||
minioConfigToConsoleFeatures()
|
minioConfigToConsoleFeatures()
|
||||||
@@ -855,6 +900,7 @@ func serverHandleEnvVars() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
globalEnableSyncBoot = env.Get("MINIO_SYNC_BOOT", config.EnableOff) == config.EnableOn
|
globalEnableSyncBoot = env.Get("MINIO_SYNC_BOOT", config.EnableOff) == config.EnableOn
|
||||||
|
globalSiteReplicationMetadataTombstones = env.Get("MINIO_SITE_REPLICATION_METADATA_TOMBSTONES", config.EnableOff) == config.EnableOn
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadRootCredentials() auth.Credentials {
|
func loadRootCredentials() auth.Credentials {
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
consoleapi "github.com/minio/console/api"
|
||||||
"github.com/minio/minio/internal/config"
|
"github.com/minio/minio/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -372,3 +373,138 @@ func TestConfigEnvFileNamedTargetDiscovery(t *testing.T) {
|
|||||||
t.Fatalf("named target %q not discovered from %s: %v", "my-hook", key, targets)
|
t.Fatalf("named target %q not discovered from %s: %v", "my-hook", key, targets)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestConsoleMinIOServerEnv locks in the loopback TLS exemption that keeps
|
||||||
|
// embedded Console login working (issue #108) while ensuring an explicitly
|
||||||
|
// configured endpoint is never silently exempted from TLS verification.
|
||||||
|
func TestConsoleMinIOServerEnv(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
endpoint string
|
||||||
|
isTLS bool
|
||||||
|
port string
|
||||||
|
wantServer string
|
||||||
|
wantSkipVerify bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "loopback TLS is exempted so embedded login works",
|
||||||
|
isTLS: true,
|
||||||
|
port: "9000",
|
||||||
|
wantServer: "https://127.0.0.1:9000",
|
||||||
|
wantSkipVerify: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "loopback plain HTTP needs no exemption",
|
||||||
|
isTLS: false,
|
||||||
|
port: "9000",
|
||||||
|
wantServer: "http://127.0.0.1:9000",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "explicit https endpoint stays verified",
|
||||||
|
endpoint: "https://silo.example:9000",
|
||||||
|
isTLS: true,
|
||||||
|
port: "9000",
|
||||||
|
wantServer: "https://silo.example:9000",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "explicit http endpoint stays verified",
|
||||||
|
endpoint: "http://silo.example:9000",
|
||||||
|
isTLS: false,
|
||||||
|
port: "9000",
|
||||||
|
wantServer: "http://silo.example:9000",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
server, skipVerify := consoleMinIOServerEnv(tt.endpoint, tt.isTLS, tt.port)
|
||||||
|
if server != tt.wantServer {
|
||||||
|
t.Fatalf("server = %q, want %q", server, tt.wantServer)
|
||||||
|
}
|
||||||
|
if skipVerify != tt.wantSkipVerify {
|
||||||
|
t.Fatalf("skipVerify = %v, want %v", skipVerify, tt.wantSkipVerify)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The startup path clears process environment, so preserve all existing Console
|
||||||
|
// variables, including ones unrelated to this test, before exercising it.
|
||||||
|
func preserveConsoleEnvironment(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
for _, entry := range os.Environ() {
|
||||||
|
if strings.HasPrefix(entry, consolePrefix) {
|
||||||
|
name, value, _ := strings.Cut(entry, "=")
|
||||||
|
t.Setenv(name, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResetConsoleEnvironment(t *testing.T) {
|
||||||
|
preserveConsoleEnvironment(t)
|
||||||
|
settings := map[string]string{
|
||||||
|
consoleapi.ConsoleWSMaxConnections: "2048",
|
||||||
|
consoleapi.ConsoleWSMaxConnectionsPerClient: "512",
|
||||||
|
consoleapi.ConsoleWSMaxAnonymousConnections: "128",
|
||||||
|
consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient: "16",
|
||||||
|
}
|
||||||
|
for key, value := range settings {
|
||||||
|
t.Setenv(key, value)
|
||||||
|
}
|
||||||
|
decoys := []string{"CONSOLE_MINIO_SERVER_TLS_SKIP_VERIFY", "CONSOLE_MINIO_SERVER", "CONSOLE_PBKDF_SALT", "CONSOLE_TRUSTED_PROXIES", "CONSOLE_WS_MAX_UNKNOWN"}
|
||||||
|
for _, key := range decoys {
|
||||||
|
t.Setenv(key, "operator-value")
|
||||||
|
}
|
||||||
|
resetConsoleEnvironment()
|
||||||
|
for key, want := range settings {
|
||||||
|
if got, present := os.LookupEnv(key); !present || got != want {
|
||||||
|
t.Errorf("%s = %q, present = %v; want %q", key, got, present, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, key := range decoys {
|
||||||
|
if _, present := os.LookupEnv(key); present {
|
||||||
|
t.Errorf("unsupported override %s survived", key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, raw := range []string{"", " 16 ", "env://missing-limit"} {
|
||||||
|
t.Setenv(consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient, raw)
|
||||||
|
resetConsoleEnvironment()
|
||||||
|
if got, present := os.LookupEnv(consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient); !present || got != raw {
|
||||||
|
t.Fatalf("raw value %q was changed to %q (present = %v)", raw, got, present)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
os.Unsetenv(consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient)
|
||||||
|
resetConsoleEnvironment()
|
||||||
|
if _, present := os.LookupEnv(consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient); present {
|
||||||
|
t.Fatal("unset setting became present")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInitConsoleServerConfigurationErrors(t *testing.T) {
|
||||||
|
for _, tt := range []struct {
|
||||||
|
name, proxy, limit, want string
|
||||||
|
}{
|
||||||
|
{"proxy error precedes limit error", "proxy.internal", "bad", "MINIO_API_TRUSTED_PROXIES"},
|
||||||
|
{"blank limit", "", "", "CONSOLE_WS_MAX_ANONYMOUS_CONNECTIONS_PER_CLIENT"},
|
||||||
|
{"non-integer limit", "", "bad", "CONSOLE_WS_MAX_ANONYMOUS_CONNECTIONS_PER_CLIENT"},
|
||||||
|
{"out-of-range limit", "", "0", "CONSOLE_WS_MAX_ANONYMOUS_CONNECTIONS_PER_CLIENT"},
|
||||||
|
{"inconsistent limits", "", "256", "must be less than"},
|
||||||
|
} {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
// Restore the process-wide library configuration after environment cleanup.
|
||||||
|
t.Cleanup(func() {
|
||||||
|
_ = consoleapi.ConfigureEmbeddedSourceIPTrust()
|
||||||
|
_ = consoleapi.ConfigureWebSocketLimits()
|
||||||
|
})
|
||||||
|
preserveConsoleEnvironment(t)
|
||||||
|
t.Setenv(consoleapi.EnvMinIOTrustedProxies, tt.proxy)
|
||||||
|
t.Setenv(consoleapi.ConsoleWSMaxConnections, "1024")
|
||||||
|
t.Setenv(consoleapi.ConsoleWSMaxConnectionsPerClient, "256")
|
||||||
|
t.Setenv(consoleapi.ConsoleWSMaxAnonymousConnections, "64")
|
||||||
|
t.Setenv(consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient, tt.limit)
|
||||||
|
server, err := initConsoleServer()
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.want) || server != nil {
|
||||||
|
t.Fatalf("initConsoleServer() = %v, %v; want nil server and %q error", server, err, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,738 @@
|
|||||||
|
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||||
|
// Copyright (c) 2026 PGSTY
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"bytes"
|
||||||
|
"crypto/md5"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/xml"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strconv"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/klauspost/compress/s2"
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
"github.com/minio/minio/internal/crypto"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
"github.com/minio/minio/internal/kms"
|
||||||
|
)
|
||||||
|
|
||||||
|
// disableCompression turns the global compression config off and returns a
|
||||||
|
// restore func. It is the replication destination that applies no transform of
|
||||||
|
// its own; setCopyChecksumCompression covers the enabled cases.
|
||||||
|
func disableCompression() func() {
|
||||||
|
globalCompressConfigMu.Lock()
|
||||||
|
previous := globalCompressConfig
|
||||||
|
globalCompressConfig.Enabled = false
|
||||||
|
globalCompressConfigMu.Unlock()
|
||||||
|
|
||||||
|
return func() {
|
||||||
|
globalCompressConfigMu.Lock()
|
||||||
|
globalCompressConfig = previous
|
||||||
|
globalCompressConfigMu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ssecTestHeaders builds the customer key headers for a key made of the given
|
||||||
|
// repeated byte.
|
||||||
|
func ssecTestHeaders(b byte) map[string]string {
|
||||||
|
key := bytes.Repeat([]byte{b}, 32)
|
||||||
|
keyMD5 := md5.Sum(key)
|
||||||
|
return map[string]string{
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key),
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(keyMD5[:]),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertStoredSSECUncompressed requires the stored object to be SSE-C sealed
|
||||||
|
// and to carry no compression marker, so that "not compressed" is never
|
||||||
|
// reported for an object that is not encrypted either.
|
||||||
|
func assertStoredSSECUncompressed(t *testing.T, obj ObjectLayer, bucketName, object string) ObjectInfo {
|
||||||
|
t.Helper()
|
||||||
|
info, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, sealed := info.UserDefined[crypto.MetaSealedKeySSEC]; !sealed {
|
||||||
|
t.Fatalf("%s is not SSE-C sealed, the fixture proves nothing (userDefined=%v)", object, info.UserDefined)
|
||||||
|
}
|
||||||
|
if marker, compressed := info.UserDefined[ReservedMetadataPrefix+"compression"]; compressed {
|
||||||
|
t.Errorf("%s was stored as a compressed SSE-C object (compression=%q); such an object cannot be replicated",
|
||||||
|
object, marker)
|
||||||
|
}
|
||||||
|
return info
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertSSECPlaintext GETs an SSE-C object with its customer key and requires
|
||||||
|
// the body to equal the plaintext.
|
||||||
|
func assertSSECPlaintext(t *testing.T, apiRouter http.Handler, credentials auth.Credentials,
|
||||||
|
bucketName, object string, sseHeaders map[string]string, want []byte,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucketName, object),
|
||||||
|
0, nil, credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("GET %s status %d: %s", object, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if bytes.Equal(rec.Body.Bytes(), want) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
body := rec.Body.Bytes()
|
||||||
|
head := body
|
||||||
|
if len(head) > 16 {
|
||||||
|
head = head[:16]
|
||||||
|
}
|
||||||
|
t.Errorf("GET %s returned %d bytes, want the %d byte plaintext; first bytes % x",
|
||||||
|
object, len(body), len(want), head)
|
||||||
|
// Name the failure mode: a body that s2-decodes to the plaintext is the raw
|
||||||
|
// S2 stream of a compressed source shipped without its compression marker.
|
||||||
|
if decoded, derr := io.ReadAll(s2.NewReader(bytes.NewReader(body))); derr == nil && bytes.Equal(decoded, want) {
|
||||||
|
t.Errorf("the returned body is the raw S2 stream: s2-decoding it yields the %d byte plaintext", len(want))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAPISSECCompressionReplicaStaysReadable replicates an SSE-C object written
|
||||||
|
// with compression enabled and allow_encryption=on, and requires the replica to
|
||||||
|
// read back as the source plaintext.
|
||||||
|
//
|
||||||
|
// The source object is read the way the replication worker reads it
|
||||||
|
// (ReplicationRequest, hence NoDecryption), its wire headers come from the
|
||||||
|
// production option builder putReplicationOpts, and the replica is written the
|
||||||
|
// way a destination that applies no transform of its own stores it: compression
|
||||||
|
// off and no default encryption.
|
||||||
|
//
|
||||||
|
// Before the SSE-C compression exclusion the source was stored as
|
||||||
|
// encrypt(s2(plaintext)) while putReplicationOpts dropped
|
||||||
|
// X-Minio-Internal-compression, so the replica decrypted to an S2 stream and a
|
||||||
|
// correct-key GET returned HTTP 200 with the wrong body.
|
||||||
|
func TestAPISSECCompressionReplicaStaysReadable(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPISSECCompressionReplicaStaysReadable,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPISSECCompressionReplicaStaysReadable(obj ObjectLayer, instanceType, bucketName string,
|
||||||
|
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
previousTLS := globalIsTLS
|
||||||
|
globalIsTLS = true
|
||||||
|
defer func() { globalIsTLS = previousTLS }()
|
||||||
|
|
||||||
|
replicator := newObjectAttributesAuthzUser(t, instanceType, bucketName, `"s3:PutObject","s3:GetObject","s3:ReplicateObject"`)
|
||||||
|
sseHeaders := ssecTestHeaders(0x42)
|
||||||
|
// Highly compressible and comfortably above minCompressibleSize (4096).
|
||||||
|
data := bytes.Repeat([]byte("silo compressed ssec replication payload "), 8192)
|
||||||
|
|
||||||
|
t.Run(instanceType+"/single-put", func(t *testing.T) {
|
||||||
|
object := "replication/ssec-single.txt"
|
||||||
|
|
||||||
|
// --- Source side: compression ON with allow_encryption ON. ---
|
||||||
|
restore := setCopyChecksumCompression(true)
|
||||||
|
srcReq, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, object),
|
||||||
|
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||||
|
if err != nil {
|
||||||
|
restore()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
srcRec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(srcRec, srcReq)
|
||||||
|
if srcRec.Code != http.StatusOK {
|
||||||
|
restore()
|
||||||
|
t.Fatalf("source PUT status %d: %s", srcRec.Code, srcRec.Body.String())
|
||||||
|
}
|
||||||
|
sourceInfo := assertStoredSSECUncompressed(t, obj, bucketName, object)
|
||||||
|
t.Logf("source: stored size=%d compression=%q plaintext=%d", sourceInfo.Size,
|
||||||
|
sourceInfo.UserDefined[ReservedMetadataPrefix+"compression"], len(data))
|
||||||
|
|
||||||
|
// The replication worker's read: raw stored bytes, no decryption.
|
||||||
|
gr, err := obj.GetObjectNInfo(t.Context(), bucketName, object, nil, http.Header{},
|
||||||
|
ObjectOptions{ReplicationRequest: true})
|
||||||
|
if err != nil {
|
||||||
|
restore()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sourceInfo = gr.ObjInfo
|
||||||
|
raw, err := io.ReadAll(gr)
|
||||||
|
gr.Close()
|
||||||
|
if err != nil {
|
||||||
|
restore()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
replicationOpts, isMP, err := putReplicationOpts(t.Context(), "", sourceInfo)
|
||||||
|
if err != nil {
|
||||||
|
restore()
|
||||||
|
t.Fatalf("putReplicationOpts rejected the source: %v", err)
|
||||||
|
}
|
||||||
|
if isMP {
|
||||||
|
restore()
|
||||||
|
t.Fatal("single PUT source classified as multipart")
|
||||||
|
}
|
||||||
|
headers := map[string]string{}
|
||||||
|
for name, values := range replicationOpts.Header() {
|
||||||
|
if len(values) > 0 {
|
||||||
|
headers[name] = values[0]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
restore()
|
||||||
|
|
||||||
|
// --- Destination side: NO compression, NO default encryption. ---
|
||||||
|
restoreDst := disableCompression()
|
||||||
|
defer restoreDst()
|
||||||
|
|
||||||
|
replReq, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, object),
|
||||||
|
int64(len(raw)), bytes.NewReader(raw), replicator.AccessKey, replicator.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
replRec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(replRec, replReq)
|
||||||
|
if replRec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("replica PUT status %d: %s", replRec.Code, replRec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Logf("replica: stored size=%d compression=%q actual-size=%q", info.Size,
|
||||||
|
info.UserDefined[ReservedMetadataPrefix+"compression"],
|
||||||
|
info.UserDefined[ReservedMetadataPrefix+"actual-size"])
|
||||||
|
assertSSECPlaintext(t, apiRouter, credentials, bucketName, object, sseHeaders, data)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run(instanceType+"/multipart", func(t *testing.T) {
|
||||||
|
object := "replication/ssec-mpu.txt"
|
||||||
|
|
||||||
|
restore := setCopyChecksumCompression(true)
|
||||||
|
newReq, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
|
||||||
|
0, nil, credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||||
|
if err != nil {
|
||||||
|
restore()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
newRec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(newRec, newReq)
|
||||||
|
if newRec.Code != http.StatusOK {
|
||||||
|
restore()
|
||||||
|
t.Fatalf("source NewMultipart status %d: %s", newRec.Code, newRec.Body.String())
|
||||||
|
}
|
||||||
|
var sourceInit InitiateMultipartUploadResponse
|
||||||
|
if err = xmlDecoder(newRec.Body, &sourceInit, int64(newRec.Body.Len())); err != nil {
|
||||||
|
restore()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
partReq, err := newTestSignedRequestV4(http.MethodPut,
|
||||||
|
getPutObjectPartURL("", bucketName, object, sourceInit.UploadID, "1"),
|
||||||
|
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||||
|
if err != nil {
|
||||||
|
restore()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
partRec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(partRec, partReq)
|
||||||
|
if partRec.Code != http.StatusOK {
|
||||||
|
restore()
|
||||||
|
t.Fatalf("source PutPart status %d: %s", partRec.Code, partRec.Body.String())
|
||||||
|
}
|
||||||
|
completeBody, err := xml.Marshal(CompleteMultipartUpload{Parts: []CompletePart{
|
||||||
|
{PartNumber: 1, ETag: canonicalizeETag(partRec.Header()[xhttp.ETag][0])},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
restore()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
completeReq, err := newTestSignedRequestV4(http.MethodPost,
|
||||||
|
getCompleteMultipartUploadURL("", bucketName, object, sourceInit.UploadID),
|
||||||
|
int64(len(completeBody)), bytes.NewReader(completeBody), credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||||
|
if err != nil {
|
||||||
|
restore()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
completeRec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(completeRec, completeReq)
|
||||||
|
if completeRec.Code != http.StatusOK {
|
||||||
|
restore()
|
||||||
|
t.Fatalf("source Complete status %d: %s", completeRec.Code, completeRec.Body.String())
|
||||||
|
}
|
||||||
|
assertStoredSSECUncompressed(t, obj, bucketName, object)
|
||||||
|
|
||||||
|
gr, err := obj.GetObjectNInfo(t.Context(), bucketName, object, nil, http.Header{},
|
||||||
|
ObjectOptions{ReplicationRequest: true})
|
||||||
|
if err != nil {
|
||||||
|
restore()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sourceInfo := gr.ObjInfo
|
||||||
|
rawPart, err := io.ReadAll(gr)
|
||||||
|
gr.Close()
|
||||||
|
if err != nil {
|
||||||
|
restore()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
actualSize, err := sourceInfo.GetActualSize()
|
||||||
|
if err != nil {
|
||||||
|
restore()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Logf("source mpu: stored size=%d actual-size=%d rawRead=%d plaintext=%d compression=%q",
|
||||||
|
sourceInfo.Size, actualSize, len(rawPart), len(data),
|
||||||
|
sourceInfo.UserDefined[ReservedMetadataPrefix+"compression"])
|
||||||
|
|
||||||
|
replicationOpts, isMP, err := putReplicationOpts(t.Context(), "", sourceInfo)
|
||||||
|
if err != nil {
|
||||||
|
restore()
|
||||||
|
t.Fatalf("putReplicationOpts rejected the source: %v", err)
|
||||||
|
}
|
||||||
|
if !isMP {
|
||||||
|
restore()
|
||||||
|
t.Fatal("SSE-C multipart source not recognized as multipart")
|
||||||
|
}
|
||||||
|
replicationOpts.Internal.SourceMTime = time.Time{}
|
||||||
|
headers := map[string]string{}
|
||||||
|
for name, values := range replicationOpts.Header() {
|
||||||
|
if len(values) > 0 {
|
||||||
|
headers[name] = values[0]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
restore()
|
||||||
|
|
||||||
|
// --- Destination: no compression, no default encryption. ---
|
||||||
|
restoreDst := disableCompression()
|
||||||
|
defer restoreDst()
|
||||||
|
|
||||||
|
replNewReq, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
|
||||||
|
0, nil, replicator.AccessKey, replicator.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
replNewRec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(replNewRec, replNewReq)
|
||||||
|
if replNewRec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("replica NewMultipart status %d: %s", replNewRec.Code, replNewRec.Body.String())
|
||||||
|
}
|
||||||
|
var replicaInit InitiateMultipartUploadResponse
|
||||||
|
if err = xmlDecoder(replNewRec.Body, &replicaInit, int64(replNewRec.Body.Len())); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
replPartReq, err := newTestSignedRequestV4(http.MethodPut,
|
||||||
|
getPutObjectPartURL("", bucketName, object, replicaInit.UploadID, "1"),
|
||||||
|
int64(len(rawPart)), bytes.NewReader(rawPart), replicator.AccessKey, replicator.SecretKey,
|
||||||
|
map[string]string{xhttp.MinIOSourceReplicationRequest: "true"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
replPartRec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(replPartRec, replPartReq)
|
||||||
|
if replPartRec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("replica PutPart status %d: %s", replPartRec.Code, replPartRec.Body.String())
|
||||||
|
}
|
||||||
|
replCompleteBody, err := xml.Marshal(CompleteMultipartUpload{Parts: []CompletePart{
|
||||||
|
{PartNumber: 1, ETag: canonicalizeETag(replPartRec.Header()[xhttp.ETag][0])},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
replCompleteReq, err := newTestSignedRequestV4(http.MethodPost,
|
||||||
|
getCompleteMultipartUploadURL("", bucketName, object, replicaInit.UploadID),
|
||||||
|
int64(len(replCompleteBody)), bytes.NewReader(replCompleteBody), replicator.AccessKey, replicator.SecretKey,
|
||||||
|
map[string]string{
|
||||||
|
xhttp.MinIOSourceReplicationRequest: "true",
|
||||||
|
xhttp.MinIOSourceMTime: sourceInfo.ModTime.Format(time.RFC3339Nano),
|
||||||
|
xhttp.MinIOSourceETag: sourceInfo.ETag,
|
||||||
|
xhttp.MinIOReplicationActualObjectSize: strconv.FormatInt(actualSize, 10),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
replCompleteRec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(replCompleteRec, replCompleteReq)
|
||||||
|
if replCompleteRec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("replica Complete status %d: %s", replCompleteRec.Code, replCompleteRec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reportedActual, aerr := info.GetActualSize()
|
||||||
|
t.Logf("replica mpu: stored size=%d compression=%q actual-size=%q GetActualSize=%d(err=%v)",
|
||||||
|
info.Size, info.UserDefined[ReservedMetadataPrefix+"compression"],
|
||||||
|
info.UserDefined[ReservedMetadataPrefix+"actual-size"], reportedActual, aerr)
|
||||||
|
assertSSECPlaintext(t, apiRouter, credentials, bucketName, object, sseHeaders, data)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAPISSECCompressionProducerMatrix pins the scope of the exclusion across
|
||||||
|
// the PutObject and NewMultipartUpload producers: SSE-C is never compressed,
|
||||||
|
// while plaintext, SSE-S3 and SSE-KMS keep following allow_encryption.
|
||||||
|
func TestAPISSECCompressionProducerMatrix(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPISSECCompressionProducerMatrix,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPISSECCompressionProducerMatrix(obj ObjectLayer, instanceType, bucketName string,
|
||||||
|
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
previousTLS := globalIsTLS
|
||||||
|
globalIsTLS = true
|
||||||
|
defer func() { globalIsTLS = previousTLS }()
|
||||||
|
|
||||||
|
ssecHeaders := ssecTestHeaders(0x5a)
|
||||||
|
sseS3Headers := map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES}
|
||||||
|
sseKMSHeaders := map[string]string{
|
||||||
|
xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionKMS,
|
||||||
|
xhttp.AmzServerSideEncryptionKmsID: "compressed-ssec-producer-matrix",
|
||||||
|
}
|
||||||
|
previousKMS := GlobalKMS
|
||||||
|
GlobalKMS = kms.NewStub("compressed-ssec-producer-matrix")
|
||||||
|
defer func() { GlobalKMS = previousKMS }()
|
||||||
|
|
||||||
|
big := bytes.Repeat([]byte("silo producer matrix payload "), 8192)
|
||||||
|
small := bytes.Repeat([]byte("s"), 1024) // below minCompressibleSize
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
allowEncrypted bool
|
||||||
|
headers map[string]string
|
||||||
|
body []byte
|
||||||
|
wantCompressed bool
|
||||||
|
}{
|
||||||
|
// SSE-C is excluded from compression in both configurations, because the
|
||||||
|
// replication wire cannot carry the compression state.
|
||||||
|
{"ssec+allow_encryption-on+large", true, ssecHeaders, big, false},
|
||||||
|
{"ssec+allow_encryption-on+small", true, ssecHeaders, small, false},
|
||||||
|
{"ssec+allow_encryption-off+large", false, ssecHeaders, big, false},
|
||||||
|
// Plaintext still compresses in both configurations.
|
||||||
|
{"plain+allow_encryption-on+large", true, nil, big, true},
|
||||||
|
{"plain+allow_encryption-off+large", false, nil, big, true},
|
||||||
|
// SSE-S3 and SSE-KMS are the reason allow_encryption exists: the server
|
||||||
|
// owns the key, so the source decompresses before replicating.
|
||||||
|
{"sse-s3+allow_encryption-on+large", true, sseS3Headers, big, true},
|
||||||
|
{"sse-s3+allow_encryption-off+large", false, sseS3Headers, big, false},
|
||||||
|
{"sse-kms+allow_encryption-on+large", true, sseKMSHeaders, big, true},
|
||||||
|
{"sse-kms+allow_encryption-off+large", false, sseKMSHeaders, big, false},
|
||||||
|
} {
|
||||||
|
t.Run(instanceType+"/put/"+tc.name, func(t *testing.T) {
|
||||||
|
restore := setCopyChecksumCompression(tc.allowEncrypted)
|
||||||
|
defer restore()
|
||||||
|
object := "producer/" + tc.name + ".txt"
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, object),
|
||||||
|
int64(len(tc.body)), bytes.NewReader(tc.body), credentials.AccessKey, credentials.SecretKey, tc.headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("PUT status %d, want %d: %s", rec.Code, http.StatusOK, rec.Body.String())
|
||||||
|
}
|
||||||
|
info, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, compressed := info.UserDefined[ReservedMetadataPrefix+"compression"]
|
||||||
|
if compressed != tc.wantCompressed {
|
||||||
|
t.Errorf("compressed=%v, want %v (userDefined=%v)", compressed, tc.wantCompressed, info.UserDefined)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewMultipartUpload has no size gate, so the exclusion turns on the SSE-C
|
||||||
|
// and allow_encryption combination alone.
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
allowEncrypted bool
|
||||||
|
headers map[string]string
|
||||||
|
wantCompressed bool
|
||||||
|
}{
|
||||||
|
{"ssec+allow_encryption-on", true, ssecHeaders, false},
|
||||||
|
{"ssec+allow_encryption-off", false, ssecHeaders, false},
|
||||||
|
{"plain+allow_encryption-on", true, nil, true},
|
||||||
|
} {
|
||||||
|
t.Run(instanceType+"/mpu/"+tc.name, func(t *testing.T) {
|
||||||
|
restore := setCopyChecksumCompression(tc.allowEncrypted)
|
||||||
|
defer restore()
|
||||||
|
object := "producer/mpu-" + tc.name + ".txt"
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
|
||||||
|
0, nil, credentials.AccessKey, credentials.SecretKey, tc.headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("NewMultipartUpload status %d, want %d: %s", rec.Code, http.StatusOK, rec.Body.String())
|
||||||
|
}
|
||||||
|
var init InitiateMultipartUploadResponse
|
||||||
|
if err = xmlDecoder(rec.Body, &init, int64(rec.Body.Len())); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mi, err := obj.GetMultipartInfo(t.Context(), bucketName, object, init.UploadID, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, compressed := mi.UserDefined[ReservedMetadataPrefix+"compression"]
|
||||||
|
if compressed != tc.wantCompressed {
|
||||||
|
t.Errorf("upload compressed=%v, want %v", compressed, tc.wantCompressed)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAPISSECCompressionSkippedOnCopyObject covers the third producer:
|
||||||
|
// CopyObjectHandler decides compression before it encrypts, so a copy with a
|
||||||
|
// destination customer key and allow_encryption=on used to store a compressed
|
||||||
|
// SSE-C object from a plaintext source. It also covers the reverse direction,
|
||||||
|
// where only copy-source customer headers are present and compression must
|
||||||
|
// still apply.
|
||||||
|
func TestAPISSECCompressionSkippedOnCopyObject(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPISSECCompressionSkippedOnCopyObject,
|
||||||
|
endpoints: []string{"CopyObject", "PutObject", "GetObject"},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPISSECCompressionSkippedOnCopyObject(obj ObjectLayer, instanceType, bucketName string,
|
||||||
|
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
previousTLS := globalIsTLS
|
||||||
|
globalIsTLS = true
|
||||||
|
defer func() { globalIsTLS = previousTLS }()
|
||||||
|
|
||||||
|
ssecHeaders := ssecTestHeaders(0x7c)
|
||||||
|
data := bytes.Repeat([]byte("copy object compressed ssec payload "), 8192)
|
||||||
|
|
||||||
|
restore := setCopyChecksumCompression(true)
|
||||||
|
defer restore()
|
||||||
|
|
||||||
|
// An unencrypted source, stored compressed because compression is on. Only
|
||||||
|
// the copy adds encryption, so only the copy can change the decision.
|
||||||
|
src := "copysrc/plain.txt"
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, src),
|
||||||
|
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("source PUT status %d: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
dst := "copydst/ssec.txt"
|
||||||
|
copyHeaders := map[string]string{"X-Amz-Copy-Source": SlashSeparator + bucketName + SlashSeparator + src}
|
||||||
|
for k, v := range ssecHeaders {
|
||||||
|
copyHeaders[k] = v
|
||||||
|
}
|
||||||
|
copyReq, err := newTestSignedRequestV4(http.MethodPut, getCopyObjectURL("", bucketName, dst),
|
||||||
|
0, nil, credentials.AccessKey, credentials.SecretKey, copyHeaders)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
copyRec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(copyRec, copyReq)
|
||||||
|
if copyRec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("CopyObject status %d: %s", copyRec.Code, copyRec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
assertStoredSSECUncompressed(t, obj, bucketName, dst)
|
||||||
|
assertSSECPlaintext(t, apiRouter, credentials, bucketName, dst, ssecHeaders, data)
|
||||||
|
|
||||||
|
// The plaintext source is untouched by the copy and stays compressed.
|
||||||
|
srcInfo, err := obj.GetObjectInfo(t.Context(), bucketName, src, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, compressed := srcInfo.UserDefined[ReservedMetadataPrefix+"compression"]; !compressed {
|
||||||
|
t.Errorf("the plaintext copy source lost compression (userDefined=%v)", srcInfo.UserDefined)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The reverse direction: a copy-source customer key is not a destination
|
||||||
|
// key, so copying the SSE-C object on to a plaintext destination still
|
||||||
|
// compresses. crypto.SSEC.IsRequested ignores the copy-source headers.
|
||||||
|
plain := "copydst/decrypted.txt"
|
||||||
|
decryptHeaders := map[string]string{
|
||||||
|
"X-Amz-Copy-Source": SlashSeparator + bucketName + SlashSeparator + dst,
|
||||||
|
xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||||
|
xhttp.AmzServerSideEncryptionCopyCustomerKey: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKey],
|
||||||
|
xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKeyMD5],
|
||||||
|
}
|
||||||
|
decryptReq, err := newTestSignedRequestV4(http.MethodPut, getCopyObjectURL("", bucketName, plain),
|
||||||
|
0, nil, credentials.AccessKey, credentials.SecretKey, decryptHeaders)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
decryptRec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(decryptRec, decryptReq)
|
||||||
|
if decryptRec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("CopyObject to a plaintext destination status %d: %s", decryptRec.Code, decryptRec.Body.String())
|
||||||
|
}
|
||||||
|
plainInfo, err := obj.GetObjectInfo(t.Context(), bucketName, plain, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, sealed := plainInfo.UserDefined[crypto.MetaSealedKeySSEC]; sealed {
|
||||||
|
t.Fatalf("%s is still SSE-C sealed, the fixture proves nothing", plain)
|
||||||
|
}
|
||||||
|
if _, compressed := plainInfo.UserDefined[ReservedMetadataPrefix+"compression"]; !compressed {
|
||||||
|
t.Errorf("a copy carrying only copy-source SSE-C headers was not compressed (userDefined=%v)", plainInfo.UserDefined)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAPISSECCompressionSkippedOnSnowballExtract covers the fourth producer:
|
||||||
|
// PutObjectExtractHandler decides compression per entry before it encrypts, so
|
||||||
|
// a tar extract carrying customer key headers used to store every entry as a
|
||||||
|
// compressed SSE-C object.
|
||||||
|
func TestAPISSECCompressionSkippedOnSnowballExtract(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPISSECCompressionSkippedOnSnowballExtract,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPISSECCompressionSkippedOnSnowballExtract(obj ObjectLayer, instanceType, bucketName string,
|
||||||
|
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
previousTLS := globalIsTLS
|
||||||
|
globalIsTLS = true
|
||||||
|
defer func() { globalIsTLS = previousTLS }()
|
||||||
|
|
||||||
|
entry := "extracted/entry.txt"
|
||||||
|
payload := bytes.Repeat([]byte("snowball compressed ssec entry "), 4096)
|
||||||
|
|
||||||
|
var body bytes.Buffer
|
||||||
|
tw := tar.NewWriter(&body)
|
||||||
|
if err := tw.WriteHeader(&tar.Header{Name: entry, Mode: 0o600, Size: int64(len(payload))}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := tw.Write(payload); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := tw.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
restore := setCopyChecksumCompression(true)
|
||||||
|
defer restore()
|
||||||
|
|
||||||
|
ssecHeaders := ssecTestHeaders(0x2d)
|
||||||
|
headers := map[string]string{xhttp.AmzSnowballExtract: "true"}
|
||||||
|
for k, v := range ssecHeaders {
|
||||||
|
headers[k] = v
|
||||||
|
}
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, "snowball.tar"),
|
||||||
|
int64(body.Len()), bytes.NewReader(body.Bytes()), credentials.AccessKey, credentials.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("snowball extract status %d: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
assertStoredSSECUncompressed(t, obj, bucketName, entry)
|
||||||
|
assertSSECPlaintext(t, apiRouter, credentials, bucketName, entry, ssecHeaders, payload)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSSECBatchReplicationCannotRead is the control for the corruption path:
|
||||||
|
// batch replication reads without ReplicationRequest, so NoDecryption is never
|
||||||
|
// set and a non-empty SSE-C source fails at read time. Batch replication
|
||||||
|
// therefore cannot reach the replica shape in
|
||||||
|
// TestAPISSECCompressionReplicaStaysReadable; it cannot replicate a non-empty
|
||||||
|
// SSE-C object at all, compressed or not. A zero-byte object takes the reader
|
||||||
|
// shortcut, whose key check passes without a customer key.
|
||||||
|
func TestSSECBatchReplicationCannotRead(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testSSECBatchReplicationCannotRead,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testSSECBatchReplicationCannotRead(obj ObjectLayer, instanceType, bucketName string,
|
||||||
|
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
previousTLS := globalIsTLS
|
||||||
|
globalIsTLS = true
|
||||||
|
defer func() { globalIsTLS = previousTLS }()
|
||||||
|
|
||||||
|
ssecHeaders := ssecTestHeaders(0x6b)
|
||||||
|
data := bytes.Repeat([]byte("batch ssec payload "), 8192)
|
||||||
|
object := "batch/ssec-plain.txt"
|
||||||
|
|
||||||
|
restore := disableCompression()
|
||||||
|
defer restore()
|
||||||
|
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, object),
|
||||||
|
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, ssecHeaders)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("source PUT status %d: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// The read shape used by BatchJobReplicateV1.ReplicateToTarget and
|
||||||
|
// writeAsArchive: no ReplicationRequest, so NoDecryption is never set.
|
||||||
|
gr, err := obj.GetObjectNInfo(t.Context(), bucketName, object, nil, http.Header{}, ObjectOptions{})
|
||||||
|
if err == nil {
|
||||||
|
gr.Close()
|
||||||
|
t.Fatal("batch-shaped read of an SSE-C object unexpectedly succeeded")
|
||||||
|
}
|
||||||
|
t.Logf("batch-shaped read of an SSE-C object fails as expected: %v", err)
|
||||||
|
|
||||||
|
// Control: the replication worker's read shape succeeds and yields ciphertext.
|
||||||
|
gr2, err := obj.GetObjectNInfo(t.Context(), bucketName, object, nil, http.Header{},
|
||||||
|
ObjectOptions{ReplicationRequest: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("replication-shaped read failed: %v", err)
|
||||||
|
}
|
||||||
|
raw, err := io.ReadAll(gr2)
|
||||||
|
gr2.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if bytes.Equal(raw, data) {
|
||||||
|
t.Fatal("replication-shaped read returned plaintext")
|
||||||
|
}
|
||||||
|
t.Logf("replication-shaped read returns %d bytes of ciphertext (plaintext %d)", len(raw), len(data))
|
||||||
|
}
|
||||||
@@ -52,7 +52,7 @@ import (
|
|||||||
"github.com/minio/minio/internal/crypto"
|
"github.com/minio/minio/internal/crypto"
|
||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/pkg/v3/env"
|
"github.com/pgsty/silo-pkg/v3/env"
|
||||||
)
|
)
|
||||||
|
|
||||||
func initHelp() {
|
func initHelp() {
|
||||||
|
|||||||
@@ -33,8 +33,8 @@ import (
|
|||||||
"github.com/minio/minio/internal/config/storageclass"
|
"github.com/minio/minio/internal/config/storageclass"
|
||||||
"github.com/minio/minio/internal/event/target"
|
"github.com/minio/minio/internal/event/target"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
xnet "github.com/minio/pkg/v3/net"
|
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||||
"github.com/minio/pkg/v3/quick"
|
"github.com/pgsty/silo-pkg/v3/quick"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Save config file to corresponding backend
|
// Save config file to corresponding backend
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ import (
|
|||||||
"github.com/minio/minio/internal/config/policy/opa"
|
"github.com/minio/minio/internal/config/policy/opa"
|
||||||
"github.com/minio/minio/internal/config/storageclass"
|
"github.com/minio/minio/internal/config/storageclass"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/pkg/v3/quick"
|
"github.com/pgsty/silo-pkg/v3/quick"
|
||||||
)
|
)
|
||||||
|
|
||||||
// FileLogger is introduced to workaround the dependency about logrus
|
// FileLogger is introduced to workaround the dependency about logrus
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ import (
|
|||||||
"github.com/minio/minio/internal/logger/target/console"
|
"github.com/minio/minio/internal/logger/target/console"
|
||||||
types "github.com/minio/minio/internal/logger/target/loggertypes"
|
types "github.com/minio/minio/internal/logger/target/loggertypes"
|
||||||
"github.com/minio/minio/internal/pubsub"
|
"github.com/minio/minio/internal/pubsub"
|
||||||
xnet "github.com/minio/pkg/v3/net"
|
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||||
)
|
)
|
||||||
|
|
||||||
// number of log messages to buffer
|
// number of log messages to buffer
|
||||||
|
|||||||
+1
-1
@@ -42,7 +42,7 @@ import (
|
|||||||
"github.com/minio/minio/internal/config/heal"
|
"github.com/minio/minio/internal/config/heal"
|
||||||
"github.com/minio/minio/internal/event"
|
"github.com/minio/minio/internal/event"
|
||||||
xioutil "github.com/minio/minio/internal/ioutil"
|
xioutil "github.com/minio/minio/internal/ioutil"
|
||||||
"github.com/minio/pkg/v3/console"
|
"github.com/pgsty/silo-pkg/v3/console"
|
||||||
uatomic "go.uber.org/atomic"
|
uatomic "go.uber.org/atomic"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user