mirror of
https://github.com/pgsty/minio.git
synced 2026-08-09 15:53:28 +03:00
Compare commits
55 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 96794aff81 | |||
| d88f46ccee | |||
| 021110b451 | |||
| aa51393694 | |||
| e064b5555f | |||
| 2ca4971d91 | |||
| 4c185d5a66 | |||
| ca674a6967 | |||
| 3b8a55deef | |||
| 11d79fddc3 | |||
| 475236c79c | |||
| 1814ae52f4 | |||
| 632ade111b | |||
| 32863c8523 | |||
| cf7df097b2 | |||
| 10c7670b80 | |||
| 9c799f42d5 | |||
| 8eae745ab2 | |||
| b42ee4e8ac | |||
| 5f4513fd40 | |||
| dfe6698627 | |||
| 2602177ef6 | |||
| 9dd1dc172d | |||
| 0c14d81510 | |||
| 162ded3438 | |||
| fe6dc47804 | |||
| 744a9dcd71 | |||
| 2f55347f78 | |||
| 97b7d28040 | |||
| 22c1e41fd2 | |||
| 38366f6543 | |||
| 1af351a702 | |||
| b6f70ab085 | |||
| 80e8eaa423 | |||
| ca7baa670d | |||
| a36fd8fffb | |||
| 8069a32ac8 | |||
| 89d346bf51 | |||
| 9247179269 | |||
| 3e14733f15 | |||
| c8590413fd | |||
| 3f192f3f0c | |||
| 15fcc3c8ac | |||
| c1aec0518a | |||
| 7babc0c390 | |||
| b7f52ca433 | |||
| 4dfc27ce32 | |||
| ce01ccbdc1 | |||
| d495d30d57 | |||
| 3e61b1d3a5 | |||
| df627ff896 | |||
| 73ac524724 | |||
| fd69c89d05 | |||
| 5e40665acd | |||
| 65795ee1f4 |
+3
-56
@@ -28,59 +28,6 @@ archives:
|
||||
- minio
|
||||
name_template: "minio_{{ .Env.PKG_VERSION }}_{{ .Os }}_{{ .Arch }}"
|
||||
|
||||
dockers:
|
||||
- id: minio-amd64
|
||||
ids:
|
||||
- minio
|
||||
goos: linux
|
||||
goarch: amd64
|
||||
dockerfile: Dockerfile.goreleaser
|
||||
use: buildx
|
||||
image_templates:
|
||||
- "pgsty/minio:{{ .Tag }}-amd64"
|
||||
- "pgsty/minio:latest-amd64"
|
||||
build_flag_templates:
|
||||
- "--platform=linux/amd64"
|
||||
- "--label=org.opencontainers.image.version={{ .Tag }}"
|
||||
- "--label=org.opencontainers.image.created={{ .Date }}"
|
||||
- "--label=org.opencontainers.image.revision={{ .FullCommit }}"
|
||||
extra_files:
|
||||
- dockerscripts/docker-entrypoint.sh
|
||||
- dockerscripts/download-static-curl.sh
|
||||
- LICENSE
|
||||
- CREDITS
|
||||
|
||||
- id: minio-arm64
|
||||
ids:
|
||||
- minio
|
||||
goos: linux
|
||||
goarch: arm64
|
||||
dockerfile: Dockerfile.goreleaser
|
||||
use: buildx
|
||||
image_templates:
|
||||
- "pgsty/minio:{{ .Tag }}-arm64"
|
||||
- "pgsty/minio:latest-arm64"
|
||||
build_flag_templates:
|
||||
- "--platform=linux/arm64"
|
||||
- "--label=org.opencontainers.image.version={{ .Tag }}"
|
||||
- "--label=org.opencontainers.image.created={{ .Date }}"
|
||||
- "--label=org.opencontainers.image.revision={{ .FullCommit }}"
|
||||
extra_files:
|
||||
- dockerscripts/docker-entrypoint.sh
|
||||
- dockerscripts/download-static-curl.sh
|
||||
- LICENSE
|
||||
- CREDITS
|
||||
|
||||
docker_manifests:
|
||||
- name_template: "pgsty/minio:{{ .Tag }}"
|
||||
image_templates:
|
||||
- "pgsty/minio:{{ .Tag }}-amd64"
|
||||
- "pgsty/minio:{{ .Tag }}-arm64"
|
||||
- name_template: "pgsty/minio:latest"
|
||||
image_templates:
|
||||
- "pgsty/minio:latest-amd64"
|
||||
- "pgsty/minio:latest-arm64"
|
||||
|
||||
checksum:
|
||||
name_template: "minio_{{ .Env.PKG_VERSION }}_checksums.txt"
|
||||
algorithm: sha256
|
||||
@@ -89,10 +36,10 @@ release:
|
||||
github:
|
||||
owner: pgsty
|
||||
name: minio
|
||||
draft: false
|
||||
draft: true
|
||||
prerelease: false
|
||||
mode: replace
|
||||
replace_existing_artifacts: true
|
||||
mode: append
|
||||
replace_existing_artifacts: false
|
||||
name_template: "{{ .Tag }}"
|
||||
|
||||
changelog:
|
||||
|
||||
+6
-1
@@ -1 +1,6 @@
|
||||
<svg data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 162.612 24.465"><path d="M52.751.414h9.108v23.63h-9.108zM41.711.74l-18.488 9.92a.919.919 0 0 1-.856 0L3.879.74A2.808 2.808 0 0 0 2.558.414h-.023A2.4 2.4 0 0 0 0 2.641v21.376h9.1V13.842a.918.918 0 0 1 1.385-.682l10.361 5.568a3.634 3.634 0 0 0 3.336.028l10.933-5.634a.917.917 0 0 1 1.371.69v10.205h9.1V2.641A2.4 2.4 0 0 0 43.055.414h-.023a2.808 2.808 0 0 0-1.321.326zm65.564-.326h-9.237v10.755a.913.913 0 0 1-1.338.706L72.762.675a2.824 2.824 0 0 0-1.191-.261h-.016a2.4 2.4 0 0 0-2.535 2.227v21.377h9.163V13.275a.914.914 0 0 1 1.337-.707l24.032 11.2a2.813 2.813 0 0 0 1.188.26 2.4 2.4 0 0 0 2.535-2.227zm7.161 23.63V.414h4.191v23.63zm28.856.421c-11.274 0-19.272-4.7-19.272-12.232C124.02 4.741 132.066 0 143.292 0s19.32 4.7 19.32 12.233-7.902 12.232-19.32 12.232zm0-21.333c-8.383 0-14.84 3.217-14.84 9.1 0 5.926 6.457 9.1 14.84 9.1s14.887-3.174 14.887-9.1c0-5.883-6.504-9.1-14.887-9.1z" fill="#c72c48"/></svg>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="255 293 539 140" fill="none">
|
||||
<path d="M255.222 385.861 304.199 383.375c1.061 6.446 3.22 11.358 6.478 14.734 5.303 5.465 12.879 8.197 22.728 8.197 7.348 0 13.011-1.397 16.988-4.19 3.977-2.794 5.966-6.032 5.966-9.716 0-3.5-1.894-6.631-5.682-9.394-3.788-2.763-12.576-5.372-26.364-7.828-22.576-4.113-38.674-9.577-48.295-16.392-9.697-6.815-14.546-15.502-14.546-26.062 0-6.938 2.481-13.491 7.443-19.662 4.962-6.17 12.424-11.02 22.386-14.55 9.962-3.53 23.617-5.295 40.966-5.295 21.288 0 37.519 3.208 48.693 9.624 11.174 6.416 17.822 16.622 19.943 30.621l-48.523 2.302c-1.288-6.078-3.996-10.498-8.125-13.261-4.129-2.763-9.83-4.144-17.103-4.144-5.984 0-10.492 1.028-13.522 3.085-3.03 2.056-4.545 4.558-4.545 7.505 0 2.149 1.25 4.083 3.75 5.802 2.424 1.78 8.182 3.438 17.273 4.973 22.5 3.93 38.617 7.905 48.352 11.926 9.735 4.022 16.818 9.01 21.25 14.965 4.432 5.956 6.648 12.617 6.648 19.984 0 8.657-2.954 16.638-8.863 23.944-5.909 7.306-14.167 12.847-24.773 16.623-10.606 3.776-23.977 5.664-40.113 5.664-28.333 0-47.954-4.42-58.863-13.261-10.909-8.841-17.083-20.076-18.522-33.706Z" fill="#287CAB"/>
|
||||
<path d="M434.578 295.519h43.304v135.007h-43.304z" fill="#287CAB"/>
|
||||
<path d="M517.881 295.519h43.209v101.762h67.436v33.245H517.881Z" fill="#287CAB"/>
|
||||
<path d="M639.4 363.115c0-22.041 6.359-39.201 19.077-51.48 12.718-12.28 30.427-18.419 53.129-18.419 23.274 0 41.206 6.032 53.796 18.096 12.591 12.064 18.886 28.963 18.886 50.697 0 15.778-2.75 28.717-8.251 38.817-5.5 10.099-13.449 17.958-23.846 23.575-10.397 5.618-23.353 8.427-38.869 8.427-15.77 0-28.822-2.425-39.155-7.275-10.333-4.85-18.711-12.525-25.133-23.023-6.423-10.499-9.634-23.637-9.634-39.416Zm43.209.184c0 13.63 2.623 23.422 7.869 29.377 5.246 5.956 12.384 8.934 21.413 8.934 9.284 0 16.47-2.916 21.557-8.749 5.087-5.833 7.631-16.301 7.631-31.403 0-12.709-2.655-21.995-7.965-27.858-5.31-5.863-12.511-8.795-21.604-8.795-8.712 0-15.707 2.978-20.985 8.933-5.278 5.956-7.916 15.81-7.916 29.561Z" fill="#287CAB"/>
|
||||
</svg>
|
||||
|
||||
|
Before Width: | Height: | Size: 978 B After Width: | Height: | Size: 2.0 KiB |
@@ -0,0 +1,38 @@
|
||||
name: minio
|
||||
arch: ${NFPM_ARCH}
|
||||
platform: linux
|
||||
version: ${PKG_VERSION}
|
||||
version_schema: none
|
||||
release: ${NFPM_RELEASE}
|
||||
section: utils
|
||||
priority: optional
|
||||
maintainer: "Ruohang Feng (@Vonng) <rh@vonng.com>"
|
||||
description: S3-Interface Libre Object Storage, Community-maintained MinIO server fork.
|
||||
vendor: PGSTY
|
||||
homepage: https://silo.pgsty.com
|
||||
license: AGPL-3.0-or-later
|
||||
|
||||
contents:
|
||||
- src: ${NFPM_SOURCE}
|
||||
dst: /usr/local/bin/minio
|
||||
expand: true
|
||||
file_info:
|
||||
mode: 0755
|
||||
owner: root
|
||||
group: root
|
||||
- src: ${NFPM_UNIT}
|
||||
dst: /usr/lib/systemd/system/minio.service
|
||||
expand: true
|
||||
file_info:
|
||||
mode: 0644
|
||||
owner: root
|
||||
group: root
|
||||
|
||||
rpm:
|
||||
group: Applications/File
|
||||
compression: gzip:9
|
||||
|
||||
deb:
|
||||
compression: gzip
|
||||
fields:
|
||||
License: AGPL-3.0-or-later
|
||||
@@ -0,0 +1,30 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg"
|
||||
width="1624" height="570" viewBox="0 0 1994 700"
|
||||
preserveAspectRatio="xMidYMid meet"
|
||||
role="img" aria-labelledby="title desc"
|
||||
shape-rendering="geometricPrecision">
|
||||
<title id="title">SILO wordmark</title>
|
||||
<desc id="desc">The SILO wordmark set in Chakra Petch Bold, outlined, with the blue-to-copper brand gradient.</desc>
|
||||
|
||||
<defs>
|
||||
<linearGradient id="silo-wordmark-color" x1="44.0" y1="-94.4" x2="1950.0" y2="794.4" gradientUnits="userSpaceOnUse">
|
||||
<stop class="wm-a" offset="0.06" stop-color="#1d588c"/>
|
||||
<stop class="wm-b" offset="0.94" stop-color="#b4762e"/>
|
||||
</linearGradient>
|
||||
<style>
|
||||
/* Light theme values of --pg-strong / --copper; dark theme swaps in its own pair. */
|
||||
@media (prefers-color-scheme: dark) {
|
||||
.wm-a { stop-color: #7fb8e8; }
|
||||
.wm-b { stop-color: #e0a35c; }
|
||||
}
|
||||
</style>
|
||||
</defs>
|
||||
|
||||
<g fill="url(#silo-wordmark-color)" fill-rule="nonzero">
|
||||
<path id="silo-s" d="M0 592V492H134V551L167 584H374L408 550V434L375 401H110L2 293V108L110 0H426L534 108V209H400V149L367 116H169L136 149V252L169 285H434L542 393V590L432 700H108Z"/>
|
||||
<path id="silo-i" d="M637 0H773V700H637Z"/>
|
||||
<path id="silo-l" d="M888 0H1024V585H1374V700H888Z"/>
|
||||
<path id="silo-o" d="M1404 585V115L1519 0H1879L1994 115V585L1879 700H1519ZM1807 584 1858 533V167L1807 116H1591L1540 167V533L1591 584Z"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.4 KiB |
@@ -0,0 +1,82 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg"
|
||||
width="570" height="570" viewBox="230 213 570 570"
|
||||
preserveAspectRatio="xMidYMid meet"
|
||||
role="img" aria-labelledby="title desc"
|
||||
shape-rendering="geometricPrecision">
|
||||
<title id="title">SILO emblem</title>
|
||||
<desc id="desc">A smooth circular SILO mark with a peaked roof, flowing left wall, columns, and a curved base.</desc>
|
||||
|
||||
<defs>
|
||||
<linearGradient id="silo-color" x1="276" y1="720" x2="742" y2="286" gradientUnits="userSpaceOnUse">
|
||||
<stop offset="0" stop-color="#064A83"/>
|
||||
<stop offset="0.5" stop-color="#007FA8"/>
|
||||
<stop offset="1" stop-color="#22C7C9"/>
|
||||
</linearGradient>
|
||||
</defs>
|
||||
|
||||
<g fill="url(#silo-color)">
|
||||
<!-- Outer circular band, intentionally opened at the lower-right plinth. -->
|
||||
<path d="
|
||||
M 734 676
|
||||
A 283.5 278.5 0 1 0 310 692
|
||||
L 359 692
|
||||
A 247 248.5 0 1 1 688 676
|
||||
Z"/>
|
||||
|
||||
<!-- Flowing left wall; its upper tangent matches the inner ellipse. -->
|
||||
<path d="
|
||||
M 300.57 375
|
||||
C 292 390 300 430 328 450
|
||||
C 343 461 357 472 374 481
|
||||
C 410 501 426 517 426 544
|
||||
L 426 676
|
||||
L 336 676
|
||||
C 308 647 286 608 274 565
|
||||
C 262 522 263 479 272 439
|
||||
C 278 413 286 389 300.57 375
|
||||
Z"/>
|
||||
|
||||
<!-- Right column with tangent-continuous upper shoulder. -->
|
||||
<path d="
|
||||
M 602 373
|
||||
Q 602 368 607 370
|
||||
C 619 374 634 381 634 389
|
||||
L 634 647
|
||||
Q 634 649 636 649
|
||||
L 708 649
|
||||
L 734 676
|
||||
L 602 676
|
||||
Z"/>
|
||||
|
||||
<!-- Lower circular cap. -->
|
||||
<path d="
|
||||
M 310 692
|
||||
L 714 692
|
||||
C 668 743 596 774 512 774
|
||||
C 428 774 355 743 310 692
|
||||
Z"/>
|
||||
|
||||
<!-- Main silo body, with a tangent-continuous right shoulder. -->
|
||||
<path d="
|
||||
M 389 389
|
||||
C 389 374 447 351 512 351
|
||||
C 540 351 565 354 580 359
|
||||
Q 583 360 583 364
|
||||
L 583 676
|
||||
L 443 676
|
||||
L 443 541
|
||||
C 443 509 426 490 389 470
|
||||
Z"/>
|
||||
|
||||
<!-- Peaked roof with softly tapered, burr-free tips. -->
|
||||
<path d="
|
||||
M 512 274
|
||||
L 647 365
|
||||
Q 649 370 647 376
|
||||
C 609 350 563 337 512 337
|
||||
C 460 337 414 350 377 376
|
||||
Q 375 370 377 365
|
||||
Z"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.2 KiB |
@@ -0,0 +1,170 @@
|
||||
name: Publish Docker Image
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: "Published RELEASE.* tag to package as pgsty/minio"
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: docker-release
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# Images are built from a published release rather than from the build
|
||||
# that produced it, so an abandoned draft can never leave :latest
|
||||
# pointing at something nobody shipped.
|
||||
- name: Validate published release
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
INPUT_TAG: ${{ inputs.tag }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TAG="${INPUT_TAG}"
|
||||
VERSION_HYPHEN="${TAG#RELEASE.}"
|
||||
PKG_VERSION="$(echo "${VERSION_HYPHEN}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')"
|
||||
if [ "${PKG_VERSION}" = "${VERSION_HYPHEN}" ]; then
|
||||
echo "Invalid release tag: ${TAG}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
IS_DRAFT="$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft --jq .isDraft)"
|
||||
IS_PRERELEASE="$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json isPrerelease --jq .isPrerelease)"
|
||||
PUBLISHED_AT="$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json publishedAt --jq .publishedAt)"
|
||||
LATEST_TAG="$(gh release view --repo "${GITHUB_REPOSITORY}" --json tagName --jq .tagName)"
|
||||
|
||||
if [ "${IS_DRAFT}" != false ] || [ "${IS_PRERELEASE}" != false ] || [ -z "${PUBLISHED_AT}" ]; then
|
||||
echo "${TAG} must be a published, non-prerelease GitHub Release"
|
||||
exit 1
|
||||
fi
|
||||
# This workflow moves :latest, so it must not run for an older tag.
|
||||
if [ "${TAG}" != "${LATEST_TAG}" ]; then
|
||||
echo "Refusing to replace Docker latest with non-latest release ${TAG} (latest is ${LATEST_TAG})"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
{
|
||||
echo "RELEASE_TAG=${TAG}"
|
||||
echo "PKG_VERSION=${PKG_VERSION}"
|
||||
echo "PUBLISHED_AT=${PUBLISHED_AT}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Validate Docker Hub credentials
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${{ secrets.DOCKERHUB_USERNAME }}" ] || [ -z "${{ secrets.DOCKERHUB_TOKEN }}" ]; then
|
||||
echo "Missing Docker Hub credentials. Set DOCKERHUB_USERNAME and DOCKERHUB_TOKEN repository secrets."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Checkout release tag
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.tag }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Prepare verified Docker contexts
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
assets_dir="docker-release/assets"
|
||||
mkdir -p "${assets_dir}"
|
||||
|
||||
amd64_archive="minio_${PKG_VERSION}_linux_amd64.tar.gz"
|
||||
arm64_archive="minio_${PKG_VERSION}_linux_arm64.tar.gz"
|
||||
checksums="minio_${PKG_VERSION}_checksums.txt"
|
||||
gh release download "${RELEASE_TAG}" --repo "${GITHUB_REPOSITORY}" \
|
||||
--dir "${assets_dir}" \
|
||||
--pattern "${amd64_archive}" \
|
||||
--pattern "${arm64_archive}" \
|
||||
--pattern "${checksums}"
|
||||
|
||||
# The binaries going into the images are the published ones, checked
|
||||
# against the published checksums, not a rebuild that merely ought to
|
||||
# match them.
|
||||
cd "${assets_dir}"
|
||||
grep -F " ${amd64_archive}" "${checksums}" | sha256sum --check
|
||||
grep -F " ${arm64_archive}" "${checksums}" | sha256sum --check
|
||||
cd "${GITHUB_WORKSPACE}"
|
||||
|
||||
# Dockerfile.goreleaser expects the binary at the context root and
|
||||
# the entrypoint scripts under dockerscripts/, which is the layout
|
||||
# GoReleaser used to assemble via extra_files.
|
||||
for arch in amd64 arm64; do
|
||||
context="docker-release/${arch}"
|
||||
archive="${assets_dir}/minio_${PKG_VERSION}_linux_${arch}.tar.gz"
|
||||
mkdir -p "${context}/dockerscripts"
|
||||
tar -xzf "${archive}" -C "${context}" minio
|
||||
cp Dockerfile.goreleaser LICENSE CREDITS "${context}/"
|
||||
cp dockerscripts/docker-entrypoint.sh dockerscripts/download-static-curl.sh \
|
||||
"${context}/dockerscripts/"
|
||||
done
|
||||
|
||||
echo "RELEASE_REVISION=$(git rev-parse HEAD)" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
with:
|
||||
platforms: arm64
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Build and push amd64 image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: docker-release/amd64
|
||||
file: docker-release/amd64/Dockerfile.goreleaser
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
tags: |
|
||||
pgsty/minio:${{ env.RELEASE_TAG }}-amd64
|
||||
pgsty/minio:latest-amd64
|
||||
labels: |
|
||||
org.opencontainers.image.version=${{ env.RELEASE_TAG }}
|
||||
org.opencontainers.image.created=${{ env.PUBLISHED_AT }}
|
||||
org.opencontainers.image.revision=${{ env.RELEASE_REVISION }}
|
||||
|
||||
- name: Build and push arm64 image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: docker-release/arm64
|
||||
file: docker-release/arm64/Dockerfile.goreleaser
|
||||
platforms: linux/arm64
|
||||
push: true
|
||||
tags: |
|
||||
pgsty/minio:${{ env.RELEASE_TAG }}-arm64
|
||||
pgsty/minio:latest-arm64
|
||||
labels: |
|
||||
org.opencontainers.image.version=${{ env.RELEASE_TAG }}
|
||||
org.opencontainers.image.created=${{ env.PUBLISHED_AT }}
|
||||
org.opencontainers.image.revision=${{ env.RELEASE_REVISION }}
|
||||
|
||||
- name: Publish multi-architecture manifests
|
||||
run: |
|
||||
set -euo pipefail
|
||||
docker buildx imagetools create \
|
||||
--tag "pgsty/minio:${RELEASE_TAG}" \
|
||||
"pgsty/minio:${RELEASE_TAG}-amd64" \
|
||||
"pgsty/minio:${RELEASE_TAG}-arm64"
|
||||
docker buildx imagetools create \
|
||||
--tag "pgsty/minio:latest" \
|
||||
"pgsty/minio:latest-amd64" \
|
||||
"pgsty/minio:latest-arm64"
|
||||
docker buildx imagetools inspect "pgsty/minio:${RELEASE_TAG}"
|
||||
docker buildx imagetools inspect "pgsty/minio:latest"
|
||||
@@ -0,0 +1,149 @@
|
||||
name: Go CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
workflow_dispatch:
|
||||
|
||||
# Cancel superseded runs for the same PR; never cancel master push runs.
|
||||
# Keyed on PR number (not head_ref) so fork PRs sharing a branch name
|
||||
# do not collide.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
name: Format, Build, Vet
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- name: Check gofmt
|
||||
run: |
|
||||
unformatted=$(gofmt -l main.go cmd internal)
|
||||
if [ -n "${unformatted}" ]; then
|
||||
echo "The following files are not gofmt-formatted:"
|
||||
echo "${unformatted}"
|
||||
gofmt -d ${unformatted}
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Build
|
||||
env:
|
||||
CGO_ENABLED: 0
|
||||
run: go build ./...
|
||||
|
||||
- name: Vet
|
||||
run: go vet ./...
|
||||
|
||||
quality:
|
||||
name: Lint, Generated Files
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- name: Lint
|
||||
run: make lint
|
||||
|
||||
- name: Check generated files
|
||||
run: make check-gen
|
||||
|
||||
race-s3select:
|
||||
name: Race, S3 Select
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- name: Run S3 Select tests under race detector
|
||||
run: go test -race ./internal/s3select/... -count=1
|
||||
|
||||
crosscompile:
|
||||
name: Cross Compile
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- name: Build supported targets
|
||||
run: make crosscompile
|
||||
|
||||
test-internal:
|
||||
name: Test internal/
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
# internal/http listener tests bind [::1]; runners disable IPv6 by default.
|
||||
- name: Enable IPv6
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
|
||||
- name: Run internal tests
|
||||
env:
|
||||
CGO_ENABLED: 0
|
||||
MINIO_API_REQUESTS_MAX: "10000"
|
||||
run: go test ./internal/... -count=1
|
||||
|
||||
test-cmd:
|
||||
name: Test cmd/
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 35
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
# Some server tests bind IPv6 listeners; runners disable IPv6 by default.
|
||||
- name: Enable IPv6
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
|
||||
- name: Run cmd tests
|
||||
env:
|
||||
CGO_ENABLED: 0
|
||||
MINIO_API_REQUESTS_MAX: "10000"
|
||||
# cmd/ is one large package; raise go test's default 10m per-package
|
||||
# timeout so slower runners fail on the job timeout, not a panic.
|
||||
run: go test ./cmd/ -count=1 -timeout 30m
|
||||
@@ -12,7 +12,6 @@ on:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
release:
|
||||
@@ -22,6 +21,11 @@ jobs:
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
# Build the code at the tag being released, not whatever branch the
|
||||
# dispatch ran from. On a tag push this is the tag ref already; on
|
||||
# workflow_dispatch it pins the checkout to the requested tag so the
|
||||
# artifacts cannot be built from one ref and published under another.
|
||||
ref: ${{ github.event.inputs.tag || github.ref }}
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
@@ -29,16 +33,34 @@ jobs:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- name: Compute release variables
|
||||
- name: Verify clean checkout
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TAG="${{ github.event.inputs.tag || github.ref_name }}"
|
||||
VERSION_HYPHEN="${TAG#RELEASE.}"
|
||||
PKG_VERSION="$(echo "${VERSION_HYPHEN}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')"
|
||||
if [ "${PKG_VERSION}" = "${VERSION_HYPHEN}" ]; then
|
||||
echo "Invalid release tag format: ${TAG}"
|
||||
if [ -n "$(git status --porcelain)" ]; then
|
||||
echo "Refusing to release from a dirty working tree:" >&2
|
||||
git status --porcelain >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Compute release variables
|
||||
env:
|
||||
# Passed through the environment, never interpolated into the script
|
||||
# body: a dispatch input reaches bash as data, so it cannot inject
|
||||
# commands the way a `${{ ... }}` splice into the source would.
|
||||
INPUT_TAG: ${{ github.event.inputs.tag }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TAG="${INPUT_TAG:-${GITHUB_REF_NAME}}"
|
||||
# Whitelist the exact tag shape before the value is used anywhere. bash
|
||||
# =~ anchors ^...$ to the whole string (not per line, as sed would), so
|
||||
# a tag carrying a newline cannot pass and then smuggle extra lines into
|
||||
# $GITHUB_ENV below.
|
||||
if [[ ! "${TAG}" =~ ^RELEASE\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}Z$ ]]; then
|
||||
echo "Invalid release tag format: ${TAG}" >&2
|
||||
exit 1
|
||||
fi
|
||||
VERSION_HYPHEN="${TAG#RELEASE.}"
|
||||
PKG_VERSION="$(echo "${VERSION_HYPHEN}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')"
|
||||
VERSION_COLON="$(echo "${VERSION_HYPHEN}" | sed -E 's/T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/T\1:\2:\3Z/')"
|
||||
LDFLAGS="$(MINIO_RELEASE=RELEASE go run buildscripts/gen-ldflags.go "${VERSION_COLON}")"
|
||||
|
||||
@@ -52,29 +74,7 @@ jobs:
|
||||
echo "Package version: ${PKG_VERSION}"
|
||||
echo "LDFLAGS: ${LDFLAGS}"
|
||||
|
||||
- name: Validate Docker Hub credentials
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${{ secrets.DOCKERHUB_USERNAME }}" ] || [ -z "${{ secrets.DOCKERHUB_TOKEN }}" ]; then
|
||||
echo "Missing Docker Hub credentials. Set DOCKERHUB_USERNAME and DOCKERHUB_TOKEN repository secrets."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
with:
|
||||
platforms: arm64
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Build and publish with GoReleaser
|
||||
- name: Build Draft release with GoReleaser
|
||||
uses: goreleaser/goreleaser-action@v6
|
||||
with:
|
||||
version: "~> v2"
|
||||
@@ -84,54 +84,34 @@ jobs:
|
||||
LDFLAGS: ${{ env.LDFLAGS }}
|
||||
PKG_VERSION: ${{ env.PKG_VERSION }}
|
||||
|
||||
- name: Install pkger
|
||||
- name: Verify binary provenance stamps
|
||||
run: |
|
||||
go install github.com/minio/pkger/v2@v2.6.18
|
||||
set -euo pipefail
|
||||
buildscripts/verify-build-provenance.sh
|
||||
|
||||
- name: Install nFPM
|
||||
run: |
|
||||
set -euo pipefail
|
||||
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.47.0
|
||||
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Prepare package layout
|
||||
- name: Build nFPM packages
|
||||
run: |
|
||||
set -euo pipefail
|
||||
copy_binary() {
|
||||
local arch="$1"
|
||||
local pattern="$2"
|
||||
local src
|
||||
src="$(find dist -maxdepth 2 -type f -path "dist/${pattern}/minio" | head -n1 || true)"
|
||||
if [ -z "${src}" ]; then
|
||||
echo "Missing GoReleaser binary for ${arch} (${pattern})"
|
||||
exit 1
|
||||
fi
|
||||
mkdir -p "dist/linux-${arch}"
|
||||
cp "${src}" "dist/linux-${arch}/minio.${RELEASE_TAG}"
|
||||
}
|
||||
buildscripts/package-release.sh
|
||||
|
||||
copy_binary amd64 "minio_linux_amd64*"
|
||||
copy_binary arm64 "minio_linux_arm64*"
|
||||
|
||||
- name: Build standard pkger packages
|
||||
run: |
|
||||
set -euo pipefail
|
||||
pkger -r "${RELEASE_TAG}" --appName minio --releaseDir dist --ignore
|
||||
|
||||
# Keep only full package files; drop convenience symlinks (minio.rpm/minio.deb/minio.apk)
|
||||
find dist/linux-* -maxdepth 1 -type l \
|
||||
\( -name 'minio.rpm' -o -name 'minio.deb' -o -name 'minio.apk' \) -delete
|
||||
|
||||
find dist -maxdepth 2 -type f \
|
||||
\( -name '*.rpm' -o -name '*.deb' -o -name '*.apk' -o -name '*.sha256sum' -o -name 'downloads-minio.json' \) | sort
|
||||
|
||||
- name: Upload pkger artifacts to GitHub release
|
||||
- name: Upload nFPM packages to Draft release
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mapfile -t files < <(find dist -maxdepth 2 -type f \
|
||||
\( -name '*.rpm' -o -name '*.deb' -o -name '*.apk' -o -name '*.sha256sum' -o -name 'downloads-minio.json' \) | sort)
|
||||
mapfile -t files < <(find dist/packages -maxdepth 1 -type f \
|
||||
\( -name '*.rpm' -o -name '*.deb' -o -name '*.apk' -o -name '*.sha256sum' \) | sort)
|
||||
if [ "${#files[@]}" -eq 0 ]; then
|
||||
echo "No packages were generated."
|
||||
exit 1
|
||||
fi
|
||||
gh release upload "${RELEASE_TAG}" "${files[@]}" --clobber
|
||||
gh release upload "${RELEASE_TAG}" "${files[@]}"
|
||||
|
||||
- name: Upload dist artifact
|
||||
if: always()
|
||||
|
||||
@@ -5,10 +5,17 @@ on:
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/goreleaser.yml"
|
||||
- ".github/nfpm.yml"
|
||||
- "Dockerfile.goreleaser"
|
||||
- "dockerscripts/docker-entrypoint.sh"
|
||||
- "minio.service"
|
||||
- "buildscripts/package-release.sh"
|
||||
- "buildscripts/sign-release-rpms.sh"
|
||||
- "buildscripts/verify-build-provenance.sh"
|
||||
- "buildscripts/gen-ldflags.go"
|
||||
- ".github/workflows/release.yml"
|
||||
- ".github/workflows/test-release.yml"
|
||||
- ".gitignore"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -35,9 +42,11 @@ jobs:
|
||||
VERSION_COLON="2026-02-14T12:00:00Z"
|
||||
PKG_VERSION="20260214120000.0.0"
|
||||
LDFLAGS="$(MINIO_RELEASE=RELEASE go run buildscripts/gen-ldflags.go "${VERSION_COLON}")"
|
||||
echo "RELEASE_TAG=${RELEASE_TAG}" >> "${GITHUB_ENV}"
|
||||
echo "PKG_VERSION=${PKG_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "LDFLAGS=${LDFLAGS}" >> "${GITHUB_ENV}"
|
||||
{
|
||||
echo "RELEASE_TAG=${RELEASE_TAG}"
|
||||
echo "PKG_VERSION=${PKG_VERSION}"
|
||||
echo "LDFLAGS=${LDFLAGS}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
echo "PKG_VERSION: ${PKG_VERSION}"
|
||||
echo "LDFLAGS: ${LDFLAGS}"
|
||||
|
||||
@@ -56,34 +65,213 @@ jobs:
|
||||
LDFLAGS: ${{ env.LDFLAGS }}
|
||||
PKG_VERSION: ${{ env.PKG_VERSION }}
|
||||
|
||||
- name: Install pkger
|
||||
run: |
|
||||
go install github.com/minio/pkger/v2@v2.6.18
|
||||
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Package snapshot binaries with pkger
|
||||
- name: Verify binary provenance stamps
|
||||
run: |
|
||||
set -euo pipefail
|
||||
copy_binary() {
|
||||
local arch="$1"
|
||||
local pattern="$2"
|
||||
local src
|
||||
src="$(find dist -maxdepth 2 -type f -path "dist/${pattern}/minio" | head -n1 || true)"
|
||||
if [ -z "${src}" ]; then
|
||||
echo "Missing GoReleaser binary for ${arch} (${pattern})"
|
||||
exit 1
|
||||
buildscripts/verify-build-provenance.sh
|
||||
|
||||
- name: Install package validation tools
|
||||
run: |
|
||||
set -euo pipefail
|
||||
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.47.0
|
||||
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
|
||||
sudo apt-get update
|
||||
sudo apt-get install --yes rpm binutils
|
||||
|
||||
- name: Package snapshot binaries with nFPM
|
||||
run: |
|
||||
set -euo pipefail
|
||||
buildscripts/package-release.sh
|
||||
|
||||
- name: Validate package names, checksums, metadata, and payload
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cd dist/packages
|
||||
|
||||
# These are the public download names; a drift here breaks every
|
||||
# script that fetches packages by URL.
|
||||
expected=(
|
||||
"minio-${PKG_VERSION}-1.aarch64.rpm"
|
||||
"minio-${PKG_VERSION}-1.x86_64.rpm"
|
||||
"minio_${PKG_VERSION}_aarch64.apk"
|
||||
"minio_${PKG_VERSION}_amd64.deb"
|
||||
"minio_${PKG_VERSION}_arm64.deb"
|
||||
"minio_${PKG_VERSION}_x86_64.apk"
|
||||
)
|
||||
|
||||
for package in "${expected[@]}"; do
|
||||
test -s "${package}"
|
||||
test -s "${package}.sha256sum"
|
||||
sha256sum --check "${package}.sha256sum"
|
||||
done
|
||||
|
||||
test "$(find . -maxdepth 1 -type f \( -name '*.rpm' -o -name '*.deb' -o -name '*.apk' \) | wc -l)" -eq 6
|
||||
|
||||
# The signing script asserts these same values, but it runs on the
|
||||
# maintainer's machine after the release workflow has already built
|
||||
# and uploaded. Take its expectations as the single source of truth
|
||||
# so nfpm.yml and the signing script cannot drift apart without
|
||||
# failing here first, while a fix is still cheap.
|
||||
#
|
||||
# This grep is deliberately limited to the seven identity variables,
|
||||
# all of which are single-line. That is what makes the eval safe:
|
||||
# should one ever become multi-line, the grep captures an
|
||||
# unterminated quote and the eval aborts on a syntax error under
|
||||
# set -e rather than quietly binding an empty value and comparing
|
||||
# against nothing. expected_payload is multi-line by design and must
|
||||
# stay out of this set for the same reason.
|
||||
eval "$(grep -E '^expected_(vendor|packager|url|summary|description|license|group)=' \
|
||||
../../buildscripts/sign-release-rpms.sh)"
|
||||
for value in "${expected_vendor}" "${expected_packager}" "${expected_url}" \
|
||||
"${expected_summary}" "${expected_description}" \
|
||||
"${expected_license}" "${expected_group}"; do
|
||||
test -n "${value}"
|
||||
done
|
||||
|
||||
service_sha="$(sha256sum ../../minio.service | awk '{print $1}')"
|
||||
|
||||
rpm_file="minio-${PKG_VERSION}-1.x86_64.rpm"
|
||||
test "$(rpm -qp --queryformat '%{VENDOR}' "${rpm_file}")" = "${expected_vendor}"
|
||||
test "$(rpm -qp --queryformat '%{PACKAGER}' "${rpm_file}")" = "${expected_packager}"
|
||||
test "$(rpm -qp --queryformat '%{URL}' "${rpm_file}")" = "${expected_url}"
|
||||
test "$(rpm -qp --queryformat '%{SUMMARY}' "${rpm_file}")" = "${expected_summary}"
|
||||
test "$(rpm -qp --queryformat '%{DESCRIPTION}' "${rpm_file}")" = "${expected_description}"
|
||||
test "$(rpm -qp --queryformat '%{LICENSE}' "${rpm_file}")" = "${expected_license}"
|
||||
test "$(rpm -qp --queryformat '%{GROUP}' "${rpm_file}")" = "${expected_group}"
|
||||
# Both payload entries: the unit file is as much a part of the
|
||||
# package as the binary, and losing it would install a server with
|
||||
# nothing to start it.
|
||||
rpm -qpl "${rpm_file}" | grep -Fx '/usr/local/bin/minio'
|
||||
rpm -qpl "${rpm_file}" | grep -Fx '/usr/lib/systemd/system/minio.service'
|
||||
test "$(rpm -qpl "${rpm_file}" | wc -l)" -eq 2
|
||||
|
||||
deb_file="minio_${PKG_VERSION}_amd64.deb"
|
||||
test "$(dpkg-deb --field "${deb_file}" Maintainer)" = "${expected_packager}"
|
||||
test "$(dpkg-deb --field "${deb_file}" Version)" = "${PKG_VERSION}"
|
||||
test "$(dpkg-deb --field "${deb_file}" License)" = "${expected_license}"
|
||||
test "$(dpkg-deb --field "${deb_file}" Section)" = "utils"
|
||||
test "$(dpkg-deb --field "${deb_file}" Homepage)" = "${expected_url}"
|
||||
test "$(dpkg-deb --field "${deb_file}" Description)" = "${expected_description}"
|
||||
dpkg-deb --contents "${deb_file}" | grep -E 'usr/local/bin/minio$'
|
||||
dpkg-deb --contents "${deb_file}" | grep -E 'usr/lib/systemd/system/minio\.service$'
|
||||
|
||||
apk_info="$(tar -xOzf "minio_${PKG_VERSION}_x86_64.apk" .PKGINFO)"
|
||||
grep -Fx "pkgver = ${PKG_VERSION}" <<< "${apk_info}"
|
||||
grep -Fx "url = ${expected_url}" <<< "${apk_info}"
|
||||
grep -Fx "maintainer = ${expected_packager}" <<< "${apk_info}"
|
||||
grep -Fx "license = ${expected_license}" <<< "${apk_info}"
|
||||
grep -Fx "pkgdesc = ${expected_description}" <<< "${apk_info}"
|
||||
tar -tzf "minio_${PKG_VERSION}_x86_64.apk" | grep -Fx 'usr/local/bin/minio'
|
||||
tar -tzf "minio_${PKG_VERSION}_x86_64.apk" | grep -Fx 'usr/lib/systemd/system/minio.service'
|
||||
|
||||
for arch in amd64 arm64; do
|
||||
if [ "${arch}" = amd64 ]; then
|
||||
rpm_arch=x86_64
|
||||
deb_arch=amd64
|
||||
apk_arch=x86_64
|
||||
else
|
||||
rpm_arch=aarch64
|
||||
deb_arch=arm64
|
||||
apk_arch=aarch64
|
||||
fi
|
||||
mkdir -p "dist/linux-${arch}"
|
||||
cp "${src}" "dist/linux-${arch}/minio.${RELEASE_TAG}"
|
||||
|
||||
test "$(rpm -qp --queryformat '%{ARCH}' "minio-${PKG_VERSION}-1.${rpm_arch}.rpm")" = "${rpm_arch}"
|
||||
test "$(dpkg-deb --field "minio_${PKG_VERSION}_${deb_arch}.deb" Architecture)" = "${deb_arch}"
|
||||
grep -Fx "arch = ${apk_arch}" <<< "$(tar -xOzf "minio_${PKG_VERSION}_${apk_arch}.apk" .PKGINFO)"
|
||||
|
||||
# Accepted weakness: this takes the first match, unsorted, where
|
||||
# find_binary in package-release.sh demands exactly one. It cannot
|
||||
# be reached with an ambiguous match today, because packaging runs
|
||||
# earlier in this same job and hard-fails on one. Revisit if
|
||||
# goamd64 gains a second level, or if find_binary's exactly-one
|
||||
# contract is ever relaxed -- at that point this weak copy would be
|
||||
# the only one left choosing silently.
|
||||
source_binary="$(find .. -maxdepth 2 -type f -path "../minio_linux_${arch}*/minio" | head -n 1)"
|
||||
source_sha="$(sha256sum "${source_binary}" | awk '{print $1}')"
|
||||
|
||||
# Do not pipe rpm2cpio here: Debian's build exits non-zero even when
|
||||
# it writes a correct payload, which trips `set -o pipefail`. Use
|
||||
# rpm's own digests instead -- -K checks the payload against the
|
||||
# header, and FILEDIGESTS is the sha256 rpm itself verifies on
|
||||
# install.
|
||||
rpm -K "minio-${PKG_VERSION}-1.${rpm_arch}.rpm"
|
||||
rpm_sha="$(rpm -qp --queryformat '[%{FILENAMES} %{FILEDIGESTS}\n]' \
|
||||
"minio-${PKG_VERSION}-1.${rpm_arch}.rpm" | awk '$1 == "/usr/local/bin/minio" { print $2 }')"
|
||||
rpm_service_sha="$(rpm -qp --queryformat '[%{FILENAMES} %{FILEDIGESTS}\n]' \
|
||||
"minio-${PKG_VERSION}-1.${rpm_arch}.rpm" | awk '$1 == "/usr/lib/systemd/system/minio.service" { print $2 }')"
|
||||
deb_sha="$(ar p "minio_${PKG_VERSION}_${deb_arch}.deb" data.tar.gz | tar -xzOf - ./usr/local/bin/minio | sha256sum | awk '{print $1}')"
|
||||
deb_service_sha="$(ar p "minio_${PKG_VERSION}_${deb_arch}.deb" data.tar.gz | tar -xzOf - ./usr/lib/systemd/system/minio.service | sha256sum | awk '{print $1}')"
|
||||
apk_sha="$(tar -xzOf "minio_${PKG_VERSION}_${apk_arch}.apk" usr/local/bin/minio | sha256sum | awk '{print $1}')"
|
||||
apk_service_sha="$(tar -xzOf "minio_${PKG_VERSION}_${apk_arch}.apk" usr/lib/systemd/system/minio.service | sha256sum | awk '{print $1}')"
|
||||
|
||||
test "${source_sha}" = "${rpm_sha}"
|
||||
test "${source_sha}" = "${deb_sha}"
|
||||
test "${source_sha}" = "${apk_sha}"
|
||||
test "${service_sha}" = "${rpm_service_sha}"
|
||||
test "${service_sha}" = "${deb_service_sha}"
|
||||
test "${service_sha}" = "${apk_service_sha}"
|
||||
done
|
||||
|
||||
find . -maxdepth 1 -type f | sort
|
||||
|
||||
- name: Build release runtime image and verify graceful shutdown
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# docker-release.yml is workflow_dispatch only, so this is the only
|
||||
# automated build of the release runtime layer and entrypoint before a
|
||||
# real publish. Assemble a minimal image from the linux/amd64 binary
|
||||
# goreleaser already produced; the mcli-download build stage is skipped
|
||||
# on purpose to keep this gate offline and deterministic.
|
||||
ctx="$(mktemp -d)"
|
||||
tar -xzf "dist/minio_${PKG_VERSION}_linux_amd64.tar.gz" -C "${ctx}" minio
|
||||
cp dockerscripts/docker-entrypoint.sh "${ctx}/docker-entrypoint.sh"
|
||||
{
|
||||
echo "FROM registry.access.redhat.com/ubi9/ubi-micro:latest"
|
||||
echo "COPY minio /usr/bin/minio"
|
||||
echo "COPY docker-entrypoint.sh /usr/bin/docker-entrypoint.sh"
|
||||
echo "RUN mkdir -p /data && chmod 0777 /data && chmod +x /usr/bin/minio /usr/bin/docker-entrypoint.sh"
|
||||
echo 'ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]'
|
||||
echo 'CMD ["minio"]'
|
||||
} > "${ctx}/Dockerfile"
|
||||
docker build -t minio-runtime-test:snapshot "${ctx}"
|
||||
|
||||
# PID 1 must be minio, not the entry shell, on every privilege path, so
|
||||
# a SIGTERM from docker stop reaches the server and it exits gracefully
|
||||
# instead of being killed at the stop timeout. Regression guard for the
|
||||
# exec-into-chroot entrypoint fix.
|
||||
assert_graceful() {
|
||||
name="$1"; shift
|
||||
docker rm -f "${name}" >/dev/null 2>&1 || true
|
||||
docker run -d --name "${name}" \
|
||||
-e MINIO_CI_CD=1 -e MINIO_ROOT_USER=ciadmin -e MINIO_ROOT_PASSWORD=ciadmin-secret-123 \
|
||||
"$@" minio-runtime-test:snapshot minio server /data --address :9000 >/dev/null
|
||||
up=""
|
||||
for _ in $(seq 1 60); do
|
||||
if docker logs "${name}" 2>&1 | grep -q "API:"; then up=1; break; fi
|
||||
if [ "$(docker inspect -f '{{.State.Running}}' "${name}")" != "true" ]; then break; fi
|
||||
sleep 1
|
||||
done
|
||||
if [ -z "${up}" ]; then echo "server did not start (${name}):"; docker logs "${name}" | tail -5; exit 1; fi
|
||||
pid1="$(docker exec "${name}" cat /proc/1/comm 2>/dev/null || echo '?')"
|
||||
start="$(date +%s)"; docker stop -t 15 "${name}" >/dev/null; end="$(date +%s)"
|
||||
code="$(docker inspect -f '{{.State.ExitCode}}' "${name}")"
|
||||
elapsed=$((end - start))
|
||||
echo "${name}: pid1=${pid1} stop=${elapsed}s exit=${code}"
|
||||
graceful=0; docker logs "${name}" 2>&1 | grep -q "Exiting on signal" && graceful=1
|
||||
docker rm -f "${name}" >/dev/null 2>&1 || true
|
||||
[ "${graceful}" = "1" ] || { echo "no graceful-shutdown log (${name}) - signal not forwarded"; exit 1; }
|
||||
[ "${code}" = "0" ] || { echo "non-zero exit (${name}): ${code}"; exit 1; }
|
||||
[ "${elapsed}" -lt 10 ] || { echo "shutdown too slow (${name}): ${elapsed}s - signal not forwarded"; exit 1; }
|
||||
}
|
||||
assert_graceful minio-rt-default
|
||||
assert_graceful minio-rt-dropuser -e MINIO_USERNAME=minio-user -e MINIO_GROUPNAME=minio-group
|
||||
|
||||
copy_binary amd64 "minio_linux_amd64*"
|
||||
copy_binary arm64 "minio_linux_arm64*"
|
||||
pkger -r "${RELEASE_TAG}" --appName minio --releaseDir dist --ignore
|
||||
|
||||
# Keep only full package files; drop convenience symlinks (minio.rpm/minio.deb/minio.apk)
|
||||
find dist/linux-* -maxdepth 1 -type l \
|
||||
\( -name 'minio.rpm' -o -name 'minio.deb' -o -name 'minio.apk' \) -delete
|
||||
|
||||
find dist -maxdepth 2 -type f \
|
||||
\( -name '*.rpm' -o -name '*.deb' -o -name '*.apk' -o -name '*.sha256sum' -o -name 'downloads-minio.json' \) | sort
|
||||
- name: Validate release scripts
|
||||
run: |
|
||||
set -euo pipefail
|
||||
bash -n buildscripts/package-release.sh
|
||||
bash -n buildscripts/sign-release-rpms.sh
|
||||
bash -n buildscripts/verify-build-provenance.sh
|
||||
test -x buildscripts/package-release.sh
|
||||
test -x buildscripts/sign-release-rpms.sh
|
||||
test -x buildscripts/verify-build-provenance.sh
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
name: VulnCheck
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
vulncheck:
|
||||
name: Analysis
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- name: Install govulncheck
|
||||
run: |
|
||||
go install golang.org/x/vuln/cmd/govulncheck@v1.6.0
|
||||
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Run govulncheck
|
||||
run: govulncheck -show verbose ./...
|
||||
+3
-1
@@ -55,12 +55,14 @@ xattr
|
||||
xl-meta
|
||||
|
||||
.gitignore
|
||||
.goreleaser.yml
|
||||
|
||||
dist/
|
||||
|
||||
|
||||
.claude/
|
||||
.codex/
|
||||
AGENTS.md
|
||||
CLAUDE.md
|
||||
_bmad/
|
||||
_bmad-output/
|
||||
docs/security/
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.26.2-alpine AS build
|
||||
FROM golang:1.26.5-alpine AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.26.2-alpine as build
|
||||
FROM golang:1.26.5-alpine as build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.26.2-alpine AS build
|
||||
FROM golang:1.26.5-alpine AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.26.2-alpine AS build
|
||||
FROM golang:1.26.5-alpine AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
|
||||
@@ -4,6 +4,7 @@ LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
|
||||
|
||||
GOOS ?= $(shell go env GOOS)
|
||||
GOARCH ?= $(shell go env GOARCH)
|
||||
GOLANGCI_VERSION ?= v2.11.3
|
||||
|
||||
VERSION ?= $(shell git describe --tags)
|
||||
REPO ?= quay.io/minio
|
||||
@@ -23,7 +24,14 @@ help: ## print this help
|
||||
|
||||
getdeps: ## fetch necessary dependencies
|
||||
@mkdir -p ${GOPATH}/bin
|
||||
@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR)
|
||||
@if [ ! -x "$(GOLANGCI)" ]; then \
|
||||
set -e; \
|
||||
echo "Installing golangci-lint $(GOLANGCI_VERSION)"; \
|
||||
script=$$(mktemp); \
|
||||
trap 'rm -f "$$script"' EXIT; \
|
||||
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/$(GOLANGCI_VERSION)/install.sh -o "$$script"; \
|
||||
sh "$$script" -b $(GOLANGCI_DIR) $(GOLANGCI_VERSION); \
|
||||
fi
|
||||
|
||||
crosscompile: ## cross compile minio
|
||||
@(env bash $(PWD)/buildscripts/cross-compile.sh)
|
||||
@@ -33,13 +41,23 @@ verifiers: lint check-gen
|
||||
check-gen: ## check for updated autogenerated files
|
||||
@go generate ./... >/dev/null
|
||||
@go mod tidy -compat=1.26
|
||||
@(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false)
|
||||
@(! git diff --name-only | grep 'go.sum') || (echo "Non-committed changes in auto-generated go.sum is detected, please commit them to proceed." && false)
|
||||
@changed=$$(git diff --name-only -- '*_gen.go' '*_gen_test.go' '*_msgp_test.go' '*_string.go' go.mod go.sum); \
|
||||
if [ -n "$$changed" ]; then \
|
||||
echo "Non-committed generated changes detected:"; \
|
||||
echo "$$changed"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@untracked=$$(git ls-files --others --exclude-standard -- '*_gen.go' '*_gen_test.go' '*_msgp_test.go' '*_string.go'); \
|
||||
if [ -n "$$untracked" ]; then \
|
||||
echo "Untracked generated files detected:"; \
|
||||
echo "$$untracked"; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
lint: getdeps ## runs golangci-lint suite of linters
|
||||
@echo "Running $@ check"
|
||||
@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml
|
||||
@command typos && typos ./ || echo "typos binary is not found.. skipping.."
|
||||
@if command -v typos >/dev/null 2>&1; then typos ./; else echo "typos binary is not found.. skipping.."; fi
|
||||
|
||||
lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes
|
||||
@echo "Running $@ check"
|
||||
|
||||
@@ -1,31 +1,165 @@
|
||||
# Silo (Community maintained fork of MinIO)
|
||||
> [!WARNING]
|
||||
> **This branch is archived and receives no further changes.**
|
||||
>
|
||||
> `minio` holds the final state of this project under the MinIO identity. Development continues on **[`main`](https://github.com/pgsty/silo/tree/main)**, where the project is named **Silo** and the binary, packages, systemd unit, container image, and Helm chart are all named `silo`. On 2026-08-06 the repository was renamed `pgsty/minio` → **[`pgsty/silo`](https://github.com/pgsty/silo)** and its default branch `master` → `main`.
|
||||
>
|
||||
> The last release cut from this branch is **[`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z)** (2026-08-04). Its 19 assets carry the `minio` artifact names, and the matching container image is `docker.io/pgsty/minio:RELEASE.2026-08-04T00-00-00Z`. Those artifacts stay published and unmodified — no tag is moved, re-signed, or removed. If you need the artifacts maintained under the original MinIO identity, this branch and the releases up to that tag are where they live. Later releases carry the `silo` names.
|
||||
>
|
||||
> **Nothing on the wire changed with the rename.** `MINIO_*` environment variables, `minio_*` Prometheus metrics, `x-minio-*` headers, `/minio/*` routes, the `.minio.sys` on-disk layout, IAM and ARN values, and the `github.com/minio/minio` Go module path are all preserved on `main`. A Silo server reads data written by this release, and the packages install side by side, so migrating or rolling back stays an explicit administrator action.
|
||||
|
||||
[](https://silo.pigsty.io)
|
||||
[](https://silo.pigsty.cc)
|
||||
[](https://github.com/pgsty/minio)
|
||||
[](https://github.com/pgsty/mc)
|
||||
[](https://github.com/pgsty/minio-docs)
|
||||
[](https://hub.docker.com/r/pgsty/minio)
|
||||
<h1 align="center">
|
||||
<img src=".github/silo-word.svg" alt="SILO" height="80">
|
||||
</h1>
|
||||
|
||||
|
||||
<p align="center">
|
||||
<strong>A conservatively maintained MinIO fork</strong><br>
|
||||
Security maintenance, versioned release artifacts, and operational continuity for existing deployments.
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://silo.pgsty.com/">Website</a> ·
|
||||
<a href="https://silo.pgsty.com/docs/">Documentation</a> ·
|
||||
<a href="https://silo.pgsty.com/download/">Download</a> ·
|
||||
<a href="https://silo.pgsty.com/blog/">Blog</a> ·
|
||||
<a href="https://github.com/pgsty/minio/releases">Releases</a> ·
|
||||
<a href="SECURITY.md">Security</a> ·
|
||||
<a href="README_ZH.md">中文</a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/pgsty/minio/releases"><img alt="GitHub Release" src="https://img.shields.io/github/v/release/pgsty/minio?include_prereleases&label=release&logo=github"></a>
|
||||
<a href="https://hub.docker.com/r/pgsty/minio"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/minio?logo=docker"></a>
|
||||
<a href="go.mod"><img alt="Go Version" src="https://img.shields.io/github/go-mod/go-version/pgsty/minio?logo=go"></a>
|
||||
<a href="LICENSE"><img alt="License" src="https://img.shields.io/badge/license-AGPLv3-blue"></a>
|
||||
</p>
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **This is a community-maintained fork of the upstream MinIO project, maintained by [Pigsty](https://pigsty.io).**
|
||||
> This project is **NOT** affiliated with, endorsed by, or sponsored by MinIO, Inc.
|
||||
> "MinIO" is a trademark of MinIO, Inc., used here solely to identify the upstream project.
|
||||
>
|
||||
> Changes from upstream are minimal:
|
||||
> - Restored the embedded management console version reference
|
||||
> - Updated documentation links and Go module paths to point to this repository
|
||||
>
|
||||
> Distributed under the original [GNU AGPLv3](LICENSE) license.
|
||||
> Silo is an independent, community-maintained fork of the open-source MinIO server, published by [Pigsty](https://pigsty.io) from [`pgsty/minio`](https://github.com/pgsty/minio). It is not affiliated with, endorsed by, or sponsored by MinIO, Inc. “MinIO” is used only to identify the upstream project and compatibility lineage.
|
||||
|
||||
Documentation: [English](https://silo.pigsty.io) | [简体中文](https://silo.pigsty.cc)
|
||||
## Overview
|
||||
|
||||
Docker Hub: [https://hub.docker.com/r/pgsty/minio](https://hub.docker.com/r/pgsty/minio) / [https://hub.docker.com/r/pgsty/mc](https://hub.docker.com/r/pgsty/mc)
|
||||
Silo maintains one downstream release line based on MinIO [`RELEASE.2025-12-03T12-00-00Z`](https://github.com/minio/minio/releases/tag/RELEASE.2025-12-03T12-00-00Z). It provides maintained builds and release artifacts for existing MinIO-compatible deployments after upstream community distribution ended. Pigsty uses this fork for object storage as an optional PG backup repo.
|
||||
|
||||
Client Repo: [`pgsty/mc`](https://github.com/pgsty/mc) CLI.
|
||||
The official project portal is [silo.pgsty.com](https://silo.pgsty.com/). It brings documentation, downloads, release and security notes, and project background together. English is served at the site root; Chinese is available under [/zh/](https://silo.pgsty.com/zh/).
|
||||
|
||||
Console: [`georgmangold/console`](https://github.com/georgmangold/console/), a community-maintained fork of restored console.
|
||||
## Find the Right Resource
|
||||
|
||||
Ansible Deployment: [https://pigsty.io/docs/minio](https://pigsty.io/docs/minio)
|
||||
| Looking for | Canonical location |
|
||||
| :-- | :-- |
|
||||
| Project overview and navigation | [Silo Website](https://silo.pgsty.com/) ([中文](https://silo.pgsty.com/zh/)) |
|
||||
| Installation methods and downloads | [Download & Install](https://silo.pgsty.com/download/) ([中文](https://silo.pgsty.com/zh/download/)) |
|
||||
| Operations, administration, development, and reference | [Documentation](https://silo.pgsty.com/docs/) ([中文](https://silo.pgsty.com/zh/docs/)) |
|
||||
| Project news, release notes, and security notes | [Blog](https://silo.pgsty.com/blog/), including [releases](https://silo.pgsty.com/blog/release/) and [security](https://silo.pgsty.com/blog/security/) |
|
||||
| Versioned binaries, checksums, and source archives | [GitHub Releases](https://github.com/pgsty/minio/releases) |
|
||||
| Bug reports and feature discussions | [GitHub Issues](https://github.com/pgsty/minio/issues) |
|
||||
| License, attribution, and trademark information | [License](https://silo.pgsty.com/about/license/), [Attribution](https://silo.pgsty.com/about/attribution/), and [Trademark](https://silo.pgsty.com/about/trademark/) |
|
||||
|
||||
APT/YUM repo for `minio` and `mcli` binary: [https://pigsty.io/docs/infra](https://pigsty.io/docs/repo/infra/list/#object-storage)
|
||||
## Maintenance Policy
|
||||
|
||||
The active release line covers:
|
||||
|
||||
- build and dependency maintenance;
|
||||
- applicable security fixes and advisories;
|
||||
- focused fixes for reproducible defects;
|
||||
- versioned binaries, packages, checksums, and multi-architecture images;
|
||||
- the web console, client, documentation, and Pigsty integration.
|
||||
|
||||
Changes are kept narrow and tested where practical. Maintenance is best effort; no response, remediation, or release schedule is guaranteed.
|
||||
|
||||
### Out of scope
|
||||
|
||||
- a separate product roadmap, new storage engine, or speculative S3 features;
|
||||
- broad rewrites or changes that materially expand the downstream delta;
|
||||
- historical releases or multiple support branches;
|
||||
- commercial support, SLAs, 24×7 coverage, or SUBNET access;
|
||||
- deployment design, access control, monitoring, backup, or recovery.
|
||||
|
||||
## Compatibility
|
||||
|
||||
Silo aims to preserve:
|
||||
|
||||
- MinIO-compatible S3 APIs, configuration, environment variables, and CLI conventions;
|
||||
- `RELEASE.YYYY-MM-DDTHH-MM-SSZ` tags, container entrypoints, and common deployment workflows.
|
||||
|
||||
Compatibility is the default constraint. Silo preserves existing wire, client, configuration, and operational behavior whenever doing so remains safe. Compatibility is broken only when necessary to close a major security issue, and the release notes must identify the affected behavior and migration path. Treat each release as a downstream upgrade: pin versions, review [release notes](https://silo.pgsty.com/blog/release/) and [security advisories](docs/security/advisories.md), keep a rollback path, and test before production use.
|
||||
|
||||
## Downloads and Release Artifacts
|
||||
|
||||
Use [Download & Install](https://silo.pgsty.com/download/) to choose an installation method. GitHub Releases remains the source for versioned server binaries, checksums, and source archives.
|
||||
|
||||
| Artifact | Location |
|
||||
| :-- | :-- |
|
||||
| Source | [`github.com/pgsty/minio`](https://github.com/pgsty/minio) |
|
||||
| Container image | [`pgsty/minio`](https://hub.docker.com/r/pgsty/minio), multi-arch for `linux/amd64` and `linux/arm64` |
|
||||
| Server binaries and checksums | [GitHub Releases](https://github.com/pgsty/minio/releases) for Linux, macOS, and Windows on `amd64` and `arm64` |
|
||||
| Linux packages | RPM, DEB, and APK artifacts, also distributed through the [Pigsty repository](https://pigsty.io/docs/repo/) |
|
||||
| Client | [`pgsty/mc`](https://github.com/pgsty/mc), bundled in the container as `mcli` with an `mc` compatibility alias |
|
||||
| Console | Maintained [`georgmangold/console`](https://github.com/georgmangold/console) fork, embedded in the server build |
|
||||
| Shared library | [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) v3.7.0, consumed through a `replace` directive while preserving `github.com/minio/pkg/v3` import paths ([release notes](https://silo.pgsty.com/blog/release/pkg-3.7.0/)) |
|
||||
|
||||
## Quick Start
|
||||
|
||||
For local evaluation:
|
||||
|
||||
```bash
|
||||
mkdir -p data
|
||||
|
||||
export MINIO_ROOT_USER=minioadmin
|
||||
export MINIO_ROOT_PASSWORD=change-me-long-password
|
||||
|
||||
docker run -d --name silo \
|
||||
-p 9000:9000 \
|
||||
-p 9001:9001 \
|
||||
-e MINIO_ROOT_USER \
|
||||
-e MINIO_ROOT_PASSWORD \
|
||||
-v "$PWD/data:/data" \
|
||||
pgsty/minio:latest server /data --console-address ":9001"
|
||||
```
|
||||
|
||||
Open the console at <http://localhost:9001>; the S3 API listens on <http://localhost:9000>.
|
||||
|
||||
The image includes the compatible client as `mcli`:
|
||||
|
||||
```bash
|
||||
docker exec silo mcli alias set local http://127.0.0.1:9000 \
|
||||
"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
|
||||
docker exec silo mcli mb local/demo
|
||||
docker exec silo mcli ls local
|
||||
```
|
||||
|
||||
> [!WARNING]
|
||||
> For production, pin a release, use unique credentials and TLS, monitor the service, keep independent backups, and test recovery.
|
||||
|
||||
Build the server from source:
|
||||
|
||||
```bash
|
||||
go build -o minio .
|
||||
./minio --version
|
||||
```
|
||||
|
||||
For other installation paths—including native packages, binaries, Podman, Kubernetes, source, and Pigsty Ansible—use [Download & Install](https://silo.pgsty.com/download/). For production deployment and administration, start with the [Silo documentation](https://silo.pgsty.com/docs/). Pigsty users can also use the [Pigsty MinIO module](https://pigsty.io/docs/minio/).
|
||||
|
||||
## Security
|
||||
|
||||
Security fixes target the active `master` branch and are recorded in the [advisory log](docs/security/advisories.md) and the portal's [security notes](https://silo.pgsty.com/blog/security/). Report vulnerabilities privately as described in [`SECURITY.md`](SECURITY.md) and [`VULNERABILITY_REPORT.md`](VULNERABILITY_REPORT.md). Report issues that also affect upstream MinIO there as well.
|
||||
|
||||
## Contributing
|
||||
|
||||
Useful contributions include security and dependency updates, reproducible bug fixes, tests, release automation, packaging, and documentation.
|
||||
|
||||
Issues and pull requests should include the affected version, reproduction steps, impact, expected behavior, tests, and compatibility notes. Discuss large changes in an issue first.
|
||||
|
||||
## Background
|
||||
|
||||
This project was created in response to changes in the upstream community distribution and maintenance model. The maintainer’s analysis, alternatives considered, and early maintenance record are documented below:
|
||||
|
||||
| Essay | Subject |
|
||||
| :-- | :-- |
|
||||
| [MinIO Is Dead](https://silo.pgsty.com/blog/post/minio-is-dead/) | Changes to the upstream project and distribution model |
|
||||
| [MinIO Is Dead, Long Live MinIO](https://silo.pgsty.com/blog/post/minio-resurrect/) | Establishing the fork and its release pipeline |
|
||||
| [Two months into maintaining a MinIO fork](https://silo.pgsty.com/blog/post/minio-promise-kept/) | Initial security and maintenance work |
|
||||
|
||||
## License and Trademark
|
||||
|
||||
The server remains licensed under the [GNU Affero General Public License v3.0](LICENSE). See [`CREDITS`](CREDITS) for upstream authorship and attribution. MinIO is a trademark of MinIO, Inc. Silo and `pgsty/minio` are independent community efforts and are not affiliated with or endorsed by MinIO, Inc.
|
||||
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
> [!WARNING]
|
||||
> **本分支已归档,不再接受任何修改。**
|
||||
>
|
||||
> `minio` 分支保留本项目以 MinIO 身份存在的最后状态。开发已迁移到 **[`main`](https://github.com/pgsty/silo/tree/main)** 分支,项目在那里以 **Silo** 的身份继续维护,二进制、软件包、systemd unit、容器镜像与 Helm chart 全部更名为 `silo`。2026-08-06,仓库由 `pgsty/minio` 更名为 **[`pgsty/silo`](https://github.com/pgsty/silo)**,默认分支由 `master` 更名为 `main`。
|
||||
>
|
||||
> 本分支上切出的最后一个版本是 **[`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z)**(2026-08-04)。它的 19 个资产使用 `minio` 命名,对应容器镜像为 `docker.io/pgsty/minio:RELEASE.2026-08-04T00-00-00Z`。这些产物保持已发布状态且不做改动 —— 不移动、不重新签名、不删除任何 tag。如果你需要以原本 MinIO 形态维持的归档构件,就在本分支以及截止到该 tag 的历次发布中。此后的版本使用 `silo` 命名。
|
||||
>
|
||||
> **更名没有改变任何对外接口。** `MINIO_*` 环境变量、`minio_*` 指标、`x-minio-*` 头、`/minio/*` 路由、`.minio.sys` 磁盘布局、IAM 与 ARN 取值,以及 Go 模块路径 `github.com/minio/minio`,在 `main` 上全部原样保留。Silo 服务端可直接读取本版本写入的数据;新旧软件包并存安装,因此迁移与回滚始终是管理员显式触发的动作。
|
||||
|
||||
<h1 align="center">
|
||||
<img src=".github/silo.svg" alt="" height="80">
|
||||
<img src=".github/silo-word.svg" alt="SILO" height="80">
|
||||
</h1>
|
||||
|
||||
<p align="center">
|
||||
<strong>审慎维护的 MinIO 社区分支</strong><br>
|
||||
为现有部署提供安全维护、带版本的发行产物与持续运维支持。
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://silo.pgsty.com/zh/">官网</a> ·
|
||||
<a href="https://silo.pgsty.com/zh/docs/">文档</a> ·
|
||||
<a href="https://silo.pgsty.com/zh/download/">下载</a> ·
|
||||
<a href="https://silo.pgsty.com/zh/blog/">博客</a> ·
|
||||
<a href="https://github.com/pgsty/minio/releases">版本发布</a> ·
|
||||
<a href="SECURITY.md">安全策略</a> ·
|
||||
<a href="README.md">English</a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/pgsty/minio/releases"><img alt="GitHub Release" src="https://img.shields.io/github/v/release/pgsty/minio?include_prereleases&label=release&logo=github"></a>
|
||||
<a href="https://hub.docker.com/r/pgsty/minio"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/minio?logo=docker"></a>
|
||||
<a href="go.mod"><img alt="Go Version" src="https://img.shields.io/github/go-mod/go-version/pgsty/minio?logo=go"></a>
|
||||
<a href="LICENSE"><img alt="License" src="https://img.shields.io/badge/license-AGPLv3-blue"></a>
|
||||
</p>
|
||||
|
||||
> [!IMPORTANT]
|
||||
> Silo 是由 [Pigsty](https://pigsty.cc) 独立维护、从 [`pgsty/minio`](https://github.com/pgsty/minio) 发布的开源 MinIO 社区分支。本项目与 MinIO, Inc. 不存在隶属、背书或赞助关系;文中使用 “MinIO” 仅用于说明上游项目及兼容谱系。
|
||||
|
||||
## 概述
|
||||
|
||||
Silo 维护一条基于 MinIO [`RELEASE.2025-12-03T12-00-00Z`](https://github.com/minio/minio/releases/tag/RELEASE.2025-12-03T12-00-00Z) 的下游版本线,为上游停止社区发行后仍在运行 MinIO 兼容部署的用户提供持续构建与发行产物。
|
||||
|
||||
Pigsty 使用本分支提供对象存储,包括 PostgreSQL 备份存储。
|
||||
|
||||
项目统一门户为 [silo.pgsty.com](https://silo.pgsty.com/zh/),集中提供文档、下载安装、版本与安全动态及项目背景。中文内容位于 `/zh/`,英文内容位于站点根路径。
|
||||
|
||||
## 按需求选择入口
|
||||
|
||||
| 需求 | 权威入口 |
|
||||
| :-- | :-- |
|
||||
| 项目概览与全站导航 | [Silo 中文门户](https://silo.pgsty.com/zh/)([English](https://silo.pgsty.com/)) |
|
||||
| 安装方式与软件下载 | [下载与安装](https://silo.pgsty.com/zh/download/)([English](https://silo.pgsty.com/download/)) |
|
||||
| 运维、管理、开发与参考指南 | [中文文档](https://silo.pgsty.com/zh/docs/)([English](https://silo.pgsty.com/docs/)) |
|
||||
| 项目动态、版本说明与安全通告 | [博客](https://silo.pgsty.com/zh/blog/),包括[版本发布](https://silo.pgsty.com/zh/blog/release/)与[安全通告](https://silo.pgsty.com/zh/blog/security/) |
|
||||
| 带版本的二进制、校验和与源码归档 | [GitHub Releases](https://github.com/pgsty/minio/releases) |
|
||||
| 缺陷报告与功能讨论 | [GitHub Issues](https://github.com/pgsty/minio/issues) |
|
||||
| 私密漏洞报告 | [`SECURITY.md`](SECURITY.md) 与 [`VULNERABILITY_REPORT.md`](VULNERABILITY_REPORT.md) |
|
||||
| 许可证、署名与商标信息 | [许可证](https://silo.pgsty.com/zh/about/license/)、[署名归属](https://silo.pgsty.com/zh/about/attribution/)与[商标政策](https://silo.pgsty.com/zh/about/trademark/) |
|
||||
|
||||
## 维护政策
|
||||
|
||||
活跃版本线的维护范围包括:
|
||||
|
||||
- 构建与依赖项维护;
|
||||
- 适用的安全修复与公告;
|
||||
- 针对可复现缺陷的范围明确的修复;
|
||||
- 带版本的二进制、软件包、校验和与多架构镜像;
|
||||
- Web Console、客户端、文档与 Pigsty 集成。
|
||||
|
||||
改动保持克制,并在可行时提供测试。所有维护均为尽力而为,不承诺固定的响应、修复或发布时间。
|
||||
|
||||
### 范围之外
|
||||
|
||||
- 独立产品路线图、新存储引擎或假设性的 S3 新特性;
|
||||
- 大规模重写或显著扩大下游差异的改动;
|
||||
- 历史版本或多条支持分支;
|
||||
- 商业支持、SLA、7×24 服务或 SUBNET 服务;
|
||||
- 部署设计、访问控制、监控、备份与恢复。
|
||||
|
||||
## 兼容策略
|
||||
|
||||
Silo 尽量保留:
|
||||
|
||||
- `minio` 可执行文件与 `github.com/minio/minio` module path;
|
||||
- MinIO 兼容的 S3 API、配置、环境变量与命令行约定;
|
||||
- `RELEASE.YYYY-MM-DDTHH-MM-SSZ` 标签、容器入口与常见部署方式。
|
||||
|
||||
兼容性是默认约束。只要不会留下安全问题,Silo 就保留既有的协议、客户端、配置与运维行为;只有在修复重大安全问题确有必要时才会打破兼容,并在版本说明中明确受影响行为与迁移方式。每个版本仍应视为下游升级:锁定版本,阅读[版本说明](https://silo.pgsty.com/zh/blog/release/)与[安全公告](docs/security/advisories.md),保留回滚路径,并在生产使用前完成测试。
|
||||
|
||||
## 下载与发行产物
|
||||
|
||||
请先在[下载与安装](https://silo.pgsty.com/zh/download/)页面选择合适的安装方式;GitHub Releases 仍是带版本服务端二进制、校验和与源码归档的获取位置。
|
||||
|
||||
| 产物 | 位置 |
|
||||
| :-- | :-- |
|
||||
| 源码 | [`github.com/pgsty/minio`](https://github.com/pgsty/minio) |
|
||||
| 容器镜像 | [`pgsty/minio`](https://hub.docker.com/r/pgsty/minio),支持 `linux/amd64` 与 `linux/arm64` 多架构清单 |
|
||||
| 服务端二进制与校验和 | [GitHub Releases](https://github.com/pgsty/minio/releases),覆盖 Linux、macOS、Windows 的 `amd64` 与 `arm64` |
|
||||
| Linux 软件包 | RPM、DEB、APK,并通过 [Pigsty 软件仓库](https://pigsty.cc/docs/repo/) 分发 |
|
||||
| 客户端 | [`pgsty/mc`](https://github.com/pgsty/mc),容器内以 `mcli` 提供,并保留 `mc` 兼容别名 |
|
||||
| 管理控制台 | 社区维护的 [`georgmangold/console`](https://github.com/georgmangold/console),嵌入服务端构建 |
|
||||
| 共享库 | [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) v3.7.0,通过 `replace` 指令使用,同时保留 `github.com/minio/pkg/v3` 导入路径([版本说明](https://silo.pgsty.com/zh/blog/release/pkg-3.7.0/)) |
|
||||
|
||||
## 快速开始
|
||||
|
||||
本地体验:
|
||||
|
||||
```bash
|
||||
mkdir -p data
|
||||
|
||||
export MINIO_ROOT_USER=minioadmin
|
||||
export MINIO_ROOT_PASSWORD=change-me-long-password
|
||||
|
||||
docker run -d --name silo \
|
||||
-p 9000:9000 \
|
||||
-p 9001:9001 \
|
||||
-e MINIO_ROOT_USER \
|
||||
-e MINIO_ROOT_PASSWORD \
|
||||
-v "$PWD/data:/data" \
|
||||
pgsty/minio:latest server /data --console-address ":9001"
|
||||
```
|
||||
|
||||
管理控制台位于 <http://localhost:9001>,S3 API 位于 <http://localhost:9000>。
|
||||
|
||||
镜像内置兼容客户端 `mcli`:
|
||||
|
||||
```bash
|
||||
docker exec silo mcli alias set local http://127.0.0.1:9000 \
|
||||
"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
|
||||
docker exec silo mcli mb local/demo
|
||||
docker exec silo mcli ls local
|
||||
```
|
||||
|
||||
> [!WARNING]
|
||||
> 生产环境应锁定版本,使用独立凭据与 TLS,配置监控,保留独立备份,并验证恢复流程。
|
||||
|
||||
从源码构建服务端:
|
||||
|
||||
```bash
|
||||
go build -o minio .
|
||||
./minio --version
|
||||
```
|
||||
|
||||
其他安装方式(包括原生软件包、二进制、Podman、Kubernetes、源码构建与 Pigsty Ansible)请前往[下载与安装](https://silo.pgsty.com/zh/download/);生产部署与管理请从 [Silo 中文文档](https://silo.pgsty.com/zh/docs/)开始。Pigsty 用户也可以直接使用 [Pigsty MinIO 模块](https://pigsty.cc/docs/minio/)。
|
||||
|
||||
## 安全
|
||||
|
||||
安全修复面向活跃的 `master` 分支,并记录在仓库[安全公告](docs/security/advisories.md)与门户[安全通告](https://silo.pgsty.com/zh/blog/security/)中。请按照 [`SECURITY.md`](SECURITY.md) 与 [`VULNERABILITY_REPORT.md`](VULNERABILITY_REPORT.md) 私密报告漏洞;同时影响上游 MinIO 的问题也应向上游报告。
|
||||
|
||||
## 参与贡献
|
||||
|
||||
欢迎安全与依赖项更新、可复现缺陷修复、测试、发布自动化、打包与文档改进。
|
||||
|
||||
Issue 与 Pull Request 应说明受影响版本、复现步骤、影响、预期行为、测试与兼容性说明。大型改动请先提交 Issue 讨论。
|
||||
|
||||
## 背景
|
||||
|
||||
本项目源于上游社区发行与维护模式的变化。维护者对相关变化的分析、替代方案评估与早期维护记录见以下文章:
|
||||
|
||||
| 文章 | 主题 |
|
||||
| :-- | :-- |
|
||||
| [MinIO已死](https://silo.pgsty.com/zh/blog/post/minio-is-dead/) | 上游项目与发行模式的变化 |
|
||||
| [MinIO已死,谁能接盘?](https://silo.pgsty.com/zh/blog/post/minio-alternative/) | 可选替代方案评估 |
|
||||
| [MinIO 已死,MinIO 复生](https://silo.pgsty.com/zh/blog/post/minio-resurrect/) | 建立分支及其发行流水线 |
|
||||
| [续命 MinIO:承诺兑现](https://silo.pgsty.com/zh/blog/post/minio-promise-kept/) | 初期安全与维护工作 |
|
||||
|
||||
## 许可证与商标
|
||||
|
||||
服务端继续采用 [GNU Affero General Public License v3.0](LICENSE) 发布。上游作者与署名信息见 [`CREDITS`](CREDITS)。
|
||||
|
||||
MinIO 是 MinIO, Inc. 的商标。Silo、Pigsty 与 `pgsty/minio` 均为独立社区项目,与 MinIO, Inc. 不存在隶属或背书关系。
|
||||
@@ -8,8 +8,11 @@ function _init() {
|
||||
## All binaries are static make sure to disable CGO.
|
||||
export CGO_ENABLED=0
|
||||
|
||||
## List of architectures and OS to test coss compilation.
|
||||
SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64 linux/riscv64"
|
||||
## Cross-compile only the OS/arch combinations we actually publish, kept in
|
||||
## sync with the goos/goarch matrix in .github/goreleaser.yml. Compile-checking
|
||||
## targets we never ship (ppc64le, s390x, mips*, riscv64, 386, arm, the BSDs)
|
||||
## spent CI minutes on unshipped code and timed the gate out on a cold cache.
|
||||
SUPPORTED_OSARCH="linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64"
|
||||
}
|
||||
|
||||
function _build() {
|
||||
|
||||
@@ -38,8 +38,12 @@ func genLDFlags(version string) string {
|
||||
ldflagsStr += " -X github.com/minio/minio/cmd.ReleaseTag=" + releaseTag
|
||||
ldflagsStr += " -X github.com/minio/minio/cmd.CommitID=" + commitID()
|
||||
ldflagsStr += " -X github.com/minio/minio/cmd.ShortCommitID=" + commitID()[:12]
|
||||
ldflagsStr += " -X github.com/minio/minio/cmd.GOPATH=" + os.Getenv("GOPATH")
|
||||
ldflagsStr += " -X github.com/minio/minio/cmd.GOROOT=" + os.Getenv("GOROOT")
|
||||
// GOPATH/GOROOT are deliberately not stamped in. They only seed the logger's
|
||||
// source-path trim list, which -trimpath already makes moot (paths are
|
||||
// relative in the binary, so there is no build-machine prefix left to trim),
|
||||
// and stamping them baked the builder's absolute paths into the released
|
||||
// binary - defeating -trimpath and reproducible builds. cmd.GOPATH/GOROOT
|
||||
// keep their empty defaults, exactly as a plain `go build` leaves them.
|
||||
return ldflagsStr
|
||||
}
|
||||
|
||||
|
||||
Executable
+120
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
repo_dir="$(cd "${script_dir}/.." && pwd)"
|
||||
dist_dir="${DIST_DIR:-${repo_dir}/dist}"
|
||||
nfpm_config="${NFPM_CONFIG:-${repo_dir}/.github/nfpm.yml}"
|
||||
|
||||
if [ -z "${PKG_VERSION:-}" ]; then
|
||||
echo "PKG_VERSION is required" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Re-validate the shape release.yml derived from the tag. The packages are
|
||||
# named from this, so a malformed value would ship under a name no repository
|
||||
# can order against.
|
||||
if ! [[ "${PKG_VERSION}" =~ ^[0-9]{14}\.0\.0$ ]]; then
|
||||
echo "Invalid PKG_VERSION: ${PKG_VERSION}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v nfpm >/dev/null 2>&1; then
|
||||
echo "nfpm is required" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "${nfpm_config}" ]; then
|
||||
echo "Missing nFPM config: ${nfpm_config}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# nfpm resolves a relative content src against the current directory, not
|
||||
# against the config file, so the unit path is passed in absolute. Otherwise
|
||||
# this only works when invoked from the repository root and fails elsewhere on
|
||||
# a message that names the file rather than the cause.
|
||||
unit_file="${repo_dir}/minio.service"
|
||||
if [ ! -f "${unit_file}" ]; then
|
||||
echo "Missing systemd unit: ${unit_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
packages_dir="${dist_dir}/packages"
|
||||
mkdir -p "${packages_dir}"
|
||||
|
||||
# Two spaces, no trailing newline: sign-release-rpms.sh parses these files to
|
||||
# check download integrity before it signs, and regenerates them afterwards in
|
||||
# the same shape.
|
||||
sha256_file() {
|
||||
local file="$1"
|
||||
local digest
|
||||
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
digest="$(sha256sum "${file}" | awk '{print $1}')"
|
||||
else
|
||||
digest="$(shasum -a 256 "${file}" | awk '{print $1}')"
|
||||
fi
|
||||
|
||||
printf '%s %s' "${digest}" "$(basename "${file}")" > "${file}.sha256sum"
|
||||
}
|
||||
|
||||
find_binary() {
|
||||
local goarch="$1"
|
||||
local matches
|
||||
local count
|
||||
|
||||
# Must resolve to exactly one binary. Picking the first of several build
|
||||
# variants (an added goamd64 level, a stale dist entry) would silently ship a
|
||||
# package whose contents do not match its name.
|
||||
matches="$(find "${dist_dir}" -maxdepth 2 -type f \
|
||||
-path "${dist_dir}/minio_linux_${goarch}*/minio" | sort)"
|
||||
count="$(printf '%s' "${matches}" | grep -c . || true)"
|
||||
|
||||
if [ "${count}" -eq 0 ]; then
|
||||
echo "Missing GoReleaser binary for linux/${goarch}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "${count}" -ne 1 ]; then
|
||||
echo "Expected exactly one GoReleaser binary for linux/${goarch}, found ${count}:" >&2
|
||||
printf '%s\n' "${matches}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf '%s\n' "${matches}"
|
||||
}
|
||||
|
||||
build_arch() {
|
||||
local goarch="$1"
|
||||
local rpm_arch="$2"
|
||||
local deb_arch="$3"
|
||||
local apk_arch="$4"
|
||||
local source
|
||||
local rpm_file
|
||||
local deb_file
|
||||
local apk_file
|
||||
|
||||
source="$(find_binary "${goarch}")"
|
||||
|
||||
# These names are the public download names and must not drift; RPM carries a
|
||||
# release number, DEB and APK do not, matching what pkger produced.
|
||||
rpm_file="${packages_dir}/minio-${PKG_VERSION}-1.${rpm_arch}.rpm"
|
||||
deb_file="${packages_dir}/minio_${PKG_VERSION}_${deb_arch}.deb"
|
||||
apk_file="${packages_dir}/minio_${PKG_VERSION}_${apk_arch}.apk"
|
||||
|
||||
PKG_VERSION="${PKG_VERSION}" NFPM_RELEASE=1 NFPM_ARCH="${goarch}" NFPM_SOURCE="${source}" NFPM_UNIT="${unit_file}" \
|
||||
nfpm package --config "${nfpm_config}" --packager rpm --target "${rpm_file}"
|
||||
PKG_VERSION="${PKG_VERSION}" NFPM_RELEASE='' NFPM_ARCH="${goarch}" NFPM_SOURCE="${source}" NFPM_UNIT="${unit_file}" \
|
||||
nfpm package --config "${nfpm_config}" --packager deb --target "${deb_file}"
|
||||
PKG_VERSION="${PKG_VERSION}" NFPM_RELEASE='' NFPM_ARCH="${goarch}" NFPM_SOURCE="${source}" NFPM_UNIT="${unit_file}" \
|
||||
nfpm package --config "${nfpm_config}" --packager apk --target "${apk_file}"
|
||||
|
||||
sha256_file "${rpm_file}"
|
||||
sha256_file "${deb_file}"
|
||||
sha256_file "${apk_file}"
|
||||
}
|
||||
|
||||
build_arch amd64 x86_64 amd64 x86_64
|
||||
build_arch arm64 aarch64 arm64 aarch64
|
||||
|
||||
find "${packages_dir}" -maxdepth 1 -type f | sort
|
||||
Executable
+282
@@ -0,0 +1,282 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# These are the single source of truth for the package identity: .github/nfpm.yml
|
||||
# must agree with them, and test-release.yml asserts that it does. Drift the
|
||||
# other way round would only surface here, on the maintainer's machine, after
|
||||
# the build has already run and uploaded.
|
||||
expected_fingerprint="9592A7BC7A682E7333376E09E7935D8DB9BD8B20"
|
||||
expected_vendor="PGSTY"
|
||||
expected_packager="Ruohang Feng (@Vonng) <rh@vonng.com>"
|
||||
expected_url="https://silo.pgsty.com"
|
||||
expected_summary="S3-Interface Libre Object Storage, Community-maintained MinIO server fork."
|
||||
expected_description="S3-Interface Libre Object Storage, Community-maintained MinIO server fork."
|
||||
expected_license="AGPL-3.0-or-later"
|
||||
expected_group="Applications/File"
|
||||
expected_payload="/usr/lib/systemd/system/minio.service
|
||||
/usr/local/bin/minio"
|
||||
repository="${GH_REPO:-pgsty/minio}"
|
||||
container="${DNFUPDATE_CONTAINER:-dnfupdate}"
|
||||
upload=false
|
||||
release_tag=""
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: buildscripts/sign-release-rpms.sh RELEASE.TAG [--upload] [--repo OWNER/REPO] [--container NAME]
|
||||
|
||||
Downloads the two unsigned RPMs from a Draft GitHub Release, signs them with
|
||||
the expected Pigsty key in the local dnfupdate container, verifies the result,
|
||||
and regenerates their .sha256sum files. Nothing is uploaded unless --upload is
|
||||
provided.
|
||||
EOF
|
||||
}
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--upload)
|
||||
upload=true
|
||||
;;
|
||||
--repo)
|
||||
shift
|
||||
if [ "$#" -eq 0 ]; then
|
||||
echo "--repo requires OWNER/REPO" >&2
|
||||
exit 1
|
||||
fi
|
||||
repository="$1"
|
||||
;;
|
||||
--container)
|
||||
shift
|
||||
if [ "$#" -eq 0 ]; then
|
||||
echo "--container requires a name" >&2
|
||||
exit 1
|
||||
fi
|
||||
container="$1"
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
-*)
|
||||
echo "Unknown option: $1" >&2
|
||||
usage >&2
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
if [ -n "${release_tag}" ]; then
|
||||
echo "Only one release tag may be specified" >&2
|
||||
exit 1
|
||||
fi
|
||||
release_tag="$1"
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
if [ -z "${release_tag}" ]; then
|
||||
usage >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for command in docker gh; do
|
||||
if ! command -v "${command}" >/dev/null 2>&1; then
|
||||
echo "${command} is required" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
version_hyphen="${release_tag#RELEASE.}"
|
||||
package_version="$(printf '%s\n' "${version_hyphen}" | sed -E \
|
||||
's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')"
|
||||
if [ "${package_version}" = "${version_hyphen}" ]; then
|
||||
echo "Invalid release tag: ${release_tag}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$(gh release view "${release_tag}" --repo "${repository}" --json isDraft --jq .isDraft)" != "true" ]; then
|
||||
echo "Refusing to sign: ${release_tag} is not a Draft release" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$(docker inspect --format '{{.State.Running}}' "${container}" 2>/dev/null || true)" != "true" ]; then
|
||||
echo "Signing container is not running: ${container}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
secret_fingerprints="$(docker exec "${container}" \
|
||||
gpg --batch --with-colons --list-secret-keys 2>/dev/null |
|
||||
awk -F: '$1 == "fpr" { print toupper($10) }')"
|
||||
if ! printf '%s\n' "${secret_fingerprints}" | grep -Fxq "${expected_fingerprint}"; then
|
||||
echo "Expected signing key is not available in ${container}: ${expected_fingerprint}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
repo_dir="$(cd "${script_dir}/.." && pwd)"
|
||||
work_root="${SIGN_WORKDIR:-${repo_dir}/.release-sign}"
|
||||
mkdir -p "${work_root}"
|
||||
work_dir="$(mktemp -d "${work_root}/${release_tag}.XXXXXX")"
|
||||
unsigned_dir="${work_dir}/unsigned"
|
||||
signed_dir="${work_dir}/signed"
|
||||
mkdir -p "${unsigned_dir}" "${signed_dir}"
|
||||
chmod 700 "${work_dir}" "${unsigned_dir}" "${signed_dir}"
|
||||
|
||||
rpm_files=(
|
||||
"minio-${package_version}-1.x86_64.rpm"
|
||||
"minio-${package_version}-1.aarch64.rpm"
|
||||
)
|
||||
|
||||
download_patterns=()
|
||||
for rpm_file in "${rpm_files[@]}"; do
|
||||
download_patterns+=(--pattern "${rpm_file}" --pattern "${rpm_file}.sha256sum")
|
||||
done
|
||||
|
||||
echo "Downloading RPMs from Draft release ${repository}@${release_tag}"
|
||||
gh release download "${release_tag}" --repo "${repository}" \
|
||||
--dir "${unsigned_dir}" "${download_patterns[@]}"
|
||||
|
||||
sha256_digest() {
|
||||
local file="$1"
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
sha256sum "${file}" | awk '{print $1}'
|
||||
else
|
||||
shasum -a 256 "${file}" | awk '{print $1}'
|
||||
fi
|
||||
}
|
||||
|
||||
for rpm_file in "${rpm_files[@]}"; do
|
||||
rpm_path="${unsigned_dir}/${rpm_file}"
|
||||
checksum_path="${rpm_path}.sha256sum"
|
||||
test -s "${rpm_path}"
|
||||
test -s "${checksum_path}"
|
||||
|
||||
actual_line="$(sha256_digest "${rpm_path}") ${rpm_file}"
|
||||
published_line="$(tr -d '\n' < "${checksum_path}")"
|
||||
if [ "${actual_line}" != "${published_line}" ]; then
|
||||
echo "Checksum mismatch for ${rpm_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
safe_tag="$(printf '%s' "${release_tag}" | tr -c 'A-Za-z0-9._-' '_')"
|
||||
container_dir="/tmp/minio-sign-${safe_tag}-$$"
|
||||
docker exec "${container}" mkdir -p "${container_dir}"
|
||||
|
||||
cleanup_container() {
|
||||
local rpm_file
|
||||
for rpm_file in "${rpm_files[@]}"; do
|
||||
docker exec "${container}" rm -f "${container_dir}/${rpm_file}" >/dev/null 2>&1 || true
|
||||
done
|
||||
docker exec "${container}" rmdir "${container_dir}" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup_container EXIT
|
||||
|
||||
assert_rpm_tag() {
|
||||
local rpm_path="$1"
|
||||
local tag="$2"
|
||||
local expected="$3"
|
||||
local actual
|
||||
|
||||
actual="$(docker exec "${container}" rpm -qp --queryformat "%{${tag}}" "${rpm_path}")"
|
||||
if [ "${actual}" != "${expected}" ]; then
|
||||
echo "Unexpected RPM ${tag}: ${actual}" >&2
|
||||
echo "Expected RPM ${tag}: ${expected}" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
for rpm_file in "${rpm_files[@]}"; do
|
||||
case "${rpm_file}" in
|
||||
*.x86_64.rpm)
|
||||
expected_arch="x86_64"
|
||||
;;
|
||||
*.aarch64.rpm)
|
||||
expected_arch="aarch64"
|
||||
;;
|
||||
*)
|
||||
echo "Unexpected RPM filename: ${rpm_file}" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "Signing ${rpm_file} with ${expected_fingerprint}"
|
||||
docker cp "${unsigned_dir}/${rpm_file}" "${container}:${container_dir}/${rpm_file}" >/dev/null
|
||||
container_rpm="${container_dir}/${rpm_file}"
|
||||
|
||||
assert_rpm_tag "${container_rpm}" NAME minio
|
||||
assert_rpm_tag "${container_rpm}" VERSION "${package_version}"
|
||||
assert_rpm_tag "${container_rpm}" RELEASE 1
|
||||
assert_rpm_tag "${container_rpm}" ARCH "${expected_arch}"
|
||||
assert_rpm_tag "${container_rpm}" VENDOR "${expected_vendor}"
|
||||
assert_rpm_tag "${container_rpm}" PACKAGER "${expected_packager}"
|
||||
assert_rpm_tag "${container_rpm}" URL "${expected_url}"
|
||||
assert_rpm_tag "${container_rpm}" LICENSE "${expected_license}"
|
||||
assert_rpm_tag "${container_rpm}" GROUP "${expected_group}"
|
||||
assert_rpm_tag "${container_rpm}" SUMMARY "${expected_summary}"
|
||||
assert_rpm_tag "${container_rpm}" DESCRIPTION "${expected_description}"
|
||||
|
||||
rpm_payload="$(docker exec "${container}" rpm -qpl "${container_rpm}")"
|
||||
if [ "${rpm_payload}" != "${expected_payload}" ]; then
|
||||
echo "Unexpected RPM payload for ${rpm_file}:" >&2
|
||||
printf '%s\n' "${rpm_payload}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker exec "${container}" rpmsign \
|
||||
--define "_gpg_name ${expected_fingerprint}" \
|
||||
--addsign "${container_rpm}"
|
||||
|
||||
signature_output="$(docker exec "${container}" rpmkeys --checksig --verbose "${container_rpm}")"
|
||||
printf '%s\n' "${signature_output}"
|
||||
if ! printf '%s\n' "${signature_output}" | tr '[:upper:]' '[:lower:]' | grep -q 'key id b9bd8b20: ok'; then
|
||||
echo "Signature verification failed for ${rpm_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker cp "${container}:${container_dir}/${rpm_file}" "${signed_dir}/${rpm_file}" >/dev/null
|
||||
signed_digest="$(sha256_digest "${signed_dir}/${rpm_file}")"
|
||||
printf '%s %s' "${signed_digest}" "${rpm_file}" > "${signed_dir}/${rpm_file}.sha256sum"
|
||||
|
||||
docker exec "${container}" rpm -qp --queryformat \
|
||||
$'Name: %{NAME}\nVersion: %{VERSION}-%{RELEASE}\nArch: %{ARCH}\nVendor: %{VENDOR}\nPackager: %{PACKAGER}\nURL: %{URL}\n' \
|
||||
"${container_rpm}"
|
||||
echo "SHA256: ${signed_digest}"
|
||||
done
|
||||
|
||||
if [ "${upload}" = true ]; then
|
||||
upload_files=()
|
||||
for rpm_file in "${rpm_files[@]}"; do
|
||||
upload_files+=("${signed_dir}/${rpm_file}" "${signed_dir}/${rpm_file}.sha256sum")
|
||||
done
|
||||
|
||||
echo "Replacing RPMs in Draft release ${release_tag}"
|
||||
gh release upload "${release_tag}" --repo "${repository}" --clobber "${upload_files[@]}"
|
||||
|
||||
for rpm_file in "${rpm_files[@]}"; do
|
||||
for asset in "${rpm_file}" "${rpm_file}.sha256sum"; do
|
||||
local_digest="sha256:$(sha256_digest "${signed_dir}/${asset}")"
|
||||
remote_digest=""
|
||||
for attempt in 1 2 3 4 5; do
|
||||
remote_digest="$(gh release view "${release_tag}" --repo "${repository}" --json assets \
|
||||
--jq ".assets[] | select(.name == \"${asset}\") | .digest")"
|
||||
if [ "${local_digest}" = "${remote_digest}" ]; then
|
||||
break
|
||||
fi
|
||||
if [ "${attempt}" -lt 5 ]; then
|
||||
sleep 2
|
||||
fi
|
||||
done
|
||||
if [ "${local_digest}" != "${remote_digest}" ]; then
|
||||
echo "GitHub asset digest mismatch for ${asset}" >&2
|
||||
echo "Local: ${local_digest}" >&2
|
||||
echo "Remote: ${remote_digest}" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Verified GitHub asset: ${asset} ${remote_digest}"
|
||||
done
|
||||
done
|
||||
else
|
||||
echo
|
||||
echo "Signed RPMs are ready for review in: ${signed_dir}"
|
||||
echo "Re-run with --upload to replace the RPM assets in the Draft release."
|
||||
fi
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Asserts that every binary GoReleaser produced is stamped by the Go toolchain
|
||||
# as built from this exact commit with a clean working tree. A stray untracked
|
||||
# file (for example an un-ignored dist/) silently turns every release binary
|
||||
# into a "+dirty" pseudo-version, which destroys the link between a published
|
||||
# artifact and its tag. Catch that here instead of after publishing.
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
repo_dir="$(cd "${script_dir}/.." && pwd)"
|
||||
dist_dir="${DIST_DIR:-${repo_dir}/dist}"
|
||||
expected_count="${EXPECTED_BINARY_COUNT:-6}"
|
||||
|
||||
if ! command -v go >/dev/null 2>&1; then
|
||||
echo "go is required" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d "${dist_dir}" ]; then
|
||||
echo "Missing GoReleaser dist directory: ${dist_dir}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
revision="$(git -C "${repo_dir}" rev-parse HEAD)"
|
||||
|
||||
count=0
|
||||
while IFS= read -r binary; do
|
||||
count=$((count + 1))
|
||||
info="$(go version -m "${binary}")"
|
||||
|
||||
if ! grep -qF "vcs.revision=${revision}" <<< "${info}"; then
|
||||
echo "Unexpected vcs.revision in ${binary} (expected ${revision})" >&2
|
||||
grep -F 'vcs.' <<< "${info}" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! grep -qF 'vcs.modified=false' <<< "${info}"; then
|
||||
echo "Binary was built from a dirty working tree: ${binary}" >&2
|
||||
grep -F 'vcs.' <<< "${info}" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
done < <(find "${dist_dir}" -maxdepth 2 -type f \( -name 'minio' -o -name 'minio.exe' \) | sort)
|
||||
|
||||
if [ "${count}" -ne "${expected_count}" ]; then
|
||||
echo "Expected ${expected_count} release binaries, found ${count}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Verified ${count} binaries built from ${revision} with a clean tree"
|
||||
+22
-4
@@ -1026,7 +1026,7 @@ type unwrapper interface {
|
||||
Unwrap() http.ResponseWriter
|
||||
}
|
||||
|
||||
// headersAlreadyWritten returns true if the headers have already been written
|
||||
// headersAlreadyWritten returns true if an HTTP status has already been written
|
||||
// to this response writer. It will unwrap the ResponseWriter if possible to try
|
||||
// and find a trackingResponseWriter.
|
||||
func headersAlreadyWritten(w http.ResponseWriter) bool {
|
||||
@@ -1041,14 +1041,18 @@ func headersAlreadyWritten(w http.ResponseWriter) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// trackingResponseWriter wraps a ResponseWriter and notes when WriterHeader has
|
||||
// been called. This allows high level request handlers to check if something
|
||||
// has already sent the header.
|
||||
// trackingResponseWriter wraps a ResponseWriter and records when an HTTP status
|
||||
// has been written, explicitly or implicitly by Write or an effective Flush.
|
||||
//
|
||||
// Informational responses are treated as final. internal/http.ResponseRecorder
|
||||
// has the same limitation, so 1xx support must be fixed in both layers.
|
||||
type trackingResponseWriter struct {
|
||||
http.ResponseWriter
|
||||
headerWritten bool
|
||||
}
|
||||
|
||||
var _ http.Flusher = (*trackingResponseWriter)(nil)
|
||||
|
||||
func (w *trackingResponseWriter) WriteHeader(statusCode int) {
|
||||
if !w.headerWritten {
|
||||
w.headerWritten = true
|
||||
@@ -1057,9 +1061,23 @@ func (w *trackingResponseWriter) WriteHeader(statusCode int) {
|
||||
}
|
||||
|
||||
func (w *trackingResponseWriter) Write(b []byte) (int, error) {
|
||||
if !w.headerWritten {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
return w.ResponseWriter.Write(b)
|
||||
}
|
||||
|
||||
func (w *trackingResponseWriter) Flush() {
|
||||
f, ok := w.ResponseWriter.(http.Flusher)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if !w.headerWritten {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
f.Flush()
|
||||
}
|
||||
|
||||
func (w *trackingResponseWriter) Unwrap() http.ResponseWriter {
|
||||
return w.ResponseWriter
|
||||
}
|
||||
|
||||
+142
-3
@@ -18,12 +18,14 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/klauspost/compress/gzhttp"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
// Tests object location.
|
||||
@@ -127,6 +129,22 @@ func TestGetURLScheme(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type writeHeaderSpy struct {
|
||||
http.ResponseWriter
|
||||
codes []int
|
||||
}
|
||||
|
||||
func (r *writeHeaderSpy) WriteHeader(code int) {
|
||||
r.codes = append(r.codes, code)
|
||||
r.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
func (r *writeHeaderSpy) Flush() {
|
||||
if f, ok := r.ResponseWriter.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrackingResponseWriter(t *testing.T) {
|
||||
rw := httptest.NewRecorder()
|
||||
trw := &trackingResponseWriter{ResponseWriter: rw}
|
||||
@@ -139,8 +157,9 @@ func TestTrackingResponseWriter(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Write unexpectedly failed: %v", err)
|
||||
}
|
||||
xhttp.Flush(trw)
|
||||
|
||||
// Check that WriteHeader and Write were called on the underlying response writer
|
||||
// Check that WriteHeader, Write, and Flush were called on the underlying response writer.
|
||||
resp := rw.Result()
|
||||
if resp.StatusCode != 299 {
|
||||
t.Fatalf("unexpected status: %v", resp.StatusCode)
|
||||
@@ -152,6 +171,9 @@ func TestTrackingResponseWriter(t *testing.T) {
|
||||
if string(body) != "hello" {
|
||||
t.Fatalf("response body incorrect: %v", string(body))
|
||||
}
|
||||
if !rw.Flushed {
|
||||
t.Fatal("underlying ResponseRecorder was not flushed")
|
||||
}
|
||||
|
||||
// Check that Unwrap works
|
||||
if trw.Unwrap() != rw {
|
||||
@@ -159,6 +181,122 @@ func TestTrackingResponseWriter(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrackingResponseWriterWriteImplicitHeader(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
body []byte
|
||||
}{
|
||||
{name: "non-empty", body: []byte("hello")},
|
||||
{name: "empty", body: nil},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
rw := &writeHeaderSpy{ResponseWriter: rec}
|
||||
trw := &trackingResponseWriter{ResponseWriter: rw}
|
||||
|
||||
n, err := trw.Write(testCase.body)
|
||||
if err != nil {
|
||||
t.Fatalf("Write unexpectedly failed: %v", err)
|
||||
}
|
||||
if n != len(testCase.body) {
|
||||
t.Fatalf("unexpected bytes written: got %d, want %d", n, len(testCase.body))
|
||||
}
|
||||
if !trw.headerWritten {
|
||||
t.Fatal("Write did not set headerWritten")
|
||||
}
|
||||
if len(rw.codes) != 1 || rw.codes[0] != http.StatusOK {
|
||||
t.Fatalf("unexpected WriteHeader calls: got %v, want [%d]", rw.codes, http.StatusOK)
|
||||
}
|
||||
if got := rec.Body.String(); got != string(testCase.body) {
|
||||
t.Fatalf("unexpected body: got %q, want %q", got, testCase.body)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrackingResponseWriterFlush(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
rw := &writeHeaderSpy{ResponseWriter: rec}
|
||||
trw := &trackingResponseWriter{ResponseWriter: rw}
|
||||
|
||||
xhttp.Flush(trw)
|
||||
if !trw.headerWritten {
|
||||
t.Fatal("Flush did not set headerWritten")
|
||||
}
|
||||
if len(rw.codes) != 1 || rw.codes[0] != http.StatusOK {
|
||||
t.Fatalf("unexpected WriteHeader calls: got %v, want [%d]", rw.codes, http.StatusOK)
|
||||
}
|
||||
if !rec.Flushed {
|
||||
t.Fatal("underlying ResponseRecorder was not flushed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrackingResponseWriterFlushUnsupported(t *testing.T) {
|
||||
rw := struct{ http.ResponseWriter }{ResponseWriter: httptest.NewRecorder()}
|
||||
trw := &trackingResponseWriter{ResponseWriter: rw}
|
||||
|
||||
trw.Flush()
|
||||
if trw.headerWritten {
|
||||
t.Fatal("unsupported Flush set headerWritten")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrackingResponseWriterGzipStreaming(t *testing.T) {
|
||||
const (
|
||||
eventPayload = "event data"
|
||||
sentinel = "<sentinel-error/>"
|
||||
)
|
||||
|
||||
rw := httptest.NewRecorder()
|
||||
trw := &trackingResponseWriter{ResponseWriter: rw}
|
||||
var (
|
||||
committed bool
|
||||
writeErr error
|
||||
)
|
||||
handler := gzipHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
setEventStreamHeaders(w)
|
||||
_, writeErr = w.Write([]byte(eventPayload))
|
||||
if writeErr != nil {
|
||||
return
|
||||
}
|
||||
xhttp.Flush(w)
|
||||
committed = headersAlreadyWritten(w)
|
||||
writeResponse(w, http.StatusInternalServerError, []byte(sentinel), mimeXML)
|
||||
}))
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
|
||||
handler.ServeHTTP(trw, req)
|
||||
|
||||
if writeErr != nil {
|
||||
t.Fatalf("Write unexpectedly failed: %v", writeErr)
|
||||
}
|
||||
if !committed {
|
||||
t.Fatal("headersAlreadyWritten returned false after Write and Flush")
|
||||
}
|
||||
resp := rw.Result()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("unexpected status: got %d, want %d", resp.StatusCode, http.StatusOK)
|
||||
}
|
||||
if got := resp.Header.Get("Content-Encoding"); got != "gzip" {
|
||||
t.Fatalf("unexpected Content-Encoding: got %q, want %q", got, "gzip")
|
||||
}
|
||||
zr, err := gzip.NewReader(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("creating gzip reader failed: %v", err)
|
||||
}
|
||||
defer zr.Close()
|
||||
body, err := io.ReadAll(zr)
|
||||
if err != nil {
|
||||
t.Fatalf("reading gzip response body failed: %v", err)
|
||||
}
|
||||
if got := string(body); got != eventPayload {
|
||||
t.Fatalf("unexpected response body: got %q, want %q (sentinel %q must be suppressed)", got, eventPayload, sentinel)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeadersAlreadyWritten(t *testing.T) {
|
||||
rw := httptest.NewRecorder()
|
||||
trw := &trackingResponseWriter{ResponseWriter: rw}
|
||||
@@ -167,7 +305,7 @@ func TestHeadersAlreadyWritten(t *testing.T) {
|
||||
t.Fatal("headers have not been written yet")
|
||||
}
|
||||
|
||||
trw.WriteHeader(123)
|
||||
trw.WriteHeader(299)
|
||||
if !headersAlreadyWritten(trw) {
|
||||
t.Fatal("headers were written")
|
||||
}
|
||||
@@ -183,7 +321,8 @@ func TestHeadersAlreadyWrittenWrapped(t *testing.T) {
|
||||
t.Fatal("headers have not been written yet")
|
||||
}
|
||||
|
||||
wrap2.WriteHeader(123)
|
||||
// Pin the current stack-wide 1xx limitation documented on trackingResponseWriter.
|
||||
wrap2.WriteHeader(http.StatusContinue)
|
||||
if !headersAlreadyWritten(wrap2) {
|
||||
t.Fatal("headers were written")
|
||||
}
|
||||
|
||||
@@ -347,8 +347,9 @@ const _APIErrorCode_name = "NoneAccessDeniedBadDigestEntityTooSmallEntityTooLarg
|
||||
var _APIErrorCode_index = [...]uint16{0, 4, 16, 25, 39, 53, 67, 81, 94, 112, 129, 144, 161, 174, 186, 208, 228, 254, 268, 289, 306, 321, 344, 361, 379, 396, 420, 435, 456, 474, 486, 506, 523, 546, 567, 579, 597, 618, 646, 676, 697, 720, 746, 783, 813, 846, 871, 903, 933, 962, 987, 1009, 1035, 1057, 1085, 1114, 1148, 1179, 1216, 1240, 1264, 1292, 1318, 1349, 1379, 1388, 1400, 1416, 1429, 1443, 1461, 1481, 1502, 1518, 1529, 1545, 1556, 1584, 1604, 1620, 1648, 1662, 1679, 1699, 1712, 1726, 1739, 1752, 1768, 1785, 1806, 1820, 1841, 1854, 1876, 1899, 1915, 1930, 1945, 1966, 1984, 1999, 2016, 2041, 2059, 2082, 2097, 2116, 2132, 2151, 2172, 2186, 2198, 2211, 2230, 2249, 2259, 2274, 2310, 2341, 2374, 2403, 2415, 2435, 2459, 2483, 2504, 2528, 2547, 2568, 2585, 2595, 2612, 2629, 2650, 2670, 2693, 2715, 2741, 2762, 2780, 2807, 2838, 2865, 2886, 2907, 2931, 2956, 2984, 3012, 3028, 3051, 3081, 3092, 3104, 3121, 3136, 3154, 3183, 3200, 3216, 3232, 3250, 3268, 3291, 3312, 3335, 3346, 3362, 3385, 3402, 3430, 3449, 3479, 3499, 3527, 3542, 3560, 3575, 3589, 3624, 3643, 3654, 3667, 3682, 3705, 3731, 3747, 3765, 3783, 3804, 3818, 3835, 3866, 3886, 3907, 3928, 3947, 3966, 3984, 4007, 4031, 4055, 4080, 4115, 4140, 4174, 4207, 4228, 4242, 4261, 4290, 4313, 4340, 4374, 4406, 4436, 4459, 4487, 4519, 4547, 4571, 4595, 4624, 4642, 4659, 4681, 4698, 4716, 4736, 4762, 4778, 4797, 4818, 4822, 4840, 4857, 4883, 4897, 4921, 4942, 4957, 4975, 4998, 5013, 5032, 5049, 5066, 5090, 5117, 5140, 5163, 5180, 5202, 5218, 5238, 5257, 5279, 5300, 5320, 5342, 5366, 5385, 5427, 5448, 5471, 5492, 5523, 5542, 5564, 5584, 5610, 5631, 5653, 5673, 5697, 5720, 5739, 5759, 5781, 5804, 5835, 5873, 5914, 5944, 5958, 5979, 5995, 6017, 6047, 6073, 6101, 6135, 6153, 6176, 6211, 6251, 6293, 6325, 6342, 6367, 6382, 6399, 6409, 6420, 6458, 6512, 6558, 6610, 6658, 6701, 6745, 6773, 6787, 6805, 6841, 6864, 6887, 6909, 6924, 6952, 6975, 6993, 7020, 7052, 7067, 7083, 7100, 7120, 7136, 7152, 7169, 7182}
|
||||
|
||||
func (i APIErrorCode) String() string {
|
||||
if i < 0 || i >= APIErrorCode(len(_APIErrorCode_index)-1) {
|
||||
idx := int(i) - 0
|
||||
if i < 0 || idx >= len(_APIErrorCode_index)-1 {
|
||||
return "APIErrorCode(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _APIErrorCode_name[_APIErrorCode_index[i]:_APIErrorCode_index[i+1]]
|
||||
return _APIErrorCode_name[_APIErrorCode_index[idx]:_APIErrorCode_index[idx+1]]
|
||||
}
|
||||
|
||||
+52
-7
@@ -343,6 +343,26 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
|
||||
return s3Err
|
||||
}
|
||||
|
||||
func checkRequestAuthTypeWithExistingTags(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName, existingTags string) (s3Err APIErrorCode) {
|
||||
logger.GetReqInfo(ctx).BucketName = bucketName
|
||||
logger.GetReqInfo(ctx).ObjectName = objectName
|
||||
|
||||
if s3Err = authenticateRequest(ctx, r, action); s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
return authorizeRequestWithExistingTags(ctx, r, action, existingTags)
|
||||
}
|
||||
|
||||
func checkRequestAuthTypeWithRequestTags(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string, requestTags *string) (s3Err APIErrorCode) {
|
||||
logger.GetReqInfo(ctx).BucketName = bucketName
|
||||
logger.GetReqInfo(ctx).ObjectName = objectName
|
||||
|
||||
if s3Err = authenticateRequest(ctx, r, action); s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
return authorizeRequestWithTags(ctx, r, action, "", requestTags)
|
||||
}
|
||||
|
||||
// checkRequestAuthTypeWithVID is similar to checkRequestAuthType
|
||||
// passes versionID additionally.
|
||||
func checkRequestAuthTypeWithVID(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName, versionID string) (s3Err APIErrorCode) {
|
||||
@@ -416,6 +436,14 @@ func authenticateRequest(ctx context.Context, r *http.Request, action policy.Act
|
||||
}
|
||||
|
||||
func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
|
||||
return authorizeRequestWithExistingTags(ctx, r, action, "")
|
||||
}
|
||||
|
||||
func authorizeRequestWithExistingTags(ctx context.Context, r *http.Request, action policy.Action, existingTags string) (s3Err APIErrorCode) {
|
||||
return authorizeRequestWithTags(ctx, r, action, existingTags, nil)
|
||||
}
|
||||
|
||||
func authorizeRequestWithTags(ctx context.Context, r *http.Request, action policy.Action, existingTags string, requestTags *string) (s3Err APIErrorCode) {
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
if reqInfo == nil {
|
||||
return ErrAccessDenied
|
||||
@@ -427,6 +455,19 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
||||
bucket := reqInfo.BucketName
|
||||
object := reqInfo.ObjectName
|
||||
versionID := reqInfo.VersionID
|
||||
conditionValuesForAuth := func(locationConstraint string, credentials auth.Credentials) map[string][]string {
|
||||
values := getConditionValuesWithTags(r, locationConstraint, credentials, existingTags, requestTags)
|
||||
if action == policy.DeleteObjectAction {
|
||||
// DeleteObjects carries the effective version ID in each XML object,
|
||||
// not in the request query. Keep authorization scoped to that entry.
|
||||
if versionID == "" {
|
||||
delete(values, "versionid")
|
||||
} else {
|
||||
values["versionid"] = []string{versionID}
|
||||
}
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
if action != policy.ListAllMyBucketsAction && cred.AccessKey == "" {
|
||||
// Anonymous checks are not meant for ListAllBuckets action
|
||||
@@ -435,7 +476,7 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
||||
Groups: cred.Groups,
|
||||
Action: action,
|
||||
BucketName: bucket,
|
||||
ConditionValues: getConditionValues(r, region, auth.AnonymousCredentials),
|
||||
ConditionValues: conditionValuesForAuth(region, auth.AnonymousCredentials),
|
||||
IsOwner: false,
|
||||
ObjectName: object,
|
||||
}) {
|
||||
@@ -451,7 +492,7 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListBucketAction,
|
||||
BucketName: bucket,
|
||||
ConditionValues: getConditionValues(r, region, auth.AnonymousCredentials),
|
||||
ConditionValues: conditionValuesForAuth(region, auth.AnonymousCredentials),
|
||||
IsOwner: false,
|
||||
ObjectName: object,
|
||||
}) {
|
||||
@@ -468,7 +509,7 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
||||
Groups: cred.Groups,
|
||||
Action: policy.Action(policy.DeleteObjectVersionAction),
|
||||
BucketName: bucket,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
ConditionValues: conditionValuesForAuth("", cred),
|
||||
ObjectName: object,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
@@ -482,7 +523,7 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
||||
Groups: cred.Groups,
|
||||
Action: action,
|
||||
BucketName: bucket,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
ConditionValues: conditionValuesForAuth("", cred),
|
||||
ObjectName: object,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
@@ -499,7 +540,7 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListBucketAction,
|
||||
BucketName: bucket,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
ConditionValues: conditionValuesForAuth("", cred),
|
||||
ObjectName: object,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
@@ -720,6 +761,10 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
|
||||
// call verifies bucket policies and IAM policies, supports multi user
|
||||
// checks etc.
|
||||
func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
|
||||
return isPutActionAllowedWithRequestTags(ctx, atype, bucketName, objectName, r, action, nil)
|
||||
}
|
||||
|
||||
func isPutActionAllowedWithRequestTags(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action policy.Action, requestTags *string) (s3Err APIErrorCode) {
|
||||
var cred auth.Credentials
|
||||
var owner bool
|
||||
region := globalSite.Region()
|
||||
@@ -760,7 +805,7 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
|
||||
Groups: cred.Groups,
|
||||
Action: action,
|
||||
BucketName: bucketName,
|
||||
ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
|
||||
ConditionValues: getConditionValuesWithTags(r, "", auth.AnonymousCredentials, "", requestTags),
|
||||
IsOwner: false,
|
||||
ObjectName: objectName,
|
||||
}) {
|
||||
@@ -774,7 +819,7 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
|
||||
Groups: cred.Groups,
|
||||
Action: action,
|
||||
BucketName: bucketName,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
ConditionValues: getConditionValuesWithTags(r, "", cred, "", requestTags),
|
||||
ObjectName: objectName,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
|
||||
@@ -27,8 +27,9 @@ const _authType_name = "UnknownAnonymousPresignedPresignedV2PostPolicyStreamingS
|
||||
var _authType_index = [...]uint8{0, 7, 16, 25, 36, 46, 61, 67, 75, 78, 81, 103, 127}
|
||||
|
||||
func (i authType) String() string {
|
||||
if i < 0 || i >= authType(len(_authType_index)-1) {
|
||||
idx := int(i) - 0
|
||||
if i < 0 || idx >= len(_authType_index)-1 {
|
||||
return "authType(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _authType_name[_authType_index[i]:_authType_index[i+1]]
|
||||
return _authType_name[_authType_index[idx]:_authType_index[idx+1]]
|
||||
}
|
||||
|
||||
@@ -18,8 +18,9 @@ const _batchJobMetric_name = "ReplicationKeyRotationExpire"
|
||||
var _batchJobMetric_index = [...]uint8{0, 11, 22, 28}
|
||||
|
||||
func (i batchJobMetric) String() string {
|
||||
if i >= batchJobMetric(len(_batchJobMetric_index)-1) {
|
||||
idx := int(i) - 0
|
||||
if i < 0 || idx >= len(_batchJobMetric_index)-1 {
|
||||
return "batchJobMetric(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _batchJobMetric_name[_batchJobMetric_index[i]:_batchJobMetric_index[i+1]]
|
||||
return _batchJobMetric_name[_batchJobMetric_index[idx]:_batchJobMetric_index[idx+1]]
|
||||
}
|
||||
|
||||
@@ -944,3 +944,130 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
|
||||
// `ExecObjectLayerAPINilTest` manages the operation.
|
||||
ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq)
|
||||
}
|
||||
|
||||
func TestAPIDeleteMultipleObjectsVersionIDNullCondition(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPIDeleteMultipleObjectsVersionIDNullCondition,
|
||||
endpoints: []string{"DeleteMultipleObjects", "PutBucketPolicy"},
|
||||
makeBucketOptions: MakeBucketOptions{VersioningEnabled: true},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPIDeleteMultipleObjectsVersionIDNullCondition(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
versionIDs := make(map[string]string, 4)
|
||||
for _, objectName := range []string{
|
||||
"without-version-id-before",
|
||||
"with-version-id",
|
||||
"without-version-id-after",
|
||||
"with-null-version-id",
|
||||
} {
|
||||
data := []byte(objectName)
|
||||
info, err := obj.PutObject(t.Context(), bucketName, objectName,
|
||||
mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{Versioned: true})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: put %q: %v", instanceType, objectName, err)
|
||||
}
|
||||
if info.VersionID == "" {
|
||||
t.Fatalf("%s: put %q did not create a version ID", instanceType, objectName)
|
||||
}
|
||||
versionIDs[objectName] = info.VersionID
|
||||
}
|
||||
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[{
|
||||
"Effect":"Allow",
|
||||
"Principal":"*",
|
||||
"Action":"s3:DeleteObject",
|
||||
"Resource":"arn:aws:s3:::%s/*",
|
||||
"Condition":{"Null":{"s3:versionid":"true"}}
|
||||
}]
|
||||
}`, bucketName)
|
||||
policyReq, err := newTestSignedRequestV4(http.MethodPut, getPutPolicyURL("", bucketName), int64(len(policyBytes)),
|
||||
bytes.NewReader(policyBytes), credentials.AccessKey, credentials.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
policyRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(policyRec, policyReq)
|
||||
if policyRec.Code != http.StatusNoContent {
|
||||
t.Fatalf("%s: put policy returned %d: %s", instanceType, policyRec.Code, policyRec.Body.String())
|
||||
}
|
||||
|
||||
deleteBody := encodeResponse(DeleteObjectsRequest{Objects: []ObjectToDelete{
|
||||
{ObjectV: ObjectV{ObjectName: "without-version-id-before"}},
|
||||
{ObjectV: ObjectV{ObjectName: "with-version-id", VersionID: versionIDs["with-version-id"]}},
|
||||
{ObjectV: ObjectV{ObjectName: "without-version-id-after"}},
|
||||
{ObjectV: ObjectV{ObjectName: "with-null-version-id", VersionID: nullVersionID}},
|
||||
}})
|
||||
// A query-level versionId is not the version of every XML entry. Each
|
||||
// object's optional VersionId remains the effective authorization value.
|
||||
deleteURL := getDeleteMultipleObjectsURL("", bucketName) + "&versionId=query-level-decoy"
|
||||
deleteReq, err := newTestRequest(http.MethodPost, deleteURL,
|
||||
int64(len(deleteBody)), bytes.NewReader(deleteBody))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deleteRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(deleteRec, deleteReq)
|
||||
if deleteRec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: delete returned %d: %s", instanceType, deleteRec.Code, deleteRec.Body.String())
|
||||
}
|
||||
|
||||
var response DeleteObjectsResponse
|
||||
if err = xml.Unmarshal(deleteRec.Body.Bytes(), &response); err != nil {
|
||||
t.Fatalf("%s: decode response: %v: %s", instanceType, err, deleteRec.Body.String())
|
||||
}
|
||||
deleted := make(map[string]DeletedObject, len(response.DeletedObjects))
|
||||
for _, object := range response.DeletedObjects {
|
||||
deleted[object.ObjectName] = object
|
||||
}
|
||||
for _, objectName := range []string{"without-version-id-before", "without-version-id-after"} {
|
||||
object, ok := deleted[objectName]
|
||||
if !ok || !object.DeleteMarker || object.DeleteMarkerVersionID == "" {
|
||||
t.Errorf("%s: %q was not a successful delete-marker creation: %+v", instanceType, objectName, response.DeletedObjects)
|
||||
}
|
||||
}
|
||||
if len(deleted) != 2 {
|
||||
t.Errorf("%s: unexpected deleted objects: %+v", instanceType, response.DeletedObjects)
|
||||
}
|
||||
errorsByKey := make(map[string]DeleteError, len(response.Errors))
|
||||
for _, deleteErr := range response.Errors {
|
||||
errorsByKey[deleteErr.Key] = deleteErr
|
||||
}
|
||||
for objectName, versionID := range map[string]string{
|
||||
"with-version-id": versionIDs["with-version-id"],
|
||||
"with-null-version-id": nullVersionID,
|
||||
} {
|
||||
deleteErr, ok := errorsByKey[objectName]
|
||||
if !ok || deleteErr.VersionID != versionID || deleteErr.Code != errorCodes[ErrAccessDenied].Code {
|
||||
t.Errorf("%s: %q did not return AccessDenied for version %q: %+v", instanceType, objectName, versionID, response.Errors)
|
||||
}
|
||||
}
|
||||
if len(errorsByKey) != 2 {
|
||||
t.Errorf("%s: unexpected delete errors: %+v", instanceType, response.Errors)
|
||||
}
|
||||
|
||||
// A simple delete adds a marker and keeps the old version. The explicitly
|
||||
// named version must also remain because its policy condition did not match.
|
||||
for objectName, versionID := range versionIDs {
|
||||
if _, err = obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{VersionID: versionID}); err != nil {
|
||||
t.Errorf("%s: version %s of %q was not preserved: %v", instanceType, versionID, objectName, err)
|
||||
}
|
||||
}
|
||||
for _, objectName := range []string{"without-version-id-before", "without-version-id-after"} {
|
||||
if _, err = obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); !isErrObjectNotFound(err) {
|
||||
t.Errorf("%s: simple delete of %q did not hide the latest object behind a delete marker: %v", instanceType, objectName, err)
|
||||
}
|
||||
}
|
||||
for _, objectName := range []string{"with-version-id", "with-null-version-id"} {
|
||||
if info, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err != nil {
|
||||
t.Errorf("%s: denied version delete removed latest %q: %v", instanceType, objectName, err)
|
||||
} else if info.VersionID != versionIDs[objectName] {
|
||||
t.Errorf("%s: latest version of %q changed from %s to %s", instanceType, objectName, versionIDs[objectName], info.VersionID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+156
-22
@@ -34,6 +34,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/policy/condition"
|
||||
)
|
||||
|
||||
// PolicySys - policy subsystem.
|
||||
@@ -75,7 +76,104 @@ func getSTSConditionValues(r *http.Request, lc string, cred auth.Credentials) ma
|
||||
return m
|
||||
}
|
||||
|
||||
type conditionValueSource uint8
|
||||
|
||||
const (
|
||||
conditionValueFromHeader conditionValueSource = 1 << iota
|
||||
conditionValueFromQuery
|
||||
)
|
||||
|
||||
// clientSuppliedConditionKeys records where each request-derived condition
|
||||
// value actually comes from. Most x-amz-* values are headers, list parameters
|
||||
// are query-only, and storage class retains the compatible query form consumed
|
||||
// by object operations.
|
||||
var clientSuppliedConditionKeys = map[string]conditionValueSource{
|
||||
"prefix": conditionValueFromQuery,
|
||||
"delimiter": conditionValueFromQuery,
|
||||
"max-keys": conditionValueFromQuery,
|
||||
// AWS explicitly excludes the query-string form from this policy key,
|
||||
// even though MinIO may consume it separately while verifying a presign.
|
||||
"x-amz-content-sha256": conditionValueFromHeader,
|
||||
"x-amz-copy-source": conditionValueFromHeader,
|
||||
"x-amz-metadata-directive": conditionValueFromHeader,
|
||||
"x-amz-server-side-encryption": conditionValueFromHeader,
|
||||
"x-amz-server-side-encryption-aws-kms-key-id": conditionValueFromHeader,
|
||||
"x-amz-server-side-encryption-customer-algorithm": conditionValueFromHeader,
|
||||
"x-amz-storage-class": conditionValueFromHeader | conditionValueFromQuery,
|
||||
}
|
||||
|
||||
func acceptsConditionValueSource(key string, source conditionValueSource) bool {
|
||||
name := strings.ToLower(key)
|
||||
allowed, ok := clientSuppliedConditionKeys[name]
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if allowed&source == 0 {
|
||||
return false
|
||||
}
|
||||
return source != conditionValueFromQuery || key == name
|
||||
}
|
||||
|
||||
// internalConditionKeys holds every other name a condition key can resolve to.
|
||||
// Those name values MinIO derives for itself - identity from the credential,
|
||||
// time from the clock, transport from the connection - and a request must never
|
||||
// write one, whether or not the server populated it this time round: a name the
|
||||
// server left empty is as forgeable as one it filled in, and the condition
|
||||
// reading it cannot tell the difference.
|
||||
//
|
||||
// Deriving the set from the condition keys rather than from what
|
||||
// getConditionValues writes is what makes it complete. The engine reads by key
|
||||
// name, so the key list is the attack surface; enumerating the writes misses
|
||||
// every key the server has no value for, which is most of jwt: and ldap:. It
|
||||
// also defaults new upstream keys to reserved, which is the safe direction.
|
||||
//
|
||||
// Reserving a name only removes it from the condition map. Request handling is
|
||||
// untouched - a handler still reads its own query parameters and headers.
|
||||
//
|
||||
// aws:SourceIp is still only as trustworthy as the forwarding headers it is
|
||||
// computed from, see the note on GetSourceIPFromHeaders.
|
||||
var internalConditionKeys = func() map[string]struct{} {
|
||||
keys := make(map[string]struct{}, 2*len(condition.AllSupportedKeys))
|
||||
for _, keyName := range condition.AllSupportedKeys {
|
||||
name := keyName.ToKey().Name()
|
||||
if _, clientSupplied := clientSuppliedConditionKeys[name]; clientSupplied {
|
||||
continue
|
||||
}
|
||||
// A condition key resolves against its exact name and falls back to the
|
||||
// canonical MIME form, so both spellings have to be held. This is also
|
||||
// what covers object lock, stored as Object-Lock-Mode and read as
|
||||
// s3:object-lock-mode.
|
||||
keys[name] = struct{}{}
|
||||
keys[http.CanonicalHeaderKey(name)] = struct{}{}
|
||||
}
|
||||
return keys
|
||||
}()
|
||||
|
||||
// Tag conditions name one tag key each, so the variable forms are reserved by
|
||||
// prefix; the bare names come from the loop above.
|
||||
var internalConditionKeyPrefixes = []string{"ExistingObjectTag/", "RequestObjectTag/"}
|
||||
|
||||
func isInternalConditionKey(key string) bool {
|
||||
if _, ok := internalConditionKeys[key]; ok {
|
||||
return true
|
||||
}
|
||||
for _, prefix := range internalConditionKeyPrefixes {
|
||||
if strings.HasPrefix(key, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[string][]string {
|
||||
return getConditionValuesWithExistingTags(r, lc, cred, "")
|
||||
}
|
||||
|
||||
func getConditionValuesWithExistingTags(r *http.Request, lc string, cred auth.Credentials, existingTags string) map[string][]string {
|
||||
return getConditionValuesWithTags(r, lc, cred, existingTags, nil)
|
||||
}
|
||||
|
||||
func getConditionValuesWithTags(r *http.Request, lc string, cred auth.Credentials, existingTags string, requestTags *string) map[string][]string {
|
||||
currTime := UTCNow()
|
||||
|
||||
var (
|
||||
@@ -100,10 +198,13 @@ func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[s
|
||||
}
|
||||
}
|
||||
|
||||
vid := r.Form.Get(xhttp.VersionID)
|
||||
// Match the version the object layer will act on: newContext and getOpts both
|
||||
// TrimSpace this value, so leaving it untrimmed here would let a padded
|
||||
// ?versionId=V%20 present a different s3:versionid than the effective version.
|
||||
vid := strings.TrimSpace(r.Form.Get(xhttp.VersionID))
|
||||
if vid == "" {
|
||||
if u, err := url.Parse(r.Header.Get(xhttp.AmzCopySource)); err == nil {
|
||||
vid = u.Query().Get(xhttp.VersionID)
|
||||
vid = strings.TrimSpace(u.Query().Get(xhttp.VersionID))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -136,34 +237,57 @@ func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[s
|
||||
"principaltype": {principalType},
|
||||
"userid": {username},
|
||||
"username": {username},
|
||||
"versionid": {vid},
|
||||
"signatureversion": {signatureVersion},
|
||||
"authType": {authtype},
|
||||
}
|
||||
|
||||
// Null conditions distinguish an absent key from a present key with an
|
||||
// empty value. Only expose s3:versionid when the request names a version.
|
||||
if vid != "" {
|
||||
args["versionid"] = []string{vid}
|
||||
}
|
||||
|
||||
if lc != "" {
|
||||
args["LocationConstraint"] = []string{lc}
|
||||
}
|
||||
|
||||
cloneHeader := r.Header.Clone()
|
||||
if v := cloneHeader.Get("x-amz-signature-age"); v != "" {
|
||||
args["signatureAge"] = []string{v}
|
||||
cloneHeader.Del("x-amz-signature-age")
|
||||
if storageClass, ok := getRequestHeaderOrQueryValue(r, xhttp.AmzStorageClass); ok {
|
||||
args[strings.ToLower(xhttp.AmzStorageClass)] = []string{storageClass}
|
||||
}
|
||||
|
||||
if userTags := cloneHeader.Get(xhttp.AmzObjectTagging); userTags != "" {
|
||||
cloneHeader := r.Header.Clone()
|
||||
signatureAge := cloneHeader.Get("x-amz-signature-age")
|
||||
cloneHeader.Del("x-amz-signature-age")
|
||||
// The presigned V4 verifier overwrites this internal scratch header after
|
||||
// validating the signature. Ignore a value supplied on every other request
|
||||
// type, where it would otherwise synthesize s3:signatureAge.
|
||||
if authType == authTypePresigned && signatureAge != "" {
|
||||
args["signatureAge"] = []string{signatureAge}
|
||||
}
|
||||
|
||||
userTags := cloneHeader.Get(xhttp.AmzObjectTagging)
|
||||
if requestTags != nil {
|
||||
userTags = *requestTags
|
||||
}
|
||||
if userTags != "" {
|
||||
tag, _ := tags.ParseObjectTags(userTags)
|
||||
if tag != nil {
|
||||
tagMap := tag.ToMap()
|
||||
keys := make([]string, 0, len(tagMap))
|
||||
for k, v := range tagMap {
|
||||
args[pathJoin("ExistingObjectTag", k)] = []string{v}
|
||||
args[pathJoin("RequestObjectTag", k)] = []string{v}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
args["RequestObjectTagKeys"] = keys
|
||||
}
|
||||
}
|
||||
if existingTags != "" {
|
||||
tag, _ := tags.ParseObjectTags(existingTags)
|
||||
if tag != nil {
|
||||
for k, v := range tag.ToMap() {
|
||||
args[pathJoin("ExistingObjectTag", k)] = []string{v}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, objLock := range []string{
|
||||
xhttp.AmzObjectLockMode,
|
||||
@@ -176,8 +300,20 @@ func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[s
|
||||
cloneHeader.Del(objLock)
|
||||
}
|
||||
|
||||
// The two loops below fold raw header and query values into the same map
|
||||
// the server just filled in. Anything they add is indistinguishable, to a
|
||||
// condition, from a value the server derived - and they merge by appending,
|
||||
// so a supplied entry sits alongside the real one rather than replacing it.
|
||||
// The source check keeps headers and query parameters in their actual roles;
|
||||
// isInternalConditionKey keeps both apart from server-derived values.
|
||||
for key, values := range cloneHeader {
|
||||
if strings.EqualFold(key, xhttp.AmzObjectTagging) {
|
||||
if strings.EqualFold(key, xhttp.AmzObjectTagging) || strings.EqualFold(key, xhttp.AmzStorageClass) {
|
||||
continue
|
||||
}
|
||||
if !acceptsConditionValueSource(key, conditionValueFromHeader) {
|
||||
continue
|
||||
}
|
||||
if isInternalConditionKey(key) {
|
||||
continue
|
||||
}
|
||||
if existingValues, found := args[key]; found {
|
||||
@@ -190,18 +326,16 @@ func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[s
|
||||
cloneURLValues := make(url.Values, len(r.Form))
|
||||
maps.Copy(cloneURLValues, r.Form)
|
||||
|
||||
for _, objLock := range []string{
|
||||
xhttp.AmzObjectLockMode,
|
||||
xhttp.AmzObjectLockLegalHold,
|
||||
xhttp.AmzObjectLockRetainUntilDate,
|
||||
} {
|
||||
if values, ok := cloneURLValues[objLock]; ok {
|
||||
args[strings.TrimPrefix(objLock, "X-Amz-")] = values
|
||||
}
|
||||
cloneURLValues.Del(objLock)
|
||||
}
|
||||
|
||||
for key, values := range cloneURLValues {
|
||||
if strings.EqualFold(key, xhttp.AmzObjectTagging) || strings.EqualFold(key, xhttp.AmzStorageClass) {
|
||||
continue
|
||||
}
|
||||
if !acceptsConditionValueSource(key, conditionValueFromQuery) {
|
||||
continue
|
||||
}
|
||||
if isInternalConditionKey(key) {
|
||||
continue
|
||||
}
|
||||
if existingValues, found := args[key]; found {
|
||||
args[key] = append(existingValues, values...)
|
||||
} else {
|
||||
|
||||
@@ -0,0 +1,679 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/policy/condition"
|
||||
)
|
||||
|
||||
const (
|
||||
testCondSourceIP = "203.0.113.5"
|
||||
testCondRemoteILP = testCondSourceIP + ":12345"
|
||||
)
|
||||
|
||||
func condValuesForRequest(t *testing.T, rawURL string, header map[string]string) map[string][]string {
|
||||
return condValuesForRequestWithTags(t, rawURL, header, "", nil)
|
||||
}
|
||||
|
||||
func condValuesForRequestWithExistingTags(t *testing.T, rawURL string, header map[string]string, existingTags string) map[string][]string {
|
||||
return condValuesForRequestWithTags(t, rawURL, header, existingTags, nil)
|
||||
}
|
||||
|
||||
func condValuesForRequestWithTags(t *testing.T, rawURL string, header map[string]string, existingTags string, requestTags *string) map[string][]string {
|
||||
t.Helper()
|
||||
r, err := http.NewRequest(http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.RemoteAddr = testCondRemoteILP
|
||||
for k, v := range header {
|
||||
r.Header.Set(k, v)
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return getConditionValuesWithTags(r, "us-east-1", auth.Credentials{AccessKey: "lowpriv"}, existingTags, requestTags)
|
||||
}
|
||||
|
||||
func resolvedConditionValues(values map[string][]string, name string) []string {
|
||||
if v := values[name]; len(v) > 0 {
|
||||
return v
|
||||
}
|
||||
return values[http.CanonicalHeaderKey(name)]
|
||||
}
|
||||
|
||||
// A client must not be able to reach a condition key that the server computes
|
||||
// for itself. Both routes are covered: a header whose canonical spelling
|
||||
// collides with the key name, and a query parameter that collides with it
|
||||
// exactly. The query route is the sharper one, because the merge appended to
|
||||
// the server's value rather than replacing it and a condition function matches
|
||||
// when any single value matches.
|
||||
func TestGetConditionValuesRejectsClientSuppliedServerKeys(t *testing.T) {
|
||||
honest := condValuesForRequest(t, "http://minio.local/bkt/obj", nil)
|
||||
|
||||
for _, kn := range condition.AllSupportedKeys {
|
||||
name := kn.ToKey().Name()
|
||||
if _, clientSupplied := clientSuppliedConditionKeys[name]; clientSupplied {
|
||||
continue // the request is where this one is supposed to come from
|
||||
}
|
||||
// Deliberately not skipped when the server left the key empty. An empty
|
||||
// name is exactly as forgeable as a populated one, and the keys the
|
||||
// server has no value for - most of jwt: and ldap: - are the ones a
|
||||
// resource variable expands.
|
||||
want := honest[name]
|
||||
canonical := http.CanonicalHeaderKey(name)
|
||||
|
||||
t.Run("query/"+name, func(t *testing.T) {
|
||||
got := condValuesForRequest(t,
|
||||
"http://minio.local/bkt/obj?"+url.Values{name: {"ATTACKER"}}.Encode(), nil)
|
||||
if slices.Contains(got[name], "ATTACKER") {
|
||||
t.Errorf("?%s= reached %v, server computed %v", name, got[name], want)
|
||||
}
|
||||
if !slices.Equal(got[name], want) {
|
||||
t.Errorf("%v changed to %v", want, got[name])
|
||||
}
|
||||
})
|
||||
|
||||
// aws:Referer is read out of the Referer header, so the header is its
|
||||
// source of truth rather than a way to forge it. aws:UserAgent is not
|
||||
// in the same position: it comes from User-Agent, which does not
|
||||
// canonicalise to "Useragent".
|
||||
if kn == condition.AWSReferer {
|
||||
continue
|
||||
}
|
||||
|
||||
t.Run("header/"+canonical, func(t *testing.T) {
|
||||
got := condValuesForRequest(t, "http://minio.local/bkt/obj",
|
||||
map[string]string{canonical: "ATTACKER"})
|
||||
// The lookup the policy engine itself performs, exact name first
|
||||
// with the canonical form as fallback.
|
||||
seen := got[name]
|
||||
if len(seen) == 0 {
|
||||
seen = got[canonical]
|
||||
}
|
||||
if slices.Contains(seen, "ATTACKER") {
|
||||
t.Errorf("%s: header reached the lookup as %v, server computed %v",
|
||||
canonical, seen, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetConditionValuesUsesActualRequestSource(t *testing.T) {
|
||||
for name, source := range clientSuppliedConditionKeys {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fromHeader := condValuesForRequest(t, "http://minio.local/bkt/obj",
|
||||
map[string]string{name: "HEADER"})
|
||||
fromQuery := condValuesForRequest(t,
|
||||
"http://minio.local/bkt/obj?"+url.Values{name: {"QUERY"}}.Encode(), nil)
|
||||
fromCanonicalQuery := condValuesForRequest(t,
|
||||
"http://minio.local/bkt/obj?"+url.Values{http.CanonicalHeaderKey(name): {"QUERY"}}.Encode(), nil)
|
||||
if got, want := slices.Contains(resolvedConditionValues(fromHeader, name), "HEADER"), source&conditionValueFromHeader != 0; got != want {
|
||||
t.Errorf("header accepted=%v, want %v: %v", got, want, fromHeader)
|
||||
}
|
||||
if got, want := slices.Contains(resolvedConditionValues(fromQuery, name), "QUERY"), source&conditionValueFromQuery != 0; got != want {
|
||||
t.Errorf("query accepted=%v, want %v: %v", got, want, fromQuery)
|
||||
}
|
||||
canonicalQueryAllowed := name == strings.ToLower(xhttp.AmzStorageClass)
|
||||
if got := slices.Contains(resolvedConditionValues(fromCanonicalQuery, name), "QUERY"); got != canonicalQueryAllowed {
|
||||
t.Errorf("case-variant query accepted=%v, want %v: %v", got, canonicalQueryAllowed, fromCanonicalQuery)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
storageURL := "http://minio.local/bkt/obj?" + url.Values{
|
||||
strings.ToLower(xhttp.AmzStorageClass): {"QUERY"},
|
||||
}.Encode()
|
||||
storageValues := condValuesForRequest(t, storageURL, map[string]string{xhttp.AmzStorageClass: "HEADER"})
|
||||
if got := resolvedConditionValues(storageValues, strings.ToLower(xhttp.AmzStorageClass)); !slices.Equal(got, []string{"HEADER"}) {
|
||||
t.Errorf("storage class did not use header precedence: %v", got)
|
||||
}
|
||||
|
||||
fromQuery := condValuesForRequest(t,
|
||||
"http://minio.local/bkt/obj?"+url.Values{xhttp.AmzObjectLockMode: {"COMPLIANCE"}}.Encode(), nil)
|
||||
if got := resolvedConditionValues(fromQuery, "object-lock-mode"); len(got) != 0 {
|
||||
t.Errorf("object-lock query value reached header condition as %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetConditionValuesVersionIDPresence(t *testing.T) {
|
||||
nullVersionID, err := condition.NewNullFunc(condition.S3VersionID.ToKey(), true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
withoutVersionID := condValuesForRequest(t, "http://minio.local/bkt/obj", nil)
|
||||
if _, ok := withoutVersionID["versionid"]; ok {
|
||||
t.Fatalf("an absent versionId was exposed to policy evaluation as %v", withoutVersionID["versionid"])
|
||||
}
|
||||
if !condition.NewFunctions(nullVersionID).Evaluate(withoutVersionID) {
|
||||
t.Fatal("Null s3:versionid=true did not match a request without versionId")
|
||||
}
|
||||
|
||||
const versionID = "7f4b6b5f-bf25-4e98-95df-90cba8070dd8"
|
||||
withVersionID := condValuesForRequest(t,
|
||||
"http://minio.local/bkt/obj?"+url.Values{xhttp.VersionID: {versionID}}.Encode(), nil)
|
||||
if got := withVersionID["versionid"]; !slices.Equal(got, []string{versionID}) {
|
||||
t.Fatalf("expected versionId %q, got %v", versionID, got)
|
||||
}
|
||||
if condition.NewFunctions(nullVersionID).Evaluate(withVersionID) {
|
||||
t.Fatal("Null s3:versionid=true matched a request with versionId")
|
||||
}
|
||||
|
||||
copySourceVersion := condValuesForRequest(t, "http://minio.local/bkt/copied", map[string]string{
|
||||
xhttp.AmzCopySource: "/source-bucket/source-object?" + url.Values{xhttp.VersionID: {versionID}}.Encode(),
|
||||
})
|
||||
if got := copySourceVersion["versionid"]; !slices.Equal(got, []string{versionID}) {
|
||||
t.Fatalf("copy source versionId was lost: got %v", got)
|
||||
}
|
||||
|
||||
// The object layer trims the version before acting on it; the condition value
|
||||
// must be the same effective string, or a padded ?versionId=V%20 would let a
|
||||
// StringEquals/Deny on s3:versionid see a different value than the one deleted.
|
||||
paddedVersion := condValuesForRequest(t,
|
||||
"http://minio.local/bkt/obj?"+url.Values{xhttp.VersionID: {versionID + " "}}.Encode(), nil)
|
||||
if got := paddedVersion["versionid"]; !slices.Equal(got, []string{versionID}) {
|
||||
t.Fatalf("a padded versionId was not trimmed to the effective value: got %v", got)
|
||||
}
|
||||
|
||||
paddedCopySource := condValuesForRequest(t, "http://minio.local/bkt/copied", map[string]string{
|
||||
xhttp.AmzCopySource: "/source-bucket/source-object?" + url.Values{xhttp.VersionID: {versionID + " "}}.Encode(),
|
||||
})
|
||||
if got := paddedCopySource["versionid"]; !slices.Equal(got, []string{versionID}) {
|
||||
t.Fatalf("a padded copy source versionId was not trimmed: got %v", got)
|
||||
}
|
||||
|
||||
// A whitespace-only versionId names no version once trimmed, exactly as the
|
||||
// object layer treats it, so the key must be absent and Null:true must match.
|
||||
blankVersion := condValuesForRequest(t,
|
||||
"http://minio.local/bkt/obj?"+url.Values{xhttp.VersionID: {" "}}.Encode(), nil)
|
||||
if _, ok := blankVersion["versionid"]; ok {
|
||||
t.Fatalf("a whitespace-only versionId was exposed to policy evaluation as %v", blankVersion["versionid"])
|
||||
}
|
||||
if !condition.NewFunctions(nullVersionID).Evaluate(blankVersion) {
|
||||
t.Fatal("Null s3:versionid=true did not match a request whose versionId was only whitespace")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetConditionValuesUsesEffectiveRequestTags(t *testing.T) {
|
||||
rawURL := "http://minio.local/bkt/obj?" + url.Values{
|
||||
strings.ToLower(xhttp.AmzObjectTagging): {"security=public&virus=true"},
|
||||
}.Encode()
|
||||
|
||||
// Generic operations such as CopyObject must not gain RequestObjectTag
|
||||
// values from a query parameter they do not consume.
|
||||
withoutEffectiveTags := condValuesForRequest(t, rawURL, nil)
|
||||
if len(withoutEffectiveTags["RequestObjectTag/security"]) != 0 || len(withoutEffectiveTags["RequestObjectTagKeys"]) != 0 {
|
||||
t.Fatalf("query tags leaked into a generic operation: %v", withoutEffectiveTags)
|
||||
}
|
||||
|
||||
effectiveTags := "security=public&virus=true"
|
||||
withEffectiveTags := condValuesForRequestWithTags(t, rawURL, nil, "", &effectiveTags)
|
||||
if !slices.Equal(withEffectiveTags["RequestObjectTag/security"], []string{"public"}) {
|
||||
t.Fatalf("effective request tag missing: %v", withEffectiveTags)
|
||||
}
|
||||
if !slices.Contains(withEffectiveTags["RequestObjectTagKeys"], "security") ||
|
||||
!slices.Contains(withEffectiveTags["RequestObjectTagKeys"], "virus") {
|
||||
t.Fatalf("effective request tag keys missing: %v", withEffectiveTags["RequestObjectTagKeys"])
|
||||
}
|
||||
|
||||
security, err := condition.NewStringEqualsFunc("", condition.NewKey(condition.RequestObjectTag, "security"), "public")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
allowedKeys, err := condition.NewStringLikeFunc("ForAllValues", condition.RequestObjectTagKeys.ToKey(), "security", "virus")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conditions := condition.NewFunctions(security, allowedKeys)
|
||||
if conditions.Evaluate(withoutEffectiveTags) {
|
||||
t.Fatal("query upload satisfied request-tag policy without effective tags")
|
||||
}
|
||||
if !conditions.Evaluate(withEffectiveTags) {
|
||||
t.Fatal("effective query tags did not satisfy request-tag policy")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketPolicySSEConditionUsesHeader(t *testing.T) {
|
||||
fn, err := condition.NewStringEqualsFunc("", condition.S3XAmzServerSideEncryption.ToKey(), "aws:kms")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conditions := condition.NewFunctions(fn)
|
||||
if conditions.Evaluate(condValuesForRequest(t,
|
||||
"http://minio.local/bkt/obj?x-amz-server-side-encryption=aws%3Akms", nil)) {
|
||||
t.Error("query parameter satisfied a condition on the SSE request header")
|
||||
}
|
||||
if !conditions.Evaluate(condValuesForRequest(t, "http://minio.local/bkt/obj",
|
||||
map[string]string{xhttp.AmzServerSideEncryption: "aws:kms"})) {
|
||||
t.Error("SSE request header did not satisfy its condition")
|
||||
}
|
||||
}
|
||||
|
||||
// The end to end shape of the bypass: an IpAddress condition restricting a
|
||||
// bucket to an internal range, against a request from outside it.
|
||||
func TestBucketPolicySourceIPCannotBeForged(t *testing.T) {
|
||||
_, cidr, err := net.ParseCIDR("10.0.0.0/8")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fn, err := condition.NewIPAddressFunc(condition.AWSSourceIP.ToKey(), cidr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bp := policy.BucketPolicy{
|
||||
Version: policy.DefaultVersion,
|
||||
Statements: []policy.BPStatement{{
|
||||
Effect: policy.Allow,
|
||||
Principal: policy.NewPrincipal("*"),
|
||||
Actions: policy.NewActionSet(policy.GetObjectAction),
|
||||
Resources: policy.NewResourceSet(policy.NewResource("bkt/*")),
|
||||
Conditions: condition.NewFunctions(fn),
|
||||
}},
|
||||
}
|
||||
allowed := func(rawURL string, header map[string]string) bool {
|
||||
return bp.IsAllowed(policy.BucketPolicyArgs{
|
||||
Action: policy.GetObjectAction,
|
||||
BucketName: "bkt",
|
||||
ObjectName: "obj",
|
||||
ConditionValues: condValuesForRequest(t, rawURL, header),
|
||||
})
|
||||
}
|
||||
|
||||
if allowed("http://minio.local/bkt/obj", nil) {
|
||||
t.Fatal("baseline: an address outside 10.0.0.0/8 must not satisfy the condition")
|
||||
}
|
||||
if allowed("http://minio.local/bkt/obj?SourceIp=10.1.2.3", nil) {
|
||||
t.Error("a query parameter forged aws:SourceIp")
|
||||
}
|
||||
if allowed("http://minio.local/bkt/obj", map[string]string{"Sourceip": "10.1.2.3"}) {
|
||||
t.Error("a header forged aws:SourceIp")
|
||||
}
|
||||
}
|
||||
|
||||
// The trust policy has to reach the decision, not merely the resolver. This
|
||||
// drives a forged X-Forwarded-For all the way through getConditionValues into a
|
||||
// real IpAddress evaluation under each mode. Everything else about the trust
|
||||
// modes is tested where the logic lives; this is the only test that would notice
|
||||
// if the resolver were correct but the policy engine were reading something else.
|
||||
func TestBucketPolicySourceIPForgeryAcrossTrustModes(t *testing.T) {
|
||||
_, cidr, err := net.ParseCIDR("10.0.0.0/8")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fn, err := condition.NewIPAddressFunc(condition.AWSSourceIP.ToKey(), cidr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bp := policy.BucketPolicy{
|
||||
Version: policy.DefaultVersion,
|
||||
Statements: []policy.BPStatement{{
|
||||
Effect: policy.Allow,
|
||||
Principal: policy.NewPrincipal("*"),
|
||||
Actions: policy.NewActionSet(policy.GetObjectAction),
|
||||
Resources: policy.NewResourceSet(policy.NewResource("bkt/*")),
|
||||
Conditions: condition.NewFunctions(fn),
|
||||
}},
|
||||
}
|
||||
|
||||
// An address inside the permitted range, asserted by a client that is not.
|
||||
const forgedClaim = "10.1.2.3"
|
||||
const outsider = "203.0.113.5:12345"
|
||||
const proxy = "192.0.2.7:9000"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
proxies string
|
||||
peer string
|
||||
allowed bool
|
||||
}{{
|
||||
// Documented, and the reason the allow-list exists.
|
||||
name: "default mode believes the claim",
|
||||
peer: outsider,
|
||||
allowed: true,
|
||||
}, {
|
||||
name: "trusting nobody ignores the claim",
|
||||
proxies: handlers.TrustNoProxies,
|
||||
peer: outsider,
|
||||
allowed: false,
|
||||
}, {
|
||||
name: "allow-list ignores an unlisted peer's claim",
|
||||
proxies: "192.0.2.7",
|
||||
peer: outsider,
|
||||
allowed: false,
|
||||
}, {
|
||||
// The allow-list must not break the deployment it exists to serve.
|
||||
name: "allow-list still honors its own proxy",
|
||||
proxies: "192.0.2.7",
|
||||
peer: proxy,
|
||||
allowed: true,
|
||||
}}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Cleanup(func() {
|
||||
os.Unsetenv(handlers.EnvTrustedProxies)
|
||||
if err := handlers.ConfigureSourceIPTrust(); err != nil {
|
||||
t.Fatalf("restoring the default policy: %v", err)
|
||||
}
|
||||
})
|
||||
if tt.proxies == "" {
|
||||
os.Unsetenv(handlers.EnvTrustedProxies)
|
||||
} else {
|
||||
t.Setenv(handlers.EnvTrustedProxies, tt.proxies)
|
||||
}
|
||||
if err := handlers.ConfigureSourceIPTrust(); err != nil {
|
||||
t.Fatalf("configuring %q: %v", tt.proxies, err)
|
||||
}
|
||||
|
||||
r, err := http.NewRequest(http.MethodGet, "http://minio.local/bkt/obj", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.RemoteAddr = tt.peer
|
||||
r.Header.Set("X-Forwarded-For", forgedClaim)
|
||||
if err := r.ParseForm(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got := bp.IsAllowed(policy.BucketPolicyArgs{
|
||||
Action: policy.GetObjectAction,
|
||||
BucketName: "bkt",
|
||||
ObjectName: "obj",
|
||||
ConditionValues: getConditionValues(r, "us-east-1", auth.Credentials{AccessKey: "lowpriv"}),
|
||||
})
|
||||
if got != tt.allowed {
|
||||
t.Errorf("IsAllowed = %v, want %v (peer %s claiming %s)", got, tt.allowed, tt.peer, forgedClaim)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// aws:SourceIp must be whatever the hardened resolver decided and nothing else.
|
||||
// The resolver is where the forwarded-header trust policy is enforced and where
|
||||
// its three modes are tested (internal/handlers/proxy_test.go); this pins the
|
||||
// join, so the condition value cannot drift onto some other derivation that the
|
||||
// policy would not cover.
|
||||
//
|
||||
// It also records the default-mode contract: with no trust policy configured,
|
||||
// each of the three forwarded headers still sets aws:SourceIp, and so an
|
||||
// IpAddress condition is only as good as the network path to the API port.
|
||||
// Enforcing such a condition against a client with direct access requires
|
||||
// MINIO_API_TRUSTED_PROXIES. Note that _MINIO_API_XFF_HEADER=off does not
|
||||
// achieve it: the loop below covers all three headers precisely because
|
||||
// suppressing one of them only moves the answer to the next.
|
||||
func TestGetConditionValuesSourceIPMatchesResolver(t *testing.T) {
|
||||
for _, header := range []map[string]string{
|
||||
nil,
|
||||
{"X-Forwarded-For": "10.1.2.3"},
|
||||
{"X-Real-IP": "10.1.2.3"},
|
||||
{"Forwarded": "for=10.1.2.3"},
|
||||
{"X-Forwarded-For": "10.1.2.3, 198.51.100.9"},
|
||||
} {
|
||||
r, err := http.NewRequest(http.MethodGet, "http://minio.local/bkt/obj", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.RemoteAddr = testCondRemoteILP
|
||||
for k, v := range header {
|
||||
r.Header.Set(k, v)
|
||||
}
|
||||
|
||||
got := resolvedConditionValues(condValuesForRequest(t, "http://minio.local/bkt/obj", header), condition.AWSSourceIP.ToKey().Name())
|
||||
want := handlers.GetSourceIPRaw(r)
|
||||
if len(got) != 1 || got[0] != want {
|
||||
t.Errorf("headers %v: aws:SourceIp = %v, resolver returned %q", header, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// "Deny unless the connection is TLS" is the usual hardening statement, and
|
||||
// aws:SecureTransport is computed from r.TLS.
|
||||
func TestBucketPolicySecureTransportCannotBeForged(t *testing.T) {
|
||||
fn, err := condition.NewBoolFunc(condition.AWSSecureTransport.ToKey(), false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bp := policy.BucketPolicy{
|
||||
Version: policy.DefaultVersion,
|
||||
Statements: []policy.BPStatement{
|
||||
{
|
||||
Effect: policy.Allow, Principal: policy.NewPrincipal("*"),
|
||||
Actions: policy.NewActionSet(policy.GetObjectAction),
|
||||
Resources: policy.NewResourceSet(policy.NewResource("bkt/*")),
|
||||
},
|
||||
{
|
||||
Effect: policy.Deny, Principal: policy.NewPrincipal("*"),
|
||||
Actions: policy.NewActionSet(policy.GetObjectAction),
|
||||
Resources: policy.NewResourceSet(policy.NewResource("bkt/*")),
|
||||
Conditions: condition.NewFunctions(fn),
|
||||
},
|
||||
},
|
||||
}
|
||||
allowed := func(rawURL string, header map[string]string) bool {
|
||||
return bp.IsAllowed(policy.BucketPolicyArgs{
|
||||
Action: policy.GetObjectAction,
|
||||
BucketName: "bkt",
|
||||
ObjectName: "obj",
|
||||
ConditionValues: condValuesForRequest(t, rawURL, header),
|
||||
})
|
||||
}
|
||||
|
||||
// r.TLS is nil throughout, so every one of these is a plaintext request.
|
||||
if allowed("http://minio.local/bkt/obj", nil) {
|
||||
t.Fatal("baseline: a plaintext request must be denied")
|
||||
}
|
||||
if allowed("http://minio.local/bkt/obj?SecureTransport=true", nil) {
|
||||
t.Error("a query parameter forged aws:SecureTransport")
|
||||
}
|
||||
if allowed("http://minio.local/bkt/obj", map[string]string{"Securetransport": "true"}) {
|
||||
t.Error("a header forged aws:SecureTransport")
|
||||
}
|
||||
}
|
||||
|
||||
// Reserving the server's own keys must not stop the request from supplying the
|
||||
// values that are client-derived by design.
|
||||
func TestGetConditionValuesKeepsClientDerivedKeys(t *testing.T) {
|
||||
got := condValuesForRequest(t, "http://minio.local/bkt/obj?prefix=team%2F",
|
||||
map[string]string{
|
||||
xhttp.AmzObjectLockMode: "GOVERNANCE",
|
||||
xhttp.AmzServerSideEncryption: "aws:kms",
|
||||
"X-Amz-Meta-Team": "storage",
|
||||
xhttp.AmzObjectTagging: "project=silo",
|
||||
})
|
||||
|
||||
for _, tc := range []struct {
|
||||
key string
|
||||
want string
|
||||
}{
|
||||
{"Object-Lock-Mode", "GOVERNANCE"},
|
||||
{xhttp.AmzServerSideEncryption, "aws:kms"},
|
||||
{"X-Amz-Meta-Team", "storage"},
|
||||
{"RequestObjectTag/project", "silo"},
|
||||
{"prefix", "team/"},
|
||||
} {
|
||||
if !slices.Contains(got[tc.key], tc.want) {
|
||||
t.Errorf("%s: expected %q, got %v", tc.key, tc.want, got[tc.key])
|
||||
}
|
||||
}
|
||||
if !slices.Contains(got["RequestObjectTagKeys"], "project") {
|
||||
t.Errorf("RequestObjectTagKeys: expected project, got %v", got["RequestObjectTagKeys"])
|
||||
}
|
||||
|
||||
if len(got["ExistingObjectTag/project"]) != 0 {
|
||||
t.Errorf("request tags leaked into ExistingObjectTag: %v", got["ExistingObjectTag/project"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetConditionValuesSeparatesRequestAndExistingTags(t *testing.T) {
|
||||
got := condValuesForRequestWithExistingTags(t, "http://minio.local/bkt/obj",
|
||||
map[string]string{xhttp.AmzObjectTagging: "project=request&new=yes"},
|
||||
"project=stored&old=yes")
|
||||
|
||||
for _, tc := range []struct {
|
||||
key string
|
||||
want string
|
||||
}{
|
||||
{"RequestObjectTag/project", "request"},
|
||||
{"RequestObjectTag/new", "yes"},
|
||||
{"ExistingObjectTag/project", "stored"},
|
||||
{"ExistingObjectTag/old", "yes"},
|
||||
} {
|
||||
if !slices.Equal(got[tc.key], []string{tc.want}) {
|
||||
t.Errorf("%s: expected %q, got %v", tc.key, tc.want, got[tc.key])
|
||||
}
|
||||
}
|
||||
if len(got["ExistingObjectTag/new"]) != 0 || len(got["RequestObjectTag/old"]) != 0 {
|
||||
t.Errorf("tag sources crossed: request new=%v, existing old=%v",
|
||||
got["ExistingObjectTag/new"], got["RequestObjectTag/old"])
|
||||
}
|
||||
}
|
||||
|
||||
// Keys the server did not populate for this request are as forgeable as ones it
|
||||
// did, so the reservation cannot depend on presence.
|
||||
func TestGetConditionValuesRejectsAbsentInternalKeys(t *testing.T) {
|
||||
for _, key := range []string{
|
||||
"signatureAge",
|
||||
"groups",
|
||||
"DurationSeconds",
|
||||
"ExistingObjectTag/security",
|
||||
"RequestObjectTag/security",
|
||||
"RequestObjectTagKeys",
|
||||
"object-lock-mode",
|
||||
"object-lock-remaining-retention-days",
|
||||
} {
|
||||
t.Run(key, func(t *testing.T) {
|
||||
got := condValuesForRequest(t,
|
||||
"http://minio.local/bkt/obj?"+url.Values{key: {"ATTACKER"}}.Encode(), nil)
|
||||
if slices.Contains(got[key], "ATTACKER") {
|
||||
t.Errorf("?%s= was accepted into the condition values as %v", key, got[key])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetConditionValuesOnlyAcceptsPresignedSignatureAge(t *testing.T) {
|
||||
const signatureAgeHeader = "x-amz-signature-age"
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
target string
|
||||
headers map[string]string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "anonymous client header",
|
||||
target: "http://minio.local/bkt/obj",
|
||||
headers: map[string]string{signatureAgeHeader: "1"},
|
||||
},
|
||||
{
|
||||
name: "header-signed client header",
|
||||
target: "http://minio.local/bkt/obj",
|
||||
headers: map[string]string{
|
||||
xhttp.Authorization: signV4Algorithm + " attacker",
|
||||
signatureAgeHeader: "1",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "presigned verifier value",
|
||||
target: "http://minio.local/bkt/obj?" + url.Values{
|
||||
xhttp.AmzCredential: {"access/20260803/us-east-1/s3/aws4_request"},
|
||||
}.Encode(),
|
||||
headers: map[string]string{signatureAgeHeader: "250"},
|
||||
want: true,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := condValuesForRequest(t, tc.target, tc.headers)
|
||||
_, ok := got["signatureAge"]
|
||||
if ok != tc.want {
|
||||
t.Fatalf("signatureAge presence: expected %v, got %v", tc.want, got["signatureAge"])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The object-lock value is stored under the header spelling while the policy key
|
||||
// that reads it is lower case. Reserving only one spelling lets the other be
|
||||
// supplied and resolved in its place - which the policy package's exact-name
|
||||
// lookup then prefers over the real one.
|
||||
func TestGetConditionValuesObjectLockSpelling(t *testing.T) {
|
||||
got := condValuesForRequest(t,
|
||||
"http://minio.local/bkt/obj?object-lock-mode=COMPLIANCE",
|
||||
map[string]string{xhttp.AmzObjectLockMode: "GOVERNANCE"})
|
||||
|
||||
if v, ok := got["object-lock-mode"]; ok {
|
||||
t.Errorf("the lower-case spelling was accepted: %v", v)
|
||||
}
|
||||
if !slices.Equal(got["Object-Lock-Mode"], []string{"GOVERNANCE"}) {
|
||||
t.Errorf("expected the header value to stand, got %v", got["Object-Lock-Mode"])
|
||||
}
|
||||
|
||||
fn, err := condition.NewStringEqualsFunc("",
|
||||
condition.S3ObjectLockMode.ToKey(), "COMPLIANCE")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if condition.NewFunctions(fn).Evaluate(got) {
|
||||
t.Error("a policy requiring COMPLIANCE was satisfied by a GOVERNANCE request")
|
||||
}
|
||||
}
|
||||
|
||||
// Resource variables read the condition map directly, so a forgeable key is a
|
||||
// forgeable resource path. ${ldap:user} and ${jwt:preferred_username} are the
|
||||
// home-directory idiom for LDAP and OIDC deployments; the server derives them
|
||||
// from the credential, and a request must not be able to answer them.
|
||||
func TestBucketPolicyResourceVariableCannotBeForged(t *testing.T) {
|
||||
for _, tc := range []struct{ variable, param, value string }{
|
||||
{"${ldap:user}", "user", "alice"},
|
||||
{"${ldap:username}", "username", "alice"},
|
||||
{"${jwt:preferred_username}", "preferred_username", "alice"},
|
||||
{"${jwt:sub}", "sub", "alice"},
|
||||
{"${aws:username}", "username", "alice"},
|
||||
} {
|
||||
t.Run(tc.variable, func(t *testing.T) {
|
||||
bp := policy.BucketPolicy{Version: policy.DefaultVersion, Statements: []policy.BPStatement{{
|
||||
Effect: policy.Allow,
|
||||
Principal: policy.NewPrincipal("*"),
|
||||
Actions: policy.NewActionSet(policy.GetObjectAction),
|
||||
Resources: policy.NewResourceSet(policy.NewResource("bkt/" + tc.variable + "/*")),
|
||||
}}}
|
||||
args := policy.BucketPolicyArgs{
|
||||
Action: policy.GetObjectAction, BucketName: "bkt", ObjectName: tc.value + "/secret",
|
||||
}
|
||||
args.ConditionValues = condValuesForRequest(t,
|
||||
"http://minio.local/bkt/"+tc.value+"/secret?"+
|
||||
url.Values{tc.param: {tc.value}}.Encode(), nil)
|
||||
if bp.IsAllowed(args) {
|
||||
t.Errorf("?%s=%s expanded %s and granted the prefix", tc.param, tc.value, tc.variable)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,379 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
)
|
||||
|
||||
// These tests pin the IAM bucket/object resource boundary end to end, through
|
||||
// the real handlers rather than the matcher alone. An object-only resource
|
||||
// pattern ("arn:aws:s3:::bucket/*") must not authorize the bucket-level writes
|
||||
// that hand a caller something its object access does not already provide —
|
||||
// upstream minio/minio issue #20449 — while every shape the fix deliberately
|
||||
// leaves alone keeps working. Both directions are asserted, because a change
|
||||
// here that only removes permissions is correct and one that adds any is not.
|
||||
|
||||
// A bucket-level write reached only through an object-only grant must be
|
||||
// refused, and refusing it must not destroy the bucket. A grant that names the
|
||||
// bucket must still succeed, and succeeding must actually remove it.
|
||||
func assertBucketDelete(ctx context.Context, c *check, admin, client *minio.Client, bucket string, wantAllowed bool) {
|
||||
c.Helper()
|
||||
err := client.RemoveBucket(ctx, bucket)
|
||||
if wantAllowed {
|
||||
if err != nil {
|
||||
c.Fatalf("RemoveBucket(%s) denied, want allowed: %v", bucket, err)
|
||||
}
|
||||
exists, existsErr := admin.BucketExists(ctx, bucket)
|
||||
if existsErr != nil {
|
||||
c.Fatalf("check removed bucket %s: %v", bucket, existsErr)
|
||||
}
|
||||
if exists {
|
||||
c.Fatalf("RemoveBucket(%s) returned nil but bucket still exists", bucket)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
c.Fatalf("RemoveBucket(%s) returned nil, want AccessDenied", bucket)
|
||||
}
|
||||
if response := minio.ToErrorResponse(err); response.Code != "AccessDenied" {
|
||||
c.Fatalf("RemoveBucket(%s) error code=%q, want AccessDenied (err=%v)", bucket, response.Code, err)
|
||||
}
|
||||
exists, existsErr := admin.BucketExists(ctx, bucket)
|
||||
if existsErr != nil {
|
||||
c.Fatalf("check protected bucket %s: %v", bucket, existsErr)
|
||||
}
|
||||
if !exists {
|
||||
c.Fatalf("RemoveBucket(%s) returned AccessDenied but bucket disappeared", bucket)
|
||||
}
|
||||
if err := admin.RemoveBucket(ctx, bucket); err != nil {
|
||||
c.Fatalf("cleanup protected bucket %s: %v", bucket, err)
|
||||
}
|
||||
}
|
||||
|
||||
func createUserWithPolicy(ctx context.Context, c *check, s *TestSuiteIAM, policyJSON []byte) (*minio.Client, string) {
|
||||
c.Helper()
|
||||
accessKey, secretKey := mustGenerateCredentials(c)
|
||||
if err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled); err != nil {
|
||||
c.Fatalf("set boundary user: %v", err)
|
||||
}
|
||||
policyName := "boundary-" + mustGetUUID()
|
||||
if err := s.adm.AddCannedPolicy(ctx, policyName, policyJSON); err != nil {
|
||||
c.Fatalf("add boundary policy: %v", err)
|
||||
}
|
||||
if _, err := s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
|
||||
Policies: []string{policyName},
|
||||
User: accessKey,
|
||||
}); err != nil {
|
||||
c.Fatalf("attach boundary policy: %v", err)
|
||||
}
|
||||
return s.getUserClient(c, accessKey, secretKey, ""), accessKey
|
||||
}
|
||||
|
||||
func TestBucketResourceBoundaryEndToEnd(t *testing.T) {
|
||||
if runtime.GOOS == globalWindowsOSName {
|
||||
t.Skip("IAM integration harness is disabled on Windows")
|
||||
}
|
||||
suite := newTestSuiteIAM(TestSuiteCommon{serverType: "ErasureSD", signer: signerV4}, false)
|
||||
c := &check{t, suite.serverType}
|
||||
suite.SetUpSuite(c)
|
||||
defer suite.TearDownSuite(c)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// The #20449 reproduction: "s3:*" on "bucket/*" and nothing else.
|
||||
coreBucket := getRandomBucketName()
|
||||
if err := suite.client.MakeBucket(ctx, coreBucket, minio.MakeBucketOptions{}); err != nil {
|
||||
c.Fatalf("create core bucket: %v", err)
|
||||
}
|
||||
corePolicy := fmt.Appendf(nil, `{
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"arn:aws:s3:::%s/*"}]
|
||||
}`, coreBucket)
|
||||
coreClient, _ := createUserWithPolicy(ctx, c, suite, corePolicy)
|
||||
assertBucketDelete(ctx, c, suite.client, coreClient, coreBucket, false)
|
||||
|
||||
// The conventional pairing of bucket and object ARNs stays authorized.
|
||||
pairedBucket := getRandomBucketName()
|
||||
if err := suite.client.MakeBucket(ctx, pairedBucket, minio.MakeBucketOptions{}); err != nil {
|
||||
c.Fatalf("create paired bucket: %v", err)
|
||||
}
|
||||
pairedPolicy := fmt.Appendf(nil, `{
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[{"Effect":"Allow","Action":"s3:*","Resource":[
|
||||
"arn:aws:s3:::%s","arn:aws:s3:::%s/*"
|
||||
]}]
|
||||
}`, pairedBucket, pairedBucket)
|
||||
pairedClient, _ := createUserWithPolicy(ctx, c, suite, pairedPolicy)
|
||||
assertBucketDelete(ctx, c, suite.client, pairedClient, pairedBucket, true)
|
||||
|
||||
// CreateBucket and ListBucket keep the historical object-pattern matching.
|
||||
compatBucket := getRandomBucketName()
|
||||
compatPolicy := fmt.Appendf(nil, `{
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"arn:aws:s3:::%s/*"}]
|
||||
}`, compatBucket)
|
||||
compatClient, _ := createUserWithPolicy(ctx, c, suite, compatPolicy)
|
||||
if err := compatClient.MakeBucket(ctx, compatBucket, minio.MakeBucketOptions{}); err != nil {
|
||||
c.Fatalf("CreateBucket compatibility path denied: %v", err)
|
||||
}
|
||||
for item := range compatClient.ListObjects(ctx, compatBucket, minio.ListObjectsOptions{}) {
|
||||
if item.Err != nil {
|
||||
c.Fatalf("ListBucket compatibility path denied: %v", item.Err)
|
||||
}
|
||||
}
|
||||
if err := suite.client.RemoveBucket(ctx, compatBucket); err != nil {
|
||||
c.Fatalf("cleanup compatibility bucket: %v", err)
|
||||
}
|
||||
|
||||
// Withholding the trailing slash changes the string patterns match against,
|
||||
// so a fixed-width wildcard can match the bare bucket name without ever
|
||||
// having matched "bucket/". Honoring it would GRANT a delete the historical
|
||||
// matcher refused, which the hardening must never do.
|
||||
wildcardBucket := getRandomBucketName()
|
||||
if err := suite.client.MakeBucket(ctx, wildcardBucket, minio.MakeBucketOptions{}); err != nil {
|
||||
c.Fatalf("create fixed-width wildcard bucket: %v", err)
|
||||
}
|
||||
wildcardPolicy := fmt.Appendf(nil, `{
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[{"Effect":"Allow","Action":"s3:DeleteBucket","Resource":"arn:aws:s3:::%s?"}]
|
||||
}`, wildcardBucket[:len(wildcardBucket)-1])
|
||||
wildcardClient, _ := createUserWithPolicy(ctx, c, suite, wildcardPolicy)
|
||||
assertBucketDelete(ctx, c, suite.client, wildcardClient, wildcardBucket, false)
|
||||
|
||||
// A NotResource exclusion keeps its historical reach, so narrowing it — and
|
||||
// thereby broadening the Allow it qualifies — cannot happen unnoticed.
|
||||
notResourceBucket := getRandomBucketName()
|
||||
if err := suite.client.MakeBucket(ctx, notResourceBucket, minio.MakeBucketOptions{}); err != nil {
|
||||
c.Fatalf("create NotResource bucket: %v", err)
|
||||
}
|
||||
notResourcePolicy := fmt.Appendf(nil, `{
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[{"Effect":"Allow","Action":"s3:*","NotResource":"arn:aws:s3:::%s/*"}]
|
||||
}`, notResourceBucket)
|
||||
notResourceClient, _ := createUserWithPolicy(ctx, c, suite, notResourcePolicy)
|
||||
assertBucketDelete(ctx, c, suite.client, notResourceClient, notResourceBucket, false)
|
||||
|
||||
// A Deny written against "bucket/*" keeps covering the bucket-level request.
|
||||
denyBucket := getRandomBucketName()
|
||||
if err := suite.client.MakeBucket(ctx, denyBucket, minio.MakeBucketOptions{}); err != nil {
|
||||
c.Fatalf("create Deny bucket: %v", err)
|
||||
}
|
||||
denyPolicy := fmt.Appendf(nil, `{
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[
|
||||
{"Effect":"Allow","Action":"s3:*","Resource":"*"},
|
||||
{"Effect":"Deny","Action":"s3:DeleteBucket","Resource":"arn:aws:s3:::%s/*"}
|
||||
]
|
||||
}`, denyBucket)
|
||||
denyClient, _ := createUserWithPolicy(ctx, c, suite, denyPolicy)
|
||||
assertBucketDelete(ctx, c, suite.client, denyClient, denyBucket, false)
|
||||
|
||||
// A service account whose parent policy allows everything but whose inline
|
||||
// policy is object-only exercises the nested AND evaluation path, where the
|
||||
// boundary has to hold on the inline side.
|
||||
serviceBucket := getRandomBucketName()
|
||||
if err := suite.client.MakeBucket(ctx, serviceBucket, minio.MakeBucketOptions{}); err != nil {
|
||||
c.Fatalf("create service-account bucket: %v", err)
|
||||
}
|
||||
parentPolicy := []byte(`{
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]
|
||||
}`)
|
||||
_, parentUser := createUserWithPolicy(ctx, c, suite, parentPolicy)
|
||||
servicePolicy := fmt.Appendf(nil, `{
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"arn:aws:s3:::%s/*"}]
|
||||
}`, serviceBucket)
|
||||
serviceAccess, serviceSecret := mustGenerateCredentials(c)
|
||||
serviceAccount, err := suite.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
|
||||
TargetUser: parentUser,
|
||||
AccessKey: serviceAccess,
|
||||
SecretKey: serviceSecret,
|
||||
Policy: bytes.Clone(servicePolicy),
|
||||
})
|
||||
if err != nil {
|
||||
c.Fatalf("create restricted service account: %v", err)
|
||||
}
|
||||
serviceClient := suite.getUserClient(c, serviceAccount.AccessKey, serviceAccount.SecretKey, "")
|
||||
assertBucketDelete(ctx, c, suite.client, serviceClient, serviceBucket, false)
|
||||
}
|
||||
|
||||
// The same boundary, asserted against an inline session policy evaluated
|
||||
// directly, so a regression in the STS and service-account paths is caught even
|
||||
// if the integration harness above is skipped.
|
||||
func TestBucketResourceBoundaryInlineSessionPolicy(t *testing.T) {
|
||||
inline := `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": "s3:*",
|
||||
"Resource": "arn:aws:s3:::mybucket/*"
|
||||
}]
|
||||
}`
|
||||
args := policy.Args{
|
||||
Action: policy.DeleteBucketAction,
|
||||
BucketName: "mybucket",
|
||||
Claims: map[string]any{
|
||||
sessionPolicyNameExtracted: inline,
|
||||
},
|
||||
}
|
||||
|
||||
for name, evaluate := range map[string]func(policy.Args) (bool, bool){
|
||||
"STS inline policy": isAllowedBySessionPolicy,
|
||||
"service-account inline policy": isAllowedBySessionPolicyForServiceAccount,
|
||||
} {
|
||||
hasPolicy, allowed := evaluate(args)
|
||||
if !hasPolicy {
|
||||
t.Errorf("%s was not detected", name)
|
||||
continue
|
||||
}
|
||||
if allowed {
|
||||
t.Errorf("%s authorized DeleteBucket through an object-only resource", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The boundary driven through the real S3 router and DeleteBucket handler,
|
||||
// without opening a TCP listener.
|
||||
func TestBucketResourceBoundaryHandler(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
endpoints: nil, // Register the full router; the focused list omits DeleteBucket.
|
||||
objAPITest: func(obj ObjectLayer, instanceType, _ string, apiRouter http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
ctx := t.Context()
|
||||
if !globalReplicationPool.IsSet() {
|
||||
// Match initTestServerWithBackend: DeleteBucket calls through this
|
||||
// singleton after the object-layer deletion, and a nil receiver is safe.
|
||||
globalReplicationPool.Set(nil)
|
||||
}
|
||||
|
||||
newPolicyClient := func(policyJSON string) auth.Credentials {
|
||||
t.Helper()
|
||||
accessKey, secretKey, err := auth.GenerateCredentials()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
credentials := auth.Credentials{AccessKey: accessKey, SecretKey: secretKey}
|
||||
if _, err = globalIAMSys.CreateUser(ctx, credentials.AccessKey, madmin.AddOrUpdateUserReq{
|
||||
SecretKey: credentials.SecretKey,
|
||||
Status: madmin.AccountEnabled,
|
||||
}); err != nil {
|
||||
t.Fatalf("%s: create boundary user: %v", instanceType, err)
|
||||
}
|
||||
parsed, err := policy.ParseConfig(strings.NewReader(policyJSON))
|
||||
if err != nil {
|
||||
t.Fatalf("%s: parse boundary policy: %v", instanceType, err)
|
||||
}
|
||||
policyName := "boundary-" + mustGetUUID()
|
||||
if _, err = globalIAMSys.SetPolicy(ctx, policyName, *parsed); err != nil {
|
||||
t.Fatalf("%s: install boundary policy: %v", instanceType, err)
|
||||
}
|
||||
if _, err = globalIAMSys.PolicyDBSet(ctx, credentials.AccessKey, policyName, regUser, false); err != nil {
|
||||
t.Fatalf("%s: attach boundary policy: %v", instanceType, err)
|
||||
}
|
||||
return credentials
|
||||
}
|
||||
|
||||
deleteCase := func(label, policyJSON string, wantAllowed bool) {
|
||||
t.Helper()
|
||||
bucket := getRandomBucketName()
|
||||
if err := obj.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatalf("%s/%s: create bucket: %v", instanceType, label, err)
|
||||
}
|
||||
policyJSON = strings.ReplaceAll(policyJSON, "BUCKET_WILDCARD", bucket[:len(bucket)-1]+"?")
|
||||
policyJSON = strings.ReplaceAll(policyJSON, "BUCKET", bucket)
|
||||
credentials := newPolicyClient(policyJSON)
|
||||
req, err := newTestSignedRequestV4(http.MethodDelete, getDeleteBucketURL("", bucket),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("%s/%s: sign DeleteBucket: %v", instanceType, label, err)
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(recorder, req)
|
||||
|
||||
if wantAllowed {
|
||||
if recorder.Code != http.StatusNoContent {
|
||||
t.Fatalf("%s/%s: DeleteBucket status=%d body=%s, want 204",
|
||||
instanceType, label, recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if _, err = obj.GetBucketInfo(ctx, bucket, BucketOptions{}); err == nil {
|
||||
t.Fatalf("%s/%s: handler returned 204 but bucket still exists", instanceType, label)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if recorder.Code != http.StatusForbidden || !strings.Contains(recorder.Body.String(), "AccessDenied") {
|
||||
t.Fatalf("%s/%s: DeleteBucket status=%d body=%s, want AccessDenied",
|
||||
instanceType, label, recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if _, err = obj.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
t.Fatalf("%s/%s: denied bucket disappeared: %v", instanceType, label, err)
|
||||
}
|
||||
if err = obj.DeleteBucket(ctx, bucket, DeleteBucketOptions{}); err != nil {
|
||||
t.Fatalf("%s/%s: cleanup bucket: %v", instanceType, label, err)
|
||||
}
|
||||
}
|
||||
|
||||
deleteCase("object-only s3:*", `{
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"arn:aws:s3:::BUCKET/*"}]
|
||||
}`, false)
|
||||
|
||||
deleteCase("paired resource", `{
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[{"Effect":"Allow","Action":"s3:*","Resource":[
|
||||
"arn:aws:s3:::BUCKET","arn:aws:s3:::BUCKET/*"
|
||||
]}]
|
||||
}`, true)
|
||||
|
||||
deleteCase("fixed-width bucket wildcard", `{
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[{"Effect":"Allow","Action":"s3:DeleteBucket","Resource":"arn:aws:s3:::BUCKET_WILDCARD"}]
|
||||
}`, false)
|
||||
|
||||
deleteCase("NotResource exclusion", `{
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[{"Effect":"Allow","Action":"s3:*","NotResource":"arn:aws:s3:::BUCKET/*"}]
|
||||
}`, false)
|
||||
|
||||
deleteCase("Deny coverage", `{
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[
|
||||
{"Effect":"Allow","Action":"s3:*","Resource":"*"},
|
||||
{"Effect":"Deny","Action":"s3:DeleteBucket","Resource":"arn:aws:s3:::BUCKET/*"}
|
||||
]
|
||||
}`, false)
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -54,6 +54,7 @@ import (
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/color"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/certs"
|
||||
@@ -711,6 +712,16 @@ func serverHandleEarlyEnvVars() {
|
||||
|
||||
func serverHandleEnvVars() {
|
||||
var err error
|
||||
|
||||
// Re-read the source-address trust policy now that loadEnvVarsFromFiles has
|
||||
// run: a policy taken at package initialisation would miss every deployment
|
||||
// configured through MINIO_CONFIG_ENV_FILE. Refuse to start on a malformed
|
||||
// allow-list rather than resolve aws:SourceIp and every audit client address
|
||||
// by a rule the operator did not write.
|
||||
if err := handlers.ConfigureSourceIPTrust(); err != nil {
|
||||
logger.Fatal(err, "Invalid %s value in environment variable", handlers.EnvTrustedProxies)
|
||||
}
|
||||
|
||||
if globalBrowserEnabled {
|
||||
if redirectURL := env.Get(config.EnvBrowserRedirectURL, ""); redirectURL != "" {
|
||||
u, err := xnet.ParseHTTPURL(redirectURL)
|
||||
|
||||
@@ -18,8 +18,9 @@ const _decomMetric_name = "DecommissionBucketDecommissionObjectDecommissionRemov
|
||||
var _decomMetric_index = [...]uint8{0, 18, 36, 60}
|
||||
|
||||
func (i decomMetric) String() string {
|
||||
if i >= decomMetric(len(_decomMetric_index)-1) {
|
||||
idx := int(i) - 0
|
||||
if i < 0 || idx >= len(_decomMetric_index)-1 {
|
||||
return "decomMetric(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _decomMetric_name[_decomMetric_index[i]:_decomMetric_index[i+1]]
|
||||
return _decomMetric_name[_decomMetric_index[idx]:_decomMetric_index[idx+1]]
|
||||
}
|
||||
|
||||
@@ -45,6 +45,17 @@ func NewErasure(ctx context.Context, dataBlocks, parityBlocks int, blockSize int
|
||||
return e, reedsolomon.ErrInvShardNum
|
||||
}
|
||||
|
||||
// blockSize reaches here from FileInfo.Erasure, i.e. from xl.meta, which a
|
||||
// peer can write, and FileInfo.IsValid() does not check it. A coder with a
|
||||
// non-positive block size cannot encode or decode anything, so refusing to
|
||||
// build one loses nothing -- but building one leaves every division by
|
||||
// e.blockSize downstream (ShardFileSize, ShardFileOffset, and the whole of
|
||||
// erasure-decode.go) as an integer divide-by-zero. Reject at the single
|
||||
// point where every Erasure value in the process is constructed.
|
||||
if blockSize <= 0 {
|
||||
return e, errInvalidArgument
|
||||
}
|
||||
|
||||
if dataBlocks+parityBlocks > 256 {
|
||||
return e, reedsolomon.ErrMaxShardNum
|
||||
}
|
||||
@@ -125,6 +136,13 @@ func (e *Erasure) ShardFileSize(totalLength int64) int64 {
|
||||
if totalLength == -1 {
|
||||
return -1
|
||||
}
|
||||
// NewErasure validates dataBlocks and parityBlocks but not blockSize, and
|
||||
// the values reach it from FileInfo.Erasure - i.e. from xl.meta, which a
|
||||
// peer can write. Mirrors the guard on ErasureInfo.ShardFileSize; without
|
||||
// it a zero block size is an integer divide-by-zero here instead.
|
||||
if e.blockSize <= 0 || e.dataBlocks <= 0 {
|
||||
return 0
|
||||
}
|
||||
numShards := totalLength / e.blockSize
|
||||
lastBlockSize := totalLength % e.blockSize
|
||||
lastShardSize := ceilFrac(lastBlockSize, int64(e.dataBlocks))
|
||||
|
||||
@@ -69,7 +69,7 @@ func newParallelReader(readers []io.ReaderAt, e Erasure, offset, totalLength int
|
||||
offset: (offset / e.blockSize) * e.ShardSize(),
|
||||
shardSize: e.ShardSize(),
|
||||
shardFileSize: e.ShardFileSize(totalLength),
|
||||
buf: make([][]byte, len(readers)),
|
||||
buf: bufs,
|
||||
readerToBuf: r2b,
|
||||
stashBuffer: b,
|
||||
}
|
||||
@@ -106,8 +106,7 @@ func (p *parallelReader) preferReaders(prefer []bool) {
|
||||
// Move reader with index i to index next.
|
||||
// Do this by swapping next and i
|
||||
p.readers[next], p.readers[i] = p.readers[i], p.readers[next]
|
||||
p.readerToBuf[next] = i
|
||||
p.readerToBuf[i] = next
|
||||
p.readerToBuf[next], p.readerToBuf[i] = p.readerToBuf[i], p.readerToBuf[next]
|
||||
next++
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,8 +26,92 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/minio/internal/bpool"
|
||||
)
|
||||
|
||||
func TestNewParallelReaderUsesPooledStashBuffer(t *testing.T) {
|
||||
erasure, err := NewErasure(t.Context(), 4, 4, blockSizeV2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
previousPool := globalBytePoolCap.Load()
|
||||
pool := bpool.NewBytePoolCap(1, blockSizeV2, blockSizeV2*2)
|
||||
globalBytePoolCap.Store(pool)
|
||||
t.Cleanup(func() {
|
||||
globalBytePoolCap.Store(previousPool)
|
||||
})
|
||||
|
||||
readers := make([]io.ReaderAt, 8)
|
||||
reader := newParallelReader(readers, erasure, 0, blockSizeV2)
|
||||
t.Cleanup(reader.Done)
|
||||
if reader.stashBuffer == nil {
|
||||
t.Fatal("expected a pooled stash buffer")
|
||||
}
|
||||
|
||||
shardSize := int(erasure.ShardSize())
|
||||
stash := reader.stashBuffer[:cap(reader.stashBuffer)]
|
||||
for i, buf := range reader.buf {
|
||||
if len(buf) != shardSize {
|
||||
t.Fatalf("buffer %d has length %d, want %d", i, len(buf), shardSize)
|
||||
}
|
||||
buf[0] = byte(i + 1)
|
||||
if got := stash[i*shardSize]; got != byte(i+1) {
|
||||
t.Fatalf("buffer %d does not use the pooled stash buffer", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParallelReaderPreferReadersMaintainsBufferMapping(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
prefer []bool
|
||||
want []int
|
||||
}{
|
||||
{
|
||||
name: "adjacent preferred readers",
|
||||
prefer: []bool{false, true, true, false},
|
||||
want: []int{1, 2, 0, 3},
|
||||
},
|
||||
{
|
||||
name: "sparse preferred readers",
|
||||
prefer: []bool{false, true, false, true},
|
||||
want: []int{1, 3, 2, 0},
|
||||
},
|
||||
{
|
||||
name: "preferred prefix",
|
||||
prefer: []bool{true, true, false, false},
|
||||
want: []int{0, 1, 2, 3},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
original := make([]io.ReaderAt, len(tt.prefer))
|
||||
readerToBuf := make([]int, len(tt.prefer))
|
||||
for i := range original {
|
||||
original[i] = bytes.NewReader([]byte{byte(i)})
|
||||
readerToBuf[i] = i
|
||||
}
|
||||
|
||||
reader := parallelReader{
|
||||
orgReaders: original,
|
||||
readerToBuf: readerToBuf,
|
||||
}
|
||||
reader.preferReaders(tt.prefer)
|
||||
|
||||
for i, originalIndex := range tt.want {
|
||||
if reader.readers[i] != original[originalIndex] {
|
||||
t.Errorf("reader %d maps to the wrong original reader", i)
|
||||
}
|
||||
if got := reader.readerToBuf[i]; got != originalIndex {
|
||||
t.Errorf("reader %d maps to buffer %d, want %d", i, got, originalIndex)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (a badDisk) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
|
||||
return 0, errFaultyDisk
|
||||
}
|
||||
|
||||
@@ -58,6 +58,14 @@ func (e ErasureInfo) ShardFileSize(totalLength int64) int64 {
|
||||
if totalLength == -1 {
|
||||
return -1
|
||||
}
|
||||
// ErasureInfo can arrive zero-valued from an untrusted internode payload:
|
||||
// CheckParts and VerifyFile hand a wire-supplied FileInfo straight here.
|
||||
// A zero BlockSize would panic with an integer divide-by-zero, and
|
||||
// CheckParts evaluates this inside xioutil.WithDeadline - a bare goroutine
|
||||
// whose panic no recover() can reach, taking the whole process down.
|
||||
if e.BlockSize <= 0 || e.DataBlocks <= 0 {
|
||||
return 0
|
||||
}
|
||||
numShards := totalLength / e.BlockSize
|
||||
lastBlockSize := totalLength % e.BlockSize
|
||||
lastShardSize := ceilFrac(lastBlockSize, int64(e.DataBlocks))
|
||||
@@ -69,6 +77,26 @@ func (e ErasureInfo) ShardSize() int64 {
|
||||
return ceilFrac(e.BlockSize, int64(e.DataBlocks))
|
||||
}
|
||||
|
||||
// HasNegativePartSize reports whether any part claims a negative size.
|
||||
//
|
||||
// Such metadata is never legitimate, and it is not merely cosmetic: a negative
|
||||
// length floors both terms of ShardFileSize to zero, and checkPart's only
|
||||
// integrity test is "st.Size() < expectedSize". A zero expectation is therefore
|
||||
// satisfied by every file that exists, including a truncated shard, so the part
|
||||
// is reported intact and a heal driven by the result skips the repair it should
|
||||
// have performed.
|
||||
//
|
||||
// Note this holds even when the erasure parameters are entirely valid, so
|
||||
// FileInfo.IsValid() - the check healing itself trusts - does not catch it.
|
||||
func (fi FileInfo) HasNegativePartSize() bool {
|
||||
for _, p := range fi.Parts {
|
||||
if p.Size < 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsValid - tells if erasure info fields are valid.
|
||||
func (fi FileInfo) IsValid() bool {
|
||||
if fi.Deleted {
|
||||
|
||||
@@ -0,0 +1,431 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
// multipartChecksumTestData is the payload used by the full object checksum
|
||||
// tests: one 5 MiB part (the minimum allowed non-final part size) and a small
|
||||
// trailing part, so part merging is actually exercised.
|
||||
func multipartChecksumTestData() (parts [][]byte, full []byte) {
|
||||
parts = [][]byte{
|
||||
bytes.Repeat([]byte("a"), 5*humanize.MiByte),
|
||||
bytes.Repeat([]byte("b"), 1*humanize.KiByte),
|
||||
}
|
||||
for _, p := range parts {
|
||||
full = append(full, p...)
|
||||
}
|
||||
return parts, full
|
||||
}
|
||||
|
||||
func mustChecksum(t *testing.T, typ hash.ChecksumType, data []byte) string {
|
||||
t.Helper()
|
||||
cs := hash.NewChecksumFromData(typ, data)
|
||||
if cs == nil {
|
||||
t.Fatalf("unable to compute %s checksum", typ.String())
|
||||
}
|
||||
return cs.Encoded
|
||||
}
|
||||
|
||||
// newMultipartUploadHTTP starts a multipart upload over the API router and
|
||||
// returns the upload ID.
|
||||
func newMultipartUploadHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
|
||||
bucket, object, algo, checksumType string,
|
||||
) string {
|
||||
t.Helper()
|
||||
hdrs := map[string]string{xhttp.AmzChecksumAlgo: algo}
|
||||
if checksumType != "" {
|
||||
hdrs[xhttp.AmzChecksumType] = checksumType
|
||||
}
|
||||
req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucket, object),
|
||||
0, nil, creds.AccessKey, creds.SecretKey, hdrs)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to build NewMultipartUpload request: %v", err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("NewMultipartUpload failed: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var res InitiateMultipartUploadResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &res); err != nil {
|
||||
t.Fatalf("failed to decode NewMultipartUpload response: %v", err)
|
||||
}
|
||||
return res.UploadID
|
||||
}
|
||||
|
||||
// uploadPartsHTTP uploads every part carrying its own checksum header, the way
|
||||
// modern AWS SDKs do by default, and returns the part ETags.
|
||||
func uploadPartsHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
|
||||
bucket, object, uploadID string, typ hash.ChecksumType, parts [][]byte,
|
||||
) []string {
|
||||
t.Helper()
|
||||
etags := make([]string, len(parts))
|
||||
for i, p := range parts {
|
||||
req, err := newTestSignedRequestV4(http.MethodPut,
|
||||
getPutObjectPartURL("", bucket, object, uploadID, strconv.Itoa(i+1)),
|
||||
int64(len(p)), bytes.NewReader(p), creds.AccessKey, creds.SecretKey,
|
||||
map[string]string{typ.Key(): mustChecksum(t, typ, p)})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to build UploadPart %d request: %v", i+1, err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("UploadPart %d failed: %d %s", i+1, rec.Code, rec.Body.String())
|
||||
}
|
||||
// MinIO writes the header under a literal "ETag" map key, which
|
||||
// http.Header.Get would canonicalize to "Etag" and miss.
|
||||
etags[i] = rec.Header()[xhttp.ETag][0]
|
||||
}
|
||||
return etags
|
||||
}
|
||||
|
||||
// completeMultipartUploadHTTP completes the upload. partCS supplies an optional
|
||||
// per-part checksum for each part; an empty string omits it, which is exactly
|
||||
// what boto3, aws-sdk-js and the Java SDK send when the caller does not track
|
||||
// part checksums.
|
||||
func completeMultipartUploadHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
|
||||
bucket, object, uploadID string, etags []string, partCS []string, hdrs map[string]string,
|
||||
) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
var body bytes.Buffer
|
||||
body.WriteString("<CompleteMultipartUpload>")
|
||||
for i, etag := range etags {
|
||||
fmt.Fprintf(&body, "<Part><PartNumber>%d</PartNumber><ETag>%s</ETag>", i+1, etag)
|
||||
if i < len(partCS) && partCS[i] != "" {
|
||||
fmt.Fprintf(&body, "<ChecksumCRC32>%s</ChecksumCRC32>", partCS[i])
|
||||
}
|
||||
body.WriteString("</Part>")
|
||||
}
|
||||
body.WriteString("</CompleteMultipartUpload>")
|
||||
|
||||
req, err := newTestSignedRequestV4(http.MethodPost,
|
||||
getCompleteMultipartUploadURL("", bucket, object, uploadID),
|
||||
int64(body.Len()), bytes.NewReader(body.Bytes()), creds.AccessKey, creds.SecretKey, hdrs)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to build CompleteMultipartUpload request: %v", err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
func apiErrorCode(t *testing.T, rec *httptest.ResponseRecorder) string {
|
||||
t.Helper()
|
||||
var e APIErrorResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &e); err != nil {
|
||||
t.Fatalf("unable to decode error response %q: %v", rec.Body.String(), err)
|
||||
}
|
||||
return e.Code
|
||||
}
|
||||
|
||||
// TestAPICompleteMultipartFullObjectChecksum covers pgsty/minio#31.
|
||||
//
|
||||
// A multipart upload created with a full object checksum type must be
|
||||
// completable by sending only PartNumber and ETag per part, plus the object
|
||||
// level checksum in the request headers. That is what AWS S3 accepts, and it is
|
||||
// the point of FULL_OBJECT: the client no longer has to retain per-part
|
||||
// checksums, only the part numbers and ETags it already tracks.
|
||||
func TestAPICompleteMultipartFullObjectChecksum(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPICompleteMultipartFullObjectChecksum,
|
||||
endpoints: []string{"NewMultipart", "PutObjectPart", "CompleteMultipart"},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPICompleteMultipartFullObjectChecksum(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
partData, full := multipartChecksumTestData()
|
||||
|
||||
// Only CRC based algorithms can linearize into a full object checksum.
|
||||
// Which one a client picks by default is SDK specific - the AWS CLI v2
|
||||
// defaults to CRC64NVME while the Go and JavaScript SDKs default to CRC32 -
|
||||
// so cover all three.
|
||||
for _, typ := range []hash.ChecksumType{hash.ChecksumCRC32, hash.ChecksumCRC32C, hash.ChecksumCRC64NVME} {
|
||||
objectName := "uploads/full-object-" + typ.String()
|
||||
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName,
|
||||
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, typ, partData)
|
||||
|
||||
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, etags, nil,
|
||||
map[string]string{
|
||||
typ.Key(): mustChecksum(t, typ, full),
|
||||
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
|
||||
})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s/%s: CompleteMultipartUpload failed: %d %s",
|
||||
instanceType, typ.String(), rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
oi, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("%s/%s: GetObjectInfo failed: %v", instanceType, typ.String(), err)
|
||||
}
|
||||
if oi.Size != int64(len(full)) {
|
||||
t.Fatalf("%s/%s: expected object size %d, got %d", instanceType, typ.String(), len(full), oi.Size)
|
||||
}
|
||||
|
||||
// The persisted checksum must be the merged full object value - not a
|
||||
// composite "<checksum>-<parts>" value - and must report FULL_OBJECT.
|
||||
cs, _ := oi.decryptChecksums(0, nil)
|
||||
if got, want := cs[typ.String()], mustChecksum(t, typ, full); got != want {
|
||||
t.Fatalf("%s/%s: expected stored checksum %q, got %q", instanceType, typ.String(), want, got)
|
||||
}
|
||||
if got := cs[xhttp.AmzChecksumType]; got != xhttp.AmzChecksumTypeFullObject {
|
||||
t.Fatalf("%s/%s: expected stored checksum type %q, got %q",
|
||||
instanceType, typ.String(), xhttp.AmzChecksumTypeFullObject, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPICompleteMultipartFullObjectChecksumMismatch asserts that accepting
|
||||
// completions without part checksums does not weaken integrity: a wrong object
|
||||
// level checksum is still rejected.
|
||||
func TestAPICompleteMultipartFullObjectChecksumMismatch(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPICompleteMultipartFullObjectChecksumMismatch,
|
||||
endpoints: []string{"NewMultipart", "PutObjectPart", "CompleteMultipart"},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPICompleteMultipartFullObjectChecksumMismatch(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
partData, _ := multipartChecksumTestData()
|
||||
objectName := "uploads/full-object-mismatch"
|
||||
typ := hash.ChecksumCRC32
|
||||
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName,
|
||||
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, typ, partData)
|
||||
|
||||
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, etags, nil,
|
||||
map[string]string{
|
||||
typ.Key(): mustChecksum(t, typ, []byte("not the object content")),
|
||||
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
|
||||
})
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("%s: CompleteMultipartUpload with a bad full object checksum returned %d, want 400",
|
||||
instanceType, rec.Code)
|
||||
}
|
||||
// NOTE: AWS S3 documents BadDigest for a full object checksum mismatch on
|
||||
// CompleteMultipartUpload. MinIO reports XAmzContentChecksumMismatch. That
|
||||
// deviation is tracked separately; assert the current code so a future
|
||||
// change to it is a deliberate one.
|
||||
if got := apiErrorCode(t, rec); got != "XAmzContentChecksumMismatch" {
|
||||
t.Fatalf("%s: expected XAmzContentChecksumMismatch, got %q", instanceType, got)
|
||||
}
|
||||
|
||||
if _, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err == nil {
|
||||
t.Fatalf("%s: object was created despite a failed checksum validation", instanceType)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPICompleteMultipartCompositeStillRequiresPartChecksums locks in that the
|
||||
// relaxation is scoped to full object checksums. For the algorithm/type pairs
|
||||
// AWS actually supports as composite, AWS requires a checksum for every part in
|
||||
// the CompleteMultipartUpload body, and so do we. (CRC64NVME is deliberately not
|
||||
// covered: AWS does not support it as composite and MinIO canonicalises it to a
|
||||
// full object checksum at initiation.)
|
||||
func TestAPICompleteMultipartCompositeStillRequiresPartChecksums(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPICompleteMultipartCompositeStillRequiresPartChecksums,
|
||||
endpoints: []string{"NewMultipart", "PutObjectPart", "CompleteMultipart"},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPICompleteMultipartCompositeStillRequiresPartChecksums(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
partData, _ := multipartChecksumTestData()
|
||||
|
||||
for _, typ := range []hash.ChecksumType{hash.ChecksumCRC32, hash.ChecksumSHA256} {
|
||||
objectName := "uploads/composite-" + typ.String()
|
||||
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName,
|
||||
typ.String(), xhttp.AmzChecksumTypeComposite)
|
||||
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, typ, partData)
|
||||
|
||||
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, etags, nil, nil)
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("%s/%s: composite CompleteMultipartUpload without part checksums returned %d, want 400",
|
||||
instanceType, typ.String(), rec.Code)
|
||||
}
|
||||
if got := apiErrorCode(t, rec); got != "InvalidPart" {
|
||||
t.Fatalf("%s/%s: expected InvalidPart, got %q", instanceType, typ.String(), got)
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err == nil {
|
||||
t.Fatalf("%s/%s: object was created despite a rejected completion", instanceType, typ.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPICompleteMultipartFullObjectVariants pins down the surrounding
|
||||
// behavior of the relaxation: what may be omitted, what must still match, and
|
||||
// that a zero length object is handled like any other.
|
||||
func TestAPICompleteMultipartFullObjectVariants(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPICompleteMultipartFullObjectVariants,
|
||||
endpoints: []string{"NewMultipart", "PutObjectPart", "CompleteMultipart"},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPICompleteMultipartFullObjectVariants(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
typ := hash.ChecksumCRC32
|
||||
partData, full := multipartChecksumTestData()
|
||||
goodCS := []string{mustChecksum(t, typ, partData[0]), mustChecksum(t, typ, partData[1])}
|
||||
|
||||
setup := func(name string) (string, []string) {
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, name,
|
||||
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||
return uploadID, uploadPartsHTTP(t, apiRouter, credentials, bucketName, name, uploadID, typ, partData)
|
||||
}
|
||||
objCSHdr := map[string]string{
|
||||
typ.Key(): mustChecksum(t, typ, full),
|
||||
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
|
||||
}
|
||||
|
||||
t.Run("mixed-present-and-omitted", func(t *testing.T) {
|
||||
name := "variants/mixed"
|
||||
uploadID, etags := setup(name)
|
||||
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, name, uploadID, etags,
|
||||
[]string{goodCS[0], ""}, objCSHdr)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: want 200, got %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("supplied-part-checksum-must-match", func(t *testing.T) {
|
||||
name := "variants/wrong-part-cs"
|
||||
uploadID, etags := setup(name)
|
||||
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, name, uploadID, etags,
|
||||
[]string{mustChecksum(t, typ, []byte("wrong")), ""}, objCSHdr)
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("%s: a non-empty but wrong part checksum must be rejected, got %d", instanceType, rec.Code)
|
||||
}
|
||||
if got := apiErrorCode(t, rec); got != "InvalidPart" {
|
||||
t.Fatalf("%s: expected InvalidPart, got %q", instanceType, got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("wrong-algorithm-part-checksum-is-rejected", func(t *testing.T) {
|
||||
// A part carrying a checksum under an algorithm other than the upload's
|
||||
// is malformed, not "omitted", and must not slip through the relaxation.
|
||||
name := "variants/wrong-algo-part-cs"
|
||||
uploadID, etags := setup(name)
|
||||
var body bytes.Buffer
|
||||
body.WriteString("<CompleteMultipartUpload>")
|
||||
for i, etag := range etags {
|
||||
fmt.Fprintf(&body, "<Part><PartNumber>%d</PartNumber><ETag>%s</ETag>"+
|
||||
"<ChecksumCRC32C>AAAAAA==</ChecksumCRC32C></Part>", i+1, etag)
|
||||
}
|
||||
body.WriteString("</CompleteMultipartUpload>")
|
||||
|
||||
req, err := newTestSignedRequestV4(http.MethodPost,
|
||||
getCompleteMultipartUploadURL("", bucketName, name, uploadID),
|
||||
int64(body.Len()), bytes.NewReader(body.Bytes()), credentials.AccessKey, credentials.SecretKey, objCSHdr)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to build CompleteMultipartUpload request: %v", err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("%s: a part checksum under the wrong algorithm must be rejected, got %d",
|
||||
instanceType, rec.Code)
|
||||
}
|
||||
if got := apiErrorCode(t, rec); got != "InvalidPart" {
|
||||
t.Fatalf("%s: expected InvalidPart, got %q", instanceType, got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no-object-checksum-supplied", func(t *testing.T) {
|
||||
// AWS treats the object level checksum on completion as optional; the
|
||||
// server stores the checksum it computed from the parts.
|
||||
name := "variants/no-object-cs"
|
||||
uploadID, etags := setup(name)
|
||||
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, name, uploadID, etags, nil, nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: want 200, got %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
oi, err := obj.GetObjectInfo(t.Context(), bucketName, name, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: GetObjectInfo failed: %v", instanceType, err)
|
||||
}
|
||||
cs, _ := oi.decryptChecksums(0, nil)
|
||||
if got, want := cs[typ.String()], mustChecksum(t, typ, full); got != want {
|
||||
t.Fatalf("%s: expected server computed checksum %q, got %q", instanceType, want, got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("zero-length-object", func(t *testing.T) {
|
||||
// A single empty part is a legal multipart upload: only non-final parts
|
||||
// have a minimum size. The merged checksum must be the checksum of no
|
||||
// bytes, not the empty string.
|
||||
name := "variants/zero-length"
|
||||
empty := []byte{}
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, name,
|
||||
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, name, uploadID, typ, [][]byte{empty})
|
||||
|
||||
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, name, uploadID, etags, nil,
|
||||
map[string]string{
|
||||
typ.Key(): mustChecksum(t, typ, empty),
|
||||
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
|
||||
})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: want 200, got %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
oi, err := obj.GetObjectInfo(t.Context(), bucketName, name, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: GetObjectInfo failed: %v", instanceType, err)
|
||||
}
|
||||
if oi.Size != 0 {
|
||||
t.Fatalf("%s: expected zero length object, got %d", instanceType, oi.Size)
|
||||
}
|
||||
cs, _ := oi.decryptChecksums(0, nil)
|
||||
if got, want := cs[typ.String()], mustChecksum(t, typ, empty); got != want {
|
||||
t.Fatalf("%s: expected stored checksum %q, got %q", instanceType, want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1290,10 +1290,26 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
hash.ChecksumSHA256.String(): part.ChecksumSHA256,
|
||||
hash.ChecksumCRC64NVME.String(): part.ChecksumCRC64NVME,
|
||||
}
|
||||
if wantCS[checksumType.String()] != crc {
|
||||
gotCS := wantCS[checksumType.String()]
|
||||
var suppliedAnyCS bool
|
||||
for _, v := range wantCS {
|
||||
if v != "" {
|
||||
suppliedAnyCS = true
|
||||
break
|
||||
}
|
||||
}
|
||||
// Part checksums are optional in the CompleteMultipartUpload body when
|
||||
// the upload was created with a full object checksum type: clients send
|
||||
// the object level checksum instead and do not retain part checksums.
|
||||
// A part that carries any checksum at all is still validated against
|
||||
// what we stored - including one sent under the wrong algorithm, which
|
||||
// cannot match and is rejected. The object level checksum, if supplied,
|
||||
// is verified against the merged part checksums below.
|
||||
allowMissingPartCS := checksumType.FullObjectRequested() && !suppliedAnyCS
|
||||
if !allowMissingPartCS && gotCS != crc {
|
||||
return oi, InvalidPart{
|
||||
PartNumber: part.PartNumber,
|
||||
ExpETag: wantCS[checksumType.String()],
|
||||
ExpETag: gotCS,
|
||||
GotETag: crc,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -469,7 +469,7 @@ func auditDanglingObjectDeletion(ctx context.Context, bucket, object, versionID
|
||||
|
||||
func joinErrs(errs []error) string {
|
||||
var s string
|
||||
for i := range s {
|
||||
for i := range errs {
|
||||
if s != "" {
|
||||
s += ","
|
||||
}
|
||||
|
||||
@@ -1281,3 +1281,24 @@ func TestGetObjectWithOutdatedDisks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestJoinErrs(t *testing.T) {
|
||||
errA := errors.New("disk not found")
|
||||
errB := errors.New("file corrupt")
|
||||
testCases := []struct {
|
||||
errs []error
|
||||
expected string
|
||||
}{
|
||||
{nil, ""},
|
||||
{[]error{}, ""},
|
||||
{[]error{nil}, "<nil>"},
|
||||
{[]error{errA}, "disk not found"},
|
||||
{[]error{nil, errA}, "<nil>,disk not found"},
|
||||
{[]error{errA, nil, errB, nil}, "disk not found,<nil>,file corrupt,<nil>"},
|
||||
}
|
||||
for i, testCase := range testCases {
|
||||
if got := joinErrs(testCase.errs); got != testCase.expected {
|
||||
t.Errorf("Test %d: expected %q, got %q", i+1, testCase.expected, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,7 +69,11 @@ func (rs *rebalanceStats) update(bucket string, fi FileInfo) {
|
||||
|
||||
rs.NumVersions++
|
||||
onDiskSz := int64(0)
|
||||
if !fi.Deleted {
|
||||
// DataBlocks comes from xl.meta and only fi.Deleted is checked above, so a
|
||||
// zero here is an integer divide-by-zero rather than a bad statistic. This
|
||||
// path does not build an Erasure, so NewErasure's validation does not cover
|
||||
// it; leave the size at zero for metadata that cannot describe a layout.
|
||||
if !fi.Deleted && fi.Erasure.DataBlocks > 0 {
|
||||
onDiskSz = fi.Size * int64(fi.Erasure.DataBlocks+fi.Erasure.ParityBlocks) / int64(fi.Erasure.DataBlocks)
|
||||
}
|
||||
rs.Bytes += uint64(onDiskSz)
|
||||
|
||||
@@ -177,7 +177,7 @@ func formatGetBackendErasureVersion(b []byte) (string, error) {
|
||||
return "", fmt.Errorf(`format.Version expected: %s, got: %s`, formatMetaVersionV1, meta.Version)
|
||||
}
|
||||
if meta.Format != formatBackendErasure && meta.Format != formatBackendErasureSingle {
|
||||
return "", fmt.Errorf(`found backend type %s, expected %s or %s - to migrate to a supported backend visit https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-migrate-fs-gateway.html`, meta.Format, formatBackendErasure, formatBackendErasureSingle)
|
||||
return "", fmt.Errorf(`found backend type %s, expected %s or %s - to migrate to a supported backend visit https://silo.pgsty.com/operations/deployments/baremetal-migrate-fs-gateway/`, meta.Format, formatBackendErasure, formatBackendErasureSingle)
|
||||
}
|
||||
// Erasure backend found, proceed to detect version.
|
||||
format := &formatErasureVersionDetect{}
|
||||
|
||||
@@ -21,8 +21,9 @@ const _format_name = "UnknownGzipZstdLZ4S2BZ2"
|
||||
var _format_index = [...]uint8{0, 7, 11, 15, 18, 20, 23}
|
||||
|
||||
func (i format) String() string {
|
||||
if i < 0 || i >= format(len(_format_index)-1) {
|
||||
idx := int(i) - 0
|
||||
if i < 0 || idx >= len(_format_index)-1 {
|
||||
return "format(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _format_name[_format_index[i]:_format_index[i+1]]
|
||||
return _format_name[_format_index[idx]:_format_index[idx+1]]
|
||||
}
|
||||
|
||||
+56
-1
@@ -142,7 +142,62 @@ var userMetadataKeyPrefixes = []string{
|
||||
|
||||
// extractMetadataFromReq extracts metadata from HTTP header and HTTP queryString.
|
||||
func extractMetadataFromReq(ctx context.Context, r *http.Request) (metadata map[string]string, err error) {
|
||||
return extractMetadata(ctx, textproto.MIMEHeader(r.Form), textproto.MIMEHeader(r.Header))
|
||||
metadata, err = extractMetadata(ctx, textproto.MIMEHeader(r.Form), textproto.MIMEHeader(r.Header))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Keep the metadata consumed by object operations in lock-step with policy
|
||||
// conditions: an explicitly present header wins, otherwise use the query
|
||||
// value accepted by the existing S3-compatible request path.
|
||||
for _, name := range []string{xhttp.AmzStorageClass, xhttp.AmzObjectTagging} {
|
||||
if value, ok := getRequestHeaderOrQueryValue(r, name); ok {
|
||||
metadata[name] = value
|
||||
}
|
||||
}
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
// getRequestHeaderOrQueryValue returns the effective value of a request field.
|
||||
// Header presence takes precedence even when its value is empty. Query lookup
|
||||
// remains case-insensitive for compatibility with extractMetadataFromReq.
|
||||
func getRequestHeaderOrQueryValue(r *http.Request, name string) (string, bool) {
|
||||
if values, ok := getRequestValues(r.Header, name, http.CanonicalHeaderKey(name)); ok {
|
||||
return strings.Join(values, ","), true
|
||||
}
|
||||
if values, ok := getRequestValues(http.Header(r.Form), name, strings.ToLower(name)); ok {
|
||||
return strings.Join(values, ","), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func getRequestValues(values http.Header, name, preferred string) ([]string, bool) {
|
||||
if value, ok := values[preferred]; ok {
|
||||
return value, true
|
||||
}
|
||||
|
||||
canonical, lower := http.CanonicalHeaderKey(name), strings.ToLower(name)
|
||||
for _, key := range []string{canonical, lower} {
|
||||
if key == preferred {
|
||||
continue
|
||||
}
|
||||
if value, ok := values[key]; ok {
|
||||
return value, true
|
||||
}
|
||||
}
|
||||
|
||||
// Multiple differently-cased spellings are malformed but were previously
|
||||
// accepted. Pick one deterministically instead of depending on map order.
|
||||
match := ""
|
||||
for key := range values {
|
||||
if strings.EqualFold(key, name) && (match == "" || key < match) {
|
||||
match = key
|
||||
}
|
||||
}
|
||||
if match != "" {
|
||||
return values[match], true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func extractMetadata(ctx context.Context, mimesHeader ...textproto.MIMEHeader) (metadata map[string]string, err error) {
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
"net/url"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/config"
|
||||
@@ -194,6 +195,63 @@ func TestExtractMetadataHeaders(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractMetadataFromRequestUsesHeaderPrecedence(t *testing.T) {
|
||||
query := make(url.Values)
|
||||
query.Set(strings.ToLower(xhttp.AmzStorageClass), "QUERY-CLASS")
|
||||
query.Set(strings.ToLower(xhttp.AmzObjectTagging), "source=query")
|
||||
req, err := http.NewRequest(http.MethodGet, "http://localhost/test?"+query.Encode(), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Header.Set(xhttp.AmzStorageClass, "HEADER-CLASS")
|
||||
req.Header.Set(xhttp.AmzObjectTagging, "source=header")
|
||||
if err = req.ParseForm(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
metadata, err := extractMetadataFromReq(t.Context(), req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := metadata[xhttp.AmzStorageClass]; got != "HEADER-CLASS" {
|
||||
t.Fatalf("storage class: expected header, got %q", got)
|
||||
}
|
||||
if got := metadata[xhttp.AmzObjectTagging]; got != "source=header" {
|
||||
t.Fatalf("tagging: expected header, got %q", got)
|
||||
}
|
||||
|
||||
// Presence, rather than a non-empty value, establishes precedence. This
|
||||
// prevents a query value from taking over when a signed header is empty.
|
||||
req.Header[xhttp.AmzObjectTagging] = []string{""}
|
||||
if got, ok := getRequestHeaderOrQueryValue(req, xhttp.AmzObjectTagging); !ok || got != "" {
|
||||
t.Fatalf("empty header did not override query: value=%q present=%v", got, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractMetadataFromRequestKeepsQueryCompatibility(t *testing.T) {
|
||||
query := make(url.Values)
|
||||
query.Set(strings.ToLower(xhttp.AmzStorageClass), "REDUCED_REDUNDANCY")
|
||||
query.Set(strings.ToLower(xhttp.AmzObjectTagging), "security=public")
|
||||
req, err := http.NewRequest(http.MethodGet, "http://localhost/test?"+query.Encode(), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = req.ParseForm(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
metadata, err := extractMetadataFromReq(t.Context(), req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := metadata[xhttp.AmzStorageClass]; got != "REDUCED_REDUNDANCY" {
|
||||
t.Fatalf("storage class query value lost: %q", got)
|
||||
}
|
||||
if got := metadata[xhttp.AmzObjectTagging]; got != "security=public" {
|
||||
t.Fatalf("tagging query value lost: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractReplicationMetadataHeaders(t *testing.T) {
|
||||
header := http.Header{
|
||||
"X-Minio-Replication-Server-Side-Encryption-Sealed-Key": []string{"sealed-key"},
|
||||
|
||||
@@ -18,8 +18,9 @@ const _healingMetric_name = "BucketObjectCheckAbandonedParts"
|
||||
var _healingMetric_index = [...]uint8{0, 6, 12, 31}
|
||||
|
||||
func (i healingMetric) String() string {
|
||||
if i >= healingMetric(len(_healingMetric_index)-1) {
|
||||
idx := int(i) - 0
|
||||
if i < 0 || idx >= len(_healingMetric_index)-1 {
|
||||
return "healingMetric(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _healingMetric_name[_healingMetric_index[i]:_healingMetric_index[i+1]]
|
||||
return _healingMetric_name[_healingMetric_index[idx]:_healingMetric_index[idx+1]]
|
||||
}
|
||||
|
||||
@@ -26,8 +26,9 @@ const _lcEventSrc_name = "NoneHealScannerDecomRebals3HeadObjects3GetObjects3List
|
||||
var _lcEventSrc_index = [...]uint8{0, 4, 8, 15, 20, 25, 37, 48, 61, 72, 84, 109}
|
||||
|
||||
func (i lcEventSrc) String() string {
|
||||
if i >= lcEventSrc(len(_lcEventSrc_index)-1) {
|
||||
idx := int(i) - 0
|
||||
if i < 0 || idx >= len(_lcEventSrc_index)-1 {
|
||||
return "lcEventSrc(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _lcEventSrc_name[_lcEventSrc_index[i]:_lcEventSrc_index[i+1]]
|
||||
return _lcEventSrc_name[_lcEventSrc_index[idx]:_lcEventSrc_index[idx+1]]
|
||||
}
|
||||
|
||||
@@ -324,14 +324,6 @@ func (d *naughtyDisk) StatInfoFile(ctx context.Context, volume, path string, glo
|
||||
return d.disk.StatInfoFile(ctx, volume, path, glob)
|
||||
}
|
||||
|
||||
func (d *naughtyDisk) ReadMultiple(ctx context.Context, req ReadMultipleReq, resp chan<- ReadMultipleResp) error {
|
||||
if err := d.calcError(); err != nil {
|
||||
close(resp)
|
||||
return err
|
||||
}
|
||||
return d.disk.ReadMultiple(ctx, req, resp)
|
||||
}
|
||||
|
||||
func (d *naughtyDisk) CleanAbandonedData(ctx context.Context, volume string, path string) error {
|
||||
if err := d.calcError(); err != nil {
|
||||
return err
|
||||
|
||||
+42
-49
@@ -387,11 +387,7 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
|
||||
return true
|
||||
}
|
||||
|
||||
if oi.UserTags != "" {
|
||||
r.Header.Set(xhttp.AmzObjectTagging, oi.UserTags)
|
||||
}
|
||||
|
||||
if s3Error := authorizeRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone {
|
||||
if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectAction, oi.UserTags); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return true
|
||||
}
|
||||
@@ -429,15 +425,17 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
|
||||
}
|
||||
}
|
||||
if reader == nil || !proxy.Proxy {
|
||||
// The conditional callback has already written 304/412. Do not
|
||||
// authorize again without the stored tags or write a second response.
|
||||
if isErrPreconditionFailed(err) {
|
||||
return
|
||||
}
|
||||
// validate if the request indeed was authorized, if it wasn't we need to return "ErrAccessDenied"
|
||||
// instead of any namespace related error.
|
||||
if s3Error := authorizeRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
if isErrPreconditionFailed(err) {
|
||||
return
|
||||
}
|
||||
if proxy.Err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, proxy.Err), r.URL)
|
||||
return
|
||||
@@ -839,12 +837,7 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
|
||||
}
|
||||
}
|
||||
|
||||
if objInfo.UserTags != "" {
|
||||
// Set this such that authorization policies can be applied on the object tags.
|
||||
r.Header.Set(xhttp.AmzObjectTagging, objInfo.UserTags)
|
||||
}
|
||||
|
||||
if s3Error := authorizeRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone {
|
||||
if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectAction, objInfo.UserTags); s3Error != ErrNone {
|
||||
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error))
|
||||
return
|
||||
}
|
||||
@@ -1055,10 +1048,7 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m
|
||||
// Storage class is special, it can be replaced regardless of the
|
||||
// metadata directive, if set should be preserved and replaced
|
||||
// to the destination metadata.
|
||||
sc := r.Header.Get(xhttp.AmzStorageClass)
|
||||
if sc == "" {
|
||||
sc = r.Form.Get(xhttp.AmzStorageClass)
|
||||
}
|
||||
sc, _ := getRequestHeaderOrQueryValue(r, xhttp.AmzStorageClass)
|
||||
|
||||
// if x-amz-metadata-directive says REPLACE then
|
||||
// we extract metadata from the input headers.
|
||||
@@ -1256,9 +1246,10 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
return
|
||||
}
|
||||
|
||||
// Validate storage class metadata if present
|
||||
dstSc := r.Header.Get(xhttp.AmzStorageClass)
|
||||
if dstSc != "" && !storageclass.IsValid(dstSc) {
|
||||
// Validate the storage class header if present. Query values retain the
|
||||
// existing compatibility path, including its historical validation behavior.
|
||||
dstSc, _ := getRequestHeaderOrQueryValue(r, xhttp.AmzStorageClass)
|
||||
if headerStorageClass := r.Header.Get(xhttp.AmzStorageClass); headerStorageClass != "" && !storageclass.IsValid(headerStorageClass) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidStorageClass), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -1857,7 +1848,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
return
|
||||
}
|
||||
|
||||
// Validate storage class metadata if present
|
||||
// Validate the storage class header if present. Query values retain the
|
||||
// existing compatibility path, including its historical validation behavior.
|
||||
if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" {
|
||||
if !storageclass.IsValid(sc) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidStorageClass), r.URL)
|
||||
@@ -1906,13 +1898,11 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
return
|
||||
}
|
||||
|
||||
if objTags := r.Header.Get(xhttp.AmzObjectTagging); objTags != "" {
|
||||
if objTags := metadata[xhttp.AmzObjectTagging]; objTags != "" {
|
||||
if _, err := tags.ParseObjectTags(objTags); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
metadata[xhttp.AmzObjectTagging] = objTags
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -1924,7 +1914,12 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
)
|
||||
|
||||
// Check if put is allowed
|
||||
if s3Err = isPutActionAllowed(ctx, rAuthType, bucket, object, r, policy.PutObjectAction); s3Err != ErrNone {
|
||||
requestTags, hasRequestTags := metadata[xhttp.AmzObjectTagging]
|
||||
var requestTagsPtr *string
|
||||
if hasRequestTags {
|
||||
requestTagsPtr = &requestTags
|
||||
}
|
||||
if s3Err = isPutActionAllowedWithRequestTags(ctx, rAuthType, bucket, object, r, policy.PutObjectAction, requestTagsPtr); s3Err != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -2271,13 +2266,12 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
|
||||
// Validate storage class metadata if present
|
||||
sc := r.Header.Get(xhttp.AmzStorageClass)
|
||||
if sc != "" {
|
||||
if !storageclass.IsValid(sc) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidStorageClass), r.URL)
|
||||
return
|
||||
}
|
||||
// Validate the storage class header if present. PutObjectExtract now also
|
||||
// consumes the compatible query value so policy and operation stay aligned.
|
||||
sc, _ := getRequestHeaderOrQueryValue(r, xhttp.AmzStorageClass)
|
||||
if headerStorageClass := r.Header.Get(xhttp.AmzStorageClass); headerStorageClass != "" && !storageclass.IsValid(headerStorageClass) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidStorageClass), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
clientETag, err := etag.FromContentMD5(r.Header)
|
||||
@@ -3205,12 +3199,7 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
|
||||
}
|
||||
}
|
||||
|
||||
// Set this such that authorization policies can be applied on the object tags.
|
||||
if tags := ot.String(); tags != "" {
|
||||
r.Header.Set(xhttp.AmzObjectTagging, tags)
|
||||
}
|
||||
|
||||
if s3Error := authorizeRequest(ctx, r, policy.GetObjectTaggingAction); s3Error != ErrNone {
|
||||
if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, ot.String()); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -3264,12 +3253,14 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
tagsStr := tags.String()
|
||||
|
||||
// Set this such that authorization policies can be applied on the object tags.
|
||||
r.Header.Set(xhttp.AmzObjectTagging, tags.String())
|
||||
r.Header.Set(xhttp.AmzObjectTagging, tagsStr)
|
||||
|
||||
// Allow putObjectTagging if policy action is set
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectTaggingAction, bucket, object); s3Error != ErrNone {
|
||||
logger.GetReqInfo(ctx).BucketName = bucket
|
||||
logger.GetReqInfo(ctx).ObjectName = object
|
||||
if s3Error := authenticateRequest(ctx, r, policy.PutObjectTaggingAction); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -3281,6 +3272,14 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
|
||||
}
|
||||
|
||||
objInfo, err := objAPI.GetObjectInfo(ctx, bucket, object, opts)
|
||||
existingTags := ""
|
||||
if err == nil {
|
||||
existingTags = objInfo.UserTags
|
||||
}
|
||||
if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.PutObjectTaggingAction, existingTags); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
// if object is not found locally, but exists on peer site - proxy
|
||||
// the tagging request to peer site. The response to client will
|
||||
@@ -3314,7 +3313,6 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
tagsStr := tags.String()
|
||||
|
||||
dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(objInfo.UserDefined, tagsStr, objInfo.ReplicationStatus, replication.MetadataReplicationType, opts))
|
||||
if dsc.ReplicateAny() {
|
||||
@@ -3413,13 +3411,8 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
if userTags := oi.UserTags; userTags != "" {
|
||||
// Set this such that authorization policies can be applied on the object tags.
|
||||
r.Header.Set(xhttp.AmzObjectTagging, oi.UserTags)
|
||||
}
|
||||
|
||||
// Allow deleteObjectTagging if policy action is set
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.DeleteObjectTaggingAction, bucket, object); s3Error != ErrNone {
|
||||
if s3Error := checkRequestAuthTypeWithExistingTags(ctx, r, policy.DeleteObjectTaggingAction, bucket, object, oi.UserTags); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -26,7 +26,6 @@ import (
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -81,7 +80,12 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, bucket, object); s3Error != ErrNone {
|
||||
requestTags, hasRequestTags := getRequestHeaderOrQueryValue(r, xhttp.AmzObjectTagging)
|
||||
var requestTagsPtr *string
|
||||
if hasRequestTags {
|
||||
requestTagsPtr = &requestTags
|
||||
}
|
||||
if s3Error := checkRequestAuthTypeWithRequestTags(ctx, r, policy.PutObjectAction, bucket, object, requestTagsPtr); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -92,7 +96,8 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
|
||||
AutoEncrypt: globalAutoEncryption,
|
||||
})
|
||||
|
||||
// Validate storage class metadata if present
|
||||
// Validate the storage class header if present. Query values retain the
|
||||
// existing compatibility path, including its historical validation behavior.
|
||||
if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" {
|
||||
if !storageclass.IsValid(sc) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidStorageClass), r.URL)
|
||||
@@ -149,13 +154,11 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
if objTags := r.Header.Get(xhttp.AmzObjectTagging); objTags != "" {
|
||||
if objTags := metadata[xhttp.AmzObjectTagging]; objTags != "" {
|
||||
if _, err := tags.ParseObjectTags(objTags); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
metadata[xhttp.AmzObjectTagging] = objTags
|
||||
}
|
||||
if r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() {
|
||||
if s3Err := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.ReplicateObjectAction); s3Err != ErrNone {
|
||||
@@ -967,11 +970,15 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
|
||||
return
|
||||
}
|
||||
|
||||
if !sort.SliceIsSorted(complMultipartUpload.Parts, func(i, j int) bool {
|
||||
return complMultipartUpload.Parts[i].PartNumber < complMultipartUpload.Parts[j].PartNumber
|
||||
}) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidPartOrder), r.URL)
|
||||
return
|
||||
// The parts list must be strictly increasing by part number. Gaps are
|
||||
// allowed, repeats are not - sort.SliceIsSorted() with a '<' predicate
|
||||
// considers equal neighbors sorted, so it is checked explicitly here,
|
||||
// before anything is assembled into the target object.
|
||||
for i := 1; i < len(complMultipartUpload.Parts); i++ {
|
||||
if complMultipartUpload.Parts[i-1].PartNumber >= complMultipartUpload.Parts[i].PartNumber {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidPartOrder), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Reject retention or governance headers if set, CompleteMultipartUpload spec
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
// Copyright (c) 2015-2025 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
)
|
||||
|
||||
// Tests the part number list validation of the CompleteMultipartUpload handler.
|
||||
// The list must be strictly increasing: duplicate or out-of-order part numbers
|
||||
// are rejected with InvalidPartOrder, and the rejection must happen before the
|
||||
// target object is assembled. Only the ordering is constrained - part numbers
|
||||
// need not start at 1 and need not be consecutive, so [1,3] and [5,9] are both
|
||||
// legal and must be accepted.
|
||||
func TestAPICompleteMultipartHandlerPartOrder(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPICompleteMultipartHandlerPartOrder,
|
||||
endpoints: []string{"NewMultipart", "PutObjectPart", "CompleteMultipart"},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPICompleteMultipartHandlerPartOrder(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Every uploaded part is >= the minimum part size so that a part is never
|
||||
// rejected for being too small when it is not the last one.
|
||||
partData := strings.Repeat("a", 5*humanize.MiByte)
|
||||
partETag := getMD5Hash([]byte(partData))
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
// Part numbers uploaded before the completion request.
|
||||
upload []int
|
||||
// Part numbers listed in the completion request, in this order.
|
||||
complete []int
|
||||
expectedRespStatus int
|
||||
// Only checked when expectedRespStatus is not http.StatusOK.
|
||||
expectedErr APIErrorCode
|
||||
}{
|
||||
// Defect reproduction. A duplicated part number is the case that
|
||||
// sort.SliceIsSorted() used to accept, because its '<' predicate treats
|
||||
// equal neighbors as sorted. Each of these assembled the same part into
|
||||
// the object more than once, inflating it past what was uploaded.
|
||||
{
|
||||
name: "duplicate-part",
|
||||
upload: []int{1},
|
||||
complete: []int{1, 1},
|
||||
expectedRespStatus: http.StatusBadRequest,
|
||||
expectedErr: ErrInvalidPartOrder,
|
||||
},
|
||||
{
|
||||
name: "trailing-duplicate",
|
||||
upload: []int{1, 2},
|
||||
complete: []int{1, 2, 2},
|
||||
expectedRespStatus: http.StatusBadRequest,
|
||||
expectedErr: ErrInvalidPartOrder,
|
||||
},
|
||||
{
|
||||
name: "duplicate-max-part-number",
|
||||
upload: []int{globalMaxPartID},
|
||||
complete: []int{globalMaxPartID, globalMaxPartID},
|
||||
expectedRespStatus: http.StatusBadRequest,
|
||||
expectedErr: ErrInvalidPartOrder,
|
||||
},
|
||||
|
||||
// Regression guard. Ordering violations that were already rejected
|
||||
// before the fix and must stay rejected with the same error.
|
||||
{
|
||||
name: "descending",
|
||||
upload: []int{1, 2},
|
||||
complete: []int{2, 1},
|
||||
expectedRespStatus: http.StatusBadRequest,
|
||||
expectedErr: ErrInvalidPartOrder,
|
||||
},
|
||||
{
|
||||
name: "repeat-after-increase",
|
||||
upload: []int{1, 2},
|
||||
complete: []int{1, 2, 1},
|
||||
expectedRespStatus: http.StatusBadRequest,
|
||||
expectedErr: ErrInvalidPartOrder,
|
||||
},
|
||||
|
||||
// Regression guard. The order loop starts at index 1 and so is a no-op
|
||||
// for an empty list; an empty list is only rejected because the handler
|
||||
// screens it out first. This pins that dependency down - without the
|
||||
// len()==0 check the empty list reaches readParts() and panics the
|
||||
// process on partMetaPaths[0] in xl-storage-disk-id-check.go.
|
||||
{
|
||||
name: "empty-parts",
|
||||
upload: []int{1},
|
||||
complete: nil,
|
||||
expectedRespStatus: http.StatusBadRequest,
|
||||
expectedErr: ErrMissingPart,
|
||||
},
|
||||
|
||||
// Regression guard. Legal lists that must keep working. Part numbers
|
||||
// only have to increase: they need not start at 1 and need not be
|
||||
// consecutive, so none of these may be rejected.
|
||||
{
|
||||
name: "single-part",
|
||||
upload: []int{1},
|
||||
complete: []int{1},
|
||||
expectedRespStatus: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "strictly-increasing",
|
||||
upload: []int{1, 2},
|
||||
complete: []int{1, 2},
|
||||
expectedRespStatus: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "non-consecutive",
|
||||
upload: []int{1, 3},
|
||||
complete: []int{1, 3},
|
||||
expectedRespStatus: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "gap-not-starting-at-one",
|
||||
upload: []int{5, 9},
|
||||
complete: []int{5, 9},
|
||||
expectedRespStatus: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "single-part-not-one",
|
||||
upload: []int{3},
|
||||
complete: []int{3},
|
||||
expectedRespStatus: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "part-number-bounds",
|
||||
upload: []int{1, globalMaxPartID},
|
||||
complete: []int{1, globalMaxPartID},
|
||||
expectedRespStatus: http.StatusOK,
|
||||
},
|
||||
}
|
||||
|
||||
// completeReq issues a CompleteMultipartUpload request listing partNumbers
|
||||
// in the given order and returns the recorded response.
|
||||
completeReq := func(objectName, uploadID string, partNumbers []int) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
|
||||
completeUploads := &CompleteMultipartUpload{}
|
||||
for _, partNumber := range partNumbers {
|
||||
completeUploads.Parts = append(completeUploads.Parts, CompletePart{
|
||||
PartNumber: partNumber,
|
||||
ETag: partETag,
|
||||
})
|
||||
}
|
||||
completeBytes, err := xml.Marshal(completeUploads)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: error XML encoding of parts: %v", instanceType, err)
|
||||
}
|
||||
|
||||
req, err := newTestSignedRequestV4(http.MethodPost,
|
||||
getCompleteMultipartUploadURL("", bucketName, objectName, uploadID),
|
||||
int64(len(completeBytes)), bytes.NewReader(completeBytes),
|
||||
credentials.AccessKey, credentials.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: failed to create HTTP request for CompleteMultipartUpload: %v", instanceType, err)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
objectName := fmt.Sprintf("test-part-order-%d-%s", i, testCase.name)
|
||||
|
||||
res, err := obj.NewMultipartUpload(ctx, bucketName, objectName, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %s: failed to initiate multipart upload: %v", instanceType, testCase.name, err)
|
||||
}
|
||||
for _, partNumber := range testCase.upload {
|
||||
_, err = obj.PutObjectPart(ctx, bucketName, objectName, res.UploadID, partNumber,
|
||||
mustGetPutObjReader(t, strings.NewReader(partData), int64(len(partData)), partETag, ""), ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %s: failed to upload part %d: %v", instanceType, testCase.name, partNumber, err)
|
||||
}
|
||||
}
|
||||
|
||||
rec := completeReq(objectName, res.UploadID, testCase.complete)
|
||||
if rec.Code != testCase.expectedRespStatus {
|
||||
t.Errorf("%s: %s: expected response status %d, got %d: %s",
|
||||
instanceType, testCase.name, testCase.expectedRespStatus, rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
objInfo, statErr := obj.GetObjectInfo(ctx, bucketName, objectName, ObjectOptions{})
|
||||
|
||||
if testCase.expectedRespStatus != http.StatusOK {
|
||||
var errResp APIErrorResponse
|
||||
if err = xml.Unmarshal(rec.Body.Bytes(), &errResp); err != nil {
|
||||
t.Errorf("%s: %s: failed parsing error response %s: %v",
|
||||
instanceType, testCase.name, rec.Body.String(), err)
|
||||
} else if wantCode := getAPIError(testCase.expectedErr).Code; errResp.Code != wantCode {
|
||||
t.Errorf("%s: %s: expected error code %s, got %s",
|
||||
instanceType, testCase.name, wantCode, errResp.Code)
|
||||
}
|
||||
// A rejected completion must not have touched the target object.
|
||||
if statErr == nil {
|
||||
t.Errorf("%s: %s: expected no object to be created, found one of size %d",
|
||||
instanceType, testCase.name, objInfo.Size)
|
||||
} else if !isErrObjectNotFound(statErr) {
|
||||
t.Errorf("%s: %s: expected ObjectNotFound after a rejected completion, got %v",
|
||||
instanceType, testCase.name, statErr)
|
||||
}
|
||||
|
||||
// The rejection must also leave the upload itself untouched, so
|
||||
// that a client can retry with a well formed list. testCase.upload
|
||||
// is always strictly increasing, so it is the corrected list.
|
||||
retry := completeReq(objectName, res.UploadID, testCase.upload)
|
||||
if retry.Code != http.StatusOK {
|
||||
t.Errorf("%s: %s: expected the upload to survive a rejected completion, retry got %d: %s",
|
||||
instanceType, testCase.name, retry.Code, retry.Body.String())
|
||||
continue
|
||||
}
|
||||
objInfo, statErr = obj.GetObjectInfo(ctx, bucketName, objectName, ObjectOptions{})
|
||||
if statErr != nil {
|
||||
t.Errorf("%s: %s: expected the object to exist after the retry: %v", instanceType, testCase.name, statErr)
|
||||
continue
|
||||
}
|
||||
if wantSize := int64(len(testCase.upload) * len(partData)); objInfo.Size != wantSize {
|
||||
t.Errorf("%s: %s: expected the retried object to be %d bytes, got %d",
|
||||
instanceType, testCase.name, wantSize, objInfo.Size)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if statErr != nil {
|
||||
t.Errorf("%s: %s: expected the object to exist after completion: %v", instanceType, testCase.name, statErr)
|
||||
continue
|
||||
}
|
||||
if wantSize := int64(len(testCase.complete) * len(partData)); objInfo.Size != wantSize {
|
||||
t.Errorf("%s: %s: expected the completed object to be %d bytes, got %d",
|
||||
instanceType, testCase.name, wantSize, objInfo.Size)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -34,8 +34,9 @@ const _osMetric_name = "RemoveAllMkdirAllMkdirRenameOpenFileWOpenFileROpenFileWF
|
||||
var _osMetric_index = [...]uint8{0, 9, 17, 22, 28, 37, 46, 57, 68, 72, 88, 93, 99, 103, 109, 115, 125, 134, 138, 142}
|
||||
|
||||
func (i osMetric) String() string {
|
||||
if i >= osMetric(len(_osMetric_index)-1) {
|
||||
idx := int(i) - 0
|
||||
if i < 0 || idx >= len(_osMetric_index)-1 {
|
||||
return "osMetric(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _osMetric_name[_osMetric_index[i]:_osMetric_index[i+1]]
|
||||
return _osMetric_name[_osMetric_index[idx]:_osMetric_index[idx+1]]
|
||||
}
|
||||
|
||||
@@ -20,8 +20,9 @@ const _rebalanceMetric_name = "RebalanceBucketsRebalanceBucketRebalanceObjectReb
|
||||
var _rebalanceMetric_index = [...]uint8{0, 16, 31, 46, 67, 79}
|
||||
|
||||
func (i rebalanceMetric) String() string {
|
||||
if i >= rebalanceMetric(len(_rebalanceMetric_index)-1) {
|
||||
idx := int(i) - 0
|
||||
if i < 0 || idx >= len(_rebalanceMetric_index)-1 {
|
||||
return "rebalanceMetric(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _rebalanceMetric_name[_rebalanceMetric_index[i]:_rebalanceMetric_index[i+1]]
|
||||
return _rebalanceMetric_name[_rebalanceMetric_index[idx]:_rebalanceMetric_index[idx+1]]
|
||||
}
|
||||
|
||||
@@ -20,8 +20,9 @@ const _rebalStatus_name = "NoneStartedCompletedStoppedFailed"
|
||||
var _rebalStatus_index = [...]uint8{0, 4, 11, 20, 27, 33}
|
||||
|
||||
func (i rebalStatus) String() string {
|
||||
if i >= rebalStatus(len(_rebalStatus_index)-1) {
|
||||
idx := int(i) - 0
|
||||
if i < 0 || idx >= len(_rebalStatus_index)-1 {
|
||||
return "rebalStatus(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _rebalStatus_name[_rebalStatus_index[i]:_rebalStatus_index[i+1]]
|
||||
return _rebalStatus_name[_rebalStatus_index[idx]:_rebalStatus_index[idx+1]]
|
||||
}
|
||||
|
||||
@@ -37,8 +37,9 @@ const _scannerMetric_name = "ReadMetadataCheckMissingSaveUsageApplyAllApplyVersi
|
||||
var _scannerMetric_index = [...]uint8{0, 12, 24, 33, 41, 53, 65, 74, 77, 93, 98, 112, 127, 147, 157, 167, 186, 198, 208, 217, 232, 245, 249}
|
||||
|
||||
func (i scannerMetric) String() string {
|
||||
if i >= scannerMetric(len(_scannerMetric_index)-1) {
|
||||
idx := int(i) - 0
|
||||
if i < 0 || idx >= len(_scannerMetric_index)-1 {
|
||||
return "scannerMetric(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _scannerMetric_name[_scannerMetric_index[i]:_scannerMetric_index[i+1]]
|
||||
return _scannerMetric_name[_scannerMetric_index[idx]:_scannerMetric_index[idx+1]]
|
||||
}
|
||||
|
||||
@@ -144,7 +144,7 @@ func printServerCommonMsg(apiEndpoints []string) {
|
||||
|
||||
// Prints startup message for Object API access, prints link to our SDK documentation.
|
||||
func printObjectAPIMsg() {
|
||||
logger.Startup(color.Blue("\nDocs: ") + "https://docs.min.io")
|
||||
logger.Startup(color.Blue("\nDocs: ") + "https://silo.pgsty.com/docs/")
|
||||
}
|
||||
|
||||
func printLambdaTargets() {
|
||||
@@ -184,7 +184,7 @@ func printCLIAccessMsg(endPoint string, alias string) {
|
||||
// Get saved credentials.
|
||||
cred := globalActiveCred
|
||||
|
||||
const mcQuickStartGuide = "https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart"
|
||||
const mcQuickStartGuide = "https://silo.pgsty.com/reference/minio-mc/#quickstart"
|
||||
|
||||
// Configure 'mc', following block prints platform specific information for minio client.
|
||||
if color.IsTerminal() && (!globalServerCtxt.Anonymous && globalAPIConfig.permitRootAccess()) {
|
||||
|
||||
@@ -396,28 +396,6 @@ func newFileInfo(object string, dataBlocks, parityBlocks int) (fi FileInfo) {
|
||||
return fi
|
||||
}
|
||||
|
||||
// ReadMultipleReq contains information of multiple files to read from disk.
|
||||
type ReadMultipleReq struct {
|
||||
Bucket string `msg:"bk"` // Bucket. Can be empty if multiple buckets.
|
||||
Prefix string `msg:"pr,omitempty"` // Shared prefix of all files. Can be empty. Will be joined to filename without modification.
|
||||
Files []string `msg:"fl"` // Individual files to read.
|
||||
MaxSize int64 `msg:"ms"` // Return error if size is exceed.
|
||||
MetadataOnly bool `msg:"mo"` // Read as XL meta and truncate data.
|
||||
AbortOn404 bool `msg:"ab"` // Stop reading after first file not found.
|
||||
MaxResults int `msg:"mr"` // Stop after this many successful results. <= 0 means all.
|
||||
}
|
||||
|
||||
// ReadMultipleResp contains a single response from a ReadMultipleReq.
|
||||
type ReadMultipleResp struct {
|
||||
Bucket string `msg:"bk"` // Bucket as given by request.
|
||||
Prefix string `msg:"pr,omitempty"` // Prefix as given by request.
|
||||
File string `msg:"fl"` // File name as given in request.
|
||||
Exists bool `msg:"ex"` // Returns whether the file existed on disk.
|
||||
Error string `msg:"er,omitempty"` // Returns any error when reading.
|
||||
Data []byte `msg:"d"` // Contains all data of file.
|
||||
Modtime time.Time `msg:"m"` // Modtime of file on disk.
|
||||
}
|
||||
|
||||
// DeleteVersionHandlerParams are parameters for DeleteVersionHandler
|
||||
type DeleteVersionHandlerParams struct {
|
||||
DiskID string `msg:"id"`
|
||||
|
||||
@@ -4234,672 +4234,6 @@ func (z ReadAllHandlerParams) Msgsize() (s int) {
|
||||
return
|
||||
}
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *ReadMultipleReq) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
var zb0001Mask uint8 /* 1 bits */
|
||||
_ = zb0001Mask
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, err = dc.ReadMapKeyPtr()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "bk":
|
||||
z.Bucket, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Bucket")
|
||||
return
|
||||
}
|
||||
case "pr":
|
||||
z.Prefix, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Prefix")
|
||||
return
|
||||
}
|
||||
zb0001Mask |= 0x1
|
||||
case "fl":
|
||||
var zb0002 uint32
|
||||
zb0002, err = dc.ReadArrayHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Files")
|
||||
return
|
||||
}
|
||||
if cap(z.Files) >= int(zb0002) {
|
||||
z.Files = (z.Files)[:zb0002]
|
||||
} else {
|
||||
z.Files = make([]string, zb0002)
|
||||
}
|
||||
for za0001 := range z.Files {
|
||||
z.Files[za0001], err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Files", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
case "ms":
|
||||
z.MaxSize, err = dc.ReadInt64()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "MaxSize")
|
||||
return
|
||||
}
|
||||
case "mo":
|
||||
z.MetadataOnly, err = dc.ReadBool()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "MetadataOnly")
|
||||
return
|
||||
}
|
||||
case "ab":
|
||||
z.AbortOn404, err = dc.ReadBool()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AbortOn404")
|
||||
return
|
||||
}
|
||||
case "mr":
|
||||
z.MaxResults, err = dc.ReadInt()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "MaxResults")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
// Clear omitted fields.
|
||||
if (zb0001Mask & 0x1) == 0 {
|
||||
z.Prefix = ""
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *ReadMultipleReq) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// check for omitted fields
|
||||
zb0001Len := uint32(7)
|
||||
var zb0001Mask uint8 /* 7 bits */
|
||||
_ = zb0001Mask
|
||||
if z.Prefix == "" {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x2
|
||||
}
|
||||
// variable map header, size zb0001Len
|
||||
err = en.Append(0x80 | uint8(zb0001Len))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// skip if no fields are to be emitted
|
||||
if zb0001Len != 0 {
|
||||
// write "bk"
|
||||
err = en.Append(0xa2, 0x62, 0x6b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.Bucket)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Bucket")
|
||||
return
|
||||
}
|
||||
if (zb0001Mask & 0x2) == 0 { // if not omitted
|
||||
// write "pr"
|
||||
err = en.Append(0xa2, 0x70, 0x72)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.Prefix)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Prefix")
|
||||
return
|
||||
}
|
||||
}
|
||||
// write "fl"
|
||||
err = en.Append(0xa2, 0x66, 0x6c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteArrayHeader(uint32(len(z.Files)))
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Files")
|
||||
return
|
||||
}
|
||||
for za0001 := range z.Files {
|
||||
err = en.WriteString(z.Files[za0001])
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Files", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
// write "ms"
|
||||
err = en.Append(0xa2, 0x6d, 0x73)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteInt64(z.MaxSize)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "MaxSize")
|
||||
return
|
||||
}
|
||||
// write "mo"
|
||||
err = en.Append(0xa2, 0x6d, 0x6f)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteBool(z.MetadataOnly)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "MetadataOnly")
|
||||
return
|
||||
}
|
||||
// write "ab"
|
||||
err = en.Append(0xa2, 0x61, 0x62)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteBool(z.AbortOn404)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AbortOn404")
|
||||
return
|
||||
}
|
||||
// write "mr"
|
||||
err = en.Append(0xa2, 0x6d, 0x72)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteInt(z.MaxResults)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "MaxResults")
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *ReadMultipleReq) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// check for omitted fields
|
||||
zb0001Len := uint32(7)
|
||||
var zb0001Mask uint8 /* 7 bits */
|
||||
_ = zb0001Mask
|
||||
if z.Prefix == "" {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x2
|
||||
}
|
||||
// variable map header, size zb0001Len
|
||||
o = append(o, 0x80|uint8(zb0001Len))
|
||||
|
||||
// skip if no fields are to be emitted
|
||||
if zb0001Len != 0 {
|
||||
// string "bk"
|
||||
o = append(o, 0xa2, 0x62, 0x6b)
|
||||
o = msgp.AppendString(o, z.Bucket)
|
||||
if (zb0001Mask & 0x2) == 0 { // if not omitted
|
||||
// string "pr"
|
||||
o = append(o, 0xa2, 0x70, 0x72)
|
||||
o = msgp.AppendString(o, z.Prefix)
|
||||
}
|
||||
// string "fl"
|
||||
o = append(o, 0xa2, 0x66, 0x6c)
|
||||
o = msgp.AppendArrayHeader(o, uint32(len(z.Files)))
|
||||
for za0001 := range z.Files {
|
||||
o = msgp.AppendString(o, z.Files[za0001])
|
||||
}
|
||||
// string "ms"
|
||||
o = append(o, 0xa2, 0x6d, 0x73)
|
||||
o = msgp.AppendInt64(o, z.MaxSize)
|
||||
// string "mo"
|
||||
o = append(o, 0xa2, 0x6d, 0x6f)
|
||||
o = msgp.AppendBool(o, z.MetadataOnly)
|
||||
// string "ab"
|
||||
o = append(o, 0xa2, 0x61, 0x62)
|
||||
o = msgp.AppendBool(o, z.AbortOn404)
|
||||
// string "mr"
|
||||
o = append(o, 0xa2, 0x6d, 0x72)
|
||||
o = msgp.AppendInt(o, z.MaxResults)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (z *ReadMultipleReq) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
var zb0001Mask uint8 /* 1 bits */
|
||||
_ = zb0001Mask
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "bk":
|
||||
z.Bucket, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Bucket")
|
||||
return
|
||||
}
|
||||
case "pr":
|
||||
z.Prefix, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Prefix")
|
||||
return
|
||||
}
|
||||
zb0001Mask |= 0x1
|
||||
case "fl":
|
||||
var zb0002 uint32
|
||||
zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Files")
|
||||
return
|
||||
}
|
||||
if cap(z.Files) >= int(zb0002) {
|
||||
z.Files = (z.Files)[:zb0002]
|
||||
} else {
|
||||
z.Files = make([]string, zb0002)
|
||||
}
|
||||
for za0001 := range z.Files {
|
||||
z.Files[za0001], bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Files", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
case "ms":
|
||||
z.MaxSize, bts, err = msgp.ReadInt64Bytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "MaxSize")
|
||||
return
|
||||
}
|
||||
case "mo":
|
||||
z.MetadataOnly, bts, err = msgp.ReadBoolBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "MetadataOnly")
|
||||
return
|
||||
}
|
||||
case "ab":
|
||||
z.AbortOn404, bts, err = msgp.ReadBoolBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AbortOn404")
|
||||
return
|
||||
}
|
||||
case "mr":
|
||||
z.MaxResults, bts, err = msgp.ReadIntBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "MaxResults")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
// Clear omitted fields.
|
||||
if (zb0001Mask & 0x1) == 0 {
|
||||
z.Prefix = ""
|
||||
}
|
||||
|
||||
o = bts
|
||||
return
|
||||
}
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *ReadMultipleReq) Msgsize() (s int) {
|
||||
s = 1 + 3 + msgp.StringPrefixSize + len(z.Bucket) + 3 + msgp.StringPrefixSize + len(z.Prefix) + 3 + msgp.ArrayHeaderSize
|
||||
for za0001 := range z.Files {
|
||||
s += msgp.StringPrefixSize + len(z.Files[za0001])
|
||||
}
|
||||
s += 3 + msgp.Int64Size + 3 + msgp.BoolSize + 3 + msgp.BoolSize + 3 + msgp.IntSize
|
||||
return
|
||||
}
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *ReadMultipleResp) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
var zb0001Mask uint8 /* 2 bits */
|
||||
_ = zb0001Mask
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, err = dc.ReadMapKeyPtr()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "bk":
|
||||
z.Bucket, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Bucket")
|
||||
return
|
||||
}
|
||||
case "pr":
|
||||
z.Prefix, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Prefix")
|
||||
return
|
||||
}
|
||||
zb0001Mask |= 0x1
|
||||
case "fl":
|
||||
z.File, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "File")
|
||||
return
|
||||
}
|
||||
case "ex":
|
||||
z.Exists, err = dc.ReadBool()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Exists")
|
||||
return
|
||||
}
|
||||
case "er":
|
||||
z.Error, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Error")
|
||||
return
|
||||
}
|
||||
zb0001Mask |= 0x2
|
||||
case "d":
|
||||
z.Data, err = dc.ReadBytes(z.Data)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Data")
|
||||
return
|
||||
}
|
||||
case "m":
|
||||
z.Modtime, err = dc.ReadTime()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Modtime")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
// Clear omitted fields.
|
||||
if zb0001Mask != 0x3 {
|
||||
if (zb0001Mask & 0x1) == 0 {
|
||||
z.Prefix = ""
|
||||
}
|
||||
if (zb0001Mask & 0x2) == 0 {
|
||||
z.Error = ""
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *ReadMultipleResp) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// check for omitted fields
|
||||
zb0001Len := uint32(7)
|
||||
var zb0001Mask uint8 /* 7 bits */
|
||||
_ = zb0001Mask
|
||||
if z.Prefix == "" {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x2
|
||||
}
|
||||
if z.Error == "" {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x10
|
||||
}
|
||||
// variable map header, size zb0001Len
|
||||
err = en.Append(0x80 | uint8(zb0001Len))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// skip if no fields are to be emitted
|
||||
if zb0001Len != 0 {
|
||||
// write "bk"
|
||||
err = en.Append(0xa2, 0x62, 0x6b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.Bucket)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Bucket")
|
||||
return
|
||||
}
|
||||
if (zb0001Mask & 0x2) == 0 { // if not omitted
|
||||
// write "pr"
|
||||
err = en.Append(0xa2, 0x70, 0x72)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.Prefix)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Prefix")
|
||||
return
|
||||
}
|
||||
}
|
||||
// write "fl"
|
||||
err = en.Append(0xa2, 0x66, 0x6c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.File)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "File")
|
||||
return
|
||||
}
|
||||
// write "ex"
|
||||
err = en.Append(0xa2, 0x65, 0x78)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteBool(z.Exists)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Exists")
|
||||
return
|
||||
}
|
||||
if (zb0001Mask & 0x10) == 0 { // if not omitted
|
||||
// write "er"
|
||||
err = en.Append(0xa2, 0x65, 0x72)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.Error)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Error")
|
||||
return
|
||||
}
|
||||
}
|
||||
// write "d"
|
||||
err = en.Append(0xa1, 0x64)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteBytes(z.Data)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Data")
|
||||
return
|
||||
}
|
||||
// write "m"
|
||||
err = en.Append(0xa1, 0x6d)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteTime(z.Modtime)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Modtime")
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *ReadMultipleResp) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// check for omitted fields
|
||||
zb0001Len := uint32(7)
|
||||
var zb0001Mask uint8 /* 7 bits */
|
||||
_ = zb0001Mask
|
||||
if z.Prefix == "" {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x2
|
||||
}
|
||||
if z.Error == "" {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x10
|
||||
}
|
||||
// variable map header, size zb0001Len
|
||||
o = append(o, 0x80|uint8(zb0001Len))
|
||||
|
||||
// skip if no fields are to be emitted
|
||||
if zb0001Len != 0 {
|
||||
// string "bk"
|
||||
o = append(o, 0xa2, 0x62, 0x6b)
|
||||
o = msgp.AppendString(o, z.Bucket)
|
||||
if (zb0001Mask & 0x2) == 0 { // if not omitted
|
||||
// string "pr"
|
||||
o = append(o, 0xa2, 0x70, 0x72)
|
||||
o = msgp.AppendString(o, z.Prefix)
|
||||
}
|
||||
// string "fl"
|
||||
o = append(o, 0xa2, 0x66, 0x6c)
|
||||
o = msgp.AppendString(o, z.File)
|
||||
// string "ex"
|
||||
o = append(o, 0xa2, 0x65, 0x78)
|
||||
o = msgp.AppendBool(o, z.Exists)
|
||||
if (zb0001Mask & 0x10) == 0 { // if not omitted
|
||||
// string "er"
|
||||
o = append(o, 0xa2, 0x65, 0x72)
|
||||
o = msgp.AppendString(o, z.Error)
|
||||
}
|
||||
// string "d"
|
||||
o = append(o, 0xa1, 0x64)
|
||||
o = msgp.AppendBytes(o, z.Data)
|
||||
// string "m"
|
||||
o = append(o, 0xa1, 0x6d)
|
||||
o = msgp.AppendTime(o, z.Modtime)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (z *ReadMultipleResp) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
var zb0001Mask uint8 /* 2 bits */
|
||||
_ = zb0001Mask
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "bk":
|
||||
z.Bucket, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Bucket")
|
||||
return
|
||||
}
|
||||
case "pr":
|
||||
z.Prefix, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Prefix")
|
||||
return
|
||||
}
|
||||
zb0001Mask |= 0x1
|
||||
case "fl":
|
||||
z.File, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "File")
|
||||
return
|
||||
}
|
||||
case "ex":
|
||||
z.Exists, bts, err = msgp.ReadBoolBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Exists")
|
||||
return
|
||||
}
|
||||
case "er":
|
||||
z.Error, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Error")
|
||||
return
|
||||
}
|
||||
zb0001Mask |= 0x2
|
||||
case "d":
|
||||
z.Data, bts, err = msgp.ReadBytesBytes(bts, z.Data)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Data")
|
||||
return
|
||||
}
|
||||
case "m":
|
||||
z.Modtime, bts, err = msgp.ReadTimeBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Modtime")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
// Clear omitted fields.
|
||||
if zb0001Mask != 0x3 {
|
||||
if (zb0001Mask & 0x1) == 0 {
|
||||
z.Prefix = ""
|
||||
}
|
||||
if (zb0001Mask & 0x2) == 0 {
|
||||
z.Error = ""
|
||||
}
|
||||
}
|
||||
o = bts
|
||||
return
|
||||
}
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *ReadMultipleResp) Msgsize() (s int) {
|
||||
s = 1 + 3 + msgp.StringPrefixSize + len(z.Bucket) + 3 + msgp.StringPrefixSize + len(z.Prefix) + 3 + msgp.StringPrefixSize + len(z.File) + 3 + msgp.BoolSize + 3 + msgp.StringPrefixSize + len(z.Error) + 2 + msgp.BytesPrefixSize + len(z.Data) + 2 + msgp.TimeSize
|
||||
return
|
||||
}
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *ReadPartsReq) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
var field []byte
|
||||
|
||||
@@ -2156,232 +2156,6 @@ func BenchmarkDecodeReadAllHandlerParams(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalReadMultipleReq(t *testing.T) {
|
||||
v := ReadMultipleReq{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgReadMultipleReq(b *testing.B) {
|
||||
v := ReadMultipleReq{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgReadMultipleReq(b *testing.B) {
|
||||
v := ReadMultipleReq{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalReadMultipleReq(b *testing.B) {
|
||||
v := ReadMultipleReq{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeReadMultipleReq(t *testing.T) {
|
||||
v := ReadMultipleReq{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeReadMultipleReq Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := ReadMultipleReq{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeReadMultipleReq(b *testing.B) {
|
||||
v := ReadMultipleReq{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeReadMultipleReq(b *testing.B) {
|
||||
v := ReadMultipleReq{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalReadMultipleResp(t *testing.T) {
|
||||
v := ReadMultipleResp{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgReadMultipleResp(b *testing.B) {
|
||||
v := ReadMultipleResp{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgReadMultipleResp(b *testing.B) {
|
||||
v := ReadMultipleResp{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalReadMultipleResp(b *testing.B) {
|
||||
v := ReadMultipleResp{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeReadMultipleResp(t *testing.T) {
|
||||
v := ReadMultipleResp{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeReadMultipleResp Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := ReadMultipleResp{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeReadMultipleResp(b *testing.B) {
|
||||
v := ReadMultipleResp{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeReadMultipleResp(b *testing.B) {
|
||||
v := ReadMultipleResp{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalReadPartsReq(t *testing.T) {
|
||||
v := ReadPartsReq{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
|
||||
@@ -101,7 +101,6 @@ type StorageAPI interface {
|
||||
VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error)
|
||||
StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error)
|
||||
ReadParts(ctx context.Context, bucket string, partMetaPaths ...string) ([]*ObjectPartInfo, error)
|
||||
ReadMultiple(ctx context.Context, req ReadMultipleReq, resp chan<- ReadMultipleResp) error
|
||||
CleanAbandonedData(ctx context.Context, volume string, path string) error
|
||||
|
||||
// Write all data, syncs the data to disk.
|
||||
|
||||
@@ -912,45 +912,6 @@ func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path
|
||||
return stat, toStorageErr(err)
|
||||
}
|
||||
|
||||
// ReadMultiple will read multiple files and send each back as response.
|
||||
// Files are read and returned in the given order.
|
||||
// The resp channel is closed before the call returns.
|
||||
// Only a canceled context or network errors returns an error.
|
||||
func (client *storageRESTClient) ReadMultiple(ctx context.Context, req ReadMultipleReq, resp chan<- ReadMultipleResp) error {
|
||||
defer xioutil.SafeClose(resp)
|
||||
body, err := req.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
respBody, err := client.call(ctx, storageRESTMethodReadMultiple, nil, bytes.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer xhttp.DrainBody(respBody)
|
||||
|
||||
pr, pw := io.Pipe()
|
||||
go func() {
|
||||
pw.CloseWithError(waitForHTTPStream(respBody, xioutil.NewDeadlineWriter(pw, globalDriveConfig.GetMaxTimeout())))
|
||||
}()
|
||||
mr := msgp.NewReader(pr)
|
||||
defer readMsgpReaderPoolPut(mr)
|
||||
for {
|
||||
var file ReadMultipleResp
|
||||
if err := file.DecodeMsg(mr); err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
err = nil
|
||||
}
|
||||
pr.CloseWithError(err)
|
||||
return toStorageErr(err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case resp <- file:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// CleanAbandonedData will read metadata of the object on disk
|
||||
// and delete any data directories and inline data that isn't referenced in metadata.
|
||||
func (client *storageRESTClient) CleanAbandonedData(ctx context.Context, volume string, path string) error {
|
||||
|
||||
@@ -41,7 +41,6 @@ const (
|
||||
storageRESTMethodRenameFile = "/rfile"
|
||||
storageRESTMethodVerifyFile = "/vfile"
|
||||
storageRESTMethodStatInfoFile = "/sfile"
|
||||
storageRESTMethodReadMultiple = "/rmpl"
|
||||
storageRESTMethodCleanAbandoned = "/cln"
|
||||
storageRESTMethodDeleteBulk = "/dblk"
|
||||
storageRESTMethodReadParts = "/rps"
|
||||
|
||||
@@ -0,0 +1,417 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"runtime"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
)
|
||||
|
||||
// guardedStorage rejects filesystem paths that arrive from an internode
|
||||
// payload and would resolve outside the volume they name.
|
||||
//
|
||||
// Why here and not in each handler: the global HTTP middleware validates only
|
||||
// r.URL.Path and r.Form (query arguments), and r.Form is never populated from
|
||||
// a request body. Paths carried in a msgpack body, or in a grid RPC frame on
|
||||
// the long-lived /minio/grid/v1 websocket, therefore reach xlStorage
|
||||
// unvalidated. Every storage-REST and grid handler obtains its StorageAPI from
|
||||
// storageRESTServer.getStorage(), so wrapping that one call covers all of them
|
||||
// at once, sees the nested struct fields a per-handler check tends to miss,
|
||||
// and cannot drift out of sync as handlers are added.
|
||||
//
|
||||
// Scope, deliberately narrow:
|
||||
//
|
||||
// - Only *path* arguments are checked here. Volume arguments are validated
|
||||
// at the sink by xlStorage.getVolDir, which additionally covers callers
|
||||
// that bypass this wrapper entirely (the peer-S3 bucket RPCs). Do not
|
||||
// conclude from the absence of a volume check here that volumes are inert.
|
||||
//
|
||||
// - Local callers - the erasure layer talking to its own drives - are not
|
||||
// wrapped. Their object names already passed IsValidObjectName at the S3
|
||||
// boundary, so wrapping them would add cost and regression risk without
|
||||
// adding a control.
|
||||
type guardedStorage struct {
|
||||
StorageAPI
|
||||
}
|
||||
|
||||
// isVolumeRootAlias reports whether p addresses the volume directory itself
|
||||
// rather than something inside it, i.e. whether pathJoin(volumeDir, p)
|
||||
// collapses back to volumeDir. That happens only for the empty string and for
|
||||
// strings made up entirely of separators.
|
||||
//
|
||||
// Whitespace must NOT be treated as a separator here, even though
|
||||
// hasBadPathComponent trims it when comparing a segment against "."/"..".
|
||||
// path.Clean does not touch spaces, so pathJoin(volumeDir, " ") is
|
||||
// "volumeDir/ " - a real directory named two spaces, not the volume root. A
|
||||
// whitespace-only object key is legal in S3 (IsValidObjectName accepts it) and
|
||||
// is committed through RenameData on the PutObject path, so rejecting it here
|
||||
// would fail those writes on every remote drive at once. See
|
||||
// TestGuardAcceptsEveryLegalObjectName.
|
||||
//
|
||||
// Three characters collapse a component on Windows but not on Unix, so the
|
||||
// rule is platform-split and exercised from either platform through
|
||||
// isVolumeRootAliasOn:
|
||||
//
|
||||
// - Backslash is a separator on Windows only. path.Clean never treats it as
|
||||
// one, so on Unix "\\" names an ordinary file and is a legal S3 object key;
|
||||
// refusing it there would make a distributed cluster reject a write that a
|
||||
// single-node server accepts.
|
||||
//
|
||||
// - Space and period are stripped from the end of a path component by the
|
||||
// Win32 normalisation layer. A component made only of those characters
|
||||
// therefore disappears and the path resolves to its parent - the volume
|
||||
// root. Go does not add the \\?\ prefix that would suppress this for the
|
||||
// short paths used here, so " " and "..." reach the syscall as an empty
|
||||
// component. On Unix they are ordinary filenames and legal object keys.
|
||||
//
|
||||
// Note the second point is reasoned from documented Win32 behavior, not from a
|
||||
// Windows test run: CI is Linux-only, so TestIsVolumeRootAliasIsPlatformCorrect
|
||||
// pins both branches of the predicate rather than the syscall behavior itself.
|
||||
func isVolumeRootAlias(p string) bool {
|
||||
return isVolumeRootAliasOn(p, runtime.GOOS == globalWindowsOSName)
|
||||
}
|
||||
|
||||
func isVolumeRootAliasOn(p string, windows bool) bool {
|
||||
for i := range len(p) {
|
||||
if p[i] == SlashSeparatorChar {
|
||||
continue
|
||||
}
|
||||
if windows && (p[i] == '\\' || p[i] == ' ' || p[i] == '.') {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// guardErasureParams rejects a FileInfo from which no meaningful expected shard
|
||||
// size can be derived, because "no meaningful size" degrades to "everything
|
||||
// passes" rather than to an error.
|
||||
//
|
||||
// checkPart's only integrity test is "st.Size() < expectedSize". Whenever
|
||||
// ShardFileSize yields 0, that comparison is false for every file that exists -
|
||||
// including a truncated shard - so the part comes back reported healthy and a
|
||||
// heal driven by the result skips a shard that actually needs repair. Two
|
||||
// distinct inputs produce that 0:
|
||||
//
|
||||
// - Unusable erasure parameters with a part of positive size. ShardFileSize
|
||||
// returns 0 early so the division cannot panic.
|
||||
//
|
||||
// - A part of NEGATIVE size, with or without usable parameters: numShards
|
||||
// floors to 0 and ceilFrac of a negative numerator is 0, so the arithmetic
|
||||
// lands on 0 by itself. This one is easy to miss precisely because valid
|
||||
// erasure parameters do not save you from it.
|
||||
//
|
||||
// Parts of zero length are left alone - ShardFileSize legitimately returns 0
|
||||
// for them, so they say nothing about whether the metadata is sound.
|
||||
//
|
||||
// This is the boundary check; ShardFileSize's own zero-value guard stays as the
|
||||
// last line of defense against a panic.
|
||||
func guardErasureParams(fi FileInfo) error {
|
||||
// Negative sizes share their rule with the storage layer, which also has to
|
||||
// cope with metadata already on disk; keep the two from drifting apart by
|
||||
// asking the same predicate.
|
||||
if fi.HasNegativePartSize() {
|
||||
return errFileCorrupt
|
||||
}
|
||||
usable := fi.Erasure.BlockSize > 0 && fi.Erasure.DataBlocks > 0
|
||||
for _, p := range fi.Parts {
|
||||
if p.Size > 0 && !usable {
|
||||
return errFileCorrupt
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// guardPaths rejects traversal in any of the supplied paths. The empty string
|
||||
// is allowed: it legitimately names the volume root for listing operations,
|
||||
// and an empty FileInfo.DataDir is normal for inline and transitioned objects.
|
||||
//
|
||||
// Callers pass the entire batch here before invoking storage, so a payload
|
||||
// mixing a valid target with a malicious one is rejected whole and performs no
|
||||
// partial work. (This returns on the first offending path; what matters is that
|
||||
// it runs to a verdict before any element has been acted on.)
|
||||
func guardPaths(paths ...string) error {
|
||||
for _, p := range paths {
|
||||
if hasBadPathComponent(p) {
|
||||
return errFileAccessDenied
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// guardObjectPaths is guardPaths plus a rejection of volume-root aliases. It
|
||||
// applies to the destructive verbs only - rename and bulk delete - because
|
||||
// those relocate or destroy the volume root when handed one, whereas reads and
|
||||
// writes of the volume root merely fail on their own.
|
||||
func guardObjectPaths(paths ...string) error {
|
||||
for _, p := range paths {
|
||||
if hasBadPathComponent(p) || isVolumeRootAlias(p) {
|
||||
return errFileAccessDenied
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// guardVersions validates every path-bearing field reachable through a
|
||||
// DeleteVersions payload: the per-object name, and the DataDir of each version.
|
||||
// DataDir is joined under the object directory, and may legitimately be empty
|
||||
// for inline and transitioned objects, so it gets guardPaths - never
|
||||
// guardObjectPaths.
|
||||
func guardVersions(versions []FileInfoVersions, opts DeleteOptions) error {
|
||||
if err := guardPaths(opts.OldDataDir); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, v := range versions {
|
||||
if err := guardPaths(v.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, fi := range v.Versions {
|
||||
if err := guardPaths(fi.DataDir); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Metadata operations
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func (g guardedStorage) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) error {
|
||||
if err := guardPaths(path, fi.DataDir, opts.OldDataDir); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.StorageAPI.DeleteVersion(ctx, volume, path, fi, forceDelMarker, opts)
|
||||
}
|
||||
|
||||
func (g guardedStorage) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions, opts DeleteOptions) []error {
|
||||
if err := guardVersions(versions, opts); err != nil {
|
||||
// The whole batch is refused: validating every element before acting on
|
||||
// any of it is what stops a payload mixing a valid target with a
|
||||
// malicious one from performing a partial delete.
|
||||
errs := make([]error, len(versions))
|
||||
for i := range errs {
|
||||
errs[i] = err
|
||||
}
|
||||
return errs
|
||||
}
|
||||
return g.StorageAPI.DeleteVersions(ctx, volume, versions, opts)
|
||||
}
|
||||
|
||||
func (g guardedStorage) DeleteBulk(ctx context.Context, volume string, paths ...string) error {
|
||||
if err := guardObjectPaths(paths...); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.StorageAPI.DeleteBulk(ctx, volume, paths...)
|
||||
}
|
||||
|
||||
func (g guardedStorage) WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) error {
|
||||
if err := guardPaths(path, fi.DataDir); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.StorageAPI.WriteMetadata(ctx, origvolume, volume, path, fi)
|
||||
}
|
||||
|
||||
func (g guardedStorage) UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo, opts UpdateMetadataOpts) error {
|
||||
if err := guardPaths(path, fi.DataDir); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.StorageAPI.UpdateMetadata(ctx, volume, path, fi, opts)
|
||||
}
|
||||
|
||||
func (g guardedStorage) ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (FileInfo, error) {
|
||||
if err := guardPaths(path); err != nil {
|
||||
return FileInfo{}, err
|
||||
}
|
||||
return g.StorageAPI.ReadVersion(ctx, origvolume, volume, path, versionID, opts)
|
||||
}
|
||||
|
||||
func (g guardedStorage) ReadXL(ctx context.Context, volume, path string, readData bool) (RawFileInfo, error) {
|
||||
if err := guardPaths(path); err != nil {
|
||||
return RawFileInfo{}, err
|
||||
}
|
||||
return g.StorageAPI.ReadXL(ctx, volume, path, readData)
|
||||
}
|
||||
|
||||
func (g guardedStorage) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) {
|
||||
if err := guardObjectPaths(srcPath, dstPath); err != nil {
|
||||
return RenameDataResp{}, err
|
||||
}
|
||||
// DataDir is empty for inline and transitioned objects, so it must not be
|
||||
// held to the non-empty rule above.
|
||||
if err := guardPaths(fi.DataDir); err != nil {
|
||||
return RenameDataResp{}, err
|
||||
}
|
||||
return g.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// File operations
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func (g guardedStorage) ListDir(ctx context.Context, origvolume, volume, dirPath string, count int) ([]string, error) {
|
||||
if err := guardPaths(dirPath); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return g.StorageAPI.ListDir(ctx, origvolume, volume, dirPath, count)
|
||||
}
|
||||
|
||||
func (g guardedStorage) ReadFile(ctx context.Context, volume, path string, offset int64, buf []byte, verifier *BitrotVerifier) (int64, error) {
|
||||
if err := guardPaths(path); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return g.StorageAPI.ReadFile(ctx, volume, path, offset, buf, verifier)
|
||||
}
|
||||
|
||||
func (g guardedStorage) AppendFile(ctx context.Context, volume, path string, buf []byte) error {
|
||||
if err := guardPaths(path); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.StorageAPI.AppendFile(ctx, volume, path, buf)
|
||||
}
|
||||
|
||||
func (g guardedStorage) CreateFile(ctx context.Context, origvolume, volume, path string, size int64, reader io.Reader) error {
|
||||
if err := guardPaths(path); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.StorageAPI.CreateFile(ctx, origvolume, volume, path, size, reader)
|
||||
}
|
||||
|
||||
func (g guardedStorage) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) {
|
||||
if err := guardPaths(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return g.StorageAPI.ReadFileStream(ctx, volume, path, offset, length)
|
||||
}
|
||||
|
||||
func (g guardedStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error {
|
||||
if err := guardObjectPaths(srcPath, dstPath); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.StorageAPI.RenameFile(ctx, srcVolume, srcPath, dstVolume, dstPath)
|
||||
}
|
||||
|
||||
func (g guardedStorage) RenamePart(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string, meta []byte, skipParent string) error {
|
||||
if err := guardObjectPaths(srcPath, dstPath); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := guardPaths(skipParent); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.StorageAPI.RenamePart(ctx, srcVolume, srcPath, dstVolume, dstPath, meta, skipParent)
|
||||
}
|
||||
|
||||
func (g guardedStorage) CheckParts(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error) {
|
||||
if err := guardPaths(path, fi.DataDir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := guardErasureParams(fi); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return g.StorageAPI.CheckParts(ctx, volume, path, fi)
|
||||
}
|
||||
|
||||
func (g guardedStorage) Delete(ctx context.Context, volume, path string, opts DeleteOptions) error {
|
||||
if err := guardPaths(path, opts.OldDataDir); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.StorageAPI.Delete(ctx, volume, path, opts)
|
||||
}
|
||||
|
||||
func (g guardedStorage) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error) {
|
||||
if err := guardPaths(path, fi.DataDir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := guardErasureParams(fi); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return g.StorageAPI.VerifyFile(ctx, volume, path, fi)
|
||||
}
|
||||
|
||||
func (g guardedStorage) StatInfoFile(ctx context.Context, volume, path string, glob bool) ([]StatInfo, error) {
|
||||
if err := guardPaths(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return g.StorageAPI.StatInfoFile(ctx, volume, path, glob)
|
||||
}
|
||||
|
||||
// ReadParts is a read, so it gets guardPaths rather than guardObjectPaths: the
|
||||
// volume-root alias rule exists because rename and delete *relocate or destroy*
|
||||
// the root, whereas a read of it merely fails to find part.N. Applying the
|
||||
// stricter rule here would widen the false-positive surface for no gain.
|
||||
func (g guardedStorage) ReadParts(ctx context.Context, bucket string, partMetaPaths ...string) ([]*ObjectPartInfo, error) {
|
||||
if err := guardPaths(partMetaPaths...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return g.StorageAPI.ReadParts(ctx, bucket, partMetaPaths...)
|
||||
}
|
||||
|
||||
func (g guardedStorage) CleanAbandonedData(ctx context.Context, volume, path string) error {
|
||||
if err := guardPaths(path); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.StorageAPI.CleanAbandonedData(ctx, volume, path)
|
||||
}
|
||||
|
||||
func (g guardedStorage) WriteAll(ctx context.Context, volume, path string, b []byte) error {
|
||||
if err := guardPaths(path); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.StorageAPI.WriteAll(ctx, volume, path, b)
|
||||
}
|
||||
|
||||
func (g guardedStorage) ReadAll(ctx context.Context, volume, path string) ([]byte, error) {
|
||||
if err := guardPaths(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return g.StorageAPI.ReadAll(ctx, volume, path)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Directory walks and scanning
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func (g guardedStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writer) error {
|
||||
// FilterPrefix and ForwardTo are only string-compared against readDir
|
||||
// output, they never reach a path join.
|
||||
if err := guardPaths(opts.Bucket, opts.BaseDir); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.StorageAPI.WalkDir(ctx, opts, wr)
|
||||
}
|
||||
|
||||
// NSScanner is the one StorageAPI method that reaches the filesystem without
|
||||
// passing through getVolDir: the scanner joins cache.Info.Name onto drivePath
|
||||
// directly (see scanFolder). It is therefore guarded here rather than at the
|
||||
// sink. Today an unregistered bucket name is also rejected further down by
|
||||
// globalBucketObjectLockSys, but that is a lookup in an unrelated subsystem,
|
||||
// not a containment boundary.
|
||||
func (g guardedStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry, scanMode madmin.HealScanMode, shouldSleep func() bool) (dataUsageCache, error) {
|
||||
if err := guardPaths(cache.Info.Name); err != nil {
|
||||
// The caller blocks until updates is closed; NSScanner owns closing it.
|
||||
xioutil.SafeClose(updates)
|
||||
return cache, err
|
||||
}
|
||||
return g.StorageAPI.NSScanner(ctx, cache, updates, scanMode, shouldSleep)
|
||||
}
|
||||
@@ -0,0 +1,392 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio-go/v7/pkg/s3utils"
|
||||
)
|
||||
|
||||
// poisonStorage implements StorageAPI with a nil embedded interface, so any
|
||||
// call that reaches it panics. guardedStorage must never delegate a traversing
|
||||
// path to it.
|
||||
type poisonStorage struct {
|
||||
StorageAPI
|
||||
}
|
||||
|
||||
// volumeOnlyOrPathless lists the StorageAPI methods guardedStorage
|
||||
// deliberately does not override, with the reason each is safe.
|
||||
//
|
||||
// Adding a method to StorageAPI without either guarding it or adding it here
|
||||
// with a reason makes TestGuardedStorageCoversEveryPathMethod fail.
|
||||
var volumeOnlyOrPathless = map[string]string{
|
||||
// No filesystem path in the signature at all.
|
||||
"String": "identity", "IsOnline": "status", "LastConn": "status",
|
||||
"IsLocal": "topology", "Hostname": "topology", "Endpoint": "topology",
|
||||
"Close": "lifecycle", "GetDiskID": "identity", "SetDiskID": "identity",
|
||||
"Healing": "status", "GetDiskLoc": "topology", "DiskInfo": "no path field",
|
||||
"ListVols": "no argument",
|
||||
|
||||
// Volume-only. xlStorage.getVolDir validates the volume at the sink, which
|
||||
// also covers the peer-S3 callers that never pass through this wrapper.
|
||||
"MakeVol": "volume-only, guarded by getVolDir",
|
||||
"MakeVolBulk": "volume-only, guarded by getVolDir",
|
||||
"StatVol": "volume-only, guarded by getVolDir",
|
||||
"DeleteVol": "volume-only, guarded by getVolDir",
|
||||
}
|
||||
|
||||
// TestGuardedStorageCoversEveryPathMethod calls every StorageAPI method on a
|
||||
// guardedStorage whose embedded storage panics, passing "../evil" in every
|
||||
// string it can reach - including strings nested inside structs and slices,
|
||||
// which is where a hand-written check is most likely to miss one.
|
||||
//
|
||||
// A method that returns without panicking rejected the path. A method that
|
||||
// panics delegated it.
|
||||
func TestGuardedStorageCoversEveryPathMethod(t *testing.T) {
|
||||
g := reflect.ValueOf(guardedStorage{poisonStorage{}})
|
||||
iface := reflect.TypeOf((*StorageAPI)(nil)).Elem()
|
||||
|
||||
for i := range iface.NumMethod() {
|
||||
name := iface.Method(i).Name
|
||||
if reason, ok := volumeOnlyOrPathless[name]; ok {
|
||||
t.Logf("skipping %s: %s", name, reason)
|
||||
continue
|
||||
}
|
||||
m := g.MethodByName(name)
|
||||
if !m.IsValid() {
|
||||
t.Errorf("%s: not found on guardedStorage", name)
|
||||
continue
|
||||
}
|
||||
mt := m.Type()
|
||||
args := make([]reflect.Value, mt.NumIn())
|
||||
for j := range args {
|
||||
args[j] = poisonArg(t, mt.In(j))
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
t.Errorf("%s delegated a traversing path to the underlying storage "+
|
||||
"- it is not guarded. Add a guardedStorage override, or add it to "+
|
||||
"volumeOnlyOrPathless with a reason. (%v)", name, r)
|
||||
}
|
||||
}()
|
||||
if mt.IsVariadic() {
|
||||
m.CallSlice(args)
|
||||
} else {
|
||||
m.Call(args)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestGuardChecksUnreachableFields pins the guard on path fields that exist on
|
||||
// the wire but that no handler currently reads, so an end-to-end test cannot
|
||||
// observe them. DeleteVersionHandler hardcodes `opts := DeleteOptions{}` and
|
||||
// discards p.Opts, which means DeleteOptions.OldDataDir - a value that reaches
|
||||
// renameAll() with no containment of its own - is unreachable by accident
|
||||
// rather than by design. If someone later plumbs p.Opts through, the guard is
|
||||
// already there; this test is what stops it from being removed as "dead".
|
||||
func TestGuardChecksUnreachableFields(t *testing.T) {
|
||||
g := guardedStorage{poisonStorage{}}
|
||||
ctx := context.Background()
|
||||
|
||||
if err := g.DeleteVersion(ctx, "foo", "obj", FileInfo{}, false,
|
||||
DeleteOptions{OldDataDir: poisonPath}); !errors.Is(err, errFileAccessDenied) {
|
||||
t.Errorf("DeleteVersion with a traversing OldDataDir: got %v, want %v", err, errFileAccessDenied)
|
||||
}
|
||||
|
||||
errs := g.DeleteVersions(ctx, "foo", []FileInfoVersions{{Name: "obj"}},
|
||||
DeleteOptions{OldDataDir: poisonPath})
|
||||
if len(errs) != 1 || !errors.Is(errs[0], errFileAccessDenied) {
|
||||
t.Errorf("DeleteVersions with a traversing OldDataDir: got %v, want %v", errs, errFileAccessDenied)
|
||||
}
|
||||
|
||||
if err := g.Delete(ctx, "foo", "obj", DeleteOptions{OldDataDir: poisonPath}); !errors.Is(err, errFileAccessDenied) {
|
||||
t.Errorf("Delete with a traversing OldDataDir: got %v, want %v", err, errFileAccessDenied)
|
||||
}
|
||||
}
|
||||
|
||||
const poisonPath = "../evil"
|
||||
|
||||
func poisonArg(t *testing.T, typ reflect.Type) reflect.Value {
|
||||
t.Helper()
|
||||
switch typ {
|
||||
case reflect.TypeOf((*context.Context)(nil)).Elem():
|
||||
return reflect.ValueOf(context.Background())
|
||||
case reflect.TypeOf((*io.Writer)(nil)).Elem():
|
||||
return reflect.ValueOf(io.Discard)
|
||||
case reflect.TypeOf((*io.Reader)(nil)).Elem():
|
||||
return reflect.ValueOf(bytes.NewReader(nil))
|
||||
}
|
||||
if typ.Kind() == reflect.Chan {
|
||||
// NSScanner's updates channel: the guard is required to close it.
|
||||
return reflect.MakeChan(reflect.ChanOf(reflect.BothDir, typ.Elem()), 1).Convert(typ)
|
||||
}
|
||||
return poisonValue(typ, 0)
|
||||
}
|
||||
|
||||
// poisonValue builds a value of typ with every settable string set to
|
||||
// poisonPath, recursing into structs, slices and pointers.
|
||||
func poisonValue(typ reflect.Type, depth int) reflect.Value {
|
||||
v := reflect.New(typ).Elem()
|
||||
if depth > 4 {
|
||||
return v
|
||||
}
|
||||
switch typ.Kind() {
|
||||
case reflect.String:
|
||||
v.SetString(poisonPath)
|
||||
case reflect.Struct:
|
||||
for i := range typ.NumField() {
|
||||
f := v.Field(i)
|
||||
if !f.CanSet() {
|
||||
continue // unexported
|
||||
}
|
||||
// Skip self-referential and container types we cannot poison
|
||||
// meaningfully; the fields that matter are plain strings.
|
||||
switch f.Kind() {
|
||||
case reflect.Map, reflect.Chan, reflect.Func, reflect.Interface, reflect.UnsafePointer:
|
||||
continue
|
||||
}
|
||||
f.Set(poisonValue(f.Type(), depth+1))
|
||||
}
|
||||
case reflect.Slice:
|
||||
s := reflect.MakeSlice(typ, 1, 1)
|
||||
s.Index(0).Set(poisonValue(typ.Elem(), depth+1))
|
||||
v.Set(s)
|
||||
case reflect.Pointer:
|
||||
p := reflect.New(typ.Elem())
|
||||
p.Elem().Set(poisonValue(typ.Elem(), depth+1))
|
||||
v.Set(p)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func TestIsVolumeRootAlias(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
path string
|
||||
want bool
|
||||
}{
|
||||
// Collapse back to the volume directory.
|
||||
{"", true},
|
||||
{"/", true},
|
||||
{"//", true},
|
||||
// Backslash is platform-dependent; see TestIsVolumeRootAliasIsPlatformCorrect.
|
||||
// Whitespace is NOT a separator. path.Clean leaves it alone, so these
|
||||
// name real directories and are legal S3 object keys.
|
||||
{" ", false},
|
||||
{" ", false},
|
||||
{"\t", false},
|
||||
{"\n", false},
|
||||
{" / ", false},
|
||||
{"/ ", false},
|
||||
{"a", false},
|
||||
{"/a", false},
|
||||
{"..", false},
|
||||
{" a ", false},
|
||||
{"obj/part.1", false},
|
||||
} {
|
||||
if got := isVolumeRootAlias(tc.path); got != tc.want {
|
||||
t.Errorf("isVolumeRootAlias(%q) = %v, want %v", tc.path, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// guardMayRefuse reports whether the guards are permitted to refuse a name that
|
||||
// IsValidObjectName accepts. On Unix the answer is never: no legal object name
|
||||
// is made up entirely of '/', so the invariant below carries NO exceptions.
|
||||
//
|
||||
// Stated in terms of the separator set rather than by calling isVolumeRootAlias,
|
||||
// so that widening that function cannot silently widen what the test forgives.
|
||||
// Two regressions have already hidden in exactly that gap: whitespace was once
|
||||
// counted as a separator (refusing the legal key " "), and an earlier version
|
||||
// of this helper excused every backslash-only name on every platform, which is
|
||||
// why the fuzzer could not see that bug at all.
|
||||
func guardMayRefuse(name string) bool {
|
||||
if runtime.GOOS != globalWindowsOSName {
|
||||
return false
|
||||
}
|
||||
// Mirrors the Windows branch of isVolumeRootAliasOn: separators plus the
|
||||
// characters Win32 strips from a component.
|
||||
return strings.Trim(name, "/\\ .") == ""
|
||||
}
|
||||
|
||||
func TestIsVolumeRootAliasIsPlatformCorrect(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
path string
|
||||
unix, windowsWant bool
|
||||
}{
|
||||
{"", true, true},
|
||||
{"/", true, true},
|
||||
{"//", true, true},
|
||||
// Backslash: an ordinary filename on Unix, a separator on Windows.
|
||||
{"\\", false, true},
|
||||
{"\\\\", false, true},
|
||||
{"/\\", false, true},
|
||||
// Space and period: ordinary filename characters on Unix, but stripped
|
||||
// from a component by Win32 normalisation, so a component made only of
|
||||
// them vanishes and the path resolves to the volume root.
|
||||
{" ", false, true},
|
||||
{" ", false, true},
|
||||
{" / ", false, true},
|
||||
{"...", false, true},
|
||||
{". .", false, true},
|
||||
// Never an alias anywhere.
|
||||
{"\t", false, false},
|
||||
{"\n", false, false},
|
||||
{"a", false, false},
|
||||
{"/a", false, false},
|
||||
{"a ", false, false},
|
||||
{" a", false, false},
|
||||
{"a.", false, false},
|
||||
} {
|
||||
if got := isVolumeRootAliasOn(tc.path, false); got != tc.unix {
|
||||
t.Errorf("isVolumeRootAliasOn(%q, unix) = %v, want %v", tc.path, got, tc.unix)
|
||||
}
|
||||
if got := isVolumeRootAliasOn(tc.path, true); got != tc.windowsWant {
|
||||
t.Errorf("isVolumeRootAliasOn(%q, windows) = %v, want %v", tc.path, got, tc.windowsWant)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestGuardAcceptsEveryLegalObjectName pins the invariant that actually matters
|
||||
// for availability: if S3 accepts a name, the guards must accept it too.
|
||||
//
|
||||
// A guard that rejects a legal object name breaks writes on every remote drive
|
||||
// simultaneously, which fails quorum - a worse outage than the vulnerability it
|
||||
// defends against. This caught a real regression: isVolumeRootAlias originally
|
||||
// treated whitespace as a separator, so a legal key of " " was refused on the
|
||||
// PutObject commit path.
|
||||
func TestGuardAcceptsEveryLegalObjectName(t *testing.T) {
|
||||
names := []string{
|
||||
// Whitespace, in every position. All legal in S3.
|
||||
" ", " ", "\t", "\n", "a b", " a", "a ", " a ", " / ", "/ ",
|
||||
// Dots that are not "." or ".." segments.
|
||||
"..foo", "foo..", ".hidden", "a/..b", "a/b..", "a.b/c..d",
|
||||
// "..." is a legal key on Unix and accepted; on Windows it is a volume-root
|
||||
// alias (Win32 strips trailing periods) and guardMayRefuse excuses it there.
|
||||
"part.1.meta", "...", "a/...",
|
||||
// Ordinary shapes.
|
||||
"a", "/a", "a/b/c", "obj/part.1", "__XLDIR__",
|
||||
"unicode/文件/名", "emoji/🙂", "pct%2e%2e/x",
|
||||
// Long-ish and punctuation-heavy.
|
||||
"a-b_c+d=e,f:g;h@i", "x!y'z(1)2*3",
|
||||
// Backslashes: ordinary filename characters on Unix.
|
||||
"\\", "\\\\", "/\\", "a\\b", "\\a", "a\\",
|
||||
}
|
||||
for _, name := range names {
|
||||
if !IsValidObjectName(name) {
|
||||
continue // S3 refuses it first; the guard is free to as well.
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if err := guardPaths(name); err != nil {
|
||||
t.Errorf("guardPaths(%q) = %v, but S3 accepts this object name", name, err)
|
||||
}
|
||||
if err := guardObjectPaths(name); err != nil {
|
||||
if guardMayRefuse(name) {
|
||||
t.Logf("allowed on this platform (%q): separator-only, addresses the volume root", name)
|
||||
return
|
||||
}
|
||||
t.Errorf("guardObjectPaths(%q) = %v, but S3 accepts this object name. "+
|
||||
"A guard that refuses a legal key fails writes on every drive at once.", name, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// FuzzGuardAcceptsLegalObjectNames searches for more of the above. The property
|
||||
// is one-directional on purpose: we assert the guards never refuse something S3
|
||||
// allows, and say nothing about names S3 already refuses.
|
||||
func FuzzGuardAcceptsLegalObjectNames(f *testing.F) {
|
||||
for _, seed := range []string{" ", " ", "\t", "a b", "..foo", "a/..b", "/", "", "\\", "\\\\\\", "a/../b"} {
|
||||
f.Add(seed)
|
||||
}
|
||||
f.Fuzz(func(t *testing.T, name string) {
|
||||
if !IsValidObjectName(name) {
|
||||
return
|
||||
}
|
||||
if guardMayRefuse(name) {
|
||||
return
|
||||
}
|
||||
if err := guardPaths(name); err != nil {
|
||||
t.Fatalf("guardPaths(%q) = %v, but IsValidObjectName accepts it", name, err)
|
||||
}
|
||||
if err := guardObjectPaths(name); err != nil {
|
||||
t.Fatalf("guardObjectPaths(%q) = %v, but IsValidObjectName accepts it", name, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// FuzzGetVolDirAcceptsLegalBucketNames is the volume-axis twin of the object
|
||||
// invariant above. G-A adds hasBadPathComponent to getVolDir, so any bucket
|
||||
// name S3 accepts must survive it, or MakeBucket/HeadBucket/DeleteBucket start
|
||||
// failing for legitimate buckets across the whole cluster.
|
||||
func FuzzGetVolDirAcceptsLegalBucketNames(f *testing.F) {
|
||||
for _, seed := range []string{"bucket", "my-bucket", "a.b.c", "1234", "x--y", "..", "a..b"} {
|
||||
f.Add(seed)
|
||||
}
|
||||
f.Fuzz(func(t *testing.T, name string) {
|
||||
if s3utils.CheckValidBucketName(name) != nil {
|
||||
return // S3 refuses it first; getVolDir is free to as well.
|
||||
}
|
||||
if hasBadPathComponent(name) {
|
||||
t.Fatalf("getVolDir would reject %q, but s3utils.CheckValidBucketName accepts it", name)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestGetVolDirAcceptsReservedVolumes covers the volume names the server uses
|
||||
// internally, which are not buckets and so are not covered by the fuzz above.
|
||||
func TestGetVolDirAcceptsReservedVolumes(t *testing.T) {
|
||||
for _, vol := range []string{
|
||||
minioMetaBucket, minioMetaTmpBucket, minioMetaTmpDeletedBucket,
|
||||
minioMetaMultipartBucket, minioReservedBucket,
|
||||
pathJoin(minioMetaBucket, bucketMetaPrefix),
|
||||
pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, "some-bucket"),
|
||||
} {
|
||||
if hasBadPathComponent(vol) {
|
||||
t.Errorf("getVolDir would reject the reserved volume %q", vol)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGuardPathsAcceptsReservedNames(t *testing.T) {
|
||||
// Reserved volumes and directories contain dot-prefixed segments. Only an
|
||||
// exact "." or ".." segment is a traversal.
|
||||
for _, p := range []string{
|
||||
"", minioMetaBucket, minioMetaTmpBucket, minioMetaTmpDeletedBucket,
|
||||
minioMetaMultipartBucket, ".deleted", "obj/part.1.meta",
|
||||
"a.b/c..d", "__XLDIR__", "bucket/.metacache/x.s2",
|
||||
} {
|
||||
if err := guardPaths(p); err != nil {
|
||||
t.Errorf("guardPaths(%q) = %v, want nil", p, err)
|
||||
}
|
||||
}
|
||||
for _, p := range traversalPaths {
|
||||
if err := guardPaths(p); err == nil {
|
||||
t.Errorf("guardPaths(%q) = nil, want %v", p, errFileAccessDenied)
|
||||
}
|
||||
}
|
||||
}
|
||||
+87
-206
@@ -19,8 +19,8 @@ package cmd
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -28,13 +28,11 @@ import (
|
||||
"net/http"
|
||||
"os/user"
|
||||
"path"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/bpool"
|
||||
"github.com/minio/minio/internal/grid"
|
||||
"github.com/tinylib/msgp/msgp"
|
||||
|
||||
@@ -51,6 +49,22 @@ import (
|
||||
|
||||
var errDiskStale = errors.New("drive stale")
|
||||
|
||||
// maxAppendFilePrealloc bounds how much AppendFileHandler reserves up front
|
||||
// from the caller-declared Content-Length. It only affects pre-reservation:
|
||||
// bodies larger than this are still read in full, they just grow into place.
|
||||
//
|
||||
// Kept small deliberately. The reservation happens before a single body byte
|
||||
// arrives, so whatever it is, an attacker gets it for free on every concurrent
|
||||
// request - a large bound simply moves the exhaustion threshold rather than
|
||||
// removing it. 1 MiB matches the common erasure block size, so the ordinary
|
||||
// append still completes in one allocation.
|
||||
const maxAppendFilePrealloc = 1 << 20
|
||||
|
||||
// maxReadFileLength caps ReadFileHandler's buffer. ReadFile serves the legacy
|
||||
// whole-file bitrot reader, whose length is ShardFileOffset over a single part,
|
||||
// and an S3 part is at most 5 GiB -- so no legitimate call can ask for more.
|
||||
const maxReadFileLength = 5 << 30
|
||||
|
||||
// To abstract a disk over network.
|
||||
type storageRESTServer struct {
|
||||
endpoint Endpoint
|
||||
@@ -76,6 +90,10 @@ var (
|
||||
storageListDirRPC = grid.NewStream[*grid.MSS, grid.NoPayload, *ListDirResult](grid.HandlerListDir, grid.NewMSS, nil, func() *ListDirResult { return &ListDirResult{} }).WithOutCapacity(1)
|
||||
)
|
||||
|
||||
// getStorageViaEndpoint returns the drive UNGUARDED. It is for local callers
|
||||
// only, whose arguments the server itself constructed. Anything serving a
|
||||
// remote peer must go through storageRESTServer.getStorage(), which wraps this
|
||||
// in guardedStorage to reject traversal in wire-supplied paths.
|
||||
func getStorageViaEndpoint(endpoint Endpoint) StorageAPI {
|
||||
globalLocalDrivesMu.RLock()
|
||||
defer globalLocalDrivesMu.RUnlock()
|
||||
@@ -86,7 +104,18 @@ func getStorageViaEndpoint(endpoint Endpoint) StorageAPI {
|
||||
}
|
||||
|
||||
func (s *storageRESTServer) getStorage() StorageAPI {
|
||||
return getStorageViaEndpoint(s.endpoint)
|
||||
st := getStorageViaEndpoint(s.endpoint)
|
||||
if st == nil {
|
||||
// Must stay an untyped nil. IsAuthValid and checkID compare the result
|
||||
// against nil before authenticating, and a nil interface wrapped in a
|
||||
// value struct does not compare equal to nil - that would turn an
|
||||
// unauthenticated request against a drive that has not come up yet
|
||||
// into a nil dereference.
|
||||
return nil
|
||||
}
|
||||
// Reject traversal in paths carried by request bodies and grid RPC frames,
|
||||
// neither of which the global HTTP middleware can see. See guardedStorage.
|
||||
return guardedStorage{st}
|
||||
}
|
||||
|
||||
func (s *storageRESTServer) writeErrorResponse(w http.ResponseWriter, err error) {
|
||||
@@ -320,12 +349,32 @@ func (s *storageRESTServer) AppendFileHandler(w http.ResponseWriter, r *http.Req
|
||||
volume := r.Form.Get(storageRESTVolume)
|
||||
filePath := r.Form.Get(storageRESTFilePath)
|
||||
|
||||
buf := make([]byte, r.ContentLength)
|
||||
_, err := io.ReadFull(r.Body, buf)
|
||||
if r.ContentLength < 0 {
|
||||
s.writeErrorResponse(w, errInvalidArgument)
|
||||
return
|
||||
}
|
||||
|
||||
// Reserve from the declared Content-Length only up to a bound, then let the
|
||||
// buffer grow with the bytes actually delivered. Content-Length is a header
|
||||
// the caller writes, and setRequestLimitMiddleware only wraps the body in a
|
||||
// MaxBytesReader sized at requestMaxBodySize (5 TiB) - it never checks the
|
||||
// declaration. Sizing the buffer from it therefore lets a request that
|
||||
// sends no body at all reserve arbitrary memory and take the node down.
|
||||
//
|
||||
// The cap governs how much is pre-reserved, never which requests are
|
||||
// accepted, so a body larger than it is still read in full.
|
||||
var body bytes.Buffer
|
||||
body.Grow(int(min(r.ContentLength, maxAppendFilePrealloc)))
|
||||
n, err := body.ReadFrom(io.LimitReader(r.Body, r.ContentLength))
|
||||
if err != nil {
|
||||
s.writeErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
if n != r.ContentLength {
|
||||
s.writeErrorResponse(w, io.ErrUnexpectedEOF)
|
||||
return
|
||||
}
|
||||
buf := body.Bytes()
|
||||
err = s.getStorage().AppendFile(r.Context(), volume, filePath, buf)
|
||||
if err != nil {
|
||||
s.writeErrorResponse(w, err)
|
||||
@@ -546,11 +595,14 @@ func (s *storageRESTServer) ReadPartsHandler(w http.ResponseWriter, r *http.Requ
|
||||
|
||||
done := keepHTTPResponseAlive(w)
|
||||
infos, err := s.getStorage().ReadParts(r.Context(), volume, preq.Paths...)
|
||||
done(nil)
|
||||
if err != nil {
|
||||
s.writeErrorResponse(w, err)
|
||||
// The keep-alive stream owns the response body from here on, so the
|
||||
// error has to travel through done(); writing a header afterwards is
|
||||
// too late and leaves the client decoding the error text as msgpack.
|
||||
done(err)
|
||||
return
|
||||
}
|
||||
done(nil)
|
||||
|
||||
presp := &ReadPartsResp{Infos: infos}
|
||||
storageLogIf(r.Context(), msgp.Encode(w, presp))
|
||||
@@ -588,6 +640,17 @@ func (s *storageRESTServer) ReadFileHandler(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
verifier = NewBitrotVerifier(BitrotAlgorithmFromString(r.Form.Get(storageRESTBitrotAlgo)), hash)
|
||||
}
|
||||
// The declared length sizes the buffer before anything is read and arrives
|
||||
// in a query argument, so an unbounded value lets a small request reserve
|
||||
// arbitrary memory. A legitimate read cannot exceed one erasure shard, and
|
||||
// a shard never exceeds the S3 part it encodes, so anything above that
|
||||
// ceiling is provably not a real read. This bounds the reservation at what
|
||||
// normal operation already reaches; it does not make GiB-scale reads free.
|
||||
if int64(length) > maxReadFileLength {
|
||||
s.writeErrorResponse(w, errInvalidArgument)
|
||||
return
|
||||
}
|
||||
|
||||
buf := make([]byte, length)
|
||||
defer metaDataPoolPut(buf) // Reuse if we can.
|
||||
_, err = s.getStorage().ReadFile(r.Context(), volume, filePath, int64(offset), buf, verifier)
|
||||
@@ -672,15 +735,27 @@ func (s *storageRESTServer) DeleteVersionsHandler(w http.ResponseWriter, r *http
|
||||
return
|
||||
}
|
||||
|
||||
versions := make([]FileInfoVersions, totalVersions)
|
||||
if totalVersions < 0 {
|
||||
s.writeErrorResponse(w, errInvalidArgument)
|
||||
return
|
||||
}
|
||||
|
||||
// Grow as the body decodes rather than trusting the declared count. The
|
||||
// count arrives in a query argument, so pre-allocating from it lets a
|
||||
// ~10-byte request reserve gigabytes (FileInfoVersions is 104 bytes, so
|
||||
// total-versions=100000000 asks for ~9.7 GiB) and take the node down by
|
||||
// memory exhaustion. Growing means the allocation stays proportional to
|
||||
// the bytes the caller actually sent.
|
||||
versions := make([]FileInfoVersions, 0, min(totalVersions, 1024))
|
||||
decoder := msgpNewReader(r.Body)
|
||||
defer readMsgpReaderPoolPut(decoder)
|
||||
for i := range totalVersions {
|
||||
dst := &versions[i]
|
||||
for range totalVersions {
|
||||
var dst FileInfoVersions
|
||||
if err := dst.DecodeMsg(decoder); err != nil {
|
||||
s.writeErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
versions = append(versions, dst)
|
||||
}
|
||||
|
||||
done := keepHTTPResponseAlive(w)
|
||||
@@ -688,7 +763,7 @@ func (s *storageRESTServer) DeleteVersionsHandler(w http.ResponseWriter, r *http
|
||||
errs := s.getStorage().DeleteVersions(r.Context(), volume, versions, opts)
|
||||
done(nil)
|
||||
|
||||
dErrsResp := &DeleteVersionsErrsResp{Errs: make([]string, totalVersions)}
|
||||
dErrsResp := &DeleteVersionsErrsResp{Errs: make([]string, len(versions))}
|
||||
for idx := range versions {
|
||||
if errs[idx] != nil {
|
||||
dErrsResp.Errs[idx] = errs[idx].Error()
|
||||
@@ -962,157 +1037,6 @@ func waitForHTTPResponse(respBody io.Reader) (io.Reader, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// httpStreamResponse allows streaming a response, but still send an error.
|
||||
type httpStreamResponse struct {
|
||||
done chan error
|
||||
block chan []byte
|
||||
err error
|
||||
}
|
||||
|
||||
// Write part of the streaming response.
|
||||
// Note that upstream errors are currently not forwarded, but may be in the future.
|
||||
func (h *httpStreamResponse) Write(b []byte) (int, error) {
|
||||
if len(b) == 0 || h.err != nil {
|
||||
// Ignore 0 length blocks
|
||||
return 0, h.err
|
||||
}
|
||||
tmp := make([]byte, len(b))
|
||||
copy(tmp, b)
|
||||
h.block <- tmp
|
||||
return len(b), h.err
|
||||
}
|
||||
|
||||
// CloseWithError will close the stream and return the specified error.
|
||||
// This can be done several times, but only the first error will be sent.
|
||||
// After calling this the stream should not be written to.
|
||||
func (h *httpStreamResponse) CloseWithError(err error) {
|
||||
if h.done == nil {
|
||||
return
|
||||
}
|
||||
h.done <- err
|
||||
h.err = err
|
||||
// Indicates that the response is done.
|
||||
<-h.done
|
||||
h.done = nil
|
||||
}
|
||||
|
||||
// streamHTTPResponse can be used to avoid timeouts with long storage
|
||||
// operations, such as bitrot verification or data usage scanning.
|
||||
// Every 10 seconds a space character is sent.
|
||||
// The returned function should always be called to release resources.
|
||||
// An optional error can be sent which will be picked as text only error,
|
||||
// without its original type by the receiver.
|
||||
// waitForHTTPStream should be used to the receiving side.
|
||||
func streamHTTPResponse(w http.ResponseWriter) *httpStreamResponse {
|
||||
doneCh := make(chan error)
|
||||
blockCh := make(chan []byte)
|
||||
h := httpStreamResponse{done: doneCh, block: blockCh}
|
||||
go func() {
|
||||
canWrite := true
|
||||
write := func(b []byte) {
|
||||
if canWrite {
|
||||
n, err := w.Write(b)
|
||||
if err != nil || n != len(b) {
|
||||
canWrite = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(time.Second * 10)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
// Response not ready, write a filler byte.
|
||||
write([]byte{32})
|
||||
if canWrite {
|
||||
xhttp.Flush(w)
|
||||
}
|
||||
case err := <-doneCh:
|
||||
if err != nil {
|
||||
write([]byte{1})
|
||||
write([]byte(err.Error()))
|
||||
} else {
|
||||
write([]byte{0})
|
||||
}
|
||||
xioutil.SafeClose(doneCh)
|
||||
return
|
||||
case block := <-blockCh:
|
||||
var tmp [5]byte
|
||||
tmp[0] = 2
|
||||
binary.LittleEndian.PutUint32(tmp[1:], uint32(len(block)))
|
||||
write(tmp[:])
|
||||
write(block)
|
||||
if canWrite {
|
||||
xhttp.Flush(w)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return &h
|
||||
}
|
||||
|
||||
var poolBuf8k = bpool.Pool[*[]byte]{
|
||||
New: func() *[]byte {
|
||||
b := make([]byte, 8192)
|
||||
return &b
|
||||
},
|
||||
}
|
||||
|
||||
// waitForHTTPStream will wait for responses where
|
||||
// streamHTTPResponse has been used.
|
||||
// The returned reader contains the payload and must be closed if no error is returned.
|
||||
func waitForHTTPStream(respBody io.ReadCloser, w io.Writer) error {
|
||||
var tmp [1]byte
|
||||
// 8K copy buffer, reused for less allocs...
|
||||
bufp := poolBuf8k.Get()
|
||||
buf := *bufp
|
||||
defer poolBuf8k.Put(bufp)
|
||||
|
||||
for {
|
||||
_, err := io.ReadFull(respBody, tmp[:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Check if we have a response ready or a filler byte.
|
||||
switch tmp[0] {
|
||||
case 0:
|
||||
// 0 is unbuffered, copy the rest.
|
||||
_, err := io.CopyBuffer(w, respBody, buf)
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
case 1:
|
||||
errorText, err := io.ReadAll(respBody)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return errors.New(string(errorText))
|
||||
case 2:
|
||||
// Block of data
|
||||
var tmp [4]byte
|
||||
_, err := io.ReadFull(respBody, tmp[:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
length := binary.LittleEndian.Uint32(tmp[:])
|
||||
n, err := io.CopyBuffer(w, io.LimitReader(respBody, int64(length)), buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != int64(length) {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
continue
|
||||
case 32:
|
||||
continue
|
||||
default:
|
||||
return fmt.Errorf("unexpected filler byte: %d", tmp[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// VerifyFileHandler - Verify all part of file for bitrot errors.
|
||||
func (s *storageRESTServer) VerifyFileHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.IsValid(w, r) {
|
||||
@@ -1283,48 +1207,6 @@ func (s *storageRESTServer) DeleteBulkHandler(w http.ResponseWriter, r *http.Req
|
||||
keepHTTPResponseAlive(w)(s.getStorage().DeleteBulk(r.Context(), volume, req.Paths...))
|
||||
}
|
||||
|
||||
// ReadMultiple returns multiple files
|
||||
func (s *storageRESTServer) ReadMultiple(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.IsValid(w, r) {
|
||||
return
|
||||
}
|
||||
rw := streamHTTPResponse(w)
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
debug.PrintStack()
|
||||
rw.CloseWithError(fmt.Errorf("panic: %v", r))
|
||||
}
|
||||
}()
|
||||
|
||||
var req ReadMultipleReq
|
||||
mr := msgpNewReader(r.Body)
|
||||
defer readMsgpReaderPoolPut(mr)
|
||||
err := req.DecodeMsg(mr)
|
||||
if err != nil {
|
||||
rw.CloseWithError(err)
|
||||
return
|
||||
}
|
||||
|
||||
mw := msgp.NewWriter(rw)
|
||||
responses := make(chan ReadMultipleResp, len(req.Files))
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for resp := range responses {
|
||||
err := resp.EncodeMsg(mw)
|
||||
if err != nil {
|
||||
rw.CloseWithError(err)
|
||||
return
|
||||
}
|
||||
mw.Flush()
|
||||
}
|
||||
}()
|
||||
err = s.getStorage().ReadMultiple(r.Context(), req, responses)
|
||||
wg.Wait()
|
||||
rw.CloseWithError(err)
|
||||
}
|
||||
|
||||
// globalLocalSetDrives is used for local drive as well as remote REST
|
||||
// API caller for other nodes to talk to this node.
|
||||
//
|
||||
@@ -1363,7 +1245,6 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
|
||||
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteVersions).HandlerFunc(h(server.DeleteVersionsHandler))
|
||||
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodVerifyFile).HandlerFunc(h(server.VerifyFileHandler))
|
||||
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatInfoFile).HandlerFunc(h(server.StatInfoFile))
|
||||
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadMultiple).HandlerFunc(h(server.ReadMultiple))
|
||||
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCleanAbandoned).HandlerFunc(h(server.CleanAbandonedDataHandler))
|
||||
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteBulk).HandlerFunc(h(server.DeleteBulkHandler))
|
||||
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadParts).HandlerFunc(h(server.ReadPartsHandler))
|
||||
|
||||
@@ -0,0 +1,818 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
// Paths that must never be accepted from an internode payload. Each is a
|
||||
// distinct evasion of the segment scanner: forward slash, backslash (Windows
|
||||
// resolves these, path.Clean does not), whitespace padding, and single-dot.
|
||||
var traversalPaths = []string{
|
||||
"../evil",
|
||||
"../../evil",
|
||||
"a/../../evil",
|
||||
"..\\evil",
|
||||
"a\\..\\..\\evil",
|
||||
" .. /evil",
|
||||
"../",
|
||||
"./../evil",
|
||||
}
|
||||
|
||||
// Paths that alias the volume root itself: pathJoin collapses each of these
|
||||
// back to the volume directory, so renaming or deleting one relocates or
|
||||
// destroys the whole volume, with no ".." required.
|
||||
//
|
||||
// Only the platform-independent forms live here. Whitespace does NOT belong:
|
||||
// path.Clean leaves spaces alone, so " " names a real directory inside the
|
||||
// volume and is a legal S3 object key. Backslash forms do not belong either:
|
||||
// they collapse on Windows but name an ordinary file on Unix. Both classes are
|
||||
// covered by TestIsVolumeRootAliasIsPlatformCorrect and, from the other
|
||||
// direction, by TestGuardAcceptsEveryLegalObjectName.
|
||||
var volumeRootAliases = []string{"", "/", "//"}
|
||||
|
||||
// sentinels plants files we can prove were neither read nor removed.
|
||||
type sentinels struct {
|
||||
drive string // drive root
|
||||
outside string // file outside the drive root entirely
|
||||
sibling string // file in a sibling volume
|
||||
legit string // a legitimate object inside "foo"
|
||||
partDir string // a readable part dir in the sibling volume
|
||||
outsideRe string // path a rename/write would land on, outside the drive
|
||||
}
|
||||
|
||||
func plantSentinels(t *testing.T, drive string) *sentinels {
|
||||
t.Helper()
|
||||
s := &sentinels{
|
||||
drive: drive,
|
||||
outside: filepath.Join(filepath.Dir(drive), "sentinel-outside.txt"),
|
||||
sibling: filepath.Join(drive, "bar", "sentinel-sibling.txt"),
|
||||
legit: filepath.Join(drive, "foo", "legit.txt"),
|
||||
partDir: filepath.Join(drive, "bar", "obj"),
|
||||
outsideRe: filepath.Join(filepath.Dir(drive), "sentinel-landing.txt"),
|
||||
}
|
||||
mustWrite(t, s.outside, "OUTSIDE-SECRET")
|
||||
t.Cleanup(func() { os.Remove(s.outside); os.Remove(s.outsideRe) })
|
||||
mustWrite(t, s.sibling, "SIBLING-SECRET")
|
||||
mustWrite(t, s.legit, "LEGIT")
|
||||
|
||||
if err := os.MkdirAll(s.partDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mustWrite(t, filepath.Join(s.partDir, "part.1"), "data")
|
||||
blob, err := (&ObjectPartInfo{Number: 1, Size: 4, ETag: "SIBLING-ETAG"}).MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(s.partDir, "part.1.meta"), blob, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func mustWrite(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// assertIntact fails if any sentinel was removed, modified, or if a file
|
||||
// appeared where a traversal would have landed.
|
||||
func (s *sentinels) assertIntact(t *testing.T, op string) {
|
||||
t.Helper()
|
||||
for _, f := range []struct{ path, want string }{
|
||||
{s.outside, "OUTSIDE-SECRET"},
|
||||
{s.sibling, "SIBLING-SECRET"},
|
||||
{s.legit, "LEGIT"},
|
||||
} {
|
||||
got, err := os.ReadFile(f.path)
|
||||
if err != nil {
|
||||
t.Errorf("%s: sentinel %s was destroyed: %v", op, f.path, err)
|
||||
continue
|
||||
}
|
||||
if string(got) != f.want {
|
||||
t.Errorf("%s: sentinel %s was modified: got %q want %q", op, f.path, got, f.want)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(s.outsideRe); err == nil {
|
||||
t.Errorf("%s: a file was created outside the drive root at %s", op, s.outsideRe)
|
||||
}
|
||||
for _, vol := range []string{"foo", "bar"} {
|
||||
st, err := os.Stat(filepath.Join(s.drive, vol))
|
||||
if err != nil || !st.IsDir() {
|
||||
t.Errorf("%s: volume %q no longer exists as a directory (err=%v)", op, vol, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// assertDenied requires the specific sentinel error, not merely "some error".
|
||||
// An operation on a nonexistent path fails anyway, so a bare err != nil check
|
||||
// passes even when the guard is absent.
|
||||
func assertDenied(t *testing.T, op string, err error) {
|
||||
t.Helper()
|
||||
if err == nil {
|
||||
t.Errorf("%s: expected %v, got nil", op, errFileAccessDenied)
|
||||
return
|
||||
}
|
||||
if !errors.Is(err, errFileAccessDenied) {
|
||||
t.Errorf("%s: expected %v, got %v (%T)", op, errFileAccessDenied, err, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStorageRESTTraversalRejected drives the real REST/grid client against a
|
||||
// real xlStorage and proves that no internode payload can read, write, move or
|
||||
// delete anything outside its own volume.
|
||||
//
|
||||
// NOTE: the grid.SetupTestGrid harness mounts a bare mux router with none of
|
||||
// globalMiddlewares, so a green run says nothing about the production HTTP
|
||||
// query-argument path (which setRequestValidityMiddleware already covers). It
|
||||
// measures the storage-layer guards and nothing else, which is the point.
|
||||
func TestStorageRESTTraversalRejected(t *testing.T) {
|
||||
restClient := newStorageRESTHTTPServerClient(t)
|
||||
drive := globalLocalSetDrives[0][0][0].Endpoint().Path
|
||||
s := plantSentinels(t, drive)
|
||||
ctx := t.Context()
|
||||
|
||||
badFI := func(dataDir string) FileInfo {
|
||||
return FileInfo{Volume: "foo", Name: "obj", DataDir: dataDir, ModTime: UTCNow()}
|
||||
}
|
||||
|
||||
// Every operation that accepts a path from the wire, driven with each
|
||||
// traversal form. The op must be denied AND must not have touched disk.
|
||||
for _, p := range traversalPaths {
|
||||
ops := []struct {
|
||||
name string
|
||||
run func() error
|
||||
}{
|
||||
{"ReadAll", func() error { _, err := restClient.ReadAll(ctx, "foo", p); return err }},
|
||||
{"WriteAll", func() error { return restClient.WriteAll(ctx, "foo", p, []byte("PWNED")) }},
|
||||
{"ReadXL", func() error { _, err := restClient.ReadXL(ctx, "foo", p, true); return err }},
|
||||
{"Delete", func() error {
|
||||
return restClient.Delete(ctx, "foo", p, DeleteOptions{Recursive: true, Immediate: true})
|
||||
}},
|
||||
{"DeleteBulk", func() error { return restClient.DeleteBulk(ctx, "foo", p) }},
|
||||
{"ReadParts", func() error { _, err := restClient.ReadParts(ctx, "foo", p); return err }},
|
||||
{"StatInfoFile", func() error { _, err := restClient.StatInfoFile(ctx, "foo", p, false); return err }},
|
||||
{"CleanAbandonedData", func() error { return restClient.CleanAbandonedData(ctx, "foo", p) }},
|
||||
{"ListDir", func() error { _, err := restClient.ListDir(ctx, "", "foo", p, -1); return err }},
|
||||
{"RenameFile-src", func() error { return restClient.RenameFile(ctx, "foo", p, "foo", "dst.txt") }},
|
||||
{"RenameFile-dst", func() error { return restClient.RenameFile(ctx, "foo", "legit.txt", "foo", p) }},
|
||||
{"RenamePart-src", func() error {
|
||||
return restClient.RenamePart(ctx, "foo", p, "foo", "dst.txt", nil, "")
|
||||
}},
|
||||
{"RenamePart-dst", func() error {
|
||||
return restClient.RenamePart(ctx, "foo", "legit.txt", "foo", p, nil, "")
|
||||
}},
|
||||
{"RenamePart-skipParent", func() error {
|
||||
return restClient.RenamePart(ctx, "foo", "legit.txt", "foo", "dst.txt", nil, p)
|
||||
}},
|
||||
{"CheckParts", func() error { _, err := restClient.CheckParts(ctx, "foo", p, badFI("")); return err }},
|
||||
{"CheckParts-DataDir", func() error {
|
||||
_, err := restClient.CheckParts(ctx, "foo", "obj", badFI(p))
|
||||
return err
|
||||
}},
|
||||
{"VerifyFile", func() error { _, err := restClient.VerifyFile(ctx, "foo", p, badFI("")); return err }},
|
||||
{"VerifyFile-DataDir", func() error {
|
||||
_, err := restClient.VerifyFile(ctx, "foo", "obj", badFI(p))
|
||||
return err
|
||||
}},
|
||||
{"WriteMetadata", func() error { return restClient.WriteMetadata(ctx, "", "foo", p, badFI("")) }},
|
||||
{"UpdateMetadata", func() error {
|
||||
return restClient.UpdateMetadata(ctx, "foo", p, badFI(""), UpdateMetadataOpts{})
|
||||
}},
|
||||
{"DeleteVersion", func() error {
|
||||
return restClient.DeleteVersion(ctx, "foo", p, badFI(""), false, DeleteOptions{})
|
||||
}},
|
||||
|
||||
// Nested path-bearing fields, poisoned one at a time. A method that
|
||||
// validates its `path` argument but forgets one of these still
|
||||
// passes every check above, so each needs its own case.
|
||||
{"WriteMetadata-DataDir", func() error {
|
||||
return restClient.WriteMetadata(ctx, "", "foo", "obj", badFI(p))
|
||||
}},
|
||||
{"UpdateMetadata-DataDir", func() error {
|
||||
return restClient.UpdateMetadata(ctx, "foo", "obj", badFI(p), UpdateMetadataOpts{})
|
||||
}},
|
||||
{"DeleteVersion-DataDir", func() error {
|
||||
return restClient.DeleteVersion(ctx, "foo", "obj", badFI(p), false, DeleteOptions{})
|
||||
}},
|
||||
// DeleteOptions.OldDataDir is guarded too, but cannot be exercised
|
||||
// from here: DeleteVersionHandler hardcodes `opts := DeleteOptions{}`
|
||||
// and never reads the wire value, so the field is unreachable today.
|
||||
// That is an accidental mitigation, not a control - see
|
||||
// TestGuardChecksUnreachableFields for the unit-level assertion that
|
||||
// the guard is ready if anyone ever plumbs it through.
|
||||
{"RenameData-srcPath", func() error {
|
||||
_, err := restClient.RenameData(ctx, "foo", p, badFI(""), "bar", "dst", RenameOptions{})
|
||||
return err
|
||||
}},
|
||||
{"RenameData-dstPath", func() error {
|
||||
_, err := restClient.RenameData(ctx, "foo", "src", badFI(""), "bar", p, RenameOptions{})
|
||||
return err
|
||||
}},
|
||||
{"RenameData-DataDir", func() error {
|
||||
_, err := restClient.RenameData(ctx, "foo", "src", badFI(p), "bar", "dst", RenameOptions{})
|
||||
return err
|
||||
}},
|
||||
}
|
||||
for _, op := range ops {
|
||||
t.Run(op.name+"/"+p, func(t *testing.T) {
|
||||
assertDenied(t, op.name, op.run())
|
||||
s.assertIntact(t, op.name)
|
||||
})
|
||||
}
|
||||
|
||||
// The nested DataDir of each version is a separate field from the
|
||||
// per-object Name; poison them independently so neither can regress
|
||||
// behind the other.
|
||||
t.Run("DeleteVersions-VersionDataDir/"+p, func(t *testing.T) {
|
||||
errs := restClient.DeleteVersions(ctx, "foo",
|
||||
[]FileInfoVersions{{Name: "obj", Versions: []FileInfo{{Name: "obj", DataDir: p}}}},
|
||||
DeleteOptions{})
|
||||
if len(errs) != 1 {
|
||||
t.Fatalf("expected 1 error, got %d", len(errs))
|
||||
}
|
||||
assertDenied(t, "DeleteVersions-VersionDataDir", errs[0])
|
||||
s.assertIntact(t, "DeleteVersions-VersionDataDir")
|
||||
})
|
||||
|
||||
t.Run("DeleteVersions/"+p, func(t *testing.T) {
|
||||
errs := restClient.DeleteVersions(ctx, "foo",
|
||||
[]FileInfoVersions{{Name: p, Versions: []FileInfo{{Name: p}}}}, DeleteOptions{})
|
||||
if len(errs) != 1 {
|
||||
t.Fatalf("expected 1 error, got %d", len(errs))
|
||||
}
|
||||
assertDenied(t, "DeleteVersions", errs[0])
|
||||
s.assertIntact(t, "DeleteVersions")
|
||||
})
|
||||
|
||||
// WalkDir and NSScanner take their paths inside an options/cache struct
|
||||
// rather than as arguments, which is exactly where a per-handler check
|
||||
// tends to miss them.
|
||||
t.Run("WalkDir/"+p, func(t *testing.T) {
|
||||
for _, opts := range []WalkDirOptions{
|
||||
{Bucket: p, BaseDir: "obj"},
|
||||
{Bucket: "foo", BaseDir: p},
|
||||
} {
|
||||
if err := restClient.WalkDir(ctx, opts, io.Discard); err == nil {
|
||||
t.Errorf("WalkDir(%+v) returned nil", opts)
|
||||
}
|
||||
}
|
||||
s.assertIntact(t, "WalkDir")
|
||||
})
|
||||
|
||||
t.Run("NSScanner/"+p, func(t *testing.T) {
|
||||
cache := dataUsageCache{Info: dataUsageCacheInfo{Name: p}}
|
||||
updates := make(chan dataUsageEntry, 1)
|
||||
if _, err := restClient.NSScanner(ctx, cache, updates, madmin.HealNormalScan, nil); err == nil {
|
||||
t.Errorf("NSScanner(cache.Info.Name=%q) returned nil", p)
|
||||
}
|
||||
s.assertIntact(t, "NSScanner")
|
||||
})
|
||||
|
||||
t.Run("ReadAll-volume/"+p, func(t *testing.T) {
|
||||
// Traversal smuggled through the volume argument rather than the path.
|
||||
buf, err := restClient.ReadAll(ctx, p, "sentinel-outside.txt")
|
||||
if err == nil {
|
||||
t.Errorf("ReadAll with volume %q: expected error, got nil (read %d bytes)", p, len(buf))
|
||||
}
|
||||
if string(buf) == "OUTSIDE-SECRET" {
|
||||
t.Errorf("ReadAll with volume %q leaked a file outside the drive root", p)
|
||||
}
|
||||
s.assertIntact(t, "ReadAll-volume")
|
||||
})
|
||||
}
|
||||
|
||||
// Volume-root aliases: no ".." involved, but a rename or bulk delete of the
|
||||
// volume root relocates or destroys the entire volume.
|
||||
for _, p := range volumeRootAliases {
|
||||
t.Run("DeleteBulk-root/"+p, func(t *testing.T) {
|
||||
assertDenied(t, "DeleteBulk", restClient.DeleteBulk(ctx, "foo", p))
|
||||
s.assertIntact(t, "DeleteBulk-root")
|
||||
})
|
||||
t.Run("RenameFile-root/"+p, func(t *testing.T) {
|
||||
assertDenied(t, "RenameFile", restClient.RenameFile(ctx, "foo", p, "bar", "captured"))
|
||||
s.assertIntact(t, "RenameFile-root")
|
||||
})
|
||||
}
|
||||
|
||||
// A batch mixing a legitimate target with a malicious one must delete
|
||||
// neither -- validate the whole set before acting on any of it.
|
||||
t.Run("MixedBatch", func(t *testing.T) {
|
||||
assertDenied(t, "DeleteBulk-mixed",
|
||||
restClient.DeleteBulk(ctx, "foo", "legit.txt", "../bar/sentinel-sibling.txt"))
|
||||
s.assertIntact(t, "DeleteBulk-mixed")
|
||||
})
|
||||
|
||||
t.Run("ReadParts-mixed", func(t *testing.T) {
|
||||
_, err := restClient.ReadParts(ctx, "foo", "legit.txt", "../bar/obj/part.1.meta")
|
||||
assertDenied(t, "ReadParts-mixed", err)
|
||||
s.assertIntact(t, "ReadParts-mixed")
|
||||
})
|
||||
}
|
||||
|
||||
// TestStorageRESTLegitimateOpsStillWork is the positive control: the guards must
|
||||
// not reject anything the cluster does in normal operation.
|
||||
func TestStorageRESTLegitimateOpsStillWork(t *testing.T) {
|
||||
restClient := newStorageRESTHTTPServerClient(t)
|
||||
drive := globalLocalSetDrives[0][0][0].Endpoint().Path
|
||||
ctx := t.Context()
|
||||
|
||||
mustWrite(t, filepath.Join(drive, "foo", "a.txt"), "HELLO")
|
||||
mustWrite(t, filepath.Join(drive, minioMetaBucket, "tmp", "sys.txt"), "SYS")
|
||||
|
||||
if err := restClient.WriteAll(ctx, "foo", "written.txt", []byte("DATA")); err != nil {
|
||||
t.Fatalf("WriteAll on a legitimate path: %v", err)
|
||||
}
|
||||
if b, err := restClient.ReadAll(ctx, "foo", "written.txt"); err != nil || string(b) != "DATA" {
|
||||
t.Fatalf("ReadAll on a legitimate path: %q %v", b, err)
|
||||
}
|
||||
// Reserved volumes contain dot-prefixed segments (".minio.sys", ".trash")
|
||||
// which must remain acceptable -- only exact "." and ".." segments are bad.
|
||||
if b, err := restClient.ReadAll(ctx, minioMetaBucket, "tmp/sys.txt"); err != nil || string(b) != "SYS" {
|
||||
t.Fatalf("ReadAll on %s: %q %v", minioMetaBucket, b, err)
|
||||
}
|
||||
if err := restClient.RenameFile(ctx, "foo", "a.txt", "foo", "b.txt"); err != nil {
|
||||
t.Fatalf("RenameFile on legitimate paths: %v", err)
|
||||
}
|
||||
if _, err := restClient.ListDir(ctx, "", "foo", "", -1); err != nil {
|
||||
t.Fatalf("ListDir on the volume root is legitimate: %v", err)
|
||||
}
|
||||
if err := restClient.DeleteBulk(ctx, "foo", "b.txt"); err != nil {
|
||||
t.Fatalf("DeleteBulk on a legitimate path: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(drive, "foo", "b.txt")); err == nil {
|
||||
t.Fatal("DeleteBulk on a legitimate path did not delete")
|
||||
}
|
||||
|
||||
// Object keys that look like separators or path components but are not.
|
||||
// A whitespace-only key is legal in S3 and is committed through the rename
|
||||
// path on PutObject, so a guard that refuses it fails the write on every
|
||||
// remote drive at once and breaks quorum. This is a real regression that
|
||||
// shipped in an earlier draft of the guard.
|
||||
oddKeys := []string{" ", " ", "\t", "a b", " lead", "trail ", "..foo", "foo..", "a/..b"}
|
||||
if runtime.GOOS != globalWindowsOSName {
|
||||
// Backslash is an ordinary filename character off Windows.
|
||||
oddKeys = append(oddKeys, "\\", "\\\\", "a\\b", "/\\")
|
||||
}
|
||||
for _, key := range oddKeys {
|
||||
if !IsValidObjectName(key) {
|
||||
t.Fatalf("test bug: %q is not a legal object name", key)
|
||||
}
|
||||
if err := restClient.WriteAll(ctx, "foo", key, []byte("ODD")); err != nil {
|
||||
t.Errorf("WriteAll(%q) on a legal object key: %v", key, err)
|
||||
continue
|
||||
}
|
||||
if b, err := restClient.ReadAll(ctx, "foo", key); err != nil || string(b) != "ODD" {
|
||||
t.Errorf("ReadAll(%q) on a legal object key: %q %v", key, b, err)
|
||||
}
|
||||
// The rename path is what PutObject uses to commit.
|
||||
if err := restClient.RenameFile(ctx, "foo", key, "foo", key+"-renamed"); err != nil {
|
||||
t.Errorf("RenameFile(%q) on a legal object key: %v", key, err)
|
||||
continue
|
||||
}
|
||||
if err := restClient.DeleteBulk(ctx, "foo", key+"-renamed"); err != nil {
|
||||
t.Errorf("DeleteBulk(%q) on a legal object key: %v", key+"-renamed", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPeerS3VolumeTraversalRejected covers the peer-S3 bucket RPCs, which reach
|
||||
// local drives through globalLocalDrivesMap and never pass through
|
||||
// storageRESTServer.getStorage(). Only the getVolDir guard protects them.
|
||||
func TestPeerS3VolumeTraversalRejected(t *testing.T) {
|
||||
newStorageRESTHTTPServerClient(t)
|
||||
drive := globalLocalSetDrives[0][0][0].Endpoint().Path
|
||||
ctx := t.Context()
|
||||
|
||||
victim := filepath.Join(filepath.Dir(drive), "peer-victim")
|
||||
mustWrite(t, filepath.Join(victim, "nested", "important.txt"), "IMPORTANT")
|
||||
t.Cleanup(func() { os.RemoveAll(victim) })
|
||||
|
||||
for _, p := range traversalPaths {
|
||||
t.Run("DeleteBucket/"+p, func(t *testing.T) {
|
||||
// Force:true reaches moveToTrash(volumeDir, recursive, immediate).
|
||||
if err := deleteBucketLocal(ctx, p+"/peer-victim", DeleteBucketOptions{Force: true}); err == nil {
|
||||
t.Errorf("deleteBucketLocal(%q) returned nil", p)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(victim, "nested", "important.txt")); err != nil {
|
||||
t.Errorf("deleteBucketLocal(%q) destroyed a tree outside the drive root: %v", p, err)
|
||||
}
|
||||
})
|
||||
t.Run("MakeBucket/"+p, func(t *testing.T) {
|
||||
created := filepath.Join(filepath.Dir(drive), "peer-created")
|
||||
t.Cleanup(func() { os.RemoveAll(created) })
|
||||
if err := makeBucketLocal(ctx, p+"/peer-created", MakeBucketOptions{}); err == nil {
|
||||
t.Errorf("makeBucketLocal(%q) returned nil", p)
|
||||
}
|
||||
if _, err := os.Stat(created); err == nil {
|
||||
t.Errorf("makeBucketLocal(%q) created a directory outside the drive root", p)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestCheckPartsMalformedErasure covers a remote node kill that is not a
|
||||
// traversal: CheckParts hands a wire-supplied FileInfo to ShardFileSize, which
|
||||
// divided by ErasureInfo.BlockSize with no validity check. The call runs inside
|
||||
// xioutil.WithDeadline, i.e. a bare goroutine, so the resulting integer
|
||||
// divide-by-zero panic cannot be recovered by the grid or net/http handlers --
|
||||
// it terminates the process. If this test regresses it does not fail, it
|
||||
// crashes the whole run.
|
||||
func TestCheckPartsMalformedErasure(t *testing.T) {
|
||||
restClient := newStorageRESTHTTPServerClient(t)
|
||||
drive := globalLocalSetDrives[0][0][0].Endpoint().Path
|
||||
ctx := t.Context()
|
||||
|
||||
// Not panicking is the floor, not the bar. ShardFileSize returns 0 for these,
|
||||
// and checkPart's "st.Size() < expectedSize" then succeeds for *any* file
|
||||
// that exists - including a truncated shard - so a request that got this far
|
||||
// would come back reporting every part intact. The boundary must refuse it
|
||||
// outright, which is what errFileCorrupt asserts here.
|
||||
for _, fi := range []FileInfo{
|
||||
{Volume: "foo", Name: "obj", Parts: []ObjectPartInfo{{Number: 1, Size: 4}}},
|
||||
// Deleted short-circuits FileInfo.IsValid() to true, so an IsValid()
|
||||
// guard would not have caught this one.
|
||||
{Volume: "foo", Name: "obj", Deleted: true, Parts: []ObjectPartInfo{{Number: 1, Size: 4}}},
|
||||
{
|
||||
Volume: "foo", Name: "obj", Parts: []ObjectPartInfo{{Number: 1, Size: 4}},
|
||||
Erasure: ErasureInfo{DataBlocks: 4},
|
||||
}, // BlockSize still zero
|
||||
{
|
||||
Volume: "foo", Name: "obj", Parts: []ObjectPartInfo{{Number: 1, Size: 4}},
|
||||
Erasure: ErasureInfo{BlockSize: blockSizeV2},
|
||||
}, // DataBlocks still zero
|
||||
} {
|
||||
if _, err := restClient.CheckParts(ctx, "foo", "obj", fi); !errors.Is(err, errFileCorrupt) {
|
||||
t.Errorf("CheckParts(%+v): got %v, want %v", fi.Erasure, err, errFileCorrupt)
|
||||
}
|
||||
if _, err := restClient.VerifyFile(ctx, "foo", "obj", fi); !errors.Is(err, errFileCorrupt) {
|
||||
t.Errorf("VerifyFile(%+v): got %v, want %v", fi.Erasure, err, errFileCorrupt)
|
||||
}
|
||||
}
|
||||
|
||||
// A part of zero length says nothing about the erasure parameters, so it
|
||||
// must not be swept up by the rule above.
|
||||
zeroPart := FileInfo{Volume: "foo", Name: "obj", Parts: []ObjectPartInfo{{Number: 1, Size: 0}}}
|
||||
if _, err := restClient.CheckParts(ctx, "foo", "obj", zeroPart); errors.Is(err, errFileCorrupt) {
|
||||
t.Errorf("CheckParts with a zero-length part was rejected as corrupt")
|
||||
}
|
||||
|
||||
// A NEGATIVE part size is the sharper case, and the one an ">0" test misses.
|
||||
// ShardFileSize returns 0 for it whether or not the erasure parameters are
|
||||
// usable (numShards and lastShardSize both floor to zero), so checkPart's
|
||||
// "st.Size() < expectedSize" is false for any file that exists and the part
|
||||
// is reported healthy. With a real short part planted on disk, a guard that
|
||||
// only rejects Size > 0 lets malformed metadata launder a truncated shard
|
||||
// into a clean bill of health.
|
||||
partDir := filepath.Join(drive, "foo", "negobj")
|
||||
if err := os.MkdirAll(partDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mustWrite(t, filepath.Join(partDir, "part.1"), "tiny")
|
||||
|
||||
for _, e := range []ErasureInfo{
|
||||
{}, // unusable parameters
|
||||
// Fully valid parameters. This is the sharper case: the FileInfo passes
|
||||
// FileInfo.IsValid(), the very check healing uses to decide the metadata
|
||||
// is trustworthy, and the negative size still lands on a zero expected
|
||||
// shard size. Valid erasure parameters do not save you here.
|
||||
{DataBlocks: 2, ParityBlocks: 2, BlockSize: blockSizeV2, Index: 1, Distribution: []int{1, 2, 3, 4}},
|
||||
} {
|
||||
fi := FileInfo{
|
||||
Volume: "foo", Name: "negobj", Erasure: e,
|
||||
Parts: []ObjectPartInfo{{Number: 1, Size: -2}},
|
||||
}
|
||||
if e.DataBlocks > 0 && !fi.IsValid() {
|
||||
t.Fatal("test bug: the second case must satisfy FileInfo.IsValid()")
|
||||
}
|
||||
resp, err := restClient.CheckParts(ctx, "foo", "negobj", fi)
|
||||
if !errors.Is(err, errFileCorrupt) {
|
||||
t.Errorf("CheckParts with a negative part size (erasure %+v): got err=%v resp=%v, want %v",
|
||||
e, err, resp, errFileCorrupt)
|
||||
if resp != nil && len(resp.Results) > 0 && resp.Results[0] == checkPartSuccess {
|
||||
t.Errorf(" -> and it reported the part HEALTHY, which is the actual damage")
|
||||
}
|
||||
}
|
||||
if _, err := restClient.VerifyFile(ctx, "foo", "negobj", fi); !errors.Is(err, errFileCorrupt) {
|
||||
t.Errorf("VerifyFile with a negative part size (erasure %+v): got %v, want %v", e, err, errFileCorrupt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeleteVersionsDeclaredCountIsNotTrusted covers a resource-exhaustion
|
||||
// vector in the same family as the CheckParts node kill: a value taken straight
|
||||
// off the wire that sizes an allocation.
|
||||
//
|
||||
// total-versions is a query argument, so a ~10 byte request used to reserve
|
||||
// len*104 bytes before a single byte of the body was read - total-versions=1e8
|
||||
// asks for ~9.7 GiB and takes the node down by memory exhaustion. A negative
|
||||
// value reached make() and panicked outright. The handler now refuses negatives
|
||||
// and grows the slice as the body decodes, so the allocation is bounded by the
|
||||
// bytes actually sent.
|
||||
//
|
||||
// The client always sends len(versions), so this has to be driven as a raw
|
||||
// request to reach the handler at all.
|
||||
func TestDeleteVersionsDeclaredCountIsNotTrusted(t *testing.T) {
|
||||
restClient := newStorageRESTHTTPServerClient(t)
|
||||
drive := globalLocalSetDrives[0][0][0].Endpoint().Path
|
||||
ctx := t.Context()
|
||||
|
||||
canary := filepath.Join(drive, "foo", "canary.txt")
|
||||
mustWrite(t, canary, "CANARY")
|
||||
|
||||
send := func(total string) error {
|
||||
values := make(url.Values)
|
||||
values.Set(storageRESTVolume, "foo")
|
||||
values.Set(storageRESTTotalVersions, total)
|
||||
// An empty body: nothing to decode, so a handler that sizes from the
|
||||
// declared count allocates for nothing at all.
|
||||
respBody, err := restClient.call(ctx, storageRESTMethodDeleteVersions, values, bytes.NewReader(nil), 0)
|
||||
if respBody != nil {
|
||||
xhttp.DrainBody(respBody)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// A negative count used to reach make() and panic. It must now be refused
|
||||
// by name, not merely produce "some error" - an empty body errors either
|
||||
// way, so a bare err != nil check here would pass against the bug.
|
||||
if err := send("-1"); err == nil || !strings.Contains(err.Error(), errInvalidArgument.Error()) {
|
||||
t.Errorf("total-versions=-1: expected %v, got %v", errInvalidArgument, err)
|
||||
}
|
||||
|
||||
// The allocation must stay proportional to the body, not to the declared
|
||||
// count. TotalAlloc is cumulative and never decreases, so it records the
|
||||
// allocation even if it is immediately collected.
|
||||
for _, total := range []string{"100000000", "9223372036854775807"} {
|
||||
var before, after runtime.MemStats
|
||||
runtime.ReadMemStats(&before)
|
||||
err := send(total)
|
||||
runtime.ReadMemStats(&after)
|
||||
|
||||
grew := after.TotalAlloc - before.TotalAlloc
|
||||
t.Logf("total-versions=%s -> err=%v, allocated %d bytes", total, err, grew)
|
||||
// 100000000 * sizeof(FileInfoVersions)(104) is ~9.7 GiB; anything in
|
||||
// that neighborhood means the declared count is still being trusted.
|
||||
if grew > 64<<20 {
|
||||
t.Errorf("total-versions=%s allocated %d bytes for an empty body - "+
|
||||
"the declared count is sizing the allocation", total, grew)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := restClient.ReadAll(ctx, "foo", "canary.txt"); err != nil {
|
||||
t.Fatalf("node stopped serving: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAppendFileDeclaredLengthIsNotTrusted is the same family again, this time
|
||||
// through the HTTP Content-Length header rather than a query argument.
|
||||
//
|
||||
// setRequestLimitMiddleware only wraps the body in a MaxBytesReader sized at
|
||||
// requestMaxBodySize (5 TiB + 64 MiB); it never checks the *declared*
|
||||
// Content-Length. Sizing a buffer from that declaration therefore lets a
|
||||
// request carrying no body at all reserve arbitrary memory.
|
||||
func TestAppendFileDeclaredLengthIsNotTrusted(t *testing.T) {
|
||||
restClient := newStorageRESTHTTPServerClient(t)
|
||||
drive := globalLocalSetDrives[0][0][0].Endpoint().Path
|
||||
ctx := t.Context()
|
||||
mustWrite(t, filepath.Join(drive, "foo", "canary.txt"), "CANARY")
|
||||
|
||||
// Go's own http client refuses to send a request whose body is shorter than
|
||||
// the declared Content-Length, so the forged header cannot be driven through
|
||||
// restClient - an attacker with a raw socket has no such scruples. Drive the
|
||||
// handler directly instead; the header reaches r.ContentLength verbatim
|
||||
// either way, and the allocation happens before a single body byte is read.
|
||||
server := &storageRESTServer{endpoint: globalLocalSetDrives[0][0][0].Endpoint()}
|
||||
call := func(declared int64, body []byte) *httptest.ResponseRecorder {
|
||||
u := "/?" + url.Values{
|
||||
storageRESTVolume: []string{"foo"},
|
||||
storageRESTFilePath: []string{"appended.bin"},
|
||||
}.Encode()
|
||||
req := httptest.NewRequest(http.MethodPost, u, bytes.NewReader(body))
|
||||
req.ContentLength = declared // what a raw client would put on the wire
|
||||
req.Header.Set("Authorization", "Bearer "+globalNodeAuthToken)
|
||||
req.Header.Set("X-Minio-Time", strconv.FormatInt(time.Now().UnixNano(), 10))
|
||||
w := httptest.NewRecorder()
|
||||
server.AppendFileHandler(w, req)
|
||||
return w
|
||||
}
|
||||
|
||||
for _, declared := range []int64{4 << 30, 64 << 30} {
|
||||
var before, after runtime.MemStats
|
||||
runtime.ReadMemStats(&before)
|
||||
w := call(declared, nil)
|
||||
runtime.ReadMemStats(&after)
|
||||
|
||||
grew := after.TotalAlloc - before.TotalAlloc
|
||||
t.Logf("Content-Length=%d, empty body -> status=%d, allocated %d bytes", declared, w.Code, grew)
|
||||
// The bound separates "reserved a fixed amount" (single-digit MiB, and
|
||||
// somewhat more under -race) from "sized by the declaration" (GiB). Keep
|
||||
// it well clear of maxAppendFilePrealloc so a race build's extra
|
||||
// bookkeeping cannot trip it.
|
||||
if grew > 64<<20 {
|
||||
t.Errorf("Content-Length=%d allocated %d bytes for an empty body - "+
|
||||
"the declared length is sizing the allocation", declared, grew)
|
||||
}
|
||||
}
|
||||
|
||||
// Chunked requests arrive with ContentLength == -1, which used to reach
|
||||
// make() directly and panic.
|
||||
if w := call(-1, nil); w.Code == http.StatusOK {
|
||||
t.Error("ContentLength=-1 was accepted; it must be refused")
|
||||
}
|
||||
|
||||
// A well-formed append must still work, and land the exact bytes.
|
||||
payload := []byte("REAL-APPEND-PAYLOAD")
|
||||
if w := call(int64(len(payload)), payload); w.Code != http.StatusOK {
|
||||
t.Fatalf("legitimate AppendFile: status %d, body %q", w.Code, w.Body.String())
|
||||
}
|
||||
got, err := restClient.ReadAll(ctx, "foo", "appended.bin")
|
||||
if err != nil || string(got) != string(payload) {
|
||||
t.Fatalf("AppendFile round trip: got %q err %v", got, err)
|
||||
}
|
||||
|
||||
if _, err := restClient.ReadAll(ctx, "foo", "canary.txt"); err != nil {
|
||||
t.Fatalf("node stopped serving: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNegativePartSizeNeverPersists covers the one defect in this family that
|
||||
// survives the request that created it.
|
||||
//
|
||||
// A malicious peer can write metadata carrying a negative part size through
|
||||
// WriteMetadata or RenameData. AddVersion used to persist PartSizes verbatim,
|
||||
// after which a *local* heal - which does not pass through the wire guards -
|
||||
// reads it back, derives a zero expected shard size, and reports every part
|
||||
// intact. The poison stays on disk and the cluster reports itself healthy.
|
||||
//
|
||||
// Both ends are covered: the write funnel refuses to persist it, and the
|
||||
// verification sinks refuse metadata already on disk.
|
||||
func TestNegativePartSizeNeverPersists(t *testing.T) {
|
||||
badFI := FileInfo{
|
||||
Volume: "foo", Name: "obj", ModTime: UTCNow(),
|
||||
VersionID: "00000000-0000-0000-0000-0000000000aa",
|
||||
Parts: []ObjectPartInfo{{Number: 1, Size: -2}},
|
||||
Erasure: ErasureInfo{
|
||||
DataBlocks: 2, ParityBlocks: 2, BlockSize: blockSizeV2,
|
||||
Index: 1, Distribution: []int{1, 2, 3, 4},
|
||||
},
|
||||
}
|
||||
if !badFI.IsValid() {
|
||||
t.Fatal("test bug: the counterexample must satisfy FileInfo.IsValid()")
|
||||
}
|
||||
|
||||
// The write funnel every version write passes through.
|
||||
var meta xlMetaV2
|
||||
if err := meta.AddVersion(badFI); !errors.Is(err, errFileCorrupt) {
|
||||
t.Errorf("xlMetaV2.AddVersion persisted a negative part size: got %v, want %v", err, errFileCorrupt)
|
||||
}
|
||||
|
||||
// The verification sinks, reached by local heals that bypass the wire guards.
|
||||
restClient := newStorageRESTHTTPServerClient(t)
|
||||
drive := globalLocalSetDrives[0][0][0].Endpoint().Path
|
||||
mustWrite(t, filepath.Join(drive, "foo", "poisoned", "part.1"), "truncated")
|
||||
|
||||
storage := globalLocalSetDrives[0][0][0]
|
||||
if _, err := storage.CheckParts(t.Context(), "foo", "poisoned", badFI); !errors.Is(err, errFileCorrupt) {
|
||||
t.Errorf("local CheckParts accepted a negative part size: got %v, want %v", err, errFileCorrupt)
|
||||
}
|
||||
if _, err := storage.VerifyFile(t.Context(), "foo", "poisoned", badFI); !errors.Is(err, errFileCorrupt) {
|
||||
t.Errorf("local VerifyFile accepted a negative part size: got %v, want %v", err, errFileCorrupt)
|
||||
}
|
||||
|
||||
// And still refused over the wire.
|
||||
if _, err := restClient.CheckParts(t.Context(), "foo", "poisoned", badFI); !errors.Is(err, errFileCorrupt) {
|
||||
t.Errorf("remote CheckParts accepted a negative part size: got %v, want %v", err, errFileCorrupt)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReadFileLengthIsBounded pins the ceiling on ReadFileHandler's buffer.
|
||||
// A legitimate read cannot exceed one erasure shard, and a shard cannot exceed
|
||||
// the S3 part it encodes.
|
||||
// Driven against the handler directly: the REST client derives the length from
|
||||
// a caller-supplied buffer, so going through it would allocate the very
|
||||
// gigabytes this test is trying to prove the server does not.
|
||||
func TestReadFileLengthIsBounded(t *testing.T) {
|
||||
newStorageRESTHTTPServerClient(t)
|
||||
drive := globalLocalSetDrives[0][0][0].Endpoint().Path
|
||||
mustWrite(t, filepath.Join(drive, "foo", "small.bin"), "tiny")
|
||||
|
||||
server := &storageRESTServer{endpoint: globalLocalSetDrives[0][0][0].Endpoint()}
|
||||
call := func(length string) (*httptest.ResponseRecorder, uint64) {
|
||||
u := "/?" + url.Values{
|
||||
storageRESTVolume: []string{"foo"},
|
||||
storageRESTFilePath: []string{"small.bin"},
|
||||
storageRESTOffset: []string{"0"},
|
||||
storageRESTLength: []string{length},
|
||||
}.Encode()
|
||||
req := httptest.NewRequest(http.MethodPost, u, nil)
|
||||
req.Header.Set("Authorization", "Bearer "+globalNodeAuthToken)
|
||||
req.Header.Set("X-Minio-Time", strconv.FormatInt(time.Now().UnixNano(), 10))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
var before, after runtime.MemStats
|
||||
runtime.ReadMemStats(&before)
|
||||
server.ReadFileHandler(w, req)
|
||||
runtime.ReadMemStats(&after)
|
||||
return w, after.TotalAlloc - before.TotalAlloc
|
||||
}
|
||||
|
||||
for _, length := range []string{"8589934592", "1099511627776"} { // 8 GiB, 1 TiB
|
||||
w, grew := call(length)
|
||||
t.Logf("length=%s against a 4 byte file -> status=%d, allocated %d bytes", length, w.Code, grew)
|
||||
if grew > 64<<20 {
|
||||
t.Errorf("length=%s allocated %d bytes; the declared length is sizing the buffer", length, grew)
|
||||
}
|
||||
}
|
||||
|
||||
// A read within the ceiling must still behave exactly as before: the file is
|
||||
// four bytes, so asking for more is a short read, not a rejected argument.
|
||||
w, _ := call("64")
|
||||
if body := w.Body.String(); strings.Contains(body, errInvalidArgument.Error()) {
|
||||
t.Errorf("a 64 byte read was rejected by the ceiling: %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
// TestShardFileSizeZeroErasure pins the arithmetic guard at the sink, which is
|
||||
// what actually protects every caller.
|
||||
//
|
||||
// Both ShardFileSize methods are covered. They are separate implementations on
|
||||
// separate types -- ErasureInfo (metadata, reached by CheckParts/VerifyFile)
|
||||
// and Erasure (the coder, reached by the object layer via NewErasure, which
|
||||
// validates dataBlocks and parityBlocks but not blockSize) -- and guarding one
|
||||
// leaves the other divisible by zero.
|
||||
func TestShardFileSizeZeroErasure(t *testing.T) {
|
||||
for _, e := range []ErasureInfo{
|
||||
{},
|
||||
{DataBlocks: 4},
|
||||
{BlockSize: blockSizeV2},
|
||||
{BlockSize: -1, DataBlocks: -1},
|
||||
} {
|
||||
if got := e.ShardFileSize(1024); got < 0 {
|
||||
t.Errorf("ShardFileSize(%+v) = %d, want a non-negative size", e, got)
|
||||
}
|
||||
if got := e.ShardSize(); got < 0 {
|
||||
t.Errorf("ShardSize(%+v) = %d, want a non-negative size", e, got)
|
||||
}
|
||||
}
|
||||
|
||||
// The coder variant. NewErasure must refuse a non-positive block size at
|
||||
// construction: guarding ShardFileSize alone would leave ShardFileOffset
|
||||
// and every division in erasure-decode.go dividing by zero, since they all
|
||||
// use e.blockSize directly. Erasure is only ever built here, so this single
|
||||
// point covers all of them.
|
||||
for _, block := range []int64{0, -1} {
|
||||
if _, err := NewErasure(t.Context(), 4, 2, block); err == nil {
|
||||
t.Errorf("NewErasure accepted blockSize=%d; every downstream division by "+
|
||||
"e.blockSize then divides by zero", block)
|
||||
}
|
||||
}
|
||||
|
||||
// A sane coder still works, and its arithmetic stays non-negative.
|
||||
coder, err := NewErasure(t.Context(), 4, 2, blockSizeV2)
|
||||
if err != nil {
|
||||
t.Fatalf("NewErasure rejected a legitimate configuration: %v", err)
|
||||
}
|
||||
if got := coder.ShardFileSize(1024); got < 0 {
|
||||
t.Errorf("Erasure.ShardFileSize = %d, want non-negative", got)
|
||||
}
|
||||
if got := coder.ShardFileOffset(0, 1024, 4096); got < 0 {
|
||||
t.Errorf("Erasure.ShardFileOffset = %d, want non-negative", got)
|
||||
}
|
||||
}
|
||||
+11
-11
@@ -33,22 +33,22 @@ func _() {
|
||||
_ = x[storageMetricReadXL-22]
|
||||
_ = x[storageMetricReadAll-23]
|
||||
_ = x[storageMetricStatInfoFile-24]
|
||||
_ = x[storageMetricReadMultiple-25]
|
||||
_ = x[storageMetricDeleteAbandonedParts-26]
|
||||
_ = x[storageMetricDiskInfo-27]
|
||||
_ = x[storageMetricDeleteBulk-28]
|
||||
_ = x[storageMetricRenamePart-29]
|
||||
_ = x[storageMetricReadParts-30]
|
||||
_ = x[storageMetricLast-31]
|
||||
_ = x[storageMetricDeleteAbandonedParts-25]
|
||||
_ = x[storageMetricDiskInfo-26]
|
||||
_ = x[storageMetricDeleteBulk-27]
|
||||
_ = x[storageMetricRenamePart-28]
|
||||
_ = x[storageMetricReadParts-29]
|
||||
_ = x[storageMetricLast-30]
|
||||
}
|
||||
|
||||
const _storageMetric_name = "MakeVolBulkMakeVolListVolsStatVolDeleteVolWalkDirListDirReadFileAppendFileCreateFileReadFileStreamRenameFileRenameDataCheckPartsDeleteDeleteVersionsVerifyFileWriteAllDeleteVersionWriteMetadataUpdateMetadataReadVersionReadXLReadAllStatInfoFileReadMultipleDeleteAbandonedPartsDiskInfoDeleteBulkRenamePartReadPartsLast"
|
||||
const _storageMetric_name = "MakeVolBulkMakeVolListVolsStatVolDeleteVolWalkDirListDirReadFileAppendFileCreateFileReadFileStreamRenameFileRenameDataCheckPartsDeleteDeleteVersionsVerifyFileWriteAllDeleteVersionWriteMetadataUpdateMetadataReadVersionReadXLReadAllStatInfoFileDeleteAbandonedPartsDiskInfoDeleteBulkRenamePartReadPartsLast"
|
||||
|
||||
var _storageMetric_index = [...]uint16{0, 11, 18, 26, 33, 42, 49, 56, 64, 74, 84, 98, 108, 118, 128, 134, 148, 158, 166, 179, 192, 206, 217, 223, 230, 242, 254, 274, 282, 292, 302, 311, 315}
|
||||
var _storageMetric_index = [...]uint16{0, 11, 18, 26, 33, 42, 49, 56, 64, 74, 84, 98, 108, 118, 128, 134, 148, 158, 166, 179, 192, 206, 217, 223, 230, 242, 262, 270, 280, 290, 299, 303}
|
||||
|
||||
func (i storageMetric) String() string {
|
||||
if i >= storageMetric(len(_storageMetric_index)-1) {
|
||||
idx := int(i) - 0
|
||||
if i < 0 || idx >= len(_storageMetric_index)-1 {
|
||||
return "storageMetric(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _storageMetric_name[_storageMetric_index[i]:_storageMetric_index[i+1]]
|
||||
return _storageMetric_name[_storageMetric_index[idx]:_storageMetric_index[idx+1]]
|
||||
}
|
||||
|
||||
+58
-51
@@ -37,7 +37,6 @@ import (
|
||||
"github.com/minio/minio/internal/auth"
|
||||
idldap "github.com/minio/minio/internal/config/identity/ldap"
|
||||
"github.com/minio/minio/internal/config/identity/openid"
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
"github.com/minio/minio/internal/hash/sha256"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
@@ -267,14 +266,22 @@ func getTokenSigningKey() (string, error) {
|
||||
return secret, nil
|
||||
}
|
||||
|
||||
// stsLDAPLoginRateLimiter throttles LDAP STS logins per source IP. It does not
|
||||
// bucket by username on purpose: a username-keyed bucket is shared across all
|
||||
// sources, so a single client sending bad-password attempts for a known account
|
||||
// could keep that account's bucket drained and lock the legitimate user out.
|
||||
//
|
||||
// Scope of protection: the per-source bucket caps the attempt rate from any one
|
||||
// source, and the uniform auth-failure response (not this limiter) is what hides
|
||||
// whether a username exists. It does not stop attackers who spread across many
|
||||
// sources (botnets, IPv6 address rotation) or the residual bind-timing side
|
||||
// channel; those are accepted limitations of an in-memory, per-source control.
|
||||
type stsLDAPLoginRateLimiter struct {
|
||||
source *stsLDAPLoginKeyLimiterSet
|
||||
user *stsLDAPLoginKeyLimiterSet
|
||||
}
|
||||
|
||||
type stsLDAPLoginReservation struct {
|
||||
source *stsLDAPLoginKeyReservation
|
||||
user *stsLDAPLoginKeyReservation
|
||||
}
|
||||
|
||||
type stsLDAPLoginKeyLimiterSet struct {
|
||||
@@ -302,7 +309,6 @@ type stsLDAPLoginKeyReservation struct {
|
||||
func newSTSLDAPLoginRateLimiter(refillEvery time.Duration, burst int, ttl time.Duration) *stsLDAPLoginRateLimiter {
|
||||
return &stsLDAPLoginRateLimiter{
|
||||
source: newSTSLDAPLoginKeyLimiterSet(refillEvery, burst, ttl),
|
||||
user: newSTSLDAPLoginKeyLimiterSet(refillEvery, burst, ttl),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -315,12 +321,8 @@ func newSTSLDAPLoginKeyLimiterSet(refillEvery time.Duration, burst int, ttl time
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeSTSLDAPUsername(username string) string {
|
||||
return strings.ToLower(strings.TrimSpace(username))
|
||||
}
|
||||
|
||||
func (l *stsLDAPLoginRateLimiter) Allow(sourceIP, username string) bool {
|
||||
reservation := l.Reserve(sourceIP, username)
|
||||
func (l *stsLDAPLoginRateLimiter) Allow(sourceIP string) bool {
|
||||
reservation := l.Reserve(sourceIP)
|
||||
if reservation == nil {
|
||||
return false
|
||||
}
|
||||
@@ -328,55 +330,34 @@ func (l *stsLDAPLoginRateLimiter) Allow(sourceIP, username string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (l *stsLDAPLoginRateLimiter) Reserve(sourceIP, username string) *stsLDAPLoginReservation {
|
||||
now := UTCNow()
|
||||
reservation := &stsLDAPLoginReservation{}
|
||||
|
||||
if sourceIP != "" {
|
||||
reservation.source = l.source.Reserve(now, sourceIP)
|
||||
if reservation.source == nil {
|
||||
return nil
|
||||
}
|
||||
func (l *stsLDAPLoginRateLimiter) Reserve(sourceIP string) *stsLDAPLoginReservation {
|
||||
// An empty source IP means we could not identify the peer; do not throttle
|
||||
// rather than collapse every such request into one shared bucket.
|
||||
if sourceIP == "" {
|
||||
return &stsLDAPLoginReservation{}
|
||||
}
|
||||
|
||||
username = normalizeSTSLDAPUsername(username)
|
||||
if username != "" {
|
||||
reservation.user = l.user.Reserve(now, username)
|
||||
if reservation.user == nil {
|
||||
reservation.Cancel()
|
||||
return nil
|
||||
}
|
||||
source := l.source.Reserve(UTCNow(), sourceIP)
|
||||
if source == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return reservation
|
||||
return &stsLDAPLoginReservation{source: source}
|
||||
}
|
||||
|
||||
func (r *stsLDAPLoginReservation) Commit() {
|
||||
if r == nil {
|
||||
if r == nil || r.source == nil {
|
||||
return
|
||||
}
|
||||
if r.source != nil {
|
||||
r.source.CommitAt(UTCNow())
|
||||
r.source = nil
|
||||
}
|
||||
if r.user != nil {
|
||||
r.user.CommitAt(UTCNow())
|
||||
r.user = nil
|
||||
}
|
||||
r.source.CommitAt(UTCNow())
|
||||
r.source = nil
|
||||
}
|
||||
|
||||
func (r *stsLDAPLoginReservation) Cancel() {
|
||||
if r == nil {
|
||||
if r == nil || r.source == nil {
|
||||
return
|
||||
}
|
||||
if r.source != nil {
|
||||
r.source.CancelAt(UTCNow())
|
||||
r.source = nil
|
||||
}
|
||||
if r.user != nil {
|
||||
r.user.CancelAt(UTCNow())
|
||||
r.user = nil
|
||||
}
|
||||
r.source.CancelAt(UTCNow())
|
||||
r.source = nil
|
||||
}
|
||||
|
||||
func (l *stsLDAPLoginKeyLimiterSet) Allow(now time.Time, key string) bool {
|
||||
@@ -510,11 +491,37 @@ func getSTSLDAPLoginSourceIP(r *http.Request) string {
|
||||
return sourceIP
|
||||
}
|
||||
|
||||
// getSTSLDAPTrustedProxySourceIP resolves the client IP for a request whose peer
|
||||
// is an allow-listed trusted proxy. A single clean X-Real-IP is preferred; for
|
||||
// X-Forwarded-For we walk the chain right-to-left and skip trusted-proxy hops,
|
||||
// returning the first untrusted address. The XFF result ignores any client-
|
||||
// supplied (left-most) value unless the entire chain to its right is trusted,
|
||||
// which an external client cannot forge.
|
||||
//
|
||||
// X-Real-IP, unlike XFF, is a single value with no chain, so it cannot be
|
||||
// validated against the allowlist: it is trusted verbatim. The deployment
|
||||
// contract is therefore that the trusted proxy MUST overwrite (not pass through)
|
||||
// any client-supplied X-Real-IP; otherwise an attacker can vary it per request
|
||||
// to evade per-source throttling. This is the standard reverse-proxy real-IP
|
||||
// contract; reordering to prefer XFF would not remove the dependency, only move
|
||||
// it (an X-Real-IP-only proxy would then be evaded via an injected XFF header).
|
||||
//
|
||||
// The RFC 7239 Forwarded header is intentionally not honored here; such
|
||||
// deployments fall back to the safe peer-address bucket.
|
||||
func getSTSLDAPTrustedProxySourceIP(r *http.Request) string {
|
||||
if realIP := getSTSLDAPLoginCanonicalIP(r.Header.Get("X-Real-IP")); realIP != "" {
|
||||
return realIP
|
||||
}
|
||||
return getSTSLDAPLoginCanonicalIP(handlers.GetSourceIPFromHeaders(r))
|
||||
|
||||
forwarded := strings.Split(r.Header.Get("X-Forwarded-For"), ",")
|
||||
for i := len(forwarded) - 1; i >= 0; i-- {
|
||||
ip := getSTSLDAPLoginCanonicalIP(forwarded[i])
|
||||
if ip == "" || globalIAMSys.LDAPConfig.IsSTSTrustedProxy(ip) {
|
||||
continue
|
||||
}
|
||||
return ip
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func getSTSLDAPLoginPeerAddr(remoteAddr string) string {
|
||||
@@ -535,11 +542,11 @@ func getSTSLDAPLoginCanonicalIP(addr string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// reserveSTSLDAPLogin acquires immediate tokens from the per-source and
|
||||
// per-username limiters before contacting LDAP. Call Commit on auth failures
|
||||
// and Cancel when the attempt should not count as an authentication failure.
|
||||
// reserveSTSLDAPLogin acquires an immediate token from the per-source limiter
|
||||
// before contacting LDAP. Call Commit on auth failures and Cancel when the
|
||||
// attempt should not count as an authentication failure.
|
||||
func reserveSTSLDAPLogin(r *http.Request) *stsLDAPLoginReservation {
|
||||
return globalSTSLDAPLoginRateLimiter.Reserve(getSTSLDAPLoginSourceIP(r), r.Form.Get(stsLDAPUsername))
|
||||
return globalSTSLDAPLoginRateLimiter.Reserve(getSTSLDAPLoginSourceIP(r))
|
||||
}
|
||||
|
||||
func ldapBindErrorToSTS(err error) (STSErrorCode, error) {
|
||||
|
||||
+206
-67
@@ -41,6 +41,8 @@ import (
|
||||
"github.com/minio/minio-go/v7"
|
||||
cr "github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/pkg/v3/ldap"
|
||||
)
|
||||
|
||||
@@ -417,6 +419,15 @@ func (s *TestSuiteIAM) TestSTSWithTags(c *check) {
|
||||
"Resource": "arn:aws:s3:::%s/*",
|
||||
"Condition": { "StringEquals": {"s3:ExistingObjectTag/security": "public" } }
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": "s3:PutObjectTagging",
|
||||
"Resource": "arn:aws:s3:::%s/*",
|
||||
"Condition": { "StringEquals": {
|
||||
"s3:ExistingObjectTag/virus": "true",
|
||||
"s3:RequestObjectTag/security": "public"
|
||||
} }
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": "s3:DeleteObject",
|
||||
@@ -431,6 +442,9 @@ func (s *TestSuiteIAM) TestSTSWithTags(c *check) {
|
||||
"arn:aws:s3:::%s/*"
|
||||
],
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:RequestObjectTag/security": "public"
|
||||
},
|
||||
"ForAllValues:StringLike": {
|
||||
"s3:RequestObjectTagKeys": [
|
||||
"security",
|
||||
@@ -440,7 +454,7 @@ func (s *TestSuiteIAM) TestSTSWithTags(c *check) {
|
||||
}
|
||||
}
|
||||
]
|
||||
}`, bucket, bucket, bucket, bucket)
|
||||
}`, bucket, bucket, bucket, bucket, bucket)
|
||||
err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
|
||||
if err != nil {
|
||||
c.Fatalf("policy add error: %v", err)
|
||||
@@ -462,8 +476,108 @@ func (s *TestSuiteIAM) TestSTSWithTags(c *check) {
|
||||
|
||||
// confirm that the user is able to access the bucket
|
||||
uClient := s.getUserClient(c, accessKey, secretKey, "")
|
||||
queryObject := object + "-query-tags"
|
||||
queryTags := "security=public&virus=true"
|
||||
// Query storage-class values were historically accepted without the strict
|
||||
// header validation. Pin that compatibility while proving they are consumed.
|
||||
presignedPut, err := uClient.Presign(ctx, http.MethodPut, bucket, queryObject, time.Minute, url.Values{
|
||||
"x-amz-storage-class": {"CUSTOM_COMPAT"},
|
||||
"x-amz-tagging": {queryTags},
|
||||
})
|
||||
if err != nil {
|
||||
c.Fatalf("unable to presign query-tagged upload: %v", err)
|
||||
}
|
||||
putReq, err := http.NewRequestWithContext(ctx, http.MethodPut, presignedPut.String(), bytes.NewReader([]byte("stuff")))
|
||||
if err != nil {
|
||||
c.Fatalf("unable to build query-tagged upload: %v", err)
|
||||
}
|
||||
putResp, err := s.TestSuiteCommon.client.Do(putReq)
|
||||
if err != nil {
|
||||
c.Fatalf("query-tagged upload failed: %v", err)
|
||||
}
|
||||
putBody, readErr := io.ReadAll(putResp.Body)
|
||||
putResp.Body.Close()
|
||||
if readErr != nil {
|
||||
c.Fatalf("unable to read query-tagged upload response: %v", readErr)
|
||||
}
|
||||
if putResp.StatusCode != http.StatusOK {
|
||||
c.Fatalf("query-tagged upload returned %s: %s", putResp.Status, putBody)
|
||||
}
|
||||
storedTags, err := s.client.GetObjectTagging(ctx, bucket, queryObject, minio.GetObjectTaggingOptions{})
|
||||
if err != nil {
|
||||
c.Fatalf("unable to read persisted query tags: %v", err)
|
||||
}
|
||||
if got := storedTags.ToMap(); got["security"] != "public" || got["virus"] != "true" {
|
||||
c.Fatalf("query tags were not persisted: %v", got)
|
||||
}
|
||||
queryObjectInfo, err := s.testServer.Obj.GetObjectInfo(ctx, bucket, queryObject, ObjectOptions{})
|
||||
if err != nil {
|
||||
c.Fatalf("unable to inspect query-tagged object: %v", err)
|
||||
}
|
||||
if queryObjectInfo.StorageClass != "CUSTOM_COMPAT" {
|
||||
c.Fatalf("query storage class was not persisted: %q", queryObjectInfo.StorageClass)
|
||||
}
|
||||
c.mustGetObject(ctx, uClient, bucket, queryObject)
|
||||
|
||||
multipartObject := object + "-multipart-query-tags"
|
||||
presignedMultipart, err := uClient.Presign(ctx, http.MethodPost, bucket, multipartObject, time.Minute, url.Values{
|
||||
"uploads": {""},
|
||||
"x-amz-tagging": {queryTags},
|
||||
})
|
||||
if err != nil {
|
||||
c.Fatalf("unable to presign query-tagged multipart upload: %v", err)
|
||||
}
|
||||
multipartReq, err := http.NewRequestWithContext(ctx, http.MethodPost, presignedMultipart.String(), nil)
|
||||
if err != nil {
|
||||
c.Fatalf("unable to build query-tagged multipart upload: %v", err)
|
||||
}
|
||||
multipartResp, err := s.TestSuiteCommon.client.Do(multipartReq)
|
||||
if err != nil {
|
||||
c.Fatalf("query-tagged multipart upload failed: %v", err)
|
||||
}
|
||||
multipartBody, readErr := io.ReadAll(multipartResp.Body)
|
||||
multipartResp.Body.Close()
|
||||
if readErr != nil {
|
||||
c.Fatalf("unable to read query-tagged multipart response: %v", readErr)
|
||||
}
|
||||
if multipartResp.StatusCode != http.StatusOK {
|
||||
c.Fatalf("query-tagged multipart upload returned %s: %s", multipartResp.Status, multipartBody)
|
||||
}
|
||||
var multipartResult InitiateMultipartUploadResponse
|
||||
if err = xml.Unmarshal(multipartBody, &multipartResult); err != nil || multipartResult.UploadID == "" {
|
||||
c.Fatalf("invalid query-tagged multipart response: uploadID=%q err=%v", multipartResult.UploadID, err)
|
||||
}
|
||||
if err = (minio.Core{Client: s.client}).AbortMultipartUpload(ctx, bucket, multipartObject, multipartResult.UploadID); err != nil {
|
||||
c.Fatalf("unable to clean up query-tagged multipart upload: %v", err)
|
||||
}
|
||||
|
||||
c.mustPutObjectWithTags(ctx, uClient, bucket, object)
|
||||
c.mustGetObject(ctx, uClient, bucket, object)
|
||||
objectInfo, err := s.client.StatObject(ctx, bucket, object, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
c.Fatalf("unable to stat object for conditional GET: %v", err)
|
||||
}
|
||||
presignedGet, err := uClient.PresignedGetObject(ctx, bucket, object, time.Minute, nil)
|
||||
if err != nil {
|
||||
c.Fatalf("unable to presign conditional GET: %v", err)
|
||||
}
|
||||
getReq, err := http.NewRequestWithContext(ctx, http.MethodGet, presignedGet.String(), nil)
|
||||
if err != nil {
|
||||
c.Fatalf("unable to build conditional GET: %v", err)
|
||||
}
|
||||
getReq.Header.Set(xhttp.IfNoneMatch, `"`+objectInfo.ETag+`"`)
|
||||
getResp := httptest.NewRecorder()
|
||||
s.testServer.Server.Config.Handler.ServeHTTP(getResp, getReq)
|
||||
if getResp.Code != http.StatusNotModified || getResp.Body.Len() != 0 {
|
||||
c.Fatalf("conditional GET returned status %d with body %q", getResp.Code, getResp.Body.String())
|
||||
}
|
||||
replacementTags, err := tags.NewTags(map[string]string{"security": "public", "reviewed": "yes"}, true)
|
||||
if err != nil {
|
||||
c.Fatalf("unable to build replacement tags: %v", err)
|
||||
}
|
||||
if err = uClient.PutObjectTagging(ctx, bucket, object, replacementTags, minio.PutObjectTaggingOptions{}); err != nil {
|
||||
c.Fatalf("user is unable to replace object tags: %v", err)
|
||||
}
|
||||
|
||||
assumeRole := cr.STSAssumeRole{
|
||||
Client: s.TestSuiteCommon.client,
|
||||
@@ -502,6 +616,9 @@ func (s *TestSuiteIAM) TestSTSWithTags(c *check) {
|
||||
if err = minioClient.RemoveObject(ctx, bucket, object, minio.RemoveObjectOptions{}); err != nil {
|
||||
c.Fatalf("user is unable to delete the object: %v", err)
|
||||
}
|
||||
if err = minioClient.RemoveObject(ctx, bucket, queryObject, minio.RemoveObjectOptions{}); err != nil {
|
||||
c.Fatalf("user is unable to delete the query-tagged object: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestSTS(c *check) {
|
||||
@@ -1415,53 +1532,51 @@ func targetIsLDAPAuthFailure(target error) bool {
|
||||
func TestSTSLDAPLoginRateLimiter(t *testing.T) {
|
||||
limiter := newSTSLDAPLoginRateLimiter(time.Hour, 2, time.Minute)
|
||||
|
||||
if !limiter.Allow("192.0.2.10", "dillon") {
|
||||
if !limiter.Allow("192.0.2.10") {
|
||||
t.Fatal("expected first attempt to be allowed")
|
||||
}
|
||||
if !limiter.Allow("192.0.2.10", "kevin") {
|
||||
t.Fatal("expected second source-IP attempt to be allowed")
|
||||
if !limiter.Allow("192.0.2.10") {
|
||||
t.Fatal("expected second attempt within burst to be allowed")
|
||||
}
|
||||
if limiter.Allow("192.0.2.10", "stuart") {
|
||||
t.Fatal("expected source IP bucket to be throttled")
|
||||
if limiter.Allow("192.0.2.10") {
|
||||
t.Fatal("expected source IP bucket to be throttled after burst")
|
||||
}
|
||||
|
||||
limiter = newSTSLDAPLoginRateLimiter(time.Hour, 2, time.Minute)
|
||||
if !limiter.Allow("192.0.2.10", "dillon") {
|
||||
t.Fatal("expected first username attempt to be allowed")
|
||||
// A different source IP has its own independent bucket, so one client
|
||||
// cannot exhaust another's budget (no per-username lockout dimension).
|
||||
if !limiter.Allow("192.0.2.11") {
|
||||
t.Fatal("expected a different source IP to be allowed")
|
||||
}
|
||||
if !limiter.Allow("192.0.2.11", "dillon") {
|
||||
t.Fatal("expected second username attempt from a different source to be allowed")
|
||||
}
|
||||
if limiter.Allow("192.0.2.12", "dillon") {
|
||||
t.Fatal("expected username bucket to be throttled")
|
||||
}
|
||||
if !limiter.Allow("192.0.2.12", "other-user") {
|
||||
t.Fatal("expected a fresh username and source tuple to be allowed")
|
||||
|
||||
// An empty source IP cannot be identified and must never be throttled,
|
||||
// otherwise all such requests would collapse into one shared bucket.
|
||||
if !limiter.Allow("") || !limiter.Allow("") {
|
||||
t.Fatal("expected unidentified source to stay unthrottled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSTSLDAPLoginRateLimiterReserveCancel(t *testing.T) {
|
||||
limiter := newSTSLDAPLoginRateLimiter(time.Hour, 1, time.Minute)
|
||||
|
||||
reservation := limiter.Reserve("192.0.2.10", "dillon")
|
||||
reservation := limiter.Reserve("192.0.2.10")
|
||||
if reservation == nil {
|
||||
t.Fatal("expected first reservation to succeed")
|
||||
}
|
||||
if limiter.Reserve("192.0.2.10", "kevin") != nil {
|
||||
if limiter.Reserve("192.0.2.10") != nil {
|
||||
t.Fatal("expected second reservation on the same source IP to be throttled before cancel")
|
||||
}
|
||||
|
||||
reservation.Cancel()
|
||||
|
||||
reservation = limiter.Reserve("192.0.2.10", "kevin")
|
||||
reservation = limiter.Reserve("192.0.2.10")
|
||||
if reservation == nil {
|
||||
t.Fatal("expected canceled reservation to restore source-IP capacity")
|
||||
}
|
||||
reservation.Cancel()
|
||||
|
||||
reservation = limiter.Reserve("192.0.2.11", "dillon")
|
||||
reservation = limiter.Reserve("192.0.2.11")
|
||||
if reservation == nil {
|
||||
t.Fatal("expected canceled reservation to restore username capacity")
|
||||
t.Fatal("expected a different source IP to have independent capacity")
|
||||
}
|
||||
reservation.Cancel()
|
||||
}
|
||||
@@ -1495,26 +1610,6 @@ func TestSTSLDAPLoginKeyLimiterCancelDoesNotOverCreditAfterRefill(t *testing.T)
|
||||
second.CancelAt(start.Add(10 * time.Millisecond))
|
||||
}
|
||||
|
||||
func TestSTSLDAPLoginRateLimiterReserveRollbackOnCompositeFailure(t *testing.T) {
|
||||
limiter := newSTSLDAPLoginRateLimiter(time.Hour, 1, time.Minute)
|
||||
|
||||
reservation := limiter.Reserve("192.0.2.10", "dillon")
|
||||
if reservation == nil {
|
||||
t.Fatal("expected initial reservation to succeed")
|
||||
}
|
||||
defer reservation.Cancel()
|
||||
|
||||
if limiter.Reserve("192.0.2.11", "dillon") != nil {
|
||||
t.Fatal("expected second reservation for the same username to be throttled")
|
||||
}
|
||||
|
||||
reservation2 := limiter.Reserve("192.0.2.11", "kevin")
|
||||
if reservation2 == nil {
|
||||
t.Fatal("expected throttled username reservation to roll back the provisional source-IP reservation")
|
||||
}
|
||||
reservation2.Cancel()
|
||||
}
|
||||
|
||||
func TestSTSLDAPLoginRateLimiterConcurrentReserveLifecycle(t *testing.T) {
|
||||
limiter := newSTSLDAPLoginRateLimiter(time.Hour, 4, time.Minute)
|
||||
|
||||
@@ -1533,7 +1628,7 @@ func TestSTSLDAPLoginRateLimiterConcurrentReserveLifecycle(t *testing.T) {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
|
||||
reservations[worker] = limiter.Reserve("192.0.2.10", "dillon")
|
||||
reservations[worker] = limiter.Reserve("192.0.2.10")
|
||||
reserveWG.Done()
|
||||
if reservations[worker] == nil {
|
||||
return
|
||||
@@ -1568,7 +1663,7 @@ func TestSTSLDAPLoginRateLimiterConcurrentReserveLifecycle(t *testing.T) {
|
||||
|
||||
remainingBudget := 0
|
||||
for {
|
||||
reservation := limiter.Reserve("192.0.2.10", "dillon")
|
||||
reservation := limiter.Reserve("192.0.2.10")
|
||||
if reservation == nil {
|
||||
break
|
||||
}
|
||||
@@ -1650,7 +1745,7 @@ func TestGetSTSLDAPLoginSourceIPUsesForwardedHeadersForTrustedProxy(t *testing.T
|
||||
{
|
||||
name: "x-forwarded-for",
|
||||
headerKey: "X-Forwarded-For",
|
||||
headerValue: "203.0.113.10, 198.51.100.24",
|
||||
headerValue: "203.0.113.10",
|
||||
want: "203.0.113.10",
|
||||
},
|
||||
{
|
||||
@@ -1659,12 +1754,6 @@ func TestGetSTSLDAPLoginSourceIPUsesForwardedHeadersForTrustedProxy(t *testing.T
|
||||
headerValue: "203.0.113.10",
|
||||
want: "203.0.113.10",
|
||||
},
|
||||
{
|
||||
name: "forwarded",
|
||||
headerKey: "Forwarded",
|
||||
headerValue: `for=203.0.113.10;proto=https`,
|
||||
want: "203.0.113.10",
|
||||
},
|
||||
}
|
||||
|
||||
withLDAPSTSTrustedProxiesForTest(t, "192.0.2.0/24", func() {
|
||||
@@ -1683,6 +1772,49 @@ func TestGetSTSLDAPLoginSourceIPUsesForwardedHeadersForTrustedProxy(t *testing.T
|
||||
})
|
||||
}
|
||||
|
||||
// A client behind a trusted, appending proxy can prepend a spoofed left-most
|
||||
// X-Forwarded-For value. The right-to-left walk must skip only trusted hops and
|
||||
// return the real (right-most untrusted) client, ignoring the spoofed value.
|
||||
func TestGetSTSLDAPLoginSourceIPTrustedProxyStripsSpoofedForwardedFor(t *testing.T) {
|
||||
withLDAPSTSTrustedProxiesForTest(t, "192.0.2.0/24", func() {
|
||||
req := &http.Request{
|
||||
Header: singleHeader("X-Forwarded-For", "1.2.3.4, 198.51.100.50"),
|
||||
RemoteAddr: "192.0.2.10:9000",
|
||||
}
|
||||
if got := getSTSLDAPLoginSourceIP(req); got != "198.51.100.50" {
|
||||
t.Fatalf("expected spoofed left-most XFF entry to be ignored and real client returned, got %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// When several hops in the chain are trusted proxies, the walk skips all of
|
||||
// them and resolves the left-most (real client) address.
|
||||
func TestGetSTSLDAPLoginSourceIPTrustedProxyWalksMultipleTrustedHops(t *testing.T) {
|
||||
withLDAPSTSTrustedProxiesForTest(t, "192.0.2.0/24", func() {
|
||||
req := &http.Request{
|
||||
Header: singleHeader("X-Forwarded-For", "203.0.113.10, 192.0.2.20, 192.0.2.21"),
|
||||
RemoteAddr: "192.0.2.10:9000",
|
||||
}
|
||||
if got := getSTSLDAPLoginSourceIP(req); got != "203.0.113.10" {
|
||||
t.Fatalf("expected walk to skip trusted hops and return real client, got %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// The RFC 7239 Forwarded header is not honored for trusted-proxy bucketing; such
|
||||
// requests fall back to the safe peer-address bucket.
|
||||
func TestGetSTSLDAPLoginSourceIPTrustedProxyIgnoresForwardedHeader(t *testing.T) {
|
||||
withLDAPSTSTrustedProxiesForTest(t, "192.0.2.0/24", func() {
|
||||
req := &http.Request{
|
||||
Header: singleHeader("Forwarded", `for=203.0.113.10;proto=https`),
|
||||
RemoteAddr: "192.0.2.10:9000",
|
||||
}
|
||||
if got := getSTSLDAPLoginSourceIP(req); got != "192.0.2.10" {
|
||||
t.Fatalf("expected RFC 7239 Forwarded to be ignored and peer address used, got %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetSTSLDAPLoginSourceIPTrustedProxyPrefersXRealIPOverXForwardedFor(t *testing.T) {
|
||||
withLDAPSTSTrustedProxiesForTest(t, "192.0.2.0/24", func() {
|
||||
req := &http.Request{
|
||||
@@ -1698,6 +1830,29 @@ func TestGetSTSLDAPLoginSourceIPTrustedProxyPrefersXRealIPOverXForwardedFor(t *t
|
||||
})
|
||||
}
|
||||
|
||||
// X-Real-IP is trusted verbatim (it cannot be chain-validated like X-Forwarded-For),
|
||||
// so a client-supplied X-Real-IP that the proxy fails to overwrite wins even over a
|
||||
// correctly appended X-Forwarded-For chain. This locks the documented deployment
|
||||
// contract: the trusted proxy MUST overwrite X-Real-IP, otherwise it is a spoofing
|
||||
// vector. If this assertion ever changes, the change must be deliberate.
|
||||
func TestGetSTSLDAPLoginSourceIPTrustedProxyTrustsXRealIPVerbatim(t *testing.T) {
|
||||
withLDAPSTSTrustedProxiesForTest(t, "192.0.2.0/24", func() {
|
||||
req := &http.Request{
|
||||
Header: make(http.Header),
|
||||
RemoteAddr: "192.0.2.10:9000",
|
||||
}
|
||||
// Attacker spoofs X-Real-IP with a value that appears nowhere in the XFF
|
||||
// chain; the trusted proxy still appends the real client (198.51.100.50).
|
||||
// The spoofed X-Real-IP wins, so the result can only have come from it.
|
||||
req.Header.Set("X-Real-IP", "10.10.10.10")
|
||||
req.Header.Set("X-Forwarded-For", "1.1.1.1, 198.51.100.50")
|
||||
|
||||
if got := getSTSLDAPLoginSourceIP(req); got != "10.10.10.10" {
|
||||
t.Fatalf("expected verbatim X-Real-IP trust (spoofable contract), got %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetSTSLDAPLoginSourceIPTrustedProxyFallsBackToPeerWithoutForwardingHeaders(t *testing.T) {
|
||||
withLDAPSTSTrustedProxiesForTest(t, "192.0.2.0/24", func() {
|
||||
req := &http.Request{RemoteAddr: "192.0.2.10:9000"}
|
||||
@@ -1890,20 +2045,6 @@ func TestLDAPBindErrorToSTS(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSTSLDAPLoginRateLimiterUsernameNormalization(t *testing.T) {
|
||||
limiter := newSTSLDAPLoginRateLimiter(time.Hour, 2, time.Minute)
|
||||
|
||||
if !limiter.Allow("192.0.2.10", "Admin") {
|
||||
t.Fatal("expected first username variant to be allowed")
|
||||
}
|
||||
if !limiter.Allow("192.0.2.11", " admin ") {
|
||||
t.Fatal("expected trimmed lowercase-equivalent username to be allowed")
|
||||
}
|
||||
if limiter.Allow("192.0.2.12", "ADMIN") {
|
||||
t.Fatal("expected username normalization to hit the same bucket")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSTSLDAPLoginRateLimiterCleanup(t *testing.T) {
|
||||
set := newSTSLDAPLoginKeyLimiterSet(time.Hour, 1, time.Minute)
|
||||
start := time.Unix(0, 0)
|
||||
@@ -1924,9 +2065,7 @@ func TestSTSLDAPLoginRateLimiterCleanup(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestWriteSTSThrottledResponse(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodPost, "http://minio.test", strings.NewReader(""))
|
||||
rr := httptest.NewRecorder()
|
||||
req = req.WithContext(newContext(req, rr, "test-throttle"))
|
||||
|
||||
writeSTSThrottledResponse(rr)
|
||||
|
||||
|
||||
@@ -30,8 +30,9 @@ const _STSErrorCode_name = "STSNoneSTSAccessDeniedSTSMissingParameterSTSInvalidP
|
||||
var _STSErrorCode_index = [...]uint16{0, 7, 22, 41, 65, 91, 118, 145, 171, 192, 219, 244, 261, 281, 297, 313}
|
||||
|
||||
func (i STSErrorCode) String() string {
|
||||
if i < 0 || i >= STSErrorCode(len(_STSErrorCode_index)-1) {
|
||||
idx := int(i) - 0
|
||||
if i < 0 || idx >= len(_STSErrorCode_index)-1 {
|
||||
return "STSErrorCode(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _STSErrorCode_name[_STSErrorCode_index[i]:_STSErrorCode_index[i+1]]
|
||||
return _STSErrorCode_name[_STSErrorCode_index[idx]:_STSErrorCode_index[idx+1]]
|
||||
}
|
||||
|
||||
+10
-15
@@ -19,6 +19,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto"
|
||||
"crypto/tls"
|
||||
"encoding/hex"
|
||||
@@ -42,7 +43,6 @@ import (
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
"github.com/minio/selfupdate"
|
||||
gopsutilcpu "github.com/shirou/gopsutil/v3/cpu"
|
||||
"github.com/valyala/bytebufferpool"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -450,10 +450,10 @@ func getLatestReleaseTime(u *url.URL, timeout time.Duration, mode string) (sha25
|
||||
|
||||
const (
|
||||
// Kubernetes deployment doc link.
|
||||
kubernetesDeploymentDoc = "https://docs.min.io/community/minio-object-store/operations/deployments/kubernetes.html"
|
||||
kubernetesDeploymentDoc = "https://silo.pgsty.com/operations/deployments/kubernetes/"
|
||||
|
||||
// Mesos deployment doc link.
|
||||
mesosDeploymentDoc = "https://docs.min.io/community/minio-object-store/operations/deployments/kubernetes.html"
|
||||
mesosDeploymentDoc = "https://silo.pgsty.com/operations/deployments/kubernetes/"
|
||||
)
|
||||
|
||||
func getDownloadURL(releaseTag string) (downloadURL string) {
|
||||
@@ -532,26 +532,21 @@ func downloadBinary(u *url.URL, mode string) (binCompressed []byte, bin []byte,
|
||||
}
|
||||
defer xhttp.DrainBody(reader)
|
||||
|
||||
b := bytebufferpool.Get()
|
||||
bc := bytebufferpool.Get()
|
||||
defer func() {
|
||||
b.Reset()
|
||||
bc.Reset()
|
||||
var b, bc bytes.Buffer
|
||||
|
||||
bytebufferpool.Put(b)
|
||||
bytebufferpool.Put(bc)
|
||||
}()
|
||||
|
||||
w, err := zstd.NewWriter(bc)
|
||||
w, err := zstd.NewWriter(&bc)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if _, err = io.Copy(w, io.TeeReader(reader, b)); err != nil {
|
||||
if _, err = io.Copy(w, io.TeeReader(reader, &b)); err != nil {
|
||||
_ = w.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
w.Close()
|
||||
if err = w.Close(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return bc.Bytes(), b.Bytes(), nil
|
||||
}
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/http"
|
||||
@@ -28,8 +29,73 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/valyala/bytebufferpool"
|
||||
)
|
||||
|
||||
func TestDownloadBinaryReturnsOwnedBuffers(t *testing.T) {
|
||||
previousMaxProcs := runtime.GOMAXPROCS(1)
|
||||
t.Cleanup(func() {
|
||||
runtime.GOMAXPROCS(previousMaxProcs)
|
||||
})
|
||||
|
||||
payload := []byte("minio update payload")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write(payload)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
u, err := url.Parse(server.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compressed, downloaded, err := downloadBinary(u, "server")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(downloaded, payload) {
|
||||
t.Fatalf("downloaded binary is %q, want %q", downloaded, payload)
|
||||
}
|
||||
decoder, err := zstd.NewReader(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(decoder.Close)
|
||||
decompressed, err := decoder.DecodeAll(compressed, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(decompressed, payload) {
|
||||
t.Fatalf("decompressed binary is %q, want %q", decompressed, payload)
|
||||
}
|
||||
wantCompressed := bytes.Clone(compressed)
|
||||
wantDownloaded := bytes.Clone(downloaded)
|
||||
|
||||
// Reuse buffers returned to bytebufferpool. downloadBinary's results must
|
||||
// remain valid after the function returns, regardless of later pool users.
|
||||
pooled := make([]*bytebufferpool.ByteBuffer, 8)
|
||||
for i := range pooled {
|
||||
pooled[i] = bytebufferpool.Get()
|
||||
if cap(pooled[i].B) > 0 {
|
||||
pooled[i].B = pooled[i].B[:cap(pooled[i].B)]
|
||||
for j := range pooled[i].B {
|
||||
pooled[i].B[j] = 0xa5
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, b := range pooled {
|
||||
bytebufferpool.Put(b)
|
||||
}
|
||||
|
||||
if !bytes.Equal(compressed, wantCompressed) {
|
||||
t.Fatal("compressed download aliases a buffer returned to bytebufferpool")
|
||||
}
|
||||
if !bytes.Equal(downloaded, wantDownloaded) {
|
||||
t.Fatal("downloaded binary aliases a buffer returned to bytebufferpool")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMinioVersionToReleaseTime(t *testing.T) {
|
||||
testCases := []struct {
|
||||
version string
|
||||
|
||||
@@ -68,7 +68,6 @@ const (
|
||||
storageMetricReadXL
|
||||
storageMetricReadAll
|
||||
storageMetricStatInfoFile
|
||||
storageMetricReadMultiple
|
||||
storageMetricDeleteAbandonedParts
|
||||
storageMetricDiskInfo
|
||||
storageMetricDeleteBulk
|
||||
@@ -714,7 +713,12 @@ func (p *xlStorageDiskIDCheck) StatInfoFile(ctx context.Context, volume, path st
|
||||
}
|
||||
|
||||
func (p *xlStorageDiskIDCheck) ReadParts(ctx context.Context, volume string, partMetaPaths ...string) ([]*ObjectPartInfo, error) {
|
||||
ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadParts, volume, path.Dir(partMetaPaths[0]))
|
||||
// Merely for tracing storage
|
||||
partPath := ""
|
||||
if len(partMetaPaths) > 0 {
|
||||
partPath = path.Dir(partMetaPaths[0])
|
||||
}
|
||||
ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadParts, volume, partPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -723,21 +727,6 @@ func (p *xlStorageDiskIDCheck) ReadParts(ctx context.Context, volume string, par
|
||||
return p.storage.ReadParts(ctx, volume, partMetaPaths...)
|
||||
}
|
||||
|
||||
// ReadMultiple will read multiple files and send each files as response.
|
||||
// Files are read and returned in the given order.
|
||||
// The resp channel is closed before the call returns.
|
||||
// Only a canceled context will return an error.
|
||||
func (p *xlStorageDiskIDCheck) ReadMultiple(ctx context.Context, req ReadMultipleReq, resp chan<- ReadMultipleResp) (err error) {
|
||||
ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadMultiple, req.Bucket, req.Prefix)
|
||||
if err != nil {
|
||||
xioutil.SafeClose(resp)
|
||||
return err
|
||||
}
|
||||
defer done(0, &err)
|
||||
|
||||
return p.storage.ReadMultiple(ctx, req, resp)
|
||||
}
|
||||
|
||||
// CleanAbandonedData will read metadata of the object on disk
|
||||
// and delete any data directories and inline data that isn't referenced in metadata.
|
||||
func (p *xlStorageDiskIDCheck) CleanAbandonedData(ctx context.Context, volume string, path string) (err error) {
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/tinylib/msgp/msgp"
|
||||
)
|
||||
|
||||
// TestReadPartsEmptyPathList covers the trace path the health decorator builds
|
||||
// before it delegates.
|
||||
//
|
||||
// ReadParts took partMetaPaths[0] unconditionally, so a caller passing no paths
|
||||
// at all indexed an empty slice. xlStorage.ReadParts itself handles an empty
|
||||
// list perfectly well - it returns an empty result - so the panic came purely
|
||||
// from the metrics bookkeeping wrapped around it.
|
||||
func TestReadPartsEmptyPathList(t *testing.T) {
|
||||
disk, _, err := newXLStorageTestSetup(t)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create test setup: %v", err)
|
||||
}
|
||||
if err := disk.MakeVol(t.Context(), "foo"); err != nil {
|
||||
t.Fatalf("MakeVol: %v", err)
|
||||
}
|
||||
|
||||
parts, err := disk.ReadParts(t.Context(), "foo")
|
||||
if err != nil {
|
||||
t.Fatalf("empty part list returned an error: %v", err)
|
||||
}
|
||||
if len(parts) != 0 {
|
||||
t.Fatalf("empty part list returned %d parts, want 0", len(parts))
|
||||
}
|
||||
}
|
||||
|
||||
// TestReadPartsEmptyPathListDoesNotLeakKeepAlive covers what the panic actually
|
||||
// cost a running node.
|
||||
//
|
||||
// ReadPartsHandler calls keepHTTPResponseAlive before it calls ReadParts. That
|
||||
// helper spawns a goroutine whose only exit is receiving from the channel that
|
||||
// done() writes. Panicking in between skips both done(err) and done(nil), so
|
||||
// net/http recovering the handler goroutine still leaves the keep-alive
|
||||
// goroutine and its 10-second ticker parked forever - one per request, driven
|
||||
// by a request body an authenticated peer fully controls.
|
||||
func TestReadPartsEmptyPathListDoesNotLeakKeepAlive(t *testing.T) {
|
||||
newStorageRESTHTTPServerClient(t)
|
||||
server := &storageRESTServer{endpoint: globalLocalSetDrives[0][0][0].Endpoint()}
|
||||
|
||||
// Recovering here mirrors net/http, which recovers a panicking handler and
|
||||
// keeps serving. That recovery is exactly why the defect is a leak rather
|
||||
// than a crash: the process survives, and the parked goroutine survives
|
||||
// with it.
|
||||
call := func() (panicked bool) {
|
||||
defer func() { panicked = recover() != nil }()
|
||||
|
||||
var body bytes.Buffer
|
||||
if err := msgp.Encode(&body, &ReadPartsReq{}); err != nil {
|
||||
t.Fatalf("encoding an empty ReadPartsReq: %v", err)
|
||||
}
|
||||
u := "/?" + url.Values{storageRESTVolume: []string{"foo"}}.Encode()
|
||||
req := httptest.NewRequest(http.MethodPost, u, bytes.NewReader(body.Bytes()))
|
||||
req.Header.Set("Authorization", "Bearer "+globalNodeAuthToken)
|
||||
req.Header.Set("X-Minio-Time", strconv.FormatInt(time.Now().UnixNano(), 10))
|
||||
w := httptest.NewRecorder()
|
||||
server.ReadPartsHandler(w, req)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("empty-path ReadParts: status %d, body %q", w.Code, w.Body.String())
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
baseline := settleKeepAliveGoroutines(0, time.Second)
|
||||
|
||||
const requests = 20
|
||||
panicked := false
|
||||
for range requests {
|
||||
panicked = call() || panicked
|
||||
}
|
||||
if panicked {
|
||||
t.Error("ReadPartsHandler panicked on an empty path list")
|
||||
}
|
||||
|
||||
// One leak per request, so anything above the baseline is the defect.
|
||||
if got := settleKeepAliveGoroutines(baseline, 5*time.Second); got > baseline {
|
||||
t.Errorf("%d empty-path requests left %d keepHTTPResponseAlive goroutines parked "+
|
||||
"(baseline %d) - done() is never reached, so each request strands one "+
|
||||
"goroutine and its ticker", requests, got, baseline)
|
||||
}
|
||||
}
|
||||
|
||||
// settleKeepAliveGoroutines counts goroutines parked inside
|
||||
// keepHTTPResponseAlive, sampling until the count falls to at most limit or the
|
||||
// deadline passes, and returns the final sample.
|
||||
//
|
||||
// Counting this one frame rather than runtime.NumGoroutine keeps the assertion
|
||||
// meaningful when the whole cmd package runs and unrelated background
|
||||
// goroutines come and go.
|
||||
func settleKeepAliveGoroutines(limit int, wait time.Duration) int {
|
||||
deadline := time.Now().Add(wait)
|
||||
for {
|
||||
n := countGoroutinesCreatedBy("github.com/minio/minio/cmd.keepHTTPResponseAlive")
|
||||
if n <= limit || !time.Now().Before(deadline) {
|
||||
return n
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// countGoroutinesCreatedBy reports how many live goroutines were started by fn.
|
||||
// Matching the "created by" line counts each goroutine once; the function name
|
||||
// on its own also appears in the running frame of every goroutine it started.
|
||||
func countGoroutinesCreatedBy(fn string) int {
|
||||
needle := []byte("created by " + fn)
|
||||
buf := make([]byte, 1<<20)
|
||||
for {
|
||||
if n := runtime.Stack(buf, true); n < len(buf) {
|
||||
return bytes.Count(buf[:n], needle)
|
||||
}
|
||||
buf = make([]byte, 2*len(buf))
|
||||
}
|
||||
}
|
||||
@@ -1590,6 +1590,13 @@ func (x *xlMetaV2) UpdateObjectVersion(fi FileInfo) error {
|
||||
|
||||
// AddVersion adds a new version
|
||||
func (x *xlMetaV2) AddVersion(fi FileInfo) error {
|
||||
// Refuse to persist metadata no shard size can be derived from. This is the
|
||||
// single funnel every version write passes through, so rejecting here keeps
|
||||
// the poison off disk rather than relying on every reader to cope with it.
|
||||
if fi.HasNegativePartSize() {
|
||||
return errFileCorrupt
|
||||
}
|
||||
|
||||
if fi.VersionID == "" {
|
||||
// this means versioning is not yet
|
||||
// enabled or suspend i.e all versions
|
||||
|
||||
@@ -20,10 +20,11 @@ const _VersionType_name = "invalidVersionTypeObjectTypeDeleteTypeLegacyTypelastV
|
||||
var _VersionType_index = [...]uint8{0, 18, 28, 38, 48, 63}
|
||||
|
||||
func (i VersionType) String() string {
|
||||
if i >= VersionType(len(_VersionType_index)-1) {
|
||||
idx := int(i) - 0
|
||||
if i < 0 || idx >= len(_VersionType_index)-1 {
|
||||
return "VersionType(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _VersionType_name[_VersionType_index[i]:_VersionType_index[i+1]]
|
||||
return _VersionType_name[_VersionType_index[idx]:_VersionType_index[idx+1]]
|
||||
}
|
||||
func _() {
|
||||
// An "invalid array index" compiler error signifies that the constant values have changed.
|
||||
@@ -39,8 +40,9 @@ const _ErasureAlgo_name = "invalidErasureAlgoReedSolomonlastErasureAlgo"
|
||||
var _ErasureAlgo_index = [...]uint8{0, 18, 29, 44}
|
||||
|
||||
func (i ErasureAlgo) String() string {
|
||||
if i >= ErasureAlgo(len(_ErasureAlgo_index)-1) {
|
||||
idx := int(i) - 0
|
||||
if i < 0 || idx >= len(_ErasureAlgo_index)-1 {
|
||||
return "ErasureAlgo(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _ErasureAlgo_name[_ErasureAlgo_index[i]:_ErasureAlgo_index[i+1]]
|
||||
return _ErasureAlgo_name[_ErasureAlgo_index[idx]:_ErasureAlgo_index[idx+1]]
|
||||
}
|
||||
|
||||
+28
-75
@@ -788,6 +788,22 @@ func (s *xlStorage) getVolDir(volume string) (string, error) {
|
||||
if volume == "" || volume == "." || volume == ".." {
|
||||
return "", errVolumeNotFound
|
||||
}
|
||||
// Reject traversal smuggled inside the volume name itself, e.g. "../" or
|
||||
// "..\", which the equality checks above do not catch.
|
||||
//
|
||||
// This must be evaluated on the raw argument and never on the joined
|
||||
// result: pathJoin() below runs path.Clean() against an absolute
|
||||
// drivePath, and Clean() *erases* leading ".." on an absolute path
|
||||
// ("/drive/../../etc" becomes "/etc"), so a check placed after the join
|
||||
// would silently accept an escaped path.
|
||||
//
|
||||
// This is also the only containment control covering callers that never
|
||||
// reach storageRESTServer.getStorage() - notably the peer-S3 bucket RPCs
|
||||
// (MakeBucket/HeadBucket/DeleteBucket/HealBucket), which drive
|
||||
// MakeVol/StatVol/DeleteVol straight off globalLocalDrivesMap.
|
||||
if hasBadPathComponent(volume) {
|
||||
return "", errVolumeNotFound
|
||||
}
|
||||
volumeDir := pathJoin(s.drivePath, volume)
|
||||
return volumeDir, nil
|
||||
}
|
||||
@@ -2407,6 +2423,12 @@ func (s *xlStorage) CheckParts(ctx context.Context, volume string, path string,
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Already-persisted metadata can carry this even though the boundary now
|
||||
// refuses it, so the check belongs here too, not only at the wire edge.
|
||||
if fi.HasNegativePartSize() {
|
||||
return nil, errFileCorrupt
|
||||
}
|
||||
|
||||
resp := CheckPartsResp{
|
||||
// By default, all results have an unknown status
|
||||
Results: make([]int, len(fi.Parts)),
|
||||
@@ -3110,6 +3132,12 @@ func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi File
|
||||
}
|
||||
}
|
||||
|
||||
// See CheckParts: metadata already on disk can carry this even though the
|
||||
// boundary now refuses it.
|
||||
if fi.HasNegativePartSize() {
|
||||
return nil, errFileCorrupt
|
||||
}
|
||||
|
||||
resp := CheckPartsResp{
|
||||
// By default, the result is unknown per part
|
||||
Results: make([]int, len(fi.Parts)),
|
||||
@@ -3187,81 +3215,6 @@ func (s *xlStorage) ReadParts(ctx context.Context, volume string, partMetaPaths
|
||||
return parts, nil
|
||||
}
|
||||
|
||||
// ReadMultiple will read multiple files and send each back as response.
|
||||
// Files are read and returned in the given order.
|
||||
// The resp channel is closed before the call returns.
|
||||
// Only a canceled context will return an error.
|
||||
func (s *xlStorage) ReadMultiple(ctx context.Context, req ReadMultipleReq, resp chan<- ReadMultipleResp) error {
|
||||
defer xioutil.SafeClose(resp)
|
||||
|
||||
volumeDir := pathJoin(s.drivePath, req.Bucket)
|
||||
found := 0
|
||||
for _, f := range req.Files {
|
||||
if contextCanceled(ctx) {
|
||||
return ctx.Err()
|
||||
}
|
||||
r := ReadMultipleResp{
|
||||
Bucket: req.Bucket,
|
||||
Prefix: req.Prefix,
|
||||
File: f,
|
||||
}
|
||||
|
||||
var data []byte
|
||||
var mt time.Time
|
||||
|
||||
fullPath := pathJoin(volumeDir, req.Prefix, f)
|
||||
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
|
||||
if err := w.Run(func() (err error) {
|
||||
if req.MetadataOnly {
|
||||
data, mt, err = s.readMetadataWithDMTime(ctx, fullPath)
|
||||
} else {
|
||||
data, mt, err = s.readAllDataWithDMTime(ctx, req.Bucket, volumeDir, fullPath)
|
||||
}
|
||||
return err
|
||||
}); err != nil {
|
||||
if !IsErr(err, errFileNotFound, errVolumeNotFound) {
|
||||
r.Exists = true
|
||||
r.Error = err.Error()
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case resp <- r:
|
||||
}
|
||||
if req.AbortOn404 && !r.Exists {
|
||||
// We stop at first file not found.
|
||||
// We have already reported the error, return nil.
|
||||
return nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
diskHealthCheckOK(ctx, nil)
|
||||
if req.MaxSize > 0 && int64(len(data)) > req.MaxSize {
|
||||
r.Exists = true
|
||||
r.Error = fmt.Sprintf("max size (%d) exceeded: %d", req.MaxSize, len(data))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case resp <- r:
|
||||
continue
|
||||
}
|
||||
}
|
||||
found++
|
||||
r.Exists = true
|
||||
r.Data = data
|
||||
r.Modtime = mt
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case resp <- r:
|
||||
}
|
||||
if req.MaxResults > 0 && found >= req.MaxResults {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *xlStorage) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
|
||||
volumeDir, err := s.getVolDir(volume)
|
||||
if err != nil {
|
||||
|
||||
@@ -11,11 +11,11 @@ fi
|
||||
docker_switch_user() {
|
||||
if [ -n "${MINIO_USERNAME}" ] && [ -n "${MINIO_GROUPNAME}" ]; then
|
||||
if [ -n "${MINIO_UID}" ] && [ -n "${MINIO_GID}" ]; then
|
||||
chroot --userspec=${MINIO_UID}:${MINIO_GID} / "$@"
|
||||
exec chroot --userspec=${MINIO_UID}:${MINIO_GID} / "$@"
|
||||
else
|
||||
echo "${MINIO_USERNAME}:x:1000:1000:${MINIO_USERNAME}:/:/sbin/nologin" >>/etc/passwd
|
||||
echo "${MINIO_GROUPNAME}:x:1000" >>/etc/group
|
||||
chroot --userspec=${MINIO_USERNAME}:${MINIO_GROUPNAME} / "$@"
|
||||
exec chroot --userspec=${MINIO_USERNAME}:${MINIO_GROUPNAME} / "$@"
|
||||
fi
|
||||
else
|
||||
exec "$@"
|
||||
|
||||
@@ -16,7 +16,7 @@ MinIO also supports multi-cluster, multi-site federation similar to AWS regions
|
||||
- [Setup Ambari](https://docs.hortonworks.com/HDPDocuments/Ambari-2.7.1.0/bk_ambari-installation/content/set_up_the_ambari_server.html) which automatically sets up YARN
|
||||
- [Installing Spark](https://docs.hortonworks.com/HDPDocuments/HDP3/HDP-3.0.1/installing-spark/content/installing_spark.html)
|
||||
- Install MinIO Distributed Server using one of the guides below.
|
||||
- [Deployment based on Kubernetes](https://docs.min.io/community/minio-object-store/operations/deployments/kubernetes.html)
|
||||
- [Deployment based on Kubernetes](https://silo.pgsty.com/operations/deployments/kubernetes/)
|
||||
- [Deployment based on MinIO Helm Chart](https://github.com/helm/charts/tree/master/stable/minio)
|
||||
|
||||
## **3. Configure Hadoop, Spark, Hive to use MinIO**
|
||||
|
||||
@@ -51,5 +51,5 @@ Tiering and lifecycle transition are applicable only to erasure/distributed MinI
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [MinIO | Golang Client API Reference](https://docs.min.io/community/minio-object-store/developers/go/API.html#SetBucketLifecycle)
|
||||
- [MinIO | Golang Client API Reference](https://pkg.go.dev/github.com/minio/minio-go/v7#Client.SetBucketLifecycle)
|
||||
- [Object Lifecycle Management](https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html)
|
||||
|
||||
@@ -4,8 +4,8 @@ Enable object lifecycle configuration on buckets to setup automatic deletion of
|
||||
|
||||
## 1. Prerequisites
|
||||
|
||||
- Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html).
|
||||
- Install `mc` - [mc Quickstart Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart)
|
||||
- Install MinIO - [MinIO Quickstart Guide](https://silo.pgsty.com/operations/deployments/baremetal-deploy-minio-on-redhat-linux/).
|
||||
- Install `mc` - [mc Quickstart Guide](https://silo.pgsty.com/reference/minio-mc/#quickstart)
|
||||
|
||||
## 2. Enable bucket lifecycle configuration
|
||||
|
||||
@@ -59,7 +59,7 @@ TempUploads | temp/ | ✓ | ✓ | 7 day(s) | ✗
|
||||
|
||||
## 3. Activate ILM versioning features
|
||||
|
||||
This will only work with a versioned bucket, take a look at [Bucket Versioning Guide](https://docs.min.io/community/minio-object-store/administration/object-management/object-versioning.html) for more understanding.
|
||||
This will only work with a versioned bucket, take a look at [Bucket Versioning Guide](https://silo.pgsty.com/administration/object-management/object-versioning/) for more understanding.
|
||||
|
||||
### 3.1 Automatic removal of non current objects versions
|
||||
|
||||
@@ -228,5 +228,5 @@ Note that transition event notification is a MinIO extension.
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [MinIO | Golang Client API Reference](https://docs.min.io/community/minio-object-store/developers/go/API.html)
|
||||
- [MinIO | Golang Client API Reference](https://pkg.go.dev/github.com/minio/minio-go/v7)
|
||||
- [Object Lifecycle Management](https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html)
|
||||
|
||||
@@ -30,7 +30,7 @@ Various event types supported by MinIO server are
|
||||
| `s3:BucketCreated` |
|
||||
| `s3:BucketRemoved` |
|
||||
|
||||
Use client tools like `mc` to set and listen for event notifications using the [`event` sub-command](https://docs.min.io/community/minio-object-store/reference/minio-mc/mc-event-add.html). MinIO SDK's [`BucketNotification` APIs](https://docs.min.io/community/minio-object-store/developers/go/API.html#setbucketnotification-ctx-context-context-bucketname-string-config-notification-configuration-error) can also be used. The notification message MinIO sends to publish an event is a JSON message with the following [structure](https://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html).
|
||||
Use client tools like `mc` to set and listen for event notifications using the [`event` sub-command](https://silo.pgsty.com/reference/minio-mc/mc-event-add/). MinIO SDK's [`BucketNotification` APIs](https://pkg.go.dev/github.com/minio/minio-go/v7#Client.SetBucketNotification) can also be used. The notification message MinIO sends to publish an event is a JSON message with the following [structure](https://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html).
|
||||
|
||||
Bucket events can be published to the following targets:
|
||||
|
||||
@@ -43,8 +43,8 @@ Bucket events can be published to the following targets:
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Install and configure MinIO Server from [here](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html#procedure).
|
||||
- Install and configure MinIO Client from [here](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart).
|
||||
- Install and configure MinIO Server from [here](https://silo.pgsty.com/operations/deployments/baremetal-deploy-minio-on-redhat-linux/#procedure).
|
||||
- Install and configure MinIO Client from [here](https://silo.pgsty.com/reference/minio-mc/#quickstart).
|
||||
|
||||
```
|
||||
$ mc admin config get myminio | grep notify
|
||||
|
||||
@@ -6,8 +6,8 @@ Buckets can be configured to have `Hard` quota - it disallows writes to the buck
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html#procedure).
|
||||
- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart)
|
||||
- Install MinIO - [MinIO Quickstart Guide](https://silo.pgsty.com/operations/deployments/baremetal-deploy-minio-on-redhat-linux/#procedure).
|
||||
- [Use `mc` with MinIO Server](https://silo.pgsty.com/reference/minio-mc/#quickstart)
|
||||
|
||||
## Set bucket quota configuration
|
||||
|
||||
|
||||
@@ -156,5 +156,5 @@ If 3 or more targets are participating in active-active replication, the replica
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [MinIO Bucket Versioning Implementation](https://docs.min.io/community/minio-object-store/administration/object-management/object-versioning.html)
|
||||
- [MinIO Client Quickstart Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart)
|
||||
- [MinIO Bucket Versioning Implementation](https://silo.pgsty.com/administration/object-management/object-versioning/)
|
||||
- [MinIO Client Quickstart Guide](https://silo.pgsty.com/reference/minio-mc/#quickstart)
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
Bucket replication is designed to replicate selected objects in a bucket to a destination bucket.
|
||||
|
||||
The contents of this page have been migrated to the new [MinIO Documentation: Bucket Replication](https://docs.min.io/community/minio-object-store/administration/bucket-replication.html) page. The [Bucket Replication](https://docs.min.io/community/minio-object-store/administration/bucket-replication/bucket-replication-requirements.html) page references dedicated tutorials for configuring one-way "Active-Passive" and two-way "Active-Active" bucket replication.
|
||||
The contents of this page have been migrated to the new [MinIO Documentation: Bucket Replication](https://silo.pgsty.com/administration/bucket-replication/) page. The [Bucket Replication](https://silo.pgsty.com/administration/bucket-replication/bucket-replication-requirements/) page references dedicated tutorials for configuring one-way "Active-Passive" and two-way "Active-Active" bucket replication.
|
||||
|
||||
To replicate objects in a bucket to a destination bucket on a target site either in the same cluster or a different cluster, start by enabling [versioning](https://docs.min.io/community/minio-object-store/administration/object-management/object-versioning.html) for both source and destination buckets. Finally, the target site and the destination bucket need to be configured on the source MinIO server.
|
||||
To replicate objects in a bucket to a destination bucket on a target site either in the same cluster or a different cluster, start by enabling [versioning](https://silo.pgsty.com/administration/object-management/object-versioning/) for both source and destination buckets. Finally, the target site and the destination bucket need to be configured on the source MinIO server.
|
||||
|
||||
## Highlights
|
||||
|
||||
@@ -155,7 +155,7 @@ The replication configuration generated has the following format and can be expo
|
||||
|
||||
The replication configuration follows [AWS S3 Spec](https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html). Any objects uploaded to the source bucket that meet replication criteria will now be automatically replicated by the MinIO server to the remote destination bucket. Replication can be disabled at any time by disabling specific rules in the configuration or deleting the replication configuration entirely.
|
||||
|
||||
When object locking is used in conjunction with replication, both source and destination buckets needs to have [object locking](https://docs.min.io/community/minio-object-store/administration/object-management/object-retention.html) enabled. Similarly objects encrypted on the server side, will be replicated if destination also supports encryption.
|
||||
When object locking is used in conjunction with replication, both source and destination buckets needs to have [object locking](https://silo.pgsty.com/administration/object-management/object-retention/) enabled. Similarly objects encrypted on the server side, will be replicated if destination also supports encryption.
|
||||
|
||||
Replication status can be seen in the metadata on the source and destination objects. On the source side, the `X-Amz-Replication-Status` changes from `PENDING` to `COMPLETED` or `FAILED` after replication attempt either succeeded or failed respectively. On the destination side, a `X-Amz-Replication-Status` status of `REPLICA` indicates that the object was replicated successfully. Any replication failures are automatically re-attempted during a periodic disk scanner cycle.
|
||||
|
||||
@@ -277,5 +277,5 @@ MinIO does not support SSE-C encrypted objects on replicated buckets, any applic
|
||||
## Explore Further
|
||||
|
||||
- [MinIO Bucket Replication Design](https://github.com/pgsty/minio/blob/master/docs/bucket/replication/DESIGN.md)
|
||||
- [MinIO Bucket Versioning Implementation](https://docs.min.io/community/minio-object-store/administration/object-management/object-retention.html)
|
||||
- [MinIO Client Quickstart Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart)
|
||||
- [MinIO Bucket Versioning Implementation](https://silo.pgsty.com/administration/object-management/object-retention/)
|
||||
- [MinIO Client Quickstart Guide](https://silo.pgsty.com/reference/minio-mc/#quickstart)
|
||||
|
||||
@@ -10,7 +10,7 @@ A default retention period and retention mode can be configured on a bucket to b
|
||||
|
||||
### 1. Prerequisites
|
||||
|
||||
- Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html)
|
||||
- Install MinIO - [MinIO Quickstart Guide](https://silo.pgsty.com/operations/deployments/baremetal-deploy-minio-on-redhat-linux/)
|
||||
- Install `awscli` - [Installing AWS Command Line Interface](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html)
|
||||
|
||||
### 2. Set bucket WORM configuration
|
||||
@@ -53,7 +53,7 @@ See <https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html>
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart)
|
||||
- [Use `aws-cli` with MinIO Server](https://docs.min.io/community/minio-object-store/integrations/aws-cli-with-minio.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html)
|
||||
- [Use `mc` with MinIO Server](https://silo.pgsty.com/reference/minio-mc/#quickstart)
|
||||
- [Use `aws-cli` with MinIO Server](https://silo.pgsty.com/integrations/aws-cli-with-minio/)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://silo.pgsty.com/developers/go/minio-go/)
|
||||
- [The MinIO documentation website](https://silo.pgsty.com/docs/)
|
||||
|
||||
@@ -211,7 +211,7 @@ public class IsVersioningEnabled {
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [Use `minio-java` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/java/minio-java.html)
|
||||
- [Object Lock and Immutability Guide](https://docs.min.io/community/minio-object-store/administration/object-management/object-retention.html)
|
||||
- [MinIO Admin Complete Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc-admin.html)
|
||||
- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html)
|
||||
- [Use `minio-java` SDK with MinIO Server](https://silo.pgsty.com/developers/java/minio-java/)
|
||||
- [Object Lock and Immutability Guide](https://silo.pgsty.com/administration/object-management/object-retention/)
|
||||
- [MinIO Admin Complete Guide](https://silo.pgsty.com/reference/minio-mc-admin/)
|
||||
- [The MinIO documentation website](https://silo.pgsty.com/docs/)
|
||||
|
||||
@@ -51,8 +51,8 @@ Instance is now accessible on the host at port 9000, proceed to access the Web b
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [MinIO Erasure Code Overview](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html)
|
||||
- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html)
|
||||
- [Use `aws-cli` with MinIO Server](https://docs.min.io/community/minio-object-store/integrations/aws-cli-with-minio.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html)
|
||||
- [MinIO Erasure Code Overview](https://silo.pgsty.com/operations/concepts/erasure-coding/)
|
||||
- [Use `mc` with MinIO Server](https://silo.pgsty.com/reference/minio-mc/)
|
||||
- [Use `aws-cli` with MinIO Server](https://silo.pgsty.com/integrations/aws-cli-with-minio/)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://silo.pgsty.com/developers/go/minio-go/)
|
||||
- [The MinIO documentation website](https://silo.pgsty.com/docs/)
|
||||
|
||||
@@ -19,7 +19,7 @@ will increase speed when the content can be compressed.
|
||||
|
||||
### 1. Prerequisites
|
||||
|
||||
Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html).
|
||||
Install MinIO - [MinIO Quickstart Guide](https://silo.pgsty.com/operations/deployments/baremetal-deploy-minio-on-redhat-linux/).
|
||||
|
||||
### 2. Run MinIO with compression
|
||||
|
||||
@@ -131,7 +131,7 @@ the data directory to view the size of the object.
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html)
|
||||
- [Use `aws-cli` with MinIO Server](https://docs.min.io/community/minio-object-store/integrations/aws-cli-with-minio.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html)
|
||||
- [Use `mc` with MinIO Server](https://silo.pgsty.com/reference/minio-mc/)
|
||||
- [Use `aws-cli` with MinIO Server](https://silo.pgsty.com/integrations/aws-cli-with-minio/)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://silo.pgsty.com/developers/go/minio-go/)
|
||||
- [The MinIO documentation website](https://silo.pgsty.com/docs/)
|
||||
|
||||
@@ -6,7 +6,7 @@ MinIO stores all its config as part of the server deployment, config is erasure
|
||||
|
||||
### Certificate Directory
|
||||
|
||||
TLS certificates by default are expected to be stored under ``${HOME}/.minio/certs`` directory. You need to place certificates here to enable `HTTPS` based access. Read more about [How to secure access to MinIO server with TLS](https://docs.min.io/community/minio-object-store/operations/network-encryption.html).
|
||||
TLS certificates by default are expected to be stored under ``${HOME}/.minio/certs`` directory. You need to place certificates here to enable `HTTPS` based access. Read more about [How to secure access to MinIO server with TLS](https://silo.pgsty.com/operations/network-encryption/).
|
||||
|
||||
Following is a sample directory structure for MinIO server with TLS certificates.
|
||||
|
||||
@@ -172,7 +172,7 @@ MINIO_API_OBJECT_MAX_VERSIONS (number) set max allowed number of
|
||||
|
||||
#### Notifications
|
||||
|
||||
Notification targets supported by MinIO are in the following list. To configure individual targets please refer to more detailed documentation [here](https://docs.min.io/community/minio-object-store/administration/monitoring.html#bucket-notifications).
|
||||
Notification targets supported by MinIO are in the following list. To configure individual targets please refer to more detailed documentation [here](https://silo.pgsty.com/administration/monitoring/#bucket-notifications).
|
||||
|
||||
```
|
||||
notify_webhook publish bucket notifications to webhook endpoints
|
||||
@@ -336,5 +336,5 @@ minio server /data
|
||||
|
||||
## Explore Further
|
||||
|
||||
* [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html)
|
||||
* [Configure MinIO Server with TLS](https://docs.min.io/community/minio-object-store/operations/network-encryption.html)
|
||||
* [MinIO Quickstart Guide](https://silo.pgsty.com/operations/deployments/baremetal-deploy-minio-on-redhat-linux/)
|
||||
* [Configure MinIO Server with TLS](https://silo.pgsty.com/operations/network-encryption/)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
## HTTP Trace
|
||||
|
||||
HTTP tracing can be enabled by using [`mc admin trace`](https://docs.min.io/community/minio-object-store/reference/minio-mc-admin/mc-admin-trace.html) command.
|
||||
HTTP tracing can be enabled by using [`mc admin trace`](https://silo.pgsty.com/reference/minio-mc-admin/mc-admin-trace/) command.
|
||||
|
||||
Example:
|
||||
|
||||
|
||||
+10
-10
@@ -8,7 +8,7 @@ MinIO in distributed mode can help you setup a highly-available storage system w
|
||||
|
||||
### Data protection
|
||||
|
||||
Distributed MinIO provides protection against multiple node/drive failures and [bit rot](https://github.com/pgsty/minio/blob/master/docs/erasure/README.md#what-is-bit-rot-protection) using [erasure code](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html). As the minimum drives required for distributed MinIO is 2 (same as minimum drives required for erasure coding), erasure code automatically kicks in as you launch distributed MinIO.
|
||||
Distributed MinIO provides protection against multiple node/drive failures and [bit rot](https://github.com/pgsty/minio/blob/master/docs/erasure/README.md#what-is-bit-rot-protection) using [erasure code](https://silo.pgsty.com/operations/concepts/erasure-coding/). As the minimum drives required for distributed MinIO is 2 (same as minimum drives required for erasure coding), erasure code automatically kicks in as you launch distributed MinIO.
|
||||
|
||||
If one or more drives are offline at the start of a PutObject or NewMultipartUpload operation the object will have additional data protection bits added automatically to provide additional safety for these objects.
|
||||
|
||||
@@ -38,11 +38,11 @@ Install MinIO either on Kubernetes or Distributed Linux.
|
||||
|
||||
Install MinIO on Kubernetes:
|
||||
|
||||
- [MinIO Quickstart Guide for Kubernetes](https://docs.min.io/community/minio-object-store/operations/deployments/kubernetes.html).
|
||||
- [Deploy a Tenant from the MinIO Operator](https://docs.min.io/community/minio-object-store/operations/deployments/k8s-deploy-minio-tenant-on-kubernetes.html)
|
||||
- [MinIO Quickstart Guide for Kubernetes](https://silo.pgsty.com/operations/deployments/kubernetes/).
|
||||
- [Deploy a Tenant from the MinIO Operator](https://silo.pgsty.com/operations/deployments/k8s-deploy-minio-tenant-on-kubernetes/)
|
||||
|
||||
Install Distributed MinIO on Linux:
|
||||
- [Deploy Distributed MinIO on Linux](https://docs.min.io/community/minio-object-store/operations/install-deploy-manage/deploy-minio-multi-node-multi-drive.html#deploy-distributed-minio)
|
||||
- [Deploy Distributed MinIO on Linux](https://silo.pgsty.com/operations/deployments/baremetal/#deploy-minio-distributed-baremetal)
|
||||
|
||||
### 2. Run distributed MinIO
|
||||
|
||||
@@ -98,12 +98,12 @@ Now the server has expanded total storage by _(newly_added_servers\*m)_ more dri
|
||||
|
||||
## 3. Test your setup
|
||||
|
||||
To test this setup, access the MinIO server via browser or [`mc`](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart).
|
||||
To test this setup, access the MinIO server via browser or [`mc`](https://silo.pgsty.com/reference/minio-mc/#quickstart).
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [MinIO Erasure Code QuickStart Guide](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html)
|
||||
- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html)
|
||||
- [Use `aws-cli` with MinIO Server](https://docs.min.io/community/minio-object-store/integrations/aws-cli-with-minio.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html)
|
||||
- [MinIO Erasure Code QuickStart Guide](https://silo.pgsty.com/operations/concepts/erasure-coding/)
|
||||
- [Use `mc` with MinIO Server](https://silo.pgsty.com/reference/minio-mc/)
|
||||
- [Use `aws-cli` with MinIO Server](https://silo.pgsty.com/integrations/aws-cli-with-minio/)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://silo.pgsty.com/developers/go/minio-go/)
|
||||
- [The MinIO documentation website](https://silo.pgsty.com/docs/)
|
||||
|
||||
@@ -10,7 +10,7 @@ Docker installed on your machine. Download the relevant installer from [here](ht
|
||||
|
||||
## Run Standalone MinIO on Docker
|
||||
|
||||
*Note*: Standalone MinIO is intended for early development and evaluation. For production clusters, deploy a [Distributed](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-as-a-container.html) MinIO deployment.
|
||||
*Note*: Standalone MinIO is intended for early development and evaluation. For production clusters, deploy a [Distributed](https://silo.pgsty.com/operations/deployments/baremetal-deploy-minio-as-a-container/) MinIO deployment.
|
||||
|
||||
MinIO needs a persistent volume to store configuration and application data. For testing purposes, you can launch MinIO by simply passing a directory (`/data` in the example below). This directory gets created in the container filesystem at the time of container start. But all the data is lost after container exits.
|
||||
|
||||
@@ -59,7 +59,7 @@ docker run \
|
||||
|
||||
We recommend kubernetes based deployment for production level deployment <https://github.com/minio/operator>.
|
||||
|
||||
See the [Kubernetes documentation](https://docs.min.io/community/minio-object-store/operations/deployments/kubernetes.html) for more information.
|
||||
See the [Kubernetes documentation](https://silo.pgsty.com/operations/deployments/kubernetes/) for more information.
|
||||
|
||||
## MinIO Docker Tips
|
||||
|
||||
@@ -213,5 +213,5 @@ docker stats <container_id>
|
||||
|
||||
## Explore Further
|
||||
|
||||
* [MinIO in a Container Installation Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-as-a-container.html)
|
||||
* [MinIO Erasure Code QuickStart Guide](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html)
|
||||
* [MinIO in a Container Installation Guide](https://silo.pgsty.com/operations/deployments/baremetal-deploy-minio-as-a-container/)
|
||||
* [MinIO Erasure Code QuickStart Guide](https://silo.pgsty.com/operations/concepts/erasure-coding/)
|
||||
|
||||
@@ -26,7 +26,7 @@ MinIO's erasure coded backend uses high speed [HighwayHash](https://github.com/m
|
||||
|
||||
MinIO divides the drives you provide into erasure-coding sets of *2 to 16* drives. Therefore, the number of drives you present must be a multiple of one of these numbers. Each object is written to a single erasure-coding set.
|
||||
|
||||
Minio uses the largest possible EC set size which divides into the number of drives given. For example, *18 drives* are configured as *2 sets of 9 drives*, and *24 drives* are configured as *2 sets of 12 drives*. This is true for scenarios when running MinIO as a standalone erasure coded deployment. In [distributed setup however node (affinity) based](https://docs.min.io/community/minio-object-store/operations/install-deploy-manage/deploy-minio-multi-node-multi-drive.html) erasure stripe sizes are chosen.
|
||||
Minio uses the largest possible EC set size which divides into the number of drives given. For example, *18 drives* are configured as *2 sets of 9 drives*, and *24 drives* are configured as *2 sets of 12 drives*. This is true for scenarios when running MinIO as a standalone erasure coded deployment. In [distributed setup however node (affinity) based](https://silo.pgsty.com/operations/deployments/baremetal/) erasure stripe sizes are chosen.
|
||||
|
||||
The drives should all be of approximately the same size.
|
||||
|
||||
@@ -34,7 +34,7 @@ The drives should all be of approximately the same size.
|
||||
|
||||
### 1. Prerequisites
|
||||
|
||||
Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html)
|
||||
Install MinIO - [MinIO Quickstart Guide](https://silo.pgsty.com/operations/deployments/baremetal-deploy-minio-on-redhat-linux/)
|
||||
|
||||
### 2. Run MinIO Server with Erasure Code
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
MinIO server supports storage class in erasure coding mode. This allows configurable data and parity drives per object.
|
||||
|
||||
This page is intended as a summary of MinIO Erasure Coding. For a more complete explanation, see <https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html>.
|
||||
This page is intended as a summary of MinIO Erasure Coding. For a more complete explanation, see <https://silo.pgsty.com/operations/concepts/erasure-coding/>.
|
||||
|
||||
## Overview
|
||||
|
||||
@@ -53,7 +53,7 @@ The default value for the `STANDARD` storage class depends on the number of volu
|
||||
| 6-7 | EC:3 |
|
||||
| 8 or more | EC:4 |
|
||||
|
||||
For more complete documentation on Erasure Set sizing, see the [MinIO Documentation on Erasure Sets](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html#erasure-sets).
|
||||
For more complete documentation on Erasure Set sizing, see the [MinIO Documentation on Erasure Sets](https://silo.pgsty.com/operations/concepts/erasure-coding/#minio-ec-erasure-set).
|
||||
|
||||
### Allowed values for REDUCED_REDUNDANCY storage class
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ This document explains how to configure MinIO with `Bucket lookup from DNS` styl
|
||||
|
||||
### 1. Prerequisites
|
||||
|
||||
Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html).
|
||||
Install MinIO - [MinIO Quickstart Guide](https://silo.pgsty.com/operations/deployments/baremetal-deploy-minio-on-redhat-linux/).
|
||||
|
||||
### 2. Run MinIO in federated mode
|
||||
|
||||
@@ -76,11 +76,11 @@ it is randomized which cluster might provision the bucket.
|
||||
|
||||
### 3. Test your setup
|
||||
|
||||
To test this setup, access the MinIO server via browser or [`mc`](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart). You’ll see the uploaded files are accessible from the all the MinIO endpoints.
|
||||
To test this setup, access the MinIO server via browser or [`mc`](https://silo.pgsty.com/reference/minio-mc/#quickstart). You’ll see the uploaded files are accessible from the all the MinIO endpoints.
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html)
|
||||
- [Use `aws-cli` with MinIO Server](https://docs.min.io/community/minio-object-store/integrations/aws-cli-with-minio.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html)
|
||||
- [Use `mc` with MinIO Server](https://silo.pgsty.com/reference/minio-mc/)
|
||||
- [Use `aws-cli` with MinIO Server](https://silo.pgsty.com/integrations/aws-cli-with-minio/)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://silo.pgsty.com/developers/go/minio-go/)
|
||||
- [The MinIO documentation website](https://silo.pgsty.com/docs/)
|
||||
|
||||
@@ -125,9 +125,6 @@ The JSON body structure can be seen from this sample:
|
||||
],
|
||||
"username": [
|
||||
"minio"
|
||||
],
|
||||
"versionid": [
|
||||
""
|
||||
]
|
||||
},
|
||||
"owner": true,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user