Compare commits

..

1 Commits

Author SHA1 Message Date
Feng Ruohang 5d57938939 add github ci/cd pipeline 2026-02-14 17:20:37 +08:00
230 changed files with 4607 additions and 12473 deletions
+54 -3
View File
@@ -28,6 +28,57 @@ archives:
- minio
name_template: "minio_{{ .Env.PKG_VERSION }}_{{ .Os }}_{{ .Arch }}"
dockers:
- id: minio-amd64
ids:
- minio
goos: linux
goarch: amd64
dockerfile: Dockerfile.goreleaser
use: buildx
image_templates:
- "pgsty/minio:{{ .Tag }}-amd64"
- "pgsty/minio:latest-amd64"
build_flag_templates:
- "--platform=linux/amd64"
- "--label=org.opencontainers.image.version={{ .Tag }}"
- "--label=org.opencontainers.image.created={{ .Date }}"
- "--label=org.opencontainers.image.revision={{ .FullCommit }}"
extra_files:
- dockerscripts/docker-entrypoint.sh
- LICENSE
- CREDITS
- id: minio-arm64
ids:
- minio
goos: linux
goarch: arm64
dockerfile: Dockerfile.goreleaser
use: buildx
image_templates:
- "pgsty/minio:{{ .Tag }}-arm64"
- "pgsty/minio:latest-arm64"
build_flag_templates:
- "--platform=linux/arm64"
- "--label=org.opencontainers.image.version={{ .Tag }}"
- "--label=org.opencontainers.image.created={{ .Date }}"
- "--label=org.opencontainers.image.revision={{ .FullCommit }}"
extra_files:
- dockerscripts/docker-entrypoint.sh
- LICENSE
- CREDITS
docker_manifests:
- name_template: "pgsty/minio:{{ .Tag }}"
image_templates:
- "pgsty/minio:{{ .Tag }}-amd64"
- "pgsty/minio:{{ .Tag }}-arm64"
- name_template: "pgsty/minio:latest"
image_templates:
- "pgsty/minio:latest-amd64"
- "pgsty/minio:latest-arm64"
checksum:
name_template: "minio_{{ .Env.PKG_VERSION }}_checksums.txt"
algorithm: sha256
@@ -36,10 +87,10 @@ release:
github:
owner: pgsty
name: minio
draft: true
draft: false
prerelease: false
mode: append
replace_existing_artifacts: false
mode: replace
replace_existing_artifacts: true
name_template: "{{ .Tag }}"
changelog:
+1 -6
View File
@@ -1,6 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="255 293 539 140" fill="none">
<path d="M255.222 385.861 304.199 383.375c1.061 6.446 3.22 11.358 6.478 14.734 5.303 5.465 12.879 8.197 22.728 8.197 7.348 0 13.011-1.397 16.988-4.19 3.977-2.794 5.966-6.032 5.966-9.716 0-3.5-1.894-6.631-5.682-9.394-3.788-2.763-12.576-5.372-26.364-7.828-22.576-4.113-38.674-9.577-48.295-16.392-9.697-6.815-14.546-15.502-14.546-26.062 0-6.938 2.481-13.491 7.443-19.662 4.962-6.17 12.424-11.02 22.386-14.55 9.962-3.53 23.617-5.295 40.966-5.295 21.288 0 37.519 3.208 48.693 9.624 11.174 6.416 17.822 16.622 19.943 30.621l-48.523 2.302c-1.288-6.078-3.996-10.498-8.125-13.261-4.129-2.763-9.83-4.144-17.103-4.144-5.984 0-10.492 1.028-13.522 3.085-3.03 2.056-4.545 4.558-4.545 7.505 0 2.149 1.25 4.083 3.75 5.802 2.424 1.78 8.182 3.438 17.273 4.973 22.5 3.93 38.617 7.905 48.352 11.926 9.735 4.022 16.818 9.01 21.25 14.965 4.432 5.956 6.648 12.617 6.648 19.984 0 8.657-2.954 16.638-8.863 23.944-5.909 7.306-14.167 12.847-24.773 16.623-10.606 3.776-23.977 5.664-40.113 5.664-28.333 0-47.954-4.42-58.863-13.261-10.909-8.841-17.083-20.076-18.522-33.706Z" fill="#287CAB"/>
<path d="M434.578 295.519h43.304v135.007h-43.304z" fill="#287CAB"/>
<path d="M517.881 295.519h43.209v101.762h67.436v33.245H517.881Z" fill="#287CAB"/>
<path d="M639.4 363.115c0-22.041 6.359-39.201 19.077-51.48 12.718-12.28 30.427-18.419 53.129-18.419 23.274 0 41.206 6.032 53.796 18.096 12.591 12.064 18.886 28.963 18.886 50.697 0 15.778-2.75 28.717-8.251 38.817-5.5 10.099-13.449 17.958-23.846 23.575-10.397 5.618-23.353 8.427-38.869 8.427-15.77 0-28.822-2.425-39.155-7.275-10.333-4.85-18.711-12.525-25.133-23.023-6.423-10.499-9.634-23.637-9.634-39.416Zm43.209.184c0 13.63 2.623 23.422 7.869 29.377 5.246 5.956 12.384 8.934 21.413 8.934 9.284 0 16.47-2.916 21.557-8.749 5.087-5.833 7.631-16.301 7.631-31.403 0-12.709-2.655-21.995-7.965-27.858-5.31-5.863-12.511-8.795-21.604-8.795-8.712 0-15.707 2.978-20.985 8.933-5.278 5.956-7.916 15.81-7.916 29.561Z" fill="#287CAB"/>
</svg>
<svg data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 162.612 24.465"><path d="M52.751.414h9.108v23.63h-9.108zM41.711.74l-18.488 9.92a.919.919 0 0 1-.856 0L3.879.74A2.808 2.808 0 0 0 2.558.414h-.023A2.4 2.4 0 0 0 0 2.641v21.376h9.1V13.842a.918.918 0 0 1 1.385-.682l10.361 5.568a3.634 3.634 0 0 0 3.336.028l10.933-5.634a.917.917 0 0 1 1.371.69v10.205h9.1V2.641A2.4 2.4 0 0 0 43.055.414h-.023a2.808 2.808 0 0 0-1.321.326zm65.564-.326h-9.237v10.755a.913.913 0 0 1-1.338.706L72.762.675a2.824 2.824 0 0 0-1.191-.261h-.016a2.4 2.4 0 0 0-2.535 2.227v21.377h9.163V13.275a.914.914 0 0 1 1.337-.707l24.032 11.2a2.813 2.813 0 0 0 1.188.26 2.4 2.4 0 0 0 2.535-2.227zm7.161 23.63V.414h4.191v23.63zm28.856.421c-11.274 0-19.272-4.7-19.272-12.232C124.02 4.741 132.066 0 143.292 0s19.32 4.7 19.32 12.233-7.902 12.232-19.32 12.232zm0-21.333c-8.383 0-14.84 3.217-14.84 9.1 0 5.926 6.457 9.1 14.84 9.1s14.887-3.174 14.887-9.1c0-5.883-6.504-9.1-14.887-9.1z" fill="#c72c48"/></svg>

Before

Width:  |  Height:  |  Size: 2.0 KiB

After

Width:  |  Height:  |  Size: 978 B

-38
View File
@@ -1,38 +0,0 @@
name: minio
arch: ${NFPM_ARCH}
platform: linux
version: ${PKG_VERSION}
version_schema: none
release: ${NFPM_RELEASE}
section: utils
priority: optional
maintainer: "Ruohang Feng (@Vonng) <rh@vonng.com>"
description: S3-Interface Libre Object Storage, Community-maintained MinIO server fork.
vendor: PGSTY
homepage: https://silo.pgsty.com
license: AGPL-3.0-or-later
contents:
- src: ${NFPM_SOURCE}
dst: /usr/local/bin/minio
expand: true
file_info:
mode: 0755
owner: root
group: root
- src: ${NFPM_UNIT}
dst: /usr/lib/systemd/system/minio.service
expand: true
file_info:
mode: 0644
owner: root
group: root
rpm:
group: Applications/File
compression: gzip:9
deb:
compression: gzip
fields:
License: AGPL-3.0-or-later
-30
View File
@@ -1,30 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg xmlns="http://www.w3.org/2000/svg"
width="1624" height="570" viewBox="0 0 1994 700"
preserveAspectRatio="xMidYMid meet"
role="img" aria-labelledby="title desc"
shape-rendering="geometricPrecision">
<title id="title">SILO wordmark</title>
<desc id="desc">The SILO wordmark set in Chakra Petch Bold, outlined, with the blue-to-copper brand gradient.</desc>
<defs>
<linearGradient id="silo-wordmark-color" x1="44.0" y1="-94.4" x2="1950.0" y2="794.4" gradientUnits="userSpaceOnUse">
<stop class="wm-a" offset="0.06" stop-color="#1d588c"/>
<stop class="wm-b" offset="0.94" stop-color="#b4762e"/>
</linearGradient>
<style>
/* Light theme values of --pg-strong / --copper; dark theme swaps in its own pair. */
@media (prefers-color-scheme: dark) {
.wm-a { stop-color: #7fb8e8; }
.wm-b { stop-color: #e0a35c; }
}
</style>
</defs>
<g fill="url(#silo-wordmark-color)" fill-rule="nonzero">
<path id="silo-s" d="M0 592V492H134V551L167 584H374L408 550V434L375 401H110L2 293V108L110 0H426L534 108V209H400V149L367 116H169L136 149V252L169 285H434L542 393V590L432 700H108Z"/>
<path id="silo-i" d="M637 0H773V700H637Z"/>
<path id="silo-l" d="M888 0H1024V585H1374V700H888Z"/>
<path id="silo-o" d="M1404 585V115L1519 0H1879L1994 115V585L1879 700H1519ZM1807 584 1858 533V167L1807 116H1591L1540 167V533L1591 584Z"/>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 1.4 KiB

-82
View File
@@ -1,82 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg xmlns="http://www.w3.org/2000/svg"
width="570" height="570" viewBox="230 213 570 570"
preserveAspectRatio="xMidYMid meet"
role="img" aria-labelledby="title desc"
shape-rendering="geometricPrecision">
<title id="title">SILO emblem</title>
<desc id="desc">A smooth circular SILO mark with a peaked roof, flowing left wall, columns, and a curved base.</desc>
<defs>
<linearGradient id="silo-color" x1="276" y1="720" x2="742" y2="286" gradientUnits="userSpaceOnUse">
<stop offset="0" stop-color="#064A83"/>
<stop offset="0.5" stop-color="#007FA8"/>
<stop offset="1" stop-color="#22C7C9"/>
</linearGradient>
</defs>
<g fill="url(#silo-color)">
<!-- Outer circular band, intentionally opened at the lower-right plinth. -->
<path d="
M 734 676
A 283.5 278.5 0 1 0 310 692
L 359 692
A 247 248.5 0 1 1 688 676
Z"/>
<!-- Flowing left wall; its upper tangent matches the inner ellipse. -->
<path d="
M 300.57 375
C 292 390 300 430 328 450
C 343 461 357 472 374 481
C 410 501 426 517 426 544
L 426 676
L 336 676
C 308 647 286 608 274 565
C 262 522 263 479 272 439
C 278 413 286 389 300.57 375
Z"/>
<!-- Right column with tangent-continuous upper shoulder. -->
<path d="
M 602 373
Q 602 368 607 370
C 619 374 634 381 634 389
L 634 647
Q 634 649 636 649
L 708 649
L 734 676
L 602 676
Z"/>
<!-- Lower circular cap. -->
<path d="
M 310 692
L 714 692
C 668 743 596 774 512 774
C 428 774 355 743 310 692
Z"/>
<!-- Main silo body, with a tangent-continuous right shoulder. -->
<path d="
M 389 389
C 389 374 447 351 512 351
C 540 351 565 354 580 359
Q 583 360 583 364
L 583 676
L 443 676
L 443 541
C 443 509 426 490 389 470
Z"/>
<!-- Peaked roof with softly tapered, burr-free tips. -->
<path d="
M 512 274
L 647 365
Q 649 370 647 376
C 609 350 563 337 512 337
C 460 337 414 350 377 376
Q 375 370 377 365
Z"/>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 2.2 KiB

+14
View File
@@ -0,0 +1,14 @@
name: 'Dependency Review'
on: [pull_request]
permissions:
contents: read
jobs:
dependency-review:
runs-on: ubuntu-latest
steps:
- name: 'Checkout Repository'
uses: actions/checkout@v4
- name: 'Dependency Review'
uses: actions/dependency-review-action@v4
-170
View File
@@ -1,170 +0,0 @@
name: Publish Docker Image
on:
workflow_dispatch:
inputs:
tag:
description: "Published RELEASE.* tag to package as pgsty/minio"
required: true
type: string
permissions:
contents: read
concurrency:
group: docker-release
cancel-in-progress: false
jobs:
publish:
runs-on: ubuntu-latest
steps:
# Images are built from a published release rather than from the build
# that produced it, so an abandoned draft can never leave :latest
# pointing at something nobody shipped.
- name: Validate published release
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
INPUT_TAG: ${{ inputs.tag }}
run: |
set -euo pipefail
TAG="${INPUT_TAG}"
VERSION_HYPHEN="${TAG#RELEASE.}"
PKG_VERSION="$(echo "${VERSION_HYPHEN}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')"
if [ "${PKG_VERSION}" = "${VERSION_HYPHEN}" ]; then
echo "Invalid release tag: ${TAG}"
exit 1
fi
IS_DRAFT="$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft --jq .isDraft)"
IS_PRERELEASE="$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json isPrerelease --jq .isPrerelease)"
PUBLISHED_AT="$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json publishedAt --jq .publishedAt)"
LATEST_TAG="$(gh release view --repo "${GITHUB_REPOSITORY}" --json tagName --jq .tagName)"
if [ "${IS_DRAFT}" != false ] || [ "${IS_PRERELEASE}" != false ] || [ -z "${PUBLISHED_AT}" ]; then
echo "${TAG} must be a published, non-prerelease GitHub Release"
exit 1
fi
# This workflow moves :latest, so it must not run for an older tag.
if [ "${TAG}" != "${LATEST_TAG}" ]; then
echo "Refusing to replace Docker latest with non-latest release ${TAG} (latest is ${LATEST_TAG})"
exit 1
fi
{
echo "RELEASE_TAG=${TAG}"
echo "PKG_VERSION=${PKG_VERSION}"
echo "PUBLISHED_AT=${PUBLISHED_AT}"
} >> "${GITHUB_ENV}"
- name: Validate Docker Hub credentials
run: |
set -euo pipefail
if [ -z "${{ secrets.DOCKERHUB_USERNAME }}" ] || [ -z "${{ secrets.DOCKERHUB_TOKEN }}" ]; then
echo "Missing Docker Hub credentials. Set DOCKERHUB_USERNAME and DOCKERHUB_TOKEN repository secrets."
exit 1
fi
- name: Checkout release tag
uses: actions/checkout@v4
with:
ref: ${{ inputs.tag }}
fetch-depth: 0
- name: Prepare verified Docker contexts
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
assets_dir="docker-release/assets"
mkdir -p "${assets_dir}"
amd64_archive="minio_${PKG_VERSION}_linux_amd64.tar.gz"
arm64_archive="minio_${PKG_VERSION}_linux_arm64.tar.gz"
checksums="minio_${PKG_VERSION}_checksums.txt"
gh release download "${RELEASE_TAG}" --repo "${GITHUB_REPOSITORY}" \
--dir "${assets_dir}" \
--pattern "${amd64_archive}" \
--pattern "${arm64_archive}" \
--pattern "${checksums}"
# The binaries going into the images are the published ones, checked
# against the published checksums, not a rebuild that merely ought to
# match them.
cd "${assets_dir}"
grep -F " ${amd64_archive}" "${checksums}" | sha256sum --check
grep -F " ${arm64_archive}" "${checksums}" | sha256sum --check
cd "${GITHUB_WORKSPACE}"
# Dockerfile.goreleaser expects the binary at the context root and
# the entrypoint scripts under dockerscripts/, which is the layout
# GoReleaser used to assemble via extra_files.
for arch in amd64 arm64; do
context="docker-release/${arch}"
archive="${assets_dir}/minio_${PKG_VERSION}_linux_${arch}.tar.gz"
mkdir -p "${context}/dockerscripts"
tar -xzf "${archive}" -C "${context}" minio
cp Dockerfile.goreleaser LICENSE CREDITS "${context}/"
cp dockerscripts/docker-entrypoint.sh dockerscripts/download-static-curl.sh \
"${context}/dockerscripts/"
done
echo "RELEASE_REVISION=$(git rev-parse HEAD)" >> "${GITHUB_ENV}"
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: arm64
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push amd64 image
uses: docker/build-push-action@v6
with:
context: docker-release/amd64
file: docker-release/amd64/Dockerfile.goreleaser
platforms: linux/amd64
push: true
tags: |
pgsty/minio:${{ env.RELEASE_TAG }}-amd64
pgsty/minio:latest-amd64
labels: |
org.opencontainers.image.version=${{ env.RELEASE_TAG }}
org.opencontainers.image.created=${{ env.PUBLISHED_AT }}
org.opencontainers.image.revision=${{ env.RELEASE_REVISION }}
- name: Build and push arm64 image
uses: docker/build-push-action@v6
with:
context: docker-release/arm64
file: docker-release/arm64/Dockerfile.goreleaser
platforms: linux/arm64
push: true
tags: |
pgsty/minio:${{ env.RELEASE_TAG }}-arm64
pgsty/minio:latest-arm64
labels: |
org.opencontainers.image.version=${{ env.RELEASE_TAG }}
org.opencontainers.image.created=${{ env.PUBLISHED_AT }}
org.opencontainers.image.revision=${{ env.RELEASE_REVISION }}
- name: Publish multi-architecture manifests
run: |
set -euo pipefail
docker buildx imagetools create \
--tag "pgsty/minio:${RELEASE_TAG}" \
"pgsty/minio:${RELEASE_TAG}-amd64" \
"pgsty/minio:${RELEASE_TAG}-arm64"
docker buildx imagetools create \
--tag "pgsty/minio:latest" \
"pgsty/minio:latest-amd64" \
"pgsty/minio:latest-arm64"
docker buildx imagetools inspect "pgsty/minio:${RELEASE_TAG}"
docker buildx imagetools inspect "pgsty/minio:latest"
+39
View File
@@ -0,0 +1,39 @@
name: Crosscompile
on:
pull_request:
branches:
- master
# This ensures that previous jobs for the PR are canceled when the PR is
# updated.
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
build:
name: Build Tests with Go ${{ matrix.go-version }} on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [1.24.x]
os: [ubuntu-latest]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
check-latest: true
- name: Build on ${{ matrix.os }}
if: matrix.os == 'ubuntu-latest'
env:
CGO_ENABLED: 0
GO111MODULE: on
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
make crosscompile
+44
View File
@@ -0,0 +1,44 @@
name: Healing Functional Tests
on:
pull_request:
branches:
- master
# This ensures that previous jobs for the PR are canceled when the PR is
# updated.
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
build:
name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [1.24.x]
os: [ubuntu-latest]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
check-latest: true
- name: Build on ${{ matrix.os }}
if: matrix.os == 'ubuntu-latest'
env:
CGO_ENABLED: 0
GO111MODULE: on
MINIO_KMS_SECRET_KEY: "my-minio-key:oyArl7zlPECEduNbB1KXgdzDn2Bdpvvw0l8VO51HQnY="
MINIO_KMS_AUTO_ENCRYPTION: on
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
make verify-healing
make verify-healing-inconsistent-versions
make verify-healing-with-root-disks
make verify-healing-with-rewrite
+42
View File
@@ -0,0 +1,42 @@
name: Linters and Tests
on:
pull_request:
branches:
- master
# This ensures that previous jobs for the PR are canceled when the PR is
# updated.
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
build:
name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [1.24.x]
os: [ubuntu-latest]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
check-latest: true
- name: Build on ${{ matrix.os }}
if: matrix.os == 'ubuntu-latest'
env:
CGO_ENABLED: 0
GO111MODULE: on
run: |
sudo apt install jq -y
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
make
make test
make test-race
+39
View File
@@ -0,0 +1,39 @@
name: Resiliency Functional Tests
on:
pull_request:
branches:
- master
# This ensures that previous jobs for the PR are canceled when the PR is
# updated.
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
build:
name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [1.24.x]
os: [ubuntu-latest]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
check-latest: true
- name: Build on ${{ matrix.os }}
if: matrix.os == 'ubuntu-latest'
env:
CGO_ENABLED: 0
GO111MODULE: on
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
make test-resiliency
+20 -127
View File
@@ -1,149 +1,42 @@
name: Go CI
name: Functional Tests
on:
pull_request:
branches:
- master
push:
branches:
- master
workflow_dispatch:
# Cancel superseded runs for the same PR; never cancel master push runs.
# Keyed on PR number (not head_ref) so fork PRs sharing a branch name
# do not collide.
# This ensures that previous jobs for the PR are canceled when the PR is
# updated.
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
group: ${{ github.workflow }}-${{ github.head_ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
verify:
name: Format, Build, Vet
runs-on: ubuntu-latest
timeout-minutes: 20
build:
name: Go ${{ matrix.go-version }} on ${{ matrix.os }} - healing
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [1.24.x]
os: [ubuntu-latest]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true
- name: Check gofmt
run: |
unformatted=$(gofmt -l main.go cmd internal)
if [ -n "${unformatted}" ]; then
echo "The following files are not gofmt-formatted:"
echo "${unformatted}"
gofmt -d ${unformatted}
exit 1
fi
- name: Build
go-version: ${{ matrix.go-version }}
check-latest: true
- name: Build on ${{ matrix.os }}
if: matrix.os == 'ubuntu-latest'
env:
CGO_ENABLED: 0
run: go build ./...
- name: Vet
run: go vet ./...
quality:
name: Lint, Generated Files
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true
- name: Lint
run: make lint
- name: Check generated files
run: make check-gen
race-s3select:
name: Race, S3 Select
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true
- name: Run S3 Select tests under race detector
run: go test -race ./internal/s3select/... -count=1
crosscompile:
name: Cross Compile
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true
- name: Build supported targets
run: make crosscompile
test-internal:
name: Test internal/
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true
# internal/http listener tests bind [::1]; runners disable IPv6 by default.
- name: Enable IPv6
GO111MODULE: on
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
MINIO_KMS_AUTO_ENCRYPTION: on
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
- name: Run internal tests
env:
CGO_ENABLED: 0
MINIO_API_REQUESTS_MAX: "10000"
run: go test ./internal/... -count=1
test-cmd:
name: Test cmd/
runs-on: ubuntu-latest
timeout-minutes: 35
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true
# Some server tests bind IPv6 listeners; runners disable IPv6 by default.
- name: Enable IPv6
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
- name: Run cmd tests
env:
CGO_ENABLED: 0
MINIO_API_REQUESTS_MAX: "10000"
# cmd/ is one large package; raise go test's default 10m per-package
# timeout so slower runners fail on the job timeout, not a panic.
run: go test ./cmd/ -count=1 -timeout 30m
make verify
make test-timeout
+30
View File
@@ -0,0 +1,30 @@
name: Helm Chart linting
on:
pull_request:
branches:
- master
# This ensures that previous jobs for the PR are canceled when the PR is
# updated.
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
release:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install Helm
uses: azure/setup-helm@v4
- name: Run helm lint
run: |
cd helm/minio
helm lint .
+161
View File
@@ -0,0 +1,161 @@
name: IAM integration
on:
pull_request:
branches:
- master
# This ensures that previous jobs for the PR are canceled when the PR is
# updated.
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
iam-matrix-test:
name: "[Go=${{ matrix.go-version }}|ldap=${{ matrix.ldap }}|etcd=${{ matrix.etcd }}|openid=${{ matrix.openid }}]"
runs-on: ubuntu-latest
services:
openldap:
image: quay.io/minio/openldap
ports:
- "389:389"
- "636:636"
env:
LDAP_ORGANIZATION: "MinIO Inc"
LDAP_DOMAIN: "min.io"
LDAP_ADMIN_PASSWORD: "admin"
etcd:
image: "quay.io/coreos/etcd:v3.5.1"
env:
ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379"
ETCD_ADVERTISE_CLIENT_URLS: "http://0.0.0.0:2379"
ports:
- "2379:2379"
options: >-
--health-cmd "etcdctl endpoint health"
--health-interval 10s
--health-timeout 5s
--health-retries 5
openid:
image: quay.io/minio/dex
ports:
- "5556:5556"
env:
DEX_LDAP_SERVER: "openldap:389"
openid2:
image: quay.io/minio/dex
ports:
- "5557:5557"
env:
DEX_LDAP_SERVER: "openldap:389"
DEX_ISSUER: "http://127.0.0.1:5557/dex"
DEX_WEB_HTTP: "0.0.0.0:5557"
strategy:
# When ldap, etcd or openid vars are empty below, those external servers
# are turned off - i.e. if ldap="", then ldap server is not enabled for
# the tests.
matrix:
go-version: [1.24.x]
ldap: ["", "localhost:389"]
etcd: ["", "http://localhost:2379"]
openid: ["", "http://127.0.0.1:5556/dex"]
exclude:
# exclude combos where all are empty.
- ldap: ""
etcd: ""
openid: ""
# exclude combos where both ldap and openid IDPs are specified.
- ldap: "localhost:389"
openid: "http://127.0.0.1:5556/dex"
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
check-latest: true
- name: Test LDAP/OpenID/Etcd combo
env:
_MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }}
_MINIO_ETCD_TEST_SERVER: ${{ matrix.etcd }}
_MINIO_OPENID_TEST_SERVER: ${{ matrix.openid }}
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
make test-iam
- name: Test with multiple OpenID providers
if: matrix.openid == 'http://127.0.0.1:5556/dex'
env:
_MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }}
_MINIO_ETCD_TEST_SERVER: ${{ matrix.etcd }}
_MINIO_OPENID_TEST_SERVER: ${{ matrix.openid }}
_MINIO_OPENID_TEST_SERVER_2: "http://127.0.0.1:5557/dex"
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
make test-iam
- name: Test with Access Management Plugin enabled
env:
_MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }}
_MINIO_ETCD_TEST_SERVER: ${{ matrix.etcd }}
_MINIO_OPENID_TEST_SERVER: ${{ matrix.openid }}
_MINIO_POLICY_PLUGIN_TEST_ENDPOINT: "http://127.0.0.1:8080"
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
go run docs/iam/access-manager-plugin.go &
make test-iam
- name: Test MinIO Old Version data to IAM import current version
if: matrix.ldap == 'ldaphost:389'
env:
_MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }}
run: |
make test-iam-ldap-upgrade-import
- name: Test LDAP for automatic site replication
if: matrix.ldap == 'localhost:389'
run: |
make test-site-replication-ldap
- name: Test OIDC for automatic site replication
if: matrix.openid == 'http://127.0.0.1:5556/dex'
run: |
make test-site-replication-oidc
iam-import-with-missing-entities:
name: Test IAM import in new cluster with missing entities
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
check-latest: true
- name: Checkout minio-iam-testing
uses: actions/checkout@v4
with:
repository: minio/minio-iam-testing
path: minio-iam-testing
- name: Test import of IAM artifacts when in fresh cluster there are missing groups etc
run: |
make test-iam-import-with-missing-entities
iam-import-with-openid:
name: Test IAM import in new cluster with opendid configurations
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
check-latest: true
- name: Checkout minio-iam-testing
uses: actions/checkout@v4
with:
repository: minio/minio-iam-testing
path: minio-iam-testing
- name: Test import of IAM artifacts when in fresh cluster with openid configurations
run: |
make test-iam-import-with-openid
+18
View File
@@ -0,0 +1,18 @@
# @format
name: Issue Workflow
on:
issues:
types:
- opened
jobs:
add-to-project:
name: Add issue to project
runs-on: ubuntu-latest
steps:
- uses: actions/add-to-project@v0.5.0
with:
project-url: https://github.com/orgs/miniohq/projects/2
github-token: ${{ secrets.BOT_PAT }}
+24
View File
@@ -0,0 +1,24 @@
name: 'Lock Threads'
on:
schedule:
- cron: '0 0 * * *'
workflow_dispatch:
permissions:
issues: write
concurrency:
group: lock
jobs:
action:
runs-on: ubuntu-latest
steps:
- uses: dessant/lock-threads@v3
with:
github-token: ${{ github.token }}
issue-inactive-days: '365'
exclude-any-issue-labels: 'do-not-close'
issue-lock-reason: 'resolved'
log-output: true
+81
View File
@@ -0,0 +1,81 @@
name: Mint Tests
on:
pull_request:
branches:
- master
# This ensures that previous jobs for the PR are canceled when the PR is
# updated.
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
mint-test:
runs-on: mint
timeout-minutes: 120
steps:
- name: cleanup #https://github.com/actions/checkout/issues/273
run: |
sudo -S rm -rf ${GITHUB_WORKSPACE}
mkdir ${GITHUB_WORKSPACE}
- name: checkout-step
uses: actions/checkout@v4
- name: setup-go-step
uses: actions/setup-go@v5
with:
go-version: 1.24.x
- name: github sha short
id: vars
run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
- name: build-minio
run: |
TAG="quay.io/minio/minio:${{ steps.vars.outputs.sha_short }}" make docker
- name: multipart uploads test
run: |
${GITHUB_WORKSPACE}/.github/workflows/multipart/migrate.sh "${{ steps.vars.outputs.sha_short }}"
- name: compress and encrypt
run: |
${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "compress-encrypt" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
- name: multiple pools
run: |
${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "pools" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
- name: standalone erasure
run: |
${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "erasure" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
# FIXME: renable this back when we have a valid way to add deadlines for PUT()s (internode CreateFile)
# - name: resiliency
# run: |
# ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "resiliency" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
- name: The job must cleanup
if: ${{ always() }}
run: |
export JOB_NAME=${{ steps.vars.outputs.sha_short }}
for mode in $(echo compress-encrypt pools erasure); do
docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml down || true
docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml rm || true
done
docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site1.yaml rm -s -f || true
docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site2.yaml rm -s -f || true
for volume in $(docker volume ls -q | grep minio); do
docker volume rm ${volume} || true
done
docker rmi -f quay.io/minio/minio:${{ steps.vars.outputs.sha_short }}
docker system prune -f || true
docker volume prune -f || true
docker volume rm $(docker volume ls -q -f dangling=true) || true
@@ -0,0 +1,80 @@
version: '3.7'
# Settings and configurations that are common for all containers
x-minio-common: &minio-common
image: quay.io/minio/minio:${JOB_NAME}
command: server --console-address ":9001" http://minio{1...4}/cdata{1...2}
expose:
- "9000"
- "9001"
environment:
MINIO_CI_CD: "on"
MINIO_ROOT_USER: "minio"
MINIO_ROOT_PASSWORD: "minio123"
MINIO_COMPRESSION_ENABLE: "on"
MINIO_COMPRESSION_MIME_TYPES: "*"
MINIO_COMPRESSION_ALLOW_ENCRYPTION: "on"
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
healthcheck:
test: ["CMD", "mc", "ready", "local"]
interval: 5s
timeout: 5s
retries: 5
# starts 4 docker containers running minio server instances.
# using nginx reverse proxy, load balancing, you can access
# it through port 9000.
services:
minio1:
<<: *minio-common
hostname: minio1
volumes:
- cdata1-1:/cdata1
- cdata1-2:/cdata2
minio2:
<<: *minio-common
hostname: minio2
volumes:
- cdata2-1:/cdata1
- cdata2-2:/cdata2
minio3:
<<: *minio-common
hostname: minio3
volumes:
- cdata3-1:/cdata1
- cdata3-2:/cdata2
minio4:
<<: *minio-common
hostname: minio4
volumes:
- cdata4-1:/cdata1
- cdata4-2:/cdata2
nginx:
image: nginx:1.19.2-alpine
hostname: nginx
volumes:
- ./nginx-4-node.conf:/etc/nginx/nginx.conf:ro
ports:
- "9000:9000"
- "9001:9001"
depends_on:
- minio1
- minio2
- minio3
- minio4
## By default this config uses default local driver,
## For custom volumes replace with volume driver configuration.
volumes:
cdata1-1:
cdata1-2:
cdata2-1:
cdata2-2:
cdata3-1:
cdata3-2:
cdata4-1:
cdata4-2:
+51
View File
@@ -0,0 +1,51 @@
version: '3.7'
# Settings and configurations that are common for all containers
x-minio-common: &minio-common
image: quay.io/minio/minio:${JOB_NAME}
command: server --console-address ":9001" edata{1...4}
expose:
- "9000"
- "9001"
environment:
MINIO_CI_CD: "on"
MINIO_ROOT_USER: "minio"
MINIO_ROOT_PASSWORD: "minio123"
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
healthcheck:
test: ["CMD", "mc", "ready", "local"]
interval: 5s
timeout: 5s
retries: 5
# starts 4 docker containers running minio server instances.
# using nginx reverse proxy, load balancing, you can access
# it through port 9000.
services:
minio1:
<<: *minio-common
hostname: minio1
volumes:
- edata1-1:/edata1
- edata1-2:/edata2
- edata1-3:/edata3
- edata1-4:/edata4
nginx:
image: nginx:1.19.2-alpine
hostname: nginx
volumes:
- ./nginx-1-node.conf:/etc/nginx/nginx.conf:ro
ports:
- "9000:9000"
- "9001:9001"
depends_on:
- minio1
## By default this config uses default local driver,
## For custom volumes replace with volume driver configuration.
volumes:
edata1-1:
edata1-2:
edata1-3:
edata1-4:
+117
View File
@@ -0,0 +1,117 @@
version: '3.7'
# Settings and configurations that are common for all containers
x-minio-common: &minio-common
image: quay.io/minio/minio:${JOB_NAME}
command: server --console-address ":9001" http://minio{1...4}/pdata{1...2} http://minio{5...8}/pdata{1...2}
expose:
- "9000"
- "9001"
environment:
MINIO_CI_CD: "on"
MINIO_ROOT_USER: "minio"
MINIO_ROOT_PASSWORD: "minio123"
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
healthcheck:
test: ["CMD", "mc", "ready", "local"]
interval: 5s
timeout: 5s
retries: 5
# starts 4 docker containers running minio server instances.
# using nginx reverse proxy, load balancing, you can access
# it through port 9000.
services:
minio1:
<<: *minio-common
hostname: minio1
volumes:
- pdata1-1:/pdata1
- pdata1-2:/pdata2
minio2:
<<: *minio-common
hostname: minio2
volumes:
- pdata2-1:/pdata1
- pdata2-2:/pdata2
minio3:
<<: *minio-common
hostname: minio3
volumes:
- pdata3-1:/pdata1
- pdata3-2:/pdata2
minio4:
<<: *minio-common
hostname: minio4
volumes:
- pdata4-1:/pdata1
- pdata4-2:/pdata2
minio5:
<<: *minio-common
hostname: minio5
volumes:
- pdata5-1:/pdata1
- pdata5-2:/pdata2
minio6:
<<: *minio-common
hostname: minio6
volumes:
- pdata6-1:/pdata1
- pdata6-2:/pdata2
minio7:
<<: *minio-common
hostname: minio7
volumes:
- pdata7-1:/pdata1
- pdata7-2:/pdata2
minio8:
<<: *minio-common
hostname: minio8
volumes:
- pdata8-1:/pdata1
- pdata8-2:/pdata2
nginx:
image: nginx:1.19.2-alpine
hostname: nginx
volumes:
- ./nginx-8-node.conf:/etc/nginx/nginx.conf:ro
ports:
- "9000:9000"
- "9001:9001"
depends_on:
- minio1
- minio2
- minio3
- minio4
- minio5
- minio6
- minio7
- minio8
## By default this config uses default local driver,
## For custom volumes replace with volume driver configuration.
volumes:
pdata1-1:
pdata1-2:
pdata2-1:
pdata2-2:
pdata3-1:
pdata3-2:
pdata4-1:
pdata4-2:
pdata5-1:
pdata5-2:
pdata6-1:
pdata6-2:
pdata7-1:
pdata7-2:
pdata8-1:
pdata8-2:
@@ -0,0 +1,78 @@
version: '3.7'
# Settings and configurations that are common for all containers
x-minio-common: &minio-common
image: quay.io/minio/minio:${JOB_NAME}
command: server --console-address ":9001" http://minio{1...4}/rdata{1...2}
expose:
- "9000"
- "9001"
environment:
MINIO_CI_CD: "on"
MINIO_ROOT_USER: "minio"
MINIO_ROOT_PASSWORD: "minio123"
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
MINIO_DRIVE_MAX_TIMEOUT: "5s"
healthcheck:
test: ["CMD", "mc", "ready", "local"]
interval: 5s
timeout: 5s
retries: 5
# starts 4 docker containers running minio server instances.
# using nginx reverse proxy, load balancing, you can access
# it through port 9000.
services:
minio1:
<<: *minio-common
hostname: minio1
volumes:
- rdata1-1:/rdata1
- rdata1-2:/rdata2
minio2:
<<: *minio-common
hostname: minio2
volumes:
- rdata2-1:/rdata1
- rdata2-2:/rdata2
minio3:
<<: *minio-common
hostname: minio3
volumes:
- rdata3-1:/rdata1
- rdata3-2:/rdata2
minio4:
<<: *minio-common
hostname: minio4
volumes:
- rdata4-1:/rdata1
- rdata4-2:/rdata2
nginx:
image: nginx:1.19.2-alpine
hostname: nginx
volumes:
- ./nginx-4-node.conf:/etc/nginx/nginx.conf:ro
ports:
- "9000:9000"
- "9001:9001"
depends_on:
- minio1
- minio2
- minio3
- minio4
## By default this config uses default local driver,
## For custom volumes replace with volume driver configuration.
volumes:
rdata1-1:
rdata1-2:
rdata2-1:
rdata2-2:
rdata3-1:
rdata3-2:
rdata4-1:
rdata4-2:
+100
View File
@@ -0,0 +1,100 @@
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 4096;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
keepalive_timeout 65;
# include /etc/nginx/conf.d/*.conf;
upstream minio {
server minio1:9000;
}
upstream console {
ip_hash;
server minio1:9001;
}
server {
listen 9000;
listen [::]:9000;
server_name localhost;
# To allow special characters in headers
ignore_invalid_headers off;
# Allow any size file to be uploaded.
# Set to a value such as 1000m; to restrict file size to a specific value
client_max_body_size 0;
# To disable buffering
proxy_buffering off;
proxy_request_buffering off;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_connect_timeout 300;
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
proxy_http_version 1.1;
proxy_set_header Connection "";
chunked_transfer_encoding off;
proxy_pass http://minio;
}
}
server {
listen 9001;
listen [::]:9001;
server_name localhost;
# To allow special characters in headers
ignore_invalid_headers off;
# Allow any size file to be uploaded.
# Set to a value such as 1000m; to restrict file size to a specific value
client_max_body_size 0;
# To disable buffering
proxy_buffering off;
proxy_request_buffering off;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-NginX-Proxy true;
# This is necessary to pass the correct IP to be hashed
real_ip_header X-Real-IP;
proxy_connect_timeout 300;
# To support websocket
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
chunked_transfer_encoding off;
proxy_pass http://console;
}
}
}
+105
View File
@@ -0,0 +1,105 @@
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 4096;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
keepalive_timeout 65;
# include /etc/nginx/conf.d/*.conf;
upstream minio {
server minio1:9000 max_fails=1 fail_timeout=10s;
server minio2:9000 max_fails=1 fail_timeout=10s;
server minio3:9000 max_fails=1 fail_timeout=10s;
}
upstream console {
ip_hash;
server minio1:9001;
server minio2:9001;
server minio3:9001;
server minio4:9001;
}
server {
listen 9000;
listen [::]:9000;
server_name localhost;
# To allow special characters in headers
ignore_invalid_headers off;
# Allow any size file to be uploaded.
# Set to a value such as 1000m; to restrict file size to a specific value
client_max_body_size 0;
# To disable buffering
proxy_buffering off;
proxy_request_buffering off;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_connect_timeout 300;
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
proxy_http_version 1.1;
proxy_set_header Connection "";
chunked_transfer_encoding off;
proxy_pass http://minio;
}
}
server {
listen 9001;
listen [::]:9001;
server_name localhost;
# To allow special characters in headers
ignore_invalid_headers off;
# Allow any size file to be uploaded.
# Set to a value such as 1000m; to restrict file size to a specific value
client_max_body_size 0;
# To disable buffering
proxy_buffering off;
proxy_request_buffering off;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-NginX-Proxy true;
# This is necessary to pass the correct IP to be hashed
real_ip_header X-Real-IP;
proxy_connect_timeout 300;
# To support websocket
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
chunked_transfer_encoding off;
proxy_pass http://console;
}
}
}
+114
View File
@@ -0,0 +1,114 @@
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 4096;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
keepalive_timeout 65;
# include /etc/nginx/conf.d/*.conf;
upstream minio {
server minio1:9000 max_fails=1 fail_timeout=10s;
server minio2:9000 max_fails=1 fail_timeout=10s;
server minio3:9000 max_fails=1 fail_timeout=10s;
server minio4:9000 max_fails=1 fail_timeout=10s;
server minio5:9000 max_fails=1 fail_timeout=10s;
server minio6:9000 max_fails=1 fail_timeout=10s;
server minio7:9000 max_fails=1 fail_timeout=10s;
server minio8:9000 max_fails=1 fail_timeout=10s;
}
upstream console {
ip_hash;
server minio1:9001;
server minio2:9001;
server minio3:9001;
server minio4:9001;
server minio5:9001;
server minio6:9001;
server minio7:9001;
server minio8:9001;
}
server {
listen 9000;
listen [::]:9000;
server_name localhost;
# To allow special characters in headers
ignore_invalid_headers off;
# Allow any size file to be uploaded.
# Set to a value such as 1000m; to restrict file size to a specific value
client_max_body_size 0;
# To disable buffering
proxy_buffering off;
proxy_request_buffering off;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_connect_timeout 300;
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
proxy_http_version 1.1;
proxy_set_header Connection "";
chunked_transfer_encoding off;
proxy_pass http://minio;
}
}
server {
listen 9001;
listen [::]:9001;
server_name localhost;
# To allow special characters in headers
ignore_invalid_headers off;
# Allow any size file to be uploaded.
# Set to a value such as 1000m; to restrict file size to a specific value
client_max_body_size 0;
# To disable buffering
proxy_buffering off;
proxy_request_buffering off;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-NginX-Proxy true;
# This is necessary to pass the correct IP to be hashed
real_ip_header X-Real-IP;
proxy_connect_timeout 300;
# To support websocket
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
chunked_transfer_encoding off;
proxy_pass http://console;
}
}
}
+106
View File
@@ -0,0 +1,106 @@
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 4096;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
keepalive_timeout 65;
# include /etc/nginx/conf.d/*.conf;
upstream minio {
server minio1:9000 max_fails=1 fail_timeout=10s;
server minio2:9000 max_fails=1 fail_timeout=10s;
server minio3:9000 max_fails=1 fail_timeout=10s;
server minio4:9000 max_fails=1 fail_timeout=10s;
}
upstream console {
ip_hash;
server minio1:9001;
server minio2:9001;
server minio3:9001;
server minio4:9001;
}
server {
listen 9000;
listen [::]:9000;
server_name localhost;
# To allow special characters in headers
ignore_invalid_headers off;
# Allow any size file to be uploaded.
# Set to a value such as 1000m; to restrict file size to a specific value
client_max_body_size 0;
# To disable buffering
proxy_buffering off;
proxy_request_buffering off;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_connect_timeout 300;
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
proxy_http_version 1.1;
proxy_set_header Connection "";
chunked_transfer_encoding off;
proxy_pass http://minio;
}
}
server {
listen 9001;
listen [::]:9001;
server_name localhost;
# To allow special characters in headers
ignore_invalid_headers off;
# Allow any size file to be uploaded.
# Set to a value such as 1000m; to restrict file size to a specific value
client_max_body_size 0;
# To disable buffering
proxy_buffering off;
proxy_request_buffering off;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-NginX-Proxy true;
# This is necessary to pass the correct IP to be hashed
real_ip_header X-Real-IP;
proxy_connect_timeout 300;
# To support websocket
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
chunked_transfer_encoding off;
proxy_pass http://console;
}
}
}
@@ -0,0 +1,66 @@
version: '3.7'
# Settings and configurations that are common for all containers
x-minio-common: &minio-common
image: quay.io/minio/minio:${RELEASE}
command: server http://site1-minio{1...4}/data{1...2}
environment:
- MINIO_PROMETHEUS_AUTH_TYPE=public
- CI=true
# starts 4 docker containers running minio server instances.
# using nginx reverse proxy, load balancing, you can access
# it through port 9000.
services:
site1-minio1:
<<: *minio-common
hostname: site1-minio1
volumes:
- site1-data1-1:/data1
- site1-data1-2:/data2
site1-minio2:
<<: *minio-common
hostname: site1-minio2
volumes:
- site1-data2-1:/data1
- site1-data2-2:/data2
site1-minio3:
<<: *minio-common
hostname: site1-minio3
volumes:
- site1-data3-1:/data1
- site1-data3-2:/data2
site1-minio4:
<<: *minio-common
hostname: site1-minio4
volumes:
- site1-data4-1:/data1
- site1-data4-2:/data2
site1-nginx:
image: nginx:1.19.2-alpine
hostname: site1-nginx
volumes:
- ./nginx-site1.conf:/etc/nginx/nginx.conf:ro
ports:
- "9001:9001"
depends_on:
- site1-minio1
- site1-minio2
- site1-minio3
- site1-minio4
## By default this config uses default local driver,
## For custom volumes replace with volume driver configuration.
volumes:
site1-data1-1:
site1-data1-2:
site1-data2-1:
site1-data2-2:
site1-data3-1:
site1-data3-2:
site1-data4-1:
site1-data4-2:
@@ -0,0 +1,66 @@
version: '3.7'
# Settings and configurations that are common for all containers
x-minio-common: &minio-common
image: quay.io/minio/minio:${RELEASE}
command: server http://site2-minio{1...4}/data{1...2}
environment:
- MINIO_PROMETHEUS_AUTH_TYPE=public
- CI=true
# starts 4 docker containers running minio server instances.
# using nginx reverse proxy, load balancing, you can access
# it through port 9000.
services:
site2-minio1:
<<: *minio-common
hostname: site2-minio1
volumes:
- site2-data1-1:/data1
- site2-data1-2:/data2
site2-minio2:
<<: *minio-common
hostname: site2-minio2
volumes:
- site2-data2-1:/data1
- site2-data2-2:/data2
site2-minio3:
<<: *minio-common
hostname: site2-minio3
volumes:
- site2-data3-1:/data1
- site2-data3-2:/data2
site2-minio4:
<<: *minio-common
hostname: site2-minio4
volumes:
- site2-data4-1:/data1
- site2-data4-2:/data2
site2-nginx:
image: nginx:1.19.2-alpine
hostname: site2-nginx
volumes:
- ./nginx-site2.conf:/etc/nginx/nginx.conf:ro
ports:
- "9002:9002"
depends_on:
- site2-minio1
- site2-minio2
- site2-minio3
- site2-minio4
## By default this config uses default local driver,
## For custom volumes replace with volume driver configuration.
volumes:
site2-data1-1:
site2-data1-2:
site2-data2-1:
site2-data2-2:
site2-data3-1:
site2-data3-2:
site2-data4-1:
site2-data4-2:
+147
View File
@@ -0,0 +1,147 @@
#!/bin/bash
set -x
## change working directory
cd .github/workflows/multipart/
function cleanup() {
docker-compose -f docker-compose-site1.yaml rm -s -f || true
docker-compose -f docker-compose-site2.yaml rm -s -f || true
for volume in $(docker volume ls -q | grep minio); do
docker volume rm ${volume} || true
done
docker system prune -f || true
docker volume prune -f || true
docker volume rm $(docker volume ls -q -f dangling=true) || true
}
cleanup
if [ ! -f ./mc ]; then
wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc &&
chmod +x mc
fi
export RELEASE=RELEASE.2023-08-29T23-07-35Z
docker-compose -f docker-compose-site1.yaml up -d
docker-compose -f docker-compose-site2.yaml up -d
sleep 30s
./mc alias set site1 http://site1-nginx:9001 minioadmin minioadmin --api s3v4
./mc alias set site2 http://site2-nginx:9002 minioadmin minioadmin --api s3v4
./mc ready site1/
./mc ready site2/
./mc admin replicate add site1 site2
./mc mb site1/testbucket/
./mc cp -r --quiet /usr/bin site1/testbucket/
sleep 5
./s3-check-md5 -h
failed_count_site1=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
failed_count_site2=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
if [ $failed_count_site1 -ne 0 ]; then
echo "failed with multipart on site1 uploads"
exit 1
fi
if [ $failed_count_site2 -ne 0 ]; then
echo "failed with multipart on site2 uploads"
exit 1
fi
./mc cp -r --quiet /usr/bin site1/testbucket/
sleep 5
failed_count_site1=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
failed_count_site2=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
## we do not need to fail here, since we are going to test
## upgrading to master, healing and being able to recover
## the last version.
if [ $failed_count_site1 -ne 0 ]; then
echo "failed with multipart on site1 uploads ${failed_count_site1}"
fi
if [ $failed_count_site2 -ne 0 ]; then
echo "failed with multipart on site2 uploads ${failed_count_site2}"
fi
export RELEASE=${1}
docker-compose -f docker-compose-site1.yaml up -d
docker-compose -f docker-compose-site2.yaml up -d
./mc ready site1/
./mc ready site2/
for i in $(seq 1 10); do
# mc admin heal -r --remove when used against a LB endpoint
# behaves flaky, let this run 10 times before giving up
./mc admin heal -r --remove --json site1/ 2>&1 >/dev/null
./mc admin heal -r --remove --json site2/ 2>&1 >/dev/null
done
failed_count_site1=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
failed_count_site2=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
if [ $failed_count_site1 -ne 0 ]; then
echo "failed with multipart on site1 uploads"
exit 1
fi
if [ $failed_count_site2 -ne 0 ]; then
echo "failed with multipart on site2 uploads"
exit 1
fi
# Add user group test
./mc admin user add site1 site-replication-issue-user site-replication-issue-password
./mc admin group add site1 site-replication-issue-group site-replication-issue-user
max_wait_attempts=30
wait_interval=5
attempt=1
while true; do
diff <(./mc admin group info site1 site-replication-issue-group) <(./mc admin group info site2 site-replication-issue-group)
if [[ $? -eq 0 ]]; then
echo "Outputs are consistent."
break
fi
remaining_attempts=$((max_wait_attempts - attempt))
if ((attempt >= max_wait_attempts)); then
echo "Outputs remain inconsistent after $max_wait_attempts attempts. Exiting with error."
exit 1
else
echo "Outputs are inconsistent. Waiting for $wait_interval seconds (attempt $attempt/$max_wait_attempts)."
sleep $wait_interval
fi
((attempt++))
done
status=$(./mc admin group info site1 site-replication-issue-group --json | jq .groupStatus | tr -d '"')
if [[ $status == "enabled" ]]; then
echo "Success"
else
echo "Expected status: enabled, actual status: $status"
exit 1
fi
cleanup
## change working directory
cd ../../../
@@ -0,0 +1,61 @@
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 4096;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
keepalive_timeout 65;
# include /etc/nginx/conf.d/*.conf;
upstream minio {
server site1-minio1:9000;
server site1-minio2:9000;
server site1-minio3:9000;
server site1-minio4:9000;
}
server {
listen 9001;
listen [::]:9001;
server_name localhost;
# To allow special characters in headers
ignore_invalid_headers off;
# Allow any size file to be uploaded.
# Set to a value such as 1000m; to restrict file size to a specific value
client_max_body_size 0;
# To disable buffering
proxy_buffering off;
proxy_request_buffering off;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_connect_timeout 300;
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
proxy_http_version 1.1;
proxy_set_header Connection "";
chunked_transfer_encoding off;
proxy_pass http://minio;
}
}
}
@@ -0,0 +1,61 @@
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 4096;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
keepalive_timeout 65;
# include /etc/nginx/conf.d/*.conf;
upstream minio {
server site2-minio1:9000;
server site2-minio2:9000;
server site2-minio3:9000;
server site2-minio4:9000;
}
server {
listen 9002;
listen [::]:9002;
server_name localhost;
# To allow special characters in headers
ignore_invalid_headers off;
# Allow any size file to be uploaded.
# Set to a value such as 1000m; to restrict file size to a specific value
client_max_body_size 0;
# To disable buffering
proxy_buffering off;
proxy_request_buffering off;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_connect_timeout 300;
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
proxy_http_version 1.1;
proxy_set_header Connection "";
chunked_transfer_encoding off;
proxy_pass http://minio;
}
}
}
+63 -48
View File
@@ -4,14 +4,10 @@ on:
push:
tags:
- "RELEASE.*"
workflow_dispatch:
inputs:
tag:
description: "Release tag (e.g. RELEASE.2026-03-24T12-00-00Z)"
required: true
permissions:
contents: write
packages: write
jobs:
release:
@@ -21,11 +17,6 @@ jobs:
uses: actions/checkout@v4
with:
fetch-depth: 0
# Build the code at the tag being released, not whatever branch the
# dispatch ran from. On a tag push this is the tag ref already; on
# workflow_dispatch it pins the checkout to the requested tag so the
# artifacts cannot be built from one ref and published under another.
ref: ${{ github.event.inputs.tag || github.ref }}
- name: Set up Go
uses: actions/setup-go@v5
@@ -33,34 +24,16 @@ jobs:
go-version-file: go.mod
cache: true
- name: Verify clean checkout
run: |
set -euo pipefail
if [ -n "$(git status --porcelain)" ]; then
echo "Refusing to release from a dirty working tree:" >&2
git status --porcelain >&2
exit 1
fi
- name: Compute release variables
env:
# Passed through the environment, never interpolated into the script
# body: a dispatch input reaches bash as data, so it cannot inject
# commands the way a `${{ ... }}` splice into the source would.
INPUT_TAG: ${{ github.event.inputs.tag }}
run: |
set -euo pipefail
TAG="${INPUT_TAG:-${GITHUB_REF_NAME}}"
# Whitelist the exact tag shape before the value is used anywhere. bash
# =~ anchors ^...$ to the whole string (not per line, as sed would), so
# a tag carrying a newline cannot pass and then smuggle extra lines into
# $GITHUB_ENV below.
if [[ ! "${TAG}" =~ ^RELEASE\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}Z$ ]]; then
echo "Invalid release tag format: ${TAG}" >&2
exit 1
fi
TAG="${GITHUB_REF_NAME}"
VERSION_HYPHEN="${TAG#RELEASE.}"
PKG_VERSION="$(echo "${VERSION_HYPHEN}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')"
if [ "${PKG_VERSION}" = "${VERSION_HYPHEN}" ]; then
echo "Invalid release tag format: ${TAG}"
exit 1
fi
VERSION_COLON="$(echo "${VERSION_HYPHEN}" | sed -E 's/T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/T\1:\2:\3Z/')"
LDFLAGS="$(MINIO_RELEASE=RELEASE go run buildscripts/gen-ldflags.go "${VERSION_COLON}")"
@@ -74,7 +47,29 @@ jobs:
echo "Package version: ${PKG_VERSION}"
echo "LDFLAGS: ${LDFLAGS}"
- name: Build Draft release with GoReleaser
- name: Validate Docker Hub credentials
run: |
set -euo pipefail
if [ -z "${{ secrets.DOCKERHUB_USERNAME }}" ] || [ -z "${{ secrets.DOCKERHUB_TOKEN }}" ]; then
echo "Missing Docker Hub credentials. Set DOCKERHUB_USERNAME and DOCKERHUB_TOKEN repository secrets."
exit 1
fi
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: arm64
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and publish with GoReleaser
uses: goreleaser/goreleaser-action@v6
with:
version: "~> v2"
@@ -84,34 +79,54 @@ jobs:
LDFLAGS: ${{ env.LDFLAGS }}
PKG_VERSION: ${{ env.PKG_VERSION }}
- name: Verify binary provenance stamps
- name: Install pkger
run: |
set -euo pipefail
buildscripts/verify-build-provenance.sh
- name: Install nFPM
run: |
set -euo pipefail
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.47.0
go install github.com/minio/pkger/v2@v2.6.18
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
- name: Build nFPM packages
- name: Prepare package layout
run: |
set -euo pipefail
buildscripts/package-release.sh
copy_binary() {
local arch="$1"
local pattern="$2"
local src
src="$(find dist -maxdepth 2 -type f -path "dist/${pattern}/minio" | head -n1 || true)"
if [ -z "${src}" ]; then
echo "Missing GoReleaser binary for ${arch} (${pattern})"
exit 1
fi
mkdir -p "dist/linux-${arch}"
cp "${src}" "dist/linux-${arch}/minio.${RELEASE_TAG}"
}
- name: Upload nFPM packages to Draft release
copy_binary amd64 "minio_linux_amd64*"
copy_binary arm64 "minio_linux_arm64*"
- name: Build standard pkger packages
run: |
set -euo pipefail
pkger -r "${RELEASE_TAG}" --appName minio --releaseDir dist --ignore
# Keep only full package files; drop convenience symlinks (minio.rpm/minio.deb/minio.apk)
find dist/linux-* -maxdepth 1 -type l \
\( -name 'minio.rpm' -o -name 'minio.deb' -o -name 'minio.apk' \) -delete
find dist -maxdepth 2 -type f \
\( -name '*.rpm' -o -name '*.deb' -o -name '*.apk' -o -name '*.sha256sum' -o -name 'downloads-minio.json' \) | sort
- name: Upload pkger artifacts to GitHub release
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
mapfile -t files < <(find dist/packages -maxdepth 1 -type f \
\( -name '*.rpm' -o -name '*.deb' -o -name '*.apk' -o -name '*.sha256sum' \) | sort)
mapfile -t files < <(find dist -maxdepth 2 -type f \
\( -name '*.rpm' -o -name '*.deb' -o -name '*.apk' -o -name '*.sha256sum' -o -name 'downloads-minio.json' \) | sort)
if [ "${#files[@]}" -eq 0 ]; then
echo "No packages were generated."
exit 1
fi
gh release upload "${RELEASE_TAG}" "${files[@]}"
gh release upload "${RELEASE_TAG}" "${files[@]}" --clobber
- name: Upload dist artifact
if: always()
+79
View File
@@ -0,0 +1,79 @@
name: MinIO advanced tests
on:
pull_request:
branches:
- master
# This ensures that previous jobs for the PR are canceled when the PR is
# updated.
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
replication-test:
name: Advanced Tests with Go ${{ matrix.go-version }}
runs-on: ubuntu-latest
strategy:
matrix:
go-version: [1.24.x]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
check-latest: true
- name: Test Decom
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
make test-decom
- name: Test ILM
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
make test-ilm
make test-ilm-transition
- name: Test PBAC
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
make test-pbac
- name: Test Config File
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
make test-configfile
- name: Test Replication
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
make test-replication
- name: Test MinIO IDP for automatic site replication
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
make test-site-replication-minio
- name: Test Versioning
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
make test-versioning
- name: Test Multipart upload with failures
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
make test-multipart
+34
View File
@@ -0,0 +1,34 @@
name: Root lockdown tests
on:
pull_request:
branches:
- master
# This ensures that previous jobs for the PR are canceled when the PR is
# updated.
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
build:
name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [1.24.x]
os: [ubuntu-latest]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
check-latest: true
- name: Start root lockdown tests
run: |
make test-root-disable
+9
View File
@@ -0,0 +1,9 @@
-----BEGIN CERTIFICATE-----
MIIBKDCB26ADAgECAhB6vebGMUfKnmBKyqoApRSOMAUGAytlcDAbMRkwFwYDVQQD
DBByb290QHBsYXkubWluLmlvMB4XDTIwMDQzMDE1MjIyNVoXDTI1MDQyOTE1MjIy
NVowGzEZMBcGA1UEAwwQcm9vdEBwbGF5Lm1pbi5pbzAqMAUGAytlcAMhALzn735W
fmSH/ghKs+4iPWziZMmWdiWr/sqvqeW+WwSxozUwMzAOBgNVHQ8BAf8EBAMCB4Aw
EwYDVR0lBAwwCgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAFBgMrZXADQQDZOrGK
b2ATkDlu2pTcP3LyhSBDpYh7V4TvjRkBTRgjkacCzwFLm+mh+7US8V4dBpIDsJ4u
uWoF0y6vbLVGIlkG
-----END CERTIFICATE-----
+3
View File
@@ -0,0 +1,3 @@
-----BEGIN PRIVATE KEY-----
MC4CAQAwBQYDK2VwBCIEID9E7FSYWrMD+VjhI6q545cYT9YOyFxZb7UnjEepYDRc
-----END PRIVATE KEY-----
+64
View File
@@ -0,0 +1,64 @@
#!/bin/bash
set -ex
export MODE="$1"
export ACCESS_KEY="$2"
export SECRET_KEY="$3"
export JOB_NAME="$4"
export MINT_MODE="full"
docker system prune -f || true
docker volume prune -f || true
docker volume rm $(docker volume ls -f dangling=true) || true
## change working directory
cd .github/workflows/mint
## always pull latest
docker pull docker.io/minio/mint:edge
docker-compose -f minio-${MODE}.yaml up -d
sleep 1m
docker system prune -f || true
docker volume prune -f || true
docker volume rm $(docker volume ls -q -f dangling=true) || true
# Stop two nodes, one of each pool, to check that all S3 calls work while quorum is still there
[ "${MODE}" == "pools" ] && docker-compose -f minio-${MODE}.yaml stop minio2
[ "${MODE}" == "pools" ] && docker-compose -f minio-${MODE}.yaml stop minio6
# Pause one node, to check that all S3 calls work while one node goes wrong
[ "${MODE}" == "resiliency" ] && docker-compose -f minio-${MODE}.yaml pause minio4
docker run --rm --net=mint_default \
--name="mint-${MODE}-${JOB_NAME}" \
-e SERVER_ENDPOINT="nginx:9000" \
-e ACCESS_KEY="${ACCESS_KEY}" \
-e SECRET_KEY="${SECRET_KEY}" \
-e ENABLE_HTTPS=0 \
-e MINT_MODE="${MINT_MODE}" \
docker.io/minio/mint:edge
# FIXME: enable this after fixing aws-sdk-java-v2 tests
# # unpause the node, to check that all S3 calls work while one node goes wrong
# [ "${MODE}" == "resiliency" ] && docker-compose -f minio-${MODE}.yaml unpause minio4
# [ "${MODE}" == "resiliency" ] && docker run --rm --net=mint_default \
# --name="mint-${MODE}-${JOB_NAME}" \
# -e SERVER_ENDPOINT="nginx:9000" \
# -e ACCESS_KEY="${ACCESS_KEY}" \
# -e SECRET_KEY="${SECRET_KEY}" \
# -e ENABLE_HTTPS=0 \
# -e MINT_MODE="${MINT_MODE}" \
# docker.io/minio/mint:edge
docker-compose -f minio-${MODE}.yaml down || true
sleep 10s
docker system prune -f || true
docker volume prune -f || true
docker volume rm $(docker volume ls -q -f dangling=true) || true
## change working directory
cd ../../../
+22
View File
@@ -0,0 +1,22 @@
name: Shell formatting checks
on:
pull_request:
branches:
- master
permissions:
contents: read
jobs:
build:
name: runner / shfmt
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: luizm/action-sh-checker@master
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SHFMT_OPTS: "-s"
with:
sh_checker_shellcheck_disable: true # disable for now
+26 -214
View File
@@ -5,17 +5,10 @@ on:
pull_request:
paths:
- ".github/goreleaser.yml"
- ".github/nfpm.yml"
- "Dockerfile.goreleaser"
- "dockerscripts/docker-entrypoint.sh"
- "minio.service"
- "buildscripts/package-release.sh"
- "buildscripts/sign-release-rpms.sh"
- "buildscripts/verify-build-provenance.sh"
- "buildscripts/gen-ldflags.go"
- ".github/workflows/release.yml"
- ".github/workflows/test-release.yml"
- ".gitignore"
permissions:
contents: read
@@ -42,11 +35,9 @@ jobs:
VERSION_COLON="2026-02-14T12:00:00Z"
PKG_VERSION="20260214120000.0.0"
LDFLAGS="$(MINIO_RELEASE=RELEASE go run buildscripts/gen-ldflags.go "${VERSION_COLON}")"
{
echo "RELEASE_TAG=${RELEASE_TAG}"
echo "PKG_VERSION=${PKG_VERSION}"
echo "LDFLAGS=${LDFLAGS}"
} >> "${GITHUB_ENV}"
echo "RELEASE_TAG=${RELEASE_TAG}" >> "${GITHUB_ENV}"
echo "PKG_VERSION=${PKG_VERSION}" >> "${GITHUB_ENV}"
echo "LDFLAGS=${LDFLAGS}" >> "${GITHUB_ENV}"
echo "PKG_VERSION: ${PKG_VERSION}"
echo "LDFLAGS: ${LDFLAGS}"
@@ -65,213 +56,34 @@ jobs:
LDFLAGS: ${{ env.LDFLAGS }}
PKG_VERSION: ${{ env.PKG_VERSION }}
- name: Verify binary provenance stamps
- name: Install pkger
run: |
set -euo pipefail
buildscripts/verify-build-provenance.sh
- name: Install package validation tools
run: |
set -euo pipefail
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.47.0
go install github.com/minio/pkger/v2@v2.6.18
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
sudo apt-get update
sudo apt-get install --yes rpm binutils
- name: Package snapshot binaries with nFPM
- name: Package snapshot binaries with pkger
run: |
set -euo pipefail
buildscripts/package-release.sh
- name: Validate package names, checksums, metadata, and payload
run: |
set -euo pipefail
cd dist/packages
# These are the public download names; a drift here breaks every
# script that fetches packages by URL.
expected=(
"minio-${PKG_VERSION}-1.aarch64.rpm"
"minio-${PKG_VERSION}-1.x86_64.rpm"
"minio_${PKG_VERSION}_aarch64.apk"
"minio_${PKG_VERSION}_amd64.deb"
"minio_${PKG_VERSION}_arm64.deb"
"minio_${PKG_VERSION}_x86_64.apk"
)
for package in "${expected[@]}"; do
test -s "${package}"
test -s "${package}.sha256sum"
sha256sum --check "${package}.sha256sum"
done
test "$(find . -maxdepth 1 -type f \( -name '*.rpm' -o -name '*.deb' -o -name '*.apk' \) | wc -l)" -eq 6
# The signing script asserts these same values, but it runs on the
# maintainer's machine after the release workflow has already built
# and uploaded. Take its expectations as the single source of truth
# so nfpm.yml and the signing script cannot drift apart without
# failing here first, while a fix is still cheap.
#
# This grep is deliberately limited to the seven identity variables,
# all of which are single-line. That is what makes the eval safe:
# should one ever become multi-line, the grep captures an
# unterminated quote and the eval aborts on a syntax error under
# set -e rather than quietly binding an empty value and comparing
# against nothing. expected_payload is multi-line by design and must
# stay out of this set for the same reason.
eval "$(grep -E '^expected_(vendor|packager|url|summary|description|license|group)=' \
../../buildscripts/sign-release-rpms.sh)"
for value in "${expected_vendor}" "${expected_packager}" "${expected_url}" \
"${expected_summary}" "${expected_description}" \
"${expected_license}" "${expected_group}"; do
test -n "${value}"
done
service_sha="$(sha256sum ../../minio.service | awk '{print $1}')"
rpm_file="minio-${PKG_VERSION}-1.x86_64.rpm"
test "$(rpm -qp --queryformat '%{VENDOR}' "${rpm_file}")" = "${expected_vendor}"
test "$(rpm -qp --queryformat '%{PACKAGER}' "${rpm_file}")" = "${expected_packager}"
test "$(rpm -qp --queryformat '%{URL}' "${rpm_file}")" = "${expected_url}"
test "$(rpm -qp --queryformat '%{SUMMARY}' "${rpm_file}")" = "${expected_summary}"
test "$(rpm -qp --queryformat '%{DESCRIPTION}' "${rpm_file}")" = "${expected_description}"
test "$(rpm -qp --queryformat '%{LICENSE}' "${rpm_file}")" = "${expected_license}"
test "$(rpm -qp --queryformat '%{GROUP}' "${rpm_file}")" = "${expected_group}"
# Both payload entries: the unit file is as much a part of the
# package as the binary, and losing it would install a server with
# nothing to start it.
rpm -qpl "${rpm_file}" | grep -Fx '/usr/local/bin/minio'
rpm -qpl "${rpm_file}" | grep -Fx '/usr/lib/systemd/system/minio.service'
test "$(rpm -qpl "${rpm_file}" | wc -l)" -eq 2
deb_file="minio_${PKG_VERSION}_amd64.deb"
test "$(dpkg-deb --field "${deb_file}" Maintainer)" = "${expected_packager}"
test "$(dpkg-deb --field "${deb_file}" Version)" = "${PKG_VERSION}"
test "$(dpkg-deb --field "${deb_file}" License)" = "${expected_license}"
test "$(dpkg-deb --field "${deb_file}" Section)" = "utils"
test "$(dpkg-deb --field "${deb_file}" Homepage)" = "${expected_url}"
test "$(dpkg-deb --field "${deb_file}" Description)" = "${expected_description}"
dpkg-deb --contents "${deb_file}" | grep -E 'usr/local/bin/minio$'
dpkg-deb --contents "${deb_file}" | grep -E 'usr/lib/systemd/system/minio\.service$'
apk_info="$(tar -xOzf "minio_${PKG_VERSION}_x86_64.apk" .PKGINFO)"
grep -Fx "pkgver = ${PKG_VERSION}" <<< "${apk_info}"
grep -Fx "url = ${expected_url}" <<< "${apk_info}"
grep -Fx "maintainer = ${expected_packager}" <<< "${apk_info}"
grep -Fx "license = ${expected_license}" <<< "${apk_info}"
grep -Fx "pkgdesc = ${expected_description}" <<< "${apk_info}"
tar -tzf "minio_${PKG_VERSION}_x86_64.apk" | grep -Fx 'usr/local/bin/minio'
tar -tzf "minio_${PKG_VERSION}_x86_64.apk" | grep -Fx 'usr/lib/systemd/system/minio.service'
for arch in amd64 arm64; do
if [ "${arch}" = amd64 ]; then
rpm_arch=x86_64
deb_arch=amd64
apk_arch=x86_64
else
rpm_arch=aarch64
deb_arch=arm64
apk_arch=aarch64
copy_binary() {
local arch="$1"
local pattern="$2"
local src
src="$(find dist -maxdepth 2 -type f -path "dist/${pattern}/minio" | head -n1 || true)"
if [ -z "${src}" ]; then
echo "Missing GoReleaser binary for ${arch} (${pattern})"
exit 1
fi
test "$(rpm -qp --queryformat '%{ARCH}' "minio-${PKG_VERSION}-1.${rpm_arch}.rpm")" = "${rpm_arch}"
test "$(dpkg-deb --field "minio_${PKG_VERSION}_${deb_arch}.deb" Architecture)" = "${deb_arch}"
grep -Fx "arch = ${apk_arch}" <<< "$(tar -xOzf "minio_${PKG_VERSION}_${apk_arch}.apk" .PKGINFO)"
# Accepted weakness: this takes the first match, unsorted, where
# find_binary in package-release.sh demands exactly one. It cannot
# be reached with an ambiguous match today, because packaging runs
# earlier in this same job and hard-fails on one. Revisit if
# goamd64 gains a second level, or if find_binary's exactly-one
# contract is ever relaxed -- at that point this weak copy would be
# the only one left choosing silently.
source_binary="$(find .. -maxdepth 2 -type f -path "../minio_linux_${arch}*/minio" | head -n 1)"
source_sha="$(sha256sum "${source_binary}" | awk '{print $1}')"
# Do not pipe rpm2cpio here: Debian's build exits non-zero even when
# it writes a correct payload, which trips `set -o pipefail`. Use
# rpm's own digests instead -- -K checks the payload against the
# header, and FILEDIGESTS is the sha256 rpm itself verifies on
# install.
rpm -K "minio-${PKG_VERSION}-1.${rpm_arch}.rpm"
rpm_sha="$(rpm -qp --queryformat '[%{FILENAMES} %{FILEDIGESTS}\n]' \
"minio-${PKG_VERSION}-1.${rpm_arch}.rpm" | awk '$1 == "/usr/local/bin/minio" { print $2 }')"
rpm_service_sha="$(rpm -qp --queryformat '[%{FILENAMES} %{FILEDIGESTS}\n]' \
"minio-${PKG_VERSION}-1.${rpm_arch}.rpm" | awk '$1 == "/usr/lib/systemd/system/minio.service" { print $2 }')"
deb_sha="$(ar p "minio_${PKG_VERSION}_${deb_arch}.deb" data.tar.gz | tar -xzOf - ./usr/local/bin/minio | sha256sum | awk '{print $1}')"
deb_service_sha="$(ar p "minio_${PKG_VERSION}_${deb_arch}.deb" data.tar.gz | tar -xzOf - ./usr/lib/systemd/system/minio.service | sha256sum | awk '{print $1}')"
apk_sha="$(tar -xzOf "minio_${PKG_VERSION}_${apk_arch}.apk" usr/local/bin/minio | sha256sum | awk '{print $1}')"
apk_service_sha="$(tar -xzOf "minio_${PKG_VERSION}_${apk_arch}.apk" usr/lib/systemd/system/minio.service | sha256sum | awk '{print $1}')"
test "${source_sha}" = "${rpm_sha}"
test "${source_sha}" = "${deb_sha}"
test "${source_sha}" = "${apk_sha}"
test "${service_sha}" = "${rpm_service_sha}"
test "${service_sha}" = "${deb_service_sha}"
test "${service_sha}" = "${apk_service_sha}"
done
find . -maxdepth 1 -type f | sort
- name: Build release runtime image and verify graceful shutdown
run: |
set -euo pipefail
# docker-release.yml is workflow_dispatch only, so this is the only
# automated build of the release runtime layer and entrypoint before a
# real publish. Assemble a minimal image from the linux/amd64 binary
# goreleaser already produced; the mcli-download build stage is skipped
# on purpose to keep this gate offline and deterministic.
ctx="$(mktemp -d)"
tar -xzf "dist/minio_${PKG_VERSION}_linux_amd64.tar.gz" -C "${ctx}" minio
cp dockerscripts/docker-entrypoint.sh "${ctx}/docker-entrypoint.sh"
{
echo "FROM registry.access.redhat.com/ubi9/ubi-micro:latest"
echo "COPY minio /usr/bin/minio"
echo "COPY docker-entrypoint.sh /usr/bin/docker-entrypoint.sh"
echo "RUN mkdir -p /data && chmod 0777 /data && chmod +x /usr/bin/minio /usr/bin/docker-entrypoint.sh"
echo 'ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]'
echo 'CMD ["minio"]'
} > "${ctx}/Dockerfile"
docker build -t minio-runtime-test:snapshot "${ctx}"
# PID 1 must be minio, not the entry shell, on every privilege path, so
# a SIGTERM from docker stop reaches the server and it exits gracefully
# instead of being killed at the stop timeout. Regression guard for the
# exec-into-chroot entrypoint fix.
assert_graceful() {
name="$1"; shift
docker rm -f "${name}" >/dev/null 2>&1 || true
docker run -d --name "${name}" \
-e MINIO_CI_CD=1 -e MINIO_ROOT_USER=ciadmin -e MINIO_ROOT_PASSWORD=ciadmin-secret-123 \
"$@" minio-runtime-test:snapshot minio server /data --address :9000 >/dev/null
up=""
for _ in $(seq 1 60); do
if docker logs "${name}" 2>&1 | grep -q "API:"; then up=1; break; fi
if [ "$(docker inspect -f '{{.State.Running}}' "${name}")" != "true" ]; then break; fi
sleep 1
done
if [ -z "${up}" ]; then echo "server did not start (${name}):"; docker logs "${name}" | tail -5; exit 1; fi
pid1="$(docker exec "${name}" cat /proc/1/comm 2>/dev/null || echo '?')"
start="$(date +%s)"; docker stop -t 15 "${name}" >/dev/null; end="$(date +%s)"
code="$(docker inspect -f '{{.State.ExitCode}}' "${name}")"
elapsed=$((end - start))
echo "${name}: pid1=${pid1} stop=${elapsed}s exit=${code}"
graceful=0; docker logs "${name}" 2>&1 | grep -q "Exiting on signal" && graceful=1
docker rm -f "${name}" >/dev/null 2>&1 || true
[ "${graceful}" = "1" ] || { echo "no graceful-shutdown log (${name}) - signal not forwarded"; exit 1; }
[ "${code}" = "0" ] || { echo "non-zero exit (${name}): ${code}"; exit 1; }
[ "${elapsed}" -lt 10 ] || { echo "shutdown too slow (${name}): ${elapsed}s - signal not forwarded"; exit 1; }
mkdir -p "dist/linux-${arch}"
cp "${src}" "dist/linux-${arch}/minio.${RELEASE_TAG}"
}
assert_graceful minio-rt-default
assert_graceful minio-rt-dropuser -e MINIO_USERNAME=minio-user -e MINIO_GROUPNAME=minio-group
- name: Validate release scripts
run: |
set -euo pipefail
bash -n buildscripts/package-release.sh
bash -n buildscripts/sign-release-rpms.sh
bash -n buildscripts/verify-build-provenance.sh
test -x buildscripts/package-release.sh
test -x buildscripts/sign-release-rpms.sh
test -x buildscripts/verify-build-provenance.sh
copy_binary amd64 "minio_linux_amd64*"
copy_binary arm64 "minio_linux_arm64*"
pkger -r "${RELEASE_TAG}" --appName minio --releaseDir dist --ignore
# Keep only full package files; drop convenience symlinks (minio.rpm/minio.deb/minio.apk)
find dist/linux-* -maxdepth 1 -type l \
\( -name 'minio.rpm' -o -name 'minio.deb' -o -name 'minio.apk' \) -delete
find dist -maxdepth 2 -type f \
\( -name '*.rpm' -o -name '*.deb' -o -name '*.apk' -o -name '*.sha256sum' -o -name 'downloads-minio.json' \) | sort
+15
View File
@@ -0,0 +1,15 @@
---
name: Spelling
on: [pull_request]
jobs:
run:
name: Spell Check with Typos
runs-on: ubuntu-latest
steps:
- name: Checkout Actions Repository
uses: actions/checkout@v4
- name: Check spelling of repo
uses: crate-ci/typos@master
+34
View File
@@ -0,0 +1,34 @@
name: Upgrade old version tests
on:
pull_request:
branches:
- master
# This ensures that previous jobs for the PR are canceled when the PR is
# updated.
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
build:
name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [1.24.x]
os: [ubuntu-latest]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
check-latest: true
- name: Start upgrade tests
run: |
make test-upgrade
+9 -14
View File
@@ -1,36 +1,31 @@
name: VulnCheck
on:
pull_request:
branches:
- master
push:
branches:
- master
workflow_dispatch:
permissions:
contents: read
contents: read # to fetch code (actions/checkout)
jobs:
vulncheck:
name: Analysis
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- name: Check out code
- name: Check out code into the Go module directory
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true
- name: Install govulncheck
run: |
go install golang.org/x/vuln/cmd/govulncheck@v1.6.0
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
go-version: 1.24.x
cached: false
- name: Get official govulncheck
run: go install golang.org/x/vuln/cmd/govulncheck@latest
shell: bash
- name: Run govulncheck
run: govulncheck -show verbose ./...
shell: bash
+1 -3
View File
@@ -55,14 +55,12 @@ xattr
xl-meta
.gitignore
.goreleaser.yml
dist/
.claude/
.codex/
AGENTS.md
CLAUDE.md
_bmad/
_bmad-output/
docs/security/
+1 -63
View File
@@ -1,62 +1,3 @@
FROM golang:1.26.5-alpine AS build
ARG TARGETARCH
ENV GOPATH=/go
ENV CGO_ENABLED=0
ARG MC_REPO=pgsty/mc
ARG MC_VERSION=latest
RUN apk add -U --no-cache \
ca-certificates \
bash \
curl \
jq && \
case "${TARGETARCH}" in \
amd64) MC_ARCH=amd64 ;; \
arm64) MC_ARCH=arm64 ;; \
*) echo "Unsupported TARGETARCH=${TARGETARCH}"; exit 1 ;; \
esac && \
if [ "${MC_VERSION}" = "latest" ]; then \
MC_RELEASE_URL="https://api.github.com/repos/${MC_REPO}/releases/latest"; \
else \
MC_RELEASE_URL="https://api.github.com/repos/${MC_REPO}/releases/tags/${MC_VERSION}"; \
fi && \
curl -fsSL "${MC_RELEASE_URL}" -o /tmp/mc-release.json && \
MC_ARCHIVE_URL=$(jq -r --arg arch "${MC_ARCH}" \
'.assets[] | select(.name | endswith("_linux_" + $arch + ".tar.gz")) | .browser_download_url' \
/tmp/mc-release.json | head -n 1) && \
MC_CHECKSUM_URL=$(jq -r \
'.assets[] | select(.name | endswith("_checksums.txt")) | .browser_download_url' \
/tmp/mc-release.json | head -n 1) && \
[ -n "${MC_ARCHIVE_URL}" ] || { echo "Cannot find mcli archive for linux/${MC_ARCH}"; exit 1; } && \
[ -n "${MC_CHECKSUM_URL}" ] || { echo "Cannot find mcli checksums file"; exit 1; } && \
ARCHIVE_NAME=$(basename "${MC_ARCHIVE_URL}") && \
echo "Downloading ${ARCHIVE_NAME} ..." && \
curl -fsSL "${MC_ARCHIVE_URL}" -o /tmp/mcli.tar.gz && \
curl -fsSL "${MC_CHECKSUM_URL}" -o /tmp/mcli_checksums.txt && \
EXPECTED=$(grep " ${ARCHIVE_NAME}$" /tmp/mcli_checksums.txt | awk '{print $1}') && \
ACTUAL=$(sha256sum /tmp/mcli.tar.gz | awk '{print $1}') && \
[ -n "${EXPECTED}" ] || { echo "Checksum entry not found for ${ARCHIVE_NAME}"; exit 1; } && \
[ "${EXPECTED}" = "${ACTUAL}" ] || { echo "Checksum mismatch: expected ${EXPECTED}, got ${ACTUAL}"; exit 1; } && \
echo "Checksum OK: ${ACTUAL}" && \
mkdir -p /tmp/mcli-extract && \
tar -xzf /tmp/mcli.tar.gz -C /tmp/mcli-extract/ && \
if [ -f /tmp/mcli-extract/mcli ]; then \
cp /tmp/mcli-extract/mcli /go/bin/mcli; \
elif [ -f /tmp/mcli-extract/mc ]; then \
cp /tmp/mcli-extract/mc /go/bin/mcli; \
else \
echo "No mc or mcli binary found in archive:"; ls -la /tmp/mcli-extract/; exit 1; \
fi && \
chmod +x /go/bin/mcli && \
ln -sf mcli /go/bin/mc
COPY dockerscripts/download-static-curl.sh /build/download-static-curl
RUN chmod +x /build/download-static-curl && \
/build/download-static-curl
FROM registry.access.redhat.com/ubi9/ubi:latest AS certs
RUN dnf -y install ca-certificates && \
update-ca-trust && \
@@ -80,14 +21,11 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
COPY --from=certs /tmp/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
COPY minio /usr/bin/minio
COPY --from=build /go/bin/mcli /usr/bin/mcli
COPY --from=build /go/bin/curl* /usr/bin/
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
COPY LICENSE /licenses/LICENSE
COPY CREDITS /licenses/CREDITS
RUN chmod +x /usr/bin/minio /usr/bin/mcli /usr/bin/docker-entrypoint.sh && \
ln -sf mcli /usr/bin/mc
RUN chmod +x /usr/bin/minio /usr/bin/docker-entrypoint.sh
EXPOSE 9000
VOLUME ["/data"]
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.26.5-alpine as build
FROM golang:1.24-alpine as build
ARG TARGETARCH
ARG RELEASE
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.26.5-alpine AS build
FROM golang:1.24-alpine AS build
ARG TARGETARCH
ARG RELEASE
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.26.5-alpine AS build
FROM golang:1.24-alpine AS build
ARG TARGETARCH
ARG RELEASE
+5 -23
View File
@@ -4,7 +4,6 @@ LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
GOOS ?= $(shell go env GOOS)
GOARCH ?= $(shell go env GOARCH)
GOLANGCI_VERSION ?= v2.11.3
VERSION ?= $(shell git describe --tags)
REPO ?= quay.io/minio
@@ -24,14 +23,7 @@ help: ## print this help
getdeps: ## fetch necessary dependencies
@mkdir -p ${GOPATH}/bin
@if [ ! -x "$(GOLANGCI)" ]; then \
set -e; \
echo "Installing golangci-lint $(GOLANGCI_VERSION)"; \
script=$$(mktemp); \
trap 'rm -f "$$script"' EXIT; \
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/$(GOLANGCI_VERSION)/install.sh -o "$$script"; \
sh "$$script" -b $(GOLANGCI_DIR) $(GOLANGCI_VERSION); \
fi
@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR)
crosscompile: ## cross compile minio
@(env bash $(PWD)/buildscripts/cross-compile.sh)
@@ -40,24 +32,14 @@ verifiers: lint check-gen
check-gen: ## check for updated autogenerated files
@go generate ./... >/dev/null
@go mod tidy -compat=1.26
@changed=$$(git diff --name-only -- '*_gen.go' '*_gen_test.go' '*_msgp_test.go' '*_string.go' go.mod go.sum); \
if [ -n "$$changed" ]; then \
echo "Non-committed generated changes detected:"; \
echo "$$changed"; \
exit 1; \
fi
@untracked=$$(git ls-files --others --exclude-standard -- '*_gen.go' '*_gen_test.go' '*_msgp_test.go' '*_string.go'); \
if [ -n "$$untracked" ]; then \
echo "Untracked generated files detected:"; \
echo "$$untracked"; \
exit 1; \
fi
@go mod tidy -compat=1.21
@(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false)
@(! git diff --name-only | grep 'go.sum') || (echo "Non-committed changes in auto-generated go.sum is detected, please commit them to proceed." && false)
lint: getdeps ## runs golangci-lint suite of linters
@echo "Running $@ check"
@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml
@if command -v typos >/dev/null 2>&1; then typos ./; else echo "typos binary is not found.. skipping.."; fi
@command typos && typos ./ || echo "typos binary is not found.. skipping.."
lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes
@echo "Running $@ check"
+102 -137
View File
@@ -1,165 +1,130 @@
> [!WARNING]
> **This branch is archived and receives no further changes.**
>
> `minio` holds the final state of this project under the MinIO identity. Development continues on **[`main`](https://github.com/pgsty/silo/tree/main)**, where the project is named **Silo** and the binary, packages, systemd unit, container image, and Helm chart are all named `silo`. On 2026-08-06 the repository was renamed `pgsty/minio` → **[`pgsty/silo`](https://github.com/pgsty/silo)** and its default branch `master` → `main`.
>
> The last release cut from this branch is **[`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z)** (2026-08-04). Its 19 assets carry the `minio` artifact names, and the matching container image is `docker.io/pgsty/minio:RELEASE.2026-08-04T00-00-00Z`. Those artifacts stay published and unmodified — no tag is moved, re-signed, or removed. If you need the artifacts maintained under the original MinIO identity, this branch and the releases up to that tag are where they live. Later releases carry the `silo` names.
>
> **Nothing on the wire changed with the rename.** `MINIO_*` environment variables, `minio_*` Prometheus metrics, `x-minio-*` headers, `/minio/*` routes, the `.minio.sys` on-disk layout, IAM and ARN values, and the `github.com/minio/minio` Go module path are all preserved on `main`. A Silo server reads data written by this release, and the packages install side by side, so migrating or rolling back stays an explicit administrator action.
<h1 align="center">
<img src=".github/silo-word.svg" alt="SILO" height="80">
</h1>
<p align="center">
<strong>A conservatively maintained MinIO fork</strong><br>
Security maintenance, versioned release artifacts, and operational continuity for existing deployments.
</p>
<p align="center">
<a href="https://silo.pgsty.com/">Website</a> ·
<a href="https://silo.pgsty.com/docs/">Documentation</a> ·
<a href="https://silo.pgsty.com/download/">Download</a> ·
<a href="https://silo.pgsty.com/blog/">Blog</a> ·
<a href="https://github.com/pgsty/minio/releases">Releases</a> ·
<a href="SECURITY.md">Security</a> ·
<a href="README_ZH.md">中文</a>
</p>
<p align="center">
<a href="https://github.com/pgsty/minio/releases"><img alt="GitHub Release" src="https://img.shields.io/github/v/release/pgsty/minio?include_prereleases&label=release&logo=github"></a>
<a href="https://hub.docker.com/r/pgsty/minio"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/minio?logo=docker"></a>
<a href="go.mod"><img alt="Go Version" src="https://img.shields.io/github/go-mod/go-version/pgsty/minio?logo=go"></a>
<a href="LICENSE"><img alt="License" src="https://img.shields.io/badge/license-AGPLv3-blue"></a>
</p>
> [!IMPORTANT]
> Silo is an independent, community-maintained fork of the open-source MinIO server, published by [Pigsty](https://pigsty.io) from [`pgsty/minio`](https://github.com/pgsty/minio). It is not affiliated with, endorsed by, or sponsored by MinIO, Inc. “MinIO” is used only to identify the upstream project and compatibility lineage.
> **This is a community-maintained fork of [minio/minio](https://github.com/minio/minio), maintained by [Pigsty](https://pigsty.io).**
> This project is **not** affiliated with, endorsed by, or sponsored by MinIO, Inc.
> "MinIO" is a trademark of MinIO, Inc., used here solely to identify the upstream project.
>
> Changes from upstream are minimal:
> - Restored the embedded management console version reference
> - Updated documentation links and Go module paths to point to this repository
>
> Distributed under the original [GNU AGPLv3](LICENSE) license.
## Overview
# MinIO Quickstart Guide
Silo maintains one downstream release line based on MinIO [`RELEASE.2025-12-03T12-00-00Z`](https://github.com/minio/minio/releases/tag/RELEASE.2025-12-03T12-00-00Z). It provides maintained builds and release artifacts for existing MinIO-compatible deployments after upstream community distribution ended. Pigsty uses this fork for object storage as an optional PG backup repo.
[![Website: minio.pigsty.io](https://img.shields.io/badge/website-minio.pigsty.io-slategray?style=flat&logo=cilium&logoColor=white)](https://pigsty.io) [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![license](https://img.shields.io/badge/license-AGPL%20V3-blue)](https://github.com/pgsty/minio/blob/master/LICENSE)
The official project portal is [silo.pgsty.com](https://silo.pgsty.com/). It brings documentation, downloads, release and security notes, and project background together. English is served at the site root; Chinese is available under [/zh/](https://silo.pgsty.com/zh/).
MinIO is a high-performance, S3-compatible object storage solution released under the GNU AGPL v3.0 license.
Designed for speed and scalability, it powers AI/ML, analytics, and data-intensive workloads with industry-leading performance.
## Find the Right Resource
- S3 API Compatible Seamless integration with existing S3 tools
- Built for AI & Analytics Optimized for large-scale data pipelines
- High Performance Ideal for demanding storage workloads.
| Looking for | Canonical location |
| :-- | :-- |
| Project overview and navigation | [Silo Website](https://silo.pgsty.com/) ([中文](https://silo.pgsty.com/zh/)) |
| Installation methods and downloads | [Download & Install](https://silo.pgsty.com/download/) ([中文](https://silo.pgsty.com/zh/download/)) |
| Operations, administration, development, and reference | [Documentation](https://silo.pgsty.com/docs/) ([中文](https://silo.pgsty.com/zh/docs/)) |
| Project news, release notes, and security notes | [Blog](https://silo.pgsty.com/blog/), including [releases](https://silo.pgsty.com/blog/release/) and [security](https://silo.pgsty.com/blog/security/) |
| Versioned binaries, checksums, and source archives | [GitHub Releases](https://github.com/pgsty/minio/releases) |
| Bug reports and feature discussions | [GitHub Issues](https://github.com/pgsty/minio/issues) |
| License, attribution, and trademark information | [License](https://silo.pgsty.com/about/license/), [Attribution](https://silo.pgsty.com/about/attribution/), and [Trademark](https://silo.pgsty.com/about/trademark/) |
This README provides instructions for building MinIO from source and deploying onto baremetal hardware.
Use the [MinIO Documentation](https://github.com/minio/docs) project to build and host a local copy of the documentation.
## Maintenance Policy
## Install from Source
The active release line covers:
Use the following commands to compile and run a standalone MinIO server from source.
If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.24](https://golang.org/dl/#stable)
- build and dependency maintenance;
- applicable security fixes and advisories;
- focused fixes for reproducible defects;
- versioned binaries, packages, checksums, and multi-architecture images;
- the web console, client, documentation, and Pigsty integration.
Changes are kept narrow and tested where practical. Maintenance is best effort; no response, remediation, or release schedule is guaranteed.
### Out of scope
- a separate product roadmap, new storage engine, or speculative S3 features;
- broad rewrites or changes that materially expand the downstream delta;
- historical releases or multiple support branches;
- commercial support, SLAs, 24×7 coverage, or SUBNET access;
- deployment design, access control, monitoring, backup, or recovery.
## Compatibility
Silo aims to preserve:
- MinIO-compatible S3 APIs, configuration, environment variables, and CLI conventions;
- `RELEASE.YYYY-MM-DDTHH-MM-SSZ` tags, container entrypoints, and common deployment workflows.
Compatibility is the default constraint. Silo preserves existing wire, client, configuration, and operational behavior whenever doing so remains safe. Compatibility is broken only when necessary to close a major security issue, and the release notes must identify the affected behavior and migration path. Treat each release as a downstream upgrade: pin versions, review [release notes](https://silo.pgsty.com/blog/release/) and [security advisories](docs/security/advisories.md), keep a rollback path, and test before production use.
## Downloads and Release Artifacts
Use [Download & Install](https://silo.pgsty.com/download/) to choose an installation method. GitHub Releases remains the source for versioned server binaries, checksums, and source archives.
| Artifact | Location |
| :-- | :-- |
| Source | [`github.com/pgsty/minio`](https://github.com/pgsty/minio) |
| Container image | [`pgsty/minio`](https://hub.docker.com/r/pgsty/minio), multi-arch for `linux/amd64` and `linux/arm64` |
| Server binaries and checksums | [GitHub Releases](https://github.com/pgsty/minio/releases) for Linux, macOS, and Windows on `amd64` and `arm64` |
| Linux packages | RPM, DEB, and APK artifacts, also distributed through the [Pigsty repository](https://pigsty.io/docs/repo/) |
| Client | [`pgsty/mc`](https://github.com/pgsty/mc), bundled in the container as `mcli` with an `mc` compatibility alias |
| Console | Maintained [`georgmangold/console`](https://github.com/georgmangold/console) fork, embedded in the server build |
| Shared library | [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) v3.7.0, consumed through a `replace` directive while preserving `github.com/minio/pkg/v3` import paths ([release notes](https://silo.pgsty.com/blog/release/pkg-3.7.0/)) |
## Quick Start
For local evaluation:
```bash
mkdir -p data
export MINIO_ROOT_USER=minioadmin
export MINIO_ROOT_PASSWORD=change-me-long-password
docker run -d --name silo \
-p 9000:9000 \
-p 9001:9001 \
-e MINIO_ROOT_USER \
-e MINIO_ROOT_PASSWORD \
-v "$PWD/data:/data" \
pgsty/minio:latest server /data --console-address ":9001"
```sh
go install github.com/pgsty/minio@latest
```
Open the console at <http://localhost:9001>; the S3 API listens on <http://localhost:9000>.
You can alternatively run `go build` and use the `GOOS` and `GOARCH` environment variables to control the OS and architecture target.
For example:
The image includes the compatible client as `mcli`:
```bash
docker exec silo mcli alias set local http://127.0.0.1:9000 \
"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
docker exec silo mcli mb local/demo
docker exec silo mcli ls local
```
env GOOS=linux GOARCH=arm64 go build
```
> [!WARNING]
> For production, pin a release, use unique credentials and TLS, monitor the service, keep independent backups, and test recovery.
Start MinIO by running `minio server PATH` where `PATH` is any empty folder on your local filesystem.
Build the server from source:
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`.
You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server.
Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials.
You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
```bash
go build -o minio .
./minio --version
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool:
```sh
mc alias set local http://localhost:9000 minioadmin minioadmin
mc admin info local
```
For other installation paths—including native packages, binaries, Podman, Kubernetes, source, and Pigsty Ansible—use [Download & Install](https://silo.pgsty.com/download/). For production deployment and administration, start with the [Silo documentation](https://silo.pgsty.com/docs/). Pigsty users can also use the [Pigsty MinIO module](https://pigsty.io/docs/minio/).
See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool.
For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
## Security
## Build Docker Image
Security fixes target the active `master` branch and are recorded in the [advisory log](docs/security/advisories.md) and the portal's [security notes](https://silo.pgsty.com/blog/security/). Report vulnerabilities privately as described in [`SECURITY.md`](SECURITY.md) and [`VULNERABILITY_REPORT.md`](VULNERABILITY_REPORT.md). Report issues that also affect upstream MinIO there as well.
You can use the `docker build .` command to build a Docker image on your local host machine.
You must first [build MinIO](#install-from-source) and ensure the `minio` binary exists in the project root.
## Contributing
The following command builds the Docker image using the default `Dockerfile` in the root project directory with the repository and image tag `myminio:minio`
Useful contributions include security and dependency updates, reproducible bug fixes, tests, release automation, packaging, and documentation.
```sh
docker build -t myminio:minio .
```
Issues and pull requests should include the affected version, reproduction steps, impact, expected behavior, tests, and compatibility notes. Discuss large changes in an issue first.
Use `docker image ls` to confirm the image exists in your local repository.
You can run the server using standard Docker invocation:
## Background
```sh
docker run -p 9000:9000 -p 9001:9001 myminio:minio server /tmp/minio --console-address :9001
```
This project was created in response to changes in the upstream community distribution and maintenance model. The maintainers analysis, alternatives considered, and early maintenance record are documented below:
Complete documentation for building Docker containers, managing custom images, or loading images into orchestration platforms is out of scope for this documentation.
You can modify the `Dockerfile` and `dockerscripts/docker-entrypoint.sh` as-needed to reflect your specific image requirements.
| Essay | Subject |
| :-- | :-- |
| [MinIO Is Dead](https://silo.pgsty.com/blog/post/minio-is-dead/) | Changes to the upstream project and distribution model |
| [MinIO Is Dead, Long Live MinIO](https://silo.pgsty.com/blog/post/minio-resurrect/) | Establishing the fork and its release pipeline |
| [Two months into maintaining a MinIO fork](https://silo.pgsty.com/blog/post/minio-promise-kept/) | Initial security and maintenance work |
## Install using Helm Charts
## License and Trademark
There are two paths for installing MinIO onto Kubernetes infrastructure:
The server remains licensed under the [GNU Affero General Public License v3.0](LICENSE). See [`CREDITS`](CREDITS) for upstream authorship and attribution. MinIO is a trademark of MinIO, Inc. Silo and `pgsty/minio` are independent community efforts and are not affiliated with or endorsed by MinIO, Inc.
- Use the [MinIO Operator](https://github.com/minio/operator)
- Use the community-maintained [Helm charts](https://github.com/pgsty/minio/tree/master/helm/minio)
The Community Helm chart has instructions in the folder-level README.
## Test MinIO Connectivity
### Test using MinIO Console
MinIO Server comes with an embedded web based object browser.
Point your web browser to <http://127.0.0.1:9000> to ensure your server has started successfully.
> [!NOTE]
> MinIO runs console on random port by default, if you wish to choose a specific port use `--console-address` to pick a specific interface and port.
### Test using MinIO Client `mc`
`mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services.
The following commands set a local alias, validate the server information, create a bucket, copy data to that bucket, and list the contents of the bucket.
```sh
mc alias set local http://localhost:9000 minioadmin minioadmin
mc admin info
mc mb data
mc cp ~/Downloads/mydata data/
mc ls data/
```
Follow the MinIO Client [Quickstart Guide](https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart) for further instructions.
## Explore Further
- [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html)
- [Use `mc` with MinIO Server](https://min.io/docs/minio/linux/reference/minio-mc.html)
- [Use `minio-go` SDK with MinIO Server](https://min.io/docs/minio/linux/developers/go/minio-go.html)
- [MinIO V3 Metrics Reference](docs/metrics/v3.md)
## Contribute
Please follow MinIO [Contributor's Guide](https://github.com/minio/minio/blob/master/CONTRIBUTING.md) for guidance on making new contributions to the repository.
## License
- MinIO source is licensed under the [GNU AGPLv3](LICENSE).
- MinIO [documentation](docs/) is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
- [License Compliance](COMPLIANCE.md)
-172
View File
@@ -1,172 +0,0 @@
> [!WARNING]
> **本分支已归档,不再接受任何修改。**
>
> `minio` 分支保留本项目以 MinIO 身份存在的最后状态。开发已迁移到 **[`main`](https://github.com/pgsty/silo/tree/main)** 分支,项目在那里以 **Silo** 的身份继续维护,二进制、软件包、systemd unit、容器镜像与 Helm chart 全部更名为 `silo`。2026-08-06,仓库由 `pgsty/minio` 更名为 **[`pgsty/silo`](https://github.com/pgsty/silo)**,默认分支由 `master` 更名为 `main`。
>
> 本分支上切出的最后一个版本是 **[`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z)**2026-08-04)。它的 19 个资产使用 `minio` 命名,对应容器镜像为 `docker.io/pgsty/minio:RELEASE.2026-08-04T00-00-00Z`。这些产物保持已发布状态且不做改动 —— 不移动、不重新签名、不删除任何 tag。如果你需要以原本 MinIO 形态维持的归档构件,就在本分支以及截止到该 tag 的历次发布中。此后的版本使用 `silo` 命名。
>
> **更名没有改变任何对外接口。** `MINIO_*` 环境变量、`minio_*` 指标、`x-minio-*` 头、`/minio/*` 路由、`.minio.sys` 磁盘布局、IAM 与 ARN 取值,以及 Go 模块路径 `github.com/minio/minio`,在 `main` 上全部原样保留。Silo 服务端可直接读取本版本写入的数据;新旧软件包并存安装,因此迁移与回滚始终是管理员显式触发的动作。
<h1 align="center">
<img src=".github/silo.svg" alt="" height="80">
<img src=".github/silo-word.svg" alt="SILO" height="80">
</h1>
<p align="center">
<strong>审慎维护的 MinIO 社区分支</strong><br>
为现有部署提供安全维护、带版本的发行产物与持续运维支持。
</p>
<p align="center">
<a href="https://silo.pgsty.com/zh/">官网</a> ·
<a href="https://silo.pgsty.com/zh/docs/">文档</a> ·
<a href="https://silo.pgsty.com/zh/download/">下载</a> ·
<a href="https://silo.pgsty.com/zh/blog/">博客</a> ·
<a href="https://github.com/pgsty/minio/releases">版本发布</a> ·
<a href="SECURITY.md">安全策略</a> ·
<a href="README.md">English</a>
</p>
<p align="center">
<a href="https://github.com/pgsty/minio/releases"><img alt="GitHub Release" src="https://img.shields.io/github/v/release/pgsty/minio?include_prereleases&label=release&logo=github"></a>
<a href="https://hub.docker.com/r/pgsty/minio"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/minio?logo=docker"></a>
<a href="go.mod"><img alt="Go Version" src="https://img.shields.io/github/go-mod/go-version/pgsty/minio?logo=go"></a>
<a href="LICENSE"><img alt="License" src="https://img.shields.io/badge/license-AGPLv3-blue"></a>
</p>
> [!IMPORTANT]
> Silo 是由 [Pigsty](https://pigsty.cc) 独立维护、从 [`pgsty/minio`](https://github.com/pgsty/minio) 发布的开源 MinIO 社区分支。本项目与 MinIO, Inc. 不存在隶属、背书或赞助关系;文中使用 “MinIO” 仅用于说明上游项目及兼容谱系。
## 概述
Silo 维护一条基于 MinIO [`RELEASE.2025-12-03T12-00-00Z`](https://github.com/minio/minio/releases/tag/RELEASE.2025-12-03T12-00-00Z) 的下游版本线,为上游停止社区发行后仍在运行 MinIO 兼容部署的用户提供持续构建与发行产物。
Pigsty 使用本分支提供对象存储,包括 PostgreSQL 备份存储。
项目统一门户为 [silo.pgsty.com](https://silo.pgsty.com/zh/),集中提供文档、下载安装、版本与安全动态及项目背景。中文内容位于 `/zh/`,英文内容位于站点根路径。
## 按需求选择入口
| 需求 | 权威入口 |
| :-- | :-- |
| 项目概览与全站导航 | [Silo 中文门户](https://silo.pgsty.com/zh/)[English](https://silo.pgsty.com/) |
| 安装方式与软件下载 | [下载与安装](https://silo.pgsty.com/zh/download/)[English](https://silo.pgsty.com/download/) |
| 运维、管理、开发与参考指南 | [中文文档](https://silo.pgsty.com/zh/docs/)[English](https://silo.pgsty.com/docs/) |
| 项目动态、版本说明与安全通告 | [博客](https://silo.pgsty.com/zh/blog/),包括[版本发布](https://silo.pgsty.com/zh/blog/release/)与[安全通告](https://silo.pgsty.com/zh/blog/security/) |
| 带版本的二进制、校验和与源码归档 | [GitHub Releases](https://github.com/pgsty/minio/releases) |
| 缺陷报告与功能讨论 | [GitHub Issues](https://github.com/pgsty/minio/issues) |
| 私密漏洞报告 | [`SECURITY.md`](SECURITY.md) 与 [`VULNERABILITY_REPORT.md`](VULNERABILITY_REPORT.md) |
| 许可证、署名与商标信息 | [许可证](https://silo.pgsty.com/zh/about/license/)、[署名归属](https://silo.pgsty.com/zh/about/attribution/)与[商标政策](https://silo.pgsty.com/zh/about/trademark/) |
## 维护政策
活跃版本线的维护范围包括:
- 构建与依赖项维护;
- 适用的安全修复与公告;
- 针对可复现缺陷的范围明确的修复;
- 带版本的二进制、软件包、校验和与多架构镜像;
- Web Console、客户端、文档与 Pigsty 集成。
改动保持克制,并在可行时提供测试。所有维护均为尽力而为,不承诺固定的响应、修复或发布时间。
### 范围之外
- 独立产品路线图、新存储引擎或假设性的 S3 新特性;
- 大规模重写或显著扩大下游差异的改动;
- 历史版本或多条支持分支;
- 商业支持、SLA、7×24 服务或 SUBNET 服务;
- 部署设计、访问控制、监控、备份与恢复。
## 兼容策略
Silo 尽量保留:
- `minio` 可执行文件与 `github.com/minio/minio` module path
- MinIO 兼容的 S3 API、配置、环境变量与命令行约定;
- `RELEASE.YYYY-MM-DDTHH-MM-SSZ` 标签、容器入口与常见部署方式。
兼容性是默认约束。只要不会留下安全问题,Silo 就保留既有的协议、客户端、配置与运维行为;只有在修复重大安全问题确有必要时才会打破兼容,并在版本说明中明确受影响行为与迁移方式。每个版本仍应视为下游升级:锁定版本,阅读[版本说明](https://silo.pgsty.com/zh/blog/release/)与[安全公告](docs/security/advisories.md),保留回滚路径,并在生产使用前完成测试。
## 下载与发行产物
请先在[下载与安装](https://silo.pgsty.com/zh/download/)页面选择合适的安装方式;GitHub Releases 仍是带版本服务端二进制、校验和与源码归档的获取位置。
| 产物 | 位置 |
| :-- | :-- |
| 源码 | [`github.com/pgsty/minio`](https://github.com/pgsty/minio) |
| 容器镜像 | [`pgsty/minio`](https://hub.docker.com/r/pgsty/minio),支持 `linux/amd64``linux/arm64` 多架构清单 |
| 服务端二进制与校验和 | [GitHub Releases](https://github.com/pgsty/minio/releases),覆盖 Linux、macOS、Windows 的 `amd64``arm64` |
| Linux 软件包 | RPM、DEB、APK,并通过 [Pigsty 软件仓库](https://pigsty.cc/docs/repo/) 分发 |
| 客户端 | [`pgsty/mc`](https://github.com/pgsty/mc),容器内以 `mcli` 提供,并保留 `mc` 兼容别名 |
| 管理控制台 | 社区维护的 [`georgmangold/console`](https://github.com/georgmangold/console),嵌入服务端构建 |
| 共享库 | [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) v3.7.0,通过 `replace` 指令使用,同时保留 `github.com/minio/pkg/v3` 导入路径([版本说明](https://silo.pgsty.com/zh/blog/release/pkg-3.7.0/) |
## 快速开始
本地体验:
```bash
mkdir -p data
export MINIO_ROOT_USER=minioadmin
export MINIO_ROOT_PASSWORD=change-me-long-password
docker run -d --name silo \
-p 9000:9000 \
-p 9001:9001 \
-e MINIO_ROOT_USER \
-e MINIO_ROOT_PASSWORD \
-v "$PWD/data:/data" \
pgsty/minio:latest server /data --console-address ":9001"
```
管理控制台位于 <http://localhost:9001>S3 API 位于 <http://localhost:9000>。
镜像内置兼容客户端 `mcli`
```bash
docker exec silo mcli alias set local http://127.0.0.1:9000 \
"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
docker exec silo mcli mb local/demo
docker exec silo mcli ls local
```
> [!WARNING]
> 生产环境应锁定版本,使用独立凭据与 TLS,配置监控,保留独立备份,并验证恢复流程。
从源码构建服务端:
```bash
go build -o minio .
./minio --version
```
其他安装方式(包括原生软件包、二进制、Podman、Kubernetes、源码构建与 Pigsty Ansible)请前往[下载与安装](https://silo.pgsty.com/zh/download/);生产部署与管理请从 [Silo 中文文档](https://silo.pgsty.com/zh/docs/)开始。Pigsty 用户也可以直接使用 [Pigsty MinIO 模块](https://pigsty.cc/docs/minio/)。
## 安全
安全修复面向活跃的 `master` 分支,并记录在仓库[安全公告](docs/security/advisories.md)与门户[安全通告](https://silo.pgsty.com/zh/blog/security/)中。请按照 [`SECURITY.md`](SECURITY.md) 与 [`VULNERABILITY_REPORT.md`](VULNERABILITY_REPORT.md) 私密报告漏洞;同时影响上游 MinIO 的问题也应向上游报告。
## 参与贡献
欢迎安全与依赖项更新、可复现缺陷修复、测试、发布自动化、打包与文档改进。
Issue 与 Pull Request 应说明受影响版本、复现步骤、影响、预期行为、测试与兼容性说明。大型改动请先提交 Issue 讨论。
## 背景
本项目源于上游社区发行与维护模式的变化。维护者对相关变化的分析、替代方案评估与早期维护记录见以下文章:
| 文章 | 主题 |
| :-- | :-- |
| [MinIO已死](https://silo.pgsty.com/zh/blog/post/minio-is-dead/) | 上游项目与发行模式的变化 |
| [MinIO已死,谁能接盘?](https://silo.pgsty.com/zh/blog/post/minio-alternative/) | 可选替代方案评估 |
| [MinIO 已死,MinIO 复生](https://silo.pgsty.com/zh/blog/post/minio-resurrect/) | 建立分支及其发行流水线 |
| [续命 MinIO:承诺兑现](https://silo.pgsty.com/zh/blog/post/minio-promise-kept/) | 初期安全与维护工作 |
## 许可证与商标
服务端继续采用 [GNU Affero General Public License v3.0](LICENSE) 发布。上游作者与署名信息见 [`CREDITS`](CREDITS)。
MinIO 是 MinIO, Inc. 的商标。Silo、Pigsty 与 `pgsty/minio` 均为独立社区项目,与 MinIO, Inc. 不存在隶属或背书关系。
+32 -10
View File
@@ -1,20 +1,42 @@
# Security Policy
This repository is the `pgsty/minio` community fork of `minio/minio`. Upstream MinIO security contacts do not handle fork-specific fixes or release notes for this repository.
## Supported Versions
Security fixes are tracked on the active `master` branch and summarized in [docs/security/advisories.md](docs/security/advisories.md).
We always provide security updates for the [latest release](https://github.com/minio/minio/releases/latest).
Whenever there is a security update you just need to upgrade to the latest version.
## Reporting a Vulnerability
For vulnerabilities in this fork:
All security bugs in [minio/minio](https://github,com/minio/minio) (or other minio/* repositories)
should be reported by email to security@min.io. Your email will be acknowledged within 48 hours,
and you'll receive a more detailed response to your email within 72 hours indicating the next steps
in handling your report.
1. Follow the fork-specific expectations in [VULNERABILITY_REPORT.md](VULNERABILITY_REPORT.md).
2. Prefer the `pgsty/minio` repository's GitHub security reporting workflow when it is available.
3. If private reporting is not available, contact the maintainers through the `pgsty/minio` repository before publishing detailed exploit information.
4. If you confirm the issue also affects upstream `minio/minio`, report it upstream separately.
Please, provide a detailed explanation of the issue. In particular, outline the type of the security
issue (DoS, authentication bypass, information disclose, ...) and the assumptions you're making (e.g. do
you need access credentials for a successful exploit).
## Disclosure Process
If you have not received a reply to your email within 48 hours or you have not heard from the security team
for the past five days please contact the security team directly:
Fork-specific fixes and user-visible upgrade notes are published in [docs/security/advisories.md](docs/security/advisories.md). The fork-specific triage and remediation process is described in [VULNERABILITY_REPORT.md](VULNERABILITY_REPORT.md).
- Primary security coordinator: aead@min.io
- Secondary coordinator: harsha@min.io
- If you receive no response: dev@min.io
### Disclosure Process
MinIO uses the following disclosure process:
1. Once the security report is received one member of the security team tries to verify and reproduce
the issue and determines the impact it has.
2. A member of the security team will respond and either confirm or reject the security report.
If the report is rejected the response explains why.
3. Code is audited to find any potential similar problems.
4. Fixes are prepared for the latest release.
5. On the date that the fixes are applied a security advisory will be published on <https://blog.min.io>.
Please inform us in your report email whether MinIO should mention your contribution w.r.t. fixing
the security issue. By default MinIO will **not** publish this information to protect your privacy.
This process can take some time, especially when coordination is required with maintainers of other projects.
Every effort will be made to handle the bug in as timely a manner as possible, however it's important that we
follow the process described above to ensure that disclosures are handled consistently.
+20 -19
View File
@@ -1,37 +1,38 @@
# Vulnerability Management Policy
This document describes how the `pgsty/minio` maintainers investigate,
assess, and remediate reported vulnerabilities affecting this fork, any
directly shipped component, or a direct / indirect dependency used by this
repository.
This document formally describes the process of addressing and managing a
reported vulnerability that has been found in the MinIO server code base,
any directly connected ecosystem component or a direct / indirect dependency
of the code base.
## Scope
This policy covers vulnerability reports opened by repository maintainers or
external third parties against `pgsty/minio` itself, its release artifacts, or
dependencies that materially affect this fork.
The vulnerability management policy described in this document covers the
process of investigating, assessing and resolving a vulnerability report
opened by a MinIO employee or an external third party.
It defines the information needed for triage and the expected remediation
workflow for supported fixes.
Therefore, it lists pre-conditions and actions that should be performed to
resolve and fix a reported vulnerability.
## Vulnerability Management Process
A useful vulnerability report should contain the following information:
The vulnerability management process requires that the vulnerability report
contains the following information:
- The project / component that contains the reported vulnerability.
- A description of the vulnerability. In particular, the type of the
reported vulnerability and how it might be exploited. Alternatively,
a well-established vulnerability identifier, such as a CVE or GHSA ID, can
be used instead.
reported vulnerability and how it might be exploited. Alternatively,
a well-established vulnerability identifier, e.g. CVE number, can be
used instead.
Based on the report, the `pgsty/minio` maintainers investigate:
Based on the description mentioned above, a MinIO engineer or security team
member investigates:
- Whether the reported vulnerability exists.
- The conditions that are required such that the vulnerability can be exploited.
- Which releases, branches, or deployment paths are affected.
- The steps required to fix the vulnerability.
If the vulnerability exists in this fork itself, the maintainers will, when
feasible, fix the issue or implement reasonable countermeasures such that the
vulnerability can no longer be exploited. Fork-specific upgrade notes and
security advisories are published in `docs/security/advisories.md`.
In general, if the vulnerability exists in one of the MinIO code bases
itself - not in a code dependency - then MinIO will, if possible, fix
the vulnerability or implement reasonable countermeasures such that the
vulnerability cannot be exploited anymore.
+2 -5
View File
@@ -8,11 +8,8 @@ function _init() {
## All binaries are static make sure to disable CGO.
export CGO_ENABLED=0
## Cross-compile only the OS/arch combinations we actually publish, kept in
## sync with the goos/goarch matrix in .github/goreleaser.yml. Compile-checking
## targets we never ship (ppc64le, s390x, mips*, riscv64, 386, arm, the BSDs)
## spent CI minutes on unshipped code and timed the gate out on a cold cache.
SUPPORTED_OSARCH="linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64"
## List of architectures and OS to test coss compilation.
SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64 linux/riscv64"
}
function _build() {
+2 -6
View File
@@ -38,12 +38,8 @@ func genLDFlags(version string) string {
ldflagsStr += " -X github.com/minio/minio/cmd.ReleaseTag=" + releaseTag
ldflagsStr += " -X github.com/minio/minio/cmd.CommitID=" + commitID()
ldflagsStr += " -X github.com/minio/minio/cmd.ShortCommitID=" + commitID()[:12]
// GOPATH/GOROOT are deliberately not stamped in. They only seed the logger's
// source-path trim list, which -trimpath already makes moot (paths are
// relative in the binary, so there is no build-machine prefix left to trim),
// and stamping them baked the builder's absolute paths into the released
// binary - defeating -trimpath and reproducible builds. cmd.GOPATH/GOROOT
// keep their empty defaults, exactly as a plain `go build` leaves them.
ldflagsStr += " -X github.com/minio/minio/cmd.GOPATH=" + os.Getenv("GOPATH")
ldflagsStr += " -X github.com/minio/minio/cmd.GOROOT=" + os.Getenv("GOROOT")
return ldflagsStr
}
-120
View File
@@ -1,120 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
repo_dir="$(cd "${script_dir}/.." && pwd)"
dist_dir="${DIST_DIR:-${repo_dir}/dist}"
nfpm_config="${NFPM_CONFIG:-${repo_dir}/.github/nfpm.yml}"
if [ -z "${PKG_VERSION:-}" ]; then
echo "PKG_VERSION is required" >&2
exit 1
fi
# Re-validate the shape release.yml derived from the tag. The packages are
# named from this, so a malformed value would ship under a name no repository
# can order against.
if ! [[ "${PKG_VERSION}" =~ ^[0-9]{14}\.0\.0$ ]]; then
echo "Invalid PKG_VERSION: ${PKG_VERSION}" >&2
exit 1
fi
if ! command -v nfpm >/dev/null 2>&1; then
echo "nfpm is required" >&2
exit 1
fi
if [ ! -f "${nfpm_config}" ]; then
echo "Missing nFPM config: ${nfpm_config}" >&2
exit 1
fi
# nfpm resolves a relative content src against the current directory, not
# against the config file, so the unit path is passed in absolute. Otherwise
# this only works when invoked from the repository root and fails elsewhere on
# a message that names the file rather than the cause.
unit_file="${repo_dir}/minio.service"
if [ ! -f "${unit_file}" ]; then
echo "Missing systemd unit: ${unit_file}" >&2
exit 1
fi
packages_dir="${dist_dir}/packages"
mkdir -p "${packages_dir}"
# Two spaces, no trailing newline: sign-release-rpms.sh parses these files to
# check download integrity before it signs, and regenerates them afterwards in
# the same shape.
sha256_file() {
local file="$1"
local digest
if command -v sha256sum >/dev/null 2>&1; then
digest="$(sha256sum "${file}" | awk '{print $1}')"
else
digest="$(shasum -a 256 "${file}" | awk '{print $1}')"
fi
printf '%s %s' "${digest}" "$(basename "${file}")" > "${file}.sha256sum"
}
find_binary() {
local goarch="$1"
local matches
local count
# Must resolve to exactly one binary. Picking the first of several build
# variants (an added goamd64 level, a stale dist entry) would silently ship a
# package whose contents do not match its name.
matches="$(find "${dist_dir}" -maxdepth 2 -type f \
-path "${dist_dir}/minio_linux_${goarch}*/minio" | sort)"
count="$(printf '%s' "${matches}" | grep -c . || true)"
if [ "${count}" -eq 0 ]; then
echo "Missing GoReleaser binary for linux/${goarch}" >&2
exit 1
fi
if [ "${count}" -ne 1 ]; then
echo "Expected exactly one GoReleaser binary for linux/${goarch}, found ${count}:" >&2
printf '%s\n' "${matches}" >&2
exit 1
fi
printf '%s\n' "${matches}"
}
build_arch() {
local goarch="$1"
local rpm_arch="$2"
local deb_arch="$3"
local apk_arch="$4"
local source
local rpm_file
local deb_file
local apk_file
source="$(find_binary "${goarch}")"
# These names are the public download names and must not drift; RPM carries a
# release number, DEB and APK do not, matching what pkger produced.
rpm_file="${packages_dir}/minio-${PKG_VERSION}-1.${rpm_arch}.rpm"
deb_file="${packages_dir}/minio_${PKG_VERSION}_${deb_arch}.deb"
apk_file="${packages_dir}/minio_${PKG_VERSION}_${apk_arch}.apk"
PKG_VERSION="${PKG_VERSION}" NFPM_RELEASE=1 NFPM_ARCH="${goarch}" NFPM_SOURCE="${source}" NFPM_UNIT="${unit_file}" \
nfpm package --config "${nfpm_config}" --packager rpm --target "${rpm_file}"
PKG_VERSION="${PKG_VERSION}" NFPM_RELEASE='' NFPM_ARCH="${goarch}" NFPM_SOURCE="${source}" NFPM_UNIT="${unit_file}" \
nfpm package --config "${nfpm_config}" --packager deb --target "${deb_file}"
PKG_VERSION="${PKG_VERSION}" NFPM_RELEASE='' NFPM_ARCH="${goarch}" NFPM_SOURCE="${source}" NFPM_UNIT="${unit_file}" \
nfpm package --config "${nfpm_config}" --packager apk --target "${apk_file}"
sha256_file "${rpm_file}"
sha256_file "${deb_file}"
sha256_file "${apk_file}"
}
build_arch amd64 x86_64 amd64 x86_64
build_arch arm64 aarch64 arm64 aarch64
find "${packages_dir}" -maxdepth 1 -type f | sort
-282
View File
@@ -1,282 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# These are the single source of truth for the package identity: .github/nfpm.yml
# must agree with them, and test-release.yml asserts that it does. Drift the
# other way round would only surface here, on the maintainer's machine, after
# the build has already run and uploaded.
expected_fingerprint="9592A7BC7A682E7333376E09E7935D8DB9BD8B20"
expected_vendor="PGSTY"
expected_packager="Ruohang Feng (@Vonng) <rh@vonng.com>"
expected_url="https://silo.pgsty.com"
expected_summary="S3-Interface Libre Object Storage, Community-maintained MinIO server fork."
expected_description="S3-Interface Libre Object Storage, Community-maintained MinIO server fork."
expected_license="AGPL-3.0-or-later"
expected_group="Applications/File"
expected_payload="/usr/lib/systemd/system/minio.service
/usr/local/bin/minio"
repository="${GH_REPO:-pgsty/minio}"
container="${DNFUPDATE_CONTAINER:-dnfupdate}"
upload=false
release_tag=""
usage() {
cat <<'EOF'
Usage: buildscripts/sign-release-rpms.sh RELEASE.TAG [--upload] [--repo OWNER/REPO] [--container NAME]
Downloads the two unsigned RPMs from a Draft GitHub Release, signs them with
the expected Pigsty key in the local dnfupdate container, verifies the result,
and regenerates their .sha256sum files. Nothing is uploaded unless --upload is
provided.
EOF
}
while [ "$#" -gt 0 ]; do
case "$1" in
--upload)
upload=true
;;
--repo)
shift
if [ "$#" -eq 0 ]; then
echo "--repo requires OWNER/REPO" >&2
exit 1
fi
repository="$1"
;;
--container)
shift
if [ "$#" -eq 0 ]; then
echo "--container requires a name" >&2
exit 1
fi
container="$1"
;;
-h|--help)
usage
exit 0
;;
-*)
echo "Unknown option: $1" >&2
usage >&2
exit 1
;;
*)
if [ -n "${release_tag}" ]; then
echo "Only one release tag may be specified" >&2
exit 1
fi
release_tag="$1"
;;
esac
shift
done
if [ -z "${release_tag}" ]; then
usage >&2
exit 1
fi
for command in docker gh; do
if ! command -v "${command}" >/dev/null 2>&1; then
echo "${command} is required" >&2
exit 1
fi
done
version_hyphen="${release_tag#RELEASE.}"
package_version="$(printf '%s\n' "${version_hyphen}" | sed -E \
's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')"
if [ "${package_version}" = "${version_hyphen}" ]; then
echo "Invalid release tag: ${release_tag}" >&2
exit 1
fi
if [ "$(gh release view "${release_tag}" --repo "${repository}" --json isDraft --jq .isDraft)" != "true" ]; then
echo "Refusing to sign: ${release_tag} is not a Draft release" >&2
exit 1
fi
if [ "$(docker inspect --format '{{.State.Running}}' "${container}" 2>/dev/null || true)" != "true" ]; then
echo "Signing container is not running: ${container}" >&2
exit 1
fi
secret_fingerprints="$(docker exec "${container}" \
gpg --batch --with-colons --list-secret-keys 2>/dev/null |
awk -F: '$1 == "fpr" { print toupper($10) }')"
if ! printf '%s\n' "${secret_fingerprints}" | grep -Fxq "${expected_fingerprint}"; then
echo "Expected signing key is not available in ${container}: ${expected_fingerprint}" >&2
exit 1
fi
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
repo_dir="$(cd "${script_dir}/.." && pwd)"
work_root="${SIGN_WORKDIR:-${repo_dir}/.release-sign}"
mkdir -p "${work_root}"
work_dir="$(mktemp -d "${work_root}/${release_tag}.XXXXXX")"
unsigned_dir="${work_dir}/unsigned"
signed_dir="${work_dir}/signed"
mkdir -p "${unsigned_dir}" "${signed_dir}"
chmod 700 "${work_dir}" "${unsigned_dir}" "${signed_dir}"
rpm_files=(
"minio-${package_version}-1.x86_64.rpm"
"minio-${package_version}-1.aarch64.rpm"
)
download_patterns=()
for rpm_file in "${rpm_files[@]}"; do
download_patterns+=(--pattern "${rpm_file}" --pattern "${rpm_file}.sha256sum")
done
echo "Downloading RPMs from Draft release ${repository}@${release_tag}"
gh release download "${release_tag}" --repo "${repository}" \
--dir "${unsigned_dir}" "${download_patterns[@]}"
sha256_digest() {
local file="$1"
if command -v sha256sum >/dev/null 2>&1; then
sha256sum "${file}" | awk '{print $1}'
else
shasum -a 256 "${file}" | awk '{print $1}'
fi
}
for rpm_file in "${rpm_files[@]}"; do
rpm_path="${unsigned_dir}/${rpm_file}"
checksum_path="${rpm_path}.sha256sum"
test -s "${rpm_path}"
test -s "${checksum_path}"
actual_line="$(sha256_digest "${rpm_path}") ${rpm_file}"
published_line="$(tr -d '\n' < "${checksum_path}")"
if [ "${actual_line}" != "${published_line}" ]; then
echo "Checksum mismatch for ${rpm_file}" >&2
exit 1
fi
done
safe_tag="$(printf '%s' "${release_tag}" | tr -c 'A-Za-z0-9._-' '_')"
container_dir="/tmp/minio-sign-${safe_tag}-$$"
docker exec "${container}" mkdir -p "${container_dir}"
cleanup_container() {
local rpm_file
for rpm_file in "${rpm_files[@]}"; do
docker exec "${container}" rm -f "${container_dir}/${rpm_file}" >/dev/null 2>&1 || true
done
docker exec "${container}" rmdir "${container_dir}" >/dev/null 2>&1 || true
}
trap cleanup_container EXIT
assert_rpm_tag() {
local rpm_path="$1"
local tag="$2"
local expected="$3"
local actual
actual="$(docker exec "${container}" rpm -qp --queryformat "%{${tag}}" "${rpm_path}")"
if [ "${actual}" != "${expected}" ]; then
echo "Unexpected RPM ${tag}: ${actual}" >&2
echo "Expected RPM ${tag}: ${expected}" >&2
exit 1
fi
}
for rpm_file in "${rpm_files[@]}"; do
case "${rpm_file}" in
*.x86_64.rpm)
expected_arch="x86_64"
;;
*.aarch64.rpm)
expected_arch="aarch64"
;;
*)
echo "Unexpected RPM filename: ${rpm_file}" >&2
exit 1
;;
esac
echo "Signing ${rpm_file} with ${expected_fingerprint}"
docker cp "${unsigned_dir}/${rpm_file}" "${container}:${container_dir}/${rpm_file}" >/dev/null
container_rpm="${container_dir}/${rpm_file}"
assert_rpm_tag "${container_rpm}" NAME minio
assert_rpm_tag "${container_rpm}" VERSION "${package_version}"
assert_rpm_tag "${container_rpm}" RELEASE 1
assert_rpm_tag "${container_rpm}" ARCH "${expected_arch}"
assert_rpm_tag "${container_rpm}" VENDOR "${expected_vendor}"
assert_rpm_tag "${container_rpm}" PACKAGER "${expected_packager}"
assert_rpm_tag "${container_rpm}" URL "${expected_url}"
assert_rpm_tag "${container_rpm}" LICENSE "${expected_license}"
assert_rpm_tag "${container_rpm}" GROUP "${expected_group}"
assert_rpm_tag "${container_rpm}" SUMMARY "${expected_summary}"
assert_rpm_tag "${container_rpm}" DESCRIPTION "${expected_description}"
rpm_payload="$(docker exec "${container}" rpm -qpl "${container_rpm}")"
if [ "${rpm_payload}" != "${expected_payload}" ]; then
echo "Unexpected RPM payload for ${rpm_file}:" >&2
printf '%s\n' "${rpm_payload}" >&2
exit 1
fi
docker exec "${container}" rpmsign \
--define "_gpg_name ${expected_fingerprint}" \
--addsign "${container_rpm}"
signature_output="$(docker exec "${container}" rpmkeys --checksig --verbose "${container_rpm}")"
printf '%s\n' "${signature_output}"
if ! printf '%s\n' "${signature_output}" | tr '[:upper:]' '[:lower:]' | grep -q 'key id b9bd8b20: ok'; then
echo "Signature verification failed for ${rpm_file}" >&2
exit 1
fi
docker cp "${container}:${container_dir}/${rpm_file}" "${signed_dir}/${rpm_file}" >/dev/null
signed_digest="$(sha256_digest "${signed_dir}/${rpm_file}")"
printf '%s %s' "${signed_digest}" "${rpm_file}" > "${signed_dir}/${rpm_file}.sha256sum"
docker exec "${container}" rpm -qp --queryformat \
$'Name: %{NAME}\nVersion: %{VERSION}-%{RELEASE}\nArch: %{ARCH}\nVendor: %{VENDOR}\nPackager: %{PACKAGER}\nURL: %{URL}\n' \
"${container_rpm}"
echo "SHA256: ${signed_digest}"
done
if [ "${upload}" = true ]; then
upload_files=()
for rpm_file in "${rpm_files[@]}"; do
upload_files+=("${signed_dir}/${rpm_file}" "${signed_dir}/${rpm_file}.sha256sum")
done
echo "Replacing RPMs in Draft release ${release_tag}"
gh release upload "${release_tag}" --repo "${repository}" --clobber "${upload_files[@]}"
for rpm_file in "${rpm_files[@]}"; do
for asset in "${rpm_file}" "${rpm_file}.sha256sum"; do
local_digest="sha256:$(sha256_digest "${signed_dir}/${asset}")"
remote_digest=""
for attempt in 1 2 3 4 5; do
remote_digest="$(gh release view "${release_tag}" --repo "${repository}" --json assets \
--jq ".assets[] | select(.name == \"${asset}\") | .digest")"
if [ "${local_digest}" = "${remote_digest}" ]; then
break
fi
if [ "${attempt}" -lt 5 ]; then
sleep 2
fi
done
if [ "${local_digest}" != "${remote_digest}" ]; then
echo "GitHub asset digest mismatch for ${asset}" >&2
echo "Local: ${local_digest}" >&2
echo "Remote: ${remote_digest}" >&2
exit 1
fi
echo "Verified GitHub asset: ${asset} ${remote_digest}"
done
done
else
echo
echo "Signed RPMs are ready for review in: ${signed_dir}"
echo "Re-run with --upload to replace the RPM assets in the Draft release."
fi
-51
View File
@@ -1,51 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# Asserts that every binary GoReleaser produced is stamped by the Go toolchain
# as built from this exact commit with a clean working tree. A stray untracked
# file (for example an un-ignored dist/) silently turns every release binary
# into a "+dirty" pseudo-version, which destroys the link between a published
# artifact and its tag. Catch that here instead of after publishing.
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
repo_dir="$(cd "${script_dir}/.." && pwd)"
dist_dir="${DIST_DIR:-${repo_dir}/dist}"
expected_count="${EXPECTED_BINARY_COUNT:-6}"
if ! command -v go >/dev/null 2>&1; then
echo "go is required" >&2
exit 1
fi
if [ ! -d "${dist_dir}" ]; then
echo "Missing GoReleaser dist directory: ${dist_dir}" >&2
exit 1
fi
revision="$(git -C "${repo_dir}" rev-parse HEAD)"
count=0
while IFS= read -r binary; do
count=$((count + 1))
info="$(go version -m "${binary}")"
if ! grep -qF "vcs.revision=${revision}" <<< "${info}"; then
echo "Unexpected vcs.revision in ${binary} (expected ${revision})" >&2
grep -F 'vcs.' <<< "${info}" >&2 || true
exit 1
fi
if ! grep -qF 'vcs.modified=false' <<< "${info}"; then
echo "Binary was built from a dirty working tree: ${binary}" >&2
grep -F 'vcs.' <<< "${info}" >&2 || true
exit 1
fi
done < <(find "${dist_dir}" -maxdepth 2 -type f \( -name 'minio' -o -name 'minio.exe' \) | sort)
if [ "${count}" -ne "${expected_count}" ]; then
echo "Expected ${expected_count} release binaries, found ${count}" >&2
exit 1
fi
echo "Verified ${count} binaries built from ${revision} with a clean tree"
+1 -1
View File
@@ -267,7 +267,7 @@ func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.R
lookupResult, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(targetUser)
if err != nil {
// if not found, check if DN
if strings.Contains(strings.ToLower(err.Error()), "user dn not found for:") {
if strings.Contains(err.Error(), "User DN not found for:") {
if isDN {
// warn user that DNs are not allowed
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminLDAPExpectedLoginName, err), r.URL)
+4 -22
View File
@@ -1026,7 +1026,7 @@ type unwrapper interface {
Unwrap() http.ResponseWriter
}
// headersAlreadyWritten returns true if an HTTP status has already been written
// headersAlreadyWritten returns true if the headers have already been written
// to this response writer. It will unwrap the ResponseWriter if possible to try
// and find a trackingResponseWriter.
func headersAlreadyWritten(w http.ResponseWriter) bool {
@@ -1041,18 +1041,14 @@ func headersAlreadyWritten(w http.ResponseWriter) bool {
}
}
// trackingResponseWriter wraps a ResponseWriter and records when an HTTP status
// has been written, explicitly or implicitly by Write or an effective Flush.
//
// Informational responses are treated as final. internal/http.ResponseRecorder
// has the same limitation, so 1xx support must be fixed in both layers.
// trackingResponseWriter wraps a ResponseWriter and notes when WriterHeader has
// been called. This allows high level request handlers to check if something
// has already sent the header.
type trackingResponseWriter struct {
http.ResponseWriter
headerWritten bool
}
var _ http.Flusher = (*trackingResponseWriter)(nil)
func (w *trackingResponseWriter) WriteHeader(statusCode int) {
if !w.headerWritten {
w.headerWritten = true
@@ -1061,23 +1057,9 @@ func (w *trackingResponseWriter) WriteHeader(statusCode int) {
}
func (w *trackingResponseWriter) Write(b []byte) (int, error) {
if !w.headerWritten {
w.WriteHeader(http.StatusOK)
}
return w.ResponseWriter.Write(b)
}
func (w *trackingResponseWriter) Flush() {
f, ok := w.ResponseWriter.(http.Flusher)
if !ok {
return
}
if !w.headerWritten {
w.WriteHeader(http.StatusOK)
}
f.Flush()
}
func (w *trackingResponseWriter) Unwrap() http.ResponseWriter {
return w.ResponseWriter
}
+5 -144
View File
@@ -18,14 +18,12 @@
package cmd
import (
"compress/gzip"
"io"
"net/http"
"net/http/httptest"
"testing"
"github.com/klauspost/compress/gzhttp"
xhttp "github.com/minio/minio/internal/http"
)
// Tests object location.
@@ -129,26 +127,10 @@ func TestGetURLScheme(t *testing.T) {
}
}
type writeHeaderSpy struct {
http.ResponseWriter
codes []int
}
func (r *writeHeaderSpy) WriteHeader(code int) {
r.codes = append(r.codes, code)
r.ResponseWriter.WriteHeader(code)
}
func (r *writeHeaderSpy) Flush() {
if f, ok := r.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
func TestTrackingResponseWriter(t *testing.T) {
rw := httptest.NewRecorder()
trw := &trackingResponseWriter{ResponseWriter: rw}
trw.WriteHeader(299)
trw.WriteHeader(123)
if !trw.headerWritten {
t.Fatal("headerWritten was not set by WriteHeader call")
}
@@ -157,11 +139,10 @@ func TestTrackingResponseWriter(t *testing.T) {
if err != nil {
t.Fatalf("Write unexpectedly failed: %v", err)
}
xhttp.Flush(trw)
// Check that WriteHeader, Write, and Flush were called on the underlying response writer.
// Check that WriteHeader and Write were called on the underlying response writer
resp := rw.Result()
if resp.StatusCode != 299 {
if resp.StatusCode != 123 {
t.Fatalf("unexpected status: %v", resp.StatusCode)
}
body, err := io.ReadAll(resp.Body)
@@ -171,9 +152,6 @@ func TestTrackingResponseWriter(t *testing.T) {
if string(body) != "hello" {
t.Fatalf("response body incorrect: %v", string(body))
}
if !rw.Flushed {
t.Fatal("underlying ResponseRecorder was not flushed")
}
// Check that Unwrap works
if trw.Unwrap() != rw {
@@ -181,122 +159,6 @@ func TestTrackingResponseWriter(t *testing.T) {
}
}
func TestTrackingResponseWriterWriteImplicitHeader(t *testing.T) {
testCases := []struct {
name string
body []byte
}{
{name: "non-empty", body: []byte("hello")},
{name: "empty", body: nil},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
rec := httptest.NewRecorder()
rw := &writeHeaderSpy{ResponseWriter: rec}
trw := &trackingResponseWriter{ResponseWriter: rw}
n, err := trw.Write(testCase.body)
if err != nil {
t.Fatalf("Write unexpectedly failed: %v", err)
}
if n != len(testCase.body) {
t.Fatalf("unexpected bytes written: got %d, want %d", n, len(testCase.body))
}
if !trw.headerWritten {
t.Fatal("Write did not set headerWritten")
}
if len(rw.codes) != 1 || rw.codes[0] != http.StatusOK {
t.Fatalf("unexpected WriteHeader calls: got %v, want [%d]", rw.codes, http.StatusOK)
}
if got := rec.Body.String(); got != string(testCase.body) {
t.Fatalf("unexpected body: got %q, want %q", got, testCase.body)
}
})
}
}
func TestTrackingResponseWriterFlush(t *testing.T) {
rec := httptest.NewRecorder()
rw := &writeHeaderSpy{ResponseWriter: rec}
trw := &trackingResponseWriter{ResponseWriter: rw}
xhttp.Flush(trw)
if !trw.headerWritten {
t.Fatal("Flush did not set headerWritten")
}
if len(rw.codes) != 1 || rw.codes[0] != http.StatusOK {
t.Fatalf("unexpected WriteHeader calls: got %v, want [%d]", rw.codes, http.StatusOK)
}
if !rec.Flushed {
t.Fatal("underlying ResponseRecorder was not flushed")
}
}
func TestTrackingResponseWriterFlushUnsupported(t *testing.T) {
rw := struct{ http.ResponseWriter }{ResponseWriter: httptest.NewRecorder()}
trw := &trackingResponseWriter{ResponseWriter: rw}
trw.Flush()
if trw.headerWritten {
t.Fatal("unsupported Flush set headerWritten")
}
}
func TestTrackingResponseWriterGzipStreaming(t *testing.T) {
const (
eventPayload = "event data"
sentinel = "<sentinel-error/>"
)
rw := httptest.NewRecorder()
trw := &trackingResponseWriter{ResponseWriter: rw}
var (
committed bool
writeErr error
)
handler := gzipHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
setEventStreamHeaders(w)
_, writeErr = w.Write([]byte(eventPayload))
if writeErr != nil {
return
}
xhttp.Flush(w)
committed = headersAlreadyWritten(w)
writeResponse(w, http.StatusInternalServerError, []byte(sentinel), mimeXML)
}))
req := httptest.NewRequest(http.MethodGet, "/", nil)
req.Header.Set("Accept-Encoding", "gzip")
handler.ServeHTTP(trw, req)
if writeErr != nil {
t.Fatalf("Write unexpectedly failed: %v", writeErr)
}
if !committed {
t.Fatal("headersAlreadyWritten returned false after Write and Flush")
}
resp := rw.Result()
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status: got %d, want %d", resp.StatusCode, http.StatusOK)
}
if got := resp.Header.Get("Content-Encoding"); got != "gzip" {
t.Fatalf("unexpected Content-Encoding: got %q, want %q", got, "gzip")
}
zr, err := gzip.NewReader(resp.Body)
if err != nil {
t.Fatalf("creating gzip reader failed: %v", err)
}
defer zr.Close()
body, err := io.ReadAll(zr)
if err != nil {
t.Fatalf("reading gzip response body failed: %v", err)
}
if got := string(body); got != eventPayload {
t.Fatalf("unexpected response body: got %q, want %q (sentinel %q must be suppressed)", got, eventPayload, sentinel)
}
}
func TestHeadersAlreadyWritten(t *testing.T) {
rw := httptest.NewRecorder()
trw := &trackingResponseWriter{ResponseWriter: rw}
@@ -305,7 +167,7 @@ func TestHeadersAlreadyWritten(t *testing.T) {
t.Fatal("headers have not been written yet")
}
trw.WriteHeader(299)
trw.WriteHeader(123)
if !headersAlreadyWritten(trw) {
t.Fatal("headers were written")
}
@@ -321,8 +183,7 @@ func TestHeadersAlreadyWrittenWrapped(t *testing.T) {
t.Fatal("headers have not been written yet")
}
// Pin the current stack-wide 1xx limitation documented on trackingResponseWriter.
wrap2.WriteHeader(http.StatusContinue)
wrap2.WriteHeader(123)
if !headersAlreadyWritten(wrap2) {
t.Fatal("headers were written")
}
+2 -3
View File
@@ -347,9 +347,8 @@ const _APIErrorCode_name = "NoneAccessDeniedBadDigestEntityTooSmallEntityTooLarg
var _APIErrorCode_index = [...]uint16{0, 4, 16, 25, 39, 53, 67, 81, 94, 112, 129, 144, 161, 174, 186, 208, 228, 254, 268, 289, 306, 321, 344, 361, 379, 396, 420, 435, 456, 474, 486, 506, 523, 546, 567, 579, 597, 618, 646, 676, 697, 720, 746, 783, 813, 846, 871, 903, 933, 962, 987, 1009, 1035, 1057, 1085, 1114, 1148, 1179, 1216, 1240, 1264, 1292, 1318, 1349, 1379, 1388, 1400, 1416, 1429, 1443, 1461, 1481, 1502, 1518, 1529, 1545, 1556, 1584, 1604, 1620, 1648, 1662, 1679, 1699, 1712, 1726, 1739, 1752, 1768, 1785, 1806, 1820, 1841, 1854, 1876, 1899, 1915, 1930, 1945, 1966, 1984, 1999, 2016, 2041, 2059, 2082, 2097, 2116, 2132, 2151, 2172, 2186, 2198, 2211, 2230, 2249, 2259, 2274, 2310, 2341, 2374, 2403, 2415, 2435, 2459, 2483, 2504, 2528, 2547, 2568, 2585, 2595, 2612, 2629, 2650, 2670, 2693, 2715, 2741, 2762, 2780, 2807, 2838, 2865, 2886, 2907, 2931, 2956, 2984, 3012, 3028, 3051, 3081, 3092, 3104, 3121, 3136, 3154, 3183, 3200, 3216, 3232, 3250, 3268, 3291, 3312, 3335, 3346, 3362, 3385, 3402, 3430, 3449, 3479, 3499, 3527, 3542, 3560, 3575, 3589, 3624, 3643, 3654, 3667, 3682, 3705, 3731, 3747, 3765, 3783, 3804, 3818, 3835, 3866, 3886, 3907, 3928, 3947, 3966, 3984, 4007, 4031, 4055, 4080, 4115, 4140, 4174, 4207, 4228, 4242, 4261, 4290, 4313, 4340, 4374, 4406, 4436, 4459, 4487, 4519, 4547, 4571, 4595, 4624, 4642, 4659, 4681, 4698, 4716, 4736, 4762, 4778, 4797, 4818, 4822, 4840, 4857, 4883, 4897, 4921, 4942, 4957, 4975, 4998, 5013, 5032, 5049, 5066, 5090, 5117, 5140, 5163, 5180, 5202, 5218, 5238, 5257, 5279, 5300, 5320, 5342, 5366, 5385, 5427, 5448, 5471, 5492, 5523, 5542, 5564, 5584, 5610, 5631, 5653, 5673, 5697, 5720, 5739, 5759, 5781, 5804, 5835, 5873, 5914, 5944, 5958, 5979, 5995, 6017, 6047, 6073, 6101, 6135, 6153, 6176, 6211, 6251, 6293, 6325, 6342, 6367, 6382, 6399, 6409, 6420, 6458, 6512, 6558, 6610, 6658, 6701, 6745, 6773, 6787, 6805, 6841, 6864, 6887, 6909, 6924, 6952, 6975, 6993, 7020, 7052, 7067, 7083, 7100, 7120, 7136, 7152, 7169, 7182}
func (i APIErrorCode) String() string {
idx := int(i) - 0
if i < 0 || idx >= len(_APIErrorCode_index)-1 {
if i < 0 || i >= APIErrorCode(len(_APIErrorCode_index)-1) {
return "APIErrorCode(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _APIErrorCode_name[_APIErrorCode_index[idx]:_APIErrorCode_index[idx+1]]
return _APIErrorCode_name[_APIErrorCode_index[i]:_APIErrorCode_index[i+1]]
}
+7 -52
View File
@@ -343,26 +343,6 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
return s3Err
}
func checkRequestAuthTypeWithExistingTags(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName, existingTags string) (s3Err APIErrorCode) {
logger.GetReqInfo(ctx).BucketName = bucketName
logger.GetReqInfo(ctx).ObjectName = objectName
if s3Err = authenticateRequest(ctx, r, action); s3Err != ErrNone {
return s3Err
}
return authorizeRequestWithExistingTags(ctx, r, action, existingTags)
}
func checkRequestAuthTypeWithRequestTags(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string, requestTags *string) (s3Err APIErrorCode) {
logger.GetReqInfo(ctx).BucketName = bucketName
logger.GetReqInfo(ctx).ObjectName = objectName
if s3Err = authenticateRequest(ctx, r, action); s3Err != ErrNone {
return s3Err
}
return authorizeRequestWithTags(ctx, r, action, "", requestTags)
}
// checkRequestAuthTypeWithVID is similar to checkRequestAuthType
// passes versionID additionally.
func checkRequestAuthTypeWithVID(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName, versionID string) (s3Err APIErrorCode) {
@@ -436,14 +416,6 @@ func authenticateRequest(ctx context.Context, r *http.Request, action policy.Act
}
func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
return authorizeRequestWithExistingTags(ctx, r, action, "")
}
func authorizeRequestWithExistingTags(ctx context.Context, r *http.Request, action policy.Action, existingTags string) (s3Err APIErrorCode) {
return authorizeRequestWithTags(ctx, r, action, existingTags, nil)
}
func authorizeRequestWithTags(ctx context.Context, r *http.Request, action policy.Action, existingTags string, requestTags *string) (s3Err APIErrorCode) {
reqInfo := logger.GetReqInfo(ctx)
if reqInfo == nil {
return ErrAccessDenied
@@ -455,19 +427,6 @@ func authorizeRequestWithTags(ctx context.Context, r *http.Request, action polic
bucket := reqInfo.BucketName
object := reqInfo.ObjectName
versionID := reqInfo.VersionID
conditionValuesForAuth := func(locationConstraint string, credentials auth.Credentials) map[string][]string {
values := getConditionValuesWithTags(r, locationConstraint, credentials, existingTags, requestTags)
if action == policy.DeleteObjectAction {
// DeleteObjects carries the effective version ID in each XML object,
// not in the request query. Keep authorization scoped to that entry.
if versionID == "" {
delete(values, "versionid")
} else {
values["versionid"] = []string{versionID}
}
}
return values
}
if action != policy.ListAllMyBucketsAction && cred.AccessKey == "" {
// Anonymous checks are not meant for ListAllBuckets action
@@ -476,7 +435,7 @@ func authorizeRequestWithTags(ctx context.Context, r *http.Request, action polic
Groups: cred.Groups,
Action: action,
BucketName: bucket,
ConditionValues: conditionValuesForAuth(region, auth.AnonymousCredentials),
ConditionValues: getConditionValues(r, region, auth.AnonymousCredentials),
IsOwner: false,
ObjectName: object,
}) {
@@ -492,7 +451,7 @@ func authorizeRequestWithTags(ctx context.Context, r *http.Request, action polic
Groups: cred.Groups,
Action: policy.ListBucketAction,
BucketName: bucket,
ConditionValues: conditionValuesForAuth(region, auth.AnonymousCredentials),
ConditionValues: getConditionValues(r, region, auth.AnonymousCredentials),
IsOwner: false,
ObjectName: object,
}) {
@@ -509,7 +468,7 @@ func authorizeRequestWithTags(ctx context.Context, r *http.Request, action polic
Groups: cred.Groups,
Action: policy.Action(policy.DeleteObjectVersionAction),
BucketName: bucket,
ConditionValues: conditionValuesForAuth("", cred),
ConditionValues: getConditionValues(r, "", cred),
ObjectName: object,
IsOwner: owner,
Claims: cred.Claims,
@@ -523,7 +482,7 @@ func authorizeRequestWithTags(ctx context.Context, r *http.Request, action polic
Groups: cred.Groups,
Action: action,
BucketName: bucket,
ConditionValues: conditionValuesForAuth("", cred),
ConditionValues: getConditionValues(r, "", cred),
ObjectName: object,
IsOwner: owner,
Claims: cred.Claims,
@@ -540,7 +499,7 @@ func authorizeRequestWithTags(ctx context.Context, r *http.Request, action polic
Groups: cred.Groups,
Action: policy.ListBucketAction,
BucketName: bucket,
ConditionValues: conditionValuesForAuth("", cred),
ConditionValues: getConditionValues(r, "", cred),
ObjectName: object,
IsOwner: owner,
Claims: cred.Claims,
@@ -761,10 +720,6 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
// call verifies bucket policies and IAM policies, supports multi user
// checks etc.
func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
return isPutActionAllowedWithRequestTags(ctx, atype, bucketName, objectName, r, action, nil)
}
func isPutActionAllowedWithRequestTags(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action policy.Action, requestTags *string) (s3Err APIErrorCode) {
var cred auth.Credentials
var owner bool
region := globalSite.Region()
@@ -805,7 +760,7 @@ func isPutActionAllowedWithRequestTags(ctx context.Context, atype authType, buck
Groups: cred.Groups,
Action: action,
BucketName: bucketName,
ConditionValues: getConditionValuesWithTags(r, "", auth.AnonymousCredentials, "", requestTags),
ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
IsOwner: false,
ObjectName: objectName,
}) {
@@ -819,7 +774,7 @@ func isPutActionAllowedWithRequestTags(ctx context.Context, atype authType, buck
Groups: cred.Groups,
Action: action,
BucketName: bucketName,
ConditionValues: getConditionValuesWithTags(r, "", cred, "", requestTags),
ConditionValues: getConditionValues(r, "", cred),
ObjectName: objectName,
IsOwner: owner,
Claims: cred.Claims,
+2 -3
View File
@@ -27,9 +27,8 @@ const _authType_name = "UnknownAnonymousPresignedPresignedV2PostPolicyStreamingS
var _authType_index = [...]uint8{0, 7, 16, 25, 36, 46, 61, 67, 75, 78, 81, 103, 127}
func (i authType) String() string {
idx := int(i) - 0
if i < 0 || idx >= len(_authType_index)-1 {
if i < 0 || i >= authType(len(_authType_index)-1) {
return "authType(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _authType_name[_authType_index[idx]:_authType_index[idx+1]]
return _authType_name[_authType_index[i]:_authType_index[i+1]]
}
+1 -1
View File
@@ -37,7 +37,7 @@ import (
"github.com/minio/pkg/v3/wildcard"
"github.com/minio/pkg/v3/workers"
"github.com/minio/pkg/v3/xtime"
"go.yaml.in/yaml/v3"
"gopkg.in/yaml.v3"
)
// expire: # Expire objects that match a condition
+1 -1
View File
@@ -21,7 +21,7 @@ import (
"slices"
"testing"
"go.yaml.in/yaml/v3"
"gopkg.in/yaml.v3"
)
func TestParseBatchJobExpire(t *testing.T) {
+1 -1
View File
@@ -52,7 +52,7 @@ import (
"github.com/minio/pkg/v3/env"
"github.com/minio/pkg/v3/policy"
"github.com/minio/pkg/v3/workers"
"go.yaml.in/yaml/v3"
"gopkg.in/yaml.v3"
)
var globalBatchConfig batch.Config
+1 -1
View File
@@ -21,7 +21,7 @@ import (
"slices"
"testing"
"go.yaml.in/yaml/v3"
"gopkg.in/yaml.v3"
)
func TestBatchJobPrefix_UnmarshalYAML(t *testing.T) {
+1 -1
View File
@@ -24,7 +24,7 @@ import (
"github.com/dustin/go-humanize"
"github.com/minio/pkg/v3/wildcard"
"go.yaml.in/yaml/v3"
"gopkg.in/yaml.v3"
)
//go:generate msgp -file $GOFILE
+1 -1
View File
@@ -21,7 +21,7 @@ import (
"slices"
"testing"
"go.yaml.in/yaml/v3"
"gopkg.in/yaml.v3"
)
func TestParseBatchJobReplicate(t *testing.T) {
+2 -3
View File
@@ -18,9 +18,8 @@ const _batchJobMetric_name = "ReplicationKeyRotationExpire"
var _batchJobMetric_index = [...]uint8{0, 11, 22, 28}
func (i batchJobMetric) String() string {
idx := int(i) - 0
if i < 0 || idx >= len(_batchJobMetric_index)-1 {
if i >= batchJobMetric(len(_batchJobMetric_index)-1) {
return "batchJobMetric(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _batchJobMetric_name[_batchJobMetric_index[idx]:_batchJobMetric_index[idx+1]]
return _batchJobMetric_name[_batchJobMetric_index[i]:_batchJobMetric_index[i+1]]
}
-127
View File
@@ -944,130 +944,3 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
// `ExecObjectLayerAPINilTest` manages the operation.
ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq)
}
func TestAPIDeleteMultipleObjectsVersionIDNullCondition(t *testing.T) {
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPIDeleteMultipleObjectsVersionIDNullCondition,
endpoints: []string{"DeleteMultipleObjects", "PutBucketPolicy"},
makeBucketOptions: MakeBucketOptions{VersioningEnabled: true},
})
}
func testAPIDeleteMultipleObjectsVersionIDNullCondition(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
credentials auth.Credentials, t *testing.T,
) {
versionIDs := make(map[string]string, 4)
for _, objectName := range []string{
"without-version-id-before",
"with-version-id",
"without-version-id-after",
"with-null-version-id",
} {
data := []byte(objectName)
info, err := obj.PutObject(t.Context(), bucketName, objectName,
mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{Versioned: true})
if err != nil {
t.Fatalf("%s: put %q: %v", instanceType, objectName, err)
}
if info.VersionID == "" {
t.Fatalf("%s: put %q did not create a version ID", instanceType, objectName)
}
versionIDs[objectName] = info.VersionID
}
policyBytes := fmt.Appendf(nil, `{
"Version":"2012-10-17",
"Statement":[{
"Effect":"Allow",
"Principal":"*",
"Action":"s3:DeleteObject",
"Resource":"arn:aws:s3:::%s/*",
"Condition":{"Null":{"s3:versionid":"true"}}
}]
}`, bucketName)
policyReq, err := newTestSignedRequestV4(http.MethodPut, getPutPolicyURL("", bucketName), int64(len(policyBytes)),
bytes.NewReader(policyBytes), credentials.AccessKey, credentials.SecretKey, nil)
if err != nil {
t.Fatal(err)
}
policyRec := httptest.NewRecorder()
apiRouter.ServeHTTP(policyRec, policyReq)
if policyRec.Code != http.StatusNoContent {
t.Fatalf("%s: put policy returned %d: %s", instanceType, policyRec.Code, policyRec.Body.String())
}
deleteBody := encodeResponse(DeleteObjectsRequest{Objects: []ObjectToDelete{
{ObjectV: ObjectV{ObjectName: "without-version-id-before"}},
{ObjectV: ObjectV{ObjectName: "with-version-id", VersionID: versionIDs["with-version-id"]}},
{ObjectV: ObjectV{ObjectName: "without-version-id-after"}},
{ObjectV: ObjectV{ObjectName: "with-null-version-id", VersionID: nullVersionID}},
}})
// A query-level versionId is not the version of every XML entry. Each
// object's optional VersionId remains the effective authorization value.
deleteURL := getDeleteMultipleObjectsURL("", bucketName) + "&versionId=query-level-decoy"
deleteReq, err := newTestRequest(http.MethodPost, deleteURL,
int64(len(deleteBody)), bytes.NewReader(deleteBody))
if err != nil {
t.Fatal(err)
}
deleteRec := httptest.NewRecorder()
apiRouter.ServeHTTP(deleteRec, deleteReq)
if deleteRec.Code != http.StatusOK {
t.Fatalf("%s: delete returned %d: %s", instanceType, deleteRec.Code, deleteRec.Body.String())
}
var response DeleteObjectsResponse
if err = xml.Unmarshal(deleteRec.Body.Bytes(), &response); err != nil {
t.Fatalf("%s: decode response: %v: %s", instanceType, err, deleteRec.Body.String())
}
deleted := make(map[string]DeletedObject, len(response.DeletedObjects))
for _, object := range response.DeletedObjects {
deleted[object.ObjectName] = object
}
for _, objectName := range []string{"without-version-id-before", "without-version-id-after"} {
object, ok := deleted[objectName]
if !ok || !object.DeleteMarker || object.DeleteMarkerVersionID == "" {
t.Errorf("%s: %q was not a successful delete-marker creation: %+v", instanceType, objectName, response.DeletedObjects)
}
}
if len(deleted) != 2 {
t.Errorf("%s: unexpected deleted objects: %+v", instanceType, response.DeletedObjects)
}
errorsByKey := make(map[string]DeleteError, len(response.Errors))
for _, deleteErr := range response.Errors {
errorsByKey[deleteErr.Key] = deleteErr
}
for objectName, versionID := range map[string]string{
"with-version-id": versionIDs["with-version-id"],
"with-null-version-id": nullVersionID,
} {
deleteErr, ok := errorsByKey[objectName]
if !ok || deleteErr.VersionID != versionID || deleteErr.Code != errorCodes[ErrAccessDenied].Code {
t.Errorf("%s: %q did not return AccessDenied for version %q: %+v", instanceType, objectName, versionID, response.Errors)
}
}
if len(errorsByKey) != 2 {
t.Errorf("%s: unexpected delete errors: %+v", instanceType, response.Errors)
}
// A simple delete adds a marker and keeps the old version. The explicitly
// named version must also remain because its policy condition did not match.
for objectName, versionID := range versionIDs {
if _, err = obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{VersionID: versionID}); err != nil {
t.Errorf("%s: version %s of %q was not preserved: %v", instanceType, versionID, objectName, err)
}
}
for _, objectName := range []string{"without-version-id-before", "without-version-id-after"} {
if _, err = obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); !isErrObjectNotFound(err) {
t.Errorf("%s: simple delete of %q did not hide the latest object behind a delete marker: %v", instanceType, objectName, err)
}
}
for _, objectName := range []string{"with-version-id", "with-null-version-id"} {
if info, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err != nil {
t.Errorf("%s: denied version delete removed latest %q: %v", instanceType, objectName, err)
} else if info.VersionID != versionIDs[objectName] {
t.Errorf("%s: latest version of %q changed from %s to %s", instanceType, objectName, versionIDs[objectName], info.VersionID)
}
}
}
+20 -154
View File
@@ -34,7 +34,6 @@ import (
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/logger"
"github.com/minio/pkg/v3/policy"
"github.com/minio/pkg/v3/policy/condition"
)
// PolicySys - policy subsystem.
@@ -76,104 +75,7 @@ func getSTSConditionValues(r *http.Request, lc string, cred auth.Credentials) ma
return m
}
type conditionValueSource uint8
const (
conditionValueFromHeader conditionValueSource = 1 << iota
conditionValueFromQuery
)
// clientSuppliedConditionKeys records where each request-derived condition
// value actually comes from. Most x-amz-* values are headers, list parameters
// are query-only, and storage class retains the compatible query form consumed
// by object operations.
var clientSuppliedConditionKeys = map[string]conditionValueSource{
"prefix": conditionValueFromQuery,
"delimiter": conditionValueFromQuery,
"max-keys": conditionValueFromQuery,
// AWS explicitly excludes the query-string form from this policy key,
// even though MinIO may consume it separately while verifying a presign.
"x-amz-content-sha256": conditionValueFromHeader,
"x-amz-copy-source": conditionValueFromHeader,
"x-amz-metadata-directive": conditionValueFromHeader,
"x-amz-server-side-encryption": conditionValueFromHeader,
"x-amz-server-side-encryption-aws-kms-key-id": conditionValueFromHeader,
"x-amz-server-side-encryption-customer-algorithm": conditionValueFromHeader,
"x-amz-storage-class": conditionValueFromHeader | conditionValueFromQuery,
}
func acceptsConditionValueSource(key string, source conditionValueSource) bool {
name := strings.ToLower(key)
allowed, ok := clientSuppliedConditionKeys[name]
if !ok {
return true
}
if allowed&source == 0 {
return false
}
return source != conditionValueFromQuery || key == name
}
// internalConditionKeys holds every other name a condition key can resolve to.
// Those name values MinIO derives for itself - identity from the credential,
// time from the clock, transport from the connection - and a request must never
// write one, whether or not the server populated it this time round: a name the
// server left empty is as forgeable as one it filled in, and the condition
// reading it cannot tell the difference.
//
// Deriving the set from the condition keys rather than from what
// getConditionValues writes is what makes it complete. The engine reads by key
// name, so the key list is the attack surface; enumerating the writes misses
// every key the server has no value for, which is most of jwt: and ldap:. It
// also defaults new upstream keys to reserved, which is the safe direction.
//
// Reserving a name only removes it from the condition map. Request handling is
// untouched - a handler still reads its own query parameters and headers.
//
// aws:SourceIp is still only as trustworthy as the forwarding headers it is
// computed from, see the note on GetSourceIPFromHeaders.
var internalConditionKeys = func() map[string]struct{} {
keys := make(map[string]struct{}, 2*len(condition.AllSupportedKeys))
for _, keyName := range condition.AllSupportedKeys {
name := keyName.ToKey().Name()
if _, clientSupplied := clientSuppliedConditionKeys[name]; clientSupplied {
continue
}
// A condition key resolves against its exact name and falls back to the
// canonical MIME form, so both spellings have to be held. This is also
// what covers object lock, stored as Object-Lock-Mode and read as
// s3:object-lock-mode.
keys[name] = struct{}{}
keys[http.CanonicalHeaderKey(name)] = struct{}{}
}
return keys
}()
// Tag conditions name one tag key each, so the variable forms are reserved by
// prefix; the bare names come from the loop above.
var internalConditionKeyPrefixes = []string{"ExistingObjectTag/", "RequestObjectTag/"}
func isInternalConditionKey(key string) bool {
if _, ok := internalConditionKeys[key]; ok {
return true
}
for _, prefix := range internalConditionKeyPrefixes {
if strings.HasPrefix(key, prefix) {
return true
}
}
return false
}
func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[string][]string {
return getConditionValuesWithExistingTags(r, lc, cred, "")
}
func getConditionValuesWithExistingTags(r *http.Request, lc string, cred auth.Credentials, existingTags string) map[string][]string {
return getConditionValuesWithTags(r, lc, cred, existingTags, nil)
}
func getConditionValuesWithTags(r *http.Request, lc string, cred auth.Credentials, existingTags string, requestTags *string) map[string][]string {
currTime := UTCNow()
var (
@@ -198,13 +100,10 @@ func getConditionValuesWithTags(r *http.Request, lc string, cred auth.Credential
}
}
// Match the version the object layer will act on: newContext and getOpts both
// TrimSpace this value, so leaving it untrimmed here would let a padded
// ?versionId=V%20 present a different s3:versionid than the effective version.
vid := strings.TrimSpace(r.Form.Get(xhttp.VersionID))
vid := r.Form.Get(xhttp.VersionID)
if vid == "" {
if u, err := url.Parse(r.Header.Get(xhttp.AmzCopySource)); err == nil {
vid = strings.TrimSpace(u.Query().Get(xhttp.VersionID))
vid = u.Query().Get(xhttp.VersionID)
}
}
@@ -237,57 +136,34 @@ func getConditionValuesWithTags(r *http.Request, lc string, cred auth.Credential
"principaltype": {principalType},
"userid": {username},
"username": {username},
"versionid": {vid},
"signatureversion": {signatureVersion},
"authType": {authtype},
}
// Null conditions distinguish an absent key from a present key with an
// empty value. Only expose s3:versionid when the request names a version.
if vid != "" {
args["versionid"] = []string{vid}
}
if lc != "" {
args["LocationConstraint"] = []string{lc}
}
if storageClass, ok := getRequestHeaderOrQueryValue(r, xhttp.AmzStorageClass); ok {
args[strings.ToLower(xhttp.AmzStorageClass)] = []string{storageClass}
}
cloneHeader := r.Header.Clone()
signatureAge := cloneHeader.Get("x-amz-signature-age")
cloneHeader.Del("x-amz-signature-age")
// The presigned V4 verifier overwrites this internal scratch header after
// validating the signature. Ignore a value supplied on every other request
// type, where it would otherwise synthesize s3:signatureAge.
if authType == authTypePresigned && signatureAge != "" {
args["signatureAge"] = []string{signatureAge}
if v := cloneHeader.Get("x-amz-signature-age"); v != "" {
args["signatureAge"] = []string{v}
cloneHeader.Del("x-amz-signature-age")
}
userTags := cloneHeader.Get(xhttp.AmzObjectTagging)
if requestTags != nil {
userTags = *requestTags
}
if userTags != "" {
if userTags := cloneHeader.Get(xhttp.AmzObjectTagging); userTags != "" {
tag, _ := tags.ParseObjectTags(userTags)
if tag != nil {
tagMap := tag.ToMap()
keys := make([]string, 0, len(tagMap))
for k, v := range tagMap {
args[pathJoin("ExistingObjectTag", k)] = []string{v}
args[pathJoin("RequestObjectTag", k)] = []string{v}
keys = append(keys, k)
}
args["RequestObjectTagKeys"] = keys
}
}
if existingTags != "" {
tag, _ := tags.ParseObjectTags(existingTags)
if tag != nil {
for k, v := range tag.ToMap() {
args[pathJoin("ExistingObjectTag", k)] = []string{v}
}
}
}
for _, objLock := range []string{
xhttp.AmzObjectLockMode,
@@ -300,20 +176,8 @@ func getConditionValuesWithTags(r *http.Request, lc string, cred auth.Credential
cloneHeader.Del(objLock)
}
// The two loops below fold raw header and query values into the same map
// the server just filled in. Anything they add is indistinguishable, to a
// condition, from a value the server derived - and they merge by appending,
// so a supplied entry sits alongside the real one rather than replacing it.
// The source check keeps headers and query parameters in their actual roles;
// isInternalConditionKey keeps both apart from server-derived values.
for key, values := range cloneHeader {
if strings.EqualFold(key, xhttp.AmzObjectTagging) || strings.EqualFold(key, xhttp.AmzStorageClass) {
continue
}
if !acceptsConditionValueSource(key, conditionValueFromHeader) {
continue
}
if isInternalConditionKey(key) {
if strings.EqualFold(key, xhttp.AmzObjectTagging) {
continue
}
if existingValues, found := args[key]; found {
@@ -326,16 +190,18 @@ func getConditionValuesWithTags(r *http.Request, lc string, cred auth.Credential
cloneURLValues := make(url.Values, len(r.Form))
maps.Copy(cloneURLValues, r.Form)
for _, objLock := range []string{
xhttp.AmzObjectLockMode,
xhttp.AmzObjectLockLegalHold,
xhttp.AmzObjectLockRetainUntilDate,
} {
if values, ok := cloneURLValues[objLock]; ok {
args[strings.TrimPrefix(objLock, "X-Amz-")] = values
}
cloneURLValues.Del(objLock)
}
for key, values := range cloneURLValues {
if strings.EqualFold(key, xhttp.AmzObjectTagging) || strings.EqualFold(key, xhttp.AmzStorageClass) {
continue
}
if !acceptsConditionValueSource(key, conditionValueFromQuery) {
continue
}
if isInternalConditionKey(key) {
continue
}
if existingValues, found := args[key]; found {
args[key] = append(existingValues, values...)
} else {
-679
View File
@@ -1,679 +0,0 @@
// Copyright (c) 2015-2026 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"net"
"net/http"
"net/url"
"os"
"slices"
"strings"
"testing"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/handlers"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/pkg/v3/policy"
"github.com/minio/pkg/v3/policy/condition"
)
const (
testCondSourceIP = "203.0.113.5"
testCondRemoteILP = testCondSourceIP + ":12345"
)
func condValuesForRequest(t *testing.T, rawURL string, header map[string]string) map[string][]string {
return condValuesForRequestWithTags(t, rawURL, header, "", nil)
}
func condValuesForRequestWithExistingTags(t *testing.T, rawURL string, header map[string]string, existingTags string) map[string][]string {
return condValuesForRequestWithTags(t, rawURL, header, existingTags, nil)
}
func condValuesForRequestWithTags(t *testing.T, rawURL string, header map[string]string, existingTags string, requestTags *string) map[string][]string {
t.Helper()
r, err := http.NewRequest(http.MethodGet, rawURL, nil)
if err != nil {
t.Fatal(err)
}
r.RemoteAddr = testCondRemoteILP
for k, v := range header {
r.Header.Set(k, v)
}
if err := r.ParseForm(); err != nil {
t.Fatal(err)
}
return getConditionValuesWithTags(r, "us-east-1", auth.Credentials{AccessKey: "lowpriv"}, existingTags, requestTags)
}
func resolvedConditionValues(values map[string][]string, name string) []string {
if v := values[name]; len(v) > 0 {
return v
}
return values[http.CanonicalHeaderKey(name)]
}
// A client must not be able to reach a condition key that the server computes
// for itself. Both routes are covered: a header whose canonical spelling
// collides with the key name, and a query parameter that collides with it
// exactly. The query route is the sharper one, because the merge appended to
// the server's value rather than replacing it and a condition function matches
// when any single value matches.
func TestGetConditionValuesRejectsClientSuppliedServerKeys(t *testing.T) {
honest := condValuesForRequest(t, "http://minio.local/bkt/obj", nil)
for _, kn := range condition.AllSupportedKeys {
name := kn.ToKey().Name()
if _, clientSupplied := clientSuppliedConditionKeys[name]; clientSupplied {
continue // the request is where this one is supposed to come from
}
// Deliberately not skipped when the server left the key empty. An empty
// name is exactly as forgeable as a populated one, and the keys the
// server has no value for - most of jwt: and ldap: - are the ones a
// resource variable expands.
want := honest[name]
canonical := http.CanonicalHeaderKey(name)
t.Run("query/"+name, func(t *testing.T) {
got := condValuesForRequest(t,
"http://minio.local/bkt/obj?"+url.Values{name: {"ATTACKER"}}.Encode(), nil)
if slices.Contains(got[name], "ATTACKER") {
t.Errorf("?%s= reached %v, server computed %v", name, got[name], want)
}
if !slices.Equal(got[name], want) {
t.Errorf("%v changed to %v", want, got[name])
}
})
// aws:Referer is read out of the Referer header, so the header is its
// source of truth rather than a way to forge it. aws:UserAgent is not
// in the same position: it comes from User-Agent, which does not
// canonicalise to "Useragent".
if kn == condition.AWSReferer {
continue
}
t.Run("header/"+canonical, func(t *testing.T) {
got := condValuesForRequest(t, "http://minio.local/bkt/obj",
map[string]string{canonical: "ATTACKER"})
// The lookup the policy engine itself performs, exact name first
// with the canonical form as fallback.
seen := got[name]
if len(seen) == 0 {
seen = got[canonical]
}
if slices.Contains(seen, "ATTACKER") {
t.Errorf("%s: header reached the lookup as %v, server computed %v",
canonical, seen, want)
}
})
}
}
func TestGetConditionValuesUsesActualRequestSource(t *testing.T) {
for name, source := range clientSuppliedConditionKeys {
t.Run(name, func(t *testing.T) {
fromHeader := condValuesForRequest(t, "http://minio.local/bkt/obj",
map[string]string{name: "HEADER"})
fromQuery := condValuesForRequest(t,
"http://minio.local/bkt/obj?"+url.Values{name: {"QUERY"}}.Encode(), nil)
fromCanonicalQuery := condValuesForRequest(t,
"http://minio.local/bkt/obj?"+url.Values{http.CanonicalHeaderKey(name): {"QUERY"}}.Encode(), nil)
if got, want := slices.Contains(resolvedConditionValues(fromHeader, name), "HEADER"), source&conditionValueFromHeader != 0; got != want {
t.Errorf("header accepted=%v, want %v: %v", got, want, fromHeader)
}
if got, want := slices.Contains(resolvedConditionValues(fromQuery, name), "QUERY"), source&conditionValueFromQuery != 0; got != want {
t.Errorf("query accepted=%v, want %v: %v", got, want, fromQuery)
}
canonicalQueryAllowed := name == strings.ToLower(xhttp.AmzStorageClass)
if got := slices.Contains(resolvedConditionValues(fromCanonicalQuery, name), "QUERY"); got != canonicalQueryAllowed {
t.Errorf("case-variant query accepted=%v, want %v: %v", got, canonicalQueryAllowed, fromCanonicalQuery)
}
})
}
storageURL := "http://minio.local/bkt/obj?" + url.Values{
strings.ToLower(xhttp.AmzStorageClass): {"QUERY"},
}.Encode()
storageValues := condValuesForRequest(t, storageURL, map[string]string{xhttp.AmzStorageClass: "HEADER"})
if got := resolvedConditionValues(storageValues, strings.ToLower(xhttp.AmzStorageClass)); !slices.Equal(got, []string{"HEADER"}) {
t.Errorf("storage class did not use header precedence: %v", got)
}
fromQuery := condValuesForRequest(t,
"http://minio.local/bkt/obj?"+url.Values{xhttp.AmzObjectLockMode: {"COMPLIANCE"}}.Encode(), nil)
if got := resolvedConditionValues(fromQuery, "object-lock-mode"); len(got) != 0 {
t.Errorf("object-lock query value reached header condition as %v", got)
}
}
func TestGetConditionValuesVersionIDPresence(t *testing.T) {
nullVersionID, err := condition.NewNullFunc(condition.S3VersionID.ToKey(), true)
if err != nil {
t.Fatal(err)
}
withoutVersionID := condValuesForRequest(t, "http://minio.local/bkt/obj", nil)
if _, ok := withoutVersionID["versionid"]; ok {
t.Fatalf("an absent versionId was exposed to policy evaluation as %v", withoutVersionID["versionid"])
}
if !condition.NewFunctions(nullVersionID).Evaluate(withoutVersionID) {
t.Fatal("Null s3:versionid=true did not match a request without versionId")
}
const versionID = "7f4b6b5f-bf25-4e98-95df-90cba8070dd8"
withVersionID := condValuesForRequest(t,
"http://minio.local/bkt/obj?"+url.Values{xhttp.VersionID: {versionID}}.Encode(), nil)
if got := withVersionID["versionid"]; !slices.Equal(got, []string{versionID}) {
t.Fatalf("expected versionId %q, got %v", versionID, got)
}
if condition.NewFunctions(nullVersionID).Evaluate(withVersionID) {
t.Fatal("Null s3:versionid=true matched a request with versionId")
}
copySourceVersion := condValuesForRequest(t, "http://minio.local/bkt/copied", map[string]string{
xhttp.AmzCopySource: "/source-bucket/source-object?" + url.Values{xhttp.VersionID: {versionID}}.Encode(),
})
if got := copySourceVersion["versionid"]; !slices.Equal(got, []string{versionID}) {
t.Fatalf("copy source versionId was lost: got %v", got)
}
// The object layer trims the version before acting on it; the condition value
// must be the same effective string, or a padded ?versionId=V%20 would let a
// StringEquals/Deny on s3:versionid see a different value than the one deleted.
paddedVersion := condValuesForRequest(t,
"http://minio.local/bkt/obj?"+url.Values{xhttp.VersionID: {versionID + " "}}.Encode(), nil)
if got := paddedVersion["versionid"]; !slices.Equal(got, []string{versionID}) {
t.Fatalf("a padded versionId was not trimmed to the effective value: got %v", got)
}
paddedCopySource := condValuesForRequest(t, "http://minio.local/bkt/copied", map[string]string{
xhttp.AmzCopySource: "/source-bucket/source-object?" + url.Values{xhttp.VersionID: {versionID + " "}}.Encode(),
})
if got := paddedCopySource["versionid"]; !slices.Equal(got, []string{versionID}) {
t.Fatalf("a padded copy source versionId was not trimmed: got %v", got)
}
// A whitespace-only versionId names no version once trimmed, exactly as the
// object layer treats it, so the key must be absent and Null:true must match.
blankVersion := condValuesForRequest(t,
"http://minio.local/bkt/obj?"+url.Values{xhttp.VersionID: {" "}}.Encode(), nil)
if _, ok := blankVersion["versionid"]; ok {
t.Fatalf("a whitespace-only versionId was exposed to policy evaluation as %v", blankVersion["versionid"])
}
if !condition.NewFunctions(nullVersionID).Evaluate(blankVersion) {
t.Fatal("Null s3:versionid=true did not match a request whose versionId was only whitespace")
}
}
func TestGetConditionValuesUsesEffectiveRequestTags(t *testing.T) {
rawURL := "http://minio.local/bkt/obj?" + url.Values{
strings.ToLower(xhttp.AmzObjectTagging): {"security=public&virus=true"},
}.Encode()
// Generic operations such as CopyObject must not gain RequestObjectTag
// values from a query parameter they do not consume.
withoutEffectiveTags := condValuesForRequest(t, rawURL, nil)
if len(withoutEffectiveTags["RequestObjectTag/security"]) != 0 || len(withoutEffectiveTags["RequestObjectTagKeys"]) != 0 {
t.Fatalf("query tags leaked into a generic operation: %v", withoutEffectiveTags)
}
effectiveTags := "security=public&virus=true"
withEffectiveTags := condValuesForRequestWithTags(t, rawURL, nil, "", &effectiveTags)
if !slices.Equal(withEffectiveTags["RequestObjectTag/security"], []string{"public"}) {
t.Fatalf("effective request tag missing: %v", withEffectiveTags)
}
if !slices.Contains(withEffectiveTags["RequestObjectTagKeys"], "security") ||
!slices.Contains(withEffectiveTags["RequestObjectTagKeys"], "virus") {
t.Fatalf("effective request tag keys missing: %v", withEffectiveTags["RequestObjectTagKeys"])
}
security, err := condition.NewStringEqualsFunc("", condition.NewKey(condition.RequestObjectTag, "security"), "public")
if err != nil {
t.Fatal(err)
}
allowedKeys, err := condition.NewStringLikeFunc("ForAllValues", condition.RequestObjectTagKeys.ToKey(), "security", "virus")
if err != nil {
t.Fatal(err)
}
conditions := condition.NewFunctions(security, allowedKeys)
if conditions.Evaluate(withoutEffectiveTags) {
t.Fatal("query upload satisfied request-tag policy without effective tags")
}
if !conditions.Evaluate(withEffectiveTags) {
t.Fatal("effective query tags did not satisfy request-tag policy")
}
}
func TestBucketPolicySSEConditionUsesHeader(t *testing.T) {
fn, err := condition.NewStringEqualsFunc("", condition.S3XAmzServerSideEncryption.ToKey(), "aws:kms")
if err != nil {
t.Fatal(err)
}
conditions := condition.NewFunctions(fn)
if conditions.Evaluate(condValuesForRequest(t,
"http://minio.local/bkt/obj?x-amz-server-side-encryption=aws%3Akms", nil)) {
t.Error("query parameter satisfied a condition on the SSE request header")
}
if !conditions.Evaluate(condValuesForRequest(t, "http://minio.local/bkt/obj",
map[string]string{xhttp.AmzServerSideEncryption: "aws:kms"})) {
t.Error("SSE request header did not satisfy its condition")
}
}
// The end to end shape of the bypass: an IpAddress condition restricting a
// bucket to an internal range, against a request from outside it.
func TestBucketPolicySourceIPCannotBeForged(t *testing.T) {
_, cidr, err := net.ParseCIDR("10.0.0.0/8")
if err != nil {
t.Fatal(err)
}
fn, err := condition.NewIPAddressFunc(condition.AWSSourceIP.ToKey(), cidr)
if err != nil {
t.Fatal(err)
}
bp := policy.BucketPolicy{
Version: policy.DefaultVersion,
Statements: []policy.BPStatement{{
Effect: policy.Allow,
Principal: policy.NewPrincipal("*"),
Actions: policy.NewActionSet(policy.GetObjectAction),
Resources: policy.NewResourceSet(policy.NewResource("bkt/*")),
Conditions: condition.NewFunctions(fn),
}},
}
allowed := func(rawURL string, header map[string]string) bool {
return bp.IsAllowed(policy.BucketPolicyArgs{
Action: policy.GetObjectAction,
BucketName: "bkt",
ObjectName: "obj",
ConditionValues: condValuesForRequest(t, rawURL, header),
})
}
if allowed("http://minio.local/bkt/obj", nil) {
t.Fatal("baseline: an address outside 10.0.0.0/8 must not satisfy the condition")
}
if allowed("http://minio.local/bkt/obj?SourceIp=10.1.2.3", nil) {
t.Error("a query parameter forged aws:SourceIp")
}
if allowed("http://minio.local/bkt/obj", map[string]string{"Sourceip": "10.1.2.3"}) {
t.Error("a header forged aws:SourceIp")
}
}
// The trust policy has to reach the decision, not merely the resolver. This
// drives a forged X-Forwarded-For all the way through getConditionValues into a
// real IpAddress evaluation under each mode. Everything else about the trust
// modes is tested where the logic lives; this is the only test that would notice
// if the resolver were correct but the policy engine were reading something else.
func TestBucketPolicySourceIPForgeryAcrossTrustModes(t *testing.T) {
_, cidr, err := net.ParseCIDR("10.0.0.0/8")
if err != nil {
t.Fatal(err)
}
fn, err := condition.NewIPAddressFunc(condition.AWSSourceIP.ToKey(), cidr)
if err != nil {
t.Fatal(err)
}
bp := policy.BucketPolicy{
Version: policy.DefaultVersion,
Statements: []policy.BPStatement{{
Effect: policy.Allow,
Principal: policy.NewPrincipal("*"),
Actions: policy.NewActionSet(policy.GetObjectAction),
Resources: policy.NewResourceSet(policy.NewResource("bkt/*")),
Conditions: condition.NewFunctions(fn),
}},
}
// An address inside the permitted range, asserted by a client that is not.
const forgedClaim = "10.1.2.3"
const outsider = "203.0.113.5:12345"
const proxy = "192.0.2.7:9000"
tests := []struct {
name string
proxies string
peer string
allowed bool
}{{
// Documented, and the reason the allow-list exists.
name: "default mode believes the claim",
peer: outsider,
allowed: true,
}, {
name: "trusting nobody ignores the claim",
proxies: handlers.TrustNoProxies,
peer: outsider,
allowed: false,
}, {
name: "allow-list ignores an unlisted peer's claim",
proxies: "192.0.2.7",
peer: outsider,
allowed: false,
}, {
// The allow-list must not break the deployment it exists to serve.
name: "allow-list still honors its own proxy",
proxies: "192.0.2.7",
peer: proxy,
allowed: true,
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Cleanup(func() {
os.Unsetenv(handlers.EnvTrustedProxies)
if err := handlers.ConfigureSourceIPTrust(); err != nil {
t.Fatalf("restoring the default policy: %v", err)
}
})
if tt.proxies == "" {
os.Unsetenv(handlers.EnvTrustedProxies)
} else {
t.Setenv(handlers.EnvTrustedProxies, tt.proxies)
}
if err := handlers.ConfigureSourceIPTrust(); err != nil {
t.Fatalf("configuring %q: %v", tt.proxies, err)
}
r, err := http.NewRequest(http.MethodGet, "http://minio.local/bkt/obj", nil)
if err != nil {
t.Fatal(err)
}
r.RemoteAddr = tt.peer
r.Header.Set("X-Forwarded-For", forgedClaim)
if err := r.ParseForm(); err != nil {
t.Fatal(err)
}
got := bp.IsAllowed(policy.BucketPolicyArgs{
Action: policy.GetObjectAction,
BucketName: "bkt",
ObjectName: "obj",
ConditionValues: getConditionValues(r, "us-east-1", auth.Credentials{AccessKey: "lowpriv"}),
})
if got != tt.allowed {
t.Errorf("IsAllowed = %v, want %v (peer %s claiming %s)", got, tt.allowed, tt.peer, forgedClaim)
}
})
}
}
// aws:SourceIp must be whatever the hardened resolver decided and nothing else.
// The resolver is where the forwarded-header trust policy is enforced and where
// its three modes are tested (internal/handlers/proxy_test.go); this pins the
// join, so the condition value cannot drift onto some other derivation that the
// policy would not cover.
//
// It also records the default-mode contract: with no trust policy configured,
// each of the three forwarded headers still sets aws:SourceIp, and so an
// IpAddress condition is only as good as the network path to the API port.
// Enforcing such a condition against a client with direct access requires
// MINIO_API_TRUSTED_PROXIES. Note that _MINIO_API_XFF_HEADER=off does not
// achieve it: the loop below covers all three headers precisely because
// suppressing one of them only moves the answer to the next.
func TestGetConditionValuesSourceIPMatchesResolver(t *testing.T) {
for _, header := range []map[string]string{
nil,
{"X-Forwarded-For": "10.1.2.3"},
{"X-Real-IP": "10.1.2.3"},
{"Forwarded": "for=10.1.2.3"},
{"X-Forwarded-For": "10.1.2.3, 198.51.100.9"},
} {
r, err := http.NewRequest(http.MethodGet, "http://minio.local/bkt/obj", nil)
if err != nil {
t.Fatal(err)
}
r.RemoteAddr = testCondRemoteILP
for k, v := range header {
r.Header.Set(k, v)
}
got := resolvedConditionValues(condValuesForRequest(t, "http://minio.local/bkt/obj", header), condition.AWSSourceIP.ToKey().Name())
want := handlers.GetSourceIPRaw(r)
if len(got) != 1 || got[0] != want {
t.Errorf("headers %v: aws:SourceIp = %v, resolver returned %q", header, got, want)
}
}
}
// "Deny unless the connection is TLS" is the usual hardening statement, and
// aws:SecureTransport is computed from r.TLS.
func TestBucketPolicySecureTransportCannotBeForged(t *testing.T) {
fn, err := condition.NewBoolFunc(condition.AWSSecureTransport.ToKey(), false)
if err != nil {
t.Fatal(err)
}
bp := policy.BucketPolicy{
Version: policy.DefaultVersion,
Statements: []policy.BPStatement{
{
Effect: policy.Allow, Principal: policy.NewPrincipal("*"),
Actions: policy.NewActionSet(policy.GetObjectAction),
Resources: policy.NewResourceSet(policy.NewResource("bkt/*")),
},
{
Effect: policy.Deny, Principal: policy.NewPrincipal("*"),
Actions: policy.NewActionSet(policy.GetObjectAction),
Resources: policy.NewResourceSet(policy.NewResource("bkt/*")),
Conditions: condition.NewFunctions(fn),
},
},
}
allowed := func(rawURL string, header map[string]string) bool {
return bp.IsAllowed(policy.BucketPolicyArgs{
Action: policy.GetObjectAction,
BucketName: "bkt",
ObjectName: "obj",
ConditionValues: condValuesForRequest(t, rawURL, header),
})
}
// r.TLS is nil throughout, so every one of these is a plaintext request.
if allowed("http://minio.local/bkt/obj", nil) {
t.Fatal("baseline: a plaintext request must be denied")
}
if allowed("http://minio.local/bkt/obj?SecureTransport=true", nil) {
t.Error("a query parameter forged aws:SecureTransport")
}
if allowed("http://minio.local/bkt/obj", map[string]string{"Securetransport": "true"}) {
t.Error("a header forged aws:SecureTransport")
}
}
// Reserving the server's own keys must not stop the request from supplying the
// values that are client-derived by design.
func TestGetConditionValuesKeepsClientDerivedKeys(t *testing.T) {
got := condValuesForRequest(t, "http://minio.local/bkt/obj?prefix=team%2F",
map[string]string{
xhttp.AmzObjectLockMode: "GOVERNANCE",
xhttp.AmzServerSideEncryption: "aws:kms",
"X-Amz-Meta-Team": "storage",
xhttp.AmzObjectTagging: "project=silo",
})
for _, tc := range []struct {
key string
want string
}{
{"Object-Lock-Mode", "GOVERNANCE"},
{xhttp.AmzServerSideEncryption, "aws:kms"},
{"X-Amz-Meta-Team", "storage"},
{"RequestObjectTag/project", "silo"},
{"prefix", "team/"},
} {
if !slices.Contains(got[tc.key], tc.want) {
t.Errorf("%s: expected %q, got %v", tc.key, tc.want, got[tc.key])
}
}
if !slices.Contains(got["RequestObjectTagKeys"], "project") {
t.Errorf("RequestObjectTagKeys: expected project, got %v", got["RequestObjectTagKeys"])
}
if len(got["ExistingObjectTag/project"]) != 0 {
t.Errorf("request tags leaked into ExistingObjectTag: %v", got["ExistingObjectTag/project"])
}
}
func TestGetConditionValuesSeparatesRequestAndExistingTags(t *testing.T) {
got := condValuesForRequestWithExistingTags(t, "http://minio.local/bkt/obj",
map[string]string{xhttp.AmzObjectTagging: "project=request&new=yes"},
"project=stored&old=yes")
for _, tc := range []struct {
key string
want string
}{
{"RequestObjectTag/project", "request"},
{"RequestObjectTag/new", "yes"},
{"ExistingObjectTag/project", "stored"},
{"ExistingObjectTag/old", "yes"},
} {
if !slices.Equal(got[tc.key], []string{tc.want}) {
t.Errorf("%s: expected %q, got %v", tc.key, tc.want, got[tc.key])
}
}
if len(got["ExistingObjectTag/new"]) != 0 || len(got["RequestObjectTag/old"]) != 0 {
t.Errorf("tag sources crossed: request new=%v, existing old=%v",
got["ExistingObjectTag/new"], got["RequestObjectTag/old"])
}
}
// Keys the server did not populate for this request are as forgeable as ones it
// did, so the reservation cannot depend on presence.
func TestGetConditionValuesRejectsAbsentInternalKeys(t *testing.T) {
for _, key := range []string{
"signatureAge",
"groups",
"DurationSeconds",
"ExistingObjectTag/security",
"RequestObjectTag/security",
"RequestObjectTagKeys",
"object-lock-mode",
"object-lock-remaining-retention-days",
} {
t.Run(key, func(t *testing.T) {
got := condValuesForRequest(t,
"http://minio.local/bkt/obj?"+url.Values{key: {"ATTACKER"}}.Encode(), nil)
if slices.Contains(got[key], "ATTACKER") {
t.Errorf("?%s= was accepted into the condition values as %v", key, got[key])
}
})
}
}
func TestGetConditionValuesOnlyAcceptsPresignedSignatureAge(t *testing.T) {
const signatureAgeHeader = "x-amz-signature-age"
for _, tc := range []struct {
name string
target string
headers map[string]string
want bool
}{
{
name: "anonymous client header",
target: "http://minio.local/bkt/obj",
headers: map[string]string{signatureAgeHeader: "1"},
},
{
name: "header-signed client header",
target: "http://minio.local/bkt/obj",
headers: map[string]string{
xhttp.Authorization: signV4Algorithm + " attacker",
signatureAgeHeader: "1",
},
},
{
name: "presigned verifier value",
target: "http://minio.local/bkt/obj?" + url.Values{
xhttp.AmzCredential: {"access/20260803/us-east-1/s3/aws4_request"},
}.Encode(),
headers: map[string]string{signatureAgeHeader: "250"},
want: true,
},
} {
t.Run(tc.name, func(t *testing.T) {
got := condValuesForRequest(t, tc.target, tc.headers)
_, ok := got["signatureAge"]
if ok != tc.want {
t.Fatalf("signatureAge presence: expected %v, got %v", tc.want, got["signatureAge"])
}
})
}
}
// The object-lock value is stored under the header spelling while the policy key
// that reads it is lower case. Reserving only one spelling lets the other be
// supplied and resolved in its place - which the policy package's exact-name
// lookup then prefers over the real one.
func TestGetConditionValuesObjectLockSpelling(t *testing.T) {
got := condValuesForRequest(t,
"http://minio.local/bkt/obj?object-lock-mode=COMPLIANCE",
map[string]string{xhttp.AmzObjectLockMode: "GOVERNANCE"})
if v, ok := got["object-lock-mode"]; ok {
t.Errorf("the lower-case spelling was accepted: %v", v)
}
if !slices.Equal(got["Object-Lock-Mode"], []string{"GOVERNANCE"}) {
t.Errorf("expected the header value to stand, got %v", got["Object-Lock-Mode"])
}
fn, err := condition.NewStringEqualsFunc("",
condition.S3ObjectLockMode.ToKey(), "COMPLIANCE")
if err != nil {
t.Fatal(err)
}
if condition.NewFunctions(fn).Evaluate(got) {
t.Error("a policy requiring COMPLIANCE was satisfied by a GOVERNANCE request")
}
}
// Resource variables read the condition map directly, so a forgeable key is a
// forgeable resource path. ${ldap:user} and ${jwt:preferred_username} are the
// home-directory idiom for LDAP and OIDC deployments; the server derives them
// from the credential, and a request must not be able to answer them.
func TestBucketPolicyResourceVariableCannotBeForged(t *testing.T) {
for _, tc := range []struct{ variable, param, value string }{
{"${ldap:user}", "user", "alice"},
{"${ldap:username}", "username", "alice"},
{"${jwt:preferred_username}", "preferred_username", "alice"},
{"${jwt:sub}", "sub", "alice"},
{"${aws:username}", "username", "alice"},
} {
t.Run(tc.variable, func(t *testing.T) {
bp := policy.BucketPolicy{Version: policy.DefaultVersion, Statements: []policy.BPStatement{{
Effect: policy.Allow,
Principal: policy.NewPrincipal("*"),
Actions: policy.NewActionSet(policy.GetObjectAction),
Resources: policy.NewResourceSet(policy.NewResource("bkt/" + tc.variable + "/*")),
}}}
args := policy.BucketPolicyArgs{
Action: policy.GetObjectAction, BucketName: "bkt", ObjectName: tc.value + "/secret",
}
args.ConditionValues = condValuesForRequest(t,
"http://minio.local/bkt/"+tc.value+"/secret?"+
url.Values{tc.param: {tc.value}}.Encode(), nil)
if bp.IsAllowed(args) {
t.Errorf("?%s=%s expanded %s and granted the prefix", tc.param, tc.value, tc.variable)
}
})
}
}
+5 -5
View File
@@ -903,7 +903,7 @@ func (z *ReplicationState) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
var za0004 VersionPurgeStatusType
err = (*replication.VersionPurgeStatusType)(&za0004).DecodeMsg(dc)
err = za0004.DecodeMsg(dc)
if err != nil {
err = msgp.WrapError(err, "PurgeTargets", za0003)
return
@@ -1060,7 +1060,7 @@ func (z *ReplicationState) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "PurgeTargets")
return
}
err = (*replication.VersionPurgeStatusType)(&za0004).EncodeMsg(en)
err = za0004.EncodeMsg(en)
if err != nil {
err = msgp.WrapError(err, "PurgeTargets", za0003)
return
@@ -1136,7 +1136,7 @@ func (z *ReplicationState) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.AppendMapHeader(o, uint32(len(z.PurgeTargets)))
for za0003, za0004 := range z.PurgeTargets {
o = msgp.AppendString(o, za0003)
o, err = (*replication.VersionPurgeStatusType)(&za0004).MarshalMsg(o)
o, err = za0004.MarshalMsg(o)
if err != nil {
err = msgp.WrapError(err, "PurgeTargets", za0003)
return
@@ -1261,7 +1261,7 @@ func (z *ReplicationState) UnmarshalMsg(bts []byte) (o []byte, err error) {
err = msgp.WrapError(err, "PurgeTargets")
return
}
bts, err = (*replication.VersionPurgeStatusType)(&za0004).UnmarshalMsg(bts)
bts, err = za0004.UnmarshalMsg(bts)
if err != nil {
err = msgp.WrapError(err, "PurgeTargets", za0003)
return
@@ -1321,7 +1321,7 @@ func (z *ReplicationState) Msgsize() (s int) {
if z.PurgeTargets != nil {
for za0003, za0004 := range z.PurgeTargets {
_ = za0004
s += msgp.StringPrefixSize + len(za0003) + (*replication.VersionPurgeStatusType)(&za0004).Msgsize()
s += msgp.StringPrefixSize + len(za0003) + za0004.Msgsize()
}
}
s += 17 + msgp.MapHeaderSize
-379
View File
@@ -1,379 +0,0 @@
// Copyright (c) 2015-2026 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"context"
"fmt"
"net/http"
"net/http/httptest"
"runtime"
"strings"
"testing"
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio-go/v7"
"github.com/minio/minio/internal/auth"
"github.com/minio/pkg/v3/policy"
)
// These tests pin the IAM bucket/object resource boundary end to end, through
// the real handlers rather than the matcher alone. An object-only resource
// pattern ("arn:aws:s3:::bucket/*") must not authorize the bucket-level writes
// that hand a caller something its object access does not already provide —
// upstream minio/minio issue #20449 — while every shape the fix deliberately
// leaves alone keeps working. Both directions are asserted, because a change
// here that only removes permissions is correct and one that adds any is not.
// A bucket-level write reached only through an object-only grant must be
// refused, and refusing it must not destroy the bucket. A grant that names the
// bucket must still succeed, and succeeding must actually remove it.
func assertBucketDelete(ctx context.Context, c *check, admin, client *minio.Client, bucket string, wantAllowed bool) {
c.Helper()
err := client.RemoveBucket(ctx, bucket)
if wantAllowed {
if err != nil {
c.Fatalf("RemoveBucket(%s) denied, want allowed: %v", bucket, err)
}
exists, existsErr := admin.BucketExists(ctx, bucket)
if existsErr != nil {
c.Fatalf("check removed bucket %s: %v", bucket, existsErr)
}
if exists {
c.Fatalf("RemoveBucket(%s) returned nil but bucket still exists", bucket)
}
return
}
if err == nil {
c.Fatalf("RemoveBucket(%s) returned nil, want AccessDenied", bucket)
}
if response := minio.ToErrorResponse(err); response.Code != "AccessDenied" {
c.Fatalf("RemoveBucket(%s) error code=%q, want AccessDenied (err=%v)", bucket, response.Code, err)
}
exists, existsErr := admin.BucketExists(ctx, bucket)
if existsErr != nil {
c.Fatalf("check protected bucket %s: %v", bucket, existsErr)
}
if !exists {
c.Fatalf("RemoveBucket(%s) returned AccessDenied but bucket disappeared", bucket)
}
if err := admin.RemoveBucket(ctx, bucket); err != nil {
c.Fatalf("cleanup protected bucket %s: %v", bucket, err)
}
}
func createUserWithPolicy(ctx context.Context, c *check, s *TestSuiteIAM, policyJSON []byte) (*minio.Client, string) {
c.Helper()
accessKey, secretKey := mustGenerateCredentials(c)
if err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled); err != nil {
c.Fatalf("set boundary user: %v", err)
}
policyName := "boundary-" + mustGetUUID()
if err := s.adm.AddCannedPolicy(ctx, policyName, policyJSON); err != nil {
c.Fatalf("add boundary policy: %v", err)
}
if _, err := s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
Policies: []string{policyName},
User: accessKey,
}); err != nil {
c.Fatalf("attach boundary policy: %v", err)
}
return s.getUserClient(c, accessKey, secretKey, ""), accessKey
}
func TestBucketResourceBoundaryEndToEnd(t *testing.T) {
if runtime.GOOS == globalWindowsOSName {
t.Skip("IAM integration harness is disabled on Windows")
}
suite := newTestSuiteIAM(TestSuiteCommon{serverType: "ErasureSD", signer: signerV4}, false)
c := &check{t, suite.serverType}
suite.SetUpSuite(c)
defer suite.TearDownSuite(c)
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
// The #20449 reproduction: "s3:*" on "bucket/*" and nothing else.
coreBucket := getRandomBucketName()
if err := suite.client.MakeBucket(ctx, coreBucket, minio.MakeBucketOptions{}); err != nil {
c.Fatalf("create core bucket: %v", err)
}
corePolicy := fmt.Appendf(nil, `{
"Version":"2012-10-17",
"Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"arn:aws:s3:::%s/*"}]
}`, coreBucket)
coreClient, _ := createUserWithPolicy(ctx, c, suite, corePolicy)
assertBucketDelete(ctx, c, suite.client, coreClient, coreBucket, false)
// The conventional pairing of bucket and object ARNs stays authorized.
pairedBucket := getRandomBucketName()
if err := suite.client.MakeBucket(ctx, pairedBucket, minio.MakeBucketOptions{}); err != nil {
c.Fatalf("create paired bucket: %v", err)
}
pairedPolicy := fmt.Appendf(nil, `{
"Version":"2012-10-17",
"Statement":[{"Effect":"Allow","Action":"s3:*","Resource":[
"arn:aws:s3:::%s","arn:aws:s3:::%s/*"
]}]
}`, pairedBucket, pairedBucket)
pairedClient, _ := createUserWithPolicy(ctx, c, suite, pairedPolicy)
assertBucketDelete(ctx, c, suite.client, pairedClient, pairedBucket, true)
// CreateBucket and ListBucket keep the historical object-pattern matching.
compatBucket := getRandomBucketName()
compatPolicy := fmt.Appendf(nil, `{
"Version":"2012-10-17",
"Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"arn:aws:s3:::%s/*"}]
}`, compatBucket)
compatClient, _ := createUserWithPolicy(ctx, c, suite, compatPolicy)
if err := compatClient.MakeBucket(ctx, compatBucket, minio.MakeBucketOptions{}); err != nil {
c.Fatalf("CreateBucket compatibility path denied: %v", err)
}
for item := range compatClient.ListObjects(ctx, compatBucket, minio.ListObjectsOptions{}) {
if item.Err != nil {
c.Fatalf("ListBucket compatibility path denied: %v", item.Err)
}
}
if err := suite.client.RemoveBucket(ctx, compatBucket); err != nil {
c.Fatalf("cleanup compatibility bucket: %v", err)
}
// Withholding the trailing slash changes the string patterns match against,
// so a fixed-width wildcard can match the bare bucket name without ever
// having matched "bucket/". Honoring it would GRANT a delete the historical
// matcher refused, which the hardening must never do.
wildcardBucket := getRandomBucketName()
if err := suite.client.MakeBucket(ctx, wildcardBucket, minio.MakeBucketOptions{}); err != nil {
c.Fatalf("create fixed-width wildcard bucket: %v", err)
}
wildcardPolicy := fmt.Appendf(nil, `{
"Version":"2012-10-17",
"Statement":[{"Effect":"Allow","Action":"s3:DeleteBucket","Resource":"arn:aws:s3:::%s?"}]
}`, wildcardBucket[:len(wildcardBucket)-1])
wildcardClient, _ := createUserWithPolicy(ctx, c, suite, wildcardPolicy)
assertBucketDelete(ctx, c, suite.client, wildcardClient, wildcardBucket, false)
// A NotResource exclusion keeps its historical reach, so narrowing it — and
// thereby broadening the Allow it qualifies — cannot happen unnoticed.
notResourceBucket := getRandomBucketName()
if err := suite.client.MakeBucket(ctx, notResourceBucket, minio.MakeBucketOptions{}); err != nil {
c.Fatalf("create NotResource bucket: %v", err)
}
notResourcePolicy := fmt.Appendf(nil, `{
"Version":"2012-10-17",
"Statement":[{"Effect":"Allow","Action":"s3:*","NotResource":"arn:aws:s3:::%s/*"}]
}`, notResourceBucket)
notResourceClient, _ := createUserWithPolicy(ctx, c, suite, notResourcePolicy)
assertBucketDelete(ctx, c, suite.client, notResourceClient, notResourceBucket, false)
// A Deny written against "bucket/*" keeps covering the bucket-level request.
denyBucket := getRandomBucketName()
if err := suite.client.MakeBucket(ctx, denyBucket, minio.MakeBucketOptions{}); err != nil {
c.Fatalf("create Deny bucket: %v", err)
}
denyPolicy := fmt.Appendf(nil, `{
"Version":"2012-10-17",
"Statement":[
{"Effect":"Allow","Action":"s3:*","Resource":"*"},
{"Effect":"Deny","Action":"s3:DeleteBucket","Resource":"arn:aws:s3:::%s/*"}
]
}`, denyBucket)
denyClient, _ := createUserWithPolicy(ctx, c, suite, denyPolicy)
assertBucketDelete(ctx, c, suite.client, denyClient, denyBucket, false)
// A service account whose parent policy allows everything but whose inline
// policy is object-only exercises the nested AND evaluation path, where the
// boundary has to hold on the inline side.
serviceBucket := getRandomBucketName()
if err := suite.client.MakeBucket(ctx, serviceBucket, minio.MakeBucketOptions{}); err != nil {
c.Fatalf("create service-account bucket: %v", err)
}
parentPolicy := []byte(`{
"Version":"2012-10-17",
"Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]
}`)
_, parentUser := createUserWithPolicy(ctx, c, suite, parentPolicy)
servicePolicy := fmt.Appendf(nil, `{
"Version":"2012-10-17",
"Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"arn:aws:s3:::%s/*"}]
}`, serviceBucket)
serviceAccess, serviceSecret := mustGenerateCredentials(c)
serviceAccount, err := suite.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
TargetUser: parentUser,
AccessKey: serviceAccess,
SecretKey: serviceSecret,
Policy: bytes.Clone(servicePolicy),
})
if err != nil {
c.Fatalf("create restricted service account: %v", err)
}
serviceClient := suite.getUserClient(c, serviceAccount.AccessKey, serviceAccount.SecretKey, "")
assertBucketDelete(ctx, c, suite.client, serviceClient, serviceBucket, false)
}
// The same boundary, asserted against an inline session policy evaluated
// directly, so a regression in the STS and service-account paths is caught even
// if the integration harness above is skipped.
func TestBucketResourceBoundaryInlineSessionPolicy(t *testing.T) {
inline := `{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": "s3:*",
"Resource": "arn:aws:s3:::mybucket/*"
}]
}`
args := policy.Args{
Action: policy.DeleteBucketAction,
BucketName: "mybucket",
Claims: map[string]any{
sessionPolicyNameExtracted: inline,
},
}
for name, evaluate := range map[string]func(policy.Args) (bool, bool){
"STS inline policy": isAllowedBySessionPolicy,
"service-account inline policy": isAllowedBySessionPolicyForServiceAccount,
} {
hasPolicy, allowed := evaluate(args)
if !hasPolicy {
t.Errorf("%s was not detected", name)
continue
}
if allowed {
t.Errorf("%s authorized DeleteBucket through an object-only resource", name)
}
}
}
// The boundary driven through the real S3 router and DeleteBucket handler,
// without opening a TCP listener.
func TestBucketResourceBoundaryHandler(t *testing.T) {
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
endpoints: nil, // Register the full router; the focused list omits DeleteBucket.
objAPITest: func(obj ObjectLayer, instanceType, _ string, apiRouter http.Handler, _ auth.Credentials, t *testing.T) {
ctx := t.Context()
if !globalReplicationPool.IsSet() {
// Match initTestServerWithBackend: DeleteBucket calls through this
// singleton after the object-layer deletion, and a nil receiver is safe.
globalReplicationPool.Set(nil)
}
newPolicyClient := func(policyJSON string) auth.Credentials {
t.Helper()
accessKey, secretKey, err := auth.GenerateCredentials()
if err != nil {
t.Fatal(err)
}
credentials := auth.Credentials{AccessKey: accessKey, SecretKey: secretKey}
if _, err = globalIAMSys.CreateUser(ctx, credentials.AccessKey, madmin.AddOrUpdateUserReq{
SecretKey: credentials.SecretKey,
Status: madmin.AccountEnabled,
}); err != nil {
t.Fatalf("%s: create boundary user: %v", instanceType, err)
}
parsed, err := policy.ParseConfig(strings.NewReader(policyJSON))
if err != nil {
t.Fatalf("%s: parse boundary policy: %v", instanceType, err)
}
policyName := "boundary-" + mustGetUUID()
if _, err = globalIAMSys.SetPolicy(ctx, policyName, *parsed); err != nil {
t.Fatalf("%s: install boundary policy: %v", instanceType, err)
}
if _, err = globalIAMSys.PolicyDBSet(ctx, credentials.AccessKey, policyName, regUser, false); err != nil {
t.Fatalf("%s: attach boundary policy: %v", instanceType, err)
}
return credentials
}
deleteCase := func(label, policyJSON string, wantAllowed bool) {
t.Helper()
bucket := getRandomBucketName()
if err := obj.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
t.Fatalf("%s/%s: create bucket: %v", instanceType, label, err)
}
policyJSON = strings.ReplaceAll(policyJSON, "BUCKET_WILDCARD", bucket[:len(bucket)-1]+"?")
policyJSON = strings.ReplaceAll(policyJSON, "BUCKET", bucket)
credentials := newPolicyClient(policyJSON)
req, err := newTestSignedRequestV4(http.MethodDelete, getDeleteBucketURL("", bucket),
0, nil, credentials.AccessKey, credentials.SecretKey, nil)
if err != nil {
t.Fatalf("%s/%s: sign DeleteBucket: %v", instanceType, label, err)
}
recorder := httptest.NewRecorder()
apiRouter.ServeHTTP(recorder, req)
if wantAllowed {
if recorder.Code != http.StatusNoContent {
t.Fatalf("%s/%s: DeleteBucket status=%d body=%s, want 204",
instanceType, label, recorder.Code, recorder.Body.String())
}
if _, err = obj.GetBucketInfo(ctx, bucket, BucketOptions{}); err == nil {
t.Fatalf("%s/%s: handler returned 204 but bucket still exists", instanceType, label)
}
return
}
if recorder.Code != http.StatusForbidden || !strings.Contains(recorder.Body.String(), "AccessDenied") {
t.Fatalf("%s/%s: DeleteBucket status=%d body=%s, want AccessDenied",
instanceType, label, recorder.Code, recorder.Body.String())
}
if _, err = obj.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
t.Fatalf("%s/%s: denied bucket disappeared: %v", instanceType, label, err)
}
if err = obj.DeleteBucket(ctx, bucket, DeleteBucketOptions{}); err != nil {
t.Fatalf("%s/%s: cleanup bucket: %v", instanceType, label, err)
}
}
deleteCase("object-only s3:*", `{
"Version":"2012-10-17",
"Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"arn:aws:s3:::BUCKET/*"}]
}`, false)
deleteCase("paired resource", `{
"Version":"2012-10-17",
"Statement":[{"Effect":"Allow","Action":"s3:*","Resource":[
"arn:aws:s3:::BUCKET","arn:aws:s3:::BUCKET/*"
]}]
}`, true)
deleteCase("fixed-width bucket wildcard", `{
"Version":"2012-10-17",
"Statement":[{"Effect":"Allow","Action":"s3:DeleteBucket","Resource":"arn:aws:s3:::BUCKET_WILDCARD"}]
}`, false)
deleteCase("NotResource exclusion", `{
"Version":"2012-10-17",
"Statement":[{"Effect":"Allow","Action":"s3:*","NotResource":"arn:aws:s3:::BUCKET/*"}]
}`, false)
deleteCase("Deny coverage", `{
"Version":"2012-10-17",
"Statement":[
{"Effect":"Allow","Action":"s3:*","Resource":"*"},
{"Effect":"Deny","Action":"s3:DeleteBucket","Resource":"arn:aws:s3:::BUCKET/*"}
]
}`, false)
},
})
}
-11
View File
@@ -54,7 +54,6 @@ import (
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/color"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/handlers"
"github.com/minio/minio/internal/kms"
"github.com/minio/minio/internal/logger"
"github.com/minio/pkg/v3/certs"
@@ -712,16 +711,6 @@ func serverHandleEarlyEnvVars() {
func serverHandleEnvVars() {
var err error
// Re-read the source-address trust policy now that loadEnvVarsFromFiles has
// run: a policy taken at package initialisation would miss every deployment
// configured through MINIO_CONFIG_ENV_FILE. Refuse to start on a malformed
// allow-list rather than resolve aws:SourceIp and every audit client address
// by a rule the operator did not write.
if err := handlers.ConfigureSourceIPTrust(); err != nil {
logger.Fatal(err, "Invalid %s value in environment variable", handlers.EnvTrustedProxies)
}
if globalBrowserEnabled {
if redirectURL := env.Get(config.EnvBrowserRedirectURL, ""); redirectURL != "" {
u, err := xnet.ParseHTTPURL(redirectURL)
+2 -3
View File
@@ -18,9 +18,8 @@ const _decomMetric_name = "DecommissionBucketDecommissionObjectDecommissionRemov
var _decomMetric_index = [...]uint8{0, 18, 36, 60}
func (i decomMetric) String() string {
idx := int(i) - 0
if i < 0 || idx >= len(_decomMetric_index)-1 {
if i >= decomMetric(len(_decomMetric_index)-1) {
return "decomMetric(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _decomMetric_name[_decomMetric_index[idx]:_decomMetric_index[idx+1]]
return _decomMetric_name[_decomMetric_index[i]:_decomMetric_index[i+1]]
}
-18
View File
@@ -45,17 +45,6 @@ func NewErasure(ctx context.Context, dataBlocks, parityBlocks int, blockSize int
return e, reedsolomon.ErrInvShardNum
}
// blockSize reaches here from FileInfo.Erasure, i.e. from xl.meta, which a
// peer can write, and FileInfo.IsValid() does not check it. A coder with a
// non-positive block size cannot encode or decode anything, so refusing to
// build one loses nothing -- but building one leaves every division by
// e.blockSize downstream (ShardFileSize, ShardFileOffset, and the whole of
// erasure-decode.go) as an integer divide-by-zero. Reject at the single
// point where every Erasure value in the process is constructed.
if blockSize <= 0 {
return e, errInvalidArgument
}
if dataBlocks+parityBlocks > 256 {
return e, reedsolomon.ErrMaxShardNum
}
@@ -136,13 +125,6 @@ func (e *Erasure) ShardFileSize(totalLength int64) int64 {
if totalLength == -1 {
return -1
}
// NewErasure validates dataBlocks and parityBlocks but not blockSize, and
// the values reach it from FileInfo.Erasure - i.e. from xl.meta, which a
// peer can write. Mirrors the guard on ErasureInfo.ShardFileSize; without
// it a zero block size is an integer divide-by-zero here instead.
if e.blockSize <= 0 || e.dataBlocks <= 0 {
return 0
}
numShards := totalLength / e.blockSize
lastBlockSize := totalLength % e.blockSize
lastShardSize := ceilFrac(lastBlockSize, int64(e.dataBlocks))
+3 -2
View File
@@ -69,7 +69,7 @@ func newParallelReader(readers []io.ReaderAt, e Erasure, offset, totalLength int
offset: (offset / e.blockSize) * e.ShardSize(),
shardSize: e.ShardSize(),
shardFileSize: e.ShardFileSize(totalLength),
buf: bufs,
buf: make([][]byte, len(readers)),
readerToBuf: r2b,
stashBuffer: b,
}
@@ -106,7 +106,8 @@ func (p *parallelReader) preferReaders(prefer []bool) {
// Move reader with index i to index next.
// Do this by swapping next and i
p.readers[next], p.readers[i] = p.readers[i], p.readers[next]
p.readerToBuf[next], p.readerToBuf[i] = p.readerToBuf[i], p.readerToBuf[next]
p.readerToBuf[next] = i
p.readerToBuf[i] = next
next++
}
}
-84
View File
@@ -26,92 +26,8 @@ import (
"testing"
"github.com/dustin/go-humanize"
"github.com/minio/minio/internal/bpool"
)
func TestNewParallelReaderUsesPooledStashBuffer(t *testing.T) {
erasure, err := NewErasure(t.Context(), 4, 4, blockSizeV2)
if err != nil {
t.Fatal(err)
}
previousPool := globalBytePoolCap.Load()
pool := bpool.NewBytePoolCap(1, blockSizeV2, blockSizeV2*2)
globalBytePoolCap.Store(pool)
t.Cleanup(func() {
globalBytePoolCap.Store(previousPool)
})
readers := make([]io.ReaderAt, 8)
reader := newParallelReader(readers, erasure, 0, blockSizeV2)
t.Cleanup(reader.Done)
if reader.stashBuffer == nil {
t.Fatal("expected a pooled stash buffer")
}
shardSize := int(erasure.ShardSize())
stash := reader.stashBuffer[:cap(reader.stashBuffer)]
for i, buf := range reader.buf {
if len(buf) != shardSize {
t.Fatalf("buffer %d has length %d, want %d", i, len(buf), shardSize)
}
buf[0] = byte(i + 1)
if got := stash[i*shardSize]; got != byte(i+1) {
t.Fatalf("buffer %d does not use the pooled stash buffer", i)
}
}
}
func TestParallelReaderPreferReadersMaintainsBufferMapping(t *testing.T) {
tests := []struct {
name string
prefer []bool
want []int
}{
{
name: "adjacent preferred readers",
prefer: []bool{false, true, true, false},
want: []int{1, 2, 0, 3},
},
{
name: "sparse preferred readers",
prefer: []bool{false, true, false, true},
want: []int{1, 3, 2, 0},
},
{
name: "preferred prefix",
prefer: []bool{true, true, false, false},
want: []int{0, 1, 2, 3},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
original := make([]io.ReaderAt, len(tt.prefer))
readerToBuf := make([]int, len(tt.prefer))
for i := range original {
original[i] = bytes.NewReader([]byte{byte(i)})
readerToBuf[i] = i
}
reader := parallelReader{
orgReaders: original,
readerToBuf: readerToBuf,
}
reader.preferReaders(tt.prefer)
for i, originalIndex := range tt.want {
if reader.readers[i] != original[originalIndex] {
t.Errorf("reader %d maps to the wrong original reader", i)
}
if got := reader.readerToBuf[i]; got != originalIndex {
t.Errorf("reader %d maps to buffer %d, want %d", i, got, originalIndex)
}
}
})
}
}
func (a badDisk) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
return 0, errFaultyDisk
}
-28
View File
@@ -58,14 +58,6 @@ func (e ErasureInfo) ShardFileSize(totalLength int64) int64 {
if totalLength == -1 {
return -1
}
// ErasureInfo can arrive zero-valued from an untrusted internode payload:
// CheckParts and VerifyFile hand a wire-supplied FileInfo straight here.
// A zero BlockSize would panic with an integer divide-by-zero, and
// CheckParts evaluates this inside xioutil.WithDeadline - a bare goroutine
// whose panic no recover() can reach, taking the whole process down.
if e.BlockSize <= 0 || e.DataBlocks <= 0 {
return 0
}
numShards := totalLength / e.BlockSize
lastBlockSize := totalLength % e.BlockSize
lastShardSize := ceilFrac(lastBlockSize, int64(e.DataBlocks))
@@ -77,26 +69,6 @@ func (e ErasureInfo) ShardSize() int64 {
return ceilFrac(e.BlockSize, int64(e.DataBlocks))
}
// HasNegativePartSize reports whether any part claims a negative size.
//
// Such metadata is never legitimate, and it is not merely cosmetic: a negative
// length floors both terms of ShardFileSize to zero, and checkPart's only
// integrity test is "st.Size() < expectedSize". A zero expectation is therefore
// satisfied by every file that exists, including a truncated shard, so the part
// is reported intact and a heal driven by the result skips the repair it should
// have performed.
//
// Note this holds even when the erasure parameters are entirely valid, so
// FileInfo.IsValid() - the check healing itself trusts - does not catch it.
func (fi FileInfo) HasNegativePartSize() bool {
for _, p := range fi.Parts {
if p.Size < 0 {
return true
}
}
return false
}
// IsValid - tells if erasure info fields are valid.
func (fi FileInfo) IsValid() bool {
if fi.Deleted {
-431
View File
@@ -1,431 +0,0 @@
// Copyright (c) 2015-2026 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"encoding/xml"
"fmt"
"net/http"
"net/http/httptest"
"strconv"
"testing"
"github.com/dustin/go-humanize"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/hash"
xhttp "github.com/minio/minio/internal/http"
)
// multipartChecksumTestData is the payload used by the full object checksum
// tests: one 5 MiB part (the minimum allowed non-final part size) and a small
// trailing part, so part merging is actually exercised.
func multipartChecksumTestData() (parts [][]byte, full []byte) {
parts = [][]byte{
bytes.Repeat([]byte("a"), 5*humanize.MiByte),
bytes.Repeat([]byte("b"), 1*humanize.KiByte),
}
for _, p := range parts {
full = append(full, p...)
}
return parts, full
}
func mustChecksum(t *testing.T, typ hash.ChecksumType, data []byte) string {
t.Helper()
cs := hash.NewChecksumFromData(typ, data)
if cs == nil {
t.Fatalf("unable to compute %s checksum", typ.String())
}
return cs.Encoded
}
// newMultipartUploadHTTP starts a multipart upload over the API router and
// returns the upload ID.
func newMultipartUploadHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
bucket, object, algo, checksumType string,
) string {
t.Helper()
hdrs := map[string]string{xhttp.AmzChecksumAlgo: algo}
if checksumType != "" {
hdrs[xhttp.AmzChecksumType] = checksumType
}
req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucket, object),
0, nil, creds.AccessKey, creds.SecretKey, hdrs)
if err != nil {
t.Fatalf("failed to build NewMultipartUpload request: %v", err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("NewMultipartUpload failed: %d %s", rec.Code, rec.Body.String())
}
var res InitiateMultipartUploadResponse
if err := xml.Unmarshal(rec.Body.Bytes(), &res); err != nil {
t.Fatalf("failed to decode NewMultipartUpload response: %v", err)
}
return res.UploadID
}
// uploadPartsHTTP uploads every part carrying its own checksum header, the way
// modern AWS SDKs do by default, and returns the part ETags.
func uploadPartsHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
bucket, object, uploadID string, typ hash.ChecksumType, parts [][]byte,
) []string {
t.Helper()
etags := make([]string, len(parts))
for i, p := range parts {
req, err := newTestSignedRequestV4(http.MethodPut,
getPutObjectPartURL("", bucket, object, uploadID, strconv.Itoa(i+1)),
int64(len(p)), bytes.NewReader(p), creds.AccessKey, creds.SecretKey,
map[string]string{typ.Key(): mustChecksum(t, typ, p)})
if err != nil {
t.Fatalf("failed to build UploadPart %d request: %v", i+1, err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("UploadPart %d failed: %d %s", i+1, rec.Code, rec.Body.String())
}
// MinIO writes the header under a literal "ETag" map key, which
// http.Header.Get would canonicalize to "Etag" and miss.
etags[i] = rec.Header()[xhttp.ETag][0]
}
return etags
}
// completeMultipartUploadHTTP completes the upload. partCS supplies an optional
// per-part checksum for each part; an empty string omits it, which is exactly
// what boto3, aws-sdk-js and the Java SDK send when the caller does not track
// part checksums.
func completeMultipartUploadHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
bucket, object, uploadID string, etags []string, partCS []string, hdrs map[string]string,
) *httptest.ResponseRecorder {
t.Helper()
var body bytes.Buffer
body.WriteString("<CompleteMultipartUpload>")
for i, etag := range etags {
fmt.Fprintf(&body, "<Part><PartNumber>%d</PartNumber><ETag>%s</ETag>", i+1, etag)
if i < len(partCS) && partCS[i] != "" {
fmt.Fprintf(&body, "<ChecksumCRC32>%s</ChecksumCRC32>", partCS[i])
}
body.WriteString("</Part>")
}
body.WriteString("</CompleteMultipartUpload>")
req, err := newTestSignedRequestV4(http.MethodPost,
getCompleteMultipartUploadURL("", bucket, object, uploadID),
int64(body.Len()), bytes.NewReader(body.Bytes()), creds.AccessKey, creds.SecretKey, hdrs)
if err != nil {
t.Fatalf("failed to build CompleteMultipartUpload request: %v", err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
return rec
}
func apiErrorCode(t *testing.T, rec *httptest.ResponseRecorder) string {
t.Helper()
var e APIErrorResponse
if err := xml.Unmarshal(rec.Body.Bytes(), &e); err != nil {
t.Fatalf("unable to decode error response %q: %v", rec.Body.String(), err)
}
return e.Code
}
// TestAPICompleteMultipartFullObjectChecksum covers pgsty/minio#31.
//
// A multipart upload created with a full object checksum type must be
// completable by sending only PartNumber and ETag per part, plus the object
// level checksum in the request headers. That is what AWS S3 accepts, and it is
// the point of FULL_OBJECT: the client no longer has to retain per-part
// checksums, only the part numbers and ETags it already tracks.
func TestAPICompleteMultipartFullObjectChecksum(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPICompleteMultipartFullObjectChecksum,
endpoints: []string{"NewMultipart", "PutObjectPart", "CompleteMultipart"},
})
}
func testAPICompleteMultipartFullObjectChecksum(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
credentials auth.Credentials, t *testing.T,
) {
partData, full := multipartChecksumTestData()
// Only CRC based algorithms can linearize into a full object checksum.
// Which one a client picks by default is SDK specific - the AWS CLI v2
// defaults to CRC64NVME while the Go and JavaScript SDKs default to CRC32 -
// so cover all three.
for _, typ := range []hash.ChecksumType{hash.ChecksumCRC32, hash.ChecksumCRC32C, hash.ChecksumCRC64NVME} {
objectName := "uploads/full-object-" + typ.String()
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName,
typ.String(), xhttp.AmzChecksumTypeFullObject)
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, typ, partData)
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, etags, nil,
map[string]string{
typ.Key(): mustChecksum(t, typ, full),
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
})
if rec.Code != http.StatusOK {
t.Fatalf("%s/%s: CompleteMultipartUpload failed: %d %s",
instanceType, typ.String(), rec.Code, rec.Body.String())
}
oi, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{})
if err != nil {
t.Fatalf("%s/%s: GetObjectInfo failed: %v", instanceType, typ.String(), err)
}
if oi.Size != int64(len(full)) {
t.Fatalf("%s/%s: expected object size %d, got %d", instanceType, typ.String(), len(full), oi.Size)
}
// The persisted checksum must be the merged full object value - not a
// composite "<checksum>-<parts>" value - and must report FULL_OBJECT.
cs, _ := oi.decryptChecksums(0, nil)
if got, want := cs[typ.String()], mustChecksum(t, typ, full); got != want {
t.Fatalf("%s/%s: expected stored checksum %q, got %q", instanceType, typ.String(), want, got)
}
if got := cs[xhttp.AmzChecksumType]; got != xhttp.AmzChecksumTypeFullObject {
t.Fatalf("%s/%s: expected stored checksum type %q, got %q",
instanceType, typ.String(), xhttp.AmzChecksumTypeFullObject, got)
}
}
}
// TestAPICompleteMultipartFullObjectChecksumMismatch asserts that accepting
// completions without part checksums does not weaken integrity: a wrong object
// level checksum is still rejected.
func TestAPICompleteMultipartFullObjectChecksumMismatch(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPICompleteMultipartFullObjectChecksumMismatch,
endpoints: []string{"NewMultipart", "PutObjectPart", "CompleteMultipart"},
})
}
func testAPICompleteMultipartFullObjectChecksumMismatch(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
credentials auth.Credentials, t *testing.T,
) {
partData, _ := multipartChecksumTestData()
objectName := "uploads/full-object-mismatch"
typ := hash.ChecksumCRC32
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName,
typ.String(), xhttp.AmzChecksumTypeFullObject)
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, typ, partData)
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, etags, nil,
map[string]string{
typ.Key(): mustChecksum(t, typ, []byte("not the object content")),
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
})
if rec.Code != http.StatusBadRequest {
t.Fatalf("%s: CompleteMultipartUpload with a bad full object checksum returned %d, want 400",
instanceType, rec.Code)
}
// NOTE: AWS S3 documents BadDigest for a full object checksum mismatch on
// CompleteMultipartUpload. MinIO reports XAmzContentChecksumMismatch. That
// deviation is tracked separately; assert the current code so a future
// change to it is a deliberate one.
if got := apiErrorCode(t, rec); got != "XAmzContentChecksumMismatch" {
t.Fatalf("%s: expected XAmzContentChecksumMismatch, got %q", instanceType, got)
}
if _, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err == nil {
t.Fatalf("%s: object was created despite a failed checksum validation", instanceType)
}
}
// TestAPICompleteMultipartCompositeStillRequiresPartChecksums locks in that the
// relaxation is scoped to full object checksums. For the algorithm/type pairs
// AWS actually supports as composite, AWS requires a checksum for every part in
// the CompleteMultipartUpload body, and so do we. (CRC64NVME is deliberately not
// covered: AWS does not support it as composite and MinIO canonicalises it to a
// full object checksum at initiation.)
func TestAPICompleteMultipartCompositeStillRequiresPartChecksums(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPICompleteMultipartCompositeStillRequiresPartChecksums,
endpoints: []string{"NewMultipart", "PutObjectPart", "CompleteMultipart"},
})
}
func testAPICompleteMultipartCompositeStillRequiresPartChecksums(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
credentials auth.Credentials, t *testing.T,
) {
partData, _ := multipartChecksumTestData()
for _, typ := range []hash.ChecksumType{hash.ChecksumCRC32, hash.ChecksumSHA256} {
objectName := "uploads/composite-" + typ.String()
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName,
typ.String(), xhttp.AmzChecksumTypeComposite)
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, typ, partData)
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, etags, nil, nil)
if rec.Code != http.StatusBadRequest {
t.Fatalf("%s/%s: composite CompleteMultipartUpload without part checksums returned %d, want 400",
instanceType, typ.String(), rec.Code)
}
if got := apiErrorCode(t, rec); got != "InvalidPart" {
t.Fatalf("%s/%s: expected InvalidPart, got %q", instanceType, typ.String(), got)
}
if _, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err == nil {
t.Fatalf("%s/%s: object was created despite a rejected completion", instanceType, typ.String())
}
}
}
// TestAPICompleteMultipartFullObjectVariants pins down the surrounding
// behavior of the relaxation: what may be omitted, what must still match, and
// that a zero length object is handled like any other.
func TestAPICompleteMultipartFullObjectVariants(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPICompleteMultipartFullObjectVariants,
endpoints: []string{"NewMultipart", "PutObjectPart", "CompleteMultipart"},
})
}
func testAPICompleteMultipartFullObjectVariants(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
credentials auth.Credentials, t *testing.T,
) {
typ := hash.ChecksumCRC32
partData, full := multipartChecksumTestData()
goodCS := []string{mustChecksum(t, typ, partData[0]), mustChecksum(t, typ, partData[1])}
setup := func(name string) (string, []string) {
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, name,
typ.String(), xhttp.AmzChecksumTypeFullObject)
return uploadID, uploadPartsHTTP(t, apiRouter, credentials, bucketName, name, uploadID, typ, partData)
}
objCSHdr := map[string]string{
typ.Key(): mustChecksum(t, typ, full),
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
}
t.Run("mixed-present-and-omitted", func(t *testing.T) {
name := "variants/mixed"
uploadID, etags := setup(name)
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, name, uploadID, etags,
[]string{goodCS[0], ""}, objCSHdr)
if rec.Code != http.StatusOK {
t.Fatalf("%s: want 200, got %d %s", instanceType, rec.Code, rec.Body.String())
}
})
t.Run("supplied-part-checksum-must-match", func(t *testing.T) {
name := "variants/wrong-part-cs"
uploadID, etags := setup(name)
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, name, uploadID, etags,
[]string{mustChecksum(t, typ, []byte("wrong")), ""}, objCSHdr)
if rec.Code != http.StatusBadRequest {
t.Fatalf("%s: a non-empty but wrong part checksum must be rejected, got %d", instanceType, rec.Code)
}
if got := apiErrorCode(t, rec); got != "InvalidPart" {
t.Fatalf("%s: expected InvalidPart, got %q", instanceType, got)
}
})
t.Run("wrong-algorithm-part-checksum-is-rejected", func(t *testing.T) {
// A part carrying a checksum under an algorithm other than the upload's
// is malformed, not "omitted", and must not slip through the relaxation.
name := "variants/wrong-algo-part-cs"
uploadID, etags := setup(name)
var body bytes.Buffer
body.WriteString("<CompleteMultipartUpload>")
for i, etag := range etags {
fmt.Fprintf(&body, "<Part><PartNumber>%d</PartNumber><ETag>%s</ETag>"+
"<ChecksumCRC32C>AAAAAA==</ChecksumCRC32C></Part>", i+1, etag)
}
body.WriteString("</CompleteMultipartUpload>")
req, err := newTestSignedRequestV4(http.MethodPost,
getCompleteMultipartUploadURL("", bucketName, name, uploadID),
int64(body.Len()), bytes.NewReader(body.Bytes()), credentials.AccessKey, credentials.SecretKey, objCSHdr)
if err != nil {
t.Fatalf("failed to build CompleteMultipartUpload request: %v", err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusBadRequest {
t.Fatalf("%s: a part checksum under the wrong algorithm must be rejected, got %d",
instanceType, rec.Code)
}
if got := apiErrorCode(t, rec); got != "InvalidPart" {
t.Fatalf("%s: expected InvalidPart, got %q", instanceType, got)
}
})
t.Run("no-object-checksum-supplied", func(t *testing.T) {
// AWS treats the object level checksum on completion as optional; the
// server stores the checksum it computed from the parts.
name := "variants/no-object-cs"
uploadID, etags := setup(name)
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, name, uploadID, etags, nil, nil)
if rec.Code != http.StatusOK {
t.Fatalf("%s: want 200, got %d %s", instanceType, rec.Code, rec.Body.String())
}
oi, err := obj.GetObjectInfo(t.Context(), bucketName, name, ObjectOptions{})
if err != nil {
t.Fatalf("%s: GetObjectInfo failed: %v", instanceType, err)
}
cs, _ := oi.decryptChecksums(0, nil)
if got, want := cs[typ.String()], mustChecksum(t, typ, full); got != want {
t.Fatalf("%s: expected server computed checksum %q, got %q", instanceType, want, got)
}
})
t.Run("zero-length-object", func(t *testing.T) {
// A single empty part is a legal multipart upload: only non-final parts
// have a minimum size. The merged checksum must be the checksum of no
// bytes, not the empty string.
name := "variants/zero-length"
empty := []byte{}
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, name,
typ.String(), xhttp.AmzChecksumTypeFullObject)
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, name, uploadID, typ, [][]byte{empty})
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, name, uploadID, etags, nil,
map[string]string{
typ.Key(): mustChecksum(t, typ, empty),
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
})
if rec.Code != http.StatusOK {
t.Fatalf("%s: want 200, got %d %s", instanceType, rec.Code, rec.Body.String())
}
oi, err := obj.GetObjectInfo(t.Context(), bucketName, name, ObjectOptions{})
if err != nil {
t.Fatalf("%s: GetObjectInfo failed: %v", instanceType, err)
}
if oi.Size != 0 {
t.Fatalf("%s: expected zero length object, got %d", instanceType, oi.Size)
}
cs, _ := oi.decryptChecksums(0, nil)
if got, want := cs[typ.String()], mustChecksum(t, typ, empty); got != want {
t.Fatalf("%s: expected stored checksum %q, got %q", instanceType, want, got)
}
})
}
+2 -18
View File
@@ -1290,26 +1290,10 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
hash.ChecksumSHA256.String(): part.ChecksumSHA256,
hash.ChecksumCRC64NVME.String(): part.ChecksumCRC64NVME,
}
gotCS := wantCS[checksumType.String()]
var suppliedAnyCS bool
for _, v := range wantCS {
if v != "" {
suppliedAnyCS = true
break
}
}
// Part checksums are optional in the CompleteMultipartUpload body when
// the upload was created with a full object checksum type: clients send
// the object level checksum instead and do not retain part checksums.
// A part that carries any checksum at all is still validated against
// what we stored - including one sent under the wrong algorithm, which
// cannot match and is rejected. The object level checksum, if supplied,
// is verified against the merged part checksums below.
allowMissingPartCS := checksumType.FullObjectRequested() && !suppliedAnyCS
if !allowMissingPartCS && gotCS != crc {
if wantCS[checksumType.String()] != crc {
return oi, InvalidPart{
PartNumber: part.PartNumber,
ExpETag: gotCS,
ExpETag: wantCS[checksumType.String()],
GotETag: crc,
}
}
+1 -1
View File
@@ -469,7 +469,7 @@ func auditDanglingObjectDeletion(ctx context.Context, bucket, object, versionID
func joinErrs(errs []error) string {
var s string
for i := range errs {
for i := range s {
if s != "" {
s += ","
}
-21
View File
@@ -1281,24 +1281,3 @@ func TestGetObjectWithOutdatedDisks(t *testing.T) {
}
}
}
func TestJoinErrs(t *testing.T) {
errA := errors.New("disk not found")
errB := errors.New("file corrupt")
testCases := []struct {
errs []error
expected string
}{
{nil, ""},
{[]error{}, ""},
{[]error{nil}, "<nil>"},
{[]error{errA}, "disk not found"},
{[]error{nil, errA}, "<nil>,disk not found"},
{[]error{errA, nil, errB, nil}, "disk not found,<nil>,file corrupt,<nil>"},
}
for i, testCase := range testCases {
if got := joinErrs(testCase.errs); got != testCase.expected {
t.Errorf("Test %d: expected %q, got %q", i+1, testCase.expected, got)
}
}
}
+1 -5
View File
@@ -69,11 +69,7 @@ func (rs *rebalanceStats) update(bucket string, fi FileInfo) {
rs.NumVersions++
onDiskSz := int64(0)
// DataBlocks comes from xl.meta and only fi.Deleted is checked above, so a
// zero here is an integer divide-by-zero rather than a bad statistic. This
// path does not build an Erasure, so NewErasure's validation does not cover
// it; leave the size at zero for metadata that cannot describe a layout.
if !fi.Deleted && fi.Erasure.DataBlocks > 0 {
if !fi.Deleted {
onDiskSz = fi.Size * int64(fi.Erasure.DataBlocks+fi.Erasure.ParityBlocks) / int64(fi.Erasure.DataBlocks)
}
rs.Bytes += uint64(onDiskSz)
+1 -1
View File
@@ -177,7 +177,7 @@ func formatGetBackendErasureVersion(b []byte) (string, error) {
return "", fmt.Errorf(`format.Version expected: %s, got: %s`, formatMetaVersionV1, meta.Version)
}
if meta.Format != formatBackendErasure && meta.Format != formatBackendErasureSingle {
return "", fmt.Errorf(`found backend type %s, expected %s or %s - to migrate to a supported backend visit https://silo.pgsty.com/operations/deployments/baremetal-migrate-fs-gateway/`, meta.Format, formatBackendErasure, formatBackendErasureSingle)
return "", fmt.Errorf(`found backend type %s, expected %s or %s - to migrate to a supported backend visit https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-migrate-fs-gateway.html`, meta.Format, formatBackendErasure, formatBackendErasureSingle)
}
// Erasure backend found, proceed to detect version.
format := &formatErasureVersionDetect{}
+2 -3
View File
@@ -21,9 +21,8 @@ const _format_name = "UnknownGzipZstdLZ4S2BZ2"
var _format_index = [...]uint8{0, 7, 11, 15, 18, 20, 23}
func (i format) String() string {
idx := int(i) - 0
if i < 0 || idx >= len(_format_index)-1 {
if i < 0 || i >= format(len(_format_index)-1) {
return "format(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _format_name[_format_index[idx]:_format_index[idx+1]]
return _format_name[_format_index[i]:_format_index[i+1]]
}
+1 -69
View File
@@ -142,62 +142,7 @@ var userMetadataKeyPrefixes = []string{
// extractMetadataFromReq extracts metadata from HTTP header and HTTP queryString.
func extractMetadataFromReq(ctx context.Context, r *http.Request) (metadata map[string]string, err error) {
metadata, err = extractMetadata(ctx, textproto.MIMEHeader(r.Form), textproto.MIMEHeader(r.Header))
if err != nil {
return nil, err
}
// Keep the metadata consumed by object operations in lock-step with policy
// conditions: an explicitly present header wins, otherwise use the query
// value accepted by the existing S3-compatible request path.
for _, name := range []string{xhttp.AmzStorageClass, xhttp.AmzObjectTagging} {
if value, ok := getRequestHeaderOrQueryValue(r, name); ok {
metadata[name] = value
}
}
return metadata, nil
}
// getRequestHeaderOrQueryValue returns the effective value of a request field.
// Header presence takes precedence even when its value is empty. Query lookup
// remains case-insensitive for compatibility with extractMetadataFromReq.
func getRequestHeaderOrQueryValue(r *http.Request, name string) (string, bool) {
if values, ok := getRequestValues(r.Header, name, http.CanonicalHeaderKey(name)); ok {
return strings.Join(values, ","), true
}
if values, ok := getRequestValues(http.Header(r.Form), name, strings.ToLower(name)); ok {
return strings.Join(values, ","), true
}
return "", false
}
func getRequestValues(values http.Header, name, preferred string) ([]string, bool) {
if value, ok := values[preferred]; ok {
return value, true
}
canonical, lower := http.CanonicalHeaderKey(name), strings.ToLower(name)
for _, key := range []string{canonical, lower} {
if key == preferred {
continue
}
if value, ok := values[key]; ok {
return value, true
}
}
// Multiple differently-cased spellings are malformed but were previously
// accepted. Pick one deterministically instead of depending on map order.
match := ""
for key := range values {
if strings.EqualFold(key, name) && (match == "" || key < match) {
match = key
}
}
if match != "" {
return values[match], true
}
return nil, false
return extractMetadata(ctx, textproto.MIMEHeader(r.Form), textproto.MIMEHeader(r.Header))
}
func extractMetadata(ctx context.Context, mimesHeader ...textproto.MIMEHeader) (metadata map[string]string, err error) {
@@ -247,16 +192,6 @@ func extractMetadata(ctx context.Context, mimesHeader ...textproto.MIMEHeader) (
// extractMetadata extracts metadata from map values.
func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error {
return extractMetadataFromMimeWithReplication(ctx, v, m, false)
}
// extractReplicationMetadataFromMime restores replication-only metadata after the
// caller has validated that the request is a trusted replication write.
func extractReplicationMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error {
return extractMetadataFromMimeWithReplication(ctx, v, m, true)
}
func extractMetadataFromMimeWithReplication(ctx context.Context, v textproto.MIMEHeader, m map[string]string, allowReplication bool) error {
if v == nil {
bugLogIf(ctx, errInvalidArgument)
return errInvalidArgument
@@ -273,9 +208,6 @@ func extractMetadataFromMimeWithReplication(ctx context.Context, v textproto.MIM
value, ok := nv[http.CanonicalHeaderKey(supportedHeader)]
if ok {
if v, ok := replicationToInternalHeaders[supportedHeader]; ok {
if !allowReplication {
continue
}
m[v] = strings.Join(value, ",")
} else {
m[supportedHeader] = strings.Join(value, ",")
-174
View File
@@ -24,14 +24,11 @@ import (
"io"
"net/http"
"net/textproto"
"net/url"
"os"
"reflect"
"strings"
"testing"
"github.com/minio/minio/internal/config"
xhttp "github.com/minio/minio/internal/http"
)
// Tests validate bucket LocationConstraint.
@@ -155,22 +152,6 @@ func TestExtractMetadataHeaders(t *testing.T) {
},
shouldFail: false,
},
// Replication-only headers must not be accepted on ordinary requests.
{
header: http.Header{
"Content-Type": []string{"image/png"},
"X-Minio-Replication-Server-Side-Encryption-Sealed-Key": []string{"sealed-key"},
"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm": []string{"DAREv2-HMAC-SHA256"},
"X-Minio-Replication-Server-Side-Encryption-Iv": []string{"iv"},
"X-Minio-Replication-Encrypted-Multipart": []string{""},
"X-Minio-Replication-Actual-Object-Size": []string{"1"},
ReplicationSsecChecksumHeader: []string{"checksum"},
},
metadata: map[string]string{
"content-type": "image/png",
},
shouldFail: false,
},
// Empty header input returns empty metadata.
{
header: nil,
@@ -195,161 +176,6 @@ func TestExtractMetadataHeaders(t *testing.T) {
}
}
func TestExtractMetadataFromRequestUsesHeaderPrecedence(t *testing.T) {
query := make(url.Values)
query.Set(strings.ToLower(xhttp.AmzStorageClass), "QUERY-CLASS")
query.Set(strings.ToLower(xhttp.AmzObjectTagging), "source=query")
req, err := http.NewRequest(http.MethodGet, "http://localhost/test?"+query.Encode(), nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set(xhttp.AmzStorageClass, "HEADER-CLASS")
req.Header.Set(xhttp.AmzObjectTagging, "source=header")
if err = req.ParseForm(); err != nil {
t.Fatal(err)
}
metadata, err := extractMetadataFromReq(t.Context(), req)
if err != nil {
t.Fatal(err)
}
if got := metadata[xhttp.AmzStorageClass]; got != "HEADER-CLASS" {
t.Fatalf("storage class: expected header, got %q", got)
}
if got := metadata[xhttp.AmzObjectTagging]; got != "source=header" {
t.Fatalf("tagging: expected header, got %q", got)
}
// Presence, rather than a non-empty value, establishes precedence. This
// prevents a query value from taking over when a signed header is empty.
req.Header[xhttp.AmzObjectTagging] = []string{""}
if got, ok := getRequestHeaderOrQueryValue(req, xhttp.AmzObjectTagging); !ok || got != "" {
t.Fatalf("empty header did not override query: value=%q present=%v", got, ok)
}
}
func TestExtractMetadataFromRequestKeepsQueryCompatibility(t *testing.T) {
query := make(url.Values)
query.Set(strings.ToLower(xhttp.AmzStorageClass), "REDUCED_REDUNDANCY")
query.Set(strings.ToLower(xhttp.AmzObjectTagging), "security=public")
req, err := http.NewRequest(http.MethodGet, "http://localhost/test?"+query.Encode(), nil)
if err != nil {
t.Fatal(err)
}
if err = req.ParseForm(); err != nil {
t.Fatal(err)
}
metadata, err := extractMetadataFromReq(t.Context(), req)
if err != nil {
t.Fatal(err)
}
if got := metadata[xhttp.AmzStorageClass]; got != "REDUCED_REDUNDANCY" {
t.Fatalf("storage class query value lost: %q", got)
}
if got := metadata[xhttp.AmzObjectTagging]; got != "security=public" {
t.Fatalf("tagging query value lost: %q", got)
}
}
func TestExtractReplicationMetadataHeaders(t *testing.T) {
header := http.Header{
"X-Minio-Replication-Server-Side-Encryption-Sealed-Key": []string{"sealed-key"},
"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm": []string{"DAREv2-HMAC-SHA256"},
"X-Minio-Replication-Server-Side-Encryption-Iv": []string{"iv"},
"X-Minio-Replication-Encrypted-Multipart": []string{""},
"X-Minio-Replication-Actual-Object-Size": []string{"1"},
ReplicationSsecChecksumHeader: []string{"checksum"},
}
metadata := make(map[string]string)
if err := extractReplicationMetadataFromMime(t.Context(), textproto.MIMEHeader(header), metadata); err != nil {
t.Fatalf("failed to extract replication metadata: %v", err)
}
expected := map[string]string{
"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "sealed-key",
"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
"X-Minio-Internal-Server-Side-Encryption-Iv": "iv",
"X-Minio-Internal-Encrypted-Multipart": "",
"X-Minio-Internal-Actual-Object-Size": "1",
ReplicationSsecChecksumHeader: "checksum",
}
if !reflect.DeepEqual(metadata, expected) {
t.Fatalf("unexpected replication metadata: expected %#v, got %#v", expected, metadata)
}
}
func TestGetCopyObjectMetadataFromHeaderReplication(t *testing.T) {
req, err := http.NewRequest(http.MethodPut, "http://localhost/test", nil)
if err != nil {
t.Fatal(err)
}
req.Form = make(url.Values)
req.Header.Set("X-Amz-Metadata-Directive", replaceDirective)
req.Header.Set("X-Minio-Replication-Server-Side-Encryption-Sealed-Key", "sealed-key")
metadata, err := getCpObjMetadataFromHeader(t.Context(), req, nil, false)
if err != nil {
t.Fatalf("copy metadata extraction failed: %v", err)
}
if _, ok := metadata["X-Minio-Internal-Server-Side-Encryption-Sealed-Key"]; ok {
t.Fatalf("unexpected replication metadata without validation: %#v", metadata)
}
metadata, err = getCpObjMetadataFromHeader(t.Context(), req, nil, true)
if err != nil {
t.Fatalf("copy metadata extraction with replication failed: %v", err)
}
if got := metadata["X-Minio-Internal-Server-Side-Encryption-Sealed-Key"]; got != "sealed-key" {
t.Fatalf("expected restored replication metadata, got %#v", metadata)
}
}
func TestCloneRequestWithoutCopyReplicationHeaders(t *testing.T) {
req, err := http.NewRequest(http.MethodPut, "http://localhost/test", nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set(xhttp.MinIOSourceReplicationRequest, "true")
req.Header.Set(xhttp.MinIOSourceETag, "etag")
req.Header.Set(xhttp.MinIOSourceMTime, "2026-04-15T10:00:00Z")
req.Header.Set(xhttp.MinIOSourceTaggingTimestamp, "2026-04-15T10:00:00Z")
req.Header.Set(xhttp.MinIOSourceObjectRetentionTimestamp, "2026-04-15T10:00:00Z")
req.Header.Set(xhttp.MinIOSourceObjectLegalHoldTimestamp, "2026-04-15T10:00:00Z")
req.Header.Set(xhttp.MinIOReplicationActualObjectSize, "123")
req.Header.Set(ReplicationSsecChecksumHeader, "checksum")
req.Header.Set("Content-Type", "application/octet-stream")
clone := cloneRequestWithoutCopyReplicationHeaders(req)
if clone == req {
t.Fatal("expected cloned request")
}
for _, header := range []string{
xhttp.MinIOSourceReplicationRequest,
xhttp.MinIOSourceETag,
xhttp.MinIOSourceMTime,
xhttp.MinIOSourceTaggingTimestamp,
xhttp.MinIOSourceObjectRetentionTimestamp,
xhttp.MinIOSourceObjectLegalHoldTimestamp,
xhttp.MinIOReplicationActualObjectSize,
ReplicationSsecChecksumHeader,
} {
if got := clone.Header.Get(header); got != "" {
t.Fatalf("expected %s to be stripped, got %q", header, got)
}
if got := req.Header.Get(header); got == "" {
t.Fatalf("expected original request to preserve %s", header)
}
}
if got := clone.Header.Get("Content-Type"); got != "application/octet-stream" {
t.Fatalf("expected non-replication headers to be preserved, got %q", got)
}
}
// Test getResource()
func TestGetResource(t *testing.T) {
testCases := []struct {
+2 -3
View File
@@ -18,9 +18,8 @@ const _healingMetric_name = "BucketObjectCheckAbandonedParts"
var _healingMetric_index = [...]uint8{0, 6, 12, 31}
func (i healingMetric) String() string {
idx := int(i) - 0
if i < 0 || idx >= len(_healingMetric_index)-1 {
if i >= healingMetric(len(_healingMetric_index)-1) {
return "healingMetric(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _healingMetric_name[_healingMetric_index[idx]:_healingMetric_index[idx+1]]
return _healingMetric_name[_healingMetric_index[i]:_healingMetric_index[i+1]]
}
+1 -1
View File
@@ -576,7 +576,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
if took := time.Since(listStartTime); took > maxIAMLoadOpTime {
var s strings.Builder
for k, v := range listedConfigItems {
fmt.Fprintf(&s, " %s: %d items\n", k, len(v))
s.WriteString(fmt.Sprintf(" %s: %d items\n", k, len(v)))
}
logger.Info("listAllIAMConfigItems took %.2fs with contents:\n%s", took.Seconds(), s.String())
}
+2 -3
View File
@@ -26,9 +26,8 @@ const _lcEventSrc_name = "NoneHealScannerDecomRebals3HeadObjects3GetObjects3List
var _lcEventSrc_index = [...]uint8{0, 4, 8, 15, 20, 25, 37, 48, 61, 72, 84, 109}
func (i lcEventSrc) String() string {
idx := int(i) - 0
if i < 0 || idx >= len(_lcEventSrc_index)-1 {
if i >= lcEventSrc(len(_lcEventSrc_index)-1) {
return "lcEventSrc(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _lcEventSrc_name[_lcEventSrc_index[idx]:_lcEventSrc_index[idx+1]]
return _lcEventSrc_name[_lcEventSrc_index[i]:_lcEventSrc_index[i+1]]
}
+8
View File
@@ -324,6 +324,14 @@ func (d *naughtyDisk) StatInfoFile(ctx context.Context, volume, path string, glo
return d.disk.StatInfoFile(ctx, volume, path, glob)
}
func (d *naughtyDisk) ReadMultiple(ctx context.Context, req ReadMultipleReq, resp chan<- ReadMultipleResp) error {
if err := d.calcError(); err != nil {
close(resp)
return err
}
return d.disk.ReadMultiple(ctx, req, resp)
}
func (d *naughtyDisk) CleanAbandonedData(ctx context.Context, volume string, path string) error {
if err := d.calcError(); err != nil {
return err
+60 -117
View File
@@ -387,7 +387,11 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
return true
}
if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectAction, oi.UserTags); s3Error != ErrNone {
if oi.UserTags != "" {
r.Header.Set(xhttp.AmzObjectTagging, oi.UserTags)
}
if s3Error := authorizeRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return true
}
@@ -425,17 +429,15 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
}
}
if reader == nil || !proxy.Proxy {
// The conditional callback has already written 304/412. Do not
// authorize again without the stored tags or write a second response.
if isErrPreconditionFailed(err) {
return
}
// validate if the request indeed was authorized, if it wasn't we need to return "ErrAccessDenied"
// instead of any namespace related error.
if s3Error := authorizeRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
if isErrPreconditionFailed(err) {
return
}
if proxy.Err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, proxy.Err), r.URL)
return
@@ -837,7 +839,12 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
}
}
if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectAction, objInfo.UserTags); s3Error != ErrNone {
if objInfo.UserTags != "" {
// Set this such that authorization policies can be applied on the object tags.
r.Header.Set(xhttp.AmzObjectTagging, objInfo.UserTags)
}
if s3Error := authorizeRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone {
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error))
return
}
@@ -1029,7 +1036,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
// Extract metadata relevant for an CopyObject operation based on conditional
// header values specified in X-Amz-Metadata-Directive.
func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta map[string]string, allowReplication bool) (map[string]string, error) {
func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta map[string]string) (map[string]string, error) {
// Make a copy of the supplied metadata to avoid
// to change the original one.
defaultMeta := make(map[string]string, len(userMeta))
@@ -1048,7 +1055,10 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m
// Storage class is special, it can be replaced regardless of the
// metadata directive, if set should be preserved and replaced
// to the destination metadata.
sc, _ := getRequestHeaderOrQueryValue(r, xhttp.AmzStorageClass)
sc := r.Header.Get(xhttp.AmzStorageClass)
if sc == "" {
sc = r.Form.Get(xhttp.AmzStorageClass)
}
// if x-amz-metadata-directive says REPLACE then
// we extract metadata from the input headers.
@@ -1057,11 +1067,6 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m
if err != nil {
return nil, err
}
if allowReplication {
if err = extractReplicationMetadataFromMime(ctx, textproto.MIMEHeader(r.Header), emetadata); err != nil {
return nil, err
}
}
if sc != "" {
emetadata[xhttp.AmzStorageClass] = sc
}
@@ -1082,31 +1087,6 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m
return defaultMeta, nil
}
func cloneRequestWithoutCopyReplicationHeaders(r *http.Request) *http.Request {
if r == nil {
return nil
}
clone := new(http.Request)
*clone = *r
clone.Header = r.Header.Clone()
for _, header := range []string{
xhttp.MinIOSourceReplicationRequest,
xhttp.MinIOSourceETag,
xhttp.MinIOSourceMTime,
xhttp.MinIOSourceTaggingTimestamp,
xhttp.MinIOSourceObjectRetentionTimestamp,
xhttp.MinIOSourceObjectLegalHoldTimestamp,
xhttp.MinIOReplicationActualObjectSize,
ReplicationSsecChecksumHeader,
} {
clone.Header.Del(header)
}
return clone
}
// getRemoteInstanceTransport contains a roundtripper for external (not peers) servers
var remoteInstanceTransport atomic.Value
@@ -1246,26 +1226,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
return
}
// Validate the storage class header if present. Query values retain the
// existing compatibility path, including its historical validation behavior.
dstSc, _ := getRequestHeaderOrQueryValue(r, xhttp.AmzStorageClass)
if headerStorageClass := r.Header.Get(xhttp.AmzStorageClass); headerStorageClass != "" && !storageclass.IsValid(headerStorageClass) {
// Validate storage class metadata if present
dstSc := r.Header.Get(xhttp.AmzStorageClass)
if dstSc != "" && !storageclass.IsValid(dstSc) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidStorageClass), r.URL)
return
}
allowReplicationMetadata := false
if r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() {
if s3Error := checkRequestAuthType(ctx, r, policy.ReplicateObjectAction, dstBucket, dstObject); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
allowReplicationMetadata = true
}
trustedReplicationRequest := allowReplicationMetadata && r.Header.Get(xhttp.MinIOSourceReplicationRequest) == "true"
optsReq := r
if !trustedReplicationRequest {
optsReq = cloneRequestWithoutCopyReplicationHeaders(r)
}
// Check if bucket encryption is enabled
sseConfig, _ := globalBucketSSEConfigSys.Get(dstBucket)
@@ -1274,7 +1240,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
})
var srcOpts, dstOpts ObjectOptions
srcOpts, err = copySrcOpts(ctx, optsReq, srcBucket, srcObject)
srcOpts, err = copySrcOpts(ctx, r, srcBucket, srcObject)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -1286,14 +1252,14 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
VersionID: srcOpts.VersionID,
Versioned: srcOpts.Versioned,
VersionSuspended: srcOpts.VersionSuspended,
ReplicationRequest: trustedReplicationRequest,
ReplicationRequest: r.Header.Get(xhttp.MinIOSourceReplicationRequest) == "true",
}
getSSE := encrypt.SSE(srcOpts.ServerSideEncryption)
if getSSE != srcOpts.ServerSideEncryption {
getOpts.ServerSideEncryption = getSSE
}
dstOpts, err = copyDstOpts(ctx, optsReq, dstBucket, dstObject, nil)
dstOpts, err = copyDstOpts(ctx, r, dstBucket, dstObject, nil)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -1303,7 +1269,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
getObjectNInfo := objectAPI.GetObjectNInfo
checkCopyPrecondFn := func(o ObjectInfo) bool {
if _, err := DecryptObjectInfo(&o, optsReq); err != nil {
if _, err := DecryptObjectInfo(&o, r); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return true
}
@@ -1414,7 +1380,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
return
}
// Encryption parameters not present for this object.
if crypto.SSEC.IsEncrypted(srcInfo.UserDefined) && !crypto.SSECopy.IsRequested(r.Header) && !trustedReplicationRequest {
if crypto.SSEC.IsEncrypted(srcInfo.UserDefined) && !crypto.SSECopy.IsRequested(r.Header) && r.Header.Get(xhttp.MinIOSourceReplicationRequest) != "true" {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidSSECustomerAlgorithm), r.URL)
return
}
@@ -1580,7 +1546,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
srcInfo.PutObjReader = pReader
srcInfo.UserDefined, err = getCpObjMetadataFromHeader(ctx, r, srcInfo.UserDefined, allowReplicationMetadata)
srcInfo.UserDefined, err = getCpObjMetadataFromHeader(ctx, r, srcInfo.UserDefined)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -1662,10 +1628,10 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
return
}
if allowReplicationMetadata {
if rs := r.Header.Get(xhttp.AmzBucketReplicationStatus); rs != "" {
srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicaStatus] = replication.Replica.String()
srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicaTimestamp] = UTCNow().Format(time.RFC3339Nano)
srcInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replication.Replica.String()
srcInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = rs
}
op := replication.ObjectReplicationType
@@ -1848,8 +1814,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
return
}
// Validate the storage class header if present. Query values retain the
// existing compatibility path, including its historical validation behavior.
// Validate storage class metadata if present
if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" {
if !storageclass.IsValid(sc) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidStorageClass), r.URL)
@@ -1898,11 +1863,13 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
return
}
if objTags := metadata[xhttp.AmzObjectTagging]; objTags != "" {
if objTags := r.Header.Get(xhttp.AmzObjectTagging); objTags != "" {
if _, err := tags.ParseObjectTags(objTags); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
metadata[xhttp.AmzObjectTagging] = objTags
}
var (
@@ -1914,12 +1881,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
)
// Check if put is allowed
requestTags, hasRequestTags := metadata[xhttp.AmzObjectTagging]
var requestTagsPtr *string
if hasRequestTags {
requestTagsPtr = &requestTags
}
if s3Err = isPutActionAllowedWithRequestTags(ctx, rAuthType, bucket, object, r, policy.PutObjectAction, requestTagsPtr); s3Err != ErrNone {
if s3Err = isPutActionAllowed(ctx, rAuthType, bucket, object, r, policy.PutObjectAction); s3Err != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
return
}
@@ -1934,7 +1896,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
}
case authTypeStreamingUnsignedTrailer:
// Initialize stream chunked reader with optional trailers.
rd, s3Err = newUnsignedV4ChunkedReader(r, true)
rd, s3Err = newUnsignedV4ChunkedReader(r, true, r.Header.Get(xhttp.Authorization) != "")
if s3Err != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
return
@@ -1971,10 +1933,6 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
return
}
if err = extractReplicationMetadataFromMime(ctx, textproto.MIMEHeader(r.Header), metadata); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
metadata[ReservedMetadataPrefixLower+ReplicaStatus] = replication.Replica.String()
metadata[ReservedMetadataPrefixLower+ReplicaTimestamp] = UTCNow().Format(time.RFC3339Nano)
defer globalReplicationStats.Load().UpdateReplicaStat(bucket, size)
@@ -2266,12 +2224,13 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
return
}
// Validate the storage class header if present. PutObjectExtract now also
// consumes the compatible query value so policy and operation stay aligned.
sc, _ := getRequestHeaderOrQueryValue(r, xhttp.AmzStorageClass)
if headerStorageClass := r.Header.Get(xhttp.AmzStorageClass); headerStorageClass != "" && !storageclass.IsValid(headerStorageClass) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidStorageClass), r.URL)
return
// Validate storage class metadata if present
sc := r.Header.Get(xhttp.AmzStorageClass)
if sc != "" {
if !storageclass.IsValid(sc) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidStorageClass), r.URL)
return
}
}
clientETag, err := etag.FromContentMD5(r.Header)
@@ -2283,7 +2242,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
// if Content-Length is unknown/missing, deny the request
size := r.ContentLength
rAuthType := getRequestAuthType(r)
if rAuthType == authTypeStreamingSigned || rAuthType == authTypeStreamingSignedTrailer || rAuthType == authTypeStreamingUnsignedTrailer {
if rAuthType == authTypeStreamingSigned || rAuthType == authTypeStreamingSignedTrailer {
if sizeStr, ok := r.Header[xhttp.AmzDecodedContentLength]; ok {
if sizeStr[0] == "" {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
@@ -2337,13 +2296,6 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
return
}
case authTypeStreamingUnsignedTrailer:
// Initialize stream chunked reader with optional trailers.
reader, s3Err = newUnsignedV4ChunkedReader(r, true)
if s3Err != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
return
}
case authTypeSignedV2, authTypePresignedV2:
s3Err = isReqAuthenticatedV2(r)
if s3Err != ErrNone {
@@ -2436,15 +2388,10 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
rawReader := hashReader
pReader := NewPutObjReader(rawReader)
allowReplicationMetadata := false
if r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() {
if s3Err = isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.ReplicateObjectAction); s3Err != ErrNone {
return errors.New(errorCodes.ToAPIErr(s3Err).Code)
}
allowReplicationMetadata = true
if err = extractReplicationMetadataFromMime(ctx, textproto.MIMEHeader(r.Header), metadata); err != nil {
return err
}
metadata[ReservedMetadataPrefixLower+ReplicaStatus] = replication.Replica.String()
metadata[ReservedMetadataPrefixLower+ReplicaTimestamp] = UTCNow().Format(time.RFC3339Nano)
}
@@ -2470,11 +2417,6 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
if err != nil {
return err
}
if allowReplicationMetadata {
if err = extractReplicationMetadataFromMime(ctx, textproto.MIMEHeader(hdrs), m); err != nil {
return err
}
}
maps.Copy(metadata, m)
} else {
versionID = r.Form.Get(xhttp.VersionID)
@@ -3199,7 +3141,12 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
}
}
if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, ot.String()); s3Error != ErrNone {
// Set this such that authorization policies can be applied on the object tags.
if tags := ot.String(); tags != "" {
r.Header.Set(xhttp.AmzObjectTagging, tags)
}
if s3Error := authorizeRequest(ctx, r, policy.GetObjectTaggingAction); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
@@ -3253,14 +3200,12 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
tagsStr := tags.String()
// Set this such that authorization policies can be applied on the object tags.
r.Header.Set(xhttp.AmzObjectTagging, tagsStr)
r.Header.Set(xhttp.AmzObjectTagging, tags.String())
logger.GetReqInfo(ctx).BucketName = bucket
logger.GetReqInfo(ctx).ObjectName = object
if s3Error := authenticateRequest(ctx, r, policy.PutObjectTaggingAction); s3Error != ErrNone {
// Allow putObjectTagging if policy action is set
if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectTaggingAction, bucket, object); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
@@ -3272,14 +3217,6 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
}
objInfo, err := objAPI.GetObjectInfo(ctx, bucket, object, opts)
existingTags := ""
if err == nil {
existingTags = objInfo.UserTags
}
if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.PutObjectTaggingAction, existingTags); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
if err != nil {
// if object is not found locally, but exists on peer site - proxy
// the tagging request to peer site. The response to client will
@@ -3313,6 +3250,7 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
tagsStr := tags.String()
dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(objInfo.UserDefined, tagsStr, objInfo.ReplicationStatus, replication.MetadataReplicationType, opts))
if dsc.ReplicateAny() {
@@ -3411,8 +3349,13 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
return
}
if userTags := oi.UserTags; userTags != "" {
// Set this such that authorization policies can be applied on the object tags.
r.Header.Set(xhttp.AmzObjectTagging, oi.UserTags)
}
// Allow deleteObjectTagging if policy action is set
if s3Error := checkRequestAuthTypeWithExistingTags(ctx, r, policy.DeleteObjectTaggingAction, bucket, object, oi.UserTags); s3Error != ErrNone {
if s3Error := checkRequestAuthType(ctx, r, policy.DeleteObjectTaggingAction, bucket, object); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
-157
View File
@@ -42,7 +42,6 @@ import (
"github.com/dustin/go-humanize"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/crypto"
"github.com/minio/minio/internal/hash/sha256"
xhttp "github.com/minio/minio/internal/http"
ioutilx "github.com/minio/minio/internal/ioutil"
@@ -61,47 +60,6 @@ const (
MissingUploadID
)
func replicationSSEPoisonHeaders() map[string]string {
return map[string]string{
"X-Minio-Replication-Server-Side-Encryption-Sealed-Key": base64.StdEncoding.EncodeToString(make([]byte, 64)),
"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm": crypto.SealAlgorithm,
"X-Minio-Replication-Server-Side-Encryption-Iv": base64.StdEncoding.EncodeToString(make([]byte, 32)),
}
}
func assertObjectMetadataKeysAbsent(t *testing.T, metadata map[string]string, keys ...string) {
t.Helper()
for _, key := range keys {
if got, ok := metadata[key]; ok {
t.Fatalf("expected metadata %q to be absent, got %q", key, got)
}
}
}
func assertObjectMetadataValueNotEqual(t *testing.T, metadata map[string]string, key, unexpected string) {
t.Helper()
if got := metadata[key]; got == unexpected {
t.Fatalf("expected metadata %q to differ from %q", key, unexpected)
}
}
func assertObjectContents(t *testing.T, obj ObjectLayer, bucketName, objectName string, expected []byte) {
t.Helper()
reader, err := obj.GetObjectNInfo(context.Background(), bucketName, objectName, nil, nil, ObjectOptions{})
if err != nil {
t.Fatalf("failed to fetch object %s/%s: %v", bucketName, objectName, err)
}
defer reader.Close()
got, err := io.ReadAll(reader)
if err != nil {
t.Fatalf("failed to read object %s/%s: %v", bucketName, objectName, err)
}
if !bytes.Equal(got, expected) {
t.Fatalf("unexpected object contents: got %d bytes, expected %d bytes", len(got), len(expected))
}
}
// Wrapper for calling HeadObject API handler tests for both Erasure multiple disks and FS single drive setup.
func TestAPIHeadObjectHandler(t *testing.T) {
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPIHeadObjectHandler, endpoints: []string{"HeadObject"}})
@@ -1861,53 +1819,6 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam
}
}
func TestAPIPutObjectReplicationHeaderPoisoning(t *testing.T) {
defer DetectTestLeak(t)()
ExecExtendedObjectLayerAPITest(t, testAPIPutObjectReplicationHeaderPoisoning, []string{"PutObject"})
}
func testAPIPutObjectReplicationHeaderPoisoning(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
credentials auth.Credentials, t *testing.T,
) {
objectName := "replication-header-poison-put"
payload := []byte("replication-header-poison-put-payload")
headers := replicationSSEPoisonHeaders()
req, err := newTestSignedRequestV4(
http.MethodPut,
getPutObjectURL("", bucketName, objectName),
int64(len(payload)),
bytes.NewReader(payload),
credentials.AccessKey,
credentials.SecretKey,
headers,
)
if err != nil {
t.Fatalf("%s: failed to create signed put request: %v", instanceType, err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("%s: expected put to succeed, got %d", instanceType, rec.Code)
}
objInfo, err := obj.GetObjectInfo(context.Background(), bucketName, objectName, ObjectOptions{})
if err != nil {
t.Fatalf("%s: failed to fetch object info: %v", instanceType, err)
}
assertObjectMetadataValueNotEqual(t, objInfo.UserDefined,
crypto.MetaSealedKeySSEC,
headers["X-Minio-Replication-Server-Side-Encryption-Sealed-Key"],
)
assertObjectMetadataValueNotEqual(t, objInfo.UserDefined,
crypto.MetaIV,
headers["X-Minio-Replication-Server-Side-Encryption-Iv"],
)
assertObjectContents(t, obj, bucketName, objectName, payload)
}
// Wrapper for calling Copy Object Part API handler tests for both Erasure multiple disks and single node setup.
func TestAPICopyObjectPartHandler(t *testing.T) {
defer DetectTestLeak(t)()
@@ -2949,74 +2860,6 @@ func testAPINewMultipartHandlerParallel(obj ObjectLayer, instanceType, bucketNam
}
}
func TestAPICopyObjectReplicationHeaderPoisoning(t *testing.T) {
defer DetectTestLeak(t)()
ExecExtendedObjectLayerAPITest(t, testAPICopyObjectReplicationHeaderPoisoning, []string{"CopyObject", "PutObject"})
}
func testAPICopyObjectReplicationHeaderPoisoning(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
credentials auth.Credentials, t *testing.T,
) {
srcObject := "replication-header-poison-copy-src"
dstObject := "replication-header-poison-copy-dst"
payload := []byte("replication-header-poison-copy-payload")
if _, err := obj.PutObject(
context.Background(),
bucketName,
srcObject,
mustGetPutObjReader(t, bytes.NewReader(payload), int64(len(payload)), "", ""),
ObjectOptions{},
); err != nil {
t.Fatalf("%s: failed to create source object: %v", instanceType, err)
}
headers := replicationSSEPoisonHeaders()
headers[xhttp.AmzCopySource] = url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + srcObject)
headers[xhttp.AmzMetadataDirective] = replaceDirective
headers[xhttp.AmzBucketReplicationStatus] = "PENDING"
headers["Content-Type"] = "application/octet-stream"
req, err := newTestSignedRequestV4(
http.MethodPut,
getCopyObjectURL("", bucketName, dstObject),
0,
nil,
credentials.AccessKey,
credentials.SecretKey,
headers,
)
if err != nil {
t.Fatalf("%s: failed to create signed copy request: %v", instanceType, err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("%s: expected copy to succeed, got %d", instanceType, rec.Code)
}
objInfo, err := obj.GetObjectInfo(context.Background(), bucketName, dstObject, ObjectOptions{})
if err != nil {
t.Fatalf("%s: failed to fetch copied object info: %v", instanceType, err)
}
assertObjectMetadataValueNotEqual(t, objInfo.UserDefined,
crypto.MetaSealedKeySSEC,
headers["X-Minio-Replication-Server-Side-Encryption-Sealed-Key"],
)
assertObjectMetadataValueNotEqual(t, objInfo.UserDefined,
crypto.MetaIV,
headers["X-Minio-Replication-Server-Side-Encryption-Iv"],
)
assertObjectMetadataKeysAbsent(t, objInfo.UserDefined,
xhttp.AmzBucketReplicationStatus,
ReservedMetadataPrefixLower+ReplicaStatus,
ReservedMetadataPrefixLower+ReplicaTimestamp,
)
assertObjectContents(t, obj, bucketName, dstObject, payload)
}
// The UploadID from the response body is parsed and its existence is asserted with an attempt to ListParts using it.
func TestAPICompleteMultipartHandler(t *testing.T) {
defer DetectTestLeak(t)()
+13 -29
View File
@@ -24,8 +24,8 @@ import (
"io"
"maps"
"net/http"
"net/textproto"
"net/url"
"sort"
"strconv"
"strings"
"time"
@@ -80,12 +80,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
return
}
requestTags, hasRequestTags := getRequestHeaderOrQueryValue(r, xhttp.AmzObjectTagging)
var requestTagsPtr *string
if hasRequestTags {
requestTagsPtr = &requestTags
}
if s3Error := checkRequestAuthTypeWithRequestTags(ctx, r, policy.PutObjectAction, bucket, object, requestTagsPtr); s3Error != ErrNone {
if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, bucket, object); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
@@ -96,8 +91,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
AutoEncrypt: globalAutoEncryption,
})
// Validate the storage class header if present. Query values retain the
// existing compatibility path, including its historical validation behavior.
// Validate storage class metadata if present
if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" {
if !storageclass.IsValid(sc) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidStorageClass), r.URL)
@@ -154,21 +148,15 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
return
}
if objTags := metadata[xhttp.AmzObjectTagging]; objTags != "" {
if objTags := r.Header.Get(xhttp.AmzObjectTagging); objTags != "" {
if _, err := tags.ParseObjectTags(objTags); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
metadata[xhttp.AmzObjectTagging] = objTags
}
if r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() {
if s3Err := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.ReplicateObjectAction); s3Err != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
return
}
if err = extractReplicationMetadataFromMime(ctx, textproto.MIMEHeader(r.Header), metadata); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
metadata[ReservedMetadataPrefixLower+ReplicaStatus] = replication.Replica.String()
metadata[ReservedMetadataPrefixLower+ReplicaTimestamp] = UTCNow().Format(time.RFC3339Nano)
}
@@ -697,8 +685,8 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
return
}
case authTypeStreamingUnsignedTrailer:
// Initialize stream chunked reader with optional trailers.
reader, s3Error = newUnsignedV4ChunkedReader(r, true)
// Initialize stream signature verifier.
reader, s3Error = newUnsignedV4ChunkedReader(r, true, r.Header.Get(xhttp.Authorization) != "")
if s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
@@ -970,15 +958,11 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
return
}
// The parts list must be strictly increasing by part number. Gaps are
// allowed, repeats are not - sort.SliceIsSorted() with a '<' predicate
// considers equal neighbors sorted, so it is checked explicitly here,
// before anything is assembled into the target object.
for i := 1; i < len(complMultipartUpload.Parts); i++ {
if complMultipartUpload.Parts[i-1].PartNumber >= complMultipartUpload.Parts[i].PartNumber {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidPartOrder), r.URL)
return
}
if !sort.SliceIsSorted(complMultipartUpload.Parts, func(i, j int) bool {
return complMultipartUpload.Parts[i].PartNumber < complMultipartUpload.Parts[j].PartNumber
}) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidPartOrder), r.URL)
return
}
// Reject retention or governance headers if set, CompleteMultipartUpload spec
-267
View File
@@ -1,267 +0,0 @@
// Copyright (c) 2015-2025 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"context"
"encoding/xml"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/dustin/go-humanize"
"github.com/minio/minio/internal/auth"
)
// Tests the part number list validation of the CompleteMultipartUpload handler.
// The list must be strictly increasing: duplicate or out-of-order part numbers
// are rejected with InvalidPartOrder, and the rejection must happen before the
// target object is assembled. Only the ordering is constrained - part numbers
// need not start at 1 and need not be consecutive, so [1,3] and [5,9] are both
// legal and must be accepted.
func TestAPICompleteMultipartHandlerPartOrder(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPICompleteMultipartHandlerPartOrder,
endpoints: []string{"NewMultipart", "PutObjectPart", "CompleteMultipart"},
})
}
func testAPICompleteMultipartHandlerPartOrder(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
credentials auth.Credentials, t *testing.T,
) {
ctx := context.Background()
// Every uploaded part is >= the minimum part size so that a part is never
// rejected for being too small when it is not the last one.
partData := strings.Repeat("a", 5*humanize.MiByte)
partETag := getMD5Hash([]byte(partData))
testCases := []struct {
name string
// Part numbers uploaded before the completion request.
upload []int
// Part numbers listed in the completion request, in this order.
complete []int
expectedRespStatus int
// Only checked when expectedRespStatus is not http.StatusOK.
expectedErr APIErrorCode
}{
// Defect reproduction. A duplicated part number is the case that
// sort.SliceIsSorted() used to accept, because its '<' predicate treats
// equal neighbors as sorted. Each of these assembled the same part into
// the object more than once, inflating it past what was uploaded.
{
name: "duplicate-part",
upload: []int{1},
complete: []int{1, 1},
expectedRespStatus: http.StatusBadRequest,
expectedErr: ErrInvalidPartOrder,
},
{
name: "trailing-duplicate",
upload: []int{1, 2},
complete: []int{1, 2, 2},
expectedRespStatus: http.StatusBadRequest,
expectedErr: ErrInvalidPartOrder,
},
{
name: "duplicate-max-part-number",
upload: []int{globalMaxPartID},
complete: []int{globalMaxPartID, globalMaxPartID},
expectedRespStatus: http.StatusBadRequest,
expectedErr: ErrInvalidPartOrder,
},
// Regression guard. Ordering violations that were already rejected
// before the fix and must stay rejected with the same error.
{
name: "descending",
upload: []int{1, 2},
complete: []int{2, 1},
expectedRespStatus: http.StatusBadRequest,
expectedErr: ErrInvalidPartOrder,
},
{
name: "repeat-after-increase",
upload: []int{1, 2},
complete: []int{1, 2, 1},
expectedRespStatus: http.StatusBadRequest,
expectedErr: ErrInvalidPartOrder,
},
// Regression guard. The order loop starts at index 1 and so is a no-op
// for an empty list; an empty list is only rejected because the handler
// screens it out first. This pins that dependency down - without the
// len()==0 check the empty list reaches readParts() and panics the
// process on partMetaPaths[0] in xl-storage-disk-id-check.go.
{
name: "empty-parts",
upload: []int{1},
complete: nil,
expectedRespStatus: http.StatusBadRequest,
expectedErr: ErrMissingPart,
},
// Regression guard. Legal lists that must keep working. Part numbers
// only have to increase: they need not start at 1 and need not be
// consecutive, so none of these may be rejected.
{
name: "single-part",
upload: []int{1},
complete: []int{1},
expectedRespStatus: http.StatusOK,
},
{
name: "strictly-increasing",
upload: []int{1, 2},
complete: []int{1, 2},
expectedRespStatus: http.StatusOK,
},
{
name: "non-consecutive",
upload: []int{1, 3},
complete: []int{1, 3},
expectedRespStatus: http.StatusOK,
},
{
name: "gap-not-starting-at-one",
upload: []int{5, 9},
complete: []int{5, 9},
expectedRespStatus: http.StatusOK,
},
{
name: "single-part-not-one",
upload: []int{3},
complete: []int{3},
expectedRespStatus: http.StatusOK,
},
{
name: "part-number-bounds",
upload: []int{1, globalMaxPartID},
complete: []int{1, globalMaxPartID},
expectedRespStatus: http.StatusOK,
},
}
// completeReq issues a CompleteMultipartUpload request listing partNumbers
// in the given order and returns the recorded response.
completeReq := func(objectName, uploadID string, partNumbers []int) *httptest.ResponseRecorder {
t.Helper()
completeUploads := &CompleteMultipartUpload{}
for _, partNumber := range partNumbers {
completeUploads.Parts = append(completeUploads.Parts, CompletePart{
PartNumber: partNumber,
ETag: partETag,
})
}
completeBytes, err := xml.Marshal(completeUploads)
if err != nil {
t.Fatalf("%s: error XML encoding of parts: %v", instanceType, err)
}
req, err := newTestSignedRequestV4(http.MethodPost,
getCompleteMultipartUploadURL("", bucketName, objectName, uploadID),
int64(len(completeBytes)), bytes.NewReader(completeBytes),
credentials.AccessKey, credentials.SecretKey, nil)
if err != nil {
t.Fatalf("%s: failed to create HTTP request for CompleteMultipartUpload: %v", instanceType, err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
return rec
}
for i, testCase := range testCases {
objectName := fmt.Sprintf("test-part-order-%d-%s", i, testCase.name)
res, err := obj.NewMultipartUpload(ctx, bucketName, objectName, ObjectOptions{})
if err != nil {
t.Fatalf("%s: %s: failed to initiate multipart upload: %v", instanceType, testCase.name, err)
}
for _, partNumber := range testCase.upload {
_, err = obj.PutObjectPart(ctx, bucketName, objectName, res.UploadID, partNumber,
mustGetPutObjReader(t, strings.NewReader(partData), int64(len(partData)), partETag, ""), ObjectOptions{})
if err != nil {
t.Fatalf("%s: %s: failed to upload part %d: %v", instanceType, testCase.name, partNumber, err)
}
}
rec := completeReq(objectName, res.UploadID, testCase.complete)
if rec.Code != testCase.expectedRespStatus {
t.Errorf("%s: %s: expected response status %d, got %d: %s",
instanceType, testCase.name, testCase.expectedRespStatus, rec.Code, rec.Body.String())
}
objInfo, statErr := obj.GetObjectInfo(ctx, bucketName, objectName, ObjectOptions{})
if testCase.expectedRespStatus != http.StatusOK {
var errResp APIErrorResponse
if err = xml.Unmarshal(rec.Body.Bytes(), &errResp); err != nil {
t.Errorf("%s: %s: failed parsing error response %s: %v",
instanceType, testCase.name, rec.Body.String(), err)
} else if wantCode := getAPIError(testCase.expectedErr).Code; errResp.Code != wantCode {
t.Errorf("%s: %s: expected error code %s, got %s",
instanceType, testCase.name, wantCode, errResp.Code)
}
// A rejected completion must not have touched the target object.
if statErr == nil {
t.Errorf("%s: %s: expected no object to be created, found one of size %d",
instanceType, testCase.name, objInfo.Size)
} else if !isErrObjectNotFound(statErr) {
t.Errorf("%s: %s: expected ObjectNotFound after a rejected completion, got %v",
instanceType, testCase.name, statErr)
}
// The rejection must also leave the upload itself untouched, so
// that a client can retry with a well formed list. testCase.upload
// is always strictly increasing, so it is the corrected list.
retry := completeReq(objectName, res.UploadID, testCase.upload)
if retry.Code != http.StatusOK {
t.Errorf("%s: %s: expected the upload to survive a rejected completion, retry got %d: %s",
instanceType, testCase.name, retry.Code, retry.Body.String())
continue
}
objInfo, statErr = obj.GetObjectInfo(ctx, bucketName, objectName, ObjectOptions{})
if statErr != nil {
t.Errorf("%s: %s: expected the object to exist after the retry: %v", instanceType, testCase.name, statErr)
continue
}
if wantSize := int64(len(testCase.upload) * len(partData)); objInfo.Size != wantSize {
t.Errorf("%s: %s: expected the retried object to be %d bytes, got %d",
instanceType, testCase.name, wantSize, objInfo.Size)
}
continue
}
if statErr != nil {
t.Errorf("%s: %s: expected the object to exist after completion: %v", instanceType, testCase.name, statErr)
continue
}
if wantSize := int64(len(testCase.complete) * len(partData)); objInfo.Size != wantSize {
t.Errorf("%s: %s: expected the completed object to be %d bytes, got %d",
instanceType, testCase.name, wantSize, objInfo.Size)
}
}
}
+2 -3
View File
@@ -34,9 +34,8 @@ const _osMetric_name = "RemoveAllMkdirAllMkdirRenameOpenFileWOpenFileROpenFileWF
var _osMetric_index = [...]uint8{0, 9, 17, 22, 28, 37, 46, 57, 68, 72, 88, 93, 99, 103, 109, 115, 125, 134, 138, 142}
func (i osMetric) String() string {
idx := int(i) - 0
if i < 0 || idx >= len(_osMetric_index)-1 {
if i >= osMetric(len(_osMetric_index)-1) {
return "osMetric(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _osMetric_name[_osMetric_index[idx]:_osMetric_index[idx+1]]
return _osMetric_name[_osMetric_index[i]:_osMetric_index[i+1]]
}
+2 -3
View File
@@ -20,9 +20,8 @@ const _rebalanceMetric_name = "RebalanceBucketsRebalanceBucketRebalanceObjectReb
var _rebalanceMetric_index = [...]uint8{0, 16, 31, 46, 67, 79}
func (i rebalanceMetric) String() string {
idx := int(i) - 0
if i < 0 || idx >= len(_rebalanceMetric_index)-1 {
if i >= rebalanceMetric(len(_rebalanceMetric_index)-1) {
return "rebalanceMetric(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _rebalanceMetric_name[_rebalanceMetric_index[idx]:_rebalanceMetric_index[idx+1]]
return _rebalanceMetric_name[_rebalanceMetric_index[i]:_rebalanceMetric_index[i+1]]
}
+2 -3
View File
@@ -20,9 +20,8 @@ const _rebalStatus_name = "NoneStartedCompletedStoppedFailed"
var _rebalStatus_index = [...]uint8{0, 4, 11, 20, 27, 33}
func (i rebalStatus) String() string {
idx := int(i) - 0
if i < 0 || idx >= len(_rebalStatus_index)-1 {
if i >= rebalStatus(len(_rebalStatus_index)-1) {
return "rebalStatus(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _rebalStatus_name[_rebalStatus_index[idx]:_rebalStatus_index[idx+1]]
return _rebalStatus_name[_rebalStatus_index[i]:_rebalStatus_index[i+1]]
}

Some files were not shown because too many files have changed in this diff Show More