mirror of
https://github.com/pgsty/minio.git
synced 2026-07-31 15:55:18 +03:00
Compare commits
27 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5c6e30d4f1 | |||
| 0ce4b0f14b | |||
| 3e61b1d3a5 | |||
| df627ff896 | |||
| 73ac524724 | |||
| fd69c89d05 | |||
| 5e40665acd | |||
| f48dbe777d | |||
| f44110890b | |||
| 9e10f6d9a0 | |||
| 18b712d49a | |||
| db4c0fd5e3 | |||
| efb6e5b00b | |||
| f444b6f37e | |||
| 3252d5b7f3 | |||
| 56fa63bfd1 | |||
| 3b950f8fa8 | |||
| d24f449e08 | |||
| e4fa063942 | |||
| ff58df9499 | |||
| 1869bd30b8 | |||
| 68e0ba9971 | |||
| ce1c537eb1 | |||
| ee55e5391a | |||
| f2f9a40dce | |||
| 377fc616d9 | |||
| 5abd9a80f6 |
@@ -46,6 +46,7 @@ dockers:
|
||||
- "--label=org.opencontainers.image.revision={{ .FullCommit }}"
|
||||
extra_files:
|
||||
- dockerscripts/docker-entrypoint.sh
|
||||
- dockerscripts/download-static-curl.sh
|
||||
- LICENSE
|
||||
- CREDITS
|
||||
|
||||
@@ -66,6 +67,7 @@ dockers:
|
||||
- "--label=org.opencontainers.image.revision={{ .FullCommit }}"
|
||||
extra_files:
|
||||
- dockerscripts/docker-entrypoint.sh
|
||||
- dockerscripts/download-static-curl.sh
|
||||
- LICENSE
|
||||
- CREDITS
|
||||
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
name: 'Dependency Review'
|
||||
on: [pull_request]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
dependency-review:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@v4
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@v4
|
||||
@@ -1,39 +0,0 @@
|
||||
name: Crosscompile
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build Tests with Go ${{ matrix.go-version }} on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.24.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- name: Build on ${{ matrix.os }}
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
env:
|
||||
CGO_ENABLED: 0
|
||||
GO111MODULE: on
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make crosscompile
|
||||
@@ -1,44 +0,0 @@
|
||||
name: Healing Functional Tests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.24.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- name: Build on ${{ matrix.os }}
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
env:
|
||||
CGO_ENABLED: 0
|
||||
GO111MODULE: on
|
||||
MINIO_KMS_SECRET_KEY: "my-minio-key:oyArl7zlPECEduNbB1KXgdzDn2Bdpvvw0l8VO51HQnY="
|
||||
MINIO_KMS_AUTO_ENCRYPTION: on
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make verify-healing
|
||||
make verify-healing-inconsistent-versions
|
||||
make verify-healing-with-root-disks
|
||||
make verify-healing-with-rewrite
|
||||
@@ -1,42 +0,0 @@
|
||||
name: Linters and Tests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.24.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- name: Build on ${{ matrix.os }}
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
env:
|
||||
CGO_ENABLED: 0
|
||||
GO111MODULE: on
|
||||
run: |
|
||||
sudo apt install jq -y
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make
|
||||
make test
|
||||
make test-race
|
||||
@@ -1,39 +0,0 @@
|
||||
name: Resiliency Functional Tests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.24.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- name: Build on ${{ matrix.os }}
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
env:
|
||||
CGO_ENABLED: 0
|
||||
GO111MODULE: on
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make test-resiliency
|
||||
@@ -1,42 +0,0 @@
|
||||
name: Functional Tests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Go ${{ matrix.go-version }} on ${{ matrix.os }} - healing
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.24.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- name: Build on ${{ matrix.os }}
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
env:
|
||||
CGO_ENABLED: 0
|
||||
GO111MODULE: on
|
||||
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
|
||||
MINIO_KMS_AUTO_ENCRYPTION: on
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make verify
|
||||
make test-timeout
|
||||
@@ -1,30 +0,0 @@
|
||||
name: Helm Chart linting
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install Helm
|
||||
uses: azure/setup-helm@v4
|
||||
|
||||
- name: Run helm lint
|
||||
run: |
|
||||
cd helm/minio
|
||||
helm lint .
|
||||
@@ -0,0 +1,110 @@
|
||||
name: Helm Chart
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/helm.yml"
|
||||
- "helm/**"
|
||||
- "helm-releases/minio-*.tgz"
|
||||
- "helm-reindex.sh"
|
||||
- "index.yaml"
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths:
|
||||
- ".github/workflows/helm.yml"
|
||||
- "helm/**"
|
||||
- "helm-releases/minio-*.tgz"
|
||||
- "helm-reindex.sh"
|
||||
- "index.yaml"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
install-smoke:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Helm
|
||||
uses: azure/setup-helm@v4
|
||||
with:
|
||||
version: v3.19.0
|
||||
|
||||
- name: Verify packaged chart
|
||||
run: |
|
||||
set -euo pipefail
|
||||
chart_version="$(awk '$1 == "version:" { print $2; exit }' helm/minio/Chart.yaml)"
|
||||
chart_package="helm-releases/minio-${chart_version}.tgz"
|
||||
test -f "${chart_package}"
|
||||
echo "CHART_PACKAGE=${chart_package}" >> "${GITHUB_ENV}"
|
||||
helm lint --strict "${chart_package}"
|
||||
|
||||
unpacked="$(mktemp -d)"
|
||||
tar -xzf "${chart_package}" -C "${unpacked}"
|
||||
diff -ru --exclude Chart.yaml helm/minio "${unpacked}/minio"
|
||||
|
||||
package_digest="$(sha256sum "${chart_package}" | awk '{ print $1 }')"
|
||||
grep -F "digest: ${package_digest}" index.yaml
|
||||
grep -F "https://raw.githubusercontent.com/pgsty/minio/master/${chart_package}" index.yaml
|
||||
|
||||
- name: Render and verify immutable images
|
||||
run: |
|
||||
set -euo pipefail
|
||||
expected_image='pgsty/minio@sha256:dacff8306a6e0a734518533992dbdcca26bc1ca47f77cf47cb9945725f92b29b'
|
||||
rendered="$(mktemp)"
|
||||
helm template silo "${CHART_PACKAGE}" \
|
||||
--namespace silo \
|
||||
--set mode=standalone \
|
||||
--set persistence.enabled=false > "${rendered}"
|
||||
grep -F "image: \"${expected_image}\"" "${rendered}"
|
||||
if grep -Eq 'image:.*(quay\.io/minio|minio/minio|minio/mc)' "${rendered}"; then
|
||||
echo "Rendered chart still references an upstream MinIO image."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Create Kind cluster
|
||||
uses: helm/kind-action@v1
|
||||
with:
|
||||
version: v0.31.0
|
||||
kubectl_version: v1.35.0
|
||||
node_image: kindest/node:v1.35.0@sha256:452d707d4862f52530247495d180205e029056831160e22870e37e3f6c1ac31f
|
||||
cluster_name: silo-chart
|
||||
wait: 120s
|
||||
|
||||
- name: Install chart
|
||||
run: |
|
||||
set -euo pipefail
|
||||
helm install silo "${CHART_PACKAGE}" \
|
||||
--namespace silo \
|
||||
--create-namespace \
|
||||
--wait \
|
||||
--wait-for-jobs \
|
||||
--timeout 10m \
|
||||
--set mode=standalone \
|
||||
--set replicas=1 \
|
||||
--set persistence.enabled=false \
|
||||
--set resources.requests.memory=512Mi \
|
||||
--set rootUser=siloadmin \
|
||||
--set rootPassword=silo-smoke-password \
|
||||
--set 'buckets[0].name=bootstrap' \
|
||||
--set 'buckets[0].policy=none' \
|
||||
--set 'buckets[0].purge=false'
|
||||
kubectl rollout status deployment/silo-minio --namespace silo --timeout=2m
|
||||
|
||||
- name: Run S3 smoke test
|
||||
run: |
|
||||
set -euo pipefail
|
||||
helm test silo --namespace silo --logs --timeout 5m
|
||||
|
||||
- name: Diagnostics
|
||||
if: failure()
|
||||
run: |
|
||||
helm status silo --namespace silo || true
|
||||
kubectl get all --namespace silo -o wide || true
|
||||
kubectl describe pods --namespace silo || true
|
||||
kubectl logs --namespace silo --all-containers=true --prefix=true --selector release=silo || true
|
||||
@@ -1,161 +0,0 @@
|
||||
name: IAM integration
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
iam-matrix-test:
|
||||
name: "[Go=${{ matrix.go-version }}|ldap=${{ matrix.ldap }}|etcd=${{ matrix.etcd }}|openid=${{ matrix.openid }}]"
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
services:
|
||||
openldap:
|
||||
image: quay.io/minio/openldap
|
||||
ports:
|
||||
- "389:389"
|
||||
- "636:636"
|
||||
env:
|
||||
LDAP_ORGANIZATION: "MinIO Inc"
|
||||
LDAP_DOMAIN: "min.io"
|
||||
LDAP_ADMIN_PASSWORD: "admin"
|
||||
etcd:
|
||||
image: "quay.io/coreos/etcd:v3.5.1"
|
||||
env:
|
||||
ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379"
|
||||
ETCD_ADVERTISE_CLIENT_URLS: "http://0.0.0.0:2379"
|
||||
ports:
|
||||
- "2379:2379"
|
||||
options: >-
|
||||
--health-cmd "etcdctl endpoint health"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
openid:
|
||||
image: quay.io/minio/dex
|
||||
ports:
|
||||
- "5556:5556"
|
||||
env:
|
||||
DEX_LDAP_SERVER: "openldap:389"
|
||||
openid2:
|
||||
image: quay.io/minio/dex
|
||||
ports:
|
||||
- "5557:5557"
|
||||
env:
|
||||
DEX_LDAP_SERVER: "openldap:389"
|
||||
DEX_ISSUER: "http://127.0.0.1:5557/dex"
|
||||
DEX_WEB_HTTP: "0.0.0.0:5557"
|
||||
|
||||
strategy:
|
||||
# When ldap, etcd or openid vars are empty below, those external servers
|
||||
# are turned off - i.e. if ldap="", then ldap server is not enabled for
|
||||
# the tests.
|
||||
matrix:
|
||||
go-version: [1.24.x]
|
||||
ldap: ["", "localhost:389"]
|
||||
etcd: ["", "http://localhost:2379"]
|
||||
openid: ["", "http://127.0.0.1:5556/dex"]
|
||||
exclude:
|
||||
# exclude combos where all are empty.
|
||||
- ldap: ""
|
||||
etcd: ""
|
||||
openid: ""
|
||||
# exclude combos where both ldap and openid IDPs are specified.
|
||||
- ldap: "localhost:389"
|
||||
openid: "http://127.0.0.1:5556/dex"
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- name: Test LDAP/OpenID/Etcd combo
|
||||
env:
|
||||
_MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }}
|
||||
_MINIO_ETCD_TEST_SERVER: ${{ matrix.etcd }}
|
||||
_MINIO_OPENID_TEST_SERVER: ${{ matrix.openid }}
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make test-iam
|
||||
- name: Test with multiple OpenID providers
|
||||
if: matrix.openid == 'http://127.0.0.1:5556/dex'
|
||||
env:
|
||||
_MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }}
|
||||
_MINIO_ETCD_TEST_SERVER: ${{ matrix.etcd }}
|
||||
_MINIO_OPENID_TEST_SERVER: ${{ matrix.openid }}
|
||||
_MINIO_OPENID_TEST_SERVER_2: "http://127.0.0.1:5557/dex"
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make test-iam
|
||||
- name: Test with Access Management Plugin enabled
|
||||
env:
|
||||
_MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }}
|
||||
_MINIO_ETCD_TEST_SERVER: ${{ matrix.etcd }}
|
||||
_MINIO_OPENID_TEST_SERVER: ${{ matrix.openid }}
|
||||
_MINIO_POLICY_PLUGIN_TEST_ENDPOINT: "http://127.0.0.1:8080"
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
go run docs/iam/access-manager-plugin.go &
|
||||
make test-iam
|
||||
- name: Test MinIO Old Version data to IAM import current version
|
||||
if: matrix.ldap == 'ldaphost:389'
|
||||
env:
|
||||
_MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }}
|
||||
run: |
|
||||
make test-iam-ldap-upgrade-import
|
||||
- name: Test LDAP for automatic site replication
|
||||
if: matrix.ldap == 'localhost:389'
|
||||
run: |
|
||||
make test-site-replication-ldap
|
||||
- name: Test OIDC for automatic site replication
|
||||
if: matrix.openid == 'http://127.0.0.1:5556/dex'
|
||||
run: |
|
||||
make test-site-replication-oidc
|
||||
iam-import-with-missing-entities:
|
||||
name: Test IAM import in new cluster with missing entities
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- name: Checkout minio-iam-testing
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: minio/minio-iam-testing
|
||||
path: minio-iam-testing
|
||||
- name: Test import of IAM artifacts when in fresh cluster there are missing groups etc
|
||||
run: |
|
||||
make test-iam-import-with-missing-entities
|
||||
iam-import-with-openid:
|
||||
name: Test IAM import in new cluster with opendid configurations
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- name: Checkout minio-iam-testing
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: minio/minio-iam-testing
|
||||
path: minio-iam-testing
|
||||
- name: Test import of IAM artifacts when in fresh cluster with openid configurations
|
||||
run: |
|
||||
make test-iam-import-with-openid
|
||||
@@ -1,18 +0,0 @@
|
||||
# @format
|
||||
|
||||
name: Issue Workflow
|
||||
|
||||
on:
|
||||
issues:
|
||||
types:
|
||||
- opened
|
||||
|
||||
jobs:
|
||||
add-to-project:
|
||||
name: Add issue to project
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/add-to-project@v0.5.0
|
||||
with:
|
||||
project-url: https://github.com/orgs/miniohq/projects/2
|
||||
github-token: ${{ secrets.BOT_PAT }}
|
||||
@@ -1,24 +0,0 @@
|
||||
name: 'Lock Threads'
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 0 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
|
||||
concurrency:
|
||||
group: lock
|
||||
|
||||
jobs:
|
||||
action:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: dessant/lock-threads@v3
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
issue-inactive-days: '365'
|
||||
exclude-any-issue-labels: 'do-not-close'
|
||||
issue-lock-reason: 'resolved'
|
||||
log-output: true
|
||||
@@ -1,81 +0,0 @@
|
||||
name: Mint Tests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
mint-test:
|
||||
runs-on: mint
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: cleanup #https://github.com/actions/checkout/issues/273
|
||||
run: |
|
||||
sudo -S rm -rf ${GITHUB_WORKSPACE}
|
||||
mkdir ${GITHUB_WORKSPACE}
|
||||
- name: checkout-step
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: setup-go-step
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 1.24.x
|
||||
|
||||
- name: github sha short
|
||||
id: vars
|
||||
run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: build-minio
|
||||
run: |
|
||||
TAG="quay.io/minio/minio:${{ steps.vars.outputs.sha_short }}" make docker
|
||||
|
||||
- name: multipart uploads test
|
||||
run: |
|
||||
${GITHUB_WORKSPACE}/.github/workflows/multipart/migrate.sh "${{ steps.vars.outputs.sha_short }}"
|
||||
|
||||
- name: compress and encrypt
|
||||
run: |
|
||||
${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "compress-encrypt" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
|
||||
|
||||
- name: multiple pools
|
||||
run: |
|
||||
${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "pools" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
|
||||
|
||||
- name: standalone erasure
|
||||
run: |
|
||||
${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "erasure" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
|
||||
|
||||
# FIXME: renable this back when we have a valid way to add deadlines for PUT()s (internode CreateFile)
|
||||
# - name: resiliency
|
||||
# run: |
|
||||
# ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "resiliency" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
|
||||
|
||||
- name: The job must cleanup
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
export JOB_NAME=${{ steps.vars.outputs.sha_short }}
|
||||
for mode in $(echo compress-encrypt pools erasure); do
|
||||
docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml down || true
|
||||
docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml rm || true
|
||||
done
|
||||
|
||||
docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site1.yaml rm -s -f || true
|
||||
docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site2.yaml rm -s -f || true
|
||||
for volume in $(docker volume ls -q | grep minio); do
|
||||
docker volume rm ${volume} || true
|
||||
done
|
||||
|
||||
docker rmi -f quay.io/minio/minio:${{ steps.vars.outputs.sha_short }}
|
||||
docker system prune -f || true
|
||||
docker volume prune -f || true
|
||||
docker volume rm $(docker volume ls -q -f dangling=true) || true
|
||||
@@ -1,80 +0,0 @@
|
||||
version: '3.7'
|
||||
|
||||
# Settings and configurations that are common for all containers
|
||||
x-minio-common: &minio-common
|
||||
image: quay.io/minio/minio:${JOB_NAME}
|
||||
command: server --console-address ":9001" http://minio{1...4}/cdata{1...2}
|
||||
expose:
|
||||
- "9000"
|
||||
- "9001"
|
||||
environment:
|
||||
MINIO_CI_CD: "on"
|
||||
MINIO_ROOT_USER: "minio"
|
||||
MINIO_ROOT_PASSWORD: "minio123"
|
||||
MINIO_COMPRESSION_ENABLE: "on"
|
||||
MINIO_COMPRESSION_MIME_TYPES: "*"
|
||||
MINIO_COMPRESSION_ALLOW_ENCRYPTION: "on"
|
||||
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
|
||||
healthcheck:
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# starts 4 docker containers running minio server instances.
|
||||
# using nginx reverse proxy, load balancing, you can access
|
||||
# it through port 9000.
|
||||
services:
|
||||
minio1:
|
||||
<<: *minio-common
|
||||
hostname: minio1
|
||||
volumes:
|
||||
- cdata1-1:/cdata1
|
||||
- cdata1-2:/cdata2
|
||||
|
||||
minio2:
|
||||
<<: *minio-common
|
||||
hostname: minio2
|
||||
volumes:
|
||||
- cdata2-1:/cdata1
|
||||
- cdata2-2:/cdata2
|
||||
|
||||
minio3:
|
||||
<<: *minio-common
|
||||
hostname: minio3
|
||||
volumes:
|
||||
- cdata3-1:/cdata1
|
||||
- cdata3-2:/cdata2
|
||||
|
||||
minio4:
|
||||
<<: *minio-common
|
||||
hostname: minio4
|
||||
volumes:
|
||||
- cdata4-1:/cdata1
|
||||
- cdata4-2:/cdata2
|
||||
|
||||
nginx:
|
||||
image: nginx:1.19.2-alpine
|
||||
hostname: nginx
|
||||
volumes:
|
||||
- ./nginx-4-node.conf:/etc/nginx/nginx.conf:ro
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
depends_on:
|
||||
- minio1
|
||||
- minio2
|
||||
- minio3
|
||||
- minio4
|
||||
|
||||
## By default this config uses default local driver,
|
||||
## For custom volumes replace with volume driver configuration.
|
||||
volumes:
|
||||
cdata1-1:
|
||||
cdata1-2:
|
||||
cdata2-1:
|
||||
cdata2-2:
|
||||
cdata3-1:
|
||||
cdata3-2:
|
||||
cdata4-1:
|
||||
cdata4-2:
|
||||
@@ -1,51 +0,0 @@
|
||||
version: '3.7'
|
||||
|
||||
# Settings and configurations that are common for all containers
|
||||
x-minio-common: &minio-common
|
||||
image: quay.io/minio/minio:${JOB_NAME}
|
||||
command: server --console-address ":9001" edata{1...4}
|
||||
expose:
|
||||
- "9000"
|
||||
- "9001"
|
||||
environment:
|
||||
MINIO_CI_CD: "on"
|
||||
MINIO_ROOT_USER: "minio"
|
||||
MINIO_ROOT_PASSWORD: "minio123"
|
||||
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
|
||||
healthcheck:
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# starts 4 docker containers running minio server instances.
|
||||
# using nginx reverse proxy, load balancing, you can access
|
||||
# it through port 9000.
|
||||
services:
|
||||
minio1:
|
||||
<<: *minio-common
|
||||
hostname: minio1
|
||||
volumes:
|
||||
- edata1-1:/edata1
|
||||
- edata1-2:/edata2
|
||||
- edata1-3:/edata3
|
||||
- edata1-4:/edata4
|
||||
|
||||
nginx:
|
||||
image: nginx:1.19.2-alpine
|
||||
hostname: nginx
|
||||
volumes:
|
||||
- ./nginx-1-node.conf:/etc/nginx/nginx.conf:ro
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
depends_on:
|
||||
- minio1
|
||||
|
||||
## By default this config uses default local driver,
|
||||
## For custom volumes replace with volume driver configuration.
|
||||
volumes:
|
||||
edata1-1:
|
||||
edata1-2:
|
||||
edata1-3:
|
||||
edata1-4:
|
||||
@@ -1,117 +0,0 @@
|
||||
version: '3.7'
|
||||
|
||||
# Settings and configurations that are common for all containers
|
||||
x-minio-common: &minio-common
|
||||
image: quay.io/minio/minio:${JOB_NAME}
|
||||
command: server --console-address ":9001" http://minio{1...4}/pdata{1...2} http://minio{5...8}/pdata{1...2}
|
||||
expose:
|
||||
- "9000"
|
||||
- "9001"
|
||||
environment:
|
||||
MINIO_CI_CD: "on"
|
||||
MINIO_ROOT_USER: "minio"
|
||||
MINIO_ROOT_PASSWORD: "minio123"
|
||||
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
|
||||
healthcheck:
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# starts 4 docker containers running minio server instances.
|
||||
# using nginx reverse proxy, load balancing, you can access
|
||||
# it through port 9000.
|
||||
services:
|
||||
minio1:
|
||||
<<: *minio-common
|
||||
hostname: minio1
|
||||
volumes:
|
||||
- pdata1-1:/pdata1
|
||||
- pdata1-2:/pdata2
|
||||
|
||||
minio2:
|
||||
<<: *minio-common
|
||||
hostname: minio2
|
||||
volumes:
|
||||
- pdata2-1:/pdata1
|
||||
- pdata2-2:/pdata2
|
||||
|
||||
minio3:
|
||||
<<: *minio-common
|
||||
hostname: minio3
|
||||
volumes:
|
||||
- pdata3-1:/pdata1
|
||||
- pdata3-2:/pdata2
|
||||
|
||||
minio4:
|
||||
<<: *minio-common
|
||||
hostname: minio4
|
||||
volumes:
|
||||
- pdata4-1:/pdata1
|
||||
- pdata4-2:/pdata2
|
||||
|
||||
minio5:
|
||||
<<: *minio-common
|
||||
hostname: minio5
|
||||
volumes:
|
||||
- pdata5-1:/pdata1
|
||||
- pdata5-2:/pdata2
|
||||
|
||||
minio6:
|
||||
<<: *minio-common
|
||||
hostname: minio6
|
||||
volumes:
|
||||
- pdata6-1:/pdata1
|
||||
- pdata6-2:/pdata2
|
||||
|
||||
minio7:
|
||||
<<: *minio-common
|
||||
hostname: minio7
|
||||
volumes:
|
||||
- pdata7-1:/pdata1
|
||||
- pdata7-2:/pdata2
|
||||
|
||||
minio8:
|
||||
<<: *minio-common
|
||||
hostname: minio8
|
||||
volumes:
|
||||
- pdata8-1:/pdata1
|
||||
- pdata8-2:/pdata2
|
||||
|
||||
nginx:
|
||||
image: nginx:1.19.2-alpine
|
||||
hostname: nginx
|
||||
volumes:
|
||||
- ./nginx-8-node.conf:/etc/nginx/nginx.conf:ro
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
depends_on:
|
||||
- minio1
|
||||
- minio2
|
||||
- minio3
|
||||
- minio4
|
||||
- minio5
|
||||
- minio6
|
||||
- minio7
|
||||
- minio8
|
||||
|
||||
## By default this config uses default local driver,
|
||||
## For custom volumes replace with volume driver configuration.
|
||||
volumes:
|
||||
pdata1-1:
|
||||
pdata1-2:
|
||||
pdata2-1:
|
||||
pdata2-2:
|
||||
pdata3-1:
|
||||
pdata3-2:
|
||||
pdata4-1:
|
||||
pdata4-2:
|
||||
pdata5-1:
|
||||
pdata5-2:
|
||||
pdata6-1:
|
||||
pdata6-2:
|
||||
pdata7-1:
|
||||
pdata7-2:
|
||||
pdata8-1:
|
||||
pdata8-2:
|
||||
@@ -1,78 +0,0 @@
|
||||
version: '3.7'
|
||||
|
||||
# Settings and configurations that are common for all containers
|
||||
x-minio-common: &minio-common
|
||||
image: quay.io/minio/minio:${JOB_NAME}
|
||||
command: server --console-address ":9001" http://minio{1...4}/rdata{1...2}
|
||||
expose:
|
||||
- "9000"
|
||||
- "9001"
|
||||
environment:
|
||||
MINIO_CI_CD: "on"
|
||||
MINIO_ROOT_USER: "minio"
|
||||
MINIO_ROOT_PASSWORD: "minio123"
|
||||
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
|
||||
MINIO_DRIVE_MAX_TIMEOUT: "5s"
|
||||
healthcheck:
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# starts 4 docker containers running minio server instances.
|
||||
# using nginx reverse proxy, load balancing, you can access
|
||||
# it through port 9000.
|
||||
services:
|
||||
minio1:
|
||||
<<: *minio-common
|
||||
hostname: minio1
|
||||
volumes:
|
||||
- rdata1-1:/rdata1
|
||||
- rdata1-2:/rdata2
|
||||
|
||||
minio2:
|
||||
<<: *minio-common
|
||||
hostname: minio2
|
||||
volumes:
|
||||
- rdata2-1:/rdata1
|
||||
- rdata2-2:/rdata2
|
||||
|
||||
minio3:
|
||||
<<: *minio-common
|
||||
hostname: minio3
|
||||
volumes:
|
||||
- rdata3-1:/rdata1
|
||||
- rdata3-2:/rdata2
|
||||
|
||||
minio4:
|
||||
<<: *minio-common
|
||||
hostname: minio4
|
||||
volumes:
|
||||
- rdata4-1:/rdata1
|
||||
- rdata4-2:/rdata2
|
||||
|
||||
nginx:
|
||||
image: nginx:1.19.2-alpine
|
||||
hostname: nginx
|
||||
volumes:
|
||||
- ./nginx-4-node.conf:/etc/nginx/nginx.conf:ro
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
depends_on:
|
||||
- minio1
|
||||
- minio2
|
||||
- minio3
|
||||
- minio4
|
||||
|
||||
## By default this config uses default local driver,
|
||||
## For custom volumes replace with volume driver configuration.
|
||||
volumes:
|
||||
rdata1-1:
|
||||
rdata1-2:
|
||||
rdata2-1:
|
||||
rdata2-2:
|
||||
rdata3-1:
|
||||
rdata3-2:
|
||||
rdata4-1:
|
||||
rdata4-2:
|
||||
@@ -1,100 +0,0 @@
|
||||
user nginx;
|
||||
worker_processes auto;
|
||||
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
pid /var/run/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 4096;
|
||||
}
|
||||
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent" "$http_x_forwarded_for"';
|
||||
|
||||
access_log /var/log/nginx/access.log main;
|
||||
sendfile on;
|
||||
keepalive_timeout 65;
|
||||
|
||||
# include /etc/nginx/conf.d/*.conf;
|
||||
|
||||
upstream minio {
|
||||
server minio1:9000;
|
||||
}
|
||||
|
||||
upstream console {
|
||||
ip_hash;
|
||||
server minio1:9001;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 9000;
|
||||
listen [::]:9000;
|
||||
server_name localhost;
|
||||
|
||||
# To allow special characters in headers
|
||||
ignore_invalid_headers off;
|
||||
# Allow any size file to be uploaded.
|
||||
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||
client_max_body_size 0;
|
||||
# To disable buffering
|
||||
proxy_buffering off;
|
||||
proxy_request_buffering off;
|
||||
|
||||
location / {
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_connect_timeout 300;
|
||||
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
chunked_transfer_encoding off;
|
||||
|
||||
proxy_pass http://minio;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 9001;
|
||||
listen [::]:9001;
|
||||
server_name localhost;
|
||||
|
||||
# To allow special characters in headers
|
||||
ignore_invalid_headers off;
|
||||
# Allow any size file to be uploaded.
|
||||
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||
client_max_body_size 0;
|
||||
# To disable buffering
|
||||
proxy_buffering off;
|
||||
proxy_request_buffering off;
|
||||
|
||||
location / {
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header X-NginX-Proxy true;
|
||||
|
||||
# This is necessary to pass the correct IP to be hashed
|
||||
real_ip_header X-Real-IP;
|
||||
|
||||
proxy_connect_timeout 300;
|
||||
|
||||
# To support websocket
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
|
||||
chunked_transfer_encoding off;
|
||||
|
||||
proxy_pass http://console;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
user nginx;
|
||||
worker_processes auto;
|
||||
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
pid /var/run/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 4096;
|
||||
}
|
||||
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent" "$http_x_forwarded_for"';
|
||||
|
||||
access_log /var/log/nginx/access.log main;
|
||||
sendfile on;
|
||||
keepalive_timeout 65;
|
||||
|
||||
# include /etc/nginx/conf.d/*.conf;
|
||||
|
||||
upstream minio {
|
||||
server minio1:9000 max_fails=1 fail_timeout=10s;
|
||||
server minio2:9000 max_fails=1 fail_timeout=10s;
|
||||
server minio3:9000 max_fails=1 fail_timeout=10s;
|
||||
}
|
||||
|
||||
upstream console {
|
||||
ip_hash;
|
||||
server minio1:9001;
|
||||
server minio2:9001;
|
||||
server minio3:9001;
|
||||
server minio4:9001;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 9000;
|
||||
listen [::]:9000;
|
||||
server_name localhost;
|
||||
|
||||
# To allow special characters in headers
|
||||
ignore_invalid_headers off;
|
||||
# Allow any size file to be uploaded.
|
||||
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||
client_max_body_size 0;
|
||||
# To disable buffering
|
||||
proxy_buffering off;
|
||||
proxy_request_buffering off;
|
||||
|
||||
location / {
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_connect_timeout 300;
|
||||
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
chunked_transfer_encoding off;
|
||||
|
||||
proxy_pass http://minio;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 9001;
|
||||
listen [::]:9001;
|
||||
server_name localhost;
|
||||
|
||||
# To allow special characters in headers
|
||||
ignore_invalid_headers off;
|
||||
# Allow any size file to be uploaded.
|
||||
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||
client_max_body_size 0;
|
||||
# To disable buffering
|
||||
proxy_buffering off;
|
||||
proxy_request_buffering off;
|
||||
|
||||
location / {
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header X-NginX-Proxy true;
|
||||
|
||||
# This is necessary to pass the correct IP to be hashed
|
||||
real_ip_header X-Real-IP;
|
||||
|
||||
proxy_connect_timeout 300;
|
||||
|
||||
# To support websocket
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
|
||||
chunked_transfer_encoding off;
|
||||
|
||||
proxy_pass http://console;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,114 +0,0 @@
|
||||
user nginx;
|
||||
worker_processes auto;
|
||||
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
pid /var/run/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 4096;
|
||||
}
|
||||
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent" "$http_x_forwarded_for"';
|
||||
|
||||
access_log /var/log/nginx/access.log main;
|
||||
sendfile on;
|
||||
keepalive_timeout 65;
|
||||
|
||||
# include /etc/nginx/conf.d/*.conf;
|
||||
|
||||
upstream minio {
|
||||
server minio1:9000 max_fails=1 fail_timeout=10s;
|
||||
server minio2:9000 max_fails=1 fail_timeout=10s;
|
||||
server minio3:9000 max_fails=1 fail_timeout=10s;
|
||||
server minio4:9000 max_fails=1 fail_timeout=10s;
|
||||
server minio5:9000 max_fails=1 fail_timeout=10s;
|
||||
server minio6:9000 max_fails=1 fail_timeout=10s;
|
||||
server minio7:9000 max_fails=1 fail_timeout=10s;
|
||||
server minio8:9000 max_fails=1 fail_timeout=10s;
|
||||
}
|
||||
|
||||
upstream console {
|
||||
ip_hash;
|
||||
server minio1:9001;
|
||||
server minio2:9001;
|
||||
server minio3:9001;
|
||||
server minio4:9001;
|
||||
server minio5:9001;
|
||||
server minio6:9001;
|
||||
server minio7:9001;
|
||||
server minio8:9001;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 9000;
|
||||
listen [::]:9000;
|
||||
server_name localhost;
|
||||
|
||||
# To allow special characters in headers
|
||||
ignore_invalid_headers off;
|
||||
# Allow any size file to be uploaded.
|
||||
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||
client_max_body_size 0;
|
||||
# To disable buffering
|
||||
proxy_buffering off;
|
||||
proxy_request_buffering off;
|
||||
|
||||
location / {
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_connect_timeout 300;
|
||||
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
chunked_transfer_encoding off;
|
||||
|
||||
proxy_pass http://minio;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 9001;
|
||||
listen [::]:9001;
|
||||
server_name localhost;
|
||||
|
||||
# To allow special characters in headers
|
||||
ignore_invalid_headers off;
|
||||
# Allow any size file to be uploaded.
|
||||
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||
client_max_body_size 0;
|
||||
# To disable buffering
|
||||
proxy_buffering off;
|
||||
proxy_request_buffering off;
|
||||
|
||||
location / {
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header X-NginX-Proxy true;
|
||||
|
||||
# This is necessary to pass the correct IP to be hashed
|
||||
real_ip_header X-Real-IP;
|
||||
|
||||
proxy_connect_timeout 300;
|
||||
|
||||
# To support websocket
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
|
||||
chunked_transfer_encoding off;
|
||||
|
||||
proxy_pass http://console;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
user nginx;
|
||||
worker_processes auto;
|
||||
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
pid /var/run/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 4096;
|
||||
}
|
||||
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent" "$http_x_forwarded_for"';
|
||||
|
||||
access_log /var/log/nginx/access.log main;
|
||||
sendfile on;
|
||||
keepalive_timeout 65;
|
||||
|
||||
# include /etc/nginx/conf.d/*.conf;
|
||||
|
||||
upstream minio {
|
||||
server minio1:9000 max_fails=1 fail_timeout=10s;
|
||||
server minio2:9000 max_fails=1 fail_timeout=10s;
|
||||
server minio3:9000 max_fails=1 fail_timeout=10s;
|
||||
server minio4:9000 max_fails=1 fail_timeout=10s;
|
||||
}
|
||||
|
||||
upstream console {
|
||||
ip_hash;
|
||||
server minio1:9001;
|
||||
server minio2:9001;
|
||||
server minio3:9001;
|
||||
server minio4:9001;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 9000;
|
||||
listen [::]:9000;
|
||||
server_name localhost;
|
||||
|
||||
# To allow special characters in headers
|
||||
ignore_invalid_headers off;
|
||||
# Allow any size file to be uploaded.
|
||||
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||
client_max_body_size 0;
|
||||
# To disable buffering
|
||||
proxy_buffering off;
|
||||
proxy_request_buffering off;
|
||||
|
||||
location / {
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_connect_timeout 300;
|
||||
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
chunked_transfer_encoding off;
|
||||
|
||||
proxy_pass http://minio;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 9001;
|
||||
listen [::]:9001;
|
||||
server_name localhost;
|
||||
|
||||
# To allow special characters in headers
|
||||
ignore_invalid_headers off;
|
||||
# Allow any size file to be uploaded.
|
||||
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||
client_max_body_size 0;
|
||||
# To disable buffering
|
||||
proxy_buffering off;
|
||||
proxy_request_buffering off;
|
||||
|
||||
location / {
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header X-NginX-Proxy true;
|
||||
|
||||
# This is necessary to pass the correct IP to be hashed
|
||||
real_ip_header X-Real-IP;
|
||||
|
||||
proxy_connect_timeout 300;
|
||||
|
||||
# To support websocket
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
|
||||
chunked_transfer_encoding off;
|
||||
|
||||
proxy_pass http://console;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
version: '3.7'
|
||||
|
||||
# Settings and configurations that are common for all containers
|
||||
x-minio-common: &minio-common
|
||||
image: quay.io/minio/minio:${RELEASE}
|
||||
command: server http://site1-minio{1...4}/data{1...2}
|
||||
environment:
|
||||
- MINIO_PROMETHEUS_AUTH_TYPE=public
|
||||
- CI=true
|
||||
|
||||
# starts 4 docker containers running minio server instances.
|
||||
# using nginx reverse proxy, load balancing, you can access
|
||||
# it through port 9000.
|
||||
services:
|
||||
site1-minio1:
|
||||
<<: *minio-common
|
||||
hostname: site1-minio1
|
||||
volumes:
|
||||
- site1-data1-1:/data1
|
||||
- site1-data1-2:/data2
|
||||
|
||||
site1-minio2:
|
||||
<<: *minio-common
|
||||
hostname: site1-minio2
|
||||
volumes:
|
||||
- site1-data2-1:/data1
|
||||
- site1-data2-2:/data2
|
||||
|
||||
site1-minio3:
|
||||
<<: *minio-common
|
||||
hostname: site1-minio3
|
||||
volumes:
|
||||
- site1-data3-1:/data1
|
||||
- site1-data3-2:/data2
|
||||
|
||||
site1-minio4:
|
||||
<<: *minio-common
|
||||
hostname: site1-minio4
|
||||
volumes:
|
||||
- site1-data4-1:/data1
|
||||
- site1-data4-2:/data2
|
||||
|
||||
site1-nginx:
|
||||
image: nginx:1.19.2-alpine
|
||||
hostname: site1-nginx
|
||||
volumes:
|
||||
- ./nginx-site1.conf:/etc/nginx/nginx.conf:ro
|
||||
ports:
|
||||
- "9001:9001"
|
||||
depends_on:
|
||||
- site1-minio1
|
||||
- site1-minio2
|
||||
- site1-minio3
|
||||
- site1-minio4
|
||||
|
||||
## By default this config uses default local driver,
|
||||
## For custom volumes replace with volume driver configuration.
|
||||
volumes:
|
||||
site1-data1-1:
|
||||
site1-data1-2:
|
||||
site1-data2-1:
|
||||
site1-data2-2:
|
||||
site1-data3-1:
|
||||
site1-data3-2:
|
||||
site1-data4-1:
|
||||
site1-data4-2:
|
||||
@@ -1,66 +0,0 @@
|
||||
version: '3.7'
|
||||
|
||||
# Settings and configurations that are common for all containers
|
||||
x-minio-common: &minio-common
|
||||
image: quay.io/minio/minio:${RELEASE}
|
||||
command: server http://site2-minio{1...4}/data{1...2}
|
||||
environment:
|
||||
- MINIO_PROMETHEUS_AUTH_TYPE=public
|
||||
- CI=true
|
||||
|
||||
# starts 4 docker containers running minio server instances.
|
||||
# using nginx reverse proxy, load balancing, you can access
|
||||
# it through port 9000.
|
||||
services:
|
||||
site2-minio1:
|
||||
<<: *minio-common
|
||||
hostname: site2-minio1
|
||||
volumes:
|
||||
- site2-data1-1:/data1
|
||||
- site2-data1-2:/data2
|
||||
|
||||
site2-minio2:
|
||||
<<: *minio-common
|
||||
hostname: site2-minio2
|
||||
volumes:
|
||||
- site2-data2-1:/data1
|
||||
- site2-data2-2:/data2
|
||||
|
||||
site2-minio3:
|
||||
<<: *minio-common
|
||||
hostname: site2-minio3
|
||||
volumes:
|
||||
- site2-data3-1:/data1
|
||||
- site2-data3-2:/data2
|
||||
|
||||
site2-minio4:
|
||||
<<: *minio-common
|
||||
hostname: site2-minio4
|
||||
volumes:
|
||||
- site2-data4-1:/data1
|
||||
- site2-data4-2:/data2
|
||||
|
||||
site2-nginx:
|
||||
image: nginx:1.19.2-alpine
|
||||
hostname: site2-nginx
|
||||
volumes:
|
||||
- ./nginx-site2.conf:/etc/nginx/nginx.conf:ro
|
||||
ports:
|
||||
- "9002:9002"
|
||||
depends_on:
|
||||
- site2-minio1
|
||||
- site2-minio2
|
||||
- site2-minio3
|
||||
- site2-minio4
|
||||
|
||||
## By default this config uses default local driver,
|
||||
## For custom volumes replace with volume driver configuration.
|
||||
volumes:
|
||||
site2-data1-1:
|
||||
site2-data1-2:
|
||||
site2-data2-1:
|
||||
site2-data2-2:
|
||||
site2-data3-1:
|
||||
site2-data3-2:
|
||||
site2-data4-1:
|
||||
site2-data4-2:
|
||||
@@ -1,147 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -x
|
||||
|
||||
## change working directory
|
||||
cd .github/workflows/multipart/
|
||||
|
||||
function cleanup() {
|
||||
docker-compose -f docker-compose-site1.yaml rm -s -f || true
|
||||
docker-compose -f docker-compose-site2.yaml rm -s -f || true
|
||||
for volume in $(docker volume ls -q | grep minio); do
|
||||
docker volume rm ${volume} || true
|
||||
done
|
||||
|
||||
docker system prune -f || true
|
||||
docker volume prune -f || true
|
||||
docker volume rm $(docker volume ls -q -f dangling=true) || true
|
||||
}
|
||||
|
||||
cleanup
|
||||
|
||||
if [ ! -f ./mc ]; then
|
||||
wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc &&
|
||||
chmod +x mc
|
||||
fi
|
||||
|
||||
export RELEASE=RELEASE.2023-08-29T23-07-35Z
|
||||
|
||||
docker-compose -f docker-compose-site1.yaml up -d
|
||||
docker-compose -f docker-compose-site2.yaml up -d
|
||||
|
||||
sleep 30s
|
||||
|
||||
./mc alias set site1 http://site1-nginx:9001 minioadmin minioadmin --api s3v4
|
||||
./mc alias set site2 http://site2-nginx:9002 minioadmin minioadmin --api s3v4
|
||||
|
||||
./mc ready site1/
|
||||
./mc ready site2/
|
||||
|
||||
./mc admin replicate add site1 site2
|
||||
./mc mb site1/testbucket/
|
||||
./mc cp -r --quiet /usr/bin site1/testbucket/
|
||||
|
||||
sleep 5
|
||||
|
||||
./s3-check-md5 -h
|
||||
|
||||
failed_count_site1=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
|
||||
failed_count_site2=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
|
||||
|
||||
if [ $failed_count_site1 -ne 0 ]; then
|
||||
echo "failed with multipart on site1 uploads"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $failed_count_site2 -ne 0 ]; then
|
||||
echo "failed with multipart on site2 uploads"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
./mc cp -r --quiet /usr/bin site1/testbucket/
|
||||
|
||||
sleep 5
|
||||
|
||||
failed_count_site1=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
|
||||
failed_count_site2=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
|
||||
|
||||
## we do not need to fail here, since we are going to test
|
||||
## upgrading to master, healing and being able to recover
|
||||
## the last version.
|
||||
if [ $failed_count_site1 -ne 0 ]; then
|
||||
echo "failed with multipart on site1 uploads ${failed_count_site1}"
|
||||
fi
|
||||
|
||||
if [ $failed_count_site2 -ne 0 ]; then
|
||||
echo "failed with multipart on site2 uploads ${failed_count_site2}"
|
||||
fi
|
||||
|
||||
export RELEASE=${1}
|
||||
|
||||
docker-compose -f docker-compose-site1.yaml up -d
|
||||
docker-compose -f docker-compose-site2.yaml up -d
|
||||
|
||||
./mc ready site1/
|
||||
./mc ready site2/
|
||||
|
||||
for i in $(seq 1 10); do
|
||||
# mc admin heal -r --remove when used against a LB endpoint
|
||||
# behaves flaky, let this run 10 times before giving up
|
||||
./mc admin heal -r --remove --json site1/ 2>&1 >/dev/null
|
||||
./mc admin heal -r --remove --json site2/ 2>&1 >/dev/null
|
||||
done
|
||||
|
||||
failed_count_site1=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
|
||||
failed_count_site2=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
|
||||
|
||||
if [ $failed_count_site1 -ne 0 ]; then
|
||||
echo "failed with multipart on site1 uploads"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $failed_count_site2 -ne 0 ]; then
|
||||
echo "failed with multipart on site2 uploads"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Add user group test
|
||||
./mc admin user add site1 site-replication-issue-user site-replication-issue-password
|
||||
./mc admin group add site1 site-replication-issue-group site-replication-issue-user
|
||||
|
||||
max_wait_attempts=30
|
||||
wait_interval=5
|
||||
|
||||
attempt=1
|
||||
while true; do
|
||||
diff <(./mc admin group info site1 site-replication-issue-group) <(./mc admin group info site2 site-replication-issue-group)
|
||||
|
||||
if [[ $? -eq 0 ]]; then
|
||||
echo "Outputs are consistent."
|
||||
break
|
||||
fi
|
||||
|
||||
remaining_attempts=$((max_wait_attempts - attempt))
|
||||
if ((attempt >= max_wait_attempts)); then
|
||||
echo "Outputs remain inconsistent after $max_wait_attempts attempts. Exiting with error."
|
||||
exit 1
|
||||
else
|
||||
echo "Outputs are inconsistent. Waiting for $wait_interval seconds (attempt $attempt/$max_wait_attempts)."
|
||||
sleep $wait_interval
|
||||
fi
|
||||
|
||||
((attempt++))
|
||||
done
|
||||
|
||||
status=$(./mc admin group info site1 site-replication-issue-group --json | jq .groupStatus | tr -d '"')
|
||||
|
||||
if [[ $status == "enabled" ]]; then
|
||||
echo "Success"
|
||||
else
|
||||
echo "Expected status: enabled, actual status: $status"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cleanup
|
||||
|
||||
## change working directory
|
||||
cd ../../../
|
||||
@@ -1,61 +0,0 @@
|
||||
user nginx;
|
||||
worker_processes auto;
|
||||
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
pid /var/run/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 4096;
|
||||
}
|
||||
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent" "$http_x_forwarded_for"';
|
||||
|
||||
access_log /var/log/nginx/access.log main;
|
||||
sendfile on;
|
||||
keepalive_timeout 65;
|
||||
|
||||
# include /etc/nginx/conf.d/*.conf;
|
||||
|
||||
upstream minio {
|
||||
server site1-minio1:9000;
|
||||
server site1-minio2:9000;
|
||||
server site1-minio3:9000;
|
||||
server site1-minio4:9000;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 9001;
|
||||
listen [::]:9001;
|
||||
server_name localhost;
|
||||
|
||||
# To allow special characters in headers
|
||||
ignore_invalid_headers off;
|
||||
# Allow any size file to be uploaded.
|
||||
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||
client_max_body_size 0;
|
||||
# To disable buffering
|
||||
proxy_buffering off;
|
||||
proxy_request_buffering off;
|
||||
|
||||
location / {
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_connect_timeout 300;
|
||||
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
chunked_transfer_encoding off;
|
||||
|
||||
proxy_pass http://minio;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
user nginx;
|
||||
worker_processes auto;
|
||||
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
pid /var/run/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 4096;
|
||||
}
|
||||
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent" "$http_x_forwarded_for"';
|
||||
|
||||
access_log /var/log/nginx/access.log main;
|
||||
sendfile on;
|
||||
keepalive_timeout 65;
|
||||
|
||||
# include /etc/nginx/conf.d/*.conf;
|
||||
|
||||
upstream minio {
|
||||
server site2-minio1:9000;
|
||||
server site2-minio2:9000;
|
||||
server site2-minio3:9000;
|
||||
server site2-minio4:9000;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 9002;
|
||||
listen [::]:9002;
|
||||
server_name localhost;
|
||||
|
||||
# To allow special characters in headers
|
||||
ignore_invalid_headers off;
|
||||
# Allow any size file to be uploaded.
|
||||
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||
client_max_body_size 0;
|
||||
# To disable buffering
|
||||
proxy_buffering off;
|
||||
proxy_request_buffering off;
|
||||
|
||||
location / {
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_connect_timeout 300;
|
||||
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
chunked_transfer_encoding off;
|
||||
|
||||
proxy_pass http://minio;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,11 @@ on:
|
||||
push:
|
||||
tags:
|
||||
- "RELEASE.*"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: "Release tag (e.g. RELEASE.2026-03-24T12-00-00Z)"
|
||||
required: true
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
@@ -27,7 +32,7 @@ jobs:
|
||||
- name: Compute release variables
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TAG="${GITHUB_REF_NAME}"
|
||||
TAG="${{ github.event.inputs.tag || github.ref_name }}"
|
||||
VERSION_HYPHEN="${TAG#RELEASE.}"
|
||||
PKG_VERSION="$(echo "${VERSION_HYPHEN}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')"
|
||||
if [ "${PKG_VERSION}" = "${VERSION_HYPHEN}" ]; then
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
name: MinIO advanced tests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
replication-test:
|
||||
name: Advanced Tests with Go ${{ matrix.go-version }}
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.24.x]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- name: Test Decom
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make test-decom
|
||||
|
||||
- name: Test ILM
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make test-ilm
|
||||
make test-ilm-transition
|
||||
|
||||
- name: Test PBAC
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make test-pbac
|
||||
|
||||
- name: Test Config File
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make test-configfile
|
||||
|
||||
- name: Test Replication
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make test-replication
|
||||
|
||||
- name: Test MinIO IDP for automatic site replication
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make test-site-replication-minio
|
||||
|
||||
- name: Test Versioning
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make test-versioning
|
||||
|
||||
- name: Test Multipart upload with failures
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make test-multipart
|
||||
@@ -1,34 +0,0 @@
|
||||
name: Root lockdown tests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.24.x]
|
||||
os: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- name: Start root lockdown tests
|
||||
run: |
|
||||
make test-root-disable
|
||||
@@ -1,9 +0,0 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIBKDCB26ADAgECAhB6vebGMUfKnmBKyqoApRSOMAUGAytlcDAbMRkwFwYDVQQD
|
||||
DBByb290QHBsYXkubWluLmlvMB4XDTIwMDQzMDE1MjIyNVoXDTI1MDQyOTE1MjIy
|
||||
NVowGzEZMBcGA1UEAwwQcm9vdEBwbGF5Lm1pbi5pbzAqMAUGAytlcAMhALzn735W
|
||||
fmSH/ghKs+4iPWziZMmWdiWr/sqvqeW+WwSxozUwMzAOBgNVHQ8BAf8EBAMCB4Aw
|
||||
EwYDVR0lBAwwCgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAFBgMrZXADQQDZOrGK
|
||||
b2ATkDlu2pTcP3LyhSBDpYh7V4TvjRkBTRgjkacCzwFLm+mh+7US8V4dBpIDsJ4u
|
||||
uWoF0y6vbLVGIlkG
|
||||
-----END CERTIFICATE-----
|
||||
@@ -1,3 +0,0 @@
|
||||
-----BEGIN PRIVATE KEY-----
|
||||
MC4CAQAwBQYDK2VwBCIEID9E7FSYWrMD+VjhI6q545cYT9YOyFxZb7UnjEepYDRc
|
||||
-----END PRIVATE KEY-----
|
||||
@@ -1,64 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
export MODE="$1"
|
||||
export ACCESS_KEY="$2"
|
||||
export SECRET_KEY="$3"
|
||||
export JOB_NAME="$4"
|
||||
export MINT_MODE="full"
|
||||
|
||||
docker system prune -f || true
|
||||
docker volume prune -f || true
|
||||
docker volume rm $(docker volume ls -f dangling=true) || true
|
||||
|
||||
## change working directory
|
||||
cd .github/workflows/mint
|
||||
|
||||
## always pull latest
|
||||
docker pull docker.io/minio/mint:edge
|
||||
|
||||
docker-compose -f minio-${MODE}.yaml up -d
|
||||
sleep 1m
|
||||
|
||||
docker system prune -f || true
|
||||
docker volume prune -f || true
|
||||
docker volume rm $(docker volume ls -q -f dangling=true) || true
|
||||
|
||||
# Stop two nodes, one of each pool, to check that all S3 calls work while quorum is still there
|
||||
[ "${MODE}" == "pools" ] && docker-compose -f minio-${MODE}.yaml stop minio2
|
||||
[ "${MODE}" == "pools" ] && docker-compose -f minio-${MODE}.yaml stop minio6
|
||||
|
||||
# Pause one node, to check that all S3 calls work while one node goes wrong
|
||||
[ "${MODE}" == "resiliency" ] && docker-compose -f minio-${MODE}.yaml pause minio4
|
||||
|
||||
docker run --rm --net=mint_default \
|
||||
--name="mint-${MODE}-${JOB_NAME}" \
|
||||
-e SERVER_ENDPOINT="nginx:9000" \
|
||||
-e ACCESS_KEY="${ACCESS_KEY}" \
|
||||
-e SECRET_KEY="${SECRET_KEY}" \
|
||||
-e ENABLE_HTTPS=0 \
|
||||
-e MINT_MODE="${MINT_MODE}" \
|
||||
docker.io/minio/mint:edge
|
||||
|
||||
# FIXME: enable this after fixing aws-sdk-java-v2 tests
|
||||
# # unpause the node, to check that all S3 calls work while one node goes wrong
|
||||
# [ "${MODE}" == "resiliency" ] && docker-compose -f minio-${MODE}.yaml unpause minio4
|
||||
# [ "${MODE}" == "resiliency" ] && docker run --rm --net=mint_default \
|
||||
# --name="mint-${MODE}-${JOB_NAME}" \
|
||||
# -e SERVER_ENDPOINT="nginx:9000" \
|
||||
# -e ACCESS_KEY="${ACCESS_KEY}" \
|
||||
# -e SECRET_KEY="${SECRET_KEY}" \
|
||||
# -e ENABLE_HTTPS=0 \
|
||||
# -e MINT_MODE="${MINT_MODE}" \
|
||||
# docker.io/minio/mint:edge
|
||||
|
||||
docker-compose -f minio-${MODE}.yaml down || true
|
||||
sleep 10s
|
||||
|
||||
docker system prune -f || true
|
||||
docker volume prune -f || true
|
||||
docker volume rm $(docker volume ls -q -f dangling=true) || true
|
||||
|
||||
## change working directory
|
||||
cd ../../../
|
||||
@@ -1,22 +0,0 @@
|
||||
name: Shell formatting checks
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: runner / shfmt
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: luizm/action-sh-checker@master
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SHFMT_OPTS: "-s"
|
||||
with:
|
||||
sh_checker_shellcheck_disable: true # disable for now
|
||||
@@ -1,15 +0,0 @@
|
||||
---
|
||||
name: Spelling
|
||||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
run:
|
||||
name: Spell Check with Typos
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout Actions Repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Check spelling of repo
|
||||
uses: crate-ci/typos@master
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
name: Upgrade old version tests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.24.x]
|
||||
os: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- name: Start upgrade tests
|
||||
run: |
|
||||
make test-upgrade
|
||||
@@ -1,31 +0,0 @@
|
||||
name: VulnCheck
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
|
||||
permissions:
|
||||
contents: read # to fetch code (actions/checkout)
|
||||
|
||||
jobs:
|
||||
vulncheck:
|
||||
name: Analysis
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v4
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 1.24.x
|
||||
cached: false
|
||||
- name: Get official govulncheck
|
||||
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
shell: bash
|
||||
- name: Run govulncheck
|
||||
run: govulncheck -show verbose ./...
|
||||
shell: bash
|
||||
+63
-1
@@ -1,3 +1,62 @@
|
||||
FROM golang:1.26.4-alpine AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
|
||||
ENV GOPATH=/go
|
||||
ENV CGO_ENABLED=0
|
||||
|
||||
ARG MC_REPO=pgsty/mc
|
||||
ARG MC_VERSION=latest
|
||||
|
||||
RUN apk add -U --no-cache \
|
||||
ca-certificates \
|
||||
bash \
|
||||
curl \
|
||||
jq && \
|
||||
case "${TARGETARCH}" in \
|
||||
amd64) MC_ARCH=amd64 ;; \
|
||||
arm64) MC_ARCH=arm64 ;; \
|
||||
*) echo "Unsupported TARGETARCH=${TARGETARCH}"; exit 1 ;; \
|
||||
esac && \
|
||||
if [ "${MC_VERSION}" = "latest" ]; then \
|
||||
MC_RELEASE_URL="https://api.github.com/repos/${MC_REPO}/releases/latest"; \
|
||||
else \
|
||||
MC_RELEASE_URL="https://api.github.com/repos/${MC_REPO}/releases/tags/${MC_VERSION}"; \
|
||||
fi && \
|
||||
curl -fsSL "${MC_RELEASE_URL}" -o /tmp/mc-release.json && \
|
||||
MC_ARCHIVE_URL=$(jq -r --arg arch "${MC_ARCH}" \
|
||||
'.assets[] | select(.name | endswith("_linux_" + $arch + ".tar.gz")) | .browser_download_url' \
|
||||
/tmp/mc-release.json | head -n 1) && \
|
||||
MC_CHECKSUM_URL=$(jq -r \
|
||||
'.assets[] | select(.name | endswith("_checksums.txt")) | .browser_download_url' \
|
||||
/tmp/mc-release.json | head -n 1) && \
|
||||
[ -n "${MC_ARCHIVE_URL}" ] || { echo "Cannot find mcli archive for linux/${MC_ARCH}"; exit 1; } && \
|
||||
[ -n "${MC_CHECKSUM_URL}" ] || { echo "Cannot find mcli checksums file"; exit 1; } && \
|
||||
ARCHIVE_NAME=$(basename "${MC_ARCHIVE_URL}") && \
|
||||
echo "Downloading ${ARCHIVE_NAME} ..." && \
|
||||
curl -fsSL "${MC_ARCHIVE_URL}" -o /tmp/mcli.tar.gz && \
|
||||
curl -fsSL "${MC_CHECKSUM_URL}" -o /tmp/mcli_checksums.txt && \
|
||||
EXPECTED=$(grep " ${ARCHIVE_NAME}$" /tmp/mcli_checksums.txt | awk '{print $1}') && \
|
||||
ACTUAL=$(sha256sum /tmp/mcli.tar.gz | awk '{print $1}') && \
|
||||
[ -n "${EXPECTED}" ] || { echo "Checksum entry not found for ${ARCHIVE_NAME}"; exit 1; } && \
|
||||
[ "${EXPECTED}" = "${ACTUAL}" ] || { echo "Checksum mismatch: expected ${EXPECTED}, got ${ACTUAL}"; exit 1; } && \
|
||||
echo "Checksum OK: ${ACTUAL}" && \
|
||||
mkdir -p /tmp/mcli-extract && \
|
||||
tar -xzf /tmp/mcli.tar.gz -C /tmp/mcli-extract/ && \
|
||||
if [ -f /tmp/mcli-extract/mcli ]; then \
|
||||
cp /tmp/mcli-extract/mcli /go/bin/mcli; \
|
||||
elif [ -f /tmp/mcli-extract/mc ]; then \
|
||||
cp /tmp/mcli-extract/mc /go/bin/mcli; \
|
||||
else \
|
||||
echo "No mc or mcli binary found in archive:"; ls -la /tmp/mcli-extract/; exit 1; \
|
||||
fi && \
|
||||
chmod +x /go/bin/mcli && \
|
||||
ln -sf mcli /go/bin/mc
|
||||
|
||||
COPY dockerscripts/download-static-curl.sh /build/download-static-curl
|
||||
RUN chmod +x /build/download-static-curl && \
|
||||
/build/download-static-curl
|
||||
|
||||
FROM registry.access.redhat.com/ubi9/ubi:latest AS certs
|
||||
RUN dnf -y install ca-certificates && \
|
||||
update-ca-trust && \
|
||||
@@ -21,11 +80,14 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
|
||||
COPY --from=certs /tmp/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
COPY minio /usr/bin/minio
|
||||
COPY --from=build /go/bin/mcli /usr/bin/mcli
|
||||
COPY --from=build /go/bin/curl* /usr/bin/
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
COPY LICENSE /licenses/LICENSE
|
||||
COPY CREDITS /licenses/CREDITS
|
||||
|
||||
RUN chmod +x /usr/bin/minio /usr/bin/docker-entrypoint.sh
|
||||
RUN chmod +x /usr/bin/minio /usr/bin/mcli /usr/bin/docker-entrypoint.sh && \
|
||||
ln -sf mcli /usr/bin/mc
|
||||
|
||||
EXPOSE 9000
|
||||
VOLUME ["/data"]
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.24-alpine as build
|
||||
FROM golang:1.26.4-alpine as build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.24-alpine AS build
|
||||
FROM golang:1.26.4-alpine AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.24-alpine AS build
|
||||
FROM golang:1.26.4-alpine AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
|
||||
@@ -32,7 +32,7 @@ verifiers: lint check-gen
|
||||
|
||||
check-gen: ## check for updated autogenerated files
|
||||
@go generate ./... >/dev/null
|
||||
@go mod tidy -compat=1.21
|
||||
@go mod tidy -compat=1.26
|
||||
@(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false)
|
||||
@(! git diff --name-only | grep 'go.sum') || (echo "Non-committed changes in auto-generated go.sum is detected, please commit them to proceed." && false)
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
[](https://hub.docker.com/r/pgsty/minio)
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **This is a community-maintained fork of [minio/minio](https://github.com/minio/minio), maintained by [Pigsty](https://pigsty.io).**
|
||||
> **This is a community-maintained fork of the upstream MinIO project, maintained by [Pigsty](https://pigsty.io).**
|
||||
> This project is **NOT** affiliated with, endorsed by, or sponsored by MinIO, Inc.
|
||||
> "MinIO" is a trademark of MinIO, Inc., used here solely to identify the upstream project.
|
||||
>
|
||||
@@ -30,3 +30,14 @@ Ansible Deployment: [https://pigsty.io/docs/minio](https://pigsty.io/docs/minio)
|
||||
|
||||
APT/YUM repo for `minio` and `mcli` binary: [https://pigsty.io/docs/infra](https://pigsty.io/docs/repo/infra/list/#object-storage)
|
||||
|
||||
## Kubernetes delivery and scope
|
||||
|
||||
This repository maintains the direct-deployment Helm chart in [`helm/minio`](helm/minio/README.md) for standalone and distributed Silo server deployments. Chart defaults use the version- and digest-pinned multi-architecture [`pgsty/minio`](https://hub.docker.com/r/pgsty/minio) image, and every chart change is checked by a real Kind install plus an S3 write/read/delete smoke test.
|
||||
|
||||
```bash
|
||||
helm repo add silo https://raw.githubusercontent.com/pgsty/minio/master
|
||||
helm repo update silo
|
||||
helm install my-release silo/minio --namespace minio --create-namespace
|
||||
```
|
||||
|
||||
The MinIO Operator, Tenant CRDs, and operator lifecycle are **not maintained by this repository**. There is no Pigsty/Silo operator fork here. Users who require an operator must select and validate a separate operator implementation; issues and pull requests for operator code should be filed with that implementation.
|
||||
|
||||
+10
-32
@@ -1,42 +1,20 @@
|
||||
# Security Policy
|
||||
|
||||
This repository is the `pgsty/minio` community fork of `minio/minio`. Upstream MinIO security contacts do not handle fork-specific fixes or release notes for this repository.
|
||||
|
||||
## Supported Versions
|
||||
|
||||
We always provide security updates for the [latest release](https://github.com/minio/minio/releases/latest).
|
||||
Whenever there is a security update you just need to upgrade to the latest version.
|
||||
Security fixes are tracked on the active `master` branch and summarized in [docs/security/advisories.md](docs/security/advisories.md).
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
All security bugs in [minio/minio](https://github,com/minio/minio) (or other minio/* repositories)
|
||||
should be reported by email to security@min.io. Your email will be acknowledged within 48 hours,
|
||||
and you'll receive a more detailed response to your email within 72 hours indicating the next steps
|
||||
in handling your report.
|
||||
For vulnerabilities in this fork:
|
||||
|
||||
Please, provide a detailed explanation of the issue. In particular, outline the type of the security
|
||||
issue (DoS, authentication bypass, information disclose, ...) and the assumptions you're making (e.g. do
|
||||
you need access credentials for a successful exploit).
|
||||
1. Follow the fork-specific expectations in [VULNERABILITY_REPORT.md](VULNERABILITY_REPORT.md).
|
||||
2. Prefer the `pgsty/minio` repository's GitHub security reporting workflow when it is available.
|
||||
3. If private reporting is not available, contact the maintainers through the `pgsty/minio` repository before publishing detailed exploit information.
|
||||
4. If you confirm the issue also affects upstream `minio/minio`, report it upstream separately.
|
||||
|
||||
If you have not received a reply to your email within 48 hours or you have not heard from the security team
|
||||
for the past five days please contact the security team directly:
|
||||
## Disclosure Process
|
||||
|
||||
- Primary security coordinator: aead@min.io
|
||||
- Secondary coordinator: harsha@min.io
|
||||
- If you receive no response: dev@min.io
|
||||
|
||||
### Disclosure Process
|
||||
|
||||
MinIO uses the following disclosure process:
|
||||
|
||||
1. Once the security report is received one member of the security team tries to verify and reproduce
|
||||
the issue and determines the impact it has.
|
||||
2. A member of the security team will respond and either confirm or reject the security report.
|
||||
If the report is rejected the response explains why.
|
||||
3. Code is audited to find any potential similar problems.
|
||||
4. Fixes are prepared for the latest release.
|
||||
5. On the date that the fixes are applied a security advisory will be published on <https://blog.min.io>.
|
||||
Please inform us in your report email whether MinIO should mention your contribution w.r.t. fixing
|
||||
the security issue. By default MinIO will **not** publish this information to protect your privacy.
|
||||
|
||||
This process can take some time, especially when coordination is required with maintainers of other projects.
|
||||
Every effort will be made to handle the bug in as timely a manner as possible, however it's important that we
|
||||
follow the process described above to ensure that disclosures are handled consistently.
|
||||
Fork-specific fixes and user-visible upgrade notes are published in [docs/security/advisories.md](docs/security/advisories.md). The fork-specific triage and remediation process is described in [VULNERABILITY_REPORT.md](VULNERABILITY_REPORT.md).
|
||||
|
||||
+19
-20
@@ -1,38 +1,37 @@
|
||||
# Vulnerability Management Policy
|
||||
|
||||
This document formally describes the process of addressing and managing a
|
||||
reported vulnerability that has been found in the MinIO server code base,
|
||||
any directly connected ecosystem component or a direct / indirect dependency
|
||||
of the code base.
|
||||
This document describes how the `pgsty/minio` maintainers investigate,
|
||||
assess, and remediate reported vulnerabilities affecting this fork, any
|
||||
directly shipped component, or a direct / indirect dependency used by this
|
||||
repository.
|
||||
|
||||
## Scope
|
||||
|
||||
The vulnerability management policy described in this document covers the
|
||||
process of investigating, assessing and resolving a vulnerability report
|
||||
opened by a MinIO employee or an external third party.
|
||||
This policy covers vulnerability reports opened by repository maintainers or
|
||||
external third parties against `pgsty/minio` itself, its release artifacts, or
|
||||
dependencies that materially affect this fork.
|
||||
|
||||
Therefore, it lists pre-conditions and actions that should be performed to
|
||||
resolve and fix a reported vulnerability.
|
||||
It defines the information needed for triage and the expected remediation
|
||||
workflow for supported fixes.
|
||||
|
||||
## Vulnerability Management Process
|
||||
|
||||
The vulnerability management process requires that the vulnerability report
|
||||
contains the following information:
|
||||
A useful vulnerability report should contain the following information:
|
||||
|
||||
- The project / component that contains the reported vulnerability.
|
||||
- A description of the vulnerability. In particular, the type of the
|
||||
reported vulnerability and how it might be exploited. Alternatively,
|
||||
a well-established vulnerability identifier, e.g. CVE number, can be
|
||||
used instead.
|
||||
reported vulnerability and how it might be exploited. Alternatively,
|
||||
a well-established vulnerability identifier, such as a CVE or GHSA ID, can
|
||||
be used instead.
|
||||
|
||||
Based on the description mentioned above, a MinIO engineer or security team
|
||||
member investigates:
|
||||
Based on the report, the `pgsty/minio` maintainers investigate:
|
||||
|
||||
- Whether the reported vulnerability exists.
|
||||
- The conditions that are required such that the vulnerability can be exploited.
|
||||
- Which releases, branches, or deployment paths are affected.
|
||||
- The steps required to fix the vulnerability.
|
||||
|
||||
In general, if the vulnerability exists in one of the MinIO code bases
|
||||
itself - not in a code dependency - then MinIO will, if possible, fix
|
||||
the vulnerability or implement reasonable countermeasures such that the
|
||||
vulnerability cannot be exploited anymore.
|
||||
If the vulnerability exists in this fork itself, the maintainers will, when
|
||||
feasible, fix the issue or implement reasonable countermeasures such that the
|
||||
vulnerability can no longer be exploited. Fork-specific upgrade notes and
|
||||
security advisories are published in `docs/security/advisories.md`.
|
||||
|
||||
@@ -267,7 +267,7 @@ func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.R
|
||||
lookupResult, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(targetUser)
|
||||
if err != nil {
|
||||
// if not found, check if DN
|
||||
if strings.Contains(err.Error(), "User DN not found for:") {
|
||||
if strings.Contains(strings.ToLower(err.Error()), "user dn not found for:") {
|
||||
if isDN {
|
||||
// warn user that DNs are not allowed
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminLDAPExpectedLoginName, err), r.URL)
|
||||
|
||||
@@ -903,7 +903,7 @@ func (z *ReplicationState) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
return
|
||||
}
|
||||
var za0004 VersionPurgeStatusType
|
||||
err = za0004.DecodeMsg(dc)
|
||||
err = (*replication.VersionPurgeStatusType)(&za0004).DecodeMsg(dc)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "PurgeTargets", za0003)
|
||||
return
|
||||
@@ -1060,7 +1060,7 @@ func (z *ReplicationState) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
err = msgp.WrapError(err, "PurgeTargets")
|
||||
return
|
||||
}
|
||||
err = za0004.EncodeMsg(en)
|
||||
err = (*replication.VersionPurgeStatusType)(&za0004).EncodeMsg(en)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "PurgeTargets", za0003)
|
||||
return
|
||||
@@ -1136,7 +1136,7 @@ func (z *ReplicationState) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.AppendMapHeader(o, uint32(len(z.PurgeTargets)))
|
||||
for za0003, za0004 := range z.PurgeTargets {
|
||||
o = msgp.AppendString(o, za0003)
|
||||
o, err = za0004.MarshalMsg(o)
|
||||
o, err = (*replication.VersionPurgeStatusType)(&za0004).MarshalMsg(o)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "PurgeTargets", za0003)
|
||||
return
|
||||
@@ -1261,7 +1261,7 @@ func (z *ReplicationState) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
err = msgp.WrapError(err, "PurgeTargets")
|
||||
return
|
||||
}
|
||||
bts, err = za0004.UnmarshalMsg(bts)
|
||||
bts, err = (*replication.VersionPurgeStatusType)(&za0004).UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "PurgeTargets", za0003)
|
||||
return
|
||||
@@ -1321,7 +1321,7 @@ func (z *ReplicationState) Msgsize() (s int) {
|
||||
if z.PurgeTargets != nil {
|
||||
for za0003, za0004 := range z.PurgeTargets {
|
||||
_ = za0004
|
||||
s += msgp.StringPrefixSize + len(za0003) + za0004.Msgsize()
|
||||
s += msgp.StringPrefixSize + len(za0003) + (*replication.VersionPurgeStatusType)(&za0004).Msgsize()
|
||||
}
|
||||
}
|
||||
s += 17 + msgp.MapHeaderSize
|
||||
|
||||
@@ -177,7 +177,7 @@ func formatGetBackendErasureVersion(b []byte) (string, error) {
|
||||
return "", fmt.Errorf(`format.Version expected: %s, got: %s`, formatMetaVersionV1, meta.Version)
|
||||
}
|
||||
if meta.Format != formatBackendErasure && meta.Format != formatBackendErasureSingle {
|
||||
return "", fmt.Errorf(`found backend type %s, expected %s or %s - to migrate to a supported backend visit https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-migrate-fs-gateway.html`, meta.Format, formatBackendErasure, formatBackendErasureSingle)
|
||||
return "", fmt.Errorf(`found backend type %s, expected %s or %s - to migrate to a supported backend visit https://silo.pigsty.io/operations/deployments/baremetal-migrate-fs-gateway.html`, meta.Format, formatBackendErasure, formatBackendErasureSingle)
|
||||
}
|
||||
// Erasure backend found, proceed to detect version.
|
||||
format := &formatErasureVersionDetect{}
|
||||
|
||||
@@ -192,6 +192,16 @@ func extractMetadata(ctx context.Context, mimesHeader ...textproto.MIMEHeader) (
|
||||
|
||||
// extractMetadata extracts metadata from map values.
|
||||
func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error {
|
||||
return extractMetadataFromMimeWithReplication(ctx, v, m, false)
|
||||
}
|
||||
|
||||
// extractReplicationMetadataFromMime restores replication-only metadata after the
|
||||
// caller has validated that the request is a trusted replication write.
|
||||
func extractReplicationMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error {
|
||||
return extractMetadataFromMimeWithReplication(ctx, v, m, true)
|
||||
}
|
||||
|
||||
func extractMetadataFromMimeWithReplication(ctx context.Context, v textproto.MIMEHeader, m map[string]string, allowReplication bool) error {
|
||||
if v == nil {
|
||||
bugLogIf(ctx, errInvalidArgument)
|
||||
return errInvalidArgument
|
||||
@@ -208,6 +218,9 @@ func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[
|
||||
value, ok := nv[http.CanonicalHeaderKey(supportedHeader)]
|
||||
if ok {
|
||||
if v, ok := replicationToInternalHeaders[supportedHeader]; ok {
|
||||
if !allowReplication {
|
||||
continue
|
||||
}
|
||||
m[v] = strings.Join(value, ",")
|
||||
} else {
|
||||
m[supportedHeader] = strings.Join(value, ",")
|
||||
|
||||
@@ -24,11 +24,13 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/config"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
// Tests validate bucket LocationConstraint.
|
||||
@@ -152,6 +154,22 @@ func TestExtractMetadataHeaders(t *testing.T) {
|
||||
},
|
||||
shouldFail: false,
|
||||
},
|
||||
// Replication-only headers must not be accepted on ordinary requests.
|
||||
{
|
||||
header: http.Header{
|
||||
"Content-Type": []string{"image/png"},
|
||||
"X-Minio-Replication-Server-Side-Encryption-Sealed-Key": []string{"sealed-key"},
|
||||
"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm": []string{"DAREv2-HMAC-SHA256"},
|
||||
"X-Minio-Replication-Server-Side-Encryption-Iv": []string{"iv"},
|
||||
"X-Minio-Replication-Encrypted-Multipart": []string{""},
|
||||
"X-Minio-Replication-Actual-Object-Size": []string{"1"},
|
||||
ReplicationSsecChecksumHeader: []string{"checksum"},
|
||||
},
|
||||
metadata: map[string]string{
|
||||
"content-type": "image/png",
|
||||
},
|
||||
shouldFail: false,
|
||||
},
|
||||
// Empty header input returns empty metadata.
|
||||
{
|
||||
header: nil,
|
||||
@@ -176,6 +194,104 @@ func TestExtractMetadataHeaders(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractReplicationMetadataHeaders(t *testing.T) {
|
||||
header := http.Header{
|
||||
"X-Minio-Replication-Server-Side-Encryption-Sealed-Key": []string{"sealed-key"},
|
||||
"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm": []string{"DAREv2-HMAC-SHA256"},
|
||||
"X-Minio-Replication-Server-Side-Encryption-Iv": []string{"iv"},
|
||||
"X-Minio-Replication-Encrypted-Multipart": []string{""},
|
||||
"X-Minio-Replication-Actual-Object-Size": []string{"1"},
|
||||
ReplicationSsecChecksumHeader: []string{"checksum"},
|
||||
}
|
||||
|
||||
metadata := make(map[string]string)
|
||||
if err := extractReplicationMetadataFromMime(t.Context(), textproto.MIMEHeader(header), metadata); err != nil {
|
||||
t.Fatalf("failed to extract replication metadata: %v", err)
|
||||
}
|
||||
|
||||
expected := map[string]string{
|
||||
"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "sealed-key",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Iv": "iv",
|
||||
"X-Minio-Internal-Encrypted-Multipart": "",
|
||||
"X-Minio-Internal-Actual-Object-Size": "1",
|
||||
ReplicationSsecChecksumHeader: "checksum",
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(metadata, expected) {
|
||||
t.Fatalf("unexpected replication metadata: expected %#v, got %#v", expected, metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetCopyObjectMetadataFromHeaderReplication(t *testing.T) {
|
||||
req, err := http.NewRequest(http.MethodPut, "http://localhost/test", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Form = make(url.Values)
|
||||
req.Header.Set("X-Amz-Metadata-Directive", replaceDirective)
|
||||
req.Header.Set("X-Minio-Replication-Server-Side-Encryption-Sealed-Key", "sealed-key")
|
||||
|
||||
metadata, err := getCpObjMetadataFromHeader(t.Context(), req, nil, false)
|
||||
if err != nil {
|
||||
t.Fatalf("copy metadata extraction failed: %v", err)
|
||||
}
|
||||
if _, ok := metadata["X-Minio-Internal-Server-Side-Encryption-Sealed-Key"]; ok {
|
||||
t.Fatalf("unexpected replication metadata without validation: %#v", metadata)
|
||||
}
|
||||
|
||||
metadata, err = getCpObjMetadataFromHeader(t.Context(), req, nil, true)
|
||||
if err != nil {
|
||||
t.Fatalf("copy metadata extraction with replication failed: %v", err)
|
||||
}
|
||||
if got := metadata["X-Minio-Internal-Server-Side-Encryption-Sealed-Key"]; got != "sealed-key" {
|
||||
t.Fatalf("expected restored replication metadata, got %#v", metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloneRequestWithoutCopyReplicationHeaders(t *testing.T) {
|
||||
req, err := http.NewRequest(http.MethodPut, "http://localhost/test", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Header.Set(xhttp.MinIOSourceReplicationRequest, "true")
|
||||
req.Header.Set(xhttp.MinIOSourceETag, "etag")
|
||||
req.Header.Set(xhttp.MinIOSourceMTime, "2026-04-15T10:00:00Z")
|
||||
req.Header.Set(xhttp.MinIOSourceTaggingTimestamp, "2026-04-15T10:00:00Z")
|
||||
req.Header.Set(xhttp.MinIOSourceObjectRetentionTimestamp, "2026-04-15T10:00:00Z")
|
||||
req.Header.Set(xhttp.MinIOSourceObjectLegalHoldTimestamp, "2026-04-15T10:00:00Z")
|
||||
req.Header.Set(xhttp.MinIOReplicationActualObjectSize, "123")
|
||||
req.Header.Set(ReplicationSsecChecksumHeader, "checksum")
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
|
||||
clone := cloneRequestWithoutCopyReplicationHeaders(req)
|
||||
if clone == req {
|
||||
t.Fatal("expected cloned request")
|
||||
}
|
||||
|
||||
for _, header := range []string{
|
||||
xhttp.MinIOSourceReplicationRequest,
|
||||
xhttp.MinIOSourceETag,
|
||||
xhttp.MinIOSourceMTime,
|
||||
xhttp.MinIOSourceTaggingTimestamp,
|
||||
xhttp.MinIOSourceObjectRetentionTimestamp,
|
||||
xhttp.MinIOSourceObjectLegalHoldTimestamp,
|
||||
xhttp.MinIOReplicationActualObjectSize,
|
||||
ReplicationSsecChecksumHeader,
|
||||
} {
|
||||
if got := clone.Header.Get(header); got != "" {
|
||||
t.Fatalf("expected %s to be stripped, got %q", header, got)
|
||||
}
|
||||
if got := req.Header.Get(header); got == "" {
|
||||
t.Fatalf("expected original request to preserve %s", header)
|
||||
}
|
||||
}
|
||||
|
||||
if got := clone.Header.Get("Content-Type"); got != "application/octet-stream" {
|
||||
t.Fatalf("expected non-replication headers to be preserved, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Test getResource()
|
||||
func TestGetResource(t *testing.T) {
|
||||
testCases := []struct {
|
||||
|
||||
@@ -576,7 +576,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
|
||||
if took := time.Since(listStartTime); took > maxIAMLoadOpTime {
|
||||
var s strings.Builder
|
||||
for k, v := range listedConfigItems {
|
||||
s.WriteString(fmt.Sprintf(" %s: %d items\n", k, len(v)))
|
||||
fmt.Fprintf(&s, " %s: %d items\n", k, len(v))
|
||||
}
|
||||
logger.Info("listAllIAMConfigItems took %.2fs with contents:\n%s", took.Seconds(), s.String())
|
||||
}
|
||||
|
||||
@@ -324,14 +324,6 @@ func (d *naughtyDisk) StatInfoFile(ctx context.Context, volume, path string, glo
|
||||
return d.disk.StatInfoFile(ctx, volume, path, glob)
|
||||
}
|
||||
|
||||
func (d *naughtyDisk) ReadMultiple(ctx context.Context, req ReadMultipleReq, resp chan<- ReadMultipleResp) error {
|
||||
if err := d.calcError(); err != nil {
|
||||
close(resp)
|
||||
return err
|
||||
}
|
||||
return d.disk.ReadMultiple(ctx, req, resp)
|
||||
}
|
||||
|
||||
func (d *naughtyDisk) CleanAbandonedData(ctx context.Context, volume string, path string) error {
|
||||
if err := d.calcError(); err != nil {
|
||||
return err
|
||||
|
||||
+75
-11
@@ -1036,7 +1036,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
|
||||
// Extract metadata relevant for an CopyObject operation based on conditional
|
||||
// header values specified in X-Amz-Metadata-Directive.
|
||||
func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta map[string]string) (map[string]string, error) {
|
||||
func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta map[string]string, allowReplication bool) (map[string]string, error) {
|
||||
// Make a copy of the supplied metadata to avoid
|
||||
// to change the original one.
|
||||
defaultMeta := make(map[string]string, len(userMeta))
|
||||
@@ -1067,6 +1067,11 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if allowReplication {
|
||||
if err = extractReplicationMetadataFromMime(ctx, textproto.MIMEHeader(r.Header), emetadata); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if sc != "" {
|
||||
emetadata[xhttp.AmzStorageClass] = sc
|
||||
}
|
||||
@@ -1087,6 +1092,31 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m
|
||||
return defaultMeta, nil
|
||||
}
|
||||
|
||||
func cloneRequestWithoutCopyReplicationHeaders(r *http.Request) *http.Request {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
clone := new(http.Request)
|
||||
*clone = *r
|
||||
clone.Header = r.Header.Clone()
|
||||
|
||||
for _, header := range []string{
|
||||
xhttp.MinIOSourceReplicationRequest,
|
||||
xhttp.MinIOSourceETag,
|
||||
xhttp.MinIOSourceMTime,
|
||||
xhttp.MinIOSourceTaggingTimestamp,
|
||||
xhttp.MinIOSourceObjectRetentionTimestamp,
|
||||
xhttp.MinIOSourceObjectLegalHoldTimestamp,
|
||||
xhttp.MinIOReplicationActualObjectSize,
|
||||
ReplicationSsecChecksumHeader,
|
||||
} {
|
||||
clone.Header.Del(header)
|
||||
}
|
||||
|
||||
return clone
|
||||
}
|
||||
|
||||
// getRemoteInstanceTransport contains a roundtripper for external (not peers) servers
|
||||
var remoteInstanceTransport atomic.Value
|
||||
|
||||
@@ -1232,6 +1262,19 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidStorageClass), r.URL)
|
||||
return
|
||||
}
|
||||
allowReplicationMetadata := false
|
||||
if r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() {
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ReplicateObjectAction, dstBucket, dstObject); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
allowReplicationMetadata = true
|
||||
}
|
||||
trustedReplicationRequest := allowReplicationMetadata && r.Header.Get(xhttp.MinIOSourceReplicationRequest) == "true"
|
||||
optsReq := r
|
||||
if !trustedReplicationRequest {
|
||||
optsReq = cloneRequestWithoutCopyReplicationHeaders(r)
|
||||
}
|
||||
|
||||
// Check if bucket encryption is enabled
|
||||
sseConfig, _ := globalBucketSSEConfigSys.Get(dstBucket)
|
||||
@@ -1240,7 +1283,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
})
|
||||
|
||||
var srcOpts, dstOpts ObjectOptions
|
||||
srcOpts, err = copySrcOpts(ctx, r, srcBucket, srcObject)
|
||||
srcOpts, err = copySrcOpts(ctx, optsReq, srcBucket, srcObject)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -1252,14 +1295,14 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
VersionID: srcOpts.VersionID,
|
||||
Versioned: srcOpts.Versioned,
|
||||
VersionSuspended: srcOpts.VersionSuspended,
|
||||
ReplicationRequest: r.Header.Get(xhttp.MinIOSourceReplicationRequest) == "true",
|
||||
ReplicationRequest: trustedReplicationRequest,
|
||||
}
|
||||
getSSE := encrypt.SSE(srcOpts.ServerSideEncryption)
|
||||
if getSSE != srcOpts.ServerSideEncryption {
|
||||
getOpts.ServerSideEncryption = getSSE
|
||||
}
|
||||
|
||||
dstOpts, err = copyDstOpts(ctx, r, dstBucket, dstObject, nil)
|
||||
dstOpts, err = copyDstOpts(ctx, optsReq, dstBucket, dstObject, nil)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -1269,7 +1312,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
getObjectNInfo := objectAPI.GetObjectNInfo
|
||||
|
||||
checkCopyPrecondFn := func(o ObjectInfo) bool {
|
||||
if _, err := DecryptObjectInfo(&o, r); err != nil {
|
||||
if _, err := DecryptObjectInfo(&o, optsReq); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return true
|
||||
}
|
||||
@@ -1380,7 +1423,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
return
|
||||
}
|
||||
// Encryption parameters not present for this object.
|
||||
if crypto.SSEC.IsEncrypted(srcInfo.UserDefined) && !crypto.SSECopy.IsRequested(r.Header) && r.Header.Get(xhttp.MinIOSourceReplicationRequest) != "true" {
|
||||
if crypto.SSEC.IsEncrypted(srcInfo.UserDefined) && !crypto.SSECopy.IsRequested(r.Header) && !trustedReplicationRequest {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidSSECustomerAlgorithm), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -1546,7 +1589,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
|
||||
srcInfo.PutObjReader = pReader
|
||||
|
||||
srcInfo.UserDefined, err = getCpObjMetadataFromHeader(ctx, r, srcInfo.UserDefined)
|
||||
srcInfo.UserDefined, err = getCpObjMetadataFromHeader(ctx, r, srcInfo.UserDefined, allowReplicationMetadata)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -1628,10 +1671,10 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
if rs := r.Header.Get(xhttp.AmzBucketReplicationStatus); rs != "" {
|
||||
if allowReplicationMetadata {
|
||||
srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicaStatus] = replication.Replica.String()
|
||||
srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicaTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
srcInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = rs
|
||||
srcInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replication.Replica.String()
|
||||
}
|
||||
|
||||
op := replication.ObjectReplicationType
|
||||
@@ -1896,7 +1939,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
case authTypeStreamingUnsignedTrailer:
|
||||
// Initialize stream chunked reader with optional trailers.
|
||||
rd, s3Err = newUnsignedV4ChunkedReader(r, true, r.Header.Get(xhttp.Authorization) != "")
|
||||
rd, s3Err = newUnsignedV4ChunkedReader(r, true)
|
||||
if s3Err != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
@@ -1933,6 +1976,10 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
if err = extractReplicationMetadataFromMime(ctx, textproto.MIMEHeader(r.Header), metadata); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
metadata[ReservedMetadataPrefixLower+ReplicaStatus] = replication.Replica.String()
|
||||
metadata[ReservedMetadataPrefixLower+ReplicaTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
defer globalReplicationStats.Load().UpdateReplicaStat(bucket, size)
|
||||
@@ -2242,7 +2289,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
|
||||
// if Content-Length is unknown/missing, deny the request
|
||||
size := r.ContentLength
|
||||
rAuthType := getRequestAuthType(r)
|
||||
if rAuthType == authTypeStreamingSigned || rAuthType == authTypeStreamingSignedTrailer {
|
||||
if rAuthType == authTypeStreamingSigned || rAuthType == authTypeStreamingSignedTrailer || rAuthType == authTypeStreamingUnsignedTrailer {
|
||||
if sizeStr, ok := r.Header[xhttp.AmzDecodedContentLength]; ok {
|
||||
if sizeStr[0] == "" {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
|
||||
@@ -2296,6 +2343,13 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
case authTypeStreamingUnsignedTrailer:
|
||||
// Initialize stream chunked reader with optional trailers.
|
||||
reader, s3Err = newUnsignedV4ChunkedReader(r, true)
|
||||
if s3Err != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
case authTypeSignedV2, authTypePresignedV2:
|
||||
s3Err = isReqAuthenticatedV2(r)
|
||||
if s3Err != ErrNone {
|
||||
@@ -2388,10 +2442,15 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
|
||||
rawReader := hashReader
|
||||
pReader := NewPutObjReader(rawReader)
|
||||
|
||||
allowReplicationMetadata := false
|
||||
if r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() {
|
||||
if s3Err = isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.ReplicateObjectAction); s3Err != ErrNone {
|
||||
return errors.New(errorCodes.ToAPIErr(s3Err).Code)
|
||||
}
|
||||
allowReplicationMetadata = true
|
||||
if err = extractReplicationMetadataFromMime(ctx, textproto.MIMEHeader(r.Header), metadata); err != nil {
|
||||
return err
|
||||
}
|
||||
metadata[ReservedMetadataPrefixLower+ReplicaStatus] = replication.Replica.String()
|
||||
metadata[ReservedMetadataPrefixLower+ReplicaTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
}
|
||||
@@ -2417,6 +2476,11 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if allowReplicationMetadata {
|
||||
if err = extractReplicationMetadataFromMime(ctx, textproto.MIMEHeader(hdrs), m); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
maps.Copy(metadata, m)
|
||||
} else {
|
||||
versionID = r.Form.Get(xhttp.VersionID)
|
||||
|
||||
@@ -42,6 +42,7 @@ import (
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
"github.com/minio/minio/internal/hash/sha256"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
ioutilx "github.com/minio/minio/internal/ioutil"
|
||||
@@ -60,6 +61,47 @@ const (
|
||||
MissingUploadID
|
||||
)
|
||||
|
||||
func replicationSSEPoisonHeaders() map[string]string {
|
||||
return map[string]string{
|
||||
"X-Minio-Replication-Server-Side-Encryption-Sealed-Key": base64.StdEncoding.EncodeToString(make([]byte, 64)),
|
||||
"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm": crypto.SealAlgorithm,
|
||||
"X-Minio-Replication-Server-Side-Encryption-Iv": base64.StdEncoding.EncodeToString(make([]byte, 32)),
|
||||
}
|
||||
}
|
||||
|
||||
func assertObjectMetadataKeysAbsent(t *testing.T, metadata map[string]string, keys ...string) {
|
||||
t.Helper()
|
||||
for _, key := range keys {
|
||||
if got, ok := metadata[key]; ok {
|
||||
t.Fatalf("expected metadata %q to be absent, got %q", key, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertObjectMetadataValueNotEqual(t *testing.T, metadata map[string]string, key, unexpected string) {
|
||||
t.Helper()
|
||||
if got := metadata[key]; got == unexpected {
|
||||
t.Fatalf("expected metadata %q to differ from %q", key, unexpected)
|
||||
}
|
||||
}
|
||||
|
||||
func assertObjectContents(t *testing.T, obj ObjectLayer, bucketName, objectName string, expected []byte) {
|
||||
t.Helper()
|
||||
reader, err := obj.GetObjectNInfo(context.Background(), bucketName, objectName, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to fetch object %s/%s: %v", bucketName, objectName, err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
got, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read object %s/%s: %v", bucketName, objectName, err)
|
||||
}
|
||||
if !bytes.Equal(got, expected) {
|
||||
t.Fatalf("unexpected object contents: got %d bytes, expected %d bytes", len(got), len(expected))
|
||||
}
|
||||
}
|
||||
|
||||
// Wrapper for calling HeadObject API handler tests for both Erasure multiple disks and FS single drive setup.
|
||||
func TestAPIHeadObjectHandler(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPIHeadObjectHandler, endpoints: []string{"HeadObject"}})
|
||||
@@ -1819,6 +1861,53 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIPutObjectReplicationHeaderPoisoning(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecExtendedObjectLayerAPITest(t, testAPIPutObjectReplicationHeaderPoisoning, []string{"PutObject"})
|
||||
}
|
||||
|
||||
func testAPIPutObjectReplicationHeaderPoisoning(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
objectName := "replication-header-poison-put"
|
||||
payload := []byte("replication-header-poison-put-payload")
|
||||
headers := replicationSSEPoisonHeaders()
|
||||
|
||||
req, err := newTestSignedRequestV4(
|
||||
http.MethodPut,
|
||||
getPutObjectURL("", bucketName, objectName),
|
||||
int64(len(payload)),
|
||||
bytes.NewReader(payload),
|
||||
credentials.AccessKey,
|
||||
credentials.SecretKey,
|
||||
headers,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: failed to create signed put request: %v", instanceType, err)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: expected put to succeed, got %d", instanceType, rec.Code)
|
||||
}
|
||||
|
||||
objInfo, err := obj.GetObjectInfo(context.Background(), bucketName, objectName, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: failed to fetch object info: %v", instanceType, err)
|
||||
}
|
||||
|
||||
assertObjectMetadataValueNotEqual(t, objInfo.UserDefined,
|
||||
crypto.MetaSealedKeySSEC,
|
||||
headers["X-Minio-Replication-Server-Side-Encryption-Sealed-Key"],
|
||||
)
|
||||
assertObjectMetadataValueNotEqual(t, objInfo.UserDefined,
|
||||
crypto.MetaIV,
|
||||
headers["X-Minio-Replication-Server-Side-Encryption-Iv"],
|
||||
)
|
||||
assertObjectContents(t, obj, bucketName, objectName, payload)
|
||||
}
|
||||
|
||||
// Wrapper for calling Copy Object Part API handler tests for both Erasure multiple disks and single node setup.
|
||||
func TestAPICopyObjectPartHandler(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
@@ -2860,6 +2949,74 @@ func testAPINewMultipartHandlerParallel(obj ObjectLayer, instanceType, bucketNam
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPICopyObjectReplicationHeaderPoisoning(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecExtendedObjectLayerAPITest(t, testAPICopyObjectReplicationHeaderPoisoning, []string{"CopyObject", "PutObject"})
|
||||
}
|
||||
|
||||
func testAPICopyObjectReplicationHeaderPoisoning(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
srcObject := "replication-header-poison-copy-src"
|
||||
dstObject := "replication-header-poison-copy-dst"
|
||||
payload := []byte("replication-header-poison-copy-payload")
|
||||
|
||||
if _, err := obj.PutObject(
|
||||
context.Background(),
|
||||
bucketName,
|
||||
srcObject,
|
||||
mustGetPutObjReader(t, bytes.NewReader(payload), int64(len(payload)), "", ""),
|
||||
ObjectOptions{},
|
||||
); err != nil {
|
||||
t.Fatalf("%s: failed to create source object: %v", instanceType, err)
|
||||
}
|
||||
|
||||
headers := replicationSSEPoisonHeaders()
|
||||
headers[xhttp.AmzCopySource] = url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + srcObject)
|
||||
headers[xhttp.AmzMetadataDirective] = replaceDirective
|
||||
headers[xhttp.AmzBucketReplicationStatus] = "PENDING"
|
||||
headers["Content-Type"] = "application/octet-stream"
|
||||
|
||||
req, err := newTestSignedRequestV4(
|
||||
http.MethodPut,
|
||||
getCopyObjectURL("", bucketName, dstObject),
|
||||
0,
|
||||
nil,
|
||||
credentials.AccessKey,
|
||||
credentials.SecretKey,
|
||||
headers,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: failed to create signed copy request: %v", instanceType, err)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: expected copy to succeed, got %d", instanceType, rec.Code)
|
||||
}
|
||||
|
||||
objInfo, err := obj.GetObjectInfo(context.Background(), bucketName, dstObject, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: failed to fetch copied object info: %v", instanceType, err)
|
||||
}
|
||||
|
||||
assertObjectMetadataValueNotEqual(t, objInfo.UserDefined,
|
||||
crypto.MetaSealedKeySSEC,
|
||||
headers["X-Minio-Replication-Server-Side-Encryption-Sealed-Key"],
|
||||
)
|
||||
assertObjectMetadataValueNotEqual(t, objInfo.UserDefined,
|
||||
crypto.MetaIV,
|
||||
headers["X-Minio-Replication-Server-Side-Encryption-Iv"],
|
||||
)
|
||||
assertObjectMetadataKeysAbsent(t, objInfo.UserDefined,
|
||||
xhttp.AmzBucketReplicationStatus,
|
||||
ReservedMetadataPrefixLower+ReplicaStatus,
|
||||
ReservedMetadataPrefixLower+ReplicaTimestamp,
|
||||
)
|
||||
assertObjectContents(t, obj, bucketName, dstObject, payload)
|
||||
}
|
||||
|
||||
// The UploadID from the response body is parsed and its existence is asserted with an attempt to ListParts using it.
|
||||
func TestAPICompleteMultipartHandler(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -157,6 +158,14 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
|
||||
metadata[xhttp.AmzObjectTagging] = objTags
|
||||
}
|
||||
if r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() {
|
||||
if s3Err := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.ReplicateObjectAction); s3Err != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
if err = extractReplicationMetadataFromMime(ctx, textproto.MIMEHeader(r.Header), metadata); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
metadata[ReservedMetadataPrefixLower+ReplicaStatus] = replication.Replica.String()
|
||||
metadata[ReservedMetadataPrefixLower+ReplicaTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
}
|
||||
@@ -685,8 +694,8 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
|
||||
return
|
||||
}
|
||||
case authTypeStreamingUnsignedTrailer:
|
||||
// Initialize stream signature verifier.
|
||||
reader, s3Error = newUnsignedV4ChunkedReader(r, true, r.Header.Get(xhttp.Authorization) != "")
|
||||
// Initialize stream chunked reader with optional trailers.
|
||||
reader, s3Error = newUnsignedV4ChunkedReader(r, true)
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
|
||||
@@ -144,7 +144,7 @@ func printServerCommonMsg(apiEndpoints []string) {
|
||||
|
||||
// Prints startup message for Object API access, prints link to our SDK documentation.
|
||||
func printObjectAPIMsg() {
|
||||
logger.Startup(color.Blue("\nDocs: ") + "https://docs.min.io")
|
||||
logger.Startup(color.Blue("\nDocs: ") + "https://silo.pigsty.io")
|
||||
}
|
||||
|
||||
func printLambdaTargets() {
|
||||
@@ -184,7 +184,7 @@ func printCLIAccessMsg(endPoint string, alias string) {
|
||||
// Get saved credentials.
|
||||
cred := globalActiveCred
|
||||
|
||||
const mcQuickStartGuide = "https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart"
|
||||
const mcQuickStartGuide = "https://silo.pigsty.io/reference/minio-mc.html#quickstart"
|
||||
|
||||
// Configure 'mc', following block prints platform specific information for minio client.
|
||||
if color.IsTerminal() && (!globalServerCtxt.Anonymous && globalAPIConfig.permitRootAccess()) {
|
||||
|
||||
+384
-19
@@ -18,10 +18,15 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
@@ -128,6 +133,13 @@ func runAllTests(suite *TestSuiteCommon, c *check) {
|
||||
suite.TestBucketSQSNotificationWebHook(c)
|
||||
suite.TestBucketSQSNotificationAMQP(c)
|
||||
suite.TestUnsignedCVE(c)
|
||||
suite.TestUnsignedQueryStringCVE(c)
|
||||
suite.TestUnsignedQueryStringCVEMultipart(c)
|
||||
suite.TestUnsignedTrailerRejectsMultipleAuthSources(c)
|
||||
suite.TestUnsignedTrailerSnowballAnonymousDenied(c)
|
||||
suite.TestUnsignedTrailerSnowballRequiresSignature(c)
|
||||
suite.TestUnsignedTrailerSnowballExtract(c)
|
||||
suite.TestAnonymousUnsignedTrailer(c)
|
||||
suite.TearDownSuite(c)
|
||||
}
|
||||
|
||||
@@ -374,24 +386,13 @@ func (s *TestSuiteCommon) TestUnsignedCVE(c *check) {
|
||||
// assert the http response status code.
|
||||
c.Assert(response.StatusCode, http.StatusOK)
|
||||
|
||||
req, err := http.NewRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, "test-cve-object.txt"), nil)
|
||||
now := UTCNow()
|
||||
req, err := newStreamingUnsignedTrailerRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, "test-cve-object.txt"), []byte("foobar!\n"), now)
|
||||
c.Assert(err, nil)
|
||||
|
||||
req.Body = io.NopCloser(bytes.NewReader([]byte("foobar!\n")))
|
||||
req.Trailer = http.Header{}
|
||||
req.Trailer.Set("x-amz-checksum-crc32", "rK0DXg==")
|
||||
|
||||
now := UTCNow()
|
||||
|
||||
req = signer.StreamingUnsignedV4(req, "", 8, now)
|
||||
|
||||
maliciousHeaders := http.Header{
|
||||
"Authorization": []string{fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s/us-east-1/s3/aws4_request, SignedHeaders=invalidheader, Signature=deadbeefdeadbeefdeadbeeddeadbeeddeadbeefdeadbeefdeadbeefdeadbeef", s.accessKey, now.Format(yyyymmdd))},
|
||||
"User-Agent": []string{"A malicious request"},
|
||||
"X-Amz-Decoded-Content-Length": []string{"8"},
|
||||
"Content-Encoding": []string{"aws-chunked"},
|
||||
"X-Amz-Trailer": []string{"x-amz-checksum-crc32"},
|
||||
"x-amz-content-sha256": []string{unsignedPayloadTrailer},
|
||||
"Authorization": []string{fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s/us-east-1/s3/aws4_request, SignedHeaders=invalidheader, Signature=deadbeefdeadbeefdeadbeeddeadbeeddeadbeefdeadbeefdeadbeefdeadbeef", s.accessKey, now.Format(yyyymmdd))},
|
||||
"User-Agent": []string{"A malicious request"},
|
||||
}
|
||||
|
||||
for k, v := range maliciousHeaders {
|
||||
@@ -409,6 +410,370 @@ func (s *TestSuiteCommon) TestUnsignedCVE(c *check) {
|
||||
c.Assert(response.StatusCode, http.StatusBadRequest)
|
||||
}
|
||||
|
||||
func (s *TestSuiteCommon) TestUnsignedQueryStringCVE(c *check) {
|
||||
c.Helper()
|
||||
|
||||
// generate a random bucket Name.
|
||||
bucketName := getRandomBucketName()
|
||||
|
||||
// HTTP request to create the bucket.
|
||||
request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName),
|
||||
0, nil, s.accessKey, s.secretKey, s.signer)
|
||||
c.Assert(err, nil)
|
||||
|
||||
// execute the request.
|
||||
response, err := s.client.Do(request)
|
||||
c.Assert(err, nil)
|
||||
|
||||
// assert the http response status code.
|
||||
c.Assert(response.StatusCode, http.StatusOK)
|
||||
|
||||
now := UTCNow()
|
||||
req, err := newStreamingUnsignedTrailerRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, "test-cve-presigned-object.txt"), []byte("foobar!\n"), now)
|
||||
c.Assert(err, nil)
|
||||
|
||||
err = presignStreamingUnsignedTrailerRequest(req, s.accessKey, s.secretKey, 60, now)
|
||||
c.Assert(err, nil)
|
||||
|
||||
// execute the request.
|
||||
response, err = s.client.Do(req)
|
||||
c.Assert(err, nil)
|
||||
|
||||
verifyError(c, response, "InvalidRequest", "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.", http.StatusBadRequest)
|
||||
}
|
||||
|
||||
func (s *TestSuiteCommon) TestUnsignedQueryStringCVEMultipart(c *check) {
|
||||
c.Helper()
|
||||
|
||||
bucketName := getRandomBucketName()
|
||||
|
||||
request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName),
|
||||
0, nil, s.accessKey, s.secretKey, s.signer)
|
||||
c.Assert(err, nil)
|
||||
|
||||
response, err := s.client.Do(request)
|
||||
c.Assert(err, nil)
|
||||
c.Assert(response.StatusCode, http.StatusOK)
|
||||
|
||||
uploadID := s.mustStartMultipartUpload(c, bucketName, "test-cve-presigned-multipart.txt")
|
||||
now := UTCNow()
|
||||
req, err := newStreamingUnsignedTrailerRequest(http.MethodPut, getPartUploadURL(s.endPoint, bucketName, "test-cve-presigned-multipart.txt", uploadID, "1"), []byte("foobar!\n"), now)
|
||||
c.Assert(err, nil)
|
||||
|
||||
err = presignStreamingUnsignedTrailerRequest(req, s.accessKey, s.secretKey, 60, now)
|
||||
c.Assert(err, nil)
|
||||
|
||||
response, err = s.client.Do(req)
|
||||
c.Assert(err, nil)
|
||||
|
||||
verifyError(c, response, "InvalidRequest", "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.", http.StatusBadRequest)
|
||||
}
|
||||
|
||||
func (s *TestSuiteCommon) TestUnsignedTrailerRejectsMultipleAuthSources(c *check) {
|
||||
c.Helper()
|
||||
|
||||
bucketName := getRandomBucketName()
|
||||
|
||||
request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName),
|
||||
0, nil, s.accessKey, s.secretKey, s.signer)
|
||||
c.Assert(err, nil)
|
||||
|
||||
response, err := s.client.Do(request)
|
||||
c.Assert(err, nil)
|
||||
c.Assert(response.StatusCode, http.StatusOK)
|
||||
|
||||
now := UTCNow()
|
||||
req, err := newStreamingUnsignedTrailerRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, "test-cve-mixed-auth.txt"), []byte("foobar!\n"), now)
|
||||
c.Assert(err, nil)
|
||||
|
||||
err = presignStreamingUnsignedTrailerRequest(req, s.accessKey, s.secretKey, 60, now)
|
||||
c.Assert(err, nil)
|
||||
err = signRequestV4(req, s.accessKey, s.secretKey)
|
||||
c.Assert(err, nil)
|
||||
|
||||
response, err = s.client.Do(req)
|
||||
c.Assert(err, nil)
|
||||
|
||||
verifyError(c, response, "InvalidRequest", "Invalid Request (request has multiple authentication types, please use one)", http.StatusBadRequest)
|
||||
}
|
||||
|
||||
func (s *TestSuiteCommon) TestUnsignedTrailerSnowballRequiresSignature(c *check) {
|
||||
c.Helper()
|
||||
|
||||
bucketName := getRandomBucketName()
|
||||
objectName := "snowball-upload.tar"
|
||||
extractedObject := "payload.txt"
|
||||
extractedData := []byte("snowball object\n")
|
||||
|
||||
request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName),
|
||||
0, nil, s.accessKey, s.secretKey, s.signer)
|
||||
c.Assert(err, nil)
|
||||
|
||||
response, err := s.client.Do(request)
|
||||
c.Assert(err, nil)
|
||||
c.Assert(response.StatusCode, http.StatusOK)
|
||||
|
||||
tarData, err := newTestTarArchive(extractedObject, extractedData)
|
||||
c.Assert(err, nil)
|
||||
|
||||
// Deliberately send raw tar bytes: before the fix this path never
|
||||
// initialized the unsigned-trailer reader, so Snowball accepted the
|
||||
// body without chunked decoding or signature verification.
|
||||
req, err := http.NewRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), bytes.NewReader(tarData))
|
||||
c.Assert(err, nil)
|
||||
|
||||
req.ContentLength = int64(len(tarData))
|
||||
req.Header.Set(xhttp.AmzSnowballExtract, "true")
|
||||
req.Header.Set(xhttp.AmzContentSha256, unsignedPayloadTrailer)
|
||||
|
||||
err = signRequestV4(req, s.accessKey, s.secretKey)
|
||||
c.Assert(err, nil)
|
||||
|
||||
req.Header.Set("Authorization",
|
||||
regexp.MustCompile(`Signature=[0-9a-f]+`).ReplaceAllString(req.Header.Get("Authorization"), "Signature="+strings.Repeat("0", 64)))
|
||||
|
||||
response, err = s.client.Do(req)
|
||||
c.Assert(err, nil)
|
||||
|
||||
verifyError(c, response, "SignatureDoesNotMatch", "The request signature we calculated does not match the signature you provided. Check your key and signing method.", http.StatusForbidden)
|
||||
}
|
||||
|
||||
func (s *TestSuiteCommon) TestUnsignedTrailerSnowballAnonymousDenied(c *check) {
|
||||
c.Helper()
|
||||
|
||||
bucketName := getRandomBucketName()
|
||||
objectName := "snowball-upload.tar"
|
||||
|
||||
request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName),
|
||||
0, nil, s.accessKey, s.secretKey, s.signer)
|
||||
c.Assert(err, nil)
|
||||
|
||||
response, err := s.client.Do(request)
|
||||
c.Assert(err, nil)
|
||||
c.Assert(response.StatusCode, http.StatusOK)
|
||||
|
||||
tarData, err := newTestTarArchive("payload.txt", []byte("snowball object\n"))
|
||||
c.Assert(err, nil)
|
||||
|
||||
req, err := http.NewRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), bytes.NewReader(tarData))
|
||||
c.Assert(err, nil)
|
||||
|
||||
req.ContentLength = int64(len(tarData))
|
||||
req.Header.Set(xhttp.AmzSnowballExtract, "true")
|
||||
req.Header.Set(xhttp.AmzContentSha256, unsignedPayloadTrailer)
|
||||
|
||||
response, err = s.client.Do(req)
|
||||
c.Assert(err, nil)
|
||||
|
||||
verifyError(c, response, "AccessDenied", "Access Denied.", http.StatusForbidden)
|
||||
}
|
||||
|
||||
func (s *TestSuiteCommon) TestUnsignedTrailerSnowballExtract(c *check) {
|
||||
c.Helper()
|
||||
|
||||
bucketName := getRandomBucketName()
|
||||
objectName := "snowball-upload.tar"
|
||||
extractedObject := "payload.txt"
|
||||
extractedData := []byte("snowball object\n")
|
||||
|
||||
request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName),
|
||||
0, nil, s.accessKey, s.secretKey, s.signer)
|
||||
c.Assert(err, nil)
|
||||
|
||||
response, err := s.client.Do(request)
|
||||
c.Assert(err, nil)
|
||||
c.Assert(response.StatusCode, http.StatusOK)
|
||||
|
||||
tarData, err := newTestTarArchive(extractedObject, extractedData)
|
||||
c.Assert(err, nil)
|
||||
|
||||
req, err := newStreamingUnsignedTrailerRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), tarData, UTCNow())
|
||||
c.Assert(err, nil)
|
||||
|
||||
req.Header.Set(xhttp.AmzSnowballExtract, "true")
|
||||
|
||||
err = signRequestV4(req, s.accessKey, s.secretKey)
|
||||
c.Assert(err, nil)
|
||||
|
||||
response, err = s.client.Do(req)
|
||||
c.Assert(err, nil)
|
||||
c.Assert(response.StatusCode, http.StatusOK)
|
||||
|
||||
request, err = newTestSignedRequest(http.MethodGet, getGetObjectURL(s.endPoint, bucketName, extractedObject),
|
||||
0, nil, s.accessKey, s.secretKey, s.signer)
|
||||
c.Assert(err, nil)
|
||||
|
||||
response, err = s.client.Do(request)
|
||||
c.Assert(err, nil)
|
||||
c.Assert(response.StatusCode, http.StatusOK)
|
||||
|
||||
got, err := io.ReadAll(response.Body)
|
||||
c.Assert(err, nil)
|
||||
c.Assert(bytes.Equal(got, extractedData), true)
|
||||
}
|
||||
|
||||
func (s *TestSuiteCommon) TestAnonymousUnsignedTrailer(c *check) {
|
||||
c.Helper()
|
||||
|
||||
bucketName := getRandomBucketName()
|
||||
objectName := "test-anonymous-unsigned-trailer.txt"
|
||||
objectData := []byte("foobar!\n")
|
||||
|
||||
request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName),
|
||||
0, nil, s.accessKey, s.secretKey, s.signer)
|
||||
c.Assert(err, nil)
|
||||
|
||||
response, err := s.client.Do(request)
|
||||
c.Assert(err, nil)
|
||||
c.Assert(response.StatusCode, http.StatusOK)
|
||||
|
||||
s.mustPutBucketPolicy(c, bucketName, getAnonWriteOnlyObjectPolicy(bucketName, objectName))
|
||||
|
||||
req, err := newStreamingUnsignedTrailerRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), objectData, UTCNow())
|
||||
c.Assert(err, nil)
|
||||
|
||||
response, err = s.client.Do(req)
|
||||
c.Assert(err, nil)
|
||||
c.Assert(response.StatusCode, http.StatusOK)
|
||||
|
||||
request, err = newTestSignedRequest(http.MethodGet, getGetObjectURL(s.endPoint, bucketName, objectName),
|
||||
0, nil, s.accessKey, s.secretKey, s.signer)
|
||||
c.Assert(err, nil)
|
||||
|
||||
response, err = s.client.Do(request)
|
||||
c.Assert(err, nil)
|
||||
c.Assert(response.StatusCode, http.StatusOK)
|
||||
|
||||
got, err := io.ReadAll(response.Body)
|
||||
c.Assert(err, nil)
|
||||
c.Assert(bytes.Equal(got, objectData), true)
|
||||
}
|
||||
|
||||
func newTestTarArchive(objectName string, objectData []byte) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
|
||||
err := tw.WriteHeader(&tar.Header{
|
||||
Name: objectName,
|
||||
Mode: 0o600,
|
||||
Size: int64(len(objectData)),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if _, err = tw.Write(objectData); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err = tw.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
func newStreamingUnsignedTrailerRequest(method, targetURL string, data []byte, reqTime time.Time) (*http.Request, error) {
|
||||
req, err := http.NewRequest(method, targetURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Body = io.NopCloser(bytes.NewReader(data))
|
||||
req.Trailer = http.Header{}
|
||||
req.Trailer.Set("x-amz-checksum-crc32", crc32Base64(data))
|
||||
req = signer.StreamingUnsignedV4(req, "", int64(len(data)), reqTime)
|
||||
req.Header.Set("X-Amz-Decoded-Content-Length", fmt.Sprintf("%d", len(data)))
|
||||
req.Header.Set("Content-Encoding", "aws-chunked")
|
||||
req.Header.Set("X-Amz-Trailer", "x-amz-checksum-crc32")
|
||||
req.Header.Set(xhttp.AmzContentSha256, unsignedPayloadTrailer)
|
||||
return req, nil
|
||||
}
|
||||
|
||||
func crc32Base64(data []byte) string {
|
||||
var crc [4]byte
|
||||
binary.BigEndian.PutUint32(crc[:], crc32.ChecksumIEEE(data))
|
||||
return base64.StdEncoding.EncodeToString(crc[:])
|
||||
}
|
||||
|
||||
func presignStreamingUnsignedTrailerRequest(req *http.Request, accessKeyID, secretAccessKey string, expires int64, reqTime time.Time) error {
|
||||
if accessKeyID == "" || secretAccessKey == "" {
|
||||
return fmt.Errorf("presign cannot be generated without access and secret keys")
|
||||
}
|
||||
|
||||
signedHeaders := []string{
|
||||
"content-encoding",
|
||||
"host",
|
||||
"x-amz-content-sha256",
|
||||
"x-amz-decoded-content-length",
|
||||
"x-amz-trailer",
|
||||
}
|
||||
region := globalSite.Region()
|
||||
scope := getScope(reqTime, region)
|
||||
credential := fmt.Sprintf("%s/%s", accessKeyID, scope)
|
||||
|
||||
query := req.URL.Query()
|
||||
query.Set(xhttp.AmzAlgorithm, signV4Algorithm)
|
||||
query.Set(xhttp.AmzDate, reqTime.Format(iso8601Format))
|
||||
query.Set(xhttp.AmzExpires, fmt.Sprintf("%d", expires))
|
||||
query.Set(xhttp.AmzSignedHeaders, strings.Join(signedHeaders, ";"))
|
||||
query.Set(xhttp.AmzCredential, credential)
|
||||
query.Set(xhttp.AmzContentSha256, unsignedPayloadTrailer)
|
||||
|
||||
req.Form = query
|
||||
extractedSignedHeaders, errCode := extractSignedHeaders(signedHeaders, req)
|
||||
if errCode != ErrNone {
|
||||
return fmt.Errorf("extractSignedHeaders failed: %v", errCode)
|
||||
}
|
||||
|
||||
queryStr := strings.ReplaceAll(query.Encode(), "+", "%20")
|
||||
canonicalRequest := getCanonicalRequest(extractedSignedHeaders, unsignedPayloadTrailer, queryStr, req.URL.Path, req.Method)
|
||||
stringToSign := getStringToSign(canonicalRequest, reqTime, scope)
|
||||
signingKey := getSigningKey(secretAccessKey, reqTime, region, serviceS3)
|
||||
signature := getSignature(signingKey, stringToSign)
|
||||
|
||||
query.Set(xhttp.AmzSignature, signature)
|
||||
req.URL.RawQuery = query.Encode()
|
||||
req.Form = req.URL.Query()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *TestSuiteCommon) mustPutBucketPolicy(c *check, bucketName string, bucketPolicy *policy.BucketPolicy) {
|
||||
c.Helper()
|
||||
|
||||
policyBytes, err := json.Marshal(bucketPolicy)
|
||||
c.Assert(err, nil)
|
||||
|
||||
request, err := newTestSignedRequest(http.MethodPut, getPutPolicyURL(s.endPoint, bucketName),
|
||||
int64(len(policyBytes)), bytes.NewReader(policyBytes), s.accessKey, s.secretKey, s.signer)
|
||||
c.Assert(err, nil)
|
||||
|
||||
response, err := s.client.Do(request)
|
||||
c.Assert(err, nil)
|
||||
c.Assert(response.StatusCode, http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (s *TestSuiteCommon) mustStartMultipartUpload(c *check, bucketName, objectName string) string {
|
||||
c.Helper()
|
||||
|
||||
request, err := newTestSignedRequest(http.MethodPost, getNewMultipartURL(s.endPoint, bucketName, objectName),
|
||||
0, nil, s.accessKey, s.secretKey, s.signer)
|
||||
c.Assert(err, nil)
|
||||
|
||||
response, err := s.client.Do(request)
|
||||
c.Assert(err, nil)
|
||||
c.Assert(response.StatusCode, http.StatusOK)
|
||||
|
||||
decoder := xml.NewDecoder(response.Body)
|
||||
newResponse := &InitiateMultipartUploadResponse{}
|
||||
err = decoder.Decode(newResponse)
|
||||
c.Assert(err, nil)
|
||||
c.Assert(len(newResponse.UploadID) > 0, true)
|
||||
|
||||
return newResponse.UploadID
|
||||
}
|
||||
|
||||
func (s *TestSuiteCommon) TestBucketSQSNotificationAMQP(c *check) {
|
||||
// Sample bucket notification.
|
||||
bucketNotificationBuf := `<NotificationConfiguration><QueueConfiguration><Event>s3:ObjectCreated:Put</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>images/</Value></FilterRule></S3Key></Filter><Id>1</Id><Queue>arn:minio:sqs:us-east-1:444455556666:amqp</Queue></QueueConfiguration></NotificationConfiguration>`
|
||||
@@ -2128,7 +2493,7 @@ func (s *TestSuiteCommon) TestGetObjectLarge10MiB(c *check) {
|
||||
1234567890,1234567890,1234567890,1234567890,1234567890,123"`
|
||||
// Create 10MiB content where each line contains 1024 characters.
|
||||
for i := range 10 * 1024 {
|
||||
buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line))
|
||||
fmt.Fprintf(&buffer, "[%05d] %s\n", i, line)
|
||||
}
|
||||
putContent := buffer.String()
|
||||
|
||||
@@ -2190,7 +2555,7 @@ func (s *TestSuiteCommon) TestGetObjectLarge11MiB(c *check) {
|
||||
1234567890,1234567890,1234567890,123`
|
||||
// Create 11MiB content where each line contains 1024 characters.
|
||||
for i := range 11 * 1024 {
|
||||
buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line))
|
||||
fmt.Fprintf(&buffer, "[%05d] %s\n", i, line)
|
||||
}
|
||||
putMD5 := getMD5Hash(buffer.Bytes())
|
||||
|
||||
@@ -2341,7 +2706,7 @@ func (s *TestSuiteCommon) TestGetPartialObjectLarge11MiB(c *check) {
|
||||
// Create 11MiB content where each line contains 1024
|
||||
// characters.
|
||||
for i := range 11 * 1024 {
|
||||
buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line))
|
||||
fmt.Fprintf(&buffer, "[%05d] %s\n", i, line)
|
||||
}
|
||||
putContent := buffer.String()
|
||||
|
||||
@@ -2407,7 +2772,7 @@ func (s *TestSuiteCommon) TestGetPartialObjectLarge10MiB(c *check) {
|
||||
1234567890,1234567890,1234567890,123`
|
||||
// Create 10MiB content where each line contains 1024 characters.
|
||||
for i := range 10 * 1024 {
|
||||
buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line))
|
||||
fmt.Fprintf(&buffer, "[%05d] %s\n", i, line)
|
||||
}
|
||||
|
||||
putContent := buffer.String()
|
||||
|
||||
@@ -396,28 +396,6 @@ func newFileInfo(object string, dataBlocks, parityBlocks int) (fi FileInfo) {
|
||||
return fi
|
||||
}
|
||||
|
||||
// ReadMultipleReq contains information of multiple files to read from disk.
|
||||
type ReadMultipleReq struct {
|
||||
Bucket string `msg:"bk"` // Bucket. Can be empty if multiple buckets.
|
||||
Prefix string `msg:"pr,omitempty"` // Shared prefix of all files. Can be empty. Will be joined to filename without modification.
|
||||
Files []string `msg:"fl"` // Individual files to read.
|
||||
MaxSize int64 `msg:"ms"` // Return error if size is exceed.
|
||||
MetadataOnly bool `msg:"mo"` // Read as XL meta and truncate data.
|
||||
AbortOn404 bool `msg:"ab"` // Stop reading after first file not found.
|
||||
MaxResults int `msg:"mr"` // Stop after this many successful results. <= 0 means all.
|
||||
}
|
||||
|
||||
// ReadMultipleResp contains a single response from a ReadMultipleReq.
|
||||
type ReadMultipleResp struct {
|
||||
Bucket string `msg:"bk"` // Bucket as given by request.
|
||||
Prefix string `msg:"pr,omitempty"` // Prefix as given by request.
|
||||
File string `msg:"fl"` // File name as given in request.
|
||||
Exists bool `msg:"ex"` // Returns whether the file existed on disk.
|
||||
Error string `msg:"er,omitempty"` // Returns any error when reading.
|
||||
Data []byte `msg:"d"` // Contains all data of file.
|
||||
Modtime time.Time `msg:"m"` // Modtime of file on disk.
|
||||
}
|
||||
|
||||
// DeleteVersionHandlerParams are parameters for DeleteVersionHandler
|
||||
type DeleteVersionHandlerParams struct {
|
||||
DiskID string `msg:"id"`
|
||||
|
||||
@@ -4234,672 +4234,6 @@ func (z ReadAllHandlerParams) Msgsize() (s int) {
|
||||
return
|
||||
}
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *ReadMultipleReq) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
var zb0001Mask uint8 /* 1 bits */
|
||||
_ = zb0001Mask
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, err = dc.ReadMapKeyPtr()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "bk":
|
||||
z.Bucket, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Bucket")
|
||||
return
|
||||
}
|
||||
case "pr":
|
||||
z.Prefix, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Prefix")
|
||||
return
|
||||
}
|
||||
zb0001Mask |= 0x1
|
||||
case "fl":
|
||||
var zb0002 uint32
|
||||
zb0002, err = dc.ReadArrayHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Files")
|
||||
return
|
||||
}
|
||||
if cap(z.Files) >= int(zb0002) {
|
||||
z.Files = (z.Files)[:zb0002]
|
||||
} else {
|
||||
z.Files = make([]string, zb0002)
|
||||
}
|
||||
for za0001 := range z.Files {
|
||||
z.Files[za0001], err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Files", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
case "ms":
|
||||
z.MaxSize, err = dc.ReadInt64()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "MaxSize")
|
||||
return
|
||||
}
|
||||
case "mo":
|
||||
z.MetadataOnly, err = dc.ReadBool()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "MetadataOnly")
|
||||
return
|
||||
}
|
||||
case "ab":
|
||||
z.AbortOn404, err = dc.ReadBool()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AbortOn404")
|
||||
return
|
||||
}
|
||||
case "mr":
|
||||
z.MaxResults, err = dc.ReadInt()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "MaxResults")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
// Clear omitted fields.
|
||||
if (zb0001Mask & 0x1) == 0 {
|
||||
z.Prefix = ""
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *ReadMultipleReq) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// check for omitted fields
|
||||
zb0001Len := uint32(7)
|
||||
var zb0001Mask uint8 /* 7 bits */
|
||||
_ = zb0001Mask
|
||||
if z.Prefix == "" {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x2
|
||||
}
|
||||
// variable map header, size zb0001Len
|
||||
err = en.Append(0x80 | uint8(zb0001Len))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// skip if no fields are to be emitted
|
||||
if zb0001Len != 0 {
|
||||
// write "bk"
|
||||
err = en.Append(0xa2, 0x62, 0x6b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.Bucket)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Bucket")
|
||||
return
|
||||
}
|
||||
if (zb0001Mask & 0x2) == 0 { // if not omitted
|
||||
// write "pr"
|
||||
err = en.Append(0xa2, 0x70, 0x72)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.Prefix)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Prefix")
|
||||
return
|
||||
}
|
||||
}
|
||||
// write "fl"
|
||||
err = en.Append(0xa2, 0x66, 0x6c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteArrayHeader(uint32(len(z.Files)))
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Files")
|
||||
return
|
||||
}
|
||||
for za0001 := range z.Files {
|
||||
err = en.WriteString(z.Files[za0001])
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Files", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
// write "ms"
|
||||
err = en.Append(0xa2, 0x6d, 0x73)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteInt64(z.MaxSize)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "MaxSize")
|
||||
return
|
||||
}
|
||||
// write "mo"
|
||||
err = en.Append(0xa2, 0x6d, 0x6f)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteBool(z.MetadataOnly)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "MetadataOnly")
|
||||
return
|
||||
}
|
||||
// write "ab"
|
||||
err = en.Append(0xa2, 0x61, 0x62)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteBool(z.AbortOn404)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AbortOn404")
|
||||
return
|
||||
}
|
||||
// write "mr"
|
||||
err = en.Append(0xa2, 0x6d, 0x72)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteInt(z.MaxResults)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "MaxResults")
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *ReadMultipleReq) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// check for omitted fields
|
||||
zb0001Len := uint32(7)
|
||||
var zb0001Mask uint8 /* 7 bits */
|
||||
_ = zb0001Mask
|
||||
if z.Prefix == "" {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x2
|
||||
}
|
||||
// variable map header, size zb0001Len
|
||||
o = append(o, 0x80|uint8(zb0001Len))
|
||||
|
||||
// skip if no fields are to be emitted
|
||||
if zb0001Len != 0 {
|
||||
// string "bk"
|
||||
o = append(o, 0xa2, 0x62, 0x6b)
|
||||
o = msgp.AppendString(o, z.Bucket)
|
||||
if (zb0001Mask & 0x2) == 0 { // if not omitted
|
||||
// string "pr"
|
||||
o = append(o, 0xa2, 0x70, 0x72)
|
||||
o = msgp.AppendString(o, z.Prefix)
|
||||
}
|
||||
// string "fl"
|
||||
o = append(o, 0xa2, 0x66, 0x6c)
|
||||
o = msgp.AppendArrayHeader(o, uint32(len(z.Files)))
|
||||
for za0001 := range z.Files {
|
||||
o = msgp.AppendString(o, z.Files[za0001])
|
||||
}
|
||||
// string "ms"
|
||||
o = append(o, 0xa2, 0x6d, 0x73)
|
||||
o = msgp.AppendInt64(o, z.MaxSize)
|
||||
// string "mo"
|
||||
o = append(o, 0xa2, 0x6d, 0x6f)
|
||||
o = msgp.AppendBool(o, z.MetadataOnly)
|
||||
// string "ab"
|
||||
o = append(o, 0xa2, 0x61, 0x62)
|
||||
o = msgp.AppendBool(o, z.AbortOn404)
|
||||
// string "mr"
|
||||
o = append(o, 0xa2, 0x6d, 0x72)
|
||||
o = msgp.AppendInt(o, z.MaxResults)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (z *ReadMultipleReq) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
var zb0001Mask uint8 /* 1 bits */
|
||||
_ = zb0001Mask
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "bk":
|
||||
z.Bucket, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Bucket")
|
||||
return
|
||||
}
|
||||
case "pr":
|
||||
z.Prefix, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Prefix")
|
||||
return
|
||||
}
|
||||
zb0001Mask |= 0x1
|
||||
case "fl":
|
||||
var zb0002 uint32
|
||||
zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Files")
|
||||
return
|
||||
}
|
||||
if cap(z.Files) >= int(zb0002) {
|
||||
z.Files = (z.Files)[:zb0002]
|
||||
} else {
|
||||
z.Files = make([]string, zb0002)
|
||||
}
|
||||
for za0001 := range z.Files {
|
||||
z.Files[za0001], bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Files", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
case "ms":
|
||||
z.MaxSize, bts, err = msgp.ReadInt64Bytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "MaxSize")
|
||||
return
|
||||
}
|
||||
case "mo":
|
||||
z.MetadataOnly, bts, err = msgp.ReadBoolBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "MetadataOnly")
|
||||
return
|
||||
}
|
||||
case "ab":
|
||||
z.AbortOn404, bts, err = msgp.ReadBoolBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AbortOn404")
|
||||
return
|
||||
}
|
||||
case "mr":
|
||||
z.MaxResults, bts, err = msgp.ReadIntBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "MaxResults")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
// Clear omitted fields.
|
||||
if (zb0001Mask & 0x1) == 0 {
|
||||
z.Prefix = ""
|
||||
}
|
||||
|
||||
o = bts
|
||||
return
|
||||
}
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *ReadMultipleReq) Msgsize() (s int) {
|
||||
s = 1 + 3 + msgp.StringPrefixSize + len(z.Bucket) + 3 + msgp.StringPrefixSize + len(z.Prefix) + 3 + msgp.ArrayHeaderSize
|
||||
for za0001 := range z.Files {
|
||||
s += msgp.StringPrefixSize + len(z.Files[za0001])
|
||||
}
|
||||
s += 3 + msgp.Int64Size + 3 + msgp.BoolSize + 3 + msgp.BoolSize + 3 + msgp.IntSize
|
||||
return
|
||||
}
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *ReadMultipleResp) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
var zb0001Mask uint8 /* 2 bits */
|
||||
_ = zb0001Mask
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, err = dc.ReadMapKeyPtr()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "bk":
|
||||
z.Bucket, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Bucket")
|
||||
return
|
||||
}
|
||||
case "pr":
|
||||
z.Prefix, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Prefix")
|
||||
return
|
||||
}
|
||||
zb0001Mask |= 0x1
|
||||
case "fl":
|
||||
z.File, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "File")
|
||||
return
|
||||
}
|
||||
case "ex":
|
||||
z.Exists, err = dc.ReadBool()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Exists")
|
||||
return
|
||||
}
|
||||
case "er":
|
||||
z.Error, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Error")
|
||||
return
|
||||
}
|
||||
zb0001Mask |= 0x2
|
||||
case "d":
|
||||
z.Data, err = dc.ReadBytes(z.Data)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Data")
|
||||
return
|
||||
}
|
||||
case "m":
|
||||
z.Modtime, err = dc.ReadTime()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Modtime")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
// Clear omitted fields.
|
||||
if zb0001Mask != 0x3 {
|
||||
if (zb0001Mask & 0x1) == 0 {
|
||||
z.Prefix = ""
|
||||
}
|
||||
if (zb0001Mask & 0x2) == 0 {
|
||||
z.Error = ""
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *ReadMultipleResp) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// check for omitted fields
|
||||
zb0001Len := uint32(7)
|
||||
var zb0001Mask uint8 /* 7 bits */
|
||||
_ = zb0001Mask
|
||||
if z.Prefix == "" {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x2
|
||||
}
|
||||
if z.Error == "" {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x10
|
||||
}
|
||||
// variable map header, size zb0001Len
|
||||
err = en.Append(0x80 | uint8(zb0001Len))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// skip if no fields are to be emitted
|
||||
if zb0001Len != 0 {
|
||||
// write "bk"
|
||||
err = en.Append(0xa2, 0x62, 0x6b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.Bucket)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Bucket")
|
||||
return
|
||||
}
|
||||
if (zb0001Mask & 0x2) == 0 { // if not omitted
|
||||
// write "pr"
|
||||
err = en.Append(0xa2, 0x70, 0x72)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.Prefix)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Prefix")
|
||||
return
|
||||
}
|
||||
}
|
||||
// write "fl"
|
||||
err = en.Append(0xa2, 0x66, 0x6c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.File)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "File")
|
||||
return
|
||||
}
|
||||
// write "ex"
|
||||
err = en.Append(0xa2, 0x65, 0x78)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteBool(z.Exists)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Exists")
|
||||
return
|
||||
}
|
||||
if (zb0001Mask & 0x10) == 0 { // if not omitted
|
||||
// write "er"
|
||||
err = en.Append(0xa2, 0x65, 0x72)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.Error)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Error")
|
||||
return
|
||||
}
|
||||
}
|
||||
// write "d"
|
||||
err = en.Append(0xa1, 0x64)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteBytes(z.Data)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Data")
|
||||
return
|
||||
}
|
||||
// write "m"
|
||||
err = en.Append(0xa1, 0x6d)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteTime(z.Modtime)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Modtime")
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *ReadMultipleResp) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// check for omitted fields
|
||||
zb0001Len := uint32(7)
|
||||
var zb0001Mask uint8 /* 7 bits */
|
||||
_ = zb0001Mask
|
||||
if z.Prefix == "" {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x2
|
||||
}
|
||||
if z.Error == "" {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x10
|
||||
}
|
||||
// variable map header, size zb0001Len
|
||||
o = append(o, 0x80|uint8(zb0001Len))
|
||||
|
||||
// skip if no fields are to be emitted
|
||||
if zb0001Len != 0 {
|
||||
// string "bk"
|
||||
o = append(o, 0xa2, 0x62, 0x6b)
|
||||
o = msgp.AppendString(o, z.Bucket)
|
||||
if (zb0001Mask & 0x2) == 0 { // if not omitted
|
||||
// string "pr"
|
||||
o = append(o, 0xa2, 0x70, 0x72)
|
||||
o = msgp.AppendString(o, z.Prefix)
|
||||
}
|
||||
// string "fl"
|
||||
o = append(o, 0xa2, 0x66, 0x6c)
|
||||
o = msgp.AppendString(o, z.File)
|
||||
// string "ex"
|
||||
o = append(o, 0xa2, 0x65, 0x78)
|
||||
o = msgp.AppendBool(o, z.Exists)
|
||||
if (zb0001Mask & 0x10) == 0 { // if not omitted
|
||||
// string "er"
|
||||
o = append(o, 0xa2, 0x65, 0x72)
|
||||
o = msgp.AppendString(o, z.Error)
|
||||
}
|
||||
// string "d"
|
||||
o = append(o, 0xa1, 0x64)
|
||||
o = msgp.AppendBytes(o, z.Data)
|
||||
// string "m"
|
||||
o = append(o, 0xa1, 0x6d)
|
||||
o = msgp.AppendTime(o, z.Modtime)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (z *ReadMultipleResp) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
var zb0001Mask uint8 /* 2 bits */
|
||||
_ = zb0001Mask
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "bk":
|
||||
z.Bucket, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Bucket")
|
||||
return
|
||||
}
|
||||
case "pr":
|
||||
z.Prefix, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Prefix")
|
||||
return
|
||||
}
|
||||
zb0001Mask |= 0x1
|
||||
case "fl":
|
||||
z.File, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "File")
|
||||
return
|
||||
}
|
||||
case "ex":
|
||||
z.Exists, bts, err = msgp.ReadBoolBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Exists")
|
||||
return
|
||||
}
|
||||
case "er":
|
||||
z.Error, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Error")
|
||||
return
|
||||
}
|
||||
zb0001Mask |= 0x2
|
||||
case "d":
|
||||
z.Data, bts, err = msgp.ReadBytesBytes(bts, z.Data)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Data")
|
||||
return
|
||||
}
|
||||
case "m":
|
||||
z.Modtime, bts, err = msgp.ReadTimeBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Modtime")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
// Clear omitted fields.
|
||||
if zb0001Mask != 0x3 {
|
||||
if (zb0001Mask & 0x1) == 0 {
|
||||
z.Prefix = ""
|
||||
}
|
||||
if (zb0001Mask & 0x2) == 0 {
|
||||
z.Error = ""
|
||||
}
|
||||
}
|
||||
o = bts
|
||||
return
|
||||
}
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *ReadMultipleResp) Msgsize() (s int) {
|
||||
s = 1 + 3 + msgp.StringPrefixSize + len(z.Bucket) + 3 + msgp.StringPrefixSize + len(z.Prefix) + 3 + msgp.StringPrefixSize + len(z.File) + 3 + msgp.BoolSize + 3 + msgp.StringPrefixSize + len(z.Error) + 2 + msgp.BytesPrefixSize + len(z.Data) + 2 + msgp.TimeSize
|
||||
return
|
||||
}
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *ReadPartsReq) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
var field []byte
|
||||
|
||||
@@ -2156,232 +2156,6 @@ func BenchmarkDecodeReadAllHandlerParams(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalReadMultipleReq(t *testing.T) {
|
||||
v := ReadMultipleReq{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgReadMultipleReq(b *testing.B) {
|
||||
v := ReadMultipleReq{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgReadMultipleReq(b *testing.B) {
|
||||
v := ReadMultipleReq{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalReadMultipleReq(b *testing.B) {
|
||||
v := ReadMultipleReq{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeReadMultipleReq(t *testing.T) {
|
||||
v := ReadMultipleReq{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeReadMultipleReq Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := ReadMultipleReq{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeReadMultipleReq(b *testing.B) {
|
||||
v := ReadMultipleReq{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeReadMultipleReq(b *testing.B) {
|
||||
v := ReadMultipleReq{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalReadMultipleResp(t *testing.T) {
|
||||
v := ReadMultipleResp{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgReadMultipleResp(b *testing.B) {
|
||||
v := ReadMultipleResp{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgReadMultipleResp(b *testing.B) {
|
||||
v := ReadMultipleResp{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalReadMultipleResp(b *testing.B) {
|
||||
v := ReadMultipleResp{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeReadMultipleResp(t *testing.T) {
|
||||
v := ReadMultipleResp{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeReadMultipleResp Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := ReadMultipleResp{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeReadMultipleResp(b *testing.B) {
|
||||
v := ReadMultipleResp{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeReadMultipleResp(b *testing.B) {
|
||||
v := ReadMultipleResp{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalReadPartsReq(t *testing.T) {
|
||||
v := ReadPartsReq{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
|
||||
@@ -101,7 +101,6 @@ type StorageAPI interface {
|
||||
VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error)
|
||||
StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error)
|
||||
ReadParts(ctx context.Context, bucket string, partMetaPaths ...string) ([]*ObjectPartInfo, error)
|
||||
ReadMultiple(ctx context.Context, req ReadMultipleReq, resp chan<- ReadMultipleResp) error
|
||||
CleanAbandonedData(ctx context.Context, volume string, path string) error
|
||||
|
||||
// Write all data, syncs the data to disk.
|
||||
|
||||
@@ -912,45 +912,6 @@ func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path
|
||||
return stat, toStorageErr(err)
|
||||
}
|
||||
|
||||
// ReadMultiple will read multiple files and send each back as response.
|
||||
// Files are read and returned in the given order.
|
||||
// The resp channel is closed before the call returns.
|
||||
// Only a canceled context or network errors returns an error.
|
||||
func (client *storageRESTClient) ReadMultiple(ctx context.Context, req ReadMultipleReq, resp chan<- ReadMultipleResp) error {
|
||||
defer xioutil.SafeClose(resp)
|
||||
body, err := req.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
respBody, err := client.call(ctx, storageRESTMethodReadMultiple, nil, bytes.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer xhttp.DrainBody(respBody)
|
||||
|
||||
pr, pw := io.Pipe()
|
||||
go func() {
|
||||
pw.CloseWithError(waitForHTTPStream(respBody, xioutil.NewDeadlineWriter(pw, globalDriveConfig.GetMaxTimeout())))
|
||||
}()
|
||||
mr := msgp.NewReader(pr)
|
||||
defer readMsgpReaderPoolPut(mr)
|
||||
for {
|
||||
var file ReadMultipleResp
|
||||
if err := file.DecodeMsg(mr); err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
err = nil
|
||||
}
|
||||
pr.CloseWithError(err)
|
||||
return toStorageErr(err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case resp <- file:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// CleanAbandonedData will read metadata of the object on disk
|
||||
// and delete any data directories and inline data that isn't referenced in metadata.
|
||||
func (client *storageRESTClient) CleanAbandonedData(ctx context.Context, volume string, path string) error {
|
||||
|
||||
@@ -41,7 +41,6 @@ const (
|
||||
storageRESTMethodRenameFile = "/rfile"
|
||||
storageRESTMethodVerifyFile = "/vfile"
|
||||
storageRESTMethodStatInfoFile = "/sfile"
|
||||
storageRESTMethodReadMultiple = "/rmpl"
|
||||
storageRESTMethodCleanAbandoned = "/cln"
|
||||
storageRESTMethodDeleteBulk = "/dblk"
|
||||
storageRESTMethodReadParts = "/rps"
|
||||
|
||||
@@ -28,7 +28,6 @@ import (
|
||||
"net/http"
|
||||
"os/user"
|
||||
"path"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -1283,48 +1282,6 @@ func (s *storageRESTServer) DeleteBulkHandler(w http.ResponseWriter, r *http.Req
|
||||
keepHTTPResponseAlive(w)(s.getStorage().DeleteBulk(r.Context(), volume, req.Paths...))
|
||||
}
|
||||
|
||||
// ReadMultiple returns multiple files
|
||||
func (s *storageRESTServer) ReadMultiple(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.IsValid(w, r) {
|
||||
return
|
||||
}
|
||||
rw := streamHTTPResponse(w)
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
debug.PrintStack()
|
||||
rw.CloseWithError(fmt.Errorf("panic: %v", r))
|
||||
}
|
||||
}()
|
||||
|
||||
var req ReadMultipleReq
|
||||
mr := msgpNewReader(r.Body)
|
||||
defer readMsgpReaderPoolPut(mr)
|
||||
err := req.DecodeMsg(mr)
|
||||
if err != nil {
|
||||
rw.CloseWithError(err)
|
||||
return
|
||||
}
|
||||
|
||||
mw := msgp.NewWriter(rw)
|
||||
responses := make(chan ReadMultipleResp, len(req.Files))
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for resp := range responses {
|
||||
err := resp.EncodeMsg(mw)
|
||||
if err != nil {
|
||||
rw.CloseWithError(err)
|
||||
return
|
||||
}
|
||||
mw.Flush()
|
||||
}
|
||||
}()
|
||||
err = s.getStorage().ReadMultiple(r.Context(), req, responses)
|
||||
wg.Wait()
|
||||
rw.CloseWithError(err)
|
||||
}
|
||||
|
||||
// globalLocalSetDrives is used for local drive as well as remote REST
|
||||
// API caller for other nodes to talk to this node.
|
||||
//
|
||||
@@ -1363,7 +1320,6 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
|
||||
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteVersions).HandlerFunc(h(server.DeleteVersionsHandler))
|
||||
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodVerifyFile).HandlerFunc(h(server.VerifyFileHandler))
|
||||
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatInfoFile).HandlerFunc(h(server.StatInfoFile))
|
||||
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadMultiple).HandlerFunc(h(server.ReadMultiple))
|
||||
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCleanAbandoned).HandlerFunc(h(server.CleanAbandonedDataHandler))
|
||||
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteBulk).HandlerFunc(h(server.DeleteBulkHandler))
|
||||
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadParts).HandlerFunc(h(server.ReadPartsHandler))
|
||||
|
||||
@@ -33,18 +33,17 @@ func _() {
|
||||
_ = x[storageMetricReadXL-22]
|
||||
_ = x[storageMetricReadAll-23]
|
||||
_ = x[storageMetricStatInfoFile-24]
|
||||
_ = x[storageMetricReadMultiple-25]
|
||||
_ = x[storageMetricDeleteAbandonedParts-26]
|
||||
_ = x[storageMetricDiskInfo-27]
|
||||
_ = x[storageMetricDeleteBulk-28]
|
||||
_ = x[storageMetricRenamePart-29]
|
||||
_ = x[storageMetricReadParts-30]
|
||||
_ = x[storageMetricLast-31]
|
||||
_ = x[storageMetricDeleteAbandonedParts-25]
|
||||
_ = x[storageMetricDiskInfo-26]
|
||||
_ = x[storageMetricDeleteBulk-27]
|
||||
_ = x[storageMetricRenamePart-28]
|
||||
_ = x[storageMetricReadParts-29]
|
||||
_ = x[storageMetricLast-30]
|
||||
}
|
||||
|
||||
const _storageMetric_name = "MakeVolBulkMakeVolListVolsStatVolDeleteVolWalkDirListDirReadFileAppendFileCreateFileReadFileStreamRenameFileRenameDataCheckPartsDeleteDeleteVersionsVerifyFileWriteAllDeleteVersionWriteMetadataUpdateMetadataReadVersionReadXLReadAllStatInfoFileReadMultipleDeleteAbandonedPartsDiskInfoDeleteBulkRenamePartReadPartsLast"
|
||||
const _storageMetric_name = "MakeVolBulkMakeVolListVolsStatVolDeleteVolWalkDirListDirReadFileAppendFileCreateFileReadFileStreamRenameFileRenameDataCheckPartsDeleteDeleteVersionsVerifyFileWriteAllDeleteVersionWriteMetadataUpdateMetadataReadVersionReadXLReadAllStatInfoFileDeleteAbandonedPartsDiskInfoDeleteBulkRenamePartReadPartsLast"
|
||||
|
||||
var _storageMetric_index = [...]uint16{0, 11, 18, 26, 33, 42, 49, 56, 64, 74, 84, 98, 108, 118, 128, 134, 148, 158, 166, 179, 192, 206, 217, 223, 230, 242, 254, 274, 282, 292, 302, 311, 315}
|
||||
var _storageMetric_index = [...]uint16{0, 11, 18, 26, 33, 42, 49, 56, 64, 74, 84, 98, 108, 118, 128, 134, 148, 158, 166, 179, 192, 206, 217, 223, 230, 242, 262, 270, 280, 290, 299, 303}
|
||||
|
||||
func (i storageMetric) String() string {
|
||||
if i >= storageMetric(len(_storageMetric_index)-1) {
|
||||
|
||||
@@ -26,12 +26,24 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// newUnsignedV4ChunkedReader returns a new s3UnsignedChunkedReader that translates the data read from r
|
||||
// out of HTTP "chunked" format before returning it.
|
||||
// newUnsignedV4ChunkedReader returns a new s3UnsignedChunkedReader that translates
|
||||
// the data read from r out of HTTP "chunked" format before returning it.
|
||||
//
|
||||
// STREAMING-UNSIGNED-PAYLOAD-TRAILER requests cross the auth boundary here because
|
||||
// the handler starts consuming the body immediately after this reader is created.
|
||||
// Header-authenticated requests must therefore be validated before any body bytes
|
||||
// are read, preserving the security boundary added in #21103, while presigned
|
||||
// query-string auth is intentionally rejected for this streaming mode.
|
||||
//
|
||||
// The s3ChunkedReader returns io.EOF when the final 0-length chunk is read.
|
||||
func newUnsignedV4ChunkedReader(req *http.Request, trailer bool, signature bool) (io.ReadCloser, APIErrorCode) {
|
||||
if signature {
|
||||
if errCode := doesSignatureMatch(unsignedPayloadTrailer, req, globalSite.Region(), serviceS3); errCode != ErrNone {
|
||||
func newUnsignedV4ChunkedReader(req *http.Request, trailer bool) (io.ReadCloser, APIErrorCode) {
|
||||
// Reject any request that claims presigned auth parameters are in use, even if
|
||||
// the query is incomplete, to avoid silently downgrading it into the anonymous path.
|
||||
if isRequestPresignedSignatureV4(req) {
|
||||
return nil, ErrSignatureVersionNotSupported
|
||||
}
|
||||
if isRequestSignatureV4(req) {
|
||||
if errCode := reqSignatureV4Verify(req, globalSite.Region(), serviceS3); errCode != ErrNone {
|
||||
return nil, errCode
|
||||
}
|
||||
}
|
||||
|
||||
+332
-4
@@ -25,14 +25,17 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
idldap "github.com/minio/minio/internal/config/identity/ldap"
|
||||
"github.com/minio/minio/internal/config/identity/openid"
|
||||
"github.com/minio/minio/internal/hash/sha256"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
@@ -91,6 +94,15 @@ const (
|
||||
|
||||
// maximum supported STS session policy size
|
||||
maxSTSSessionPolicySize = 2048
|
||||
|
||||
stsLDAPLoginBurst = 10
|
||||
stsLDAPLoginEntryTTL = 15 * time.Minute
|
||||
stsLDAPLoginRetryAfterSec = int((time.Minute / stsLDAPLoginBurst) / time.Second)
|
||||
)
|
||||
|
||||
var (
|
||||
errLDAPAuthenticationFailed = errors.New("LDAP authentication failed")
|
||||
globalSTSLDAPLoginRateLimiter = newSTSLDAPLoginRateLimiter(time.Minute/time.Duration(stsLDAPLoginBurst), stsLDAPLoginBurst, stsLDAPLoginEntryTTL)
|
||||
)
|
||||
|
||||
type stsClaims map[string]any
|
||||
@@ -254,6 +266,308 @@ func getTokenSigningKey() (string, error) {
|
||||
return secret, nil
|
||||
}
|
||||
|
||||
// stsLDAPLoginRateLimiter throttles LDAP STS logins per source IP. It does not
|
||||
// bucket by username on purpose: a username-keyed bucket is shared across all
|
||||
// sources, so a single client sending bad-password attempts for a known account
|
||||
// could keep that account's bucket drained and lock the legitimate user out.
|
||||
//
|
||||
// Scope of protection: the per-source bucket caps the attempt rate from any one
|
||||
// source, and the uniform auth-failure response (not this limiter) is what hides
|
||||
// whether a username exists. It does not stop attackers who spread across many
|
||||
// sources (botnets, IPv6 address rotation) or the residual bind-timing side
|
||||
// channel; those are accepted limitations of an in-memory, per-source control.
|
||||
type stsLDAPLoginRateLimiter struct {
|
||||
source *stsLDAPLoginKeyLimiterSet
|
||||
}
|
||||
|
||||
type stsLDAPLoginReservation struct {
|
||||
source *stsLDAPLoginKeyReservation
|
||||
}
|
||||
|
||||
type stsLDAPLoginKeyLimiterSet struct {
|
||||
mu sync.Mutex
|
||||
refillEvery time.Duration
|
||||
burst int
|
||||
ttl time.Duration
|
||||
lastCleanup time.Time
|
||||
entries map[string]*stsLDAPLoginKeyLimiter
|
||||
}
|
||||
|
||||
type stsLDAPLoginKeyLimiter struct {
|
||||
tokens float64
|
||||
lastRefill time.Time
|
||||
lastSeen time.Time
|
||||
inFlight int
|
||||
}
|
||||
|
||||
type stsLDAPLoginKeyReservation struct {
|
||||
set *stsLDAPLoginKeyLimiterSet
|
||||
entry *stsLDAPLoginKeyLimiter
|
||||
finalized bool
|
||||
}
|
||||
|
||||
func newSTSLDAPLoginRateLimiter(refillEvery time.Duration, burst int, ttl time.Duration) *stsLDAPLoginRateLimiter {
|
||||
return &stsLDAPLoginRateLimiter{
|
||||
source: newSTSLDAPLoginKeyLimiterSet(refillEvery, burst, ttl),
|
||||
}
|
||||
}
|
||||
|
||||
func newSTSLDAPLoginKeyLimiterSet(refillEvery time.Duration, burst int, ttl time.Duration) *stsLDAPLoginKeyLimiterSet {
|
||||
return &stsLDAPLoginKeyLimiterSet{
|
||||
refillEvery: refillEvery,
|
||||
burst: burst,
|
||||
ttl: ttl,
|
||||
entries: make(map[string]*stsLDAPLoginKeyLimiter),
|
||||
}
|
||||
}
|
||||
|
||||
func (l *stsLDAPLoginRateLimiter) Allow(sourceIP string) bool {
|
||||
reservation := l.Reserve(sourceIP)
|
||||
if reservation == nil {
|
||||
return false
|
||||
}
|
||||
reservation.Commit()
|
||||
return true
|
||||
}
|
||||
|
||||
func (l *stsLDAPLoginRateLimiter) Reserve(sourceIP string) *stsLDAPLoginReservation {
|
||||
// An empty source IP means we could not identify the peer; do not throttle
|
||||
// rather than collapse every such request into one shared bucket.
|
||||
if sourceIP == "" {
|
||||
return &stsLDAPLoginReservation{}
|
||||
}
|
||||
|
||||
source := l.source.Reserve(UTCNow(), sourceIP)
|
||||
if source == nil {
|
||||
return nil
|
||||
}
|
||||
return &stsLDAPLoginReservation{source: source}
|
||||
}
|
||||
|
||||
func (r *stsLDAPLoginReservation) Commit() {
|
||||
if r == nil || r.source == nil {
|
||||
return
|
||||
}
|
||||
r.source.CommitAt(UTCNow())
|
||||
r.source = nil
|
||||
}
|
||||
|
||||
func (r *stsLDAPLoginReservation) Cancel() {
|
||||
if r == nil || r.source == nil {
|
||||
return
|
||||
}
|
||||
r.source.CancelAt(UTCNow())
|
||||
r.source = nil
|
||||
}
|
||||
|
||||
func (l *stsLDAPLoginKeyLimiterSet) Allow(now time.Time, key string) bool {
|
||||
reservation := l.Reserve(now, key)
|
||||
if reservation == nil {
|
||||
return false
|
||||
}
|
||||
reservation.CommitAt(now)
|
||||
return true
|
||||
}
|
||||
|
||||
func (l *stsLDAPLoginKeyLimiterSet) Reserve(now time.Time, key string) *stsLDAPLoginKeyReservation {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
if l.lastCleanup.IsZero() || now.Sub(l.lastCleanup) >= l.ttl {
|
||||
l.cleanup(now)
|
||||
l.lastCleanup = now
|
||||
}
|
||||
|
||||
entry := l.getOrCreateLocked(now, key)
|
||||
l.refillLocked(now, entry)
|
||||
if entry.tokens < 1 {
|
||||
entry.lastSeen = now
|
||||
return nil
|
||||
}
|
||||
|
||||
entry.tokens--
|
||||
entry.inFlight++
|
||||
entry.lastSeen = now
|
||||
return &stsLDAPLoginKeyReservation{set: l, entry: entry}
|
||||
}
|
||||
|
||||
func (l *stsLDAPLoginKeyLimiterSet) cleanup(now time.Time) {
|
||||
for key, entry := range l.entries {
|
||||
if entry.inFlight == 0 && now.Sub(entry.lastSeen) > l.ttl {
|
||||
delete(l.entries, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (l *stsLDAPLoginKeyLimiterSet) getOrCreateLocked(now time.Time, key string) *stsLDAPLoginKeyLimiter {
|
||||
entry, ok := l.entries[key]
|
||||
if !ok {
|
||||
entry = &stsLDAPLoginKeyLimiter{
|
||||
tokens: float64(l.burst),
|
||||
lastRefill: now,
|
||||
lastSeen: now,
|
||||
}
|
||||
l.entries[key] = entry
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
func (l *stsLDAPLoginKeyLimiterSet) refillLocked(now time.Time, entry *stsLDAPLoginKeyLimiter) {
|
||||
if entry.lastRefill.IsZero() {
|
||||
entry.lastRefill = now
|
||||
}
|
||||
|
||||
lastRefill := entry.lastRefill
|
||||
if now.Before(lastRefill) {
|
||||
lastRefill = now
|
||||
}
|
||||
if l.refillEvery <= 0 {
|
||||
entry.lastRefill = now
|
||||
entry.tokens = l.maxTokensLocked(entry)
|
||||
return
|
||||
}
|
||||
|
||||
entry.tokens += float64(now.Sub(lastRefill)) / float64(l.refillEvery)
|
||||
if maxTokens := l.maxTokensLocked(entry); entry.tokens > maxTokens {
|
||||
entry.tokens = maxTokens
|
||||
}
|
||||
entry.lastRefill = now
|
||||
}
|
||||
|
||||
func (l *stsLDAPLoginKeyLimiterSet) maxTokensLocked(entry *stsLDAPLoginKeyLimiter) float64 {
|
||||
maxTokens := l.burst - entry.inFlight
|
||||
if maxTokens < 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(maxTokens)
|
||||
}
|
||||
|
||||
func (r *stsLDAPLoginKeyReservation) CommitAt(now time.Time) {
|
||||
r.finalize(now, false)
|
||||
}
|
||||
|
||||
func (r *stsLDAPLoginKeyReservation) CancelAt(now time.Time) {
|
||||
r.finalize(now, true)
|
||||
}
|
||||
|
||||
func (r *stsLDAPLoginKeyReservation) finalize(now time.Time, refund bool) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
|
||||
r.set.mu.Lock()
|
||||
defer r.set.mu.Unlock()
|
||||
|
||||
if r.finalized {
|
||||
return
|
||||
}
|
||||
|
||||
r.set.refillLocked(now, r.entry)
|
||||
r.entry.lastSeen = now
|
||||
if r.entry.inFlight > 0 {
|
||||
r.entry.inFlight--
|
||||
}
|
||||
if refund {
|
||||
r.entry.tokens++
|
||||
if maxTokens := r.set.maxTokensLocked(r.entry); r.entry.tokens > maxTokens {
|
||||
r.entry.tokens = maxTokens
|
||||
}
|
||||
}
|
||||
|
||||
r.finalized = true
|
||||
}
|
||||
|
||||
func getSTSLDAPLoginSourceIP(r *http.Request) string {
|
||||
peerIP := getSTSLDAPLoginCanonicalIP(r.RemoteAddr)
|
||||
sourceIP := peerIP
|
||||
if sourceIP == "" {
|
||||
sourceIP = getSTSLDAPLoginPeerAddr(r.RemoteAddr)
|
||||
}
|
||||
if peerIP != "" && globalIAMSys != nil && globalIAMSys.LDAPConfig.IsSTSTrustedProxy(peerIP) {
|
||||
if forwardedIP := getSTSLDAPTrustedProxySourceIP(r); forwardedIP != "" {
|
||||
return forwardedIP
|
||||
}
|
||||
}
|
||||
return sourceIP
|
||||
}
|
||||
|
||||
// getSTSLDAPTrustedProxySourceIP resolves the client IP for a request whose peer
|
||||
// is an allow-listed trusted proxy. A single clean X-Real-IP is preferred; for
|
||||
// X-Forwarded-For we walk the chain right-to-left and skip trusted-proxy hops,
|
||||
// returning the first untrusted address. The XFF result ignores any client-
|
||||
// supplied (left-most) value unless the entire chain to its right is trusted,
|
||||
// which an external client cannot forge.
|
||||
//
|
||||
// X-Real-IP, unlike XFF, is a single value with no chain, so it cannot be
|
||||
// validated against the allowlist: it is trusted verbatim. The deployment
|
||||
// contract is therefore that the trusted proxy MUST overwrite (not pass through)
|
||||
// any client-supplied X-Real-IP; otherwise an attacker can vary it per request
|
||||
// to evade per-source throttling. This is the standard reverse-proxy real-IP
|
||||
// contract; reordering to prefer XFF would not remove the dependency, only move
|
||||
// it (an X-Real-IP-only proxy would then be evaded via an injected XFF header).
|
||||
//
|
||||
// The RFC 7239 Forwarded header is intentionally not honored here; such
|
||||
// deployments fall back to the safe peer-address bucket.
|
||||
func getSTSLDAPTrustedProxySourceIP(r *http.Request) string {
|
||||
if realIP := getSTSLDAPLoginCanonicalIP(r.Header.Get("X-Real-IP")); realIP != "" {
|
||||
return realIP
|
||||
}
|
||||
|
||||
forwarded := strings.Split(r.Header.Get("X-Forwarded-For"), ",")
|
||||
for i := len(forwarded) - 1; i >= 0; i-- {
|
||||
ip := getSTSLDAPLoginCanonicalIP(forwarded[i])
|
||||
if ip == "" || globalIAMSys.LDAPConfig.IsSTSTrustedProxy(ip) {
|
||||
continue
|
||||
}
|
||||
return ip
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func getSTSLDAPLoginPeerAddr(remoteAddr string) string {
|
||||
sourceIP, _, err := net.SplitHostPort(remoteAddr)
|
||||
if err == nil {
|
||||
return sourceIP
|
||||
}
|
||||
return remoteAddr
|
||||
}
|
||||
|
||||
func getSTSLDAPLoginCanonicalIP(addr string) string {
|
||||
addr = strings.TrimSpace(getSTSLDAPLoginPeerAddr(addr))
|
||||
addr = strings.TrimPrefix(addr, "[")
|
||||
addr = strings.TrimSuffix(addr, "]")
|
||||
if ip := net.ParseIP(addr); ip != nil {
|
||||
return ip.String()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// reserveSTSLDAPLogin acquires an immediate token from the per-source limiter
|
||||
// before contacting LDAP. Call Commit on auth failures and Cancel when the
|
||||
// attempt should not count as an authentication failure.
|
||||
func reserveSTSLDAPLogin(r *http.Request) *stsLDAPLoginReservation {
|
||||
return globalSTSLDAPLoginRateLimiter.Reserve(getSTSLDAPLoginSourceIP(r))
|
||||
}
|
||||
|
||||
func ldapBindErrorToSTS(err error) (STSErrorCode, error) {
|
||||
if idldap.IsAuthError(err) {
|
||||
return ErrSTSInvalidParameterValue, errLDAPAuthenticationFailed
|
||||
}
|
||||
return ErrSTSUpstreamError, nil
|
||||
}
|
||||
|
||||
func writeSTSThrottledResponse(w http.ResponseWriter) {
|
||||
stsErrorResponse := STSErrorResponse{}
|
||||
stsErrorResponse.Error.Code = "ThrottlingException"
|
||||
stsErrorResponse.Error.Message = "Request throttled, please retry later."
|
||||
stsErrorResponse.RequestID = w.Header().Get(xhttp.AmzRequestID)
|
||||
|
||||
w.Header().Set("Retry-After", strconv.Itoa(stsLDAPLoginRetryAfterSec))
|
||||
|
||||
encodedErrorResponse := encodeResponse(stsErrorResponse)
|
||||
writeResponse(w, http.StatusTooManyRequests, encodedErrorResponse, mimeXML)
|
||||
}
|
||||
|
||||
// AssumeRole - implementation of AWS STS API AssumeRole to get temporary
|
||||
// credentials for regular users on Minio.
|
||||
// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
|
||||
@@ -690,12 +1004,26 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
|
||||
return
|
||||
}
|
||||
|
||||
lookupResult, groupDistNames, err := globalIAMSys.LDAPConfig.Bind(ldapUsername, ldapPassword)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("LDAP server error: %w", err)
|
||||
writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
|
||||
loginReservation := reserveSTSLDAPLogin(r)
|
||||
if loginReservation == nil {
|
||||
writeSTSThrottledResponse(w)
|
||||
return
|
||||
}
|
||||
defer loginReservation.Cancel()
|
||||
|
||||
lookupResult, groupDistNames, err := globalIAMSys.LDAPConfig.Bind(ldapUsername, ldapPassword)
|
||||
if err != nil {
|
||||
errCode, errResp := ldapBindErrorToSTS(err)
|
||||
if errCode == ErrSTSUpstreamError {
|
||||
loginReservation.Cancel()
|
||||
stsLogIf(ctx, err, logger.ErrorKind)
|
||||
} else {
|
||||
loginReservation.Commit()
|
||||
}
|
||||
writeSTSErrorResponse(ctx, w, errCode, errResp)
|
||||
return
|
||||
}
|
||||
loginReservation.Cancel()
|
||||
ldapUserDN := lookupResult.NormDN
|
||||
ldapActualUserDN := lookupResult.ActualDN
|
||||
|
||||
|
||||
@@ -20,12 +20,19 @@ package cmd
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -977,6 +984,345 @@ func TestIAMWithLDAPServerSuite(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type ldapSTSErrorResult struct {
|
||||
StatusCode int
|
||||
RetryAfter string
|
||||
Code string
|
||||
Message string
|
||||
Body string
|
||||
}
|
||||
|
||||
type ldapSTSHTTPResult struct {
|
||||
StatusCode int
|
||||
RetryAfter string
|
||||
Body string
|
||||
}
|
||||
|
||||
func withGlobalSTSLDAPLoginRateLimiterForTest(limiter *stsLDAPLoginRateLimiter, fn func()) {
|
||||
previous := globalSTSLDAPLoginRateLimiter
|
||||
globalSTSLDAPLoginRateLimiter = limiter
|
||||
defer func() {
|
||||
globalSTSLDAPLoginRateLimiter = previous
|
||||
}()
|
||||
|
||||
fn()
|
||||
}
|
||||
|
||||
func withLDAPSTSTrustedProxiesForTest(t *testing.T, trustedProxies string, fn func()) {
|
||||
t.Helper()
|
||||
|
||||
previousIAMSys := globalIAMSys
|
||||
if globalIAMSys == nil {
|
||||
globalIAMSys = &IAMSys{}
|
||||
}
|
||||
previous := globalIAMSys.LDAPConfig.Clone()
|
||||
if err := globalIAMSys.LDAPConfig.SetSTSTrustedProxies(trustedProxies); err != nil {
|
||||
t.Fatalf("unable to set LDAP STS trusted proxies for test: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
globalIAMSys.LDAPConfig = previous
|
||||
globalIAMSys = previousIAMSys
|
||||
}()
|
||||
|
||||
fn()
|
||||
}
|
||||
|
||||
func singleHeader(key, value string) http.Header {
|
||||
header := make(http.Header)
|
||||
if key != "" {
|
||||
header.Set(key, value)
|
||||
}
|
||||
return header
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) postLDAPSTSWithHeaders(c *check, username, password string, headers http.Header) ldapSTSHTTPResult {
|
||||
c.Helper()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("Action", ldapIdentity)
|
||||
form.Set("Version", stsAPIVersion)
|
||||
form.Set(stsLDAPUsername, username)
|
||||
form.Set(stsLDAPPassword, password)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.endPoint, strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
c.Fatalf("unexpected request creation error: %v", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
for key, values := range headers {
|
||||
for _, value := range values {
|
||||
req.Header.Add(key, value)
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := s.TestSuiteCommon.client.Do(req)
|
||||
if err != nil {
|
||||
c.Fatalf("unexpected LDAP STS request error: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
c.Fatalf("unexpected LDAP STS response read error: %v", err)
|
||||
}
|
||||
|
||||
return ldapSTSHTTPResult{
|
||||
StatusCode: resp.StatusCode,
|
||||
RetryAfter: resp.Header.Get("Retry-After"),
|
||||
Body: string(body),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) postLDAPSTS(c *check, username, password string) ldapSTSHTTPResult {
|
||||
c.Helper()
|
||||
return s.postLDAPSTSWithHeaders(c, username, password, nil)
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) postLDAPSTSForError(c *check, username, password string) ldapSTSErrorResult {
|
||||
c.Helper()
|
||||
|
||||
result := s.postLDAPSTS(c, username, password)
|
||||
if result.StatusCode == http.StatusOK {
|
||||
c.Fatalf("expected LDAP STS request to fail, got success: %s", result.Body)
|
||||
}
|
||||
|
||||
var stsErr STSErrorResponse
|
||||
if err := xml.Unmarshal([]byte(result.Body), &stsErr); err != nil {
|
||||
c.Fatalf("unexpected LDAP STS XML decode error: %v, body: %s", err, result.Body)
|
||||
}
|
||||
|
||||
return ldapSTSErrorResult{
|
||||
StatusCode: result.StatusCode,
|
||||
RetryAfter: result.RetryAfter,
|
||||
Code: stsErr.Error.Code,
|
||||
Message: stsErr.Error.Message,
|
||||
Body: result.Body,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestLDAPSTSAuthFailureUniformResponse(c *check) {
|
||||
withGlobalSTSLDAPLoginRateLimiterForTest(
|
||||
newSTSLDAPLoginRateLimiter(time.Minute, stsLDAPLoginBurst, stsLDAPLoginEntryTTL),
|
||||
func() {
|
||||
missingUser := s.postLDAPSTSForError(c, "missing-user", "nottherightpassword")
|
||||
wrongPassword := s.postLDAPSTSForError(c, "dillon", "nottherightpassword")
|
||||
|
||||
if missingUser.StatusCode != http.StatusBadRequest {
|
||||
c.Fatalf("expected missing-user request status %d, got %d", http.StatusBadRequest, missingUser.StatusCode)
|
||||
}
|
||||
if wrongPassword.StatusCode != http.StatusBadRequest {
|
||||
c.Fatalf("expected wrong-password request status %d, got %d", http.StatusBadRequest, wrongPassword.StatusCode)
|
||||
}
|
||||
if missingUser.Code != "InvalidParameterValue" || wrongPassword.Code != "InvalidParameterValue" {
|
||||
c.Fatalf("expected InvalidParameterValue for both auth failures, got missing=%q wrong=%q", missingUser.Code, wrongPassword.Code)
|
||||
}
|
||||
if missingUser.Message != errLDAPAuthenticationFailed.Error() || wrongPassword.Message != errLDAPAuthenticationFailed.Error() {
|
||||
c.Fatalf("expected uniform LDAP auth failure message, got missing=%q wrong=%q", missingUser.Message, wrongPassword.Message)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(missingUser.Body), "unable to find user dn") {
|
||||
c.Fatalf("missing-user response leaked lookup details: %s", missingUser.Body)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(wrongPassword.Body), "ldap auth failed for dn") {
|
||||
c.Fatalf("wrong-password response leaked bind details: %s", wrongPassword.Body)
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestLDAPSTSRateLimit(c *check) {
|
||||
withGlobalSTSLDAPLoginRateLimiterForTest(
|
||||
newSTSLDAPLoginRateLimiter(time.Hour, 2, stsLDAPLoginEntryTTL),
|
||||
func() {
|
||||
first := s.postLDAPSTSForError(c, "dillon", "nottherightpassword")
|
||||
second := s.postLDAPSTSForError(c, "dillon", "nottherightpassword")
|
||||
throttled := s.postLDAPSTSForError(c, "dillon", "nottherightpassword")
|
||||
|
||||
if first.StatusCode != http.StatusBadRequest || second.StatusCode != http.StatusBadRequest {
|
||||
c.Fatalf("expected first two failed auth attempts to return %d, got %d and %d", http.StatusBadRequest, first.StatusCode, second.StatusCode)
|
||||
}
|
||||
if throttled.StatusCode != http.StatusTooManyRequests {
|
||||
c.Fatalf("expected throttled request status %d, got %d", http.StatusTooManyRequests, throttled.StatusCode)
|
||||
}
|
||||
if throttled.Code != "ThrottlingException" {
|
||||
c.Fatalf("expected throttled code %q, got %q", "ThrottlingException", throttled.Code)
|
||||
}
|
||||
if throttled.Message != "Request throttled, please retry later." {
|
||||
c.Fatalf("expected throttled message %q, got %q", "Request throttled, please retry later.", throttled.Message)
|
||||
}
|
||||
if throttled.RetryAfter != fmt.Sprintf("%d", stsLDAPLoginRetryAfterSec) {
|
||||
c.Fatalf("expected Retry-After %d, got %q", stsLDAPLoginRetryAfterSec, throttled.RetryAfter)
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestLDAPSTSSuccessDoesNotConsumeRateLimit(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
|
||||
userReq := madmin.PolicyAssociationReq{
|
||||
Policies: []string{"consoleAdmin"},
|
||||
User: "uid=dillon,ou=people,ou=swengg,dc=min,dc=io",
|
||||
}
|
||||
if _, err := s.adm.AttachPolicyLDAP(ctx, userReq); err != nil {
|
||||
c.Fatalf("unable to attach LDAP policy for success rate-limit test: %v", err)
|
||||
}
|
||||
|
||||
withGlobalSTSLDAPLoginRateLimiterForTest(
|
||||
newSTSLDAPLoginRateLimiter(time.Hour, 2, stsLDAPLoginEntryTTL),
|
||||
func() {
|
||||
for attempt := 1; attempt <= 3; attempt++ {
|
||||
success := s.postLDAPSTS(c, "dillon", "dillon")
|
||||
if success.StatusCode != http.StatusOK {
|
||||
c.Fatalf("expected successful LDAP STS login on attempt %d, got status %d body: %s", attempt, success.StatusCode, success.Body)
|
||||
}
|
||||
}
|
||||
|
||||
firstFailure := s.postLDAPSTSForError(c, "dillon", "nottherightpassword")
|
||||
secondFailure := s.postLDAPSTSForError(c, "dillon", "nottherightpassword")
|
||||
throttled := s.postLDAPSTSForError(c, "dillon", "nottherightpassword")
|
||||
|
||||
if firstFailure.StatusCode != http.StatusBadRequest || secondFailure.StatusCode != http.StatusBadRequest {
|
||||
c.Fatalf("expected failed auth attempts after successful logins to return %d, got %d and %d", http.StatusBadRequest, firstFailure.StatusCode, secondFailure.StatusCode)
|
||||
}
|
||||
if throttled.StatusCode != http.StatusTooManyRequests {
|
||||
c.Fatalf("expected third failed auth attempt after successful logins to be throttled with %d, got %d", http.StatusTooManyRequests, throttled.StatusCode)
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestLDAPSTSUpstreamFailure(c *check) {
|
||||
original := globalIAMSys.LDAPConfig.Clone()
|
||||
globalIAMSys.LDAPConfig.LDAP.ServerAddr = "127.0.0.1:1"
|
||||
defer func() {
|
||||
globalIAMSys.LDAPConfig = original
|
||||
}()
|
||||
|
||||
withGlobalSTSLDAPLoginRateLimiterForTest(
|
||||
newSTSLDAPLoginRateLimiter(time.Hour, 2, stsLDAPLoginEntryTTL),
|
||||
func() {
|
||||
for range 3 {
|
||||
upstreamFailure := s.postLDAPSTSForError(c, "dillon", "dillon")
|
||||
|
||||
if upstreamFailure.StatusCode != http.StatusInternalServerError {
|
||||
c.Fatalf("expected upstream failure status %d, got %d", http.StatusInternalServerError, upstreamFailure.StatusCode)
|
||||
}
|
||||
if upstreamFailure.Code != "InternalError" {
|
||||
c.Fatalf("expected upstream failure code %q, got %q", "InternalError", upstreamFailure.Code)
|
||||
}
|
||||
if upstreamFailure.Message != stsErrCodes.ToSTSErr(ErrSTSUpstreamError).Description {
|
||||
c.Fatalf("expected upstream failure message %q, got %q", stsErrCodes.ToSTSErr(ErrSTSUpstreamError).Description, upstreamFailure.Message)
|
||||
}
|
||||
if upstreamFailure.Message == errLDAPAuthenticationFailed.Error() {
|
||||
c.Fatalf("expected upstream failure to stay distinct from auth failure, got %q", upstreamFailure.Message)
|
||||
}
|
||||
}
|
||||
|
||||
globalIAMSys.LDAPConfig = original
|
||||
|
||||
authFailure := s.postLDAPSTSForError(c, "dillon", "nottherightpassword")
|
||||
if authFailure.StatusCode == http.StatusTooManyRequests {
|
||||
c.Fatalf("expected upstream failures not to consume rate limit budget, got throttled response: %+v", authFailure)
|
||||
}
|
||||
if authFailure.StatusCode != http.StatusBadRequest {
|
||||
c.Fatalf("expected auth failure after upstream recovery to return %d, got %d", http.StatusBadRequest, authFailure.StatusCode)
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestLDAPSTSTrustedProxyRateLimit(c *check) {
|
||||
withLDAPSTSTrustedProxiesForTest(c.T, "127.0.0.0/8,::1/128", func() {
|
||||
withGlobalSTSLDAPLoginRateLimiterForTest(
|
||||
newSTSLDAPLoginRateLimiter(time.Hour, 1, stsLDAPLoginEntryTTL),
|
||||
func() {
|
||||
// These usernames intentionally do not exist. The test asserts that
|
||||
// LDAP user-not-found stays classified as an auth error, so failed
|
||||
// attempts still commit the reservation and hit the source bucket.
|
||||
first := s.postLDAPSTSWithHeaders(c, "missing-user-a", "nottherightpassword", singleHeader("X-Real-IP", "203.0.113.10"))
|
||||
second := s.postLDAPSTSWithHeaders(c, "missing-user-b", "nottherightpassword", singleHeader("X-Real-IP", "198.51.100.23"))
|
||||
third := s.postLDAPSTSWithHeaders(c, "missing-user-c", "nottherightpassword", singleHeader("X-Real-IP", "203.0.113.10"))
|
||||
|
||||
if first.StatusCode != http.StatusBadRequest {
|
||||
c.Fatalf("expected first trusted-proxy LDAP STS auth failure to return %d, got %d body: %s", http.StatusBadRequest, first.StatusCode, first.Body)
|
||||
}
|
||||
if second.StatusCode != http.StatusBadRequest {
|
||||
c.Fatalf("expected a different forwarded client IP behind the same trusted proxy to avoid source throttling, got %d body: %s", second.StatusCode, second.Body)
|
||||
}
|
||||
if third.StatusCode != http.StatusTooManyRequests {
|
||||
c.Fatalf("expected the same forwarded client IP behind the trusted proxy to be throttled with %d, got %d body: %s", http.StatusTooManyRequests, third.StatusCode, third.Body)
|
||||
}
|
||||
},
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
func TestIAMWithLDAPSecurityServerSuite(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
run func(*TestSuiteIAM, *check, string)
|
||||
}{
|
||||
{
|
||||
name: "AuthFailureUniformResponse",
|
||||
run: func(suite *TestSuiteIAM, c *check, ldapServer string) {
|
||||
suite.TestLDAPSTSAuthFailureUniformResponse(c)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "RateLimit",
|
||||
run: func(suite *TestSuiteIAM, c *check, ldapServer string) {
|
||||
suite.TestLDAPSTSRateLimit(c)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "SuccessDoesNotConsumeRateLimit",
|
||||
run: func(suite *TestSuiteIAM, c *check, ldapServer string) {
|
||||
suite.TestLDAPSTSSuccessDoesNotConsumeRateLimit(c)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "UpstreamFailure",
|
||||
run: func(suite *TestSuiteIAM, c *check, ldapServer string) {
|
||||
suite.TestLDAPSTSUpstreamFailure(c)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "TrustedProxyRateLimit",
|
||||
run: func(suite *TestSuiteIAM, c *check, ldapServer string) {
|
||||
suite.TestLDAPSTSTrustedProxyRateLimit(c)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for i, testCase := range iamTestSuites {
|
||||
t.Run(
|
||||
fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription),
|
||||
func(t *testing.T) {
|
||||
ldapServer := os.Getenv(EnvTestLDAPServer)
|
||||
if ldapServer == "" {
|
||||
t.Skipf("Skipping LDAP security test as no LDAP server is provided via %s", EnvTestLDAPServer)
|
||||
}
|
||||
|
||||
suite := testCase
|
||||
for _, tc := range tests {
|
||||
tc := tc
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
c := &check{t, testCase.serverType}
|
||||
suite.SetUpSuite(c)
|
||||
suite.SetUpLDAP(c, ldapServer)
|
||||
tc.run(suite, c, ldapServer)
|
||||
suite.TearDownSuite(c)
|
||||
})
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// This test is for a fix added to handle non-normalized base DN values in the
|
||||
// LDAP configuration. It runs the existing LDAP sub-tests with a non-normalized
|
||||
// LDAP configuration.
|
||||
@@ -1052,6 +1398,578 @@ func TestIAMExportImportWithLDAP(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type matchingAuthError struct{}
|
||||
|
||||
func (matchingAuthError) Error() string {
|
||||
return "ldap auth failed"
|
||||
}
|
||||
|
||||
func (matchingAuthError) Is(target error) bool {
|
||||
return targetIsLDAPAuthFailure(target)
|
||||
}
|
||||
|
||||
func targetIsLDAPAuthFailure(target error) bool {
|
||||
return target != nil && target.Error() == "ldap authentication failed"
|
||||
}
|
||||
|
||||
func TestSTSLDAPLoginRateLimiter(t *testing.T) {
|
||||
limiter := newSTSLDAPLoginRateLimiter(time.Hour, 2, time.Minute)
|
||||
|
||||
if !limiter.Allow("192.0.2.10") {
|
||||
t.Fatal("expected first attempt to be allowed")
|
||||
}
|
||||
if !limiter.Allow("192.0.2.10") {
|
||||
t.Fatal("expected second attempt within burst to be allowed")
|
||||
}
|
||||
if limiter.Allow("192.0.2.10") {
|
||||
t.Fatal("expected source IP bucket to be throttled after burst")
|
||||
}
|
||||
|
||||
// A different source IP has its own independent bucket, so one client
|
||||
// cannot exhaust another's budget (no per-username lockout dimension).
|
||||
if !limiter.Allow("192.0.2.11") {
|
||||
t.Fatal("expected a different source IP to be allowed")
|
||||
}
|
||||
|
||||
// An empty source IP cannot be identified and must never be throttled,
|
||||
// otherwise all such requests would collapse into one shared bucket.
|
||||
if !limiter.Allow("") || !limiter.Allow("") {
|
||||
t.Fatal("expected unidentified source to stay unthrottled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSTSLDAPLoginRateLimiterReserveCancel(t *testing.T) {
|
||||
limiter := newSTSLDAPLoginRateLimiter(time.Hour, 1, time.Minute)
|
||||
|
||||
reservation := limiter.Reserve("192.0.2.10")
|
||||
if reservation == nil {
|
||||
t.Fatal("expected first reservation to succeed")
|
||||
}
|
||||
if limiter.Reserve("192.0.2.10") != nil {
|
||||
t.Fatal("expected second reservation on the same source IP to be throttled before cancel")
|
||||
}
|
||||
|
||||
reservation.Cancel()
|
||||
|
||||
reservation = limiter.Reserve("192.0.2.10")
|
||||
if reservation == nil {
|
||||
t.Fatal("expected canceled reservation to restore source-IP capacity")
|
||||
}
|
||||
reservation.Cancel()
|
||||
|
||||
reservation = limiter.Reserve("192.0.2.11")
|
||||
if reservation == nil {
|
||||
t.Fatal("expected a different source IP to have independent capacity")
|
||||
}
|
||||
reservation.Cancel()
|
||||
}
|
||||
|
||||
func TestSTSLDAPLoginKeyLimiterCancelDoesNotOverCreditAfterRefill(t *testing.T) {
|
||||
set := newSTSLDAPLoginKeyLimiterSet(10*time.Millisecond, 2, time.Minute)
|
||||
start := time.Unix(0, 0)
|
||||
|
||||
first := set.Reserve(start, "192.0.2.10")
|
||||
if first == nil {
|
||||
t.Fatal("expected first reservation to succeed")
|
||||
}
|
||||
second := set.Reserve(start.Add(5*time.Millisecond), "192.0.2.10")
|
||||
if second == nil {
|
||||
t.Fatal("expected second reservation to succeed while one token remains available")
|
||||
}
|
||||
|
||||
first.CancelAt(start.Add(10 * time.Millisecond))
|
||||
|
||||
third := set.Reserve(start.Add(10*time.Millisecond), "192.0.2.10")
|
||||
if third == nil {
|
||||
t.Fatal("expected canceled reservation to restore exactly one slot")
|
||||
}
|
||||
defer third.CancelAt(start.Add(10 * time.Millisecond))
|
||||
|
||||
if extra := set.Reserve(start.Add(10*time.Millisecond), "192.0.2.10"); extra != nil {
|
||||
extra.CancelAt(start.Add(10 * time.Millisecond))
|
||||
t.Fatal("expected only one slot to be restored after cancel; got over-credit from refill")
|
||||
}
|
||||
|
||||
second.CancelAt(start.Add(10 * time.Millisecond))
|
||||
}
|
||||
|
||||
func TestSTSLDAPLoginRateLimiterConcurrentReserveLifecycle(t *testing.T) {
|
||||
limiter := newSTSLDAPLoginRateLimiter(time.Hour, 4, time.Minute)
|
||||
|
||||
const workers = 8
|
||||
start := make(chan struct{})
|
||||
finish := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
var reserveWG sync.WaitGroup
|
||||
reservations := make([]*stsLDAPLoginReservation, workers)
|
||||
var canceledReservations atomic.Int32
|
||||
|
||||
reserveWG.Add(workers)
|
||||
for i := 0; i < workers; i++ {
|
||||
wg.Add(1)
|
||||
go func(worker int) {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
|
||||
reservations[worker] = limiter.Reserve("192.0.2.10")
|
||||
reserveWG.Done()
|
||||
if reservations[worker] == nil {
|
||||
return
|
||||
}
|
||||
|
||||
<-finish
|
||||
if worker%2 == 0 {
|
||||
canceledReservations.Add(1)
|
||||
reservations[worker].Cancel()
|
||||
return
|
||||
}
|
||||
|
||||
reservations[worker].Commit()
|
||||
}(i)
|
||||
}
|
||||
|
||||
close(start)
|
||||
reserveWG.Wait()
|
||||
|
||||
successfulReservations := 0
|
||||
for _, reservation := range reservations {
|
||||
if reservation != nil {
|
||||
successfulReservations++
|
||||
}
|
||||
}
|
||||
if got := successfulReservations; got != 4 {
|
||||
t.Fatalf("expected exactly 4 successful concurrent reservations, got %d", got)
|
||||
}
|
||||
|
||||
close(finish)
|
||||
wg.Wait()
|
||||
|
||||
remainingBudget := 0
|
||||
for {
|
||||
reservation := limiter.Reserve("192.0.2.10")
|
||||
if reservation == nil {
|
||||
break
|
||||
}
|
||||
remainingBudget++
|
||||
reservation.Commit()
|
||||
}
|
||||
|
||||
if want, got := int(canceledReservations.Load()), remainingBudget; got != want {
|
||||
t.Fatalf("expected %d tokens to remain after concurrent commit/cancel mix, got %d", want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetSTSLDAPLoginSourceIP(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
remoteAddr string
|
||||
want string
|
||||
}{
|
||||
{name: "empty", remoteAddr: "", want: ""},
|
||||
{name: "ipv4", remoteAddr: "192.0.2.10:9000", want: "192.0.2.10"},
|
||||
{name: "ipv6", remoteAddr: "[2001:db8::10]:9000", want: "2001:db8::10"},
|
||||
{name: "bare-host", remoteAddr: "192.0.2.10", want: "192.0.2.10"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req := &http.Request{RemoteAddr: tt.remoteAddr}
|
||||
if got := getSTSLDAPLoginSourceIP(req); got != tt.want {
|
||||
t.Fatalf("expected %q, got %q", tt.want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetSTSLDAPLoginSourceIPIgnoresSpoofedForwardingHeaders(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
headerKey string
|
||||
headerValue string
|
||||
}{
|
||||
{
|
||||
name: "x-forwarded-for",
|
||||
headerKey: "X-Forwarded-For",
|
||||
headerValue: "203.0.113.10, 198.51.100.24",
|
||||
},
|
||||
{
|
||||
name: "x-real-ip",
|
||||
headerKey: "X-Real-IP",
|
||||
headerValue: "203.0.113.10",
|
||||
},
|
||||
{
|
||||
name: "forwarded",
|
||||
headerKey: "Forwarded",
|
||||
headerValue: `for=203.0.113.10;proto=https`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req := &http.Request{
|
||||
Header: singleHeader(tt.headerKey, tt.headerValue),
|
||||
RemoteAddr: "192.0.2.10:9000",
|
||||
}
|
||||
|
||||
if got := getSTSLDAPLoginSourceIP(req); got != "192.0.2.10" {
|
||||
t.Fatalf("expected helper to ignore spoofed %s and return peer address, got %q", tt.headerKey, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetSTSLDAPLoginSourceIPUsesForwardedHeadersForTrustedProxy(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
headerKey string
|
||||
headerValue string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "x-forwarded-for",
|
||||
headerKey: "X-Forwarded-For",
|
||||
headerValue: "203.0.113.10",
|
||||
want: "203.0.113.10",
|
||||
},
|
||||
{
|
||||
name: "x-real-ip",
|
||||
headerKey: "X-Real-IP",
|
||||
headerValue: "203.0.113.10",
|
||||
want: "203.0.113.10",
|
||||
},
|
||||
}
|
||||
|
||||
withLDAPSTSTrustedProxiesForTest(t, "192.0.2.0/24", func() {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req := &http.Request{
|
||||
Header: singleHeader(tt.headerKey, tt.headerValue),
|
||||
RemoteAddr: "192.0.2.10:9000",
|
||||
}
|
||||
|
||||
if got := getSTSLDAPLoginSourceIP(req); got != tt.want {
|
||||
t.Fatalf("expected trusted proxy header %s to resolve %q, got %q", tt.headerKey, tt.want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// A client behind a trusted, appending proxy can prepend a spoofed left-most
|
||||
// X-Forwarded-For value. The right-to-left walk must skip only trusted hops and
|
||||
// return the real (right-most untrusted) client, ignoring the spoofed value.
|
||||
func TestGetSTSLDAPLoginSourceIPTrustedProxyStripsSpoofedForwardedFor(t *testing.T) {
|
||||
withLDAPSTSTrustedProxiesForTest(t, "192.0.2.0/24", func() {
|
||||
req := &http.Request{
|
||||
Header: singleHeader("X-Forwarded-For", "1.2.3.4, 198.51.100.50"),
|
||||
RemoteAddr: "192.0.2.10:9000",
|
||||
}
|
||||
if got := getSTSLDAPLoginSourceIP(req); got != "198.51.100.50" {
|
||||
t.Fatalf("expected spoofed left-most XFF entry to be ignored and real client returned, got %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// When several hops in the chain are trusted proxies, the walk skips all of
|
||||
// them and resolves the left-most (real client) address.
|
||||
func TestGetSTSLDAPLoginSourceIPTrustedProxyWalksMultipleTrustedHops(t *testing.T) {
|
||||
withLDAPSTSTrustedProxiesForTest(t, "192.0.2.0/24", func() {
|
||||
req := &http.Request{
|
||||
Header: singleHeader("X-Forwarded-For", "203.0.113.10, 192.0.2.20, 192.0.2.21"),
|
||||
RemoteAddr: "192.0.2.10:9000",
|
||||
}
|
||||
if got := getSTSLDAPLoginSourceIP(req); got != "203.0.113.10" {
|
||||
t.Fatalf("expected walk to skip trusted hops and return real client, got %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// The RFC 7239 Forwarded header is not honored for trusted-proxy bucketing; such
|
||||
// requests fall back to the safe peer-address bucket.
|
||||
func TestGetSTSLDAPLoginSourceIPTrustedProxyIgnoresForwardedHeader(t *testing.T) {
|
||||
withLDAPSTSTrustedProxiesForTest(t, "192.0.2.0/24", func() {
|
||||
req := &http.Request{
|
||||
Header: singleHeader("Forwarded", `for=203.0.113.10;proto=https`),
|
||||
RemoteAddr: "192.0.2.10:9000",
|
||||
}
|
||||
if got := getSTSLDAPLoginSourceIP(req); got != "192.0.2.10" {
|
||||
t.Fatalf("expected RFC 7239 Forwarded to be ignored and peer address used, got %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetSTSLDAPLoginSourceIPTrustedProxyPrefersXRealIPOverXForwardedFor(t *testing.T) {
|
||||
withLDAPSTSTrustedProxiesForTest(t, "192.0.2.0/24", func() {
|
||||
req := &http.Request{
|
||||
Header: make(http.Header),
|
||||
RemoteAddr: "192.0.2.10:9000",
|
||||
}
|
||||
req.Header.Set("X-Forwarded-For", "198.51.100.99, 203.0.113.10")
|
||||
req.Header.Set("X-Real-IP", "203.0.113.10")
|
||||
|
||||
if got := getSTSLDAPLoginSourceIP(req); got != "203.0.113.10" {
|
||||
t.Fatalf("expected trusted proxy path to prefer X-Real-IP over appended X-Forwarded-For, got %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// X-Real-IP is trusted verbatim (it cannot be chain-validated like X-Forwarded-For),
|
||||
// so a client-supplied X-Real-IP that the proxy fails to overwrite wins even over a
|
||||
// correctly appended X-Forwarded-For chain. This locks the documented deployment
|
||||
// contract: the trusted proxy MUST overwrite X-Real-IP, otherwise it is a spoofing
|
||||
// vector. If this assertion ever changes, the change must be deliberate.
|
||||
func TestGetSTSLDAPLoginSourceIPTrustedProxyTrustsXRealIPVerbatim(t *testing.T) {
|
||||
withLDAPSTSTrustedProxiesForTest(t, "192.0.2.0/24", func() {
|
||||
req := &http.Request{
|
||||
Header: make(http.Header),
|
||||
RemoteAddr: "192.0.2.10:9000",
|
||||
}
|
||||
// Attacker spoofs X-Real-IP with a value that appears nowhere in the XFF
|
||||
// chain; the trusted proxy still appends the real client (198.51.100.50).
|
||||
// The spoofed X-Real-IP wins, so the result can only have come from it.
|
||||
req.Header.Set("X-Real-IP", "10.10.10.10")
|
||||
req.Header.Set("X-Forwarded-For", "1.1.1.1, 198.51.100.50")
|
||||
|
||||
if got := getSTSLDAPLoginSourceIP(req); got != "10.10.10.10" {
|
||||
t.Fatalf("expected verbatim X-Real-IP trust (spoofable contract), got %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetSTSLDAPLoginSourceIPTrustedProxyFallsBackToPeerWithoutForwardingHeaders(t *testing.T) {
|
||||
withLDAPSTSTrustedProxiesForTest(t, "192.0.2.0/24", func() {
|
||||
req := &http.Request{RemoteAddr: "192.0.2.10:9000"}
|
||||
if got := getSTSLDAPLoginSourceIP(req); got != "192.0.2.10" {
|
||||
t.Fatalf("expected trusted proxy path without forwarding headers to fall back to peer address, got %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestReserveSTSLDAPLoginUsesPeerAddressBuckets(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
headerKey string
|
||||
firstValue string
|
||||
secondValue string
|
||||
}{
|
||||
{
|
||||
name: "x-forwarded-for",
|
||||
headerKey: "X-Forwarded-For",
|
||||
firstValue: "203.0.113.10",
|
||||
secondValue: "198.51.100.23, 198.51.100.24",
|
||||
},
|
||||
{
|
||||
name: "x-real-ip",
|
||||
headerKey: "X-Real-IP",
|
||||
firstValue: "203.0.113.10",
|
||||
secondValue: "198.51.100.23",
|
||||
},
|
||||
{
|
||||
name: "forwarded",
|
||||
headerKey: "Forwarded",
|
||||
firstValue: `for=203.0.113.10;proto=https`,
|
||||
secondValue: `for=198.51.100.23;proto=https`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name+"-same-peer", func(t *testing.T) {
|
||||
limiter := newSTSLDAPLoginRateLimiter(time.Hour, 2, time.Minute)
|
||||
withGlobalSTSLDAPLoginRateLimiterForTest(limiter, func() {
|
||||
req1 := &http.Request{
|
||||
Header: singleHeader(tt.headerKey, tt.firstValue),
|
||||
RemoteAddr: "192.0.2.10:9000",
|
||||
Form: url.Values{stsLDAPUsername: []string{"alice"}},
|
||||
}
|
||||
req2 := &http.Request{
|
||||
Header: singleHeader(tt.headerKey, tt.secondValue),
|
||||
RemoteAddr: "192.0.2.10:9001",
|
||||
Form: url.Values{stsLDAPUsername: []string{"bob"}},
|
||||
}
|
||||
|
||||
reservation1 := reserveSTSLDAPLogin(req1)
|
||||
if reservation1 == nil {
|
||||
t.Fatal("expected first reservation to succeed")
|
||||
}
|
||||
defer reservation1.Cancel()
|
||||
|
||||
reservation2 := reserveSTSLDAPLogin(req2)
|
||||
if reservation2 == nil {
|
||||
t.Fatal("expected second reservation to succeed with shared source bucket capacity")
|
||||
}
|
||||
defer reservation2.Cancel()
|
||||
|
||||
if got := len(limiter.source.entries); got != 1 {
|
||||
t.Fatalf("expected requests from the same peer address to share one source bucket, got %d", got)
|
||||
}
|
||||
if _, ok := limiter.source.entries["192.0.2.10"]; !ok {
|
||||
t.Fatalf("expected source bucket for peer address %q, got keys %v", "192.0.2.10", limiter.source.entries)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
t.Run(tt.name+"-different-peers", func(t *testing.T) {
|
||||
limiter := newSTSLDAPLoginRateLimiter(time.Hour, 1, time.Minute)
|
||||
withGlobalSTSLDAPLoginRateLimiterForTest(limiter, func() {
|
||||
req1 := &http.Request{
|
||||
Header: singleHeader(tt.headerKey, tt.firstValue),
|
||||
RemoteAddr: "192.0.2.10:9000",
|
||||
Form: url.Values{stsLDAPUsername: []string{"alice"}},
|
||||
}
|
||||
req2 := &http.Request{
|
||||
Header: singleHeader(tt.headerKey, tt.firstValue),
|
||||
RemoteAddr: "192.0.2.11:9000",
|
||||
Form: url.Values{stsLDAPUsername: []string{"bob"}},
|
||||
}
|
||||
|
||||
reservation1 := reserveSTSLDAPLogin(req1)
|
||||
if reservation1 == nil {
|
||||
t.Fatal("expected first reservation to succeed")
|
||||
}
|
||||
defer reservation1.Cancel()
|
||||
|
||||
reservation2 := reserveSTSLDAPLogin(req2)
|
||||
if reservation2 == nil {
|
||||
t.Fatal("expected second reservation from a different peer address to succeed")
|
||||
}
|
||||
defer reservation2.Cancel()
|
||||
|
||||
if got := len(limiter.source.entries); got != 2 {
|
||||
t.Fatalf("expected requests from different peer addresses to use different source buckets, got %d", got)
|
||||
}
|
||||
if _, ok := limiter.source.entries["192.0.2.10"]; !ok {
|
||||
t.Fatalf("expected source bucket for peer address %q, got keys %v", "192.0.2.10", limiter.source.entries)
|
||||
}
|
||||
if _, ok := limiter.source.entries["192.0.2.11"]; !ok {
|
||||
t.Fatalf("expected source bucket for peer address %q, got keys %v", "192.0.2.11", limiter.source.entries)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReserveSTSLDAPLoginUsesForwardedBucketsForTrustedProxy(t *testing.T) {
|
||||
limiter := newSTSLDAPLoginRateLimiter(time.Hour, 1, time.Minute)
|
||||
withGlobalSTSLDAPLoginRateLimiterForTest(limiter, func() {
|
||||
withLDAPSTSTrustedProxiesForTest(t, "192.0.2.0/24", func() {
|
||||
req1 := &http.Request{
|
||||
Header: singleHeader("X-Forwarded-For", "203.0.113.10"),
|
||||
RemoteAddr: "192.0.2.10:9000",
|
||||
Form: url.Values{stsLDAPUsername: []string{"alice"}},
|
||||
}
|
||||
req2 := &http.Request{
|
||||
Header: singleHeader("X-Forwarded-For", "198.51.100.23"),
|
||||
RemoteAddr: "192.0.2.10:9001",
|
||||
Form: url.Values{stsLDAPUsername: []string{"bob"}},
|
||||
}
|
||||
|
||||
reservation1 := reserveSTSLDAPLogin(req1)
|
||||
if reservation1 == nil {
|
||||
t.Fatal("expected first reservation through trusted proxy to succeed")
|
||||
}
|
||||
defer reservation1.Cancel()
|
||||
|
||||
reservation2 := reserveSTSLDAPLogin(req2)
|
||||
if reservation2 == nil {
|
||||
t.Fatal("expected forwarded client IPs behind the same trusted proxy to use distinct source buckets")
|
||||
}
|
||||
defer reservation2.Cancel()
|
||||
|
||||
if got := len(limiter.source.entries); got != 2 {
|
||||
t.Fatalf("expected distinct forwarded client IPs to use two source buckets, got %d", got)
|
||||
}
|
||||
if _, ok := limiter.source.entries["203.0.113.10"]; !ok {
|
||||
t.Fatalf("expected source bucket for forwarded client IP %q, got keys %v", "203.0.113.10", limiter.source.entries)
|
||||
}
|
||||
if _, ok := limiter.source.entries["198.51.100.23"]; !ok {
|
||||
t.Fatalf("expected source bucket for forwarded client IP %q, got keys %v", "198.51.100.23", limiter.source.entries)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestLDAPBindErrorToSTS(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
code STSErrorCode
|
||||
message string
|
||||
}{
|
||||
{
|
||||
name: "auth failure",
|
||||
err: matchingAuthError{},
|
||||
code: ErrSTSInvalidParameterValue,
|
||||
message: errLDAPAuthenticationFailed.Error(),
|
||||
},
|
||||
{
|
||||
name: "upstream failure",
|
||||
err: errors.New("ldap server unavailable"),
|
||||
code: ErrSTSUpstreamError,
|
||||
message: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
code, err := ldapBindErrorToSTS(tt.err)
|
||||
if code != tt.code {
|
||||
t.Fatalf("expected code %v, got %v", tt.code, code)
|
||||
}
|
||||
if tt.message == "" {
|
||||
if err != nil {
|
||||
t.Fatalf("expected nil response error, got %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil || err.Error() != tt.message {
|
||||
t.Fatalf("expected %q, got %v", tt.message, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSTSLDAPLoginRateLimiterCleanup(t *testing.T) {
|
||||
set := newSTSLDAPLoginKeyLimiterSet(time.Hour, 1, time.Minute)
|
||||
start := time.Unix(0, 0)
|
||||
|
||||
if !set.Allow(start, "old-key") {
|
||||
t.Fatal("expected initial key to be allowed")
|
||||
}
|
||||
if len(set.entries) != 1 {
|
||||
t.Fatalf("expected one entry, got %d", len(set.entries))
|
||||
}
|
||||
|
||||
if !set.Allow(start.Add(2*time.Minute), "new-key") {
|
||||
t.Fatal("expected new key to be allowed after ttl expiry")
|
||||
}
|
||||
if _, ok := set.entries["old-key"]; ok {
|
||||
t.Fatal("expected expired key to be cleaned up")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteSTSThrottledResponse(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodPost, "http://minio.test", strings.NewReader(""))
|
||||
rr := httptest.NewRecorder()
|
||||
req = req.WithContext(newContext(req, rr, "test-throttle"))
|
||||
|
||||
writeSTSThrottledResponse(rr)
|
||||
|
||||
if rr.Code != http.StatusTooManyRequests {
|
||||
t.Fatalf("expected status %d, got %d", http.StatusTooManyRequests, rr.Code)
|
||||
}
|
||||
if got := rr.Header().Get("Retry-After"); got != "6" {
|
||||
t.Fatalf("expected Retry-After header %q, got %q", "6", got)
|
||||
}
|
||||
|
||||
body := rr.Body.String()
|
||||
if !strings.Contains(body, "<Code>ThrottlingException</Code>") {
|
||||
t.Fatalf("expected throttling code in response, got %s", body)
|
||||
}
|
||||
if !strings.Contains(body, "<Message>Request throttled, please retry later.</Message>") {
|
||||
t.Fatalf("expected throttling message in response, got %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMImportAssetWithLDAP(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(t.Context(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
|
||||
+2
-2
@@ -450,10 +450,10 @@ func getLatestReleaseTime(u *url.URL, timeout time.Duration, mode string) (sha25
|
||||
|
||||
const (
|
||||
// Kubernetes deployment doc link.
|
||||
kubernetesDeploymentDoc = "https://docs.min.io/community/minio-object-store/operations/deployments/kubernetes.html"
|
||||
kubernetesDeploymentDoc = "https://silo.pigsty.io/operations/deployments/kubernetes.html"
|
||||
|
||||
// Mesos deployment doc link.
|
||||
mesosDeploymentDoc = "https://docs.min.io/community/minio-object-store/operations/deployments/kubernetes.html"
|
||||
mesosDeploymentDoc = "https://silo.pigsty.io/operations/deployments/kubernetes.html"
|
||||
)
|
||||
|
||||
func getDownloadURL(releaseTag string) (downloadURL string) {
|
||||
|
||||
@@ -68,7 +68,6 @@ const (
|
||||
storageMetricReadXL
|
||||
storageMetricReadAll
|
||||
storageMetricStatInfoFile
|
||||
storageMetricReadMultiple
|
||||
storageMetricDeleteAbandonedParts
|
||||
storageMetricDiskInfo
|
||||
storageMetricDeleteBulk
|
||||
@@ -723,21 +722,6 @@ func (p *xlStorageDiskIDCheck) ReadParts(ctx context.Context, volume string, par
|
||||
return p.storage.ReadParts(ctx, volume, partMetaPaths...)
|
||||
}
|
||||
|
||||
// ReadMultiple will read multiple files and send each files as response.
|
||||
// Files are read and returned in the given order.
|
||||
// The resp channel is closed before the call returns.
|
||||
// Only a canceled context will return an error.
|
||||
func (p *xlStorageDiskIDCheck) ReadMultiple(ctx context.Context, req ReadMultipleReq, resp chan<- ReadMultipleResp) (err error) {
|
||||
ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadMultiple, req.Bucket, req.Prefix)
|
||||
if err != nil {
|
||||
xioutil.SafeClose(resp)
|
||||
return err
|
||||
}
|
||||
defer done(0, &err)
|
||||
|
||||
return p.storage.ReadMultiple(ctx, req, resp)
|
||||
}
|
||||
|
||||
// CleanAbandonedData will read metadata of the object on disk
|
||||
// and delete any data directories and inline data that isn't referenced in metadata.
|
||||
func (p *xlStorageDiskIDCheck) CleanAbandonedData(ctx context.Context, volume string, path string) (err error) {
|
||||
|
||||
@@ -3187,81 +3187,6 @@ func (s *xlStorage) ReadParts(ctx context.Context, volume string, partMetaPaths
|
||||
return parts, nil
|
||||
}
|
||||
|
||||
// ReadMultiple will read multiple files and send each back as response.
|
||||
// Files are read and returned in the given order.
|
||||
// The resp channel is closed before the call returns.
|
||||
// Only a canceled context will return an error.
|
||||
func (s *xlStorage) ReadMultiple(ctx context.Context, req ReadMultipleReq, resp chan<- ReadMultipleResp) error {
|
||||
defer xioutil.SafeClose(resp)
|
||||
|
||||
volumeDir := pathJoin(s.drivePath, req.Bucket)
|
||||
found := 0
|
||||
for _, f := range req.Files {
|
||||
if contextCanceled(ctx) {
|
||||
return ctx.Err()
|
||||
}
|
||||
r := ReadMultipleResp{
|
||||
Bucket: req.Bucket,
|
||||
Prefix: req.Prefix,
|
||||
File: f,
|
||||
}
|
||||
|
||||
var data []byte
|
||||
var mt time.Time
|
||||
|
||||
fullPath := pathJoin(volumeDir, req.Prefix, f)
|
||||
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
|
||||
if err := w.Run(func() (err error) {
|
||||
if req.MetadataOnly {
|
||||
data, mt, err = s.readMetadataWithDMTime(ctx, fullPath)
|
||||
} else {
|
||||
data, mt, err = s.readAllDataWithDMTime(ctx, req.Bucket, volumeDir, fullPath)
|
||||
}
|
||||
return err
|
||||
}); err != nil {
|
||||
if !IsErr(err, errFileNotFound, errVolumeNotFound) {
|
||||
r.Exists = true
|
||||
r.Error = err.Error()
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case resp <- r:
|
||||
}
|
||||
if req.AbortOn404 && !r.Exists {
|
||||
// We stop at first file not found.
|
||||
// We have already reported the error, return nil.
|
||||
return nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
diskHealthCheckOK(ctx, nil)
|
||||
if req.MaxSize > 0 && int64(len(data)) > req.MaxSize {
|
||||
r.Exists = true
|
||||
r.Error = fmt.Sprintf("max size (%d) exceeded: %d", req.MaxSize, len(data))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case resp <- r:
|
||||
continue
|
||||
}
|
||||
}
|
||||
found++
|
||||
r.Exists = true
|
||||
r.Data = data
|
||||
r.Modtime = mt
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case resp <- r:
|
||||
}
|
||||
if req.MaxResults > 0 && found >= req.MaxResults {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *xlStorage) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
|
||||
volumeDir, err := s.getVolDir(volume)
|
||||
if err != nil {
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Pin to v8.11.0 – the last release that includes curl-aarch64.
|
||||
# v8.17.0 (current latest) dropped the aarch64 build.
|
||||
STATIC_CURL_VERSION="v8.11.0"
|
||||
|
||||
function download_arch_specific_executable {
|
||||
curl -f -L -s -q \
|
||||
https://github.com/moparisthebest/static-curl/releases/latest/download/curl-$1 \
|
||||
"https://github.com/moparisthebest/static-curl/releases/download/${STATIC_CURL_VERSION}/curl-$1" \
|
||||
-o /go/bin/curl || exit 1
|
||||
chmod +x /go/bin/curl
|
||||
}
|
||||
|
||||
+13
-13
@@ -2,7 +2,7 @@
|
||||
|
||||
## **1. Cloud-native Architecture**
|
||||
|
||||

|
||||

|
||||
|
||||
Kubernetes manages stateless Spark and Hive containers elastically on the compute nodes. Spark has native scheduler integration with Kubernetes. Hive, for legacy reasons, uses YARN scheduler on top of Kubernetes.
|
||||
|
||||
@@ -16,24 +16,24 @@ MinIO also supports multi-cluster, multi-site federation similar to AWS regions
|
||||
- [Setup Ambari](https://docs.hortonworks.com/HDPDocuments/Ambari-2.7.1.0/bk_ambari-installation/content/set_up_the_ambari_server.html) which automatically sets up YARN
|
||||
- [Installing Spark](https://docs.hortonworks.com/HDPDocuments/HDP3/HDP-3.0.1/installing-spark/content/installing_spark.html)
|
||||
- Install MinIO Distributed Server using one of the guides below.
|
||||
- [Deployment based on Kubernetes](https://docs.min.io/community/minio-object-store/operations/deployments/kubernetes.html)
|
||||
- [Deployment based on Kubernetes](https://silo.pigsty.io/operations/deployments/kubernetes.html)
|
||||
- [Deployment based on MinIO Helm Chart](https://github.com/helm/charts/tree/master/stable/minio)
|
||||
|
||||
## **3. Configure Hadoop, Spark, Hive to use MinIO**
|
||||
|
||||
After successful installation navigate to the Ambari UI `http://<ambari-server>:8080/` and login using the default credentials: [**_username: admin, password: admin_**]
|
||||
|
||||

|
||||

|
||||
|
||||
### **3.1 Configure Hadoop**
|
||||
|
||||
Navigate to **Services** -> **HDFS** -> **CONFIGS** -> **ADVANCED** as shown below
|
||||
|
||||

|
||||

|
||||
|
||||
Navigate to **Custom core-site** to configure MinIO parameters for `_s3a_` connector
|
||||
|
||||

|
||||

|
||||
|
||||
```
|
||||
sudo pip install yq
|
||||
@@ -100,17 +100,17 @@ The rest of the other optimization options are discussed in the links below
|
||||
|
||||
Once the config changes are applied, proceed to restart **Hadoop** services.
|
||||
|
||||

|
||||

|
||||
|
||||
### **3.2 Configure Spark2**
|
||||
|
||||
Navigate to **Services** -> **Spark2** -> **CONFIGS** as shown below
|
||||
|
||||

|
||||

|
||||
|
||||
Navigate to “**Custom spark-defaults**” to configure MinIO parameters for `_s3a_` connector
|
||||
|
||||

|
||||

|
||||
|
||||
Add the following optimal entries for _spark-defaults.conf_ to configure Spark with **MinIO**.
|
||||
|
||||
@@ -146,17 +146,17 @@ spark.hadoop.fs.s3a.threads.max 2048 # maximum number of threads for S3A
|
||||
|
||||
Once the config changes are applied, proceed to restart **Spark** services.
|
||||
|
||||

|
||||

|
||||
|
||||
### **3.3 Configure Hive**
|
||||
|
||||
Navigate to **Services** -> **Hive** -> **CONFIGS**-> **ADVANCED** as shown below
|
||||
|
||||

|
||||

|
||||
|
||||
Navigate to “**Custom hive-site**” to configure MinIO parameters for `_s3a_` connector
|
||||
|
||||

|
||||

|
||||
|
||||
Add the following optimal entries for `hive-site.xml` to configure Hive with **MinIO**.
|
||||
|
||||
@@ -171,11 +171,11 @@ mapreduce.input.fileinputformat.list-status.num-threads=50
|
||||
|
||||
For more information about these options please visit [https://www.cloudera.com/documentation/enterprise/5-11-x/topics/admin_hive_on_s3_tuning.html](https://www.cloudera.com/documentation/enterprise/5-11-x/topics/admin_hive_on_s3_tuning.html)
|
||||
|
||||

|
||||

|
||||
|
||||
Once the config changes are applied, proceed to restart all Hive services.
|
||||
|
||||

|
||||

|
||||
|
||||
## **4. Run Sample Applications**
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# ILM Tiering Design [](https://slack.min.io) [](https://hub.docker.com/r/minio/minio/)
|
||||
|
||||
Lifecycle transition functionality provided in [bucket lifecycle guide](https://github.com/minio/minio/master/docs/bucket/lifecycle/README.md) allows tiering of content from MinIO object store to public clouds or other MinIO clusters.
|
||||
Lifecycle transition functionality provided in [bucket lifecycle guide](https://github.com/pgsty/minio/blob/master/docs/bucket/lifecycle/README.md) allows tiering of content from MinIO object store to public clouds or other MinIO clusters.
|
||||
|
||||
Transition tiers can be added to MinIO using `mc admin tier add` command to associate a `gcs`, `s3` or `azure` bucket or prefix path on a bucket to the tier name.
|
||||
Lifecycle transition rules can be applied to buckets (both versioned and un-versioned) by specifying the tier name defined above as the transition storage class for the lifecycle rule.
|
||||
@@ -51,5 +51,5 @@ Tiering and lifecycle transition are applicable only to erasure/distributed MinI
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [MinIO | Golang Client API Reference](https://docs.min.io/community/minio-object-store/developers/go/API.html#SetBucketLifecycle)
|
||||
- [MinIO | Golang Client API Reference](https://silo.pigsty.io/developers/go/API.html#SetBucketLifecycle)
|
||||
- [Object Lifecycle Management](https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html)
|
||||
|
||||
@@ -4,8 +4,8 @@ Enable object lifecycle configuration on buckets to setup automatic deletion of
|
||||
|
||||
## 1. Prerequisites
|
||||
|
||||
- Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html).
|
||||
- Install `mc` - [mc Quickstart Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart)
|
||||
- Install MinIO - [MinIO Quickstart Guide](https://silo.pigsty.io/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html).
|
||||
- Install `mc` - [mc Quickstart Guide](https://silo.pigsty.io/reference/minio-mc.html#quickstart)
|
||||
|
||||
## 2. Enable bucket lifecycle configuration
|
||||
|
||||
@@ -59,7 +59,7 @@ TempUploads | temp/ | ✓ | ✓ | 7 day(s) | ✗
|
||||
|
||||
## 3. Activate ILM versioning features
|
||||
|
||||
This will only work with a versioned bucket, take a look at [Bucket Versioning Guide](https://docs.min.io/community/minio-object-store/administration/object-management/object-versioning.html) for more understanding.
|
||||
This will only work with a versioned bucket, take a look at [Bucket Versioning Guide](https://silo.pigsty.io/administration/object-management/object-versioning.html) for more understanding.
|
||||
|
||||
### 3.1 Automatic removal of non current objects versions
|
||||
|
||||
@@ -228,5 +228,5 @@ Note that transition event notification is a MinIO extension.
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [MinIO | Golang Client API Reference](https://docs.min.io/community/minio-object-store/developers/go/API.html)
|
||||
- [MinIO | Golang Client API Reference](https://silo.pigsty.io/developers/go/API.html)
|
||||
- [Object Lifecycle Management](https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html)
|
||||
|
||||
@@ -30,7 +30,7 @@ Various event types supported by MinIO server are
|
||||
| `s3:BucketCreated` |
|
||||
| `s3:BucketRemoved` |
|
||||
|
||||
Use client tools like `mc` to set and listen for event notifications using the [`event` sub-command](https://docs.min.io/community/minio-object-store/reference/minio-mc/mc-event-add.html). MinIO SDK's [`BucketNotification` APIs](https://docs.min.io/community/minio-object-store/developers/go/API.html#setbucketnotification-ctx-context-context-bucketname-string-config-notification-configuration-error) can also be used. The notification message MinIO sends to publish an event is a JSON message with the following [structure](https://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html).
|
||||
Use client tools like `mc` to set and listen for event notifications using the [`event` sub-command](https://silo.pigsty.io/reference/minio-mc/mc-event-add.html). MinIO SDK's [`BucketNotification` APIs](https://silo.pigsty.io/developers/go/API.html#setbucketnotification-ctx-context-context-bucketname-string-config-notification-configuration-error) can also be used. The notification message MinIO sends to publish an event is a JSON message with the following [structure](https://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html).
|
||||
|
||||
Bucket events can be published to the following targets:
|
||||
|
||||
@@ -43,8 +43,8 @@ Bucket events can be published to the following targets:
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Install and configure MinIO Server from [here](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html#procedure).
|
||||
- Install and configure MinIO Client from [here](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart).
|
||||
- Install and configure MinIO Server from [here](https://silo.pigsty.io/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html#procedure).
|
||||
- Install and configure MinIO Client from [here](https://silo.pigsty.io/reference/minio-mc.html#quickstart).
|
||||
|
||||
```
|
||||
$ mc admin config get myminio | grep notify
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
# Bucket Quota Configuration Quickstart Guide [](https://slack.min.io) [](https://hub.docker.com/r/minio/minio/)
|
||||
|
||||

|
||||

|
||||
|
||||
Buckets can be configured to have `Hard` quota - it disallows writes to the bucket after configured quota limit is reached.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html#procedure).
|
||||
- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart)
|
||||
- Install MinIO - [MinIO Quickstart Guide](https://silo.pigsty.io/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html#procedure).
|
||||
- [Use `mc` with MinIO Server](https://silo.pigsty.io/reference/minio-mc.html#quickstart)
|
||||
|
||||
## Set bucket quota configuration
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Bucket Replication Design [](https://slack.min.io) [](https://hub.docker.com/r/minio/minio/)
|
||||
|
||||
This document explains the design approach of server side bucket replication. If you're looking to get started with replication, we suggest you go through the [Bucket replication guide](https://github.com/minio/minio/blob/master/docs/bucket/replication/README.md) first.
|
||||
This document explains the design approach of server side bucket replication. If you're looking to get started with replication, we suggest you go through the [Bucket replication guide](https://github.com/pgsty/minio/blob/master/docs/bucket/replication/README.md) first.
|
||||
|
||||
## Overview
|
||||
|
||||
@@ -59,7 +59,7 @@ If 3 or more targets are participating in active-active replication, the replica
|
||||
|
||||
### Internal metadata for replication
|
||||
|
||||
`xl.meta` that is in use for [versioning](https://github.com/minio/minio/blob/master/docs/bucket/versioning/DESIGN.md) has additional metadata for replication of objects,delete markers and versioned deletes.
|
||||
`xl.meta` that is in use for [versioning](https://github.com/pgsty/minio/blob/master/docs/bucket/versioning/DESIGN.md) has additional metadata for replication of objects,delete markers and versioned deletes.
|
||||
|
||||
### Metadata for object replication - on source
|
||||
|
||||
@@ -156,5 +156,5 @@ If 3 or more targets are participating in active-active replication, the replica
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [MinIO Bucket Versioning Implementation](https://docs.min.io/community/minio-object-store/administration/object-management/object-versioning.html)
|
||||
- [MinIO Client Quickstart Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart)
|
||||
- [MinIO Bucket Versioning Implementation](https://silo.pigsty.io/administration/object-management/object-versioning.html)
|
||||
- [MinIO Client Quickstart Guide](https://silo.pigsty.io/reference/minio-mc.html#quickstart)
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
Bucket replication is designed to replicate selected objects in a bucket to a destination bucket.
|
||||
|
||||
The contents of this page have been migrated to the new [MinIO Documentation: Bucket Replication](https://docs.min.io/community/minio-object-store/administration/bucket-replication.html) page. The [Bucket Replication](https://docs.min.io/community/minio-object-store/administration/bucket-replication/bucket-replication-requirements.html) page references dedicated tutorials for configuring one-way "Active-Passive" and two-way "Active-Active" bucket replication.
|
||||
The contents of this page have been migrated to the new [MinIO Documentation: Bucket Replication](https://silo.pigsty.io/administration/bucket-replication.html) page. The [Bucket Replication](https://silo.pigsty.io/administration/bucket-replication/bucket-replication-requirements.html) page references dedicated tutorials for configuring one-way "Active-Passive" and two-way "Active-Active" bucket replication.
|
||||
|
||||
To replicate objects in a bucket to a destination bucket on a target site either in the same cluster or a different cluster, start by enabling [versioning](https://docs.min.io/community/minio-object-store/administration/object-management/object-versioning.html) for both source and destination buckets. Finally, the target site and the destination bucket need to be configured on the source MinIO server.
|
||||
To replicate objects in a bucket to a destination bucket on a target site either in the same cluster or a different cluster, start by enabling [versioning](https://silo.pigsty.io/administration/object-management/object-versioning.html) for both source and destination buckets. Finally, the target site and the destination bucket need to be configured on the source MinIO server.
|
||||
|
||||
## Highlights
|
||||
|
||||
@@ -94,7 +94,7 @@ The access key provided for the replication *target* cluster should have these m
|
||||
}
|
||||
```
|
||||
|
||||
Please note that the permissions required by the admin user on the target cluster can be more fine grained to exclude permissions like "s3:ReplicateDelete", "s3:GetBucketObjectLockConfiguration" etc depending on whether delete replication rules are set up or if object locking is disabled on `destbucket`. The above policies assume that replication of objects, tags and delete marker replication are all enabled on object lock enabled buckets. A sample script to setup replication is provided [here](https://github.com/minio/minio/blob/master/docs/bucket/replication/setup_replication.sh)
|
||||
Please note that the permissions required by the admin user on the target cluster can be more fine grained to exclude permissions like "s3:ReplicateDelete", "s3:GetBucketObjectLockConfiguration" etc depending on whether delete replication rules are set up or if object locking is disabled on `destbucket`. The above policies assume that replication of objects, tags and delete marker replication are all enabled on object lock enabled buckets. A sample script to setup replication is provided [here](https://github.com/pgsty/minio/blob/master/docs/bucket/replication/setup_replication.sh)
|
||||
|
||||
To set up replication from a source bucket `srcbucket` on myminio cluster to a bucket `destbucket` on the target minio cluster with endpoint https://replica-endpoint:9000, use:
|
||||
```
|
||||
@@ -155,15 +155,15 @@ The replication configuration generated has the following format and can be expo
|
||||
|
||||
The replication configuration follows [AWS S3 Spec](https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html). Any objects uploaded to the source bucket that meet replication criteria will now be automatically replicated by the MinIO server to the remote destination bucket. Replication can be disabled at any time by disabling specific rules in the configuration or deleting the replication configuration entirely.
|
||||
|
||||
When object locking is used in conjunction with replication, both source and destination buckets needs to have [object locking](https://docs.min.io/community/minio-object-store/administration/object-management/object-retention.html) enabled. Similarly objects encrypted on the server side, will be replicated if destination also supports encryption.
|
||||
When object locking is used in conjunction with replication, both source and destination buckets needs to have [object locking](https://silo.pigsty.io/administration/object-management/object-retention.html) enabled. Similarly objects encrypted on the server side, will be replicated if destination also supports encryption.
|
||||
|
||||
Replication status can be seen in the metadata on the source and destination objects. On the source side, the `X-Amz-Replication-Status` changes from `PENDING` to `COMPLETED` or `FAILED` after replication attempt either succeeded or failed respectively. On the destination side, a `X-Amz-Replication-Status` status of `REPLICA` indicates that the object was replicated successfully. Any replication failures are automatically re-attempted during a periodic disk scanner cycle.
|
||||
|
||||
To perform bi-directional replication, repeat the above process on the target site - this time setting the source bucket as the replication target. It is recommended that replication be run in a system with at least two CPU's available to the process, so that replication can run in its own thread.
|
||||
|
||||

|
||||

|
||||
|
||||

|
||||

|
||||
|
||||
## Replica Modification sync
|
||||
|
||||
@@ -210,11 +210,11 @@ Also note that `mc` version `RELEASE.2021-09-02T09-21-27Z` or older supports onl
|
||||
|
||||
Status of delete marker replication can be viewed by doing a GET/HEAD on the object version - it will return a `X-Minio-Replication-DeleteMarker-Status` header and http response code of `405`. In the case of permanent deletes, if the delete replication is pending or failed to propagate to the target cluster, GET/HEAD will return additional `X-Minio-Replication-Delete-Status` header and a http response code of `405`.
|
||||
|
||||

|
||||

|
||||
|
||||
The status of replication can be monitored by configuring event notifications on the source and target buckets using `mc event add`.On the source side, the `s3:PutObject`, `s3:Replication:OperationCompletedReplication` and `s3:Replication:OperationFailedReplication` events show the status of replication in the `X-Amz-Replication-Status` metadata.
|
||||
|
||||
On the target bucket, `s3:PutObject` event shows `X-Amz-Replication-Status` status of `REPLICA` in the metadata. Additional metrics to monitor backlog state for the purpose of bandwidth management and resource allocation are exposed via Prometheus - see <https://github.com/minio/minio/blob/master/docs/metrics/prometheus/list.md> for more details.
|
||||
On the target bucket, `s3:PutObject` event shows `X-Amz-Replication-Status` status of `REPLICA` in the metadata. Additional metrics to monitor backlog state for the purpose of bandwidth management and resource allocation are exposed via Prometheus - see <https://github.com/pgsty/minio/blob/master/docs/metrics/prometheus/list.md> for more details.
|
||||
|
||||
### Sync/Async Replication
|
||||
|
||||
@@ -276,6 +276,6 @@ MinIO does not support SSE-C encrypted objects on replicated buckets, any applic
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [MinIO Bucket Replication Design](https://github.com/minio/minio/blob/master/docs/bucket/replication/DESIGN.md)
|
||||
- [MinIO Bucket Versioning Implementation](https://docs.min.io/community/minio-object-store/administration/object-management/object-retention.html)
|
||||
- [MinIO Client Quickstart Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart)
|
||||
- [MinIO Bucket Replication Design](https://github.com/pgsty/minio/blob/master/docs/bucket/replication/DESIGN.md)
|
||||
- [MinIO Bucket Versioning Implementation](https://silo.pigsty.io/administration/object-management/object-retention.html)
|
||||
- [MinIO Client Quickstart Guide](https://silo.pigsty.io/reference/minio-mc.html#quickstart)
|
||||
|
||||
@@ -10,7 +10,7 @@ A default retention period and retention mode can be configured on a bucket to b
|
||||
|
||||
### 1. Prerequisites
|
||||
|
||||
- Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html)
|
||||
- Install MinIO - [MinIO Quickstart Guide](https://silo.pigsty.io/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html)
|
||||
- Install `awscli` - [Installing AWS Command Line Interface](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html)
|
||||
|
||||
### 2. Set bucket WORM configuration
|
||||
@@ -53,7 +53,7 @@ See <https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html>
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart)
|
||||
- [Use `aws-cli` with MinIO Server](https://docs.min.io/community/minio-object-store/integrations/aws-cli-with-minio.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html)
|
||||
- [Use `mc` with MinIO Server](https://silo.pigsty.io/reference/minio-mc.html#quickstart)
|
||||
- [Use `aws-cli` with MinIO Server](https://silo.pigsty.io/integrations/aws-cli-with-minio.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://silo.pigsty.io/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://silo.pigsty.io/index.html)
|
||||
|
||||
@@ -42,7 +42,7 @@ Therefore, the metadata is wrapped as a binary array for easy skipping.
|
||||
- LegacyObjectType (preserves existing deployments and older xl.json format)
|
||||
- DeleteMarker (a versionId to capture the DELETE sequences implemented primarily for AWS spec compatibility)
|
||||
|
||||
A sample msgpack-JSON `xl.meta`, you can debug the content inside `xl.meta` using [xl-meta.go](https://github.com/minio/minio/tree/master/docs/debugging#decoding-metadata) program.
|
||||
A sample msgpack-JSON `xl.meta`, you can debug the content inside `xl.meta` using [xl-meta.go](https://github.com/pgsty/minio/tree/master/docs/debugging#decoding-metadata) program.
|
||||
|
||||
```json
|
||||
{
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
MinIO versioning is designed to keep multiple versions of an object in one bucket. For example, you could store `spark.csv` (version `ede336f2`) and `spark.csv` (version `fae684da`) in a single bucket. Versioning protects you from unintended overwrites, deletions, protect objects with retention policies.
|
||||
|
||||
To control data retention and storage usage, use object versioning with [object lifecycle management](https://github.com/minio/minio/blob/master/docs/bucket/lifecycle/README.md). If you have an object expiration lifecycle policy in your non-versioned bucket and you want to maintain the same permanent delete behavior when on versioning-enabled bucket, you must add a noncurrent expiration policy. The noncurrent expiration lifecycle policy will manage the deletes of the noncurrent object versions in the versioning-enabled bucket. (A version-enabled bucket maintains one current and zero or more noncurrent object versions.)
|
||||
To control data retention and storage usage, use object versioning with [object lifecycle management](https://github.com/pgsty/minio/blob/master/docs/bucket/lifecycle/README.md). If you have an object expiration lifecycle policy in your non-versioned bucket and you want to maintain the same permanent delete behavior when on versioning-enabled bucket, you must add a noncurrent expiration policy. The noncurrent expiration lifecycle policy will manage the deletes of the noncurrent object versions in the versioning-enabled bucket. (A version-enabled bucket maintains one current and zero or more noncurrent object versions.)
|
||||
|
||||
Versioning must be explicitly enabled on a bucket, versioning is not enabled by default. Object locking enabled buckets have versioning enabled automatically. Enabling and suspending versioning is done at the bucket level.
|
||||
|
||||
@@ -10,25 +10,25 @@ Only MinIO generates version IDs, and they can't be edited. Version IDs are simp
|
||||
|
||||
When you PUT an object in a versioning-enabled bucket, the noncurrent version is not overwritten. The following figure shows that when a new version of `spark.csv` is PUT into a bucket that already contains an object with the same name, the original object (ID = `ede336f2`) remains in the bucket, MinIO generates a new version (ID = `fae684da`), and adds the newer version to the bucket.
|
||||
|
||||

|
||||

|
||||
|
||||
This protects against accidental overwrites or deletes of objects, allows previous versions to be retrieved.
|
||||
|
||||
When you DELETE an object, all versions remain in the bucket and MinIO adds a delete marker, as shown below:
|
||||
|
||||

|
||||

|
||||
|
||||
Now the delete marker becomes the current version of the object. GET requests by default always retrieve the latest stored version. So performing a simple GET object request when the current version is a delete marker would return `404` `The specified key does not exist` as shown below:
|
||||
|
||||

|
||||

|
||||
|
||||
GET requests by specifying a version ID as shown below, you can retrieve the specific object version `fae684da`.
|
||||
|
||||

|
||||

|
||||
|
||||
To permanently delete an object you need to specify the version you want to delete, only the user with appropriate permissions can permanently delete a version. As shown below DELETE request called with a specific version id permanently deletes an object from a bucket. Delete marker is not added for DELETE requests with version id.
|
||||
|
||||

|
||||

|
||||
|
||||
## Concepts
|
||||
|
||||
@@ -211,7 +211,7 @@ public class IsVersioningEnabled {
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [Use `minio-java` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/java/minio-java.html)
|
||||
- [Object Lock and Immutability Guide](https://docs.min.io/community/minio-object-store/administration/object-management/object-retention.html)
|
||||
- [MinIO Admin Complete Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc-admin.html)
|
||||
- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html)
|
||||
- [Use `minio-java` SDK with MinIO Server](https://silo.pigsty.io/developers/java/minio-java.html)
|
||||
- [Object Lock and Immutability Guide](https://silo.pigsty.io/administration/object-management/object-retention.html)
|
||||
- [MinIO Admin Complete Guide](https://silo.pigsty.io/reference/minio-mc-admin.html)
|
||||
- [The MinIO documentation website](https://silo.pigsty.io/index.html)
|
||||
|
||||
@@ -51,8 +51,8 @@ Instance is now accessible on the host at port 9000, proceed to access the Web b
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [MinIO Erasure Code Overview](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html)
|
||||
- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html)
|
||||
- [Use `aws-cli` with MinIO Server](https://docs.min.io/community/minio-object-store/integrations/aws-cli-with-minio.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html)
|
||||
- [MinIO Erasure Code Overview](https://silo.pigsty.io/operations/concepts/erasure-coding.html)
|
||||
- [Use `mc` with MinIO Server](https://silo.pigsty.io/reference/minio-mc.html)
|
||||
- [Use `aws-cli` with MinIO Server](https://silo.pigsty.io/integrations/aws-cli-with-minio.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://silo.pigsty.io/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://silo.pigsty.io/index.html)
|
||||
|
||||
@@ -19,7 +19,7 @@ will increase speed when the content can be compressed.
|
||||
|
||||
### 1. Prerequisites
|
||||
|
||||
Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html).
|
||||
Install MinIO - [MinIO Quickstart Guide](https://silo.pigsty.io/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html).
|
||||
|
||||
### 2. Run MinIO with compression
|
||||
|
||||
@@ -131,7 +131,7 @@ the data directory to view the size of the object.
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html)
|
||||
- [Use `aws-cli` with MinIO Server](https://docs.min.io/community/minio-object-store/integrations/aws-cli-with-minio.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html)
|
||||
- [Use `mc` with MinIO Server](https://silo.pigsty.io/reference/minio-mc.html)
|
||||
- [Use `aws-cli` with MinIO Server](https://silo.pigsty.io/integrations/aws-cli-with-minio.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://silo.pigsty.io/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://silo.pigsty.io/index.html)
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
|
||||
## Configuration Directory
|
||||
|
||||
MinIO stores all its config as part of the server deployment, config is erasure coded on MinIO. On a fresh deployment MinIO automatically generates a new `config` and this config is available to be configured via `mc admin config` command. MinIO also encrypts all the config, IAM and policies content if KMS is configured. Please refer to how to encrypt your config and IAM credentials [here](https://github.com/minio/minio/blob/master/docs/kms/IAM.md).
|
||||
MinIO stores all its config as part of the server deployment, config is erasure coded on MinIO. On a fresh deployment MinIO automatically generates a new `config` and this config is available to be configured via `mc admin config` command. MinIO also encrypts all the config, IAM and policies content if KMS is configured. Please refer to how to encrypt your config and IAM credentials [here](https://github.com/pgsty/minio/blob/master/docs/kms/IAM.md).
|
||||
|
||||
### Certificate Directory
|
||||
|
||||
TLS certificates by default are expected to be stored under ``${HOME}/.minio/certs`` directory. You need to place certificates here to enable `HTTPS` based access. Read more about [How to secure access to MinIO server with TLS](https://docs.min.io/community/minio-object-store/operations/network-encryption.html).
|
||||
TLS certificates by default are expected to be stored under ``${HOME}/.minio/certs`` directory. You need to place certificates here to enable `HTTPS` based access. Read more about [How to secure access to MinIO server with TLS](https://silo.pigsty.io/operations/network-encryption.html).
|
||||
|
||||
Following is a sample directory structure for MinIO server with TLS certificates.
|
||||
|
||||
@@ -65,7 +65,7 @@ minio server /data
|
||||
|
||||
### Storage Class
|
||||
|
||||
By default, parity for objects with standard storage class is set to `N/2`, and parity for objects with reduced redundancy storage class objects is set to `2`. Read more about storage class support in MinIO server [here](https://github.com/minio/minio/blob/master/docs/erasure/storage-class/README.md).
|
||||
By default, parity for objects with standard storage class is set to `N/2`, and parity for objects with reduced redundancy storage class objects is set to `2`. Read more about storage class support in MinIO server [here](https://github.com/pgsty/minio/blob/master/docs/erasure/storage-class/README.md).
|
||||
|
||||
```
|
||||
KEY:
|
||||
@@ -91,7 +91,7 @@ MINIO_STORAGE_CLASS_COMMENT (sentence) optionally add a comment to this setti
|
||||
|
||||
#### Etcd
|
||||
|
||||
MinIO supports storing encrypted IAM assets in etcd, if KMS is configured. Please refer to how to encrypt your config and IAM credentials [here](https://github.com/minio/minio/blob/master/docs/kms/IAM.md).
|
||||
MinIO supports storing encrypted IAM assets in etcd, if KMS is configured. Please refer to how to encrypt your config and IAM credentials [here](https://github.com/pgsty/minio/blob/master/docs/kms/IAM.md).
|
||||
|
||||
> NOTE: if *path_prefix* is set then MinIO will not federate your buckets, namespaced IAM assets are assumed as isolated tenants, only buckets are considered globally unique but performing a lookup with a *bucket* which belongs to a different tenant will fail unlike federated setups where MinIO would port-forward and route the request to relevant cluster accordingly. This is a special feature, federated deployments should not need to set *path_prefix*.
|
||||
|
||||
@@ -125,7 +125,7 @@ MINIO_ETCD_COMMENT (sentence) optionally add a comment to this setting
|
||||
|
||||
### API
|
||||
|
||||
By default, there is no limitation on the number of concurrent requests that a server/cluster processes at the same time. However, it is possible to impose such limitation using the API subsystem. Read more about throttling limitation in MinIO server [here](https://github.com/minio/minio/blob/master/docs/throttle/README.md).
|
||||
By default, there is no limitation on the number of concurrent requests that a server/cluster processes at the same time. However, it is possible to impose such limitation using the API subsystem. Read more about throttling limitation in MinIO server [here](https://github.com/pgsty/minio/blob/master/docs/throttle/README.md).
|
||||
|
||||
```
|
||||
KEY:
|
||||
@@ -172,7 +172,7 @@ MINIO_API_OBJECT_MAX_VERSIONS (number) set max allowed number of
|
||||
|
||||
#### Notifications
|
||||
|
||||
Notification targets supported by MinIO are in the following list. To configure individual targets please refer to more detailed documentation [here](https://docs.min.io/community/minio-object-store/administration/monitoring.html#bucket-notifications).
|
||||
Notification targets supported by MinIO are in the following list. To configure individual targets please refer to more detailed documentation [here](https://silo.pigsty.io/administration/monitoring.html#bucket-notifications).
|
||||
|
||||
```
|
||||
notify_webhook publish bucket notifications to webhook endpoints
|
||||
@@ -336,5 +336,5 @@ minio server /data
|
||||
|
||||
## Explore Further
|
||||
|
||||
* [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html)
|
||||
* [Configure MinIO Server with TLS](https://docs.min.io/community/minio-object-store/operations/network-encryption.html)
|
||||
* [MinIO Quickstart Guide](https://silo.pigsty.io/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html)
|
||||
* [Configure MinIO Server with TLS](https://silo.pigsty.io/operations/network-encryption.html)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
## HTTP Trace
|
||||
|
||||
HTTP tracing can be enabled by using [`mc admin trace`](https://docs.min.io/community/minio-object-store/reference/minio-mc-admin/mc-admin-trace.html) command.
|
||||
HTTP tracing can be enabled by using [`mc admin trace`](https://silo.pigsty.io/reference/minio-mc-admin/mc-admin-trace.html) command.
|
||||
|
||||
Example:
|
||||
|
||||
|
||||
@@ -115,7 +115,7 @@ In above example there are two server pools
|
||||
|
||||
> Notice the requirement of common SLA here original cluster had 1024 drives with 16 drives per erasure set with default parity of '4', second pool is expected to have a minimum of 8 drives per erasure set to match the original cluster SLA (parity count) of '4'. '12' drives stripe per erasure set in the second pool satisfies the original pool's parity count.
|
||||
|
||||
Refer to the sizing guide with details on the default parity count chosen for different erasure stripe sizes [here](https://github.com/minio/minio/blob/master/docs/distributed/SIZING.md)
|
||||
Refer to the sizing guide with details on the default parity count chosen for different erasure stripe sizes [here](https://github.com/pgsty/minio/blob/master/docs/distributed/SIZING.md)
|
||||
|
||||
MinIO places new objects in server pools based on proportionate free space, per pool. Following pseudo code demonstrates this behavior.
|
||||
|
||||
|
||||
+12
-12
@@ -8,7 +8,7 @@ MinIO in distributed mode can help you setup a highly-available storage system w
|
||||
|
||||
### Data protection
|
||||
|
||||
Distributed MinIO provides protection against multiple node/drive failures and [bit rot](https://github.com/minio/minio/blob/master/docs/erasure/README.md#what-is-bit-rot-protection) using [erasure code](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html). As the minimum drives required for distributed MinIO is 2 (same as minimum drives required for erasure coding), erasure code automatically kicks in as you launch distributed MinIO.
|
||||
Distributed MinIO provides protection against multiple node/drive failures and [bit rot](https://github.com/pgsty/minio/blob/master/docs/erasure/README.md#what-is-bit-rot-protection) using [erasure code](https://silo.pigsty.io/operations/concepts/erasure-coding.html). As the minimum drives required for distributed MinIO is 2 (same as minimum drives required for erasure coding), erasure code automatically kicks in as you launch distributed MinIO.
|
||||
|
||||
If one or more drives are offline at the start of a PutObject or NewMultipartUpload operation the object will have additional data protection bits added automatically to provide additional safety for these objects.
|
||||
|
||||
@@ -18,7 +18,7 @@ A stand-alone MinIO server would go down if the server hosting the drives goes o
|
||||
|
||||
For example, an 16-server distributed setup with 200 drives per node would continue serving files, up to 4 servers can be offline in default configuration i.e around 800 drives down MinIO would continue to read and write objects.
|
||||
|
||||
Refer to sizing guide for more understanding on default values chosen depending on your erasure stripe size [here](https://github.com/minio/minio/blob/master/docs/distributed/SIZING.md). Parity settings can be changed using [storage classes](https://github.com/minio/minio/tree/master/docs/erasure/storage-class).
|
||||
Refer to sizing guide for more understanding on default values chosen depending on your erasure stripe size [here](https://github.com/pgsty/minio/blob/master/docs/distributed/SIZING.md). Parity settings can be changed using [storage classes](https://github.com/pgsty/minio/tree/master/docs/erasure/storage-class).
|
||||
|
||||
### Consistency Guarantees
|
||||
|
||||
@@ -38,11 +38,11 @@ Install MinIO either on Kubernetes or Distributed Linux.
|
||||
|
||||
Install MinIO on Kubernetes:
|
||||
|
||||
- [MinIO Quickstart Guide for Kubernetes](https://docs.min.io/community/minio-object-store/operations/deployments/kubernetes.html).
|
||||
- [Deploy a Tenant from the MinIO Operator](https://docs.min.io/community/minio-object-store/operations/deployments/k8s-deploy-minio-tenant-on-kubernetes.html)
|
||||
- [MinIO Quickstart Guide for Kubernetes](https://silo.pigsty.io/operations/deployments/kubernetes.html).
|
||||
- [Deploy a Tenant from the MinIO Operator](https://silo.pigsty.io/operations/deployments/k8s-deploy-minio-tenant-on-kubernetes.html)
|
||||
|
||||
Install Distributed MinIO on Linux:
|
||||
- [Deploy Distributed MinIO on Linux](https://docs.min.io/community/minio-object-store/operations/install-deploy-manage/deploy-minio-multi-node-multi-drive.html#deploy-distributed-minio)
|
||||
- [Deploy Distributed MinIO on Linux](https://silo.pigsty.io/operations/install-deploy-manage/deploy-minio-multi-node-multi-drive.html#deploy-distributed-minio)
|
||||
|
||||
### 2. Run distributed MinIO
|
||||
|
||||
@@ -63,7 +63,7 @@ To start a distributed MinIO instance, you just need to pass drive locations as
|
||||
|
||||
Example 1: Start distributed MinIO instance on n nodes with m drives each mounted at `/export1` to `/exportm` (pictured below), by running this command on all the n nodes:
|
||||
|
||||

|
||||

|
||||
|
||||
### GNU/Linux and macOS
|
||||
|
||||
@@ -98,12 +98,12 @@ Now the server has expanded total storage by _(newly_added_servers\*m)_ more dri
|
||||
|
||||
## 3. Test your setup
|
||||
|
||||
To test this setup, access the MinIO server via browser or [`mc`](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart).
|
||||
To test this setup, access the MinIO server via browser or [`mc`](https://silo.pigsty.io/reference/minio-mc.html#quickstart).
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [MinIO Erasure Code QuickStart Guide](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html)
|
||||
- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html)
|
||||
- [Use `aws-cli` with MinIO Server](https://docs.min.io/community/minio-object-store/integrations/aws-cli-with-minio.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html)
|
||||
- [MinIO Erasure Code QuickStart Guide](https://silo.pigsty.io/operations/concepts/erasure-coding.html)
|
||||
- [Use `mc` with MinIO Server](https://silo.pigsty.io/reference/minio-mc.html)
|
||||
- [Use `aws-cli` with MinIO Server](https://silo.pigsty.io/integrations/aws-cli-with-minio.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://silo.pigsty.io/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://silo.pigsty.io/index.html)
|
||||
|
||||
@@ -2,13 +2,15 @@
|
||||
|
||||
See our web documentation on [Deploying MinIO in Standalone Mode](Deploy Standalone MinIO in a Container) for a more structured tutorial on deploying MinIO in a container.
|
||||
|
||||
For images built from this `pgsty/minio` fork, the container also bundles `mcli` and a compatibility `mc` symlink from `pgsty/mc`.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Docker installed on your machine. Download the relevant installer from [here](https://www.docker.com/community-edition#/download).
|
||||
|
||||
## Run Standalone MinIO on Docker
|
||||
|
||||
*Note*: Standalone MinIO is intended for early development and evaluation. For production clusters, deploy a [Distributed](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-as-a-container.html) MinIO deployment.
|
||||
*Note*: Standalone MinIO is intended for early development and evaluation. For production clusters, deploy a [Distributed](https://silo.pigsty.io/operations/deployments/baremetal-deploy-minio-as-a-container.html) MinIO deployment.
|
||||
|
||||
MinIO needs a persistent volume to store configuration and application data. For testing purposes, you can launch MinIO by simply passing a directory (`/data` in the example below). This directory gets created in the container filesystem at the time of container start. But all the data is lost after container exits.
|
||||
|
||||
@@ -57,7 +59,7 @@ docker run \
|
||||
|
||||
We recommend kubernetes based deployment for production level deployment <https://github.com/minio/operator>.
|
||||
|
||||
See the [Kubernetes documentation](https://docs.min.io/community/minio-object-store/operations/deployments/kubernetes.html) for more information.
|
||||
See the [Kubernetes documentation](https://silo.pigsty.io/operations/deployments/kubernetes.html) for more information.
|
||||
|
||||
## MinIO Docker Tips
|
||||
|
||||
@@ -211,5 +213,5 @@ docker stats <container_id>
|
||||
|
||||
## Explore Further
|
||||
|
||||
* [MinIO in a Container Installation Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-as-a-container.html)
|
||||
* [MinIO Erasure Code QuickStart Guide](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html)
|
||||
* [MinIO in a Container Installation Guide](https://silo.pigsty.io/operations/deployments/baremetal-deploy-minio-as-a-container.html)
|
||||
* [MinIO Erasure Code QuickStart Guide](https://silo.pigsty.io/operations/concepts/erasure-coding.html)
|
||||
|
||||
@@ -6,7 +6,7 @@ MinIO protects data against hardware failures and silent data corruption using e
|
||||
|
||||
Erasure code is a mathematical algorithm to reconstruct missing or corrupted data. MinIO uses Reed-Solomon code to shard objects into variable data and parity blocks. For example, in a 12 drive setup, an object can be sharded to a variable number of data and parity blocks across all the drives - ranging from six data and six parity blocks to ten data and two parity blocks.
|
||||
|
||||
By default, MinIO shards the objects across N/2 data and N/2 parity drives. Though, you can use [storage classes](https://github.com/minio/minio/tree/master/docs/erasure/storage-class) to use a custom configuration. We recommend N/2 data and parity blocks, as it ensures the best protection from drive failures.
|
||||
By default, MinIO shards the objects across N/2 data and N/2 parity drives. Though, you can use [storage classes](https://github.com/pgsty/minio/tree/master/docs/erasure/storage-class) to use a custom configuration. We recommend N/2 data and parity blocks, as it ensures the best protection from drive failures.
|
||||
|
||||
In 12 drive example above, with MinIO server running in the default configuration, you can lose any of the six drives and still reconstruct the data reliably from the remaining drives.
|
||||
|
||||
@@ -14,7 +14,7 @@ In 12 drive example above, with MinIO server running in the default configuratio
|
||||
|
||||
Erasure code protects data from multiple drives failure, unlike RAID or replication. For example, RAID6 can protect against two drive failure whereas in MinIO erasure code you can lose as many as half of drives and still the data remains safe. Further, MinIO's erasure code is at the object level and can heal one object at a time. For RAID, healing can be done only at the volume level which translates into high downtime. As MinIO encodes each object individually, it can heal objects incrementally. Storage servers once deployed should not require drive replacement or healing for the lifetime of the server. MinIO's erasure coded backend is designed for operational efficiency and takes full advantage of hardware acceleration whenever available.
|
||||
|
||||

|
||||

|
||||
|
||||
## What is Bit Rot protection?
|
||||
|
||||
@@ -26,7 +26,7 @@ MinIO's erasure coded backend uses high speed [HighwayHash](https://github.com/m
|
||||
|
||||
MinIO divides the drives you provide into erasure-coding sets of *2 to 16* drives. Therefore, the number of drives you present must be a multiple of one of these numbers. Each object is written to a single erasure-coding set.
|
||||
|
||||
Minio uses the largest possible EC set size which divides into the number of drives given. For example, *18 drives* are configured as *2 sets of 9 drives*, and *24 drives* are configured as *2 sets of 12 drives*. This is true for scenarios when running MinIO as a standalone erasure coded deployment. In [distributed setup however node (affinity) based](https://docs.min.io/community/minio-object-store/operations/install-deploy-manage/deploy-minio-multi-node-multi-drive.html) erasure stripe sizes are chosen.
|
||||
Minio uses the largest possible EC set size which divides into the number of drives given. For example, *18 drives* are configured as *2 sets of 9 drives*, and *24 drives* are configured as *2 sets of 12 drives*. This is true for scenarios when running MinIO as a standalone erasure coded deployment. In [distributed setup however node (affinity) based](https://silo.pigsty.io/operations/install-deploy-manage/deploy-minio-multi-node-multi-drive.html) erasure stripe sizes are chosen.
|
||||
|
||||
The drives should all be of approximately the same size.
|
||||
|
||||
@@ -34,7 +34,7 @@ The drives should all be of approximately the same size.
|
||||
|
||||
### 1. Prerequisites
|
||||
|
||||
Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html)
|
||||
Install MinIO - [MinIO Quickstart Guide](https://silo.pigsty.io/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html)
|
||||
|
||||
### 2. Run MinIO Server with Erasure Code
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
MinIO server supports storage class in erasure coding mode. This allows configurable data and parity drives per object.
|
||||
|
||||
This page is intended as a summary of MinIO Erasure Coding. For a more complete explanation, see <https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html>.
|
||||
This page is intended as a summary of MinIO Erasure Coding. For a more complete explanation, see <https://silo.pigsty.io/operations/concepts/erasure-coding.html>.
|
||||
|
||||
## Overview
|
||||
|
||||
@@ -53,7 +53,7 @@ The default value for the `STANDARD` storage class depends on the number of volu
|
||||
| 6-7 | EC:3 |
|
||||
| 8 or more | EC:4 |
|
||||
|
||||
For more complete documentation on Erasure Set sizing, see the [MinIO Documentation on Erasure Sets](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html#erasure-sets).
|
||||
For more complete documentation on Erasure Set sizing, see the [MinIO Documentation on Erasure Sets](https://silo.pigsty.io/operations/concepts/erasure-coding.html#erasure-sets).
|
||||
|
||||
### Allowed values for REDUCED_REDUNDANCY storage class
|
||||
|
||||
@@ -80,7 +80,7 @@ export MINIO_STORAGE_CLASS_STANDARD=EC:3
|
||||
export MINIO_STORAGE_CLASS_RRS=EC:2
|
||||
```
|
||||
|
||||
Storage class can also be set via `mc admin config` get/set commands to update the configuration. Refer [storage class](https://github.com/minio/minio/tree/master/docs/config#storage-class) for
|
||||
Storage class can also be set via `mc admin config` get/set commands to update the configuration. Refer [storage class](https://github.com/pgsty/minio/tree/master/docs/config#storage-class) for
|
||||
more details.
|
||||
|
||||
#### Note
|
||||
|
||||
@@ -23,9 +23,9 @@ All properties except the file size are tied to the zip file. This means that mo
|
||||
|
||||
## Code Examples
|
||||
|
||||
[Using minio-go library](https://github.com/minio/minio/blob/master/docs/extensions/s3zip/examples/minio-go/main.go)
|
||||
[Using AWS JS SDK v2](https://github.com/minio/minio/blob/master/docs/extensions/s3zip/examples/aws-js/main.js)
|
||||
[Using boto3](https://github.com/minio/minio/blob/master/docs/extensions/s3zip/examples/boto3/main.py)
|
||||
[Using minio-go library](https://github.com/pgsty/minio/blob/master/docs/extensions/s3zip/examples/minio-go/main.go)
|
||||
[Using AWS JS SDK v2](https://github.com/pgsty/minio/blob/master/docs/extensions/s3zip/examples/aws-js/main.js)
|
||||
[Using boto3](https://github.com/pgsty/minio/blob/master/docs/extensions/s3zip/examples/boto3/main.py)
|
||||
|
||||
## Requirements and limits
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ This document explains how to configure MinIO with `Bucket lookup from DNS` styl
|
||||
|
||||
### 1. Prerequisites
|
||||
|
||||
Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html).
|
||||
Install MinIO - [MinIO Quickstart Guide](https://silo.pigsty.io/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html).
|
||||
|
||||
### 2. Run MinIO in federated mode
|
||||
|
||||
@@ -17,7 +17,7 @@ Bucket lookup from DNS federation requires two dependencies
|
||||
|
||||
## Architecture
|
||||
|
||||

|
||||

|
||||
|
||||
### Environment variables
|
||||
|
||||
@@ -76,11 +76,11 @@ it is randomized which cluster might provision the bucket.
|
||||
|
||||
### 3. Test your setup
|
||||
|
||||
To test this setup, access the MinIO server via browser or [`mc`](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart). You’ll see the uploaded files are accessible from the all the MinIO endpoints.
|
||||
To test this setup, access the MinIO server via browser or [`mc`](https://silo.pigsty.io/reference/minio-mc.html#quickstart). You’ll see the uploaded files are accessible from the all the MinIO endpoints.
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html)
|
||||
- [Use `aws-cli` with MinIO Server](https://docs.min.io/community/minio-object-store/integrations/aws-cli-with-minio.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html)
|
||||
- [Use `mc` with MinIO Server](https://silo.pigsty.io/reference/minio-mc.html)
|
||||
- [Use `aws-cli` with MinIO Server](https://silo.pigsty.io/integrations/aws-cli-with-minio.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://silo.pigsty.io/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://silo.pigsty.io/index.html)
|
||||
|
||||
+1
-1
@@ -64,7 +64,7 @@ Developer is advised to clone the MinIO source and checkout the MinIO release ta
|
||||
```
|
||||
|
||||
Create a branch and proceed to push the branch **upstream**
|
||||
> (upstream here points to git@github.com:minio/minio.git)
|
||||
> (upstream here points to git@github.com:pgsty/minio.git)
|
||||
|
||||
```
|
||||
λ git branch -m RELEASE.2021-04-22T15-44-28Z.hotfix
|
||||
|
||||
@@ -5,10 +5,10 @@ When using Veeam Backup and Replication, you can use S3 compatible object storag
|
||||
## Prerequisites
|
||||
|
||||
- One or both of Veeam Backup and Replication with support for S3 compatible object store (e.g. 9.5.4) and Veeam Backup for Office365 (VBO)
|
||||
- MinIO object storage set up per <https://docs.min.io/community/minio-object-store/index.html>
|
||||
- Veeam requires TLS connections to the object storage. This can be configured per <https://docs.min.io/community/minio-object-store/operations/network-encryption.html>
|
||||
- MinIO object storage set up per <https://silo.pigsty.io/index.html>
|
||||
- Veeam requires TLS connections to the object storage. This can be configured per <https://silo.pigsty.io/operations/network-encryption.html>
|
||||
- The S3 bucket, Access Key and Secret Key have to be created before and outside of Veeam.
|
||||
- Configure the minio client for the Veeam MinIO endpoint - <https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html>
|
||||
- Configure the minio client for the Veeam MinIO endpoint - <https://silo.pigsty.io/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html>
|
||||
|
||||
## Setting up an S3 compatible object store for Veeam Backup and Replication
|
||||
|
||||
@@ -26,7 +26,7 @@ mc mb myminio/veeambackup
|
||||
mc mb -l myminio/veeambackup
|
||||
```
|
||||
|
||||
> Object locking requires erasure coding enabled on the minio server. For more information see <https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html>.
|
||||
> Object locking requires erasure coding enabled on the minio server. For more information see <https://silo.pigsty.io/operations/concepts/erasure-coding.html>.
|
||||
|
||||
### Add MinIO as an object store for Veeam
|
||||
|
||||
@@ -34,7 +34,7 @@ Follow the steps from the Veeam documentation for adding MinIO as an object stor
|
||||
|
||||
For Veeam Backup with Immutability, choose the amount of days you want to make backups immutable for
|
||||
|
||||

|
||||

|
||||
|
||||
### Creating the Scale-out Backup Repository
|
||||
|
||||
@@ -58,7 +58,7 @@ For Veeam Backup with Immutability, choose the amount of days you want to make b
|
||||
|
||||
- For Veeam Backup with Immutability, you can choose a number of restore points or days to make backups immutable.
|
||||
|
||||

|
||||

|
||||
|
||||
#### Backup Office 365 with VBO
|
||||
|
||||
@@ -70,7 +70,7 @@ mc mb -l myminio/vbo
|
||||
|
||||
- Under Backup Infrastructure, right click on Object Storage Repositories and choose "Add object storage"
|
||||
|
||||

|
||||

|
||||
|
||||
- Follow through the wizard as above for Veeam Backup and Replication as the steps are the same between both products
|
||||
|
||||
@@ -78,7 +78,7 @@ mc mb -l myminio/vbo
|
||||
|
||||
- Follow the wizard. Under the "Object Storage Backup Repository" section, choose the MinIO object storage you created above
|
||||
|
||||

|
||||

|
||||
|
||||
- When you create your backup job, choose the backup repository you created above.
|
||||
|
||||
|
||||
+2
-2
@@ -7,7 +7,7 @@ MinIO supports encrypting config, IAM assets with KMS provided keys. If the KMS
|
||||
MinIO supports two ways of encrypting IAM and configuration data.
|
||||
You can either use KES - together with an external KMS - or, much simpler,
|
||||
set the env. variable `MINIO_KMS_SECRET_KEY` and start/restart the MinIO server. For more details about KES and how
|
||||
to set it up refer to our [KMS Guide](https://github.com/minio/minio/blob/master/docs/kms/README.md).
|
||||
to set it up refer to our [KMS Guide](https://github.com/pgsty/minio/blob/master/docs/kms/README.md).
|
||||
|
||||
Instead of configuring an external KMS you can start with a single key by
|
||||
setting the env. variable `MINIO_KMS_SECRET_KEY`. It expects the following
|
||||
@@ -45,7 +45,7 @@ kes key create my-minio-key OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=
|
||||
|
||||
- For instructions on setting up KES, see the [KES Getting Started guide](https://github.com/minio/kes/wiki/Getting-Started)
|
||||
|
||||
- For instructions on using KES for encrypting the MinIO backend, follow the [KMS Quick Start](https://github.com/minio/minio/tree/master/docs/kms). The SSE-S3 configuration setup also supports MinIO KMS backend encryption.
|
||||
- For instructions on using KES for encrypting the MinIO backend, follow the [KMS Quick Start](https://github.com/pgsty/minio/tree/master/docs/kms). The SSE-S3 configuration setup also supports MinIO KMS backend encryption.
|
||||
|
||||
## FAQ
|
||||
|
||||
|
||||
+6
-6
@@ -4,7 +4,7 @@ MinIO uses a key-management-system (KMS) to support SSE-S3. If a client requests
|
||||
|
||||
## Quick Start
|
||||
|
||||
MinIO supports multiple KMS implementations via our [KES](https://github.com/minio/kes#kes) project. We run a KES instance at `https://play.min.io:7373` for you to experiment and quickly get started. To run MinIO with a KMS just fetch the root identity, set the following environment variables and then start your MinIO server. If you haven't installed MinIO, yet, then follow the MinIO [install instructions](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html) first.
|
||||
MinIO supports multiple KMS implementations via our [KES](https://github.com/minio/kes#kes) project. We run a KES instance at `https://play.min.io:7373` for you to experiment and quickly get started. To run MinIO with a KMS just fetch the root identity, set the following environment variables and then start your MinIO server. If you haven't installed MinIO, yet, then follow the MinIO [install instructions](https://silo.pigsty.io/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html) first.
|
||||
|
||||
### 1. Fetch the root identity
|
||||
|
||||
@@ -67,7 +67,7 @@ The MinIO-KES configuration is always the same - regardless of the underlying KM
|
||||
|
||||
### Further references
|
||||
|
||||
- [Run MinIO with TLS / HTTPS](https://docs.min.io/community/minio-object-store/operations/network-encryption.html)
|
||||
- [Run MinIO with TLS / HTTPS](https://silo.pigsty.io/operations/network-encryption.html)
|
||||
- [Tweak the KES server configuration](https://github.com/minio/kes/wiki/Configuration)
|
||||
- [Run a load balancer in front of KES](https://github.com/minio/kes/wiki/TLS-Proxy)
|
||||
- [Understand the KES server concepts](https://github.com/minio/kes/wiki/Concepts)
|
||||
@@ -137,7 +137,7 @@ Certificates are no secrets and sent in plaintext as part of the TLS handshake.
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html)
|
||||
- [Use `aws-cli` with MinIO Server](https://docs.min.io/community/minio-object-store/integrations/aws-cli-with-minio.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html)
|
||||
- [Use `mc` with MinIO Server](https://silo.pigsty.io/reference/minio-mc.html)
|
||||
- [Use `aws-cli` with MinIO Server](https://silo.pigsty.io/integrations/aws-cli-with-minio.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://silo.pigsty.io/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://silo.pigsty.io/index.html)
|
||||
|
||||
@@ -4,7 +4,7 @@ MinIO's Object Lambda implementation allows for transforming your data to serve
|
||||
|
||||
MinIO's Object Lambda, enables application developers to process data retrieved from MinIO before returning it to an application. You can register a Lambda Function target on MinIO, once successfully registered it can be used to transform the data for application GET requests on demand.
|
||||
|
||||
This document focuses on showing a working example on how to use Object Lambda with MinIO, you must have [MinIO deployed in your environment](https://docs.min.io/community/minio-object-store/operations/installation.html) before you can start using external lambda functions. You also must install Python version 3.8 or later for the lambda handlers to work.
|
||||
This document focuses on showing a working example on how to use Object Lambda with MinIO, you must have [MinIO deployed in your environment](https://silo.pigsty.io/operations/installation.html) before you can start using external lambda functions. You also must install Python version 3.8 or later for the lambda handlers to work.
|
||||
|
||||
## Example Lambda handler
|
||||
|
||||
@@ -134,7 +134,7 @@ mc cp testobject myminio/functionbucket/
|
||||
|
||||
## Invoke Lambda transformation via PresignedGET
|
||||
|
||||
Following example shows how you can use [`minio-go` PresignedGetObject](https://docs.min.io/community/minio-object-store/developers/go/API.html#presignedgetobject-ctx-context-context-bucketname-objectname-string-expiry-time-duration-reqparams-url-values-url-url-error)
|
||||
Following example shows how you can use [`minio-go` PresignedGetObject](https://silo.pigsty.io/developers/go/API.html#presignedgetobject-ctx-context-context-bucketname-objectname-string-expiry-time-duration-reqparams-url-values-url-url-error)
|
||||
```go
|
||||
package main
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ Console target is on always and cannot be disabled.
|
||||
|
||||
HTTP target logs to a generic HTTP endpoint in JSON format and is not enabled by default. To enable HTTP target logging you would have to update your MinIO server configuration using `mc admin config set` command.
|
||||
|
||||
Assuming `mc` is already [configured](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart)
|
||||
Assuming `mc` is already [configured](https://silo.pigsty.io/reference/minio-mc.html#quickstart)
|
||||
|
||||
```
|
||||
mc admin config get myminio/ logger_webhook
|
||||
@@ -42,7 +42,7 @@ minio server /mnt/data
|
||||
|
||||
## Audit Targets
|
||||
|
||||
Assuming `mc` is already [configured](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart)
|
||||
Assuming `mc` is already [configured](https://silo.pigsty.io/reference/minio-mc.html#quickstart)
|
||||
|
||||
### Audit HTTP Target
|
||||
|
||||
@@ -224,5 +224,5 @@ NOTE:
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html)
|
||||
- [Configure MinIO Server with TLS](https://docs.min.io/community/minio-object-store/operations/network-encryption.html)
|
||||
- [MinIO Quickstart Guide](https://silo.pigsty.io/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html)
|
||||
- [Configure MinIO Server with TLS](https://silo.pigsty.io/operations/network-encryption.html)
|
||||
|
||||
@@ -9,7 +9,7 @@ MinIO server has two healthcheck related un-authenticated endpoints, a liveness
|
||||
- Liveness probe available at `/minio/health/live`
|
||||
- Cluster probe available at `/minio/health/cluster`
|
||||
|
||||
Read more on how to use these endpoints in [MinIO healthcheck guide](https://github.com/minio/minio/blob/master/docs/metrics/healthcheck/README.md).
|
||||
Read more on how to use these endpoints in [MinIO healthcheck guide](https://github.com/pgsty/minio/blob/master/docs/metrics/healthcheck/README.md).
|
||||
|
||||
## Prometheus Probe
|
||||
|
||||
@@ -25,7 +25,7 @@ The additional bucket specific metrics which include additional go metrics or pr
|
||||
The additional resource specific metrics which include additional go metrics or process metrics are exposed at
|
||||
`<Address for MinIO Node>/minio/v2/metrics/resource`.
|
||||
|
||||
To use this endpoint, setup Prometheus to scrape data from this endpoint. Read more on how to configure and use Prometheus to monitor MinIO server in [How to monitor MinIO server with Prometheus](https://github.com/minio/minio/blob/master/docs/metrics/prometheus/README.md).
|
||||
To use this endpoint, setup Prometheus to scrape data from this endpoint. Read more on how to configure and use Prometheus to monitor MinIO server in [How to monitor MinIO server with Prometheus](https://github.com/pgsty/minio/blob/master/docs/metrics/prometheus/README.md).
|
||||
|
||||
### **Deprecated metrics monitoring**
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ This document explains how to setup Prometheus and configure it to scrape data f
|
||||
|
||||
## Prerequisites
|
||||
|
||||
To get started with MinIO, refer [MinIO QuickStart Document](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html).
|
||||
To get started with MinIO, refer [MinIO QuickStart Document](https://silo.pigsty.io/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html).
|
||||
Follow below steps to get started with MinIO monitoring using Prometheus.
|
||||
|
||||
### 1. Download Prometheus
|
||||
@@ -49,7 +49,7 @@ minio server ~/test
|
||||
|
||||
> If MinIO is configured to expose metrics without authentication, you don't need to use `mc` to generate prometheus config. You can skip reading further and move to 3.2 section.
|
||||
|
||||
The Prometheus endpoint in MinIO requires authentication by default. Prometheus supports a bearer token approach to authenticate prometheus scrape requests, override the default Prometheus config with the one generated using mc. To generate a Prometheus config for an alias, use [mc](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart) as follows `mc admin prometheus generate <alias> [METRIC-TYPE]`. The valid values for METRIC-TYPE are `cluster`, `node`, `bucket` and `resource` and if not mentioned, it defaults to `cluster`.
|
||||
The Prometheus endpoint in MinIO requires authentication by default. Prometheus supports a bearer token approach to authenticate prometheus scrape requests, override the default Prometheus config with the one generated using mc. To generate a Prometheus config for an alias, use [mc](https://silo.pigsty.io/reference/minio-mc.html#quickstart) as follows `mc admin prometheus generate <alias> [METRIC-TYPE]`. The valid values for METRIC-TYPE are `cluster`, `node`, `bucket` and `resource` and if not mentioned, it defaults to `cluster`.
|
||||
|
||||
The command will generate the `scrape_configs` section of the prometheus.yml as follows:
|
||||
|
||||
@@ -171,7 +171,7 @@ Prometheus sets the `Host` header to `domain:port` as part of HTTP operations ag
|
||||
|
||||
### 6. Configure Grafana
|
||||
|
||||
After Prometheus is configured, you can use Grafana to visualize MinIO metrics. Refer the [document here to setup Grafana with MinIO prometheus metrics](https://github.com/minio/minio/blob/master/docs/metrics/prometheus/grafana/README.md).
|
||||
After Prometheus is configured, you can use Grafana to visualize MinIO metrics. Refer the [document here to setup Grafana with MinIO prometheus metrics](https://github.com/pgsty/minio/blob/master/docs/metrics/prometheus/grafana/README.md).
|
||||
|
||||
## List of metrics exposed by MinIO
|
||||
|
||||
@@ -188,8 +188,8 @@ curl https://play.min.io/minio/v2/metrics/cluster
|
||||
|
||||
### List of metrics reported Cluster and Bucket level
|
||||
|
||||
[The list of metrics reported can be here](https://github.com/minio/minio/blob/master/docs/metrics/prometheus/list.md)
|
||||
[The list of metrics reported can be here](https://github.com/pgsty/minio/blob/master/docs/metrics/prometheus/list.md)
|
||||
|
||||
### Configure Alerts for Prometheus
|
||||
|
||||
[The Prometheus AlertManager and alerts can be configured following this](https://github.com/minio/minio/blob/master/docs/metrics/prometheus/alerts.md)
|
||||
[The Prometheus AlertManager and alerts can be configured following this](https://github.com/pgsty/minio/blob/master/docs/metrics/prometheus/alerts.md)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user