mirror of
https://github.com/pgsty/minio.git
synced 2026-08-10 16:23:28 +03:00
Compare commits
289 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 07c3a429bf | |||
| 0cde982902 | |||
| d0f50cdd9b | |||
| da532ab93d | |||
| 558fc1c09c | |||
| 9fdbf6fe83 | |||
| 5c87d4ae87 | |||
| f0b91e5504 | |||
| 3b7cb6512c | |||
| 4ea6f3b06b | |||
| 86d9d9b55e | |||
| 5a35585acd | |||
| 0848e69602 | |||
| 02ba581ecf | |||
| b44b2a090c | |||
| c7d6a9722d | |||
| a8abdc797e | |||
| 0638ccc5f3 | |||
| b1a34fd63f | |||
| ffcfa36b13 | |||
| 376fbd11a7 | |||
| c76f209ccc | |||
| 7a6a2256b1 | |||
| d002beaee3 | |||
| 71f293d9ab | |||
| e3d183b6a4 | |||
| 752abc2e2c | |||
| b9f0e8c712 | |||
| 7ced9663e6 | |||
| 50fcf9b670 | |||
| 64f5c6103f | |||
| e909be6380 | |||
| 83b2ad418b | |||
| 7a64bb9766 | |||
| 34679befef | |||
| 4021d8c8e2 | |||
| de234b888c | |||
| 2718d9a430 | |||
| a65292cab1 | |||
| e0c79be251 | |||
| a6c538c5a1 | |||
| e1fcaebc77 | |||
| 21409f112d | |||
| 417c8648f0 | |||
| e2245a0b12 | |||
| b4b3d208dd | |||
| 0a36d41dcd | |||
| ea77bcfc98 | |||
| 9f24ca5d66 | |||
| 816666a4c6 | |||
| 2c7fe094d1 | |||
| 9ebe168782 | |||
| ee2028cde6 | |||
| ecde75f911 | |||
| 12a6ea89cc | |||
| 63e102c049 | |||
| 160f8a901b | |||
| ef9b03fbf5 | |||
| 1d50cae43d | |||
| c0a33952c6 | |||
| 8cad40a483 | |||
| 6d18dba9a2 | |||
| 9ea14c88d8 | |||
| 30a1261c22 | |||
| 0e017ab071 | |||
| f14198e3dc | |||
| 93c389dbc9 | |||
| ddd9a84cd7 | |||
| b7540169a2 | |||
| f01374950f | |||
| 18aceae620 | |||
| 427826abc5 | |||
| 2780778c10 | |||
| 2d8ba15b9e | |||
| bd6dd55e7f | |||
| 0d7408fc99 | |||
| 864f80e226 | |||
| 0379d6a37f | |||
| 43aa8e4259 | |||
| e2ed696619 | |||
| fb3f67a597 | |||
| 7ee75368e0 | |||
| 1d6478b8ae | |||
| 0581001b6f | |||
| 479303e7e9 | |||
| 89aec6804b | |||
| eb33bc6bf5 | |||
| 3310f740f0 | |||
| 4595293ca0 | |||
| 02a67cbd2a | |||
| 2b34e5b9ae | |||
| a6258668a6 | |||
| d0cada583f | |||
| 0bd8f06b62 | |||
| 6640be3bed | |||
| eafeb27e90 | |||
| f2c9eb0f79 | |||
| f2619d1f62 | |||
| 8c70975283 | |||
| 01447d2438 | |||
| 07f31e574c | |||
| 8d223e07fb | |||
| 4041a8727c | |||
| 5f243fde9a | |||
| a0e3f1cc18 | |||
| b1bc641105 | |||
| e0c8738230 | |||
| 9aa24b1920 | |||
| 53d40e41bc | |||
| e88d494775 | |||
| b67f0cf721 | |||
| 46922c71b7 | |||
| 670edb4fcf | |||
| 42d4ab2a0a | |||
| 5e2eb372bf | |||
| cccb37a5ac | |||
| dbf31af6cb | |||
| 93e40c3ab4 | |||
| 8aa0e9ff7c | |||
| bbd6f18afb | |||
| 2a3acc4f24 | |||
| 11507d46da | |||
| f9c62dea55 | |||
| 8c2c92f7af | |||
| 4c71f1b4ec | |||
| 6cd8a372cb | |||
| 953a3e2bbd | |||
| 7cc0c69228 | |||
| f129fd48f2 | |||
| bc4008ced4 | |||
| 526053339b | |||
| 62a35b3e77 | |||
| 39df134204 | |||
| ad4cbce22d | |||
| 90f5e1e5f6 | |||
| aeabac9181 | |||
| b312f13473 | |||
| 727a803bc0 | |||
| d0e443172d | |||
| 60446e7ac0 | |||
| b8544266e5 | |||
| 437dd4e32a | |||
| 447054b841 | |||
| 9bf43e54cd | |||
| 60f8423157 | |||
| 4355ea3c3f | |||
| e30f1ad7bd | |||
| f00c8c4cce | |||
| 703f51164d | |||
| b8dde47d4e | |||
| 7fa3e39f85 | |||
| 4df7a3aa8f | |||
| 64a8f2e554 | |||
| f4fd4ea66d | |||
| 712fe1a8df | |||
| 4a319bedc9 | |||
| bdb3db6dad | |||
| abb385af41 | |||
| 4ee62606e4 | |||
| 079d64c801 | |||
| dcc000ae2c | |||
| c5d19ecebb | |||
| ed29a525b3 | |||
| 020c46cd3c | |||
| 827004cd6d | |||
| 779ec8f0d4 | |||
| 3d0f513ee2 | |||
| 4b6eadbd80 | |||
| 6f47414b23 | |||
| 224a27992a | |||
| 232544e1d8 | |||
| dbcb71828d | |||
| b9196757fd | |||
| b4ac53d157 | |||
| 4952bdb770 | |||
| 00b2ef2932 | |||
| 4536ecfaa4 | |||
| 43a7402968 | |||
| 330dca9a35 | |||
| ddd137d317 | |||
| 06ddd8770e | |||
| 16f8cf1c52 | |||
| 01e520eb23 | |||
| 02f770a0c0 | |||
| 2f4c79bc0f | |||
| 969ee7dfbe | |||
| 5f0b086b05 | |||
| 68b004a48f | |||
| 54ecce66f0 | |||
| 2b008c598b | |||
| 86d02b17cf | |||
| c1a95a70ac | |||
| f246c9053f | |||
| 9cdd204ae4 | |||
| 7b3eb9f7f8 | |||
| d56ef8dbe1 | |||
| a248ed5ff5 | |||
| 5bb31e4883 | |||
| aff2a76d80 | |||
| eddbe6bca2 | |||
| 734d1e320a | |||
| b8dab7b1a9 | |||
| abd6bf060d | |||
| f0d4ef604c | |||
| 2712f75762 | |||
| 4c46668da8 | |||
| 02e93fd6ba | |||
| 366876e98b | |||
| d202fdd022 | |||
| c07e5b49d4 | |||
| 9a39f8ad4d | |||
| 7e0c1c9413 | |||
| 485d833cd7 | |||
| e8a476ef5a | |||
| 267f0ecea2 | |||
| 4ee3434854 | |||
| 0e9854372e | |||
| b5177993b3 | |||
| 55f5c18fd9 | |||
| 8ce101c174 | |||
| 4972735507 | |||
| e6ca6de194 | |||
| cefc43e4da | |||
| 25e34fda5f | |||
| 4208d7af5a | |||
| 8d42f37e4b | |||
| 7cb4b5c636 | |||
| 1615920f48 | |||
| 7ee42b3ff5 | |||
| a6f1e727fb | |||
| c1fc7779ca | |||
| b3ab7546ee | |||
| ad88a81e3d | |||
| c4239ced22 | |||
| f85c28e960 | |||
| f7e176d4ca | |||
| 72a0d14195 | |||
| 6abe4128d7 | |||
| ed5ed7e490 | |||
| 51410c9023 | |||
| 96ca402dcd | |||
| 3da7c9cce3 | |||
| a14e19ec54 | |||
| e091dde041 | |||
| d10bb7e1b6 | |||
| 7ebceacac6 | |||
| 1593cb615d | |||
| 86a41d1631 | |||
| d4157b819c | |||
| e0aceca1b7 | |||
| 87804624fe | |||
| e029f8a9d7 | |||
| 1bc6681176 | |||
| 28322124e2 | |||
| cbfe9de3e7 | |||
| dc86b8d9d4 | |||
| ba70118e2b | |||
| cb1d3e50f7 | |||
| ded0b19d97 | |||
| d0bb3dd136 | |||
| ab7714b01e | |||
| e5b18df6db | |||
| 0abfd1bcb1 | |||
| 6186d11761 | |||
| e8b457e8a6 | |||
| afea40cc0f | |||
| 402b798f1b | |||
| 4759532e90 | |||
| 7f1e1713ab | |||
| b2c5819dbc | |||
| 2b0156b1fc | |||
| f6f0807c86 | |||
| 0c53d86017 | |||
| 6a6ee46d76 | |||
| 974cbb3bb7 | |||
| 03e996320e | |||
| 78fcb76294 | |||
| 3d152015eb | |||
| ade8925155 | |||
| 05a6c170bf | |||
| e1c2344591 | |||
| 48a591e9b4 | |||
| fa5d9c02ef | |||
| 5bd27346ac | |||
| 3c82cf9327 | |||
| 70d40083e9 | |||
| 8a30967542 | |||
| 229f04ab79 | |||
| 1123dc3676 |
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.22.x]
|
||||
go-version: [1.24.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
name: FIPS Build Test
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Go BoringCrypto ${{ matrix.go-version }} on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.22.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
|
||||
- name: Setup dockerfile for build test
|
||||
run: |
|
||||
GO_VERSION=$(go version | cut -d ' ' -f 3 | sed 's/go//')
|
||||
echo Detected go version $GO_VERSION
|
||||
cat > Dockerfile.fips.test <<EOF
|
||||
FROM golang:${GO_VERSION}
|
||||
COPY . /minio
|
||||
WORKDIR /minio
|
||||
ENV GOEXPERIMENT=boringcrypto
|
||||
RUN make
|
||||
EOF
|
||||
|
||||
- name: Build
|
||||
uses: docker/build-push-action@v3
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile.fips.test
|
||||
push: false
|
||||
load: true
|
||||
tags: minio/fips-test:latest
|
||||
|
||||
# This should fail if grep returns non-zero exit
|
||||
- name: Test binary
|
||||
run: |
|
||||
docker run --rm minio/fips-test:latest ./minio --version
|
||||
docker run --rm -i minio/fips-test:latest /bin/bash -c 'go tool nm ./minio | grep FIPS | grep -q FIPS'
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.22.x]
|
||||
go-version: [1.24.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -20,24 +20,14 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.22.x]
|
||||
os: [ubuntu-latest, Windows]
|
||||
go-version: [1.24.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- name: Build on ${{ matrix.os }}
|
||||
if: matrix.os == 'Windows'
|
||||
env:
|
||||
CGO_ENABLED: 0
|
||||
GO111MODULE: on
|
||||
run: |
|
||||
Set-MpPreference -DisableRealtimeMonitoring $true
|
||||
netsh int ipv4 set dynamicport tcp start=60000 num=61000
|
||||
go build --ldflags="-s -w" -o %GOPATH%\bin\minio.exe
|
||||
go test -v --timeout 120m ./...
|
||||
- name: Build on ${{ matrix.os }}
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
env:
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
name: Resiliency Functional Tests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.24.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- name: Build on ${{ matrix.os }}
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
env:
|
||||
CGO_ENABLED: 0
|
||||
GO111MODULE: on
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make test-resiliency
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.22.x]
|
||||
go-version: [1.24.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -39,3 +39,4 @@ jobs:
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make verify
|
||||
make test-timeout
|
||||
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
# are turned off - i.e. if ldap="", then ldap server is not enabled for
|
||||
# the tests.
|
||||
matrix:
|
||||
go-version: [1.22.x]
|
||||
go-version: [1.24.x]
|
||||
ldap: ["", "localhost:389"]
|
||||
etcd: ["", "http://localhost:2379"]
|
||||
openid: ["", "http://127.0.0.1:5556/dex"]
|
||||
@@ -142,3 +142,20 @@ jobs:
|
||||
- name: Test import of IAM artifacts when in fresh cluster there are missing groups etc
|
||||
run: |
|
||||
make test-iam-import-with-missing-entities
|
||||
iam-import-with-openid:
|
||||
name: Test IAM import in new cluster with opendid configurations
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- name: Checkout minio-iam-testing
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: minio/minio-iam-testing
|
||||
path: minio-iam-testing
|
||||
- name: Test import of IAM artifacts when in fresh cluster with openid configurations
|
||||
run: |
|
||||
make test-iam-import-with-openid
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
- name: setup-go-step
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 1.22.x
|
||||
go-version: 1.24.x
|
||||
|
||||
- name: github sha short
|
||||
id: vars
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.22.x]
|
||||
go-version: [1.24.x]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -40,6 +40,7 @@ jobs:
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make test-ilm
|
||||
make test-ilm-transition
|
||||
|
||||
- name: Test PBAC
|
||||
run: |
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.22.x]
|
||||
go-version: [1.24.x]
|
||||
os: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
|
||||
@@ -15,6 +15,9 @@ docker volume rm $(docker volume ls -f dangling=true) || true
|
||||
## change working directory
|
||||
cd .github/workflows/mint
|
||||
|
||||
## always pull latest
|
||||
docker pull docker.io/minio/mint:edge
|
||||
|
||||
docker-compose -f minio-${MODE}.yaml up -d
|
||||
sleep 1m
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.22.x]
|
||||
go-version: [1.24.x]
|
||||
os: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
|
||||
@@ -21,7 +21,8 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 1.22.7
|
||||
go-version: 1.24.x
|
||||
cached: false
|
||||
- name: Get official govulncheck
|
||||
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
shell: bash
|
||||
|
||||
+51
-23
@@ -1,36 +1,64 @@
|
||||
linters-settings:
|
||||
gofumpt:
|
||||
simplify: true
|
||||
|
||||
misspell:
|
||||
locale: US
|
||||
|
||||
staticcheck:
|
||||
checks: ['all', '-ST1005', '-ST1000', '-SA4000', '-SA9004', '-SA1019', '-SA1008', '-U1000', '-ST1016']
|
||||
|
||||
version: "2"
|
||||
linters:
|
||||
disable-all: true
|
||||
default: none
|
||||
enable:
|
||||
- durationcheck
|
||||
- forcetypeassert
|
||||
- gocritic
|
||||
- gofumpt
|
||||
- goimports
|
||||
- gomodguard
|
||||
- govet
|
||||
- ineffassign
|
||||
- misspell
|
||||
- revive
|
||||
- staticcheck
|
||||
- tenv
|
||||
- typecheck
|
||||
- unconvert
|
||||
- unused
|
||||
|
||||
- usetesting
|
||||
- whitespace
|
||||
settings:
|
||||
misspell:
|
||||
locale: US
|
||||
staticcheck:
|
||||
checks:
|
||||
- all
|
||||
- -SA1008
|
||||
- -SA1019
|
||||
- -SA4000
|
||||
- -SA9004
|
||||
- -ST1000
|
||||
- -ST1005
|
||||
- -ST1016
|
||||
- -U1000
|
||||
exclusions:
|
||||
generated: lax
|
||||
rules:
|
||||
- linters:
|
||||
- forcetypeassert
|
||||
path: _test\.go
|
||||
- path: (.+)\.go$
|
||||
text: 'empty-block:'
|
||||
- path: (.+)\.go$
|
||||
text: 'unused-parameter:'
|
||||
- path: (.+)\.go$
|
||||
text: 'dot-imports:'
|
||||
- path: (.+)\.go$
|
||||
text: should have a package comment
|
||||
- path: (.+)\.go$
|
||||
text: error strings should not be capitalized or end with punctuation or a newline
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
issues:
|
||||
exclude-use-default: false
|
||||
exclude:
|
||||
- "empty-block:"
|
||||
- "unused-parameter:"
|
||||
- "dot-imports:"
|
||||
- should have a package comment
|
||||
- error strings should not be capitalized or end with punctuation or a newline
|
||||
max-issues-per-linter: 100
|
||||
max-same-issues: 100
|
||||
formatters:
|
||||
enable:
|
||||
- gofumpt
|
||||
- goimports
|
||||
exclusions:
|
||||
generated: lax
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
|
||||
+10
-7
@@ -1,11 +1,5 @@
|
||||
[files]
|
||||
extend-exclude = [
|
||||
".git/",
|
||||
"docs/",
|
||||
"CREDITS",
|
||||
"go.mod",
|
||||
"go.sum",
|
||||
]
|
||||
extend-exclude = [".git/", "docs/", "CREDITS", "go.mod", "go.sum"]
|
||||
ignore-hidden = false
|
||||
|
||||
[default]
|
||||
@@ -20,6 +14,7 @@ extend-ignore-re = [
|
||||
'http\.Header\{"X-Amz-Server-Side-Encryptio":',
|
||||
"ZoEoZdLlzVbOlT9rbhD7ZN7TLyiYXSAlB79uGEge",
|
||||
"ERRO:",
|
||||
"(?Rm)^.*(#|//)\\s*spellchecker:disable-line$", # ignore line
|
||||
]
|
||||
|
||||
[default.extend-words]
|
||||
@@ -40,3 +35,11 @@ extend-ignore-re = [
|
||||
"TestGetPartialObjectMisAligned" = "TestGetPartialObjectMisAligned"
|
||||
"thr" = "thr"
|
||||
"toi" = "toi"
|
||||
|
||||
[type.go]
|
||||
extend-ignore-identifiers-re = [
|
||||
# Variants of `typ` used to mean `type` in golang as it is otherwise a
|
||||
# keyword - some of these (like typ1 -> type1) can be fixed, but probably
|
||||
# not worth the effort.
|
||||
"[tT]yp[0-9]*",
|
||||
]
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
FROM minio/minio:latest
|
||||
|
||||
RUN chmod 1777 /usr/bin
|
||||
RUN chmod -R 777 /usr/bin
|
||||
|
||||
COPY ./minio /usr/bin/minio
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.22-alpine as build
|
||||
FROM golang:1.24-alpine as build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
@@ -53,7 +53,7 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_CONFIG_ENV_FILE=config.env \
|
||||
MC_CONFIG_DIR=/tmp/.mc
|
||||
|
||||
RUN chmod 1777 /usr/bin
|
||||
RUN chmod -R 777 /usr/bin
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /go/bin/minio* /usr/bin/
|
||||
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.22-alpine AS build
|
||||
FROM golang:1.24-alpine AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
@@ -55,7 +55,7 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_CONFIG_ENV_FILE=config.env \
|
||||
MC_CONFIG_DIR=/tmp/.mc
|
||||
|
||||
RUN chmod 1777 /usr/bin
|
||||
RUN chmod -R 777 /usr/bin
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /go/bin/minio* /usr/bin/
|
||||
|
||||
@@ -1,69 +0,0 @@
|
||||
FROM golang:1.22-alpine AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
|
||||
ENV GOPATH=/go
|
||||
ENV CGO_ENABLED=0
|
||||
|
||||
# Install curl and minisign
|
||||
RUN apk add -U --no-cache ca-certificates && \
|
||||
apk add -U --no-cache curl && \
|
||||
go install aead.dev/minisign/cmd/minisign@v0.2.1
|
||||
|
||||
# Download minio binary and signature files
|
||||
RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /go/bin/minio && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /go/bin/minio.minisig && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.sha256sum -o /go/bin/minio.sha256sum && \
|
||||
chmod +x /go/bin/minio
|
||||
|
||||
# Download mc binary and signature files
|
||||
RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.fips -o /go/bin/mc && \
|
||||
curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.fips.minisig -o /go/bin/mc.minisig && \
|
||||
curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.fips.sha256sum -o /go/bin/mc.sha256sum && \
|
||||
chmod +x /go/bin/mc
|
||||
|
||||
RUN if [ "$TARGETARCH" = "amd64" ]; then \
|
||||
curl -L -s -q https://github.com/moparisthebest/static-curl/releases/latest/download/curl-${TARGETARCH} -o /go/bin/curl; \
|
||||
chmod +x /go/bin/curl; \
|
||||
fi
|
||||
|
||||
# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
|
||||
RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
|
||||
|
||||
FROM registry.access.redhat.com/ubi9/ubi-micro:latest
|
||||
|
||||
ARG RELEASE
|
||||
|
||||
LABEL name="MinIO" \
|
||||
vendor="MinIO Inc <dev@min.io>" \
|
||||
maintainer="MinIO Inc <dev@min.io>" \
|
||||
version="${RELEASE}" \
|
||||
release="${RELEASE}" \
|
||||
summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \
|
||||
description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads."
|
||||
|
||||
ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_SECRET_KEY_FILE=secret_key \
|
||||
MINIO_ROOT_USER_FILE=access_key \
|
||||
MINIO_ROOT_PASSWORD_FILE=secret_key \
|
||||
MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
|
||||
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
|
||||
MINIO_CONFIG_ENV_FILE=config.env
|
||||
|
||||
RUN chmod 1777 /usr/bin
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /go/bin/minio* /usr/bin/
|
||||
COPY --from=build /go/bin/mc* /usr/bin/
|
||||
COPY --from=build /go/bin/cur* /usr/bin/
|
||||
|
||||
COPY CREDITS /licenses/CREDITS
|
||||
COPY LICENSE /licenses/LICENSE
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
|
||||
EXPOSE 9000
|
||||
VOLUME ["/data"]
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
CMD ["minio"]
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.22-alpine AS build
|
||||
FROM golang:1.24-alpine AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
@@ -53,7 +53,7 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_CONFIG_ENV_FILE=config.env \
|
||||
MC_CONFIG_DIR=/tmp/.mc
|
||||
|
||||
RUN chmod 1777 /usr/bin
|
||||
RUN chmod -R 777 /usr/bin
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /go/bin/minio* /usr/bin/
|
||||
|
||||
@@ -2,8 +2,8 @@ PWD := $(shell pwd)
|
||||
GOPATH := $(shell go env GOPATH)
|
||||
LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
|
||||
|
||||
GOARCH := $(shell go env GOARCH)
|
||||
GOOS := $(shell go env GOOS)
|
||||
GOOS ?= $(shell go env GOOS)
|
||||
GOARCH ?= $(shell go env GOARCH)
|
||||
|
||||
VERSION ?= $(shell git describe --tags)
|
||||
REPO ?= quay.io/minio
|
||||
@@ -24,8 +24,6 @@ help: ## print this help
|
||||
getdeps: ## fetch necessary dependencies
|
||||
@mkdir -p ${GOPATH}/bin
|
||||
@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR)
|
||||
@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.10-0.20240227114326-6d6f813fff1b
|
||||
@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
|
||||
|
||||
crosscompile: ## cross compile minio
|
||||
@(env bash $(PWD)/buildscripts/cross-compile.sh)
|
||||
@@ -60,6 +58,10 @@ test-ilm: install-race
|
||||
@echo "Running ILM tests"
|
||||
@env bash $(PWD)/docs/bucket/replication/setup_ilm_expiry_replication.sh
|
||||
|
||||
test-ilm-transition: install-race
|
||||
@echo "Running ILM tiering tests with healing"
|
||||
@env bash $(PWD)/docs/bucket/lifecycle/setup_ilm_transition.sh
|
||||
|
||||
test-pbac: install-race
|
||||
@echo "Running bucket policies tests"
|
||||
@env bash $(PWD)/docs/iam/policies/pbac-tests.sh
|
||||
@@ -101,6 +103,10 @@ test-iam-import-with-missing-entities: install-race ## test import of external i
|
||||
@echo "Test IAM import configurations with missing entities"
|
||||
@env bash $(PWD)/docs/distributed/iam-import-with-missing-entities.sh
|
||||
|
||||
test-iam-import-with-openid: install-race
|
||||
@echo "Test IAM import configurations with openid"
|
||||
@env bash $(PWD)/docs/distributed/iam-import-with-openid.sh
|
||||
|
||||
test-sio-error:
|
||||
@(env bash $(PWD)/docs/bucket/replication/sio-error.sh)
|
||||
|
||||
@@ -141,6 +147,10 @@ test-multipart: install-race ## test multipart
|
||||
@echo "Test multipart behavior when part files are missing"
|
||||
@(env bash $(PWD)/buildscripts/multipart-quorum-test.sh)
|
||||
|
||||
test-timeout: install-race ## test multipart
|
||||
@echo "Test server timeout"
|
||||
@(env bash $(PWD)/buildscripts/test-timeout.sh)
|
||||
|
||||
verify: install-race ## verify minio various setups
|
||||
@echo "Verifying build with race"
|
||||
@(env bash $(PWD)/buildscripts/verify-build.sh)
|
||||
@@ -168,7 +178,7 @@ build-debugging:
|
||||
|
||||
build: checks build-debugging ## builds minio to $(PWD)
|
||||
@echo "Building minio binary to './minio'"
|
||||
@CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
||||
@CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
||||
|
||||
hotfix-vars:
|
||||
$(eval LDFLAGS := $(shell MINIO_RELEASE="RELEASE" MINIO_HOTFIX="hotfix.$(shell git rev-parse --short HEAD)" go run buildscripts/gen-ldflags.go $(shell git describe --tags --abbrev=0 | \
|
||||
@@ -176,9 +186,9 @@ hotfix-vars:
|
||||
$(eval VERSION := $(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD))
|
||||
|
||||
hotfix: hotfix-vars clean install ## builds minio binary with hotfix tags
|
||||
@wget -q -c https://github.com/minio/pkger/releases/download/v2.3.1/pkger_2.3.1_linux_amd64.deb
|
||||
@wget -q -c https://raw.githubusercontent.com/minio/minio-service/v1.0.1/linux-systemd/distributed/minio.service
|
||||
@sudo apt install ./pkger_2.3.1_linux_amd64.deb --yes
|
||||
@wget -q -c https://github.com/minio/pkger/releases/download/v2.3.11/pkger_2.3.11_linux_amd64.deb
|
||||
@wget -q -c https://raw.githubusercontent.com/minio/minio-service/v1.1.1/linux-systemd/distributed/minio.service
|
||||
@sudo apt install ./pkger_2.3.11_linux_amd64.deb --yes
|
||||
@mkdir -p minio-release/$(GOOS)-$(GOARCH)/archive
|
||||
@cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio
|
||||
@cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION)
|
||||
@@ -188,11 +198,11 @@ hotfix: hotfix-vars clean install ## builds minio binary with hotfix tags
|
||||
@pkger -r $(VERSION) --ignore
|
||||
|
||||
hotfix-push: hotfix
|
||||
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-amd64/
|
||||
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive
|
||||
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-amd64/
|
||||
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive
|
||||
@echo "Published new hotfix binaries at https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.$(VERSION)"
|
||||
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-$(GOOS)/
|
||||
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-$(GOOS)/archive
|
||||
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-$(GOOS)/
|
||||
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-$(GOOS)/archive
|
||||
@echo "Published new hotfix binaries at https://dl.min.io/server/minio/hotfixes/linux-$(GOOS)/archive/minio.$(VERSION)"
|
||||
|
||||
docker-hotfix-push: docker-hotfix
|
||||
@docker push -q $(TAG) && echo "Published new container $(TAG)"
|
||||
@@ -205,6 +215,10 @@ docker: build ## builds minio docker container
|
||||
@echo "Building minio docker image '$(TAG)'"
|
||||
@docker build -q --no-cache -t $(TAG) . -f Dockerfile
|
||||
|
||||
test-resiliency: build
|
||||
@echo "Running resiliency tests"
|
||||
@(DOCKER_COMPOSE_FILE=$(PWD)/docs/resiliency/docker-compose.yaml env bash $(PWD)/docs/resiliency/resiliency-tests.sh)
|
||||
|
||||
install-race: checks build-debugging ## builds minio to $(PWD)
|
||||
@echo "Building minio binary with -race to './minio'"
|
||||
@GORACE=history_size=7 CGO_ENABLED=1 go build -tags kqueue,dev -race -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
# MinIO Pull Request Guidelines
|
||||
|
||||
These guidelines ensure high-quality commits in MinIO’s GitHub repositories, maintaining
|
||||
a clear, valuable commit history for our open-source projects. They apply to all contributors,
|
||||
fostering efficient reviews and robust code.
|
||||
|
||||
## Why Pull Requests?
|
||||
|
||||
Pull Requests (PRs) drive quality in MinIO’s codebase by:
|
||||
- Enabling peer review without pair programming.
|
||||
- Documenting changes for future reference.
|
||||
- Ensuring commits tell a clear story of development.
|
||||
|
||||
**A poor commit lasts forever, even if code is refactored.**
|
||||
|
||||
## Crafting a Quality PR
|
||||
|
||||
A strong MinIO PR:
|
||||
- Delivers a complete, valuable change (feature, bug fix, or improvement).
|
||||
- Has a concise title (e.g., `[S3] Fix bucket policy parsing #1234`) and a summary with context, referencing issues (e.g., `#1234`).
|
||||
- Contains well-written, logical commits explaining *why* changes were made (e.g., “Add S3 bucket tagging support so that users can organize resources efficiently”).
|
||||
- Is small, focused, and easy to review—ideally one commit, unless multiple commits better narrate complex work.
|
||||
- Adheres to MinIO’s coding standards (e.g., Go style, error handling, testing).
|
||||
|
||||
PRs must flow smoothly through review to reach production. Large PRs should be split into smaller, manageable ones.
|
||||
|
||||
## Submitting PRs
|
||||
|
||||
1. **Title and Summary**:
|
||||
- Use a scannable title: `[Subsystem] Action Description #Issue` (e.g., `[IAM] Add role-based access control #567`).
|
||||
- Include context in the summary: what changed, why, and any issue references.
|
||||
- Use `[WIP]` for in-progress PRs to avoid premature merging or choose GitHub draft PRs.
|
||||
|
||||
2. **Commits**:
|
||||
- Write clear messages: what changed and why (e.g., “Refactor S3 API handler to reduce latency so that requests process 20% faster”).
|
||||
- Rebase to tidy commits before submitting (e.g., `git rebase -i main` to squash typos or reword messages), unless multiple contributors worked on the branch.
|
||||
- Keep PRs focused—one feature or fix. Split large changes into multiple PRs.
|
||||
|
||||
3. **Testing**:
|
||||
- Include unit tests for new functionality or bug fixes.
|
||||
- Ensure existing tests pass (`make test`).
|
||||
- Document testing steps in the PR summary if manual testing was performed.
|
||||
|
||||
4. **Before Submitting**:
|
||||
- Run `make verify` to check formatting, linting, and tests.
|
||||
- Reference related issues (e.g., “Closes #1234”).
|
||||
- Notify team members via GitHub `@mentions` if urgent or complex.
|
||||
|
||||
## Reviewing PRs
|
||||
|
||||
Reviewers ensure MinIO’s commit history remains a clear, reliable record. Responsibilities include:
|
||||
|
||||
1. **Commit Quality**:
|
||||
- Verify each commit explains *why* the change was made (e.g., “So that…”).
|
||||
- Request rebasing if commits are unclear, redundant, or lack context (e.g., “Please squash typo fixes into the parent commit”).
|
||||
|
||||
2. **Code Quality**:
|
||||
- Check adherence to MinIO’s Go standards (e.g., error handling, documentation).
|
||||
- Ensure tests cover new code and pass CI.
|
||||
- Flag bugs or critical issues for immediate fixes; suggest non-blocking improvements as follow-up issues.
|
||||
|
||||
3. **Flow**:
|
||||
- Review promptly to avoid blocking progress.
|
||||
- Balance quality and speed—minor issues can be addressed later via issues, not PR blocks.
|
||||
- If unable to complete the review, tag another reviewer (e.g., `@username please take over`).
|
||||
|
||||
4. **Shared Responsibility**:
|
||||
- All MinIO contributors are reviewers. The first commenter on a PR owns the review unless they delegate.
|
||||
- Multiple reviewers are encouraged for complex PRs.
|
||||
|
||||
5. **No Self-Edits**:
|
||||
- Don’t modify the PR directly (e.g., fixing bugs). Request changes from the submitter or create a follow-up PR.
|
||||
- If you edit, you’re a collaborator, not a reviewer, and cannot merge.
|
||||
|
||||
6. **Testing**:
|
||||
- Assume the submitter tested the code. If testing is unclear, ask for details (e.g., “How was this tested?”).
|
||||
- Reject untested PRs unless testing is infeasible, then assist with test setup.
|
||||
|
||||
## Tips for Success
|
||||
|
||||
- **Small PRs**: Easier to review, faster to merge. Split large changes logically.
|
||||
- **Clear Commits**: Use `git rebase -i` to refine history before submitting.
|
||||
- **Engage Early**: Discuss complex changes in issues or Slack (https://slack.min.io) before coding.
|
||||
- **Be Responsive**: Address reviewer feedback promptly to keep PRs moving.
|
||||
- **Learn from Reviews**: Use feedback to improve future contributions.
|
||||
|
||||
## Resources
|
||||
|
||||
- [MinIO Coding Standards](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)
|
||||
- [Effective Commit Messages](https://mislav.net/2014/02/hidden-documentation/)
|
||||
- [GitHub PR Tips](https://github.com/blog/1943-how-to-write-the-perfect-pull-request)
|
||||
|
||||
By following these guidelines, we ensure MinIO’s codebase remains high-quality, maintainable, and a joy to contribute to. Happy coding!
|
||||
@@ -1,7 +0,0 @@
|
||||
# MinIO FIPS Builds
|
||||
|
||||
MinIO creates FIPS builds using a patched version of the Go compiler (that uses BoringCrypto, from BoringSSL, which is [FIPS 140-2 validated](https://csrc.nist.gov/csrc/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp2964.pdf)) published by the Golang Team [here](https://github.com/golang/go/tree/dev.boringcrypto/misc/boring).
|
||||
|
||||
MinIO FIPS executables are available at <http://dl.min.io> - they are only published for `linux-amd64` architecture as binary files with the suffix `.fips`. We also publish corresponding container images to our official image repositories.
|
||||
|
||||
We are not making any statements or representations about the suitability of this code or build in relation to the FIPS 140-2 standard. Interested users will have to evaluate for themselves whether this is useful for their own purposes.
|
||||
@@ -4,7 +4,13 @@
|
||||
|
||||
[](https://min.io)
|
||||
|
||||
MinIO is a High Performance Object Storage released under GNU Affero General Public License v3.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads.
|
||||
MinIO is a high-performance, S3-compatible object storage solution released under the GNU AGPL v3.0 license. Designed for speed and scalability, it powers AI/ML, analytics, and data-intensive workloads with industry-leading performance.
|
||||
|
||||
🔹 S3 API Compatible – Seamless integration with existing S3 tools
|
||||
🔹 Built for AI & Analytics – Optimized for large-scale data pipelines
|
||||
🔹 High Performance – Ideal for demanding storage workloads.
|
||||
|
||||
AI storage documentation (https://min.io/solutions/object-storage-for-ai).
|
||||
|
||||
This README provides quickstart instructions on running MinIO on bare metal hardware, including container-based installations. For Kubernetes environments, use the [MinIO Kubernetes Operator](https://github.com/minio/operator/blob/master/README.md).
|
||||
|
||||
@@ -14,7 +20,7 @@ Use the following commands to run a standalone MinIO server as a container.
|
||||
|
||||
Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication
|
||||
require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically,
|
||||
with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html)
|
||||
with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html)
|
||||
for more complete documentation.
|
||||
|
||||
### Stable
|
||||
@@ -32,15 +38,17 @@ root credentials. You can use the Browser to create buckets, upload objects, and
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See
|
||||
[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
|
||||
see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
|
||||
see <https://docs.min.io/community/minio-object-store/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
|
||||
|
||||
> NOTE: To deploy MinIO on with persistent storage, you must map local persistent directories from the host OS to the container using the `podman -v` option. For example, `-v /mnt/data:/data` maps the host OS drive at `/mnt/data` to `/data` on the container.
|
||||
> [!NOTE]
|
||||
> To deploy MinIO on with persistent storage, you must map local persistent directories from the host OS to the container using the `podman -v` option.
|
||||
> For example, `-v /mnt/data:/data` maps the host OS drive at `/mnt/data` to `/data` on the container.
|
||||
|
||||
## macOS
|
||||
|
||||
Use the following commands to run a standalone MinIO server on macOS.
|
||||
|
||||
Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html) for more complete documentation.
|
||||
Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html) for more complete documentation.
|
||||
|
||||
### Homebrew (recommended)
|
||||
|
||||
@@ -51,7 +59,8 @@ brew install minio/stable/minio
|
||||
minio server /data
|
||||
```
|
||||
|
||||
> NOTE: If you previously installed minio using `brew install minio` then it is recommended that you reinstall minio from `minio/stable/minio` official repo instead.
|
||||
> [!NOTE]
|
||||
> If you previously installed minio using `brew install minio` then it is recommended that you reinstall minio from `minio/stable/minio` official repo instead.
|
||||
|
||||
```sh
|
||||
brew uninstall minio
|
||||
@@ -60,7 +69,7 @@ brew install minio/stable/minio
|
||||
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html/> to view MinIO SDKs for supported languages.
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/community/minio-object-store/developers/minio-drivers.html/> to view MinIO SDKs for supported languages.
|
||||
|
||||
### Binary Download
|
||||
|
||||
@@ -74,7 +83,7 @@ chmod +x minio
|
||||
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/community/minio-object-store/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
|
||||
|
||||
## GNU/Linux
|
||||
|
||||
@@ -93,13 +102,13 @@ The following table lists supported architectures. Replace the `wget` URL with t
|
||||
| 64-bit Intel/AMD | <https://dl.min.io/server/minio/release/linux-amd64/minio> |
|
||||
| 64-bit ARM | <https://dl.min.io/server/minio/release/linux-arm64/minio> |
|
||||
| 64-bit PowerPC LE (ppc64le) | <https://dl.min.io/server/minio/release/linux-ppc64le/minio> |
|
||||
| IBM Z-Series (S390X) | <https://dl.min.io/server/minio/release/linux-s390x/minio> |
|
||||
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/community/minio-object-store/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
|
||||
|
||||
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html#) for more complete documentation.
|
||||
> [!NOTE]
|
||||
> Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html) for more complete documentation.
|
||||
|
||||
## Microsoft Windows
|
||||
|
||||
@@ -117,13 +126,14 @@ minio.exe server D:\
|
||||
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/community/minio-object-store/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
|
||||
|
||||
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html#) for more complete documentation.
|
||||
> [!NOTE]
|
||||
> Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html) for more complete documentation.
|
||||
|
||||
## Install from Source
|
||||
|
||||
Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.21](https://golang.org/dl/#stable)
|
||||
Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.24](https://golang.org/dl/#stable)
|
||||
|
||||
```sh
|
||||
go install github.com/minio/minio@latest
|
||||
@@ -131,9 +141,10 @@ go install github.com/minio/minio@latest
|
||||
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/community/minio-object-store/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
|
||||
|
||||
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html) for more complete documentation.
|
||||
> [!NOTE]
|
||||
> Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html) for more complete documentation.
|
||||
|
||||
MinIO strongly recommends *against* using compiled-from-source MinIO servers for production environments.
|
||||
|
||||
@@ -171,7 +182,8 @@ This command gets the active zone(s). Now, apply port rules to the relevant zone
|
||||
firewall-cmd --zone=public --add-port=9000/tcp --permanent
|
||||
```
|
||||
|
||||
Note that `permanent` makes sure the rules are persistent across firewall start, restart or reload. Finally reload the firewall for changes to take effect.
|
||||
> [!NOTE]
|
||||
> `permanent` makes sure the rules are persistent across firewall start, restart or reload. Finally reload the firewall for changes to take effect.
|
||||
|
||||
```sh
|
||||
firewall-cmd --reload
|
||||
@@ -200,7 +212,8 @@ service iptables restart
|
||||
|
||||
MinIO Server comes with an embedded web based object browser. Point your web browser to <http://127.0.0.1:9000> to ensure your server has started successfully.
|
||||
|
||||
> NOTE: MinIO runs console on random port by default, if you wish to choose a specific port use `--console-address` to pick a specific interface and port.
|
||||
> [!NOTE]
|
||||
> MinIO runs console on random port by default, if you wish to choose a specific port use `--console-address` to pick a specific interface and port.
|
||||
|
||||
### Things to consider
|
||||
|
||||
@@ -216,15 +229,16 @@ For example, consider a MinIO deployment behind a proxy `https://minio.example.n
|
||||
|
||||
## Test using MinIO Client `mc`
|
||||
|
||||
`mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the MinIO Client [Quickstart Guide](https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart) for further instructions.
|
||||
`mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the MinIO Client [Quickstart Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart) for further instructions.
|
||||
|
||||
## Upgrading MinIO
|
||||
|
||||
Upgrades require zero downtime in MinIO, all upgrades are non-disruptive, all transactions on MinIO are atomic. So upgrading all the servers simultaneously is the recommended way to upgrade MinIO.
|
||||
|
||||
> NOTE: requires internet access to update directly from <https://dl.min.io>, optionally you can host any mirrors at <https://my-artifactory.example.com/minio/>
|
||||
> [!NOTE]
|
||||
> requires internet access to update directly from <https://dl.min.io>, optionally you can host any mirrors at <https://my-artifactory.example.com/minio/>
|
||||
|
||||
- For deployments that installed the MinIO server binary by hand, use [`mc admin update`](https://min.io/docs/minio/linux/reference/minio-mc-admin/mc-admin-update.html)
|
||||
- For deployments that installed the MinIO server binary by hand, use [`mc admin update`](https://docs.min.io/community/minio-object-store/reference/minio-mc-admin/mc-admin-update.html)
|
||||
|
||||
```sh
|
||||
mc admin update <minio alias, e.g., myminio>
|
||||
@@ -244,10 +258,10 @@ mc admin update <minio alias, e.g., myminio>
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html)
|
||||
- [Use `mc` with MinIO Server](https://min.io/docs/minio/linux/reference/minio-mc.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://min.io/docs/minio/linux/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://min.io/docs/minio/linux/index.html)
|
||||
- [MinIO Erasure Code Overview](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html)
|
||||
- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html)
|
||||
|
||||
## Contribute to MinIO Project
|
||||
|
||||
|
||||
@@ -74,11 +74,11 @@ check_minimum_version() {
|
||||
|
||||
assert_is_supported_arch() {
|
||||
case "${ARCH}" in
|
||||
x86_64 | amd64 | aarch64 | ppc64le | arm* | s390x | loong64 | loongarch64)
|
||||
x86_64 | amd64 | aarch64 | ppc64le | arm* | s390x | loong64 | loongarch64 | riscv64)
|
||||
return
|
||||
;;
|
||||
*)
|
||||
echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, ppc64le, arm*, s390x, loong64, loongarch64]"
|
||||
echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, ppc64le, arm*, s390x, loong64, loongarch64, riscv64]"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -9,7 +9,7 @@ function _init() {
|
||||
export CGO_ENABLED=0
|
||||
|
||||
## List of architectures and OS to test coss compilation.
|
||||
SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64"
|
||||
SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64 linux/riscv64"
|
||||
}
|
||||
|
||||
function _build() {
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
#
|
||||
# This script assumes that LDAP server is at:
|
||||
#
|
||||
# `localhost:1389`
|
||||
# `localhost:389`
|
||||
#
|
||||
# if this is not the case, set the environment variable
|
||||
# `_MINIO_LDAP_TEST_SERVER`.
|
||||
@@ -41,7 +41,7 @@ __init__() {
|
||||
fi
|
||||
|
||||
if [ -z "$_MINIO_LDAP_TEST_SERVER" ]; then
|
||||
export _MINIO_LDAP_TEST_SERVER=localhost:1389
|
||||
export _MINIO_LDAP_TEST_SERVER=localhost:389
|
||||
echo "Using default LDAP endpoint: $_MINIO_LDAP_TEST_SERVER"
|
||||
fi
|
||||
|
||||
@@ -58,7 +58,7 @@ create_iam_content_in_old_minio() {
|
||||
mc alias set old-minio http://localhost:9000 minioadmin minioadmin
|
||||
mc ready old-minio
|
||||
mc idp ldap add old-minio \
|
||||
server_addr=localhost:1389 \
|
||||
server_addr=localhost:389 \
|
||||
server_insecure=on \
|
||||
lookup_bind_dn=cn=admin,dc=min,dc=io \
|
||||
lookup_bind_password=admin \
|
||||
|
||||
@@ -69,8 +69,10 @@ __init__() {
|
||||
|
||||
## this is needed because github actions don't have
|
||||
## docker-compose on all runners
|
||||
go install github.com/docker/compose/v2/cmd@latest
|
||||
mv -v /tmp/gopath/bin/cmd /tmp/gopath/bin/docker-compose
|
||||
COMPOSE_VERSION=v2.35.1
|
||||
mkdir -p /tmp/gopath/bin/
|
||||
wget -O /tmp/gopath/bin/docker-compose https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-linux-x86_64
|
||||
chmod +x /tmp/gopath/bin/docker-compose
|
||||
|
||||
cleanup
|
||||
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ -n "$TEST_DEBUG" ]; then
|
||||
set -x
|
||||
fi
|
||||
|
||||
WORK_DIR="$PWD/.verify-$RANDOM"
|
||||
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
|
||||
MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
|
||||
|
||||
if [ ! -x "$PWD/minio" ]; then
|
||||
echo "minio executable binary not found in current directory"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -x "$PWD/minio" ]; then
|
||||
echo "minio executable binary not found in current directory"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
trap 'catch $LINENO' ERR
|
||||
|
||||
function purge() {
|
||||
rm -rf "$1"
|
||||
}
|
||||
|
||||
# shellcheck disable=SC2120
|
||||
catch() {
|
||||
if [ $# -ne 0 ]; then
|
||||
echo "error on line $1"
|
||||
fi
|
||||
|
||||
echo "Cleaning up instances of MinIO"
|
||||
pkill minio || true
|
||||
pkill -9 minio || true
|
||||
purge "$WORK_DIR"
|
||||
if [ $# -ne 0 ]; then
|
||||
exit $#
|
||||
fi
|
||||
}
|
||||
|
||||
catch
|
||||
|
||||
function gen_put_request() {
|
||||
hdr_sleep=$1
|
||||
body_sleep=$2
|
||||
|
||||
echo "PUT /testbucket/testobject HTTP/1.1"
|
||||
sleep $hdr_sleep
|
||||
echo "Host: foo-header"
|
||||
echo "User-Agent: curl/8.2.1"
|
||||
echo "Accept: */*"
|
||||
echo "Content-Length: 30"
|
||||
echo ""
|
||||
|
||||
sleep $body_sleep
|
||||
echo "random line 0"
|
||||
echo "random line 1"
|
||||
echo ""
|
||||
echo ""
|
||||
}
|
||||
|
||||
function send_put_object_request() {
|
||||
hdr_timeout=$1
|
||||
body_timeout=$2
|
||||
|
||||
start=$(date +%s)
|
||||
timeout 5m bash -c "gen_put_request $hdr_timeout $body_timeout | netcat 127.0.0.1 $start_port | read" || return -1
|
||||
[ $(($(date +%s) - start)) -gt $((srv_hdr_timeout + srv_idle_timeout + 1)) ] && return -1
|
||||
return 0
|
||||
}
|
||||
|
||||
function test_minio_with_timeout() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ROOT_USER=minio
|
||||
export MINIO_ROOT_PASSWORD=minio123
|
||||
export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
mkdir ${WORK_DIR}
|
||||
C_PWD=${PWD}
|
||||
if [ ! -x "$PWD/mc" ]; then
|
||||
MC_BUILD_DIR="mc-$RANDOM"
|
||||
if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
|
||||
echo "failed to download https://github.com/minio/mc"
|
||||
purge "${MC_BUILD_DIR}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
(cd "${MC_BUILD_DIR}" && go build -o "$C_PWD/mc")
|
||||
|
||||
# remove mc source.
|
||||
purge "${MC_BUILD_DIR}"
|
||||
fi
|
||||
|
||||
"${MINIO[@]}" --address ":$start_port" --read-header-timeout ${srv_hdr_timeout}s --idle-timeout ${srv_idle_timeout}s "${WORK_DIR}/disk/" >"${WORK_DIR}/server1.log" 2>&1 &
|
||||
pid=$!
|
||||
disown $pid
|
||||
sleep 1
|
||||
|
||||
if ! ps -p ${pid} 1>&2 >/dev/null; then
|
||||
echo "server1 log:"
|
||||
cat "${WORK_DIR}/server1.log"
|
||||
echo "FAILED"
|
||||
purge "$WORK_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -e
|
||||
|
||||
"${PWD}/mc" mb minio/testbucket
|
||||
"${PWD}/mc" anonymous set public minio/testbucket
|
||||
|
||||
# slow header writing
|
||||
send_put_object_request 20 0 && exit -1
|
||||
"${PWD}/mc" stat minio/testbucket/testobject && exit -1
|
||||
|
||||
# quick header write and slow bodywrite
|
||||
send_put_object_request 0 40 && exit -1
|
||||
"${PWD}/mc" stat minio/testbucket/testobject && exit -1
|
||||
|
||||
# quick header and body write
|
||||
send_put_object_request 1 1 || exit -1
|
||||
"${PWD}/mc" stat minio/testbucket/testobject || exit -1
|
||||
}
|
||||
|
||||
function main() {
|
||||
export start_port=$(shuf -i 10000-65000 -n 1)
|
||||
export srv_hdr_timeout=5
|
||||
export srv_idle_timeout=5
|
||||
export -f gen_put_request
|
||||
|
||||
test_minio_with_timeout ${start_port}
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -38,6 +38,7 @@ import (
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
"github.com/minio/minio/internal/event"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
@@ -980,7 +981,6 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
rpt.SetStatus(bucket, "", err)
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
rptData, err := json.Marshal(rpt.BucketMetaImportErrs)
|
||||
@@ -1039,7 +1039,7 @@ func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.
|
||||
}
|
||||
if len(diffCh) == 0 {
|
||||
// Flush if nothing is queued
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
}
|
||||
case <-keepAliveTicker.C:
|
||||
if len(diffCh) > 0 {
|
||||
@@ -1048,7 +1048,7 @@ func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.
|
||||
if _, err := w.Write([]byte(" ")); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
@@ -1098,7 +1098,7 @@ func (a adminAPIHandlers) ReplicationMRFHandler(w http.ResponseWriter, r *http.R
|
||||
}
|
||||
if len(mrfCh) == 0 {
|
||||
// Flush if nothing is queued
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
}
|
||||
case <-keepAliveTicker.C:
|
||||
if len(mrfCh) > 0 {
|
||||
@@ -1107,7 +1107,7 @@ func (a adminAPIHandlers) ReplicationMRFHandler(w http.ResponseWriter, r *http.R
|
||||
if _, err := w.Write([]byte(" ")); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
|
||||
@@ -125,7 +125,6 @@ func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.R
|
||||
}
|
||||
|
||||
if err = validateConfig(ctx, cfg, subSys); err != nil {
|
||||
|
||||
var validationErr ldap.Validation
|
||||
if errors.As(err, &validationErr) {
|
||||
// If we got an LDAP validation error, we need to send appropriate
|
||||
@@ -416,7 +415,6 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
|
||||
return
|
||||
}
|
||||
if err = validateConfig(ctx, cfg, subSys); err != nil {
|
||||
|
||||
var validationErr ldap.Validation
|
||||
if errors.As(err, &validationErr) {
|
||||
// If we got an LDAP validation error, we need to send appropriate
|
||||
|
||||
@@ -190,7 +190,7 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
|
||||
//
|
||||
// PUT /minio/admin/v3/idp/ldap/add-service-account
|
||||
func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r)
|
||||
ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r, true)
|
||||
if APIError.Code != "" {
|
||||
writeErrorResponseJSON(ctx, w, APIError, r.URL)
|
||||
return
|
||||
@@ -214,10 +214,7 @@ func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.R
|
||||
}
|
||||
|
||||
// Check if we are creating svc account for request sender.
|
||||
isSvcAccForRequestor := false
|
||||
if targetUser == requestorUser || targetUser == requestorParentUser {
|
||||
isSvcAccForRequestor = true
|
||||
}
|
||||
isSvcAccForRequestor := targetUser == requestorUser || targetUser == requestorParentUser
|
||||
|
||||
var (
|
||||
targetGroups []string
|
||||
@@ -345,7 +342,7 @@ func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.R
|
||||
Name: newCred.Name,
|
||||
Description: newCred.Description,
|
||||
Claims: opts.claims,
|
||||
SessionPolicy: createReq.Policy,
|
||||
SessionPolicy: madmin.SRSessionPolicy(createReq.Policy),
|
||||
Status: auth.AccountOn,
|
||||
Expiration: createReq.Expiration,
|
||||
},
|
||||
@@ -448,8 +445,10 @@ func (a adminAPIHandlers) ListAccessKeysLDAP(w http.ResponseWriter, r *http.Requ
|
||||
for _, svc := range serviceAccounts {
|
||||
expiryTime := svc.Expiration
|
||||
serviceAccountList = append(serviceAccountList, madmin.ServiceAccountInfo{
|
||||
AccessKey: svc.AccessKey,
|
||||
Expiration: &expiryTime,
|
||||
AccessKey: svc.AccessKey,
|
||||
Expiration: &expiryTime,
|
||||
Name: svc.Name,
|
||||
Description: svc.Description,
|
||||
})
|
||||
}
|
||||
for _, sts := range stsKeys {
|
||||
@@ -499,7 +498,7 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
|
||||
|
||||
dnList := r.Form["userDNs"]
|
||||
isAll := r.Form.Get("all") == "true"
|
||||
onlySelf := !isAll && len(dnList) == 0
|
||||
selfOnly := !isAll && len(dnList) == 0
|
||||
|
||||
if isAll && len(dnList) > 0 {
|
||||
// This should be checked on client side, so return generic error
|
||||
@@ -527,7 +526,7 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
|
||||
dn = foundResult.NormDN
|
||||
}
|
||||
if dn == cred.ParentUser || dnList[0] == cred.ParentUser {
|
||||
onlySelf = true
|
||||
selfOnly = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -538,13 +537,13 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: onlySelf,
|
||||
DenyOnly: selfOnly,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if onlySelf && len(dnList) == 0 {
|
||||
if selfOnly && len(dnList) == 0 {
|
||||
selfDN := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
selfDN = cred.ParentUser
|
||||
@@ -609,10 +608,9 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
for _, sts := range stsKeys {
|
||||
expiryTime := sts.Expiration
|
||||
accessKeys.STSKeys = append(accessKeys.STSKeys, madmin.ServiceAccountInfo{
|
||||
AccessKey: sts.AccessKey,
|
||||
Expiration: &expiryTime,
|
||||
Expiration: &sts.Expiration,
|
||||
})
|
||||
}
|
||||
// if only STS keys, skip if user has no STS keys
|
||||
@@ -628,10 +626,11 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
for _, svc := range serviceAccounts {
|
||||
expiryTime := svc.Expiration
|
||||
accessKeys.ServiceAccounts = append(accessKeys.ServiceAccounts, madmin.ServiceAccountInfo{
|
||||
AccessKey: svc.AccessKey,
|
||||
Expiration: &expiryTime,
|
||||
AccessKey: svc.AccessKey,
|
||||
Expiration: &svc.Expiration,
|
||||
Name: svc.Name,
|
||||
Description: svc.Description,
|
||||
})
|
||||
}
|
||||
// if only service accounts, skip if user has no service accounts
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
// Copyright (c) 2015-2025 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"sort"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
)
|
||||
|
||||
const dummyRoleARN = "dummy-internal"
|
||||
|
||||
// ListAccessKeysOpenIDBulk - GET /minio/admin/v3/idp/openid/list-access-keys-bulk
|
||||
func (a adminAPIHandlers) ListAccessKeysOpenIDBulk(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil || globalNotificationSys == nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
cred, owner, s3Err := validateAdminSignature(ctx, r, "")
|
||||
if s3Err != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if !globalIAMSys.OpenIDConfig.Enabled {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminOpenIDNotEnabled), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
userList := r.Form["users"]
|
||||
isAll := r.Form.Get("all") == "true"
|
||||
selfOnly := !isAll && len(userList) == 0
|
||||
cfgName := r.Form.Get("configName")
|
||||
allConfigs := r.Form.Get("allConfigs") == "true"
|
||||
if cfgName == "" && !allConfigs {
|
||||
cfgName = madmin.Default
|
||||
}
|
||||
|
||||
if isAll && len(userList) > 0 {
|
||||
// This should be checked on client side, so return generic error
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Empty DN list and not self, list access keys for all users
|
||||
if isAll {
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListUsersAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
} else if len(userList) == 1 && userList[0] == cred.ParentUser {
|
||||
selfOnly = true
|
||||
}
|
||||
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: selfOnly,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if selfOnly && len(userList) == 0 {
|
||||
selfDN := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
selfDN = cred.ParentUser
|
||||
}
|
||||
userList = append(userList, selfDN)
|
||||
}
|
||||
|
||||
listType := r.Form.Get("listType")
|
||||
var listSTSKeys, listServiceAccounts bool
|
||||
switch listType {
|
||||
case madmin.AccessKeyListUsersOnly:
|
||||
listSTSKeys = false
|
||||
listServiceAccounts = false
|
||||
case madmin.AccessKeyListSTSOnly:
|
||||
listSTSKeys = true
|
||||
listServiceAccounts = false
|
||||
case madmin.AccessKeyListSvcaccOnly:
|
||||
listSTSKeys = false
|
||||
listServiceAccounts = true
|
||||
case madmin.AccessKeyListAll:
|
||||
listSTSKeys = true
|
||||
listServiceAccounts = true
|
||||
default:
|
||||
err := errors.New("invalid list type")
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
s := globalServerConfig.Clone()
|
||||
roleArnMap := make(map[string]string)
|
||||
// Map of configs to a map of users to their access keys
|
||||
cfgToUsersMap := make(map[string]map[string]madmin.OpenIDUserAccessKeys)
|
||||
configs, err := globalIAMSys.OpenIDConfig.GetConfigList(s)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
for _, config := range configs {
|
||||
if !allConfigs && cfgName != config.Name {
|
||||
continue
|
||||
}
|
||||
arn := dummyRoleARN
|
||||
if config.RoleARN != "" {
|
||||
arn = config.RoleARN
|
||||
}
|
||||
roleArnMap[arn] = config.Name
|
||||
newResp := make(map[string]madmin.OpenIDUserAccessKeys)
|
||||
cfgToUsersMap[config.Name] = newResp
|
||||
}
|
||||
if len(roleArnMap) == 0 {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
userSet := set.CreateStringSet(userList...)
|
||||
accessKeys, err := globalIAMSys.ListAllAccessKeys(ctx)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
for _, accessKey := range accessKeys {
|
||||
// Filter out any disqualifying access keys
|
||||
_, ok := accessKey.Claims[subClaim]
|
||||
if !ok {
|
||||
continue // OpenID access keys must have a sub claim
|
||||
}
|
||||
if (!listSTSKeys && !accessKey.IsServiceAccount()) || (!listServiceAccounts && accessKey.IsServiceAccount()) {
|
||||
continue // skip if not the type we want
|
||||
}
|
||||
arn, ok := accessKey.Claims[roleArnClaim].(string)
|
||||
if !ok {
|
||||
if _, ok := accessKey.Claims[iamPolicyClaimNameOpenID()]; !ok {
|
||||
continue // skip if no roleArn and no policy claim
|
||||
}
|
||||
// claim-based provider is in the roleArnMap under dummy ARN
|
||||
arn = dummyRoleARN
|
||||
}
|
||||
matchingCfgName, ok := roleArnMap[arn]
|
||||
if !ok {
|
||||
continue // skip if not part of the target config
|
||||
}
|
||||
var id string
|
||||
if idClaim := globalIAMSys.OpenIDConfig.GetUserIDClaim(matchingCfgName); idClaim != "" {
|
||||
id, _ = accessKey.Claims[idClaim].(string)
|
||||
}
|
||||
if !userSet.IsEmpty() && !userSet.Contains(accessKey.ParentUser) && !userSet.Contains(id) {
|
||||
continue // skip if not in the user list
|
||||
}
|
||||
openIDUserAccessKeys, ok := cfgToUsersMap[matchingCfgName][accessKey.ParentUser]
|
||||
|
||||
// Add new user to map if not already present
|
||||
if !ok {
|
||||
var readableClaim string
|
||||
if rc := globalIAMSys.OpenIDConfig.GetUserReadableClaim(matchingCfgName); rc != "" {
|
||||
readableClaim, _ = accessKey.Claims[rc].(string)
|
||||
}
|
||||
openIDUserAccessKeys = madmin.OpenIDUserAccessKeys{
|
||||
MinioAccessKey: accessKey.ParentUser,
|
||||
ID: id,
|
||||
ReadableName: readableClaim,
|
||||
}
|
||||
}
|
||||
svcAccInfo := madmin.ServiceAccountInfo{
|
||||
AccessKey: accessKey.AccessKey,
|
||||
Expiration: &accessKey.Expiration,
|
||||
}
|
||||
if accessKey.IsServiceAccount() {
|
||||
openIDUserAccessKeys.ServiceAccounts = append(openIDUserAccessKeys.ServiceAccounts, svcAccInfo)
|
||||
} else {
|
||||
openIDUserAccessKeys.STSKeys = append(openIDUserAccessKeys.STSKeys, svcAccInfo)
|
||||
}
|
||||
cfgToUsersMap[matchingCfgName][accessKey.ParentUser] = openIDUserAccessKeys
|
||||
}
|
||||
|
||||
// Convert map to slice and sort
|
||||
resp := make([]madmin.ListAccessKeysOpenIDResp, 0, len(cfgToUsersMap))
|
||||
for cfgName, usersMap := range cfgToUsersMap {
|
||||
users := make([]madmin.OpenIDUserAccessKeys, 0, len(usersMap))
|
||||
for _, user := range usersMap {
|
||||
users = append(users, user)
|
||||
}
|
||||
sort.Slice(users, func(i, j int) bool {
|
||||
return users[i].MinioAccessKey < users[j].MinioAccessKey
|
||||
})
|
||||
resp = append(resp, madmin.ListAccessKeysOpenIDResp{
|
||||
ConfigName: cfgName,
|
||||
Users: users,
|
||||
})
|
||||
}
|
||||
sort.Slice(resp, func(i, j int) bool {
|
||||
return resp[i].ConfigName < resp[j].ConfigName
|
||||
})
|
||||
|
||||
data, err := json.Marshal(resp)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseJSON(w, encryptedData)
|
||||
}
|
||||
@@ -61,7 +61,7 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
|
||||
return
|
||||
}
|
||||
|
||||
if z.IsRebalanceStarted() {
|
||||
if z.IsRebalanceStarted(ctx) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminRebalanceAlreadyStarted), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -258,8 +258,8 @@ func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request)
|
||||
// concurrent rebalance-start commands.
|
||||
if ep := globalEndpoints[0].Endpoints[0]; !ep.IsLocal {
|
||||
for nodeIdx, proxyEp := range globalProxyEndpoints {
|
||||
if proxyEp.Endpoint.Host == ep.Host {
|
||||
if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
|
||||
if proxyEp.Host == ep.Host {
|
||||
if proxied, success := proxyRequestByNodeIndex(ctx, w, r, nodeIdx, false); proxied && success {
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -277,7 +277,7 @@ func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
|
||||
if pools.IsRebalanceStarted() {
|
||||
if pools.IsRebalanceStarted(ctx) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminRebalanceAlreadyStarted), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -329,8 +329,8 @@ func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request
|
||||
// pools may temporarily have out of date info on the others.
|
||||
if ep := globalEndpoints[0].Endpoints[0]; !ep.IsLocal {
|
||||
for nodeIdx, proxyEp := range globalProxyEndpoints {
|
||||
if proxyEp.Endpoint.Host == ep.Host {
|
||||
if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
|
||||
if proxyEp.Host == ep.Host {
|
||||
if proxied, success := proxyRequestByNodeIndex(ctx, w, r, nodeIdx, false); proxied && success {
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -383,8 +383,8 @@ func proxyDecommissionRequest(ctx context.Context, defaultEndPoint Endpoint, w h
|
||||
return
|
||||
}
|
||||
for nodeIdx, proxyEp := range globalProxyEndpoints {
|
||||
if proxyEp.Endpoint.Host == host && !proxyEp.IsLocal {
|
||||
if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
|
||||
if proxyEp.Host == host && !proxyEp.IsLocal {
|
||||
if proxied, success := proxyRequestByNodeIndex(ctx, w, r, nodeIdx, false); proxied && success {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -304,7 +304,7 @@ func (a adminAPIHandlers) SRPeerGetIDPSettings(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
}
|
||||
|
||||
func parseJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptionKey string) error {
|
||||
func parseJSONBody(ctx context.Context, body io.Reader, v any, encryptionKey string) error {
|
||||
data, err := io.ReadAll(body)
|
||||
if err != nil {
|
||||
return SRError{
|
||||
|
||||
@@ -89,7 +89,7 @@ func (s *TestSuiteIAM) TestDeleteUserRace(c *check) {
|
||||
|
||||
// Create a policy policy
|
||||
policy := "mypolicy"
|
||||
policyBytes := []byte(fmt.Sprintf(`{
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
@@ -104,7 +104,7 @@ func (s *TestSuiteIAM) TestDeleteUserRace(c *check) {
|
||||
]
|
||||
}
|
||||
]
|
||||
}`, bucket))
|
||||
}`, bucket)
|
||||
err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
|
||||
if err != nil {
|
||||
c.Fatalf("policy add error: %v", err)
|
||||
@@ -113,7 +113,7 @@ func (s *TestSuiteIAM) TestDeleteUserRace(c *check) {
|
||||
userCount := 50
|
||||
accessKeys := make([]string, userCount)
|
||||
secretKeys := make([]string, userCount)
|
||||
for i := 0; i < userCount; i++ {
|
||||
for i := range userCount {
|
||||
accessKey, secretKey := mustGenerateCredentials(c)
|
||||
err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
|
||||
if err != nil {
|
||||
@@ -133,7 +133,7 @@ func (s *TestSuiteIAM) TestDeleteUserRace(c *check) {
|
||||
}
|
||||
|
||||
g := errgroup.Group{}
|
||||
for i := 0; i < userCount; i++ {
|
||||
for i := range userCount {
|
||||
g.Go(func(i int) func() error {
|
||||
return func() error {
|
||||
uClient := s.getUserClient(c, accessKeys[i], secretKeys[i], "")
|
||||
|
||||
+454
-78
@@ -24,6 +24,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"os"
|
||||
"slices"
|
||||
@@ -37,6 +38,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/config/dns"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
xldap "github.com/minio/pkg/v3/ldap"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
@@ -65,6 +67,17 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// This API only supports removal of internal users not service accounts.
|
||||
ok, _, err = globalIAMSys.IsServiceAccount(accessKey)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if ok {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// When the user is root credential you are not allowed to
|
||||
// remove the root user. Also you cannot delete yourself.
|
||||
if accessKey == globalActiveCred.AccessKey || accessKey == cred.AccessKey {
|
||||
@@ -145,9 +158,7 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
for k, v := range ldapUsers {
|
||||
allCredentials[k] = v
|
||||
}
|
||||
maps.Copy(allCredentials, ldapUsers)
|
||||
|
||||
// Marshal the response
|
||||
data, err := json.Marshal(allCredentials)
|
||||
@@ -185,12 +196,7 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
checkDenyOnly := false
|
||||
if name == cred.AccessKey {
|
||||
// Check that there is no explicit deny - otherwise it's allowed
|
||||
// to view one's own info.
|
||||
checkDenyOnly = true
|
||||
}
|
||||
checkDenyOnly := name == cred.AccessKey
|
||||
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
@@ -481,12 +487,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
checkDenyOnly := false
|
||||
if accessKey == cred.AccessKey {
|
||||
// Check that there is no explicit deny - otherwise it's allowed
|
||||
// to change one's own password.
|
||||
checkDenyOnly = true
|
||||
}
|
||||
checkDenyOnly := accessKey == cred.AccessKey
|
||||
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
@@ -637,7 +638,7 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
|
||||
|
||||
// AddServiceAccount - PUT /minio/admin/v3/add-service-account
|
||||
func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r)
|
||||
ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r, false)
|
||||
if APIError.Code != "" {
|
||||
writeErrorResponseJSON(ctx, w, APIError, r.URL)
|
||||
return
|
||||
@@ -677,10 +678,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
|
||||
// Check if we are creating svc account for request sender.
|
||||
isSvcAccForRequestor := false
|
||||
if targetUser == requestorUser || targetUser == requestorParentUser {
|
||||
isSvcAccForRequestor = true
|
||||
}
|
||||
isSvcAccForRequestor := targetUser == requestorUser || targetUser == requestorParentUser
|
||||
|
||||
// If we are creating svc account for request sender, ensure
|
||||
// that targetUser is a real user (i.e. not derived
|
||||
@@ -771,7 +769,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
|
||||
Name: newCred.Name,
|
||||
Description: newCred.Description,
|
||||
Claims: opts.claims,
|
||||
SessionPolicy: createReq.Policy,
|
||||
SessionPolicy: madmin.SRSessionPolicy(createReq.Policy),
|
||||
Status: auth.AccountOn,
|
||||
Expiration: createReq.Expiration,
|
||||
},
|
||||
@@ -895,7 +893,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
Status: opts.status,
|
||||
Name: opts.name,
|
||||
Description: opts.description,
|
||||
SessionPolicy: updateReq.NewPolicy,
|
||||
SessionPolicy: madmin.SRSessionPolicy(updateReq.NewPolicy),
|
||||
Expiration: updateReq.NewExpiration,
|
||||
},
|
||||
},
|
||||
@@ -1167,6 +1165,172 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
writeSuccessNoContent(w)
|
||||
}
|
||||
|
||||
// ListAccessKeysBulk - GET /minio/admin/v3/list-access-keys-bulk
|
||||
func (a adminAPIHandlers) ListAccessKeysBulk(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil || globalNotificationSys == nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
cred, owner, s3Err := validateAdminSignature(ctx, r, "")
|
||||
if s3Err != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
users := r.Form["users"]
|
||||
isAll := r.Form.Get("all") == "true"
|
||||
selfOnly := !isAll && len(users) == 0
|
||||
|
||||
if isAll && len(users) > 0 {
|
||||
// This should be checked on client side, so return generic error
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Empty user list and not self, list access keys for all users
|
||||
if isAll {
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListUsersAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
} else if len(users) == 1 {
|
||||
if users[0] == cred.AccessKey || users[0] == cred.ParentUser {
|
||||
selfOnly = true
|
||||
}
|
||||
}
|
||||
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: selfOnly,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if selfOnly && len(users) == 0 {
|
||||
selfUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
selfUser = cred.ParentUser
|
||||
}
|
||||
users = append(users, selfUser)
|
||||
}
|
||||
|
||||
var checkedUserList []string
|
||||
if isAll {
|
||||
users, err := globalIAMSys.ListUsers(ctx)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
for user := range users {
|
||||
checkedUserList = append(checkedUserList, user)
|
||||
}
|
||||
checkedUserList = append(checkedUserList, globalActiveCred.AccessKey)
|
||||
} else {
|
||||
for _, user := range users {
|
||||
// Validate the user
|
||||
_, ok := globalIAMSys.GetUser(ctx, user)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
checkedUserList = append(checkedUserList, user)
|
||||
}
|
||||
}
|
||||
|
||||
listType := r.Form.Get("listType")
|
||||
var listSTSKeys, listServiceAccounts bool
|
||||
switch listType {
|
||||
case madmin.AccessKeyListUsersOnly:
|
||||
listSTSKeys = false
|
||||
listServiceAccounts = false
|
||||
case madmin.AccessKeyListSTSOnly:
|
||||
listSTSKeys = true
|
||||
listServiceAccounts = false
|
||||
case madmin.AccessKeyListSvcaccOnly:
|
||||
listSTSKeys = false
|
||||
listServiceAccounts = true
|
||||
case madmin.AccessKeyListAll:
|
||||
listSTSKeys = true
|
||||
listServiceAccounts = true
|
||||
default:
|
||||
err := errors.New("invalid list type")
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
accessKeyMap := make(map[string]madmin.ListAccessKeysResp)
|
||||
for _, user := range checkedUserList {
|
||||
accessKeys := madmin.ListAccessKeysResp{}
|
||||
if listSTSKeys {
|
||||
stsKeys, err := globalIAMSys.ListSTSAccounts(ctx, user)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
for _, sts := range stsKeys {
|
||||
accessKeys.STSKeys = append(accessKeys.STSKeys, madmin.ServiceAccountInfo{
|
||||
AccessKey: sts.AccessKey,
|
||||
Expiration: &sts.Expiration,
|
||||
})
|
||||
}
|
||||
// if only STS keys, skip if user has no STS keys
|
||||
if !listServiceAccounts && len(stsKeys) == 0 {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if listServiceAccounts {
|
||||
serviceAccounts, err := globalIAMSys.ListServiceAccounts(ctx, user)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
for _, svc := range serviceAccounts {
|
||||
accessKeys.ServiceAccounts = append(accessKeys.ServiceAccounts, madmin.ServiceAccountInfo{
|
||||
AccessKey: svc.AccessKey,
|
||||
Expiration: &svc.Expiration,
|
||||
})
|
||||
}
|
||||
// if only service accounts, skip if user has no service accounts
|
||||
if !listSTSKeys && len(serviceAccounts) == 0 {
|
||||
continue
|
||||
}
|
||||
}
|
||||
accessKeyMap[user] = accessKeys
|
||||
}
|
||||
|
||||
data, err := json.Marshal(accessKeyMap)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseJSON(w, encryptedData)
|
||||
}
|
||||
|
||||
// AccountInfoHandler returns usage, permissions and other bucket metadata for incoming us
|
||||
func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
@@ -1276,7 +1440,12 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
|
||||
var buf []byte
|
||||
switch {
|
||||
case accountName == globalActiveCred.AccessKey:
|
||||
case accountName == globalActiveCred.AccessKey || newGlobalAuthZPluginFn() != nil:
|
||||
// For owner account and when plugin authZ is configured always set
|
||||
// effective policy as `consoleAdmin`.
|
||||
//
|
||||
// In the latter case, we let the UI render everything, but individual
|
||||
// actions would fail if not permitted by the external authZ service.
|
||||
for _, policy := range policy.DefaultPolicies {
|
||||
if policy.Name == "consoleAdmin" {
|
||||
effectivePolicy = policy.Definition
|
||||
@@ -1304,8 +1473,8 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
effectivePolicy = globalIAMSys.GetCombinedPolicy(policies...)
|
||||
|
||||
}
|
||||
|
||||
buf, err = json.MarshalIndent(effectivePolicy, "", " ")
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
@@ -1397,6 +1566,7 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errTooManyPolicies), r.URL)
|
||||
return
|
||||
}
|
||||
setReqInfoPolicyName(ctx, name)
|
||||
|
||||
policyDoc, err := globalIAMSys.InfoPolicy(name)
|
||||
if err != nil {
|
||||
@@ -1500,6 +1670,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
|
||||
|
||||
vars := mux.Vars(r)
|
||||
policyName := vars["name"]
|
||||
setReqInfoPolicyName(ctx, policyName)
|
||||
|
||||
if err := globalIAMSys.DeletePolicy(ctx, policyName, true); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
@@ -1532,6 +1703,13 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
|
||||
return
|
||||
}
|
||||
setReqInfoPolicyName(ctx, policyName)
|
||||
|
||||
// Reject policy names with commas.
|
||||
if strings.Contains(policyName, ",") {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrPolicyInvalidName), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Error out if Content-Length is missing.
|
||||
if r.ContentLength <= 0 {
|
||||
@@ -1597,6 +1775,7 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
|
||||
policyName := vars["policyName"]
|
||||
entityName := vars["userOrGroup"]
|
||||
isGroup := vars["isGroup"] == "true"
|
||||
setReqInfoPolicyName(ctx, policyName)
|
||||
|
||||
if !isGroup {
|
||||
ok, _, err := globalIAMSys.IsTempUser(entityName)
|
||||
@@ -1647,16 +1826,18 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
|
||||
iamLogIf(ctx, err)
|
||||
} else if foundGroupDN == nil || !underBaseDN {
|
||||
err = errNoSuchGroup
|
||||
} else {
|
||||
entityName = foundGroupDN.NormDN
|
||||
}
|
||||
entityName = foundGroupDN.NormDN
|
||||
} else {
|
||||
var foundUserDN *xldap.DNSearchResult
|
||||
if foundUserDN, err = globalIAMSys.LDAPConfig.GetValidatedDNForUsername(entityName); err != nil {
|
||||
iamLogIf(ctx, err)
|
||||
} else if foundUserDN == nil {
|
||||
err = errNoSuchUser
|
||||
} else {
|
||||
entityName = foundUserDN.NormDN
|
||||
}
|
||||
entityName = foundUserDN.NormDN
|
||||
}
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
@@ -1682,7 +1863,7 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
|
||||
}))
|
||||
}
|
||||
|
||||
// ListPolicyMappingEntities - GET /minio/admin/v3/idp/builtin/polciy-entities?policy=xxx&user=xxx&group=xxx
|
||||
// ListPolicyMappingEntities - GET /minio/admin/v3/idp/builtin/policy-entities?policy=xxx&user=xxx&group=xxx
|
||||
func (a adminAPIHandlers) ListPolicyMappingEntities(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
@@ -1784,6 +1965,7 @@ func (a adminAPIHandlers) AttachDetachPolicyBuiltin(w http.ResponseWriter, r *ht
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
setReqInfoPolicyName(ctx, strings.Join(addedOrRemoved, ","))
|
||||
|
||||
respBody := madmin.PolicyAssociationResp{
|
||||
UpdatedAt: updatedAt,
|
||||
@@ -1809,6 +1991,227 @@ func (a adminAPIHandlers) AttachDetachPolicyBuiltin(w http.ResponseWriter, r *ht
|
||||
writeSuccessResponseJSON(w, encryptedData)
|
||||
}
|
||||
|
||||
// RevokeTokens - POST /minio/admin/v3/revoke-tokens/{userProvider}
|
||||
func (a adminAPIHandlers) RevokeTokens(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil || globalNotificationSys == nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
cred, owner, s3Err := validateAdminSignature(ctx, r, "")
|
||||
if s3Err != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
userProvider := mux.Vars(r)["userProvider"]
|
||||
|
||||
user := r.Form.Get("user")
|
||||
tokenRevokeType := r.Form.Get("tokenRevokeType")
|
||||
fullRevoke := r.Form.Get("fullRevoke") == "true"
|
||||
isTokenSelfRevoke := user == ""
|
||||
if !isTokenSelfRevoke {
|
||||
var err error
|
||||
user, err = getUserWithProvider(ctx, userProvider, user, false)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if (user != "" && tokenRevokeType == "" && !fullRevoke) || (tokenRevokeType != "" && fullRevoke) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
adminPrivilege := globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.RemoveServiceAccountAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
})
|
||||
|
||||
if !adminPrivilege || isTokenSelfRevoke {
|
||||
parentUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
parentUser = cred.ParentUser
|
||||
}
|
||||
if !isTokenSelfRevoke && user != parentUser {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
user = parentUser
|
||||
}
|
||||
|
||||
// Infer token revoke type from the request if requestor is STS.
|
||||
if isTokenSelfRevoke && tokenRevokeType == "" && !fullRevoke {
|
||||
if cred.IsTemp() {
|
||||
tokenRevokeType, _ = cred.Claims[tokenRevokeTypeClaim].(string)
|
||||
}
|
||||
if tokenRevokeType == "" {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNoTokenRevokeType), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
err := globalIAMSys.RevokeTokens(ctx, user, tokenRevokeType)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessNoContent(w)
|
||||
}
|
||||
|
||||
// InfoAccessKey - GET /minio/admin/v3/info-access-key?access-key=<access-key>
|
||||
func (a adminAPIHandlers) InfoAccessKey(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil || globalNotificationSys == nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
cred, owner, s3Err := validateAdminSignature(ctx, r, "")
|
||||
if s3Err != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
accessKey := mux.Vars(r)["accessKey"]
|
||||
if accessKey == "" {
|
||||
accessKey = cred.AccessKey
|
||||
}
|
||||
|
||||
u, ok := globalIAMSys.GetUser(ctx, accessKey)
|
||||
targetCred := u.Credentials
|
||||
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
// If requested user does not exist and requestor is not allowed to list service accounts, return access denied.
|
||||
if !ok {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
requestUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
requestUser = cred.ParentUser
|
||||
}
|
||||
|
||||
if requestUser != targetCred.ParentUser {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if !ok {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchAccessKey), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var (
|
||||
sessionPolicy *policy.Policy
|
||||
err error
|
||||
userType string
|
||||
)
|
||||
switch {
|
||||
case targetCred.IsTemp():
|
||||
userType = "STS"
|
||||
_, sessionPolicy, err = globalIAMSys.GetTemporaryAccount(ctx, accessKey)
|
||||
if err == errNoSuchTempAccount {
|
||||
err = errNoSuchAccessKey
|
||||
}
|
||||
case targetCred.IsServiceAccount():
|
||||
userType = "Service Account"
|
||||
_, sessionPolicy, err = globalIAMSys.GetServiceAccount(ctx, accessKey)
|
||||
if err == errNoSuchServiceAccount {
|
||||
err = errNoSuchAccessKey
|
||||
}
|
||||
default:
|
||||
err = errNoSuchAccessKey
|
||||
}
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// if session policy is nil or empty, then it is implied policy
|
||||
impliedPolicy := sessionPolicy == nil || (sessionPolicy.Version == "" && len(sessionPolicy.Statements) == 0)
|
||||
|
||||
var svcAccountPolicy policy.Policy
|
||||
|
||||
if !impliedPolicy {
|
||||
svcAccountPolicy = *sessionPolicy
|
||||
} else {
|
||||
policiesNames, err := globalIAMSys.PolicyDBGet(targetCred.ParentUser, targetCred.Groups...)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
svcAccountPolicy = globalIAMSys.GetCombinedPolicy(policiesNames...)
|
||||
}
|
||||
|
||||
policyJSON, err := json.MarshalIndent(svcAccountPolicy, "", " ")
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var expiration *time.Time
|
||||
if !targetCred.Expiration.IsZero() && !targetCred.Expiration.Equal(timeSentinel) {
|
||||
expiration = &targetCred.Expiration
|
||||
}
|
||||
|
||||
userProvider := guessUserProvider(targetCred)
|
||||
|
||||
infoResp := madmin.InfoAccessKeyResp{
|
||||
AccessKey: accessKey,
|
||||
InfoServiceAccountResp: madmin.InfoServiceAccountResp{
|
||||
ParentUser: targetCred.ParentUser,
|
||||
Name: targetCred.Name,
|
||||
Description: targetCred.Description,
|
||||
AccountStatus: targetCred.Status,
|
||||
ImpliedPolicy: impliedPolicy,
|
||||
Policy: string(policyJSON),
|
||||
Expiration: expiration,
|
||||
},
|
||||
|
||||
UserType: userType,
|
||||
UserProvider: userProvider,
|
||||
}
|
||||
|
||||
populateProviderInfoFromClaims(targetCred.Claims, userProvider, &infoResp)
|
||||
|
||||
data, err := json.Marshal(infoResp)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseJSON(w, encryptedData)
|
||||
}
|
||||
|
||||
const (
|
||||
allPoliciesFile = "policies.json"
|
||||
allUsersFile = "users.json"
|
||||
@@ -1838,6 +2241,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ExportIAMAction)
|
||||
if objectAPI == nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
// Initialize a zip writer which will provide a zipped content
|
||||
@@ -1974,7 +2378,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
SecretKey: acc.Credentials.SecretKey,
|
||||
Groups: acc.Credentials.Groups,
|
||||
Claims: claims,
|
||||
SessionPolicy: json.RawMessage(policyJSON),
|
||||
SessionPolicy: policyJSON,
|
||||
Status: acc.Credentials.Status,
|
||||
Name: sa.Name,
|
||||
Description: sa.Description,
|
||||
@@ -2060,17 +2464,13 @@ func (a adminAPIHandlers) ImportIAMV2(w http.ResponseWriter, r *http.Request) {
|
||||
func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiVer string) {
|
||||
ctx := r.Context()
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
// Validate signature, permissions and get current object layer instance.
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ImportIAMAction)
|
||||
if objectAPI == nil || globalNotificationSys == nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
cred, owner, s3Err := validateAdminSignature(ctx, r, "")
|
||||
if s3Err != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||
@@ -2088,7 +2488,6 @@ func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiV
|
||||
|
||||
// import policies first
|
||||
{
|
||||
|
||||
f, err := zr.Open(pathJoin(iamAssetsDir, allPoliciesFile))
|
||||
switch {
|
||||
case errors.Is(err, os.ErrNotExist):
|
||||
@@ -2160,44 +2559,17 @@ func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiV
|
||||
return
|
||||
}
|
||||
|
||||
if (cred.IsTemp() || cred.IsServiceAccount()) && cred.ParentUser == accessKey {
|
||||
// Incoming access key matches parent user then we should
|
||||
// reject password change requests.
|
||||
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAddUserInvalidArgument, err, allUsersFile, accessKey), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if accessKey has beginning and end space characters, this only applies to new users.
|
||||
if !exists && hasSpaceBE(accessKey) {
|
||||
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminResourceInvalidArgument, err, allUsersFile, accessKey), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
checkDenyOnly := false
|
||||
if accessKey == cred.AccessKey {
|
||||
// Check that there is no explicit deny - otherwise it's allowed
|
||||
// to change one's own password.
|
||||
checkDenyOnly = true
|
||||
}
|
||||
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.CreateUserAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: checkDenyOnly,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAccessDenied, err, allUsersFile, accessKey), r.URL)
|
||||
return
|
||||
}
|
||||
if _, err = globalIAMSys.CreateUser(ctx, accessKey, ureq); err != nil {
|
||||
failed.Users = append(failed.Users, madmin.IAMErrEntity{Name: accessKey, Error: err})
|
||||
} else {
|
||||
added.Users = append(added.Users, accessKey)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2291,17 +2663,6 @@ func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiV
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
|
||||
return
|
||||
}
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.CreateServiceAccountAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAccessDenied, err, allSvcAcctsFile, user), r.URL)
|
||||
return
|
||||
}
|
||||
updateReq := true
|
||||
_, _, err = globalIAMSys.GetServiceAccount(ctx, svcAcctReq.AccessKey)
|
||||
if err != nil {
|
||||
@@ -2521,7 +2882,7 @@ func addExpirationToCondValues(exp *time.Time, condValues map[string][]string) e
|
||||
if exp == nil || exp.IsZero() || exp.Equal(timeSentinel) {
|
||||
return nil
|
||||
}
|
||||
dur := exp.Sub(time.Now())
|
||||
dur := time.Until(*exp)
|
||||
if dur <= 0 {
|
||||
return errors.New("unsupported expiration time")
|
||||
}
|
||||
@@ -2529,7 +2890,7 @@ func addExpirationToCondValues(exp *time.Time, condValues map[string][]string) e
|
||||
return nil
|
||||
}
|
||||
|
||||
func commonAddServiceAccount(r *http.Request) (context.Context, auth.Credentials, newServiceAccountOpts, madmin.AddServiceAccountReq, string, APIError) {
|
||||
func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.Credentials, newServiceAccountOpts, madmin.AddServiceAccountReq, string, APIError) {
|
||||
ctx := r.Context()
|
||||
|
||||
// Get current object layer instance.
|
||||
@@ -2587,7 +2948,7 @@ func commonAddServiceAccount(r *http.Request) (context.Context, auth.Credentials
|
||||
name: createReq.Name,
|
||||
description: description,
|
||||
expiration: createReq.Expiration,
|
||||
claims: make(map[string]interface{}),
|
||||
claims: make(map[string]any),
|
||||
}
|
||||
|
||||
condValues := getConditionValues(r, "", cred)
|
||||
@@ -2596,6 +2957,14 @@ func commonAddServiceAccount(r *http.Request) (context.Context, auth.Credentials
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err)
|
||||
}
|
||||
|
||||
denyOnly := (targetUser == cred.AccessKey || targetUser == cred.ParentUser)
|
||||
if ldap && !denyOnly {
|
||||
res, _ := globalIAMSys.LDAPConfig.GetValidatedDNForUsername(targetUser)
|
||||
if res != nil && res.NormDN == cred.ParentUser {
|
||||
denyOnly = true
|
||||
}
|
||||
}
|
||||
|
||||
// Check if action is allowed if creating access key for another user
|
||||
// Check if action is explicitly denied if for self
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
@@ -2605,7 +2974,7 @@ func commonAddServiceAccount(r *http.Request) (context.Context, auth.Credentials
|
||||
ConditionValues: condValues,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: (targetUser == cred.AccessKey || targetUser == cred.ParentUser),
|
||||
DenyOnly: denyOnly,
|
||||
}) {
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAccessDenied)
|
||||
}
|
||||
@@ -2622,3 +2991,10 @@ func commonAddServiceAccount(r *http.Request) (context.Context, auth.Credentials
|
||||
|
||||
return ctx, cred, opts, createReq, targetUser, APIError{}
|
||||
}
|
||||
|
||||
// setReqInfoPolicyName will set the given policyName as a tag on the context's request info,
|
||||
// so that it appears in audit logs.
|
||||
func setReqInfoPolicyName(ctx context.Context, policyName string) {
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
reqInfo.SetTags("policyName", policyName)
|
||||
}
|
||||
|
||||
@@ -160,7 +160,7 @@ func (s *TestSuiteIAM) SetUpSuite(c *check) {
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) RestartIAMSuite(c *check) {
|
||||
s.TestSuiteCommon.RestartTestServer(c)
|
||||
s.RestartTestServer(c)
|
||||
|
||||
s.iamSetup(c)
|
||||
}
|
||||
@@ -332,22 +332,30 @@ func (s *TestSuiteIAM) TestUserPolicyEscalationBug(c *check) {
|
||||
|
||||
// 2.2 create and associate policy to user
|
||||
policy := "mypolicy-test-user-update"
|
||||
policyBytes := []byte(fmt.Sprintf(`{
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject",
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::%s"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::%s/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}`, bucket))
|
||||
}`, bucket, bucket)
|
||||
err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
|
||||
if err != nil {
|
||||
c.Fatalf("policy add error: %v", err)
|
||||
@@ -443,7 +451,7 @@ func (s *TestSuiteIAM) TestAddServiceAccountPerms(c *check) {
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::testbucket/*"
|
||||
"arn:aws:s3:::testbucket"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -554,22 +562,30 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) {
|
||||
|
||||
// 1. Create a policy
|
||||
policy := "mypolicy"
|
||||
policyBytes := []byte(fmt.Sprintf(`{
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject",
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::%s"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::%s/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}`, bucket))
|
||||
}`, bucket, bucket)
|
||||
err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
|
||||
if err != nil {
|
||||
c.Fatalf("policy add error: %v", err)
|
||||
@@ -664,22 +680,30 @@ func (s *TestSuiteIAM) TestCannedPolicies(c *check) {
|
||||
c.Fatalf("bucket creat error: %v", err)
|
||||
}
|
||||
|
||||
policyBytes := []byte(fmt.Sprintf(`{
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject",
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::%s"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::%s/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}`, bucket))
|
||||
}`, bucket, bucket)
|
||||
|
||||
// Check that default policies can be overwritten.
|
||||
err = s.adm.AddCannedPolicy(ctx, "readwrite", policyBytes)
|
||||
@@ -692,6 +716,12 @@ func (s *TestSuiteIAM) TestCannedPolicies(c *check) {
|
||||
c.Fatalf("policy info err: %v", err)
|
||||
}
|
||||
|
||||
// Check that policy with comma is rejected.
|
||||
err = s.adm.AddCannedPolicy(ctx, "invalid,policy", policyBytes)
|
||||
if err == nil {
|
||||
c.Fatalf("invalid policy created successfully")
|
||||
}
|
||||
|
||||
infoStr := string(info)
|
||||
if !strings.Contains(infoStr, `"s3:PutObject"`) || !strings.Contains(infoStr, ":"+bucket+"/") {
|
||||
c.Fatalf("policy contains unexpected content!")
|
||||
@@ -709,22 +739,30 @@ func (s *TestSuiteIAM) TestGroupAddRemove(c *check) {
|
||||
}
|
||||
|
||||
policy := "mypolicy"
|
||||
policyBytes := []byte(fmt.Sprintf(`{
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject",
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::%s"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::%s/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}`, bucket))
|
||||
}`, bucket, bucket)
|
||||
err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
|
||||
if err != nil {
|
||||
c.Fatalf("policy add error: %v", err)
|
||||
@@ -873,22 +911,30 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByUser(c *check) {
|
||||
|
||||
// Create policy, user and associate policy
|
||||
policy := "mypolicy"
|
||||
policyBytes := []byte(fmt.Sprintf(`{
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject",
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::%s"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::%s/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}`, bucket))
|
||||
}`, bucket, bucket)
|
||||
err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
|
||||
if err != nil {
|
||||
c.Fatalf("policy add error: %v", err)
|
||||
@@ -949,7 +995,7 @@ func (s *TestSuiteIAM) TestServiceAccountDurationSecondsCondition(c *check) {
|
||||
|
||||
// Create policy, user and associate policy
|
||||
policy := "mypolicy"
|
||||
policyBytes := []byte(fmt.Sprintf(`{
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
@@ -963,16 +1009,24 @@ func (s *TestSuiteIAM) TestServiceAccountDurationSecondsCondition(c *check) {
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject",
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::%s"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::%s/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}`, bucket))
|
||||
}`, bucket, bucket)
|
||||
err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
|
||||
if err != nil {
|
||||
c.Fatalf("policy add error: %v", err)
|
||||
@@ -1039,22 +1093,30 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByAdmin(c *check) {
|
||||
|
||||
// Create policy, user and associate policy
|
||||
policy := "mypolicy"
|
||||
policyBytes := []byte(fmt.Sprintf(`{
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject",
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::%s"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::%s/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}`, bucket))
|
||||
}`, bucket, bucket)
|
||||
err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
|
||||
if err != nil {
|
||||
c.Fatalf("policy add error: %v", err)
|
||||
@@ -1305,7 +1367,7 @@ func (s *TestSuiteIAM) TestAccMgmtPlugin(c *check) {
|
||||
svcAK, svcSK := mustGenerateCredentials(c)
|
||||
|
||||
// This policy does not allow listing objects.
|
||||
policyBytes := []byte(fmt.Sprintf(`{
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
@@ -1319,7 +1381,7 @@ func (s *TestSuiteIAM) TestAccMgmtPlugin(c *check) {
|
||||
]
|
||||
}
|
||||
]
|
||||
}`, bucket))
|
||||
}`, bucket)
|
||||
cr, err := userAdmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
|
||||
Policy: policyBytes,
|
||||
TargetUser: accessKey,
|
||||
@@ -1496,7 +1558,7 @@ func (c *check) mustDownload(ctx context.Context, client *minio.Client, bucket s
|
||||
func (c *check) mustUploadReturnVersions(ctx context.Context, client *minio.Client, bucket string) []string {
|
||||
c.Helper()
|
||||
versions := []string{}
|
||||
for i := 0; i < 5; i++ {
|
||||
for range 5 {
|
||||
ui, err := client.PutObject(ctx, bucket, "some-object", bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{})
|
||||
if err != nil {
|
||||
c.Fatalf("upload did not succeed got %#v", err)
|
||||
@@ -1565,7 +1627,7 @@ func (c *check) assertSvcAccSessionPolicyUpdate(ctx context.Context, s *TestSuit
|
||||
svcAK, svcSK := mustGenerateCredentials(c)
|
||||
|
||||
// This policy does not allow listing objects.
|
||||
policyBytes := []byte(fmt.Sprintf(`{
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
@@ -1579,7 +1641,7 @@ func (c *check) assertSvcAccSessionPolicyUpdate(ctx context.Context, s *TestSuit
|
||||
]
|
||||
}
|
||||
]
|
||||
}`, bucket))
|
||||
}`, bucket)
|
||||
cr, err := madmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
|
||||
Policy: policyBytes,
|
||||
TargetUser: accessKey,
|
||||
@@ -1593,7 +1655,7 @@ func (c *check) assertSvcAccSessionPolicyUpdate(ctx context.Context, s *TestSuit
|
||||
c.mustNotListObjects(ctx, svcClient, bucket)
|
||||
|
||||
// This policy allows listing objects.
|
||||
newPolicyBytes := []byte(fmt.Sprintf(`{
|
||||
newPolicyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
@@ -1602,11 +1664,11 @@ func (c *check) assertSvcAccSessionPolicyUpdate(ctx context.Context, s *TestSuit
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::%s/*"
|
||||
"arn:aws:s3:::%s"
|
||||
]
|
||||
}
|
||||
]
|
||||
}`, bucket))
|
||||
}`, bucket)
|
||||
err = madmClient.UpdateServiceAccount(ctx, svcAK, madmin.UpdateServiceAccountReq{
|
||||
NewPolicy: newPolicyBytes,
|
||||
})
|
||||
|
||||
+83
-56
@@ -49,6 +49,7 @@ import (
|
||||
"github.com/klauspost/compress/zip"
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/madmin-go/v3/estream"
|
||||
"github.com/minio/madmin-go/v3/logger/log"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
@@ -59,7 +60,6 @@ import (
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/logger/message/log"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/secure-io/sio-go"
|
||||
@@ -93,11 +93,18 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
|
||||
return
|
||||
}
|
||||
|
||||
if globalInplaceUpdateDisabled || currentReleaseTime.IsZero() {
|
||||
if globalInplaceUpdateDisabled {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if currentReleaseTime.IsZero() || currentReleaseTime.Equal(timeSentinel) {
|
||||
apiErr := errorCodes.ToAPIErr(ErrMethodNotAllowed)
|
||||
apiErr.Description = fmt.Sprintf("unable to perform in-place update, release time is unrecognized: %s", currentReleaseTime)
|
||||
writeErrorResponseJSON(ctx, w, apiErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
updateURL := vars["updateURL"]
|
||||
dryRun := r.Form.Get("dry-run") == "true"
|
||||
@@ -110,6 +117,11 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
|
||||
}
|
||||
}
|
||||
|
||||
local := globalLocalNodeName
|
||||
if local == "" {
|
||||
local = "127.0.0.1"
|
||||
}
|
||||
|
||||
u, err := url.Parse(updateURL)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
@@ -128,6 +140,39 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
|
||||
return
|
||||
}
|
||||
|
||||
updateStatus := madmin.ServerUpdateStatusV2{
|
||||
DryRun: dryRun,
|
||||
Results: make([]madmin.ServerPeerUpdateStatus, 0, len(globalNotificationSys.allPeerClients)),
|
||||
}
|
||||
peerResults := make(map[string]madmin.ServerPeerUpdateStatus, len(globalNotificationSys.allPeerClients))
|
||||
failedClients := make(map[int]bool, len(globalNotificationSys.allPeerClients))
|
||||
|
||||
if lrTime.Sub(currentReleaseTime) <= 0 {
|
||||
updateStatus.Results = append(updateStatus.Results, madmin.ServerPeerUpdateStatus{
|
||||
Host: local,
|
||||
Err: fmt.Sprintf("server is running the latest version: %s", Version),
|
||||
CurrentVersion: Version,
|
||||
})
|
||||
|
||||
for _, client := range globalNotificationSys.peerClients {
|
||||
updateStatus.Results = append(updateStatus.Results, madmin.ServerPeerUpdateStatus{
|
||||
Host: client.String(),
|
||||
Err: fmt.Sprintf("server is running the latest version: %s", Version),
|
||||
CurrentVersion: Version,
|
||||
})
|
||||
}
|
||||
|
||||
// Marshal API response
|
||||
jsonBytes, err := json.Marshal(updateStatus)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseJSON(w, jsonBytes)
|
||||
return
|
||||
}
|
||||
|
||||
u.Path = path.Dir(u.Path) + SlashSeparator + releaseInfo
|
||||
// Download Binary Once
|
||||
binC, bin, err := downloadBinary(u, mode)
|
||||
@@ -137,16 +182,6 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
|
||||
return
|
||||
}
|
||||
|
||||
updateStatus := madmin.ServerUpdateStatusV2{DryRun: dryRun}
|
||||
peerResults := make(map[string]madmin.ServerPeerUpdateStatus)
|
||||
|
||||
local := globalLocalNodeName
|
||||
if local == "" {
|
||||
local = "127.0.0.1"
|
||||
}
|
||||
|
||||
failedClients := make(map[int]struct{})
|
||||
|
||||
if globalIsDistErasure {
|
||||
// Push binary to other servers
|
||||
for idx, nerr := range globalNotificationSys.VerifyBinary(ctx, u, sha256Sum, releaseInfo, binC) {
|
||||
@@ -156,7 +191,7 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
|
||||
Err: nerr.Err.Error(),
|
||||
CurrentVersion: Version,
|
||||
}
|
||||
failedClients[idx] = struct{}{}
|
||||
failedClients[idx] = true
|
||||
} else {
|
||||
peerResults[nerr.Host.String()] = madmin.ServerPeerUpdateStatus{
|
||||
Host: nerr.Host.String(),
|
||||
@@ -167,25 +202,17 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
|
||||
}
|
||||
}
|
||||
|
||||
if lrTime.Sub(currentReleaseTime) > 0 {
|
||||
if err = verifyBinary(u, sha256Sum, releaseInfo, mode, bytes.NewReader(bin)); err != nil {
|
||||
peerResults[local] = madmin.ServerPeerUpdateStatus{
|
||||
Host: local,
|
||||
Err: err.Error(),
|
||||
CurrentVersion: Version,
|
||||
}
|
||||
} else {
|
||||
peerResults[local] = madmin.ServerPeerUpdateStatus{
|
||||
Host: local,
|
||||
CurrentVersion: Version,
|
||||
UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
|
||||
}
|
||||
if err = verifyBinary(u, sha256Sum, releaseInfo, mode, bytes.NewReader(bin)); err != nil {
|
||||
peerResults[local] = madmin.ServerPeerUpdateStatus{
|
||||
Host: local,
|
||||
Err: err.Error(),
|
||||
CurrentVersion: Version,
|
||||
}
|
||||
} else {
|
||||
peerResults[local] = madmin.ServerPeerUpdateStatus{
|
||||
Host: local,
|
||||
Err: fmt.Sprintf("server is already running the latest version: %s", Version),
|
||||
CurrentVersion: Version,
|
||||
UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -193,8 +220,7 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
|
||||
if globalIsDistErasure {
|
||||
ng := WithNPeers(len(globalNotificationSys.peerClients))
|
||||
for idx, client := range globalNotificationSys.peerClients {
|
||||
_, ok := failedClients[idx]
|
||||
if ok {
|
||||
if failedClients[idx] {
|
||||
continue
|
||||
}
|
||||
client := client
|
||||
@@ -240,14 +266,14 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
|
||||
startTime := time.Now().Add(restartUpdateDelay)
|
||||
ng := WithNPeers(len(globalNotificationSys.peerClients))
|
||||
for idx, client := range globalNotificationSys.peerClients {
|
||||
_, ok := failedClients[idx]
|
||||
if ok {
|
||||
if failedClients[idx] {
|
||||
continue
|
||||
}
|
||||
client := client
|
||||
ng.Go(ctx, func() error {
|
||||
prs, ok := peerResults[client.String()]
|
||||
if ok && prs.CurrentVersion != prs.UpdatedVersion && prs.UpdatedVersion != "" {
|
||||
// We restart only on success, not for any failures.
|
||||
if ok && prs.Err == "" {
|
||||
return client.SignalService(serviceRestart, "", dryRun, &startTime)
|
||||
}
|
||||
return nil
|
||||
@@ -284,7 +310,9 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
|
||||
writeSuccessResponseJSON(w, jsonBytes)
|
||||
|
||||
if !dryRun {
|
||||
if lrTime.Sub(currentReleaseTime) > 0 {
|
||||
prs, ok := peerResults[local]
|
||||
// We restart only on success, not for any failures.
|
||||
if ok && prs.Err == "" {
|
||||
globalServiceSignalCh <- serviceRestart
|
||||
}
|
||||
}
|
||||
@@ -801,7 +829,7 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
|
||||
// Flush before waiting for next...
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
|
||||
select {
|
||||
case <-ticker.C:
|
||||
@@ -926,7 +954,7 @@ func (a adminAPIHandlers) ForceUnlockHandler(w http.ResponseWriter, r *http.Requ
|
||||
|
||||
var args dsync.LockArgs
|
||||
var lockers []dsync.NetLocker
|
||||
for _, path := range strings.Split(vars["paths"], ",") {
|
||||
for path := range strings.SplitSeq(vars["paths"], ",") {
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
@@ -1165,7 +1193,7 @@ type dummyFileInfo struct {
|
||||
mode os.FileMode
|
||||
modTime time.Time
|
||||
isDir bool
|
||||
sys interface{}
|
||||
sys any
|
||||
}
|
||||
|
||||
func (f dummyFileInfo) Name() string { return f.name }
|
||||
@@ -1173,7 +1201,7 @@ func (f dummyFileInfo) Size() int64 { return f.size }
|
||||
func (f dummyFileInfo) Mode() os.FileMode { return f.mode }
|
||||
func (f dummyFileInfo) ModTime() time.Time { return f.modTime }
|
||||
func (f dummyFileInfo) IsDir() bool { return f.isDir }
|
||||
func (f dummyFileInfo) Sys() interface{} { return f.sys }
|
||||
func (f dummyFileInfo) Sys() any { return f.sys }
|
||||
|
||||
// DownloadProfilingHandler - POST /minio/admin/v3/profiling/download
|
||||
// ----------
|
||||
@@ -1292,7 +1320,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// Analyze the heal token and route the request accordingly
|
||||
token, success := proxyRequestByToken(ctx, w, r, hip.clientToken)
|
||||
token, _, success := proxyRequestByToken(ctx, w, r, hip.clientToken, false)
|
||||
if success {
|
||||
return
|
||||
}
|
||||
@@ -1331,7 +1359,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if _, err := w.Write([]byte(" ")); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
case hr := <-respCh:
|
||||
switch hr.apiErr {
|
||||
case noError:
|
||||
@@ -1339,7 +1367,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if _, err := w.Write(hr.respBytes); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
} else {
|
||||
writeSuccessResponseJSON(w, hr.respBytes)
|
||||
}
|
||||
@@ -1366,7 +1394,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if _, err := w.Write(errorRespJSON); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
}
|
||||
break forLoop
|
||||
}
|
||||
@@ -1379,7 +1407,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if exists && !nh.hasEnded() && len(nh.currentStatus.Items) > 0 {
|
||||
clientToken := nh.clientToken
|
||||
if globalIsDistErasure {
|
||||
clientToken = fmt.Sprintf("%s:%d", nh.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
|
||||
clientToken = fmt.Sprintf("%s%s%d", nh.clientToken, getKeySeparator(), GetProxyEndpointLocalIndex(globalProxyEndpoints))
|
||||
}
|
||||
b, err := json.Marshal(madmin.HealStartSuccess{
|
||||
ClientToken: clientToken,
|
||||
@@ -1583,7 +1611,6 @@ func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request)
|
||||
if err != nil || ctx.Err() != nil || totalRx > 100*humanize.GiByte {
|
||||
break
|
||||
}
|
||||
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
@@ -1812,7 +1839,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
case result, ok := <-ch:
|
||||
if !ok {
|
||||
return
|
||||
@@ -1821,7 +1848,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
prevResult = result
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1930,7 +1957,7 @@ func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.R
|
||||
if err := enc.Encode(madmin.DriveSpeedTestResult{}); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
case result, ok := <-ch:
|
||||
if !ok {
|
||||
return
|
||||
@@ -1938,7 +1965,7 @@ func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.R
|
||||
if err := enc.Encode(result); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2055,7 +2082,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
|
||||
grid.PutByteBuffer(entry)
|
||||
if len(traceCh) == 0 {
|
||||
// Flush if nothing is queued
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
}
|
||||
case <-keepAliveTicker.C:
|
||||
if len(traceCh) > 0 {
|
||||
@@ -2064,7 +2091,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if _, err := w.Write([]byte(" ")); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
@@ -2156,7 +2183,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
|
||||
grid.PutByteBuffer(log)
|
||||
if len(logCh) == 0 {
|
||||
// Flush if nothing is queued
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
}
|
||||
case <-keepAliveTicker.C:
|
||||
if len(logCh) > 0 {
|
||||
@@ -2165,7 +2192,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
|
||||
if _, err := w.Write([]byte(" ")); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
@@ -2649,7 +2676,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
|
||||
// disk metrics are already included under drive info of each server
|
||||
getRealtimeMetrics := func() *madmin.RealtimeMetrics {
|
||||
var m madmin.RealtimeMetrics
|
||||
var types madmin.MetricType = madmin.MetricsAll &^ madmin.MetricsDisk
|
||||
types := madmin.MetricsAll &^ madmin.MetricsDisk
|
||||
mLocal := collectLocalMetrics(types, collectMetricsOpts{})
|
||||
m.Merge(&mLocal)
|
||||
cctx, cancel := context.WithTimeout(healthCtx, time.Second/2)
|
||||
@@ -2693,7 +2720,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
|
||||
poolsArgs := re.ReplaceAllString(cmdLine, `$3`)
|
||||
var anonPools []string
|
||||
|
||||
if !(strings.Contains(poolsArgs, "{") && strings.Contains(poolsArgs, "}")) {
|
||||
if !strings.Contains(poolsArgs, "{") || !strings.Contains(poolsArgs, "}") {
|
||||
// No ellipses pattern. Anonymize host name from every pool arg
|
||||
pools := strings.Fields(poolsArgs)
|
||||
anonPools = make([]string, len(pools))
|
||||
@@ -2935,13 +2962,13 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
if len(healthInfoCh) == 0 {
|
||||
// Flush if nothing is queued
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
}
|
||||
case <-ticker.C:
|
||||
if _, err := w.Write([]byte(" ")); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
case <-healthCtx.Done():
|
||||
return
|
||||
}
|
||||
@@ -3393,7 +3420,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
|
||||
// save the format.json as part of inspect by default
|
||||
if !(volume == minioMetaBucket && file == formatConfigFile) {
|
||||
if volume != minioMetaBucket || file != formatConfigFile {
|
||||
err = o.GetRawData(ctx, minioMetaBucket, formatConfigFile, rawDataFn)
|
||||
}
|
||||
if !errors.Is(err, errFileNotFound) {
|
||||
|
||||
@@ -263,7 +263,7 @@ func buildAdminRequest(queryVal url.Values, method, path string,
|
||||
}
|
||||
|
||||
func TestAdminServerInfo(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
|
||||
adminTestBed, err := prepareAdminErasureTestBed(ctx)
|
||||
@@ -402,7 +402,7 @@ func (b byResourceUID) Less(i, j int) bool {
|
||||
func TestTopLockEntries(t *testing.T) {
|
||||
locksHeld := make(map[string][]lockRequesterInfo)
|
||||
var owners []string
|
||||
for i := 0; i < 4; i++ {
|
||||
for i := range 4 {
|
||||
owners = append(owners, fmt.Sprintf("node-%d", i))
|
||||
}
|
||||
|
||||
@@ -410,7 +410,7 @@ func TestTopLockEntries(t *testing.T) {
|
||||
// request UID, but 10 different resource names associated with it.
|
||||
var lris []lockRequesterInfo
|
||||
uuid := mustGetUUID()
|
||||
for i := 0; i < 10; i++ {
|
||||
for i := range 10 {
|
||||
resource := fmt.Sprintf("bucket/delete-object-%d", i)
|
||||
lri := lockRequesterInfo{
|
||||
Name: resource,
|
||||
@@ -425,7 +425,7 @@ func TestTopLockEntries(t *testing.T) {
|
||||
}
|
||||
|
||||
// Add a few concurrent read locks to the mix
|
||||
for i := 0; i < 50; i++ {
|
||||
for i := range 50 {
|
||||
resource := fmt.Sprintf("bucket/get-object-%d", i)
|
||||
lri := lockRequesterInfo{
|
||||
Name: resource,
|
||||
|
||||
+17
-30
@@ -22,6 +22,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/http"
|
||||
"sort"
|
||||
"sync"
|
||||
@@ -260,7 +261,7 @@ func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) {
|
||||
} else {
|
||||
clientToken := he.clientToken
|
||||
if globalIsDistErasure {
|
||||
clientToken = fmt.Sprintf("%s:%d", he.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
|
||||
clientToken = fmt.Sprintf("%s%s%d", he.clientToken, getKeySeparator(), GetProxyEndpointLocalIndex(globalProxyEndpoints))
|
||||
}
|
||||
|
||||
hsp = madmin.HealStopSuccess{
|
||||
@@ -331,7 +332,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence, objAPI ObjectLay
|
||||
|
||||
clientToken := h.clientToken
|
||||
if globalIsDistErasure {
|
||||
clientToken = fmt.Sprintf("%s:%d", h.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
|
||||
clientToken = fmt.Sprintf("%s%s%d", h.clientToken, getKeySeparator(), GetProxyEndpointLocalIndex(globalProxyEndpoints))
|
||||
}
|
||||
|
||||
if h.clientToken == bgHealingUUID {
|
||||
@@ -520,9 +521,7 @@ func (h *healSequence) getScannedItemsMap() map[madmin.HealItemType]int64 {
|
||||
|
||||
// Make a copy before returning the value
|
||||
retMap := make(map[madmin.HealItemType]int64, len(h.scannedItemsMap))
|
||||
for k, v := range h.scannedItemsMap {
|
||||
retMap[k] = v
|
||||
}
|
||||
maps.Copy(retMap, h.scannedItemsMap)
|
||||
|
||||
return retMap
|
||||
}
|
||||
@@ -534,9 +533,7 @@ func (h *healSequence) getHealedItemsMap() map[madmin.HealItemType]int64 {
|
||||
|
||||
// Make a copy before returning the value
|
||||
retMap := make(map[madmin.HealItemType]int64, len(h.healedItemsMap))
|
||||
for k, v := range h.healedItemsMap {
|
||||
retMap[k] = v
|
||||
}
|
||||
maps.Copy(retMap, h.healedItemsMap)
|
||||
|
||||
return retMap
|
||||
}
|
||||
@@ -549,9 +546,7 @@ func (h *healSequence) getHealFailedItemsMap() map[madmin.HealItemType]int64 {
|
||||
|
||||
// Make a copy before returning the value
|
||||
retMap := make(map[madmin.HealItemType]int64, len(h.healFailedItemsMap))
|
||||
for k, v := range h.healFailedItemsMap {
|
||||
retMap[k] = v
|
||||
}
|
||||
maps.Copy(retMap, h.healFailedItemsMap)
|
||||
|
||||
return retMap
|
||||
}
|
||||
@@ -806,18 +801,20 @@ func (h *healSequence) healDiskMeta(objAPI ObjectLayer) error {
|
||||
return h.healMinioSysMeta(objAPI, minioConfigPrefix)()
|
||||
}
|
||||
|
||||
func (h *healSequence) healItems(objAPI ObjectLayer, bucketsOnly bool) error {
|
||||
func (h *healSequence) healItems(objAPI ObjectLayer) error {
|
||||
if h.clientToken == bgHealingUUID {
|
||||
// For background heal do nothing.
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := h.healDiskMeta(objAPI); err != nil {
|
||||
return err
|
||||
if h.bucket == "" { // heal internal meta only during a site-wide heal
|
||||
if err := h.healDiskMeta(objAPI); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Heal buckets and objects
|
||||
return h.healBuckets(objAPI, bucketsOnly)
|
||||
return h.healBuckets(objAPI)
|
||||
}
|
||||
|
||||
// traverseAndHeal - traverses on-disk data and performs healing
|
||||
@@ -828,8 +825,7 @@ func (h *healSequence) healItems(objAPI ObjectLayer, bucketsOnly bool) error {
|
||||
// has to wait until a safe point is reached, such as between scanning
|
||||
// two objects.
|
||||
func (h *healSequence) traverseAndHeal(objAPI ObjectLayer) {
|
||||
bucketsOnly := false // Heals buckets and objects also.
|
||||
h.traverseAndHealDoneCh <- h.healItems(objAPI, bucketsOnly)
|
||||
h.traverseAndHealDoneCh <- h.healItems(objAPI)
|
||||
xioutil.SafeClose(h.traverseAndHealDoneCh)
|
||||
}
|
||||
|
||||
@@ -840,6 +836,7 @@ func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) f
|
||||
// NOTE: Healing on meta is run regardless
|
||||
// of any bucket being selected, this is to ensure that
|
||||
// meta are always upto date and correct.
|
||||
h.settings.Recursive = true
|
||||
return objAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string, scanMode madmin.HealScanMode) error {
|
||||
if h.isQuitting() {
|
||||
return errHealStopSignalled
|
||||
@@ -856,14 +853,14 @@ func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) f
|
||||
}
|
||||
|
||||
// healBuckets - check for all buckets heal or just particular bucket.
|
||||
func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
|
||||
func (h *healSequence) healBuckets(objAPI ObjectLayer) error {
|
||||
if h.isQuitting() {
|
||||
return errHealStopSignalled
|
||||
}
|
||||
|
||||
// 1. If a bucket was specified, heal only the bucket.
|
||||
if h.bucket != "" {
|
||||
return h.healBucket(objAPI, h.bucket, bucketsOnly)
|
||||
return h.healBucket(objAPI, h.bucket, false)
|
||||
}
|
||||
|
||||
buckets, err := objAPI.ListBuckets(h.ctx, BucketOptions{})
|
||||
@@ -877,7 +874,7 @@ func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
|
||||
})
|
||||
|
||||
for _, bucket := range buckets {
|
||||
if err = h.healBucket(objAPI, bucket.Name, bucketsOnly); err != nil {
|
||||
if err = h.healBucket(objAPI, bucket.Name, false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -895,16 +892,6 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly
|
||||
return nil
|
||||
}
|
||||
|
||||
if !h.settings.Recursive {
|
||||
if h.object != "" {
|
||||
if err := h.healObject(bucket, h.object, "", h.settings.ScanMode); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := objAPI.HealObjects(h.ctx, bucket, h.object, h.settings, h.healObject); err != nil {
|
||||
return errFnHealFromAPIErr(h.ctx, err)
|
||||
}
|
||||
|
||||
@@ -244,6 +244,10 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
// STS accounts ops
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/temporary-account-info").HandlerFunc(adminMiddleware(adminAPI.TemporaryAccountInfo)).Queries("accessKey", "{accessKey:.*}")
|
||||
|
||||
// Access key (service account/STS) operations
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-access-keys-bulk").HandlerFunc(adminMiddleware(adminAPI.ListAccessKeysBulk)).Queries("listType", "{listType:.*}")
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-access-key").HandlerFunc(adminMiddleware(adminAPI.InfoAccessKey)).Queries("accessKey", "{accessKey:.*}")
|
||||
|
||||
// Info policy IAM latest
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(adminMiddleware(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
|
||||
// List policies latest
|
||||
@@ -309,6 +313,11 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
// LDAP IAM operations
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/ldap/policy-entities").HandlerFunc(adminMiddleware(adminAPI.ListLDAPPolicyMappingEntities))
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/ldap/policy/{operation}").HandlerFunc(adminMiddleware(adminAPI.AttachDetachPolicyLDAP))
|
||||
|
||||
// OpenID specific service accounts ops
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/idp/openid/list-access-keys-bulk").
|
||||
HandlerFunc(adminMiddleware(adminAPI.ListAccessKeysOpenIDBulk)).Queries("listType", "{listType:.*}")
|
||||
|
||||
// -- END IAM APIs --
|
||||
|
||||
// GetBucketQuotaConfig
|
||||
@@ -421,6 +430,9 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
// -- Health API --
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/healthinfo").
|
||||
HandlerFunc(adminMiddleware(adminAPI.HealthInfoHandler))
|
||||
|
||||
// STS Revocation
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/revoke-tokens/{userProvider}").HandlerFunc(adminMiddleware(adminAPI.RevokeTokens))
|
||||
}
|
||||
|
||||
// If none of the routes match add default error handler routes
|
||||
|
||||
@@ -32,6 +32,8 @@ type DeletedObject struct {
|
||||
DeleteMarkerMTime DeleteMarkerMTime `xml:"-"`
|
||||
// MinIO extensions to support delete marker replication
|
||||
ReplicationState ReplicationState `xml:"-"`
|
||||
|
||||
found bool // the object was found during deletion
|
||||
}
|
||||
|
||||
// DeleteMarkerMTime is an embedded type containing time.Time for XML marshal
|
||||
@@ -42,10 +44,10 @@ type DeleteMarkerMTime struct {
|
||||
// MarshalXML encodes expiration date if it is non-zero and encodes
|
||||
// empty string otherwise
|
||||
func (t DeleteMarkerMTime) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
|
||||
if t.Time.IsZero() {
|
||||
if t.IsZero() {
|
||||
return nil
|
||||
}
|
||||
return e.EncodeElement(t.Time.Format(time.RFC3339), startElement)
|
||||
return e.EncodeElement(t.Format(time.RFC3339), startElement)
|
||||
}
|
||||
|
||||
// ObjectV object version key/versionId
|
||||
|
||||
+30
-2
@@ -213,6 +213,10 @@ const (
|
||||
ErrPolicyAlreadyAttached
|
||||
ErrPolicyNotAttached
|
||||
ErrExcessData
|
||||
ErrPolicyInvalidName
|
||||
ErrNoTokenRevokeType
|
||||
ErrAdminOpenIDNotEnabled
|
||||
ErrAdminNoSuchAccessKey
|
||||
// Add new error codes here.
|
||||
|
||||
// SSE-S3/SSE-KMS related API errors
|
||||
@@ -561,6 +565,16 @@ var errorCodes = errorCodeMap{
|
||||
Description: "More data provided than indicated content length",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrPolicyInvalidName: {
|
||||
Code: "PolicyInvalidName",
|
||||
Description: "Policy name may not contain comma",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminOpenIDNotEnabled: {
|
||||
Code: "OpenIDNotEnabled",
|
||||
Description: "No enabled OpenID Connect identity providers",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrPolicyTooLarge: {
|
||||
Code: "PolicyTooLarge",
|
||||
Description: "Policy exceeds the maximum allowed document size.",
|
||||
@@ -623,7 +637,7 @@ var errorCodes = errorCodeMap{
|
||||
},
|
||||
ErrMissingContentMD5: {
|
||||
Code: "MissingContentMD5",
|
||||
Description: "Missing required header for this request: Content-Md5.",
|
||||
Description: "Missing or invalid required header for this request: Content-Md5 or Amz-Content-Checksum",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrMissingSecurityHeader: {
|
||||
@@ -978,7 +992,7 @@ var errorCodes = errorCodeMap{
|
||||
},
|
||||
ErrReplicationNoExistingObjects: {
|
||||
Code: "XMinioReplicationNoExistingObjects",
|
||||
Description: "No matching ExistingsObjects rule enabled",
|
||||
Description: "No matching ExistingObjects rule enabled",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrRemoteTargetDenyAddError: {
|
||||
@@ -1258,6 +1272,16 @@ var errorCodes = errorCodeMap{
|
||||
Description: "The security token included in the request is invalid",
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
},
|
||||
ErrNoTokenRevokeType: {
|
||||
Code: "InvalidArgument",
|
||||
Description: "No token revoke type specified and one could not be inferred from the request",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminNoSuchAccessKey: {
|
||||
Code: "XMinioAdminNoSuchAccessKey",
|
||||
Description: "The specified access key does not exist.",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
|
||||
// S3 extensions.
|
||||
ErrContentSHA256Mismatch: {
|
||||
@@ -2155,6 +2179,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrAdminNoSuchUserLDAPWarn
|
||||
case errNoSuchServiceAccount:
|
||||
apiErr = ErrAdminServiceAccountNotFound
|
||||
case errNoSuchAccessKey:
|
||||
apiErr = ErrAdminNoSuchAccessKey
|
||||
case errNoSuchGroup:
|
||||
apiErr = ErrAdminNoSuchGroup
|
||||
case errGroupNotEmpty:
|
||||
@@ -2248,6 +2274,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrServerNotInitialized
|
||||
case errBucketMetadataNotInitialized:
|
||||
apiErr = ErrBucketMetadataNotInitialized
|
||||
case hash.ErrInvalidChecksum:
|
||||
apiErr = ErrInvalidChecksum
|
||||
}
|
||||
|
||||
// Compression errors
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
@@ -64,7 +63,7 @@ var toAPIErrorTests = []struct {
|
||||
}
|
||||
|
||||
func TestAPIErrCode(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctx := t.Context()
|
||||
for i, testCase := range toAPIErrorTests {
|
||||
errCode := toAPIErrorCode(ctx, testCase.err)
|
||||
if errCode != testCase.errCode {
|
||||
|
||||
+41
-3
@@ -23,6 +23,7 @@ import (
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"mime"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -64,7 +65,7 @@ func setCommonHeaders(w http.ResponseWriter) {
|
||||
}
|
||||
|
||||
// Encodes the response headers into XML format.
|
||||
func encodeResponse(response interface{}) []byte {
|
||||
func encodeResponse(response any) []byte {
|
||||
var buf bytes.Buffer
|
||||
buf.WriteString(xml.Header)
|
||||
if err := xml.NewEncoder(&buf).Encode(response); err != nil {
|
||||
@@ -82,7 +83,7 @@ func encodeResponse(response interface{}) []byte {
|
||||
// Do not use this function for anything other than ListObjects()
|
||||
// variants, please open a github discussion if you wish to use
|
||||
// this in other places.
|
||||
func encodeResponseList(response interface{}) []byte {
|
||||
func encodeResponseList(response any) []byte {
|
||||
var buf bytes.Buffer
|
||||
buf.WriteString(xxml.Header)
|
||||
if err := xxml.NewEncoder(&buf).Encode(response); err != nil {
|
||||
@@ -93,7 +94,7 @@ func encodeResponseList(response interface{}) []byte {
|
||||
}
|
||||
|
||||
// Encodes the response headers into JSON format.
|
||||
func encodeResponseJSON(response interface{}) []byte {
|
||||
func encodeResponseJSON(response any) []byte {
|
||||
var bytesBuffer bytes.Buffer
|
||||
e := json.NewEncoder(&bytesBuffer)
|
||||
e.Encode(response)
|
||||
@@ -168,6 +169,32 @@ func setObjectHeaders(ctx context.Context, w http.ResponseWriter, objInfo Object
|
||||
if !stringsHasPrefixFold(k, userMetadataPrefix) {
|
||||
continue
|
||||
}
|
||||
// check the doc https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html
|
||||
// For metadata values like "ö", "ÄMÄZÕÑ S3", and "öha, das sollte eigentlich
|
||||
// funktionieren", tested against a real AWS S3 bucket, S3 may encode incorrectly. For
|
||||
// example, "ö" was encoded as =?UTF-8?B?w4PCtg==?=, producing invalid UTF-8 instead
|
||||
// of =?UTF-8?B?w7Y=?=. This mirrors errors like the ä½ in another string.
|
||||
//
|
||||
// S3 uses B-encoding (Base64) for non-ASCII-heavy metadata and Q-encoding
|
||||
// (quoted-printable) for mostly ASCII strings. Long strings are split at word
|
||||
// boundaries to fit RFC 2047’s 75-character limit, ensuring HTTP parser
|
||||
// compatibility.
|
||||
//
|
||||
// However, this splitting increases header size and can introduce errors, unlike Go’s
|
||||
// mime package in MinIO, which correctly encodes strings with fixed B/Q encodings,
|
||||
// avoiding S3’s heuristic-driven issues.
|
||||
//
|
||||
// For MinIO developers, decode S3 metadata with mime.WordDecoder, validate outputs,
|
||||
// report encoding bugs to AWS, and use ASCII-only metadata to ensure reliable S3 API
|
||||
// compatibility.
|
||||
if needsMimeEncoding(v) {
|
||||
// see https://github.com/golang/go/blob/release-branch.go1.24/src/net/mail/message.go#L325
|
||||
if strings.ContainsAny(v, "\"#$%&'(),.:;<>@[]^`{|}~") {
|
||||
v = mime.BEncoding.Encode("UTF-8", v)
|
||||
} else {
|
||||
v = mime.QEncoding.Encode("UTF-8", v)
|
||||
}
|
||||
}
|
||||
w.Header()[strings.ToLower(k)] = []string{v}
|
||||
isSet = true
|
||||
break
|
||||
@@ -229,3 +256,14 @@ func setObjectHeaders(ctx context.Context, w http.ResponseWriter, objInfo Object
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// needsEncoding reports whether s contains any bytes that need to be encoded.
|
||||
// see mime.needsEncoding
|
||||
func needsMimeEncoding(s string) bool {
|
||||
for _, b := range s {
|
||||
if (b < ' ' || b > '~') && b != '\t' {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -34,7 +34,8 @@ func TestNewRequestID(t *testing.T) {
|
||||
e = char
|
||||
|
||||
// Ensure that it is alphanumeric, in this case, between 0-9 and A-Z.
|
||||
if !(('0' <= e && e <= '9') || ('A' <= e && e <= 'Z')) {
|
||||
isAlnum := ('0' <= e && e <= '9') || ('A' <= e && e <= 'Z')
|
||||
if !isAlnum {
|
||||
t.Fail()
|
||||
}
|
||||
}
|
||||
|
||||
+21
-18
@@ -166,10 +166,11 @@ type Part struct {
|
||||
Size int64
|
||||
|
||||
// Checksum values
|
||||
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
|
||||
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
|
||||
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
|
||||
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
|
||||
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
|
||||
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
|
||||
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
|
||||
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
|
||||
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||
}
|
||||
|
||||
// ListPartsResponse - format for list parts response.
|
||||
@@ -192,6 +193,8 @@ type ListPartsResponse struct {
|
||||
IsTruncated bool
|
||||
|
||||
ChecksumAlgorithm string
|
||||
ChecksumType string
|
||||
|
||||
// List of parts.
|
||||
Parts []Part `xml:"Part"`
|
||||
}
|
||||
@@ -413,10 +416,11 @@ type CompleteMultipartUploadResponse struct {
|
||||
Key string
|
||||
ETag string
|
||||
|
||||
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
|
||||
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
|
||||
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
|
||||
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
|
||||
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
|
||||
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
|
||||
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
|
||||
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
|
||||
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||
}
|
||||
|
||||
// DeleteError structure.
|
||||
@@ -516,7 +520,6 @@ func cleanReservedKeys(metadata map[string]string) map[string]string {
|
||||
}
|
||||
case crypto.SSEC:
|
||||
m[xhttp.AmzServerSideEncryptionCustomerAlgorithm] = xhttp.AmzEncryptionAES
|
||||
|
||||
}
|
||||
|
||||
var toRemove []string
|
||||
@@ -593,8 +596,6 @@ func generateListVersionsResponse(ctx context.Context, bucket, prefix, marker, v
|
||||
for k, v := range cleanReservedKeys(object.UserDefined) {
|
||||
content.UserMetadata.Set(k, v)
|
||||
}
|
||||
|
||||
content.UserMetadata.Set("expires", object.Expires.Format(http.TimeFormat))
|
||||
content.Internal = &ObjectInternalInfo{
|
||||
K: object.DataBlocks,
|
||||
M: object.ParityBlocks,
|
||||
@@ -729,7 +730,6 @@ func generateListObjectsV2Response(ctx context.Context, bucket, prefix, token, n
|
||||
for k, v := range cleanReservedKeys(object.UserDefined) {
|
||||
content.UserMetadata.Set(k, v)
|
||||
}
|
||||
content.UserMetadata.Set("expires", object.Expires.Format(http.TimeFormat))
|
||||
content.Internal = &ObjectInternalInfo{
|
||||
K: object.DataBlocks,
|
||||
M: object.ParityBlocks,
|
||||
@@ -790,17 +790,18 @@ func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) Initi
|
||||
|
||||
// generates CompleteMultipartUploadResponse for given bucket, key, location and ETag.
|
||||
func generateCompleteMultipartUploadResponse(bucket, key, location string, oi ObjectInfo, h http.Header) CompleteMultipartUploadResponse {
|
||||
cs := oi.decryptChecksums(0, h)
|
||||
cs, _ := oi.decryptChecksums(0, h)
|
||||
c := CompleteMultipartUploadResponse{
|
||||
Location: location,
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
// AWS S3 quotes the ETag in XML, make sure we are compatible here.
|
||||
ETag: "\"" + oi.ETag + "\"",
|
||||
ChecksumSHA1: cs[hash.ChecksumSHA1.String()],
|
||||
ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
|
||||
ChecksumCRC32: cs[hash.ChecksumCRC32.String()],
|
||||
ChecksumCRC32C: cs[hash.ChecksumCRC32C.String()],
|
||||
ETag: "\"" + oi.ETag + "\"",
|
||||
ChecksumSHA1: cs[hash.ChecksumSHA1.String()],
|
||||
ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
|
||||
ChecksumCRC32: cs[hash.ChecksumCRC32.String()],
|
||||
ChecksumCRC32C: cs[hash.ChecksumCRC32C.String()],
|
||||
ChecksumCRC64NVME: cs[hash.ChecksumCRC64NVME.String()],
|
||||
}
|
||||
return c
|
||||
}
|
||||
@@ -828,6 +829,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
|
||||
listPartsResponse.IsTruncated = partsInfo.IsTruncated
|
||||
listPartsResponse.NextPartNumberMarker = partsInfo.NextPartNumberMarker
|
||||
listPartsResponse.ChecksumAlgorithm = partsInfo.ChecksumAlgorithm
|
||||
listPartsResponse.ChecksumType = partsInfo.ChecksumType
|
||||
|
||||
listPartsResponse.Parts = make([]Part, len(partsInfo.Parts))
|
||||
for index, part := range partsInfo.Parts {
|
||||
@@ -840,6 +842,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
|
||||
newPart.ChecksumCRC32C = part.ChecksumCRC32C
|
||||
newPart.ChecksumSHA1 = part.ChecksumSHA1
|
||||
newPart.ChecksumSHA256 = part.ChecksumSHA256
|
||||
newPart.ChecksumCRC64NVME = part.ChecksumCRC64NVME
|
||||
listPartsResponse.Parts[index] = newPart
|
||||
}
|
||||
return listPartsResponse
|
||||
|
||||
@@ -100,7 +100,6 @@ func TestObjectLocation(t *testing.T) {
|
||||
},
|
||||
}
|
||||
for _, testCase := range testCases {
|
||||
testCase := testCase
|
||||
t.Run("", func(t *testing.T) {
|
||||
gotLocation := getObjectLocation(testCase.request, testCase.domains, testCase.bucket, testCase.object)
|
||||
if testCase.expectedLocation != gotLocation {
|
||||
|
||||
+7
-2
@@ -227,13 +227,13 @@ func s3APIMiddleware(f http.HandlerFunc, flags ...s3HFlag) http.HandlerFunc {
|
||||
}
|
||||
|
||||
// Skip wrapping with the gzip middleware if specified.
|
||||
var gzippedHandler http.HandlerFunc = tracedHandler
|
||||
gzippedHandler := tracedHandler
|
||||
if !handlerFlags.has(noGZS3HFlag) {
|
||||
gzippedHandler = gzipHandler(gzippedHandler)
|
||||
}
|
||||
|
||||
// Skip wrapping with throttling middleware if specified.
|
||||
var throttledHandler http.HandlerFunc = gzippedHandler
|
||||
throttledHandler := gzippedHandler
|
||||
if !handlerFlags.has(noThrottleS3HFlag) {
|
||||
throttledHandler = maxClients(throttledHandler)
|
||||
}
|
||||
@@ -387,6 +387,11 @@ func registerAPIRouter(router *mux.Router) {
|
||||
HeadersRegexp(xhttp.AmzSnowballExtract, "true").
|
||||
HandlerFunc(s3APIMiddleware(api.PutObjectExtractHandler, traceHdrsS3HFlag))
|
||||
|
||||
// AppendObject to be rejected
|
||||
router.Methods(http.MethodPut).Path("/{object:.+}").
|
||||
HeadersRegexp(xhttp.AmzWriteOffsetBytes, "").
|
||||
HandlerFunc(s3APIMiddleware(errorResponseHandler))
|
||||
|
||||
// PutObject
|
||||
router.Methods(http.MethodPut).Path("/{object:.+}").
|
||||
HandlerFunc(s3APIMiddleware(api.PutObjectHandler, traceHdrsS3HFlag))
|
||||
|
||||
+3
-3
@@ -43,7 +43,7 @@ func shouldEscape(c byte) bool {
|
||||
// - Force encoding of '~'
|
||||
func s3URLEncode(s string) string {
|
||||
spaceCount, hexCount := 0, 0
|
||||
for i := 0; i < len(s); i++ {
|
||||
for i := range len(s) {
|
||||
c := s[i]
|
||||
if shouldEscape(c) {
|
||||
if c == ' ' {
|
||||
@@ -70,7 +70,7 @@ func s3URLEncode(s string) string {
|
||||
|
||||
if hexCount == 0 {
|
||||
copy(t, s)
|
||||
for i := 0; i < len(s); i++ {
|
||||
for i := range len(s) {
|
||||
if s[i] == ' ' {
|
||||
t[i] = '+'
|
||||
}
|
||||
@@ -79,7 +79,7 @@ func s3URLEncode(s string) string {
|
||||
}
|
||||
|
||||
j := 0
|
||||
for i := 0; i < len(s); i++ {
|
||||
for i := range len(s) {
|
||||
switch c := s[i]; {
|
||||
case c == ' ':
|
||||
t[j] = '+'
|
||||
|
||||
+203
-199
File diff suppressed because one or more lines are too long
+21
-38
@@ -96,7 +96,7 @@ func isRequestSignStreamingTrailerV4(r *http.Request) bool {
|
||||
// Verify if the request has AWS Streaming Signature Version '4', with unsigned content and trailer.
|
||||
func isRequestUnsignedTrailerV4(r *http.Request) bool {
|
||||
return r.Header.Get(xhttp.AmzContentSha256) == unsignedPayloadTrailer &&
|
||||
r.Method == http.MethodPut && strings.Contains(r.Header.Get(xhttp.ContentEncoding), streamingContentEncoding)
|
||||
r.Method == http.MethodPut
|
||||
}
|
||||
|
||||
// Authorization type.
|
||||
@@ -162,7 +162,6 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
|
||||
s3Err := ErrAccessDenied
|
||||
if _, ok := r.Header[xhttp.AmzContentSha256]; ok &&
|
||||
getRequestAuthType(r) == authTypeSigned {
|
||||
|
||||
// Get credential information from the request.
|
||||
cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
|
||||
if s3Err != ErrNone {
|
||||
@@ -217,12 +216,12 @@ func getSessionToken(r *http.Request) (token string) {
|
||||
|
||||
// Fetch claims in the security token returned by the client, doesn't return
|
||||
// errors - upon errors the returned claims map will be empty.
|
||||
func mustGetClaimsFromToken(r *http.Request) map[string]interface{} {
|
||||
func mustGetClaimsFromToken(r *http.Request) map[string]any {
|
||||
claims, _ := getClaimsFromToken(getSessionToken(r))
|
||||
return claims
|
||||
}
|
||||
|
||||
func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{}, error) {
|
||||
func getClaimsFromTokenWithSecret(token, secret string) (*xjwt.MapClaims, error) {
|
||||
// JWT token for x-amz-security-token is signed with admin
|
||||
// secret key, temporary credentials become invalid if
|
||||
// server admin credentials change. This is done to ensure
|
||||
@@ -244,7 +243,7 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
|
||||
|
||||
// If AuthZPlugin is set, return without any further checks.
|
||||
if newGlobalAuthZPluginFn() != nil {
|
||||
return claims.Map(), nil
|
||||
return claims, nil
|
||||
}
|
||||
|
||||
// Check if a session policy is set. If so, decode it here.
|
||||
@@ -263,16 +262,20 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
|
||||
claims.MapClaims[sessionPolicyNameExtracted] = string(spBytes)
|
||||
}
|
||||
|
||||
return claims.Map(), nil
|
||||
return claims, nil
|
||||
}
|
||||
|
||||
// Fetch claims in the security token returned by the client.
|
||||
func getClaimsFromToken(token string) (map[string]interface{}, error) {
|
||||
return getClaimsFromTokenWithSecret(token, globalActiveCred.SecretKey)
|
||||
func getClaimsFromToken(token string) (map[string]any, error) {
|
||||
jwtClaims, err := getClaimsFromTokenWithSecret(token, globalActiveCred.SecretKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return jwtClaims.Map(), nil
|
||||
}
|
||||
|
||||
// Fetch claims in the security token returned by the client and validate the token.
|
||||
func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]interface{}, APIErrorCode) {
|
||||
func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]any, APIErrorCode) {
|
||||
token := getSessionToken(r)
|
||||
if token != "" && cred.AccessKey == "" {
|
||||
// x-amz-security-token is not allowed for anonymous access.
|
||||
@@ -319,7 +322,7 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
|
||||
if err != nil {
|
||||
return nil, toAPIErrorCode(r.Context(), err)
|
||||
}
|
||||
return claims, ErrNone
|
||||
return claims.Map(), ErrNone
|
||||
}
|
||||
|
||||
claims := xjwt.NewMapClaims()
|
||||
@@ -360,7 +363,7 @@ func authenticateRequest(ctx context.Context, r *http.Request, action policy.Act
|
||||
var cred auth.Credentials
|
||||
var owner bool
|
||||
switch getRequestAuthType(r) {
|
||||
case authTypeUnknown, authTypeStreamingSigned:
|
||||
case authTypeUnknown, authTypeStreamingSigned, authTypeStreamingSignedTrailer, authTypeStreamingUnsignedTrailer:
|
||||
return ErrSignatureVersionNotSupported
|
||||
case authTypePresignedV2, authTypeSignedV2:
|
||||
if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone {
|
||||
@@ -671,32 +674,6 @@ func setAuthMiddleware(h http.Handler) http.Handler {
|
||||
})
|
||||
}
|
||||
|
||||
func validateSignature(atype authType, r *http.Request) (auth.Credentials, bool, APIErrorCode) {
|
||||
var cred auth.Credentials
|
||||
var owner bool
|
||||
var s3Err APIErrorCode
|
||||
switch atype {
|
||||
case authTypeUnknown, authTypeStreamingSigned:
|
||||
return cred, owner, ErrSignatureVersionNotSupported
|
||||
case authTypeSignedV2, authTypePresignedV2:
|
||||
if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone {
|
||||
return cred, owner, s3Err
|
||||
}
|
||||
cred, owner, s3Err = getReqAccessKeyV2(r)
|
||||
case authTypePresigned, authTypeSigned:
|
||||
region := globalSite.Region()
|
||||
if s3Err = isReqAuthenticated(GlobalContext, r, region, serviceS3); s3Err != ErrNone {
|
||||
return cred, owner, s3Err
|
||||
}
|
||||
cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
|
||||
}
|
||||
if s3Err != ErrNone {
|
||||
return cred, owner, s3Err
|
||||
}
|
||||
|
||||
return cred, owner, ErrNone
|
||||
}
|
||||
|
||||
func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate time.Time, retMode objectlock.RetMode, byPassSet bool, r *http.Request, cred auth.Credentials, owner bool) (s3Err APIErrorCode) {
|
||||
var retSet bool
|
||||
if cred.AccessKey == "" {
|
||||
@@ -751,8 +728,14 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
|
||||
return ErrSignatureVersionNotSupported
|
||||
case authTypeSignedV2, authTypePresignedV2:
|
||||
cred, owner, s3Err = getReqAccessKeyV2(r)
|
||||
case authTypeStreamingSigned, authTypePresigned, authTypeSigned, authTypeStreamingSignedTrailer, authTypeStreamingUnsignedTrailer:
|
||||
case authTypeStreamingSigned, authTypePresigned, authTypeSigned, authTypeStreamingSignedTrailer:
|
||||
cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
|
||||
case authTypeStreamingUnsignedTrailer:
|
||||
cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
|
||||
if s3Err == ErrMissingFields {
|
||||
// Could be anonymous. cred + owner is zero value.
|
||||
s3Err = ErrNone
|
||||
}
|
||||
}
|
||||
if s3Err != ErrNone {
|
||||
return s3Err
|
||||
|
||||
@@ -413,7 +413,7 @@ func TestIsReqAuthenticated(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCheckAdminRequestAuthType(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
|
||||
objLayer, fsDir, err := prepareFS(ctx)
|
||||
@@ -450,7 +450,7 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestValidateAdminSignature(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
|
||||
objLayer, fsDir, err := prepareFS(ctx)
|
||||
|
||||
@@ -102,7 +102,7 @@ func waitForLowHTTPReq() {
|
||||
func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
|
||||
bgSeq := newBgHealSequence()
|
||||
// Run the background healer
|
||||
for i := 0; i < globalBackgroundHealRoutine.workers; i++ {
|
||||
for range globalBackgroundHealRoutine.workers {
|
||||
go globalBackgroundHealRoutine.AddWorker(ctx, objAPI, bgSeq)
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -223,9 +224,6 @@ func (h *healingTracker) updateProgress(success, skipped bool, bytes uint64) {
|
||||
// update will update the tracker on the disk.
|
||||
// If the tracker has been deleted an error is returned.
|
||||
func (h *healingTracker) update(ctx context.Context) error {
|
||||
if h.disk.Healing() == nil {
|
||||
return fmt.Errorf("healingTracker: drive %q is not marked as healing", h.ID)
|
||||
}
|
||||
h.mu.Lock()
|
||||
if h.ID == "" || h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 {
|
||||
h.ID, _ = h.disk.GetDiskID()
|
||||
@@ -272,12 +270,7 @@ func (h *healingTracker) delete(ctx context.Context) error {
|
||||
func (h *healingTracker) isHealed(bucket string) bool {
|
||||
h.mu.RLock()
|
||||
defer h.mu.RUnlock()
|
||||
for _, v := range h.HealedBuckets {
|
||||
if v == bucket {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
return slices.Contains(h.HealedBuckets, bucket)
|
||||
}
|
||||
|
||||
// resume will reset progress to the numbers at the start of the bucket.
|
||||
|
||||
+129
-84
@@ -27,7 +27,6 @@ import (
|
||||
"net/http"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
@@ -196,8 +195,8 @@ func (ef BatchJobExpireFilter) Matches(obj ObjectInfo, now time.Time) bool {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if len(ef.Metadata) > 0 && !obj.DeleteMarker {
|
||||
for _, kv := range ef.Metadata {
|
||||
// Object (version) must match all x-amz-meta and
|
||||
@@ -290,6 +289,16 @@ type BatchJobExpire struct {
|
||||
|
||||
var _ yaml.Unmarshaler = &BatchJobExpire{}
|
||||
|
||||
// RedactSensitive will redact any sensitive information in b.
|
||||
func (r *BatchJobExpire) RedactSensitive() {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
if r.NotificationCfg.Token != "" {
|
||||
r.NotificationCfg.Token = redactedText
|
||||
}
|
||||
}
|
||||
|
||||
// UnmarshalYAML - BatchJobExpire extends default unmarshal to extract line, col information.
|
||||
func (r *BatchJobExpire) UnmarshalYAML(val *yaml.Node) error {
|
||||
type expireJob BatchJobExpire
|
||||
@@ -389,9 +398,12 @@ func (oiCache objInfoCache) Get(toDel ObjectToDelete) (*ObjectInfo, bool) {
|
||||
|
||||
func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo, job BatchJobRequest, api ObjectLayer, wk *workers.Workers, expireCh <-chan []expireObjInfo) {
|
||||
vc, _ := globalBucketVersioningSys.Get(r.Bucket)
|
||||
retryAttempts := r.Retry.Attempts
|
||||
retryAttempts := job.Expire.Retry.Attempts
|
||||
if retryAttempts <= 0 {
|
||||
retryAttempts = batchExpireJobDefaultRetries
|
||||
}
|
||||
delay := job.Expire.Retry.Delay
|
||||
if delay == 0 {
|
||||
if delay <= 0 {
|
||||
delay = batchExpireJobDefaultRetryDelay
|
||||
}
|
||||
|
||||
@@ -412,12 +424,12 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
|
||||
go func(toExpire []expireObjInfo) {
|
||||
defer wk.Give()
|
||||
|
||||
toExpireAll := make([]ObjectInfo, 0, len(toExpire))
|
||||
toExpireAll := make([]expireObjInfo, 0, len(toExpire))
|
||||
toDel := make([]ObjectToDelete, 0, len(toExpire))
|
||||
oiCache := newObjInfoCache()
|
||||
for _, exp := range toExpire {
|
||||
if exp.ExpireAll {
|
||||
toExpireAll = append(toExpireAll, exp.ObjectInfo)
|
||||
toExpireAll = append(toExpireAll, exp)
|
||||
continue
|
||||
}
|
||||
// Cache ObjectInfo value via pointers for
|
||||
@@ -433,14 +445,14 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
|
||||
oiCache.Add(od, &exp.ObjectInfo)
|
||||
}
|
||||
|
||||
var done bool
|
||||
// DeleteObject(deletePrefix: true) to expire all versions of an object
|
||||
for _, exp := range toExpireAll {
|
||||
var success bool
|
||||
for attempts := 1; attempts <= retryAttempts; attempts++ {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
done = true
|
||||
ri.trackMultipleObjectVersions(exp, success)
|
||||
return
|
||||
default:
|
||||
}
|
||||
stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
|
||||
@@ -457,14 +469,7 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
|
||||
break
|
||||
}
|
||||
}
|
||||
ri.trackMultipleObjectVersions(r.Bucket, exp, success)
|
||||
if done {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if done {
|
||||
return
|
||||
ri.trackMultipleObjectVersions(exp, success)
|
||||
}
|
||||
|
||||
// DeleteMultiple objects
|
||||
@@ -522,7 +527,8 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
|
||||
|
||||
type expireObjInfo struct {
|
||||
ObjectInfo
|
||||
ExpireAll bool
|
||||
ExpireAll bool
|
||||
DeleteMarkerCount int64
|
||||
}
|
||||
|
||||
// Start the batch expiration job, resumes if there was a pending job via "job.ID"
|
||||
@@ -557,9 +563,13 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
|
||||
|
||||
results := make(chan itemOrErr[ObjectInfo], workerSize)
|
||||
go func() {
|
||||
for _, prefix := range r.Prefix.F() {
|
||||
prefixes := r.Prefix.F()
|
||||
if len(prefixes) == 0 {
|
||||
prefixes = []string{""}
|
||||
}
|
||||
for _, prefix := range prefixes {
|
||||
prefixResultCh := make(chan itemOrErr[ObjectInfo], workerSize)
|
||||
err := api.Walk(ctx, r.Bucket, strings.TrimSpace(prefix), prefixResultCh, WalkOptions{
|
||||
err := api.Walk(ctx, r.Bucket, prefix, prefixResultCh, WalkOptions{
|
||||
Marker: lastObject,
|
||||
LatestOnly: false, // we need to visit all versions of the object to implement purge: retainVersions
|
||||
VersionsSort: WalkVersionsSortDesc,
|
||||
@@ -615,80 +625,115 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
|
||||
matchedFilter BatchJobExpireFilter
|
||||
versionsCount int
|
||||
toDel []expireObjInfo
|
||||
failed bool
|
||||
done bool
|
||||
)
|
||||
failed := false
|
||||
for result := range results {
|
||||
if result.Err != nil {
|
||||
failed = true
|
||||
batchLogIf(ctx, result.Err)
|
||||
continue
|
||||
deleteMarkerCountMap := map[string]int64{}
|
||||
pushToExpire := func() {
|
||||
// set preObject deleteMarkerCount
|
||||
if len(toDel) > 0 {
|
||||
lastDelIndex := len(toDel) - 1
|
||||
lastDel := toDel[lastDelIndex]
|
||||
if lastDel.ExpireAll {
|
||||
toDel[lastDelIndex].DeleteMarkerCount = deleteMarkerCountMap[lastDel.Name]
|
||||
// delete the key
|
||||
delete(deleteMarkerCountMap, lastDel.Name)
|
||||
}
|
||||
}
|
||||
|
||||
// Apply filter to find the matching rule to apply expiry
|
||||
// actions accordingly.
|
||||
// nolint:gocritic
|
||||
if result.Item.IsLatest {
|
||||
// send down filtered entries to be deleted using
|
||||
// DeleteObjects method
|
||||
if len(toDel) > 10 { // batch up to 10 objects/versions to be expired simultaneously.
|
||||
xfer := make([]expireObjInfo, len(toDel))
|
||||
copy(xfer, toDel)
|
||||
|
||||
var done bool
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
done = true
|
||||
case expireCh <- xfer:
|
||||
toDel = toDel[:0] // resetting toDel
|
||||
}
|
||||
if done {
|
||||
break
|
||||
}
|
||||
// send down filtered entries to be deleted using
|
||||
// DeleteObjects method
|
||||
if len(toDel) > 10 { // batch up to 10 objects/versions to be expired simultaneously.
|
||||
xfer := make([]expireObjInfo, len(toDel))
|
||||
copy(xfer, toDel)
|
||||
select {
|
||||
case expireCh <- xfer:
|
||||
toDel = toDel[:0] // resetting toDel
|
||||
case <-ctx.Done():
|
||||
done = true
|
||||
}
|
||||
var match BatchJobExpireFilter
|
||||
var found bool
|
||||
for _, rule := range r.Rules {
|
||||
if rule.Matches(result.Item, now) {
|
||||
match = rule
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
|
||||
prevObj = result.Item
|
||||
matchedFilter = match
|
||||
versionsCount = 1
|
||||
// Include the latest version
|
||||
if matchedFilter.Purge.RetainVersions == 0 {
|
||||
toDel = append(toDel, expireObjInfo{
|
||||
ObjectInfo: result.Item,
|
||||
ExpireAll: true,
|
||||
})
|
||||
continue
|
||||
}
|
||||
} else if prevObj.Name == result.Item.Name {
|
||||
if matchedFilter.Purge.RetainVersions == 0 {
|
||||
continue // including latest version in toDel suffices, skipping other versions
|
||||
}
|
||||
versionsCount++
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
|
||||
if versionsCount <= matchedFilter.Purge.RetainVersions {
|
||||
continue // retain versions
|
||||
}
|
||||
toDel = append(toDel, expireObjInfo{
|
||||
ObjectInfo: result.Item,
|
||||
})
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case result, ok := <-results:
|
||||
if !ok {
|
||||
done = true
|
||||
break
|
||||
}
|
||||
if result.Err != nil {
|
||||
failed = true
|
||||
batchLogIf(ctx, result.Err)
|
||||
continue
|
||||
}
|
||||
if result.Item.DeleteMarker {
|
||||
deleteMarkerCountMap[result.Item.Name]++
|
||||
}
|
||||
// Apply filter to find the matching rule to apply expiry
|
||||
// actions accordingly.
|
||||
// nolint:gocritic
|
||||
if result.Item.IsLatest {
|
||||
var match BatchJobExpireFilter
|
||||
var found bool
|
||||
for _, rule := range r.Rules {
|
||||
if rule.Matches(result.Item, now) {
|
||||
match = rule
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
|
||||
if prevObj.Name != result.Item.Name {
|
||||
// switch the object
|
||||
pushToExpire()
|
||||
}
|
||||
|
||||
prevObj = result.Item
|
||||
matchedFilter = match
|
||||
versionsCount = 1
|
||||
// Include the latest version
|
||||
if matchedFilter.Purge.RetainVersions == 0 {
|
||||
toDel = append(toDel, expireObjInfo{
|
||||
ObjectInfo: result.Item,
|
||||
ExpireAll: true,
|
||||
})
|
||||
continue
|
||||
}
|
||||
} else if prevObj.Name == result.Item.Name {
|
||||
if matchedFilter.Purge.RetainVersions == 0 {
|
||||
continue // including latest version in toDel suffices, skipping other versions
|
||||
}
|
||||
versionsCount++
|
||||
} else {
|
||||
// switch the object
|
||||
pushToExpire()
|
||||
// a file switched with no LatestVersion, logging it
|
||||
batchLogIf(ctx, fmt.Errorf("skipping object %s, no latest version found", result.Item.Name))
|
||||
continue
|
||||
}
|
||||
|
||||
if versionsCount <= matchedFilter.Purge.RetainVersions {
|
||||
continue // retain versions
|
||||
}
|
||||
toDel = append(toDel, expireObjInfo{
|
||||
ObjectInfo: result.Item,
|
||||
})
|
||||
pushToExpire()
|
||||
case <-ctx.Done():
|
||||
done = true
|
||||
}
|
||||
if done {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if context.Cause(ctx) != nil {
|
||||
xioutil.SafeClose(expireCh)
|
||||
return context.Cause(ctx)
|
||||
}
|
||||
pushToExpire()
|
||||
// Send any remaining objects downstream
|
||||
if len(toDel) > 0 {
|
||||
select {
|
||||
|
||||
+169
-105
@@ -25,6 +25,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -39,7 +40,6 @@ import (
|
||||
"github.com/lithammer/shortuuid/v4"
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7"
|
||||
miniogo "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio-go/v7/pkg/encrypt"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
@@ -47,7 +47,6 @@ import (
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/ioutil"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/pkg/v3/console"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
@@ -61,6 +60,8 @@ var globalBatchConfig batch.Config
|
||||
const (
|
||||
// Keep the completed/failed job stats 3 days before removing it
|
||||
oldJobsExpiration = 3 * 24 * time.Hour
|
||||
|
||||
redactedText = "**REDACTED**"
|
||||
)
|
||||
|
||||
// BatchJobRequest this is an internal data structure not for external consumption.
|
||||
@@ -74,6 +75,29 @@ type BatchJobRequest struct {
|
||||
ctx context.Context `msg:"-"`
|
||||
}
|
||||
|
||||
// RedactSensitive will redact any sensitive information in b.
|
||||
func (j *BatchJobRequest) RedactSensitive() {
|
||||
j.Replicate.RedactSensitive()
|
||||
j.Expire.RedactSensitive()
|
||||
j.KeyRotate.RedactSensitive()
|
||||
}
|
||||
|
||||
// RedactSensitive will redact any sensitive information in b.
|
||||
func (r *BatchJobReplicateV1) RedactSensitive() {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
if r.Target.Creds.SecretKey != "" {
|
||||
r.Target.Creds.SecretKey = redactedText
|
||||
}
|
||||
if r.Target.Creds.SessionToken != "" {
|
||||
r.Target.Creds.SessionToken = redactedText
|
||||
}
|
||||
}
|
||||
|
||||
// RedactSensitive will redact any sensitive information in b.
|
||||
func (r *BatchJobKeyRotateV1) RedactSensitive() {}
|
||||
|
||||
func notifyEndpoint(ctx context.Context, ri *batchJobInfo, endpoint, token string) error {
|
||||
if endpoint == "" {
|
||||
return nil
|
||||
@@ -117,7 +141,7 @@ func (r BatchJobReplicateV1) Notify(ctx context.Context, ri *batchJobInfo) error
|
||||
}
|
||||
|
||||
// ReplicateFromSource - this is not implemented yet where source is 'remote' and target is local.
|
||||
func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api ObjectLayer, core *miniogo.Core, srcObjInfo ObjectInfo, retry bool) error {
|
||||
func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api ObjectLayer, core *minio.Core, srcObjInfo ObjectInfo, retry bool) error {
|
||||
srcBucket := r.Source.Bucket
|
||||
tgtBucket := r.Target.Bucket
|
||||
srcObject := srcObjInfo.Name
|
||||
@@ -164,7 +188,7 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
|
||||
}
|
||||
return r.copyWithMultipartfromSource(ctx, api, core, srcObjInfo, opts, partsCount)
|
||||
}
|
||||
gopts := miniogo.GetObjectOptions{
|
||||
gopts := minio.GetObjectOptions{
|
||||
VersionID: srcObjInfo.VersionID,
|
||||
}
|
||||
if err := gopts.SetMatchETag(srcObjInfo.ETag); err != nil {
|
||||
@@ -185,7 +209,7 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, api ObjectLayer, c *miniogo.Core, srcObjInfo ObjectInfo, opts ObjectOptions, partsCount int) (err error) {
|
||||
func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, api ObjectLayer, c *minio.Core, srcObjInfo ObjectInfo, opts ObjectOptions, partsCount int) (err error) {
|
||||
srcBucket := r.Source.Bucket
|
||||
tgtBucket := r.Target.Bucket
|
||||
srcObject := srcObjInfo.Name
|
||||
@@ -225,8 +249,8 @@ func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, a
|
||||
pInfo PartInfo
|
||||
)
|
||||
|
||||
for i := 0; i < partsCount; i++ {
|
||||
gopts := miniogo.GetObjectOptions{
|
||||
for i := range partsCount {
|
||||
gopts := minio.GetObjectOptions{
|
||||
VersionID: srcObjInfo.VersionID,
|
||||
PartNumber: i + 1,
|
||||
}
|
||||
@@ -276,8 +300,12 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
|
||||
}
|
||||
globalBatchJobsMetrics.save(job.ID, ri)
|
||||
|
||||
retryAttempts := job.Replicate.Flags.Retry.Attempts
|
||||
if retryAttempts <= 0 {
|
||||
retryAttempts = batchReplJobDefaultRetries
|
||||
}
|
||||
delay := job.Replicate.Flags.Retry.Delay
|
||||
if delay == 0 {
|
||||
if delay <= 0 {
|
||||
delay = batchReplJobDefaultRetryDelay
|
||||
}
|
||||
rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
@@ -297,12 +325,12 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
|
||||
return true
|
||||
}
|
||||
|
||||
if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.Before(oi.ModTime) {
|
||||
if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.After(oi.ModTime) {
|
||||
// skip all objects that are created before the specified time.
|
||||
return true
|
||||
}
|
||||
|
||||
if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.After(oi.ModTime) {
|
||||
if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.Before(oi.ModTime) {
|
||||
// skip all objects that are created after the specified time.
|
||||
return true
|
||||
}
|
||||
@@ -353,7 +381,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
|
||||
|
||||
cred := r.Source.Creds
|
||||
|
||||
c, err := miniogo.New(u.Host, &miniogo.Options{
|
||||
c, err := minio.New(u.Host, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
|
||||
Secure: u.Scheme == "https",
|
||||
Transport: getRemoteInstanceTransport(),
|
||||
@@ -364,7 +392,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
|
||||
}
|
||||
|
||||
c.SetAppInfo("minio-"+batchJobPrefix, r.APIVersion+" "+job.ID)
|
||||
core := &miniogo.Core{Client: c}
|
||||
core := &minio.Core{Client: c}
|
||||
|
||||
workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_REPLICATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
|
||||
if err != nil {
|
||||
@@ -377,7 +405,6 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
|
||||
return err
|
||||
}
|
||||
|
||||
retryAttempts := ri.RetryAttempts
|
||||
retry := false
|
||||
for attempts := 1; attempts <= retryAttempts; attempts++ {
|
||||
attempts := attempts
|
||||
@@ -386,15 +413,15 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
|
||||
minioSrc := r.Source.Type == BatchJobReplicateResourceMinIO
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
|
||||
objInfoCh := make(chan miniogo.ObjectInfo, 1)
|
||||
objInfoCh := make(chan minio.ObjectInfo, 1)
|
||||
go func() {
|
||||
prefixes := r.Source.Prefix.F()
|
||||
if len(prefixes) == 0 {
|
||||
prefixes = []string{""}
|
||||
}
|
||||
for _, prefix := range prefixes {
|
||||
prefixObjInfoCh := c.ListObjects(ctx, r.Source.Bucket, miniogo.ListObjectsOptions{
|
||||
Prefix: strings.TrimSpace(prefix),
|
||||
prefixObjInfoCh := c.ListObjects(ctx, r.Source.Bucket, minio.ListObjectsOptions{
|
||||
Prefix: prefix,
|
||||
WithVersions: minioSrc,
|
||||
Recursive: true,
|
||||
WithMetadata: true,
|
||||
@@ -416,7 +443,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
|
||||
// all user metadata or just storageClass. If its only storageClass
|
||||
// List() already returns relevant information for filter to be applied.
|
||||
if isMetadata && !isStorageClassOnly {
|
||||
oi2, err := c.StatObject(ctx, r.Source.Bucket, obj.Key, miniogo.StatObjectOptions{})
|
||||
oi2, err := c.StatObject(ctx, r.Source.Bucket, obj.Key, minio.StatObjectOptions{})
|
||||
if err == nil {
|
||||
oi = toObjectInfo(r.Source.Bucket, obj.Key, oi2)
|
||||
} else {
|
||||
@@ -512,7 +539,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
|
||||
}
|
||||
|
||||
// toObjectInfo converts minio.ObjectInfo to ObjectInfo
|
||||
func toObjectInfo(bucket, object string, objInfo miniogo.ObjectInfo) ObjectInfo {
|
||||
func toObjectInfo(bucket, object string, objInfo minio.ObjectInfo) ObjectInfo {
|
||||
tags, _ := tags.MapToObjectTags(objInfo.UserTags)
|
||||
oi := ObjectInfo{
|
||||
Bucket: bucket,
|
||||
@@ -548,9 +575,7 @@ func toObjectInfo(bucket, object string, objInfo miniogo.ObjectInfo) ObjectInfo
|
||||
oi.UserDefined[xhttp.AmzStorageClass] = objInfo.StorageClass
|
||||
}
|
||||
|
||||
for k, v := range objInfo.UserMetadata {
|
||||
oi.UserDefined[k] = v
|
||||
}
|
||||
maps.Copy(oi.UserDefined, objInfo.UserMetadata)
|
||||
|
||||
return oi
|
||||
}
|
||||
@@ -594,12 +619,12 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa
|
||||
},
|
||||
}
|
||||
|
||||
opts, err := batchReplicationOpts(ctx, "", gr.ObjInfo)
|
||||
opts, _, err := batchReplicationOpts(ctx, "", gr.ObjInfo)
|
||||
if err != nil {
|
||||
batchLogIf(ctx, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// TODO: I am not sure we read it back, but we aren't sending whether checksums are single/multipart.
|
||||
for k, vals := range opts.Header() {
|
||||
for _, v := range vals {
|
||||
snowballObj.Headers.Add(k, v)
|
||||
@@ -615,7 +640,7 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa
|
||||
}
|
||||
|
||||
// ReplicateToTarget read from source and replicate to configured target
|
||||
func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectLayer, c *miniogo.Core, srcObjInfo ObjectInfo, retry bool) error {
|
||||
func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectLayer, c *minio.Core, srcObjInfo ObjectInfo, retry bool) error {
|
||||
srcBucket := r.Source.Bucket
|
||||
tgtBucket := r.Target.Bucket
|
||||
tgtPrefix := r.Target.Prefix
|
||||
@@ -624,9 +649,9 @@ func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectL
|
||||
|
||||
if srcObjInfo.DeleteMarker || !srcObjInfo.VersionPurgeStatus.Empty() {
|
||||
if retry && !s3Type {
|
||||
if _, err := c.StatObject(ctx, tgtBucket, pathJoin(tgtPrefix, srcObject), miniogo.StatObjectOptions{
|
||||
if _, err := c.StatObject(ctx, tgtBucket, pathJoin(tgtPrefix, srcObject), minio.StatObjectOptions{
|
||||
VersionID: srcObjInfo.VersionID,
|
||||
Internal: miniogo.AdvancedGetOptions{
|
||||
Internal: minio.AdvancedGetOptions{
|
||||
ReplicationProxyRequest: "false",
|
||||
},
|
||||
}); isErrMethodNotAllowed(ErrorRespToObjectError(err, tgtBucket, pathJoin(tgtPrefix, srcObject))) {
|
||||
@@ -643,19 +668,19 @@ func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectL
|
||||
dmVersionID = ""
|
||||
versionID = ""
|
||||
}
|
||||
return c.RemoveObject(ctx, tgtBucket, pathJoin(tgtPrefix, srcObject), miniogo.RemoveObjectOptions{
|
||||
return c.RemoveObject(ctx, tgtBucket, pathJoin(tgtPrefix, srcObject), minio.RemoveObjectOptions{
|
||||
VersionID: versionID,
|
||||
Internal: miniogo.AdvancedRemoveOptions{
|
||||
Internal: minio.AdvancedRemoveOptions{
|
||||
ReplicationDeleteMarker: dmVersionID != "",
|
||||
ReplicationMTime: srcObjInfo.ModTime,
|
||||
ReplicationStatus: miniogo.ReplicationStatusReplica,
|
||||
ReplicationStatus: minio.ReplicationStatusReplica,
|
||||
ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
if retry && !s3Type { // when we are retrying avoid copying if necessary.
|
||||
gopts := miniogo.GetObjectOptions{}
|
||||
gopts := minio.GetObjectOptions{}
|
||||
if err := gopts.SetMatchETag(srcObjInfo.ETag); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -684,14 +709,14 @@ func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectL
|
||||
return err
|
||||
}
|
||||
|
||||
putOpts, err := batchReplicationOpts(ctx, "", objInfo)
|
||||
putOpts, isMP, err := batchReplicationOpts(ctx, "", objInfo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
|
||||
putOpts.Internal = miniogo.AdvancedPutOptions{}
|
||||
putOpts.Internal = minio.AdvancedPutOptions{}
|
||||
}
|
||||
if objInfo.isMultipart() {
|
||||
if isMP {
|
||||
if err := replicateObjectWithMultipart(ctx, c, tgtBucket, pathJoin(tgtPrefix, objInfo.Name), rd, objInfo, putOpts); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -772,22 +797,10 @@ func (ri *batchJobInfo) loadOrInit(ctx context.Context, api ObjectLayer, job Bat
|
||||
switch {
|
||||
case job.Replicate != nil:
|
||||
ri.Version = batchReplVersionV1
|
||||
ri.RetryAttempts = batchReplJobDefaultRetries
|
||||
if job.Replicate.Flags.Retry.Attempts > 0 {
|
||||
ri.RetryAttempts = job.Replicate.Flags.Retry.Attempts
|
||||
}
|
||||
case job.KeyRotate != nil:
|
||||
ri.Version = batchKeyRotateVersionV1
|
||||
ri.RetryAttempts = batchKeyRotateJobDefaultRetries
|
||||
if job.KeyRotate.Flags.Retry.Attempts > 0 {
|
||||
ri.RetryAttempts = job.KeyRotate.Flags.Retry.Attempts
|
||||
}
|
||||
case job.Expire != nil:
|
||||
ri.Version = batchExpireVersionV1
|
||||
ri.RetryAttempts = batchExpireJobDefaultRetries
|
||||
if job.Expire.Retry.Attempts > 0 {
|
||||
ri.RetryAttempts = job.Expire.Retry.Attempts
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -867,21 +880,23 @@ func (ri *batchJobInfo) clone() *batchJobInfo {
|
||||
defer ri.mu.RUnlock()
|
||||
|
||||
return &batchJobInfo{
|
||||
Version: ri.Version,
|
||||
JobID: ri.JobID,
|
||||
JobType: ri.JobType,
|
||||
RetryAttempts: ri.RetryAttempts,
|
||||
Complete: ri.Complete,
|
||||
Failed: ri.Failed,
|
||||
StartTime: ri.StartTime,
|
||||
LastUpdate: ri.LastUpdate,
|
||||
Bucket: ri.Bucket,
|
||||
Object: ri.Object,
|
||||
Objects: ri.Objects,
|
||||
ObjectsFailed: ri.ObjectsFailed,
|
||||
BytesTransferred: ri.BytesTransferred,
|
||||
BytesFailed: ri.BytesFailed,
|
||||
Attempts: ri.Attempts,
|
||||
Version: ri.Version,
|
||||
JobID: ri.JobID,
|
||||
JobType: ri.JobType,
|
||||
RetryAttempts: ri.RetryAttempts,
|
||||
Complete: ri.Complete,
|
||||
Failed: ri.Failed,
|
||||
StartTime: ri.StartTime,
|
||||
LastUpdate: ri.LastUpdate,
|
||||
Bucket: ri.Bucket,
|
||||
Object: ri.Object,
|
||||
Objects: ri.Objects,
|
||||
ObjectsFailed: ri.ObjectsFailed,
|
||||
DeleteMarkers: ri.DeleteMarkers,
|
||||
DeleteMarkersFailed: ri.DeleteMarkersFailed,
|
||||
BytesTransferred: ri.BytesTransferred,
|
||||
BytesFailed: ri.BytesFailed,
|
||||
Attempts: ri.Attempts,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -980,11 +995,22 @@ func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, durati
|
||||
// Note: to be used only with batch jobs that affect multiple versions through
|
||||
// a single action. e.g batch-expire has an option to expire all versions of an
|
||||
// object which matches the given filters.
|
||||
func (ri *batchJobInfo) trackMultipleObjectVersions(bucket string, info ObjectInfo, success bool) {
|
||||
func (ri *batchJobInfo) trackMultipleObjectVersions(info expireObjInfo, success bool) {
|
||||
if ri == nil {
|
||||
return
|
||||
}
|
||||
|
||||
ri.mu.Lock()
|
||||
defer ri.mu.Unlock()
|
||||
|
||||
if success {
|
||||
ri.Objects += int64(info.NumVersions)
|
||||
ri.Bucket = info.Bucket
|
||||
ri.Object = info.Name
|
||||
ri.Objects += int64(info.NumVersions) - info.DeleteMarkerCount
|
||||
ri.DeleteMarkers += info.DeleteMarkerCount
|
||||
} else {
|
||||
ri.ObjectsFailed += int64(info.NumVersions)
|
||||
ri.ObjectsFailed += int64(info.NumVersions) - info.DeleteMarkerCount
|
||||
ri.DeleteMarkersFailed += info.DeleteMarkerCount
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1034,10 +1060,15 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
globalBatchJobsMetrics.save(job.ID, ri)
|
||||
lastObject := ri.Object
|
||||
|
||||
retryAttempts := job.Replicate.Flags.Retry.Attempts
|
||||
if retryAttempts <= 0 {
|
||||
retryAttempts = batchReplJobDefaultRetries
|
||||
}
|
||||
delay := job.Replicate.Flags.Retry.Delay
|
||||
if delay < time.Second {
|
||||
if delay <= 0 {
|
||||
delay = batchReplJobDefaultRetryDelay
|
||||
}
|
||||
|
||||
rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
selectObj := func(info FileInfo) (ok bool) {
|
||||
@@ -1051,12 +1082,12 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
return false
|
||||
}
|
||||
|
||||
if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.Before(info.ModTime) {
|
||||
if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.After(info.ModTime) {
|
||||
// skip all objects that are created before the specified time.
|
||||
return false
|
||||
}
|
||||
|
||||
if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.After(info.ModTime) {
|
||||
if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.Before(info.ModTime) {
|
||||
// skip all objects that are created after the specified time.
|
||||
return false
|
||||
}
|
||||
@@ -1103,7 +1134,8 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
}
|
||||
|
||||
// if one of source or target is non MinIO, just replicate the top most version like `mc mirror`
|
||||
return !((r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3) && !info.IsLatest)
|
||||
isSourceOrTargetS3 := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3
|
||||
return !isSourceOrTargetS3 || info.IsLatest
|
||||
}
|
||||
|
||||
u, err := url.Parse(r.Target.Endpoint)
|
||||
@@ -1113,7 +1145,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
|
||||
cred := r.Target.Creds
|
||||
|
||||
c, err := miniogo.NewCore(u.Host, &miniogo.Options{
|
||||
c, err := minio.NewCore(u.Host, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
|
||||
Secure: u.Scheme == "https",
|
||||
Transport: getRemoteInstanceTransport(),
|
||||
@@ -1125,7 +1157,6 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
|
||||
c.SetAppInfo("minio-"+batchJobPrefix, r.APIVersion+" "+job.ID)
|
||||
|
||||
retryAttempts := ri.RetryAttempts
|
||||
retry := false
|
||||
for attempts := 1; attempts <= retryAttempts; attempts++ {
|
||||
attempts := attempts
|
||||
@@ -1137,14 +1168,14 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
if r.Source.Snowball.Disable != nil && !*r.Source.Snowball.Disable && r.Source.Type.isMinio() && r.Target.Type.isMinio() {
|
||||
go func() {
|
||||
// Snowball currently needs the high level minio-go Client, not the Core one
|
||||
cl, err := miniogo.New(u.Host, &miniogo.Options{
|
||||
cl, err := minio.New(u.Host, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
|
||||
Secure: u.Scheme == "https",
|
||||
Transport: getRemoteInstanceTransport(),
|
||||
BucketLookup: lookupStyle(r.Target.Path),
|
||||
})
|
||||
if err != nil {
|
||||
batchLogOnceIf(ctx, err, job.ID+"miniogo.New")
|
||||
batchLogOnceIf(ctx, err, job.ID+"minio.New")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1214,7 +1245,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
}
|
||||
for _, prefix := range prefixes {
|
||||
prefixWalkCh := make(chan itemOrErr[ObjectInfo], 100)
|
||||
if err := api.Walk(ctx, r.Source.Bucket, strings.TrimSpace(prefix), prefixWalkCh, WalkOptions{
|
||||
if err := api.Walk(ctx, r.Source.Bucket, prefix, prefixWalkCh, WalkOptions{
|
||||
Marker: lastObject,
|
||||
Filter: selectObj,
|
||||
AskDisks: walkQuorum,
|
||||
@@ -1254,7 +1285,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
stopFn := globalBatchJobsMetrics.trace(batchJobMetricReplication, job.ID, attempts)
|
||||
success := true
|
||||
if err := r.ReplicateToTarget(ctx, api, c, result, retry); err != nil {
|
||||
if miniogo.ToErrorResponse(err).Code == "PreconditionFailed" {
|
||||
if minio.ToErrorResponse(err).Code == "PreconditionFailed" {
|
||||
// pre-condition failed means we already have the object copied over.
|
||||
return
|
||||
}
|
||||
@@ -1430,7 +1461,6 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
|
||||
cred = r.Source.Creds
|
||||
remoteBkt = r.Source.Bucket
|
||||
pathStyle = r.Source.Path
|
||||
|
||||
}
|
||||
|
||||
u, err := url.Parse(remoteEp)
|
||||
@@ -1438,7 +1468,7 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
|
||||
return err
|
||||
}
|
||||
|
||||
c, err := miniogo.NewCore(u.Host, &miniogo.Options{
|
||||
c, err := minio.NewCore(u.Host, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
|
||||
Secure: u.Scheme == "https",
|
||||
Transport: getRemoteInstanceTransport(),
|
||||
@@ -1451,7 +1481,7 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
|
||||
|
||||
vcfg, err := c.GetBucketVersioning(ctx, remoteBkt)
|
||||
if err != nil {
|
||||
if miniogo.ToErrorResponse(err).Code == "NoSuchBucket" {
|
||||
if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
|
||||
return batchReplicationJobError{
|
||||
Code: "NoSuchTargetBucket",
|
||||
Description: "The specified target bucket does not exist",
|
||||
@@ -1556,19 +1586,19 @@ func (j *BatchJobRequest) load(ctx context.Context, api ObjectLayer, name string
|
||||
return err
|
||||
}
|
||||
|
||||
func batchReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions, err error) {
|
||||
func batchReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts minio.PutObjectOptions, isMP bool, err error) {
|
||||
// TODO: support custom storage class for remote replication
|
||||
putOpts, err = putReplicationOpts(ctx, "", objInfo, 0)
|
||||
putOpts, isMP, err = putReplicationOpts(ctx, "", objInfo)
|
||||
if err != nil {
|
||||
return putOpts, err
|
||||
return putOpts, isMP, err
|
||||
}
|
||||
putOpts.Internal = miniogo.AdvancedPutOptions{
|
||||
putOpts.Internal = minio.AdvancedPutOptions{
|
||||
SourceVersionID: objInfo.VersionID,
|
||||
SourceMTime: objInfo.ModTime,
|
||||
SourceETag: objInfo.ETag,
|
||||
ReplicationRequest: true,
|
||||
}
|
||||
return putOpts, nil
|
||||
return putOpts, isMP, nil
|
||||
}
|
||||
|
||||
// ListBatchJobs - lists all currently active batch jobs, optionally takes {jobType}
|
||||
@@ -1582,9 +1612,6 @@ func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
|
||||
jobType := r.Form.Get("jobType")
|
||||
if jobType == "" {
|
||||
jobType = string(madmin.BatchJobReplicate)
|
||||
}
|
||||
|
||||
resultCh := make(chan itemOrErr[ObjectInfo])
|
||||
|
||||
@@ -1613,7 +1640,7 @@ func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request)
|
||||
continue
|
||||
}
|
||||
|
||||
if jobType == string(req.Type()) {
|
||||
if jobType == string(req.Type()) || jobType == "" {
|
||||
listResult.Jobs = append(listResult.Jobs, madmin.BatchJobResult{
|
||||
ID: req.ID,
|
||||
Type: req.Type(),
|
||||
@@ -1703,6 +1730,8 @@ func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Reques
|
||||
return
|
||||
}
|
||||
|
||||
// Remove sensitive fields.
|
||||
req.RedactSensitive()
|
||||
buf, err := yaml.Marshal(req)
|
||||
if err != nil {
|
||||
batchLogIf(ctx, err)
|
||||
@@ -1722,7 +1751,7 @@ func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
|
||||
buf, err := io.ReadAll(ioutil.HardLimitReader(r.Body, humanize.MiByte*4))
|
||||
buf, err := io.ReadAll(xioutil.HardLimitReader(r.Body, humanize.MiByte*4))
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -1810,7 +1839,7 @@ func (a adminAPIHandlers) CancelBatchJob(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
|
||||
if _, success := proxyRequestByToken(ctx, w, r, jobID); success {
|
||||
if _, proxied, _ := proxyRequestByToken(ctx, w, r, jobID, true); proxied {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1994,7 +2023,6 @@ func (j *BatchJobPool) AddWorker() {
|
||||
}
|
||||
}
|
||||
}
|
||||
job.delete(j.ctx, j.objLayer)
|
||||
j.canceler(job.ID, false)
|
||||
case <-j.workerKillCh:
|
||||
return
|
||||
@@ -2055,7 +2083,9 @@ func (j *BatchJobPool) canceler(jobID string, cancel bool) error {
|
||||
canceler()
|
||||
}
|
||||
}
|
||||
delete(j.jobCancelers, jobID)
|
||||
if cancel {
|
||||
delete(j.jobCancelers, jobID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -2116,12 +2146,14 @@ func (ri *batchJobInfo) metric() madmin.JobMetric {
|
||||
switch ri.JobType {
|
||||
case string(madmin.BatchJobReplicate):
|
||||
m.Replicate = &madmin.ReplicateInfo{
|
||||
Bucket: ri.Bucket,
|
||||
Object: ri.Object,
|
||||
Objects: ri.Objects,
|
||||
ObjectsFailed: ri.ObjectsFailed,
|
||||
BytesTransferred: ri.BytesTransferred,
|
||||
BytesFailed: ri.BytesFailed,
|
||||
Bucket: ri.Bucket,
|
||||
Object: ri.Object,
|
||||
Objects: ri.Objects,
|
||||
DeleteMarkers: ri.DeleteMarkers,
|
||||
ObjectsFailed: ri.ObjectsFailed,
|
||||
DeleteMarkersFailed: ri.DeleteMarkersFailed,
|
||||
BytesTransferred: ri.BytesTransferred,
|
||||
BytesFailed: ri.BytesFailed,
|
||||
}
|
||||
case string(madmin.BatchJobKeyRotate):
|
||||
m.KeyRotate = &madmin.KeyRotationInfo{
|
||||
@@ -2132,10 +2164,12 @@ func (ri *batchJobInfo) metric() madmin.JobMetric {
|
||||
}
|
||||
case string(madmin.BatchJobExpire):
|
||||
m.Expired = &madmin.ExpirationInfo{
|
||||
Bucket: ri.Bucket,
|
||||
Object: ri.Object,
|
||||
Objects: ri.Objects,
|
||||
ObjectsFailed: ri.ObjectsFailed,
|
||||
Bucket: ri.Bucket,
|
||||
Object: ri.Object,
|
||||
Objects: ri.Objects,
|
||||
DeleteMarkers: ri.DeleteMarkers,
|
||||
ObjectsFailed: ri.ObjectsFailed,
|
||||
DeleteMarkersFailed: ri.DeleteMarkersFailed,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2181,11 +2215,42 @@ func (m *batchJobMetrics) purgeJobMetrics() {
|
||||
m.RUnlock()
|
||||
for _, jobID := range toDeleteJobMetrics {
|
||||
m.delete(jobID)
|
||||
j := BatchJobRequest{
|
||||
ID: jobID,
|
||||
}
|
||||
j.delete(GlobalContext, newObjectLayerFn())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// load metrics from disk on startup
|
||||
func (m *batchJobMetrics) init(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
resultCh := make(chan itemOrErr[ObjectInfo])
|
||||
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
if err := objectAPI.Walk(ctx, minioMetaBucket, batchJobReportsPrefix, resultCh, WalkOptions{}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for result := range resultCh {
|
||||
if result.Err != nil {
|
||||
return result.Err
|
||||
}
|
||||
ri := &batchJobInfo{}
|
||||
if err := ri.loadByPath(ctx, objectAPI, result.Item.Name); err != nil {
|
||||
if !errors.Is(err, errNoSuchJob) {
|
||||
batchLogIf(ctx, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
m.metrics[ri.JobID] = ri
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *batchJobMetrics) delete(jobID string) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
@@ -2250,16 +2315,15 @@ func (m *batchJobMetrics) trace(d batchJobMetric, job string, attempts int) func
|
||||
}
|
||||
}
|
||||
|
||||
func lookupStyle(s string) miniogo.BucketLookupType {
|
||||
var lookup miniogo.BucketLookupType
|
||||
func lookupStyle(s string) minio.BucketLookupType {
|
||||
var lookup minio.BucketLookupType
|
||||
switch s {
|
||||
case "on":
|
||||
lookup = miniogo.BucketLookupPath
|
||||
lookup = minio.BucketLookupPath
|
||||
case "off":
|
||||
lookup = miniogo.BucketLookupDNS
|
||||
lookup = minio.BucketLookupDNS
|
||||
default:
|
||||
lookup = miniogo.BucketLookupAuto
|
||||
|
||||
lookup = minio.BucketLookupAuto
|
||||
}
|
||||
return lookup
|
||||
}
|
||||
|
||||
@@ -275,7 +275,7 @@ func (sf BatchJobSizeFilter) Validate() error {
|
||||
type BatchJobSize int64
|
||||
|
||||
// UnmarshalYAML to parse humanized byte values
|
||||
func (s *BatchJobSize) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
func (s *BatchJobSize) UnmarshalYAML(unmarshal func(any) error) error {
|
||||
var batchExpireSz string
|
||||
err := unmarshal(&batchExpireSz)
|
||||
if err != nil {
|
||||
|
||||
+8
-8
@@ -21,6 +21,7 @@ import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"maps"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"runtime"
|
||||
@@ -110,9 +111,7 @@ func (e BatchJobKeyRotateEncryption) Validate() error {
|
||||
}
|
||||
}
|
||||
e.kmsContext = kms.Context{}
|
||||
for k, v := range ctx {
|
||||
e.kmsContext[k] = v
|
||||
}
|
||||
maps.Copy(e.kmsContext, ctx)
|
||||
ctx["MinIO batch API"] = "batchrotate" // Context for a test key operation
|
||||
if _, err := GlobalKMS.GenerateKey(GlobalContext, &kms.GenerateKeyRequest{Name: e.Key, AssociatedData: ctx}); err != nil {
|
||||
return err
|
||||
@@ -225,9 +224,7 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
|
||||
// Since we are rotating the keys, make sure to update the metadata.
|
||||
oi.metadataOnly = true
|
||||
oi.keyRotation = true
|
||||
for k, v := range encMetadata {
|
||||
oi.UserDefined[k] = v
|
||||
}
|
||||
maps.Copy(oi.UserDefined, encMetadata)
|
||||
if _, err := api.CopyObject(ctx, r.Bucket, oi.Name, r.Bucket, oi.Name, oi, ObjectOptions{
|
||||
VersionID: oi.VersionID,
|
||||
}, ObjectOptions{
|
||||
@@ -267,8 +264,12 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
globalBatchJobsMetrics.save(job.ID, ri)
|
||||
lastObject := ri.Object
|
||||
|
||||
retryAttempts := job.KeyRotate.Flags.Retry.Attempts
|
||||
if retryAttempts <= 0 {
|
||||
retryAttempts = batchKeyRotateJobDefaultRetries
|
||||
}
|
||||
delay := job.KeyRotate.Flags.Retry.Delay
|
||||
if delay == 0 {
|
||||
if delay <= 0 {
|
||||
delay = batchKeyRotateJobDefaultRetryDelay
|
||||
}
|
||||
|
||||
@@ -354,7 +355,6 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
return err
|
||||
}
|
||||
|
||||
retryAttempts := ri.RetryAttempts
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
|
||||
results := make(chan itemOrErr[ObjectInfo], 100)
|
||||
|
||||
+15
-15
@@ -35,7 +35,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// obtains random bucket name.
|
||||
bucket := getRandomBucketName()
|
||||
// create bucket.
|
||||
err = obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{})
|
||||
err = obj.MakeBucket(b.Context(), bucket, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -51,10 +51,10 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
|
||||
b.ReportAllocs()
|
||||
// the actual benchmark for PutObject starts here. Reset the benchmark timer.
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
|
||||
for i := 0; b.Loop(); i++ {
|
||||
// insert the object.
|
||||
objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
|
||||
objInfo, err := obj.PutObject(b.Context(), bucket, "object"+strconv.Itoa(i),
|
||||
mustGetPutObjReader(b, bytes.NewReader(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
@@ -76,7 +76,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
|
||||
object := getRandomObjectName()
|
||||
|
||||
// create bucket.
|
||||
err = obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{})
|
||||
err = obj.MakeBucket(b.Context(), bucket, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -90,7 +90,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
|
||||
textData := generateBytesData(objSize)
|
||||
// generate md5sum for the generated data.
|
||||
// md5sum of the data to written is required as input for NewMultipartUpload.
|
||||
res, err := obj.NewMultipartUpload(context.Background(), bucket, object, ObjectOptions{})
|
||||
res, err := obj.NewMultipartUpload(b.Context(), bucket, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -101,11 +101,11 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
|
||||
// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
|
||||
b.ReportAllocs()
|
||||
// the actual benchmark for PutObjectPart starts here. Reset the benchmark timer.
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
|
||||
for i := 0; b.Loop(); i++ {
|
||||
// insert the object.
|
||||
totalPartsNR := int(math.Ceil(float64(objSize) / float64(partSize)))
|
||||
for j := 0; j < totalPartsNR; j++ {
|
||||
for j := range totalPartsNR {
|
||||
if j < totalPartsNR-1 {
|
||||
textPartData = textData[j*partSize : (j+1)*partSize-1]
|
||||
} else {
|
||||
@@ -113,7 +113,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
|
||||
}
|
||||
md5hex := getMD5Hash(textPartData)
|
||||
var partInfo PartInfo
|
||||
partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, res.UploadID, j,
|
||||
partInfo, err = obj.PutObjectPart(b.Context(), bucket, object, res.UploadID, j,
|
||||
mustGetPutObjReader(b, bytes.NewReader(textPartData), int64(len(textPartData)), md5hex, sha256hex), ObjectOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
@@ -130,7 +130,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
|
||||
// creates Erasure/FS backend setup, obtains the object layer and calls the runPutObjectPartBenchmark function.
|
||||
func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
|
||||
// create a temp Erasure/FS backend.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
ctx, cancel := context.WithCancel(b.Context())
|
||||
defer cancel()
|
||||
objLayer, disks, err := prepareTestBackend(ctx, instanceType)
|
||||
if err != nil {
|
||||
@@ -146,7 +146,7 @@ func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
|
||||
// creates Erasure/FS backend setup, obtains the object layer and calls the runPutObjectBenchmark function.
|
||||
func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
|
||||
// create a temp Erasure/FS backend.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
ctx, cancel := context.WithCancel(b.Context())
|
||||
defer cancel()
|
||||
objLayer, disks, err := prepareTestBackend(ctx, instanceType)
|
||||
if err != nil {
|
||||
@@ -162,7 +162,7 @@ func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
|
||||
// creates Erasure/FS backend setup, obtains the object layer and runs parallel benchmark for put object.
|
||||
func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int) {
|
||||
// create a temp Erasure/FS backend.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
ctx, cancel := context.WithCancel(b.Context())
|
||||
defer cancel()
|
||||
objLayer, disks, err := prepareTestBackend(ctx, instanceType)
|
||||
if err != nil {
|
||||
@@ -196,7 +196,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// obtains random bucket name.
|
||||
bucket := getRandomBucketName()
|
||||
// create bucket.
|
||||
err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{})
|
||||
err := obj.MakeBucket(b.Context(), bucket, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -218,7 +218,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
i := 0
|
||||
for pb.Next() {
|
||||
// insert the object.
|
||||
objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
|
||||
objInfo, err := obj.PutObject(b.Context(), bucket, "object"+strconv.Itoa(i),
|
||||
mustGetPutObjReader(b, bytes.NewReader(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
|
||||
@@ -20,6 +20,7 @@ package cmd
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"hash"
|
||||
"io"
|
||||
"sync"
|
||||
@@ -37,12 +38,22 @@ type streamingBitrotWriter struct {
|
||||
shardSize int64
|
||||
canClose *sync.WaitGroup
|
||||
byteBuf []byte
|
||||
finished bool
|
||||
}
|
||||
|
||||
func (b *streamingBitrotWriter) Write(p []byte) (int, error) {
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if b.finished {
|
||||
return 0, errors.New("bitrot write not allowed")
|
||||
}
|
||||
if int64(len(p)) > b.shardSize {
|
||||
return 0, errors.New("unexpected bitrot buffer size")
|
||||
}
|
||||
if int64(len(p)) < b.shardSize {
|
||||
b.finished = true
|
||||
}
|
||||
b.h.Reset()
|
||||
b.h.Write(p)
|
||||
hashBytes := b.h.Sum(nil)
|
||||
|
||||
+13
-7
@@ -128,14 +128,20 @@ func closeBitrotReaders(rs []io.ReaderAt) {
|
||||
}
|
||||
|
||||
// Close all the writers.
|
||||
func closeBitrotWriters(ws []io.Writer) {
|
||||
for _, w := range ws {
|
||||
if w != nil {
|
||||
if bw, ok := w.(io.Closer); ok {
|
||||
bw.Close()
|
||||
}
|
||||
func closeBitrotWriters(ws []io.Writer) []error {
|
||||
errs := make([]error, len(ws))
|
||||
for i, w := range ws {
|
||||
if w == nil {
|
||||
errs[i] = errDiskNotFound
|
||||
continue
|
||||
}
|
||||
if bw, ok := w.(io.Closer); ok {
|
||||
errs[i] = bw.Close()
|
||||
} else {
|
||||
errs[i] = nil
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
// Returns hash sum for whole-bitrot, nil for streaming-bitrot.
|
||||
@@ -178,7 +184,7 @@ func bitrotVerify(r io.Reader, wantSize, partSize int64, algo BitrotAlgorithm, w
|
||||
return errFileCorrupt
|
||||
}
|
||||
|
||||
bufp := xioutil.ODirectPoolSmall.Get().(*[]byte)
|
||||
bufp := xioutil.ODirectPoolSmall.Get()
|
||||
defer xioutil.ODirectPoolSmall.Put(bufp)
|
||||
|
||||
for left > 0 {
|
||||
|
||||
+1
-2
@@ -18,7 +18,6 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
)
|
||||
@@ -34,7 +33,7 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
disk.MakeVol(context.Background(), volume)
|
||||
disk.MakeVol(t.Context(), volume)
|
||||
|
||||
writer := newBitrotWriter(disk, "", volume, filePath, 35, bitrotAlgo, 10)
|
||||
|
||||
|
||||
@@ -48,9 +48,7 @@ func (bs *bootstrapTracer) Events() []madmin.TraceInfo {
|
||||
traceInfo := make([]madmin.TraceInfo, 0, bootstrapTraceLimit)
|
||||
|
||||
bs.mu.RLock()
|
||||
for _, i := range bs.info {
|
||||
traceInfo = append(traceInfo, i)
|
||||
}
|
||||
traceInfo = append(traceInfo, bs.info...)
|
||||
bs.mu.RUnlock()
|
||||
|
||||
return traceInfo
|
||||
|
||||
@@ -183,11 +183,19 @@ var binaryChecksum = getBinaryChecksum()
|
||||
|
||||
func getBinaryChecksum() string {
|
||||
mw := md5.New()
|
||||
b, err := os.Open(os.Args[0])
|
||||
if err == nil {
|
||||
defer b.Close()
|
||||
io.Copy(mw, b)
|
||||
binPath, err := os.Executable()
|
||||
if err != nil {
|
||||
logger.Error("Calculating checksum failed: %s", err)
|
||||
return "00000000000000000000000000000000"
|
||||
}
|
||||
b, err := os.Open(binPath)
|
||||
if err != nil {
|
||||
logger.Error("Calculating checksum failed: %s", err)
|
||||
return "00000000000000000000000000000000"
|
||||
}
|
||||
|
||||
defer b.Close()
|
||||
io.Copy(mw, b)
|
||||
return hex.EncodeToString(mw.Sum(nil))
|
||||
}
|
||||
|
||||
|
||||
+22
-14
@@ -154,7 +154,6 @@ func initFederatorBackend(buckets []string, objLayer ObjectLayer) {
|
||||
g := errgroup.WithNErrs(len(bucketsToBeUpdatedSlice)).WithConcurrency(50)
|
||||
|
||||
for index := range bucketsToBeUpdatedSlice {
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
return globalDNSConfig.Put(bucketsToBeUpdatedSlice[index])
|
||||
}, index)
|
||||
@@ -344,11 +343,9 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
Created: dnsRecords[0].CreationDate,
|
||||
})
|
||||
}
|
||||
|
||||
sort.Slice(bucketsInfo, func(i, j int) bool {
|
||||
return bucketsInfo[i].Name < bucketsInfo[j].Name
|
||||
})
|
||||
|
||||
} else {
|
||||
// Invoke the list buckets.
|
||||
var err error
|
||||
@@ -429,7 +426,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
|
||||
// Content-Md5 is required should be set
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
|
||||
if _, ok := r.Header[xhttp.ContentMD5]; !ok {
|
||||
if !validateLengthAndChecksum(r) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -561,7 +558,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
}, goi, opts, gerr)
|
||||
if dsc.ReplicateAny() {
|
||||
if object.VersionID != "" {
|
||||
object.VersionPurgeStatus = Pending
|
||||
object.VersionPurgeStatus = replication.VersionPurgePending
|
||||
object.VersionPurgeStatuses = dsc.PendingStatus()
|
||||
} else {
|
||||
object.DeleteMarkerReplicationStatus = dsc.PendingStatus()
|
||||
@@ -671,7 +668,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
continue
|
||||
}
|
||||
|
||||
if replicateDeletes && (dobj.DeleteMarkerReplicationStatus() == replication.Pending || dobj.VersionPurgeStatus() == Pending) {
|
||||
if replicateDeletes && (dobj.DeleteMarkerReplicationStatus() == replication.Pending || dobj.VersionPurgeStatus() == replication.VersionPurgePending) {
|
||||
// copy so we can re-add null ID.
|
||||
dobj := dobj
|
||||
if isDirObject(dobj.ObjectName) && dobj.VersionID == "" {
|
||||
@@ -841,7 +838,6 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
|
||||
}
|
||||
apiErr := ErrBucketAlreadyExists
|
||||
if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() {
|
||||
@@ -1092,6 +1088,14 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
break
|
||||
}
|
||||
|
||||
// check if have a file
|
||||
if reader == nil {
|
||||
apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
|
||||
apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, errors.New("The file or text content is missing"))
|
||||
writeErrorResponse(ctx, w, apiErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if keyName, ok := formValues["Key"]; !ok {
|
||||
apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
|
||||
apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, errors.New("The name of the uploaded key is missing"))
|
||||
@@ -1338,6 +1342,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
}
|
||||
opts.EncryptFn = metadataEncrypter(objectEncryptionKey)
|
||||
pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
@@ -1381,10 +1386,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
// Set the correct hex md5sum for the fan-out stream.
|
||||
fanOutOpts.MD5Hex = hex.EncodeToString(md5w.Sum(nil))
|
||||
|
||||
concurrentSize := 100
|
||||
if runtime.GOMAXPROCS(0) < concurrentSize {
|
||||
concurrentSize = runtime.GOMAXPROCS(0)
|
||||
}
|
||||
concurrentSize := min(runtime.GOMAXPROCS(0), 100)
|
||||
|
||||
fanOutResp := make([]minio.PutObjectFanOutResponse, 0, len(fanOutEntries))
|
||||
eventArgsList := make([]eventArgs, 0, len(fanOutEntries))
|
||||
@@ -1655,9 +1657,11 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error))
|
||||
return
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.HeadBucketAction, bucket, ""); s3Error != ErrNone {
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
getBucketInfo := objectAPI.GetBucketInfo
|
||||
@@ -1809,6 +1813,10 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
|
||||
return
|
||||
}
|
||||
|
||||
// Audit log tags.
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
reqInfo.SetTags("retention", config.String())
|
||||
|
||||
configData, err := xml.Marshal(config)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
|
||||
@@ -188,7 +188,6 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
|
||||
if errorResponse.Code != testCase.errorResponse.Code {
|
||||
t.Errorf("Test %d: %s: Expected the error code to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Code, errorResponse.Code)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Test for Anonymous/unsigned http request.
|
||||
@@ -290,7 +289,6 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
|
||||
if recV2.Code != testCase.expectedRespStatus {
|
||||
t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Test for Anonymous/unsigned http request.
|
||||
@@ -659,7 +657,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
|
||||
|
||||
sha256sum := ""
|
||||
var objectNames []string
|
||||
for i := 0; i < 10; i++ {
|
||||
for i := range 10 {
|
||||
contentBytes := []byte("hello")
|
||||
objectName := "test-object-" + strconv.Itoa(i)
|
||||
if i == 0 {
|
||||
@@ -689,7 +687,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
|
||||
|
||||
// The following block will create a bucket policy with delete object to 'public/*'. This is
|
||||
// to test a mixed response of a successful & failure while deleting objects in a single request
|
||||
policyBytes := []byte(fmt.Sprintf(`{"Id": "Policy1637752602639", "Version": "2012-10-17", "Statement": [{"Sid": "Stmt1637752600730", "Action": "s3:DeleteObject", "Effect": "Allow", "Resource": "arn:aws:s3:::%s/public/*", "Principal": "*"}]}`, bucketName))
|
||||
policyBytes := fmt.Appendf(nil, `{"Id": "Policy1637752602639", "Version": "2012-10-17", "Statement": [{"Sid": "Stmt1637752600730", "Action": "s3:DeleteObject", "Effect": "Allow", "Resource": "arn:aws:s3:::%s/public/*", "Principal": "*"}]}`, bucketName)
|
||||
rec := httptest.NewRecorder()
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, getPutPolicyURL("", bucketName), int64(len(policyBytes)), bytes.NewReader(policyBytes),
|
||||
credentials.AccessKey, credentials.SecretKey, nil)
|
||||
|
||||
@@ -26,7 +26,7 @@ import (
|
||||
//go:generate stringer -type lcEventSrc -trimprefix lcEventSrc_ $GOFILE
|
||||
type lcEventSrc uint8
|
||||
|
||||
//revive:disable:var-naming Underscores is used here to indicate where common prefix ends and the enumeration name begins
|
||||
//nolint:staticcheck,revive // Underscores are used here to indicate where common prefix ends and the enumeration name begins
|
||||
const (
|
||||
lcEventSrc_None lcEventSrc = iota
|
||||
lcEventSrc_Heal
|
||||
|
||||
@@ -19,7 +19,6 @@ package cmd
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
@@ -53,7 +52,7 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
|
||||
bucket := vars["bucket"]
|
||||
|
||||
// PutBucketLifecycle always needs a Content-Md5
|
||||
if _, ok := r.Header[xhttp.ContentMD5]; !ok {
|
||||
if !validateLengthAndChecksum(r) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -70,7 +69,7 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
bucketLifecycle, err := lifecycle.ParseLifecycleConfigWithID(io.LimitReader(r.Body, r.ContentLength))
|
||||
bucketLifecycle, err := lifecycle.ParseLifecycleConfigWithID(r.Body)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
|
||||
+45
-34
@@ -23,6 +23,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -71,8 +72,12 @@ func NewLifecycleSys() *LifecycleSys {
|
||||
return &LifecycleSys{}
|
||||
}
|
||||
|
||||
func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event string, metadata map[string]string) madmin.TraceInfo {
|
||||
func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event string, metadata map[string]string, err string) madmin.TraceInfo {
|
||||
sz, _ := oi.GetActualSize()
|
||||
if metadata == nil {
|
||||
metadata = make(map[string]string)
|
||||
}
|
||||
metadata["version-id"] = oi.VersionID
|
||||
return madmin.TraceInfo{
|
||||
TraceType: madmin.TraceILM,
|
||||
Time: startTime,
|
||||
@@ -81,18 +86,22 @@ func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event
|
||||
Duration: duration,
|
||||
Path: pathJoin(oi.Bucket, oi.Name),
|
||||
Bytes: sz,
|
||||
Error: "",
|
||||
Error: err,
|
||||
Message: getSource(4),
|
||||
Custom: metadata,
|
||||
}
|
||||
}
|
||||
|
||||
func (sys *LifecycleSys) trace(oi ObjectInfo) func(event string, metadata map[string]string) {
|
||||
func (sys *LifecycleSys) trace(oi ObjectInfo) func(event string, metadata map[string]string, err error) {
|
||||
startTime := time.Now()
|
||||
return func(event string, metadata map[string]string) {
|
||||
return func(event string, metadata map[string]string, err error) {
|
||||
duration := time.Since(startTime)
|
||||
if globalTrace.NumSubscribers(madmin.TraceILM) > 0 {
|
||||
globalTrace.Publish(ilmTrace(startTime, duration, oi, event, metadata))
|
||||
e := ""
|
||||
if err != nil {
|
||||
e = err.Error()
|
||||
}
|
||||
globalTrace.Publish(ilmTrace(startTime, duration, oi, event, metadata, e))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -147,8 +156,8 @@ func (f freeVersionTask) OpHash() uint64 {
|
||||
return xxh3.HashString(f.TransitionedObject.Tier + f.TransitionedObject.Name)
|
||||
}
|
||||
|
||||
func (n newerNoncurrentTask) OpHash() uint64 {
|
||||
return xxh3.HashString(n.bucket + n.versions[0].ObjectV.ObjectName)
|
||||
func (n noncurrentVersionsTask) OpHash() uint64 {
|
||||
return xxh3.HashString(n.bucket + n.versions[0].ObjectName)
|
||||
}
|
||||
|
||||
func (j jentry) OpHash() uint64 {
|
||||
@@ -232,14 +241,16 @@ func (es *expiryState) enqueueByDays(oi ObjectInfo, event lifecycle.Event, src l
|
||||
}
|
||||
}
|
||||
|
||||
// enqueueByNewerNoncurrent enqueues object versions expired by
|
||||
// NewerNoncurrentVersions limit for expiry.
|
||||
func (es *expiryState) enqueueByNewerNoncurrent(bucket string, versions []ObjectToDelete, lcEvent lifecycle.Event) {
|
||||
func (es *expiryState) enqueueNoncurrentVersions(bucket string, versions []ObjectToDelete, events []lifecycle.Event) {
|
||||
if len(versions) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
task := newerNoncurrentTask{bucket: bucket, versions: versions, event: lcEvent}
|
||||
task := noncurrentVersionsTask{
|
||||
bucket: bucket,
|
||||
versions: versions,
|
||||
events: events,
|
||||
}
|
||||
wrkr := es.getWorkerCh(task.OpHash())
|
||||
if wrkr == nil {
|
||||
es.stats.missedExpiryTasks.Add(1)
|
||||
@@ -339,8 +350,8 @@ func (es *expiryState) Worker(input <-chan expiryOp) {
|
||||
} else {
|
||||
applyExpiryOnNonTransitionedObjects(es.ctx, es.objAPI, v.objInfo, v.event, v.src)
|
||||
}
|
||||
case newerNoncurrentTask:
|
||||
deleteObjectVersions(es.ctx, es.objAPI, v.bucket, v.versions, v.event)
|
||||
case noncurrentVersionsTask:
|
||||
deleteObjectVersions(es.ctx, es.objAPI, v.bucket, v.versions, v.events)
|
||||
case jentry:
|
||||
transitionLogIf(es.ctx, deleteObjectFromRemoteTier(es.ctx, v.ObjName, v.VersionID, v.TierName))
|
||||
case freeVersionTask:
|
||||
@@ -348,7 +359,7 @@ func (es *expiryState) Worker(input <-chan expiryOp) {
|
||||
traceFn := globalLifecycleSys.trace(oi)
|
||||
if !oi.TransitionedObject.FreeVersion {
|
||||
// nothing to be done
|
||||
return
|
||||
continue
|
||||
}
|
||||
|
||||
ignoreNotFoundErr := func(err error) error {
|
||||
@@ -362,7 +373,8 @@ func (es *expiryState) Worker(input <-chan expiryOp) {
|
||||
err := deleteObjectFromRemoteTier(es.ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier)
|
||||
if ignoreNotFoundErr(err) != nil {
|
||||
transitionLogIf(es.ctx, err)
|
||||
return
|
||||
traceFn(ILMFreeVersionDelete, nil, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Remove this free version
|
||||
@@ -387,12 +399,10 @@ func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
|
||||
globalExpiryState = newExpiryState(ctx, objectAPI, globalILMConfig.getExpirationWorkers())
|
||||
}
|
||||
|
||||
// newerNoncurrentTask encapsulates arguments required by worker to expire objects
|
||||
// by NewerNoncurrentVersions
|
||||
type newerNoncurrentTask struct {
|
||||
type noncurrentVersionsTask struct {
|
||||
bucket string
|
||||
versions []ObjectToDelete
|
||||
event lifecycle.Event
|
||||
events []lifecycle.Event
|
||||
}
|
||||
|
||||
type transitionTask struct {
|
||||
@@ -950,9 +960,7 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
|
||||
UserDefined: meta,
|
||||
}
|
||||
}
|
||||
for k, v := range objInfo.UserDefined {
|
||||
meta[k] = v
|
||||
}
|
||||
maps.Copy(meta, objInfo.UserDefined)
|
||||
if len(objInfo.UserTags) != 0 {
|
||||
meta[xhttp.AmzObjectTagging] = objInfo.UserTags
|
||||
}
|
||||
@@ -1099,17 +1107,20 @@ func isRestoredObjectOnDisk(meta map[string]string) (onDisk bool) {
|
||||
// ToLifecycleOpts returns lifecycle.ObjectOpts value for oi.
|
||||
func (oi ObjectInfo) ToLifecycleOpts() lifecycle.ObjectOpts {
|
||||
return lifecycle.ObjectOpts{
|
||||
Name: oi.Name,
|
||||
UserTags: oi.UserTags,
|
||||
VersionID: oi.VersionID,
|
||||
ModTime: oi.ModTime,
|
||||
Size: oi.Size,
|
||||
IsLatest: oi.IsLatest,
|
||||
NumVersions: oi.NumVersions,
|
||||
DeleteMarker: oi.DeleteMarker,
|
||||
SuccessorModTime: oi.SuccessorModTime,
|
||||
RestoreOngoing: oi.RestoreOngoing,
|
||||
RestoreExpires: oi.RestoreExpires,
|
||||
TransitionStatus: oi.TransitionedObject.Status,
|
||||
Name: oi.Name,
|
||||
UserTags: oi.UserTags,
|
||||
VersionID: oi.VersionID,
|
||||
ModTime: oi.ModTime,
|
||||
Size: oi.Size,
|
||||
IsLatest: oi.IsLatest,
|
||||
NumVersions: oi.NumVersions,
|
||||
DeleteMarker: oi.DeleteMarker,
|
||||
SuccessorModTime: oi.SuccessorModTime,
|
||||
RestoreOngoing: oi.RestoreOngoing,
|
||||
RestoreExpires: oi.RestoreExpires,
|
||||
TransitionStatus: oi.TransitionedObject.Status,
|
||||
UserDefined: oi.UserDefined,
|
||||
VersionPurgeStatus: oi.VersionPurgeStatus,
|
||||
ReplicationStatus: oi.ReplicationStatus,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -243,26 +243,26 @@ func parseRequestToken(token string) (subToken string, nodeIndex int) {
|
||||
return subToken, nodeIndex
|
||||
}
|
||||
|
||||
func proxyRequestByToken(ctx context.Context, w http.ResponseWriter, r *http.Request, token string) (string, bool) {
|
||||
subToken, nodeIndex := parseRequestToken(token)
|
||||
if nodeIndex > 0 {
|
||||
return subToken, proxyRequestByNodeIndex(ctx, w, r, nodeIndex)
|
||||
func proxyRequestByToken(ctx context.Context, w http.ResponseWriter, r *http.Request, token string, returnErr bool) (subToken string, proxied bool, success bool) {
|
||||
var nodeIndex int
|
||||
if subToken, nodeIndex = parseRequestToken(token); nodeIndex >= 0 {
|
||||
proxied, success = proxyRequestByNodeIndex(ctx, w, r, nodeIndex, returnErr)
|
||||
}
|
||||
return subToken, false
|
||||
return
|
||||
}
|
||||
|
||||
func proxyRequestByNodeIndex(ctx context.Context, w http.ResponseWriter, r *http.Request, index int) (success bool) {
|
||||
func proxyRequestByNodeIndex(ctx context.Context, w http.ResponseWriter, r *http.Request, index int, returnErr bool) (proxied, success bool) {
|
||||
if len(globalProxyEndpoints) == 0 {
|
||||
return false
|
||||
return
|
||||
}
|
||||
if index < 0 || index >= len(globalProxyEndpoints) {
|
||||
return false
|
||||
return
|
||||
}
|
||||
ep := globalProxyEndpoints[index]
|
||||
if ep.IsLocal {
|
||||
return false
|
||||
return
|
||||
}
|
||||
return proxyRequest(ctx, w, r, ep)
|
||||
return true, proxyRequest(ctx, w, r, ep, returnErr)
|
||||
}
|
||||
|
||||
// ListObjectsV1Handler - GET Bucket (List Objects) Version 1.
|
||||
|
||||
@@ -472,7 +472,7 @@ func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (met
|
||||
return meta, reloaded, nil
|
||||
}
|
||||
|
||||
val, err, _ := sys.group.Do(bucket, func() (val interface{}, err error) {
|
||||
val, err, _ := sys.group.Do(bucket, func() (val any, err error) {
|
||||
meta, err = loadBucketMetadata(ctx, objAPI, bucket)
|
||||
if err != nil {
|
||||
if !sys.Initialized() {
|
||||
@@ -511,7 +511,6 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []stri
|
||||
g := errgroup.WithNErrs(len(buckets))
|
||||
bucketMetas := make([]BucketMetadata, len(buckets))
|
||||
for index := range buckets {
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
// Sleep and stagger to avoid blocked CPU and thundering
|
||||
// herd upon start up sequence.
|
||||
@@ -647,9 +646,7 @@ func (sys *BucketMetadataSys) init(ctx context.Context, buckets []string) {
|
||||
// Reset the state of the BucketMetadataSys.
|
||||
func (sys *BucketMetadataSys) Reset() {
|
||||
sys.Lock()
|
||||
for k := range sys.metadataMap {
|
||||
delete(sys.metadataMap, k)
|
||||
}
|
||||
clear(sys.metadataMap)
|
||||
sys.Unlock()
|
||||
}
|
||||
|
||||
|
||||
@@ -38,7 +38,6 @@ import (
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/fips"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
@@ -556,7 +555,7 @@ func encryptBucketMetadata(ctx context.Context, bucket string, input []byte, kms
|
||||
objectKey := crypto.GenerateKey(key.Plaintext, rand.Reader)
|
||||
sealedKey := objectKey.Seal(key.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, "")
|
||||
crypto.S3.CreateMetadata(metadata, key.KeyID, key.Ciphertext, sealedKey)
|
||||
_, err = sio.Encrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20, CipherSuites: fips.DARECiphers()})
|
||||
_, err = sio.Encrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20})
|
||||
if err != nil {
|
||||
return output, metabytes, err
|
||||
}
|
||||
@@ -590,6 +589,6 @@ func decryptBucketMetadata(input []byte, bucket string, meta map[string]string,
|
||||
}
|
||||
|
||||
outbuf := bytes.NewBuffer(nil)
|
||||
_, err = sio.Decrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20, CipherSuites: fips.DARECiphers()})
|
||||
_, err = sio.Decrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20})
|
||||
return outbuf.Bytes(), err
|
||||
}
|
||||
|
||||
@@ -295,7 +295,10 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
||||
if legalHoldRequested {
|
||||
var lerr error
|
||||
if legalHold, lerr = objectlock.ParseObjectLockLegalHoldHeaders(rq.Header); lerr != nil {
|
||||
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
|
||||
return mode, retainDate, legalHold, toAPIErrorCode(ctx, lerr)
|
||||
}
|
||||
if legalHoldPermErr != ErrNone {
|
||||
return mode, retainDate, legalHold, legalHoldPermErr
|
||||
}
|
||||
}
|
||||
|
||||
@@ -305,7 +308,7 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
||||
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
|
||||
}
|
||||
rMode, rDate, err := objectlock.ParseObjectLockRetentionHeaders(rq.Header)
|
||||
if err != nil && !(replica && rMode == "" && rDate.IsZero()) {
|
||||
if err != nil && (!replica || rMode != "" || !rDate.IsZero()) {
|
||||
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
|
||||
}
|
||||
if retentionPermErr != ErrNone {
|
||||
|
||||
@@ -122,7 +122,7 @@ func testCreateBucket(obj ObjectLayer, instanceType, bucketName string, apiRoute
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
wg.Add(n)
|
||||
for i := 0; i < n; i++ {
|
||||
for range n {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
// Sync start.
|
||||
@@ -187,7 +187,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
|
||||
// Test case - 1.
|
||||
{
|
||||
bucketName: bucketName,
|
||||
bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName))),
|
||||
bucketPolicyReader: bytes.NewReader(fmt.Appendf(nil, bucketPolicyTemplate, bucketName, bucketName)),
|
||||
|
||||
policyLen: len(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName)),
|
||||
accessKey: credentials.AccessKey,
|
||||
@@ -199,7 +199,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
|
||||
// Expecting StatusBadRequest (400).
|
||||
{
|
||||
bucketName: bucketName,
|
||||
bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName))),
|
||||
bucketPolicyReader: bytes.NewReader(fmt.Appendf(nil, bucketPolicyTemplate, bucketName, bucketName)),
|
||||
|
||||
policyLen: maxBucketPolicySize + 1,
|
||||
accessKey: credentials.AccessKey,
|
||||
@@ -211,7 +211,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
|
||||
// Expecting the HTTP response status to be StatusLengthRequired (411).
|
||||
{
|
||||
bucketName: bucketName,
|
||||
bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName))),
|
||||
bucketPolicyReader: bytes.NewReader(fmt.Appendf(nil, bucketPolicyTemplate, bucketName, bucketName)),
|
||||
|
||||
policyLen: 0,
|
||||
accessKey: credentials.AccessKey,
|
||||
@@ -258,7 +258,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
|
||||
// checkBucketPolicyResources should fail.
|
||||
{
|
||||
bucketName: bucketName1,
|
||||
bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName))),
|
||||
bucketPolicyReader: bytes.NewReader(fmt.Appendf(nil, bucketPolicyTemplate, bucketName, bucketName)),
|
||||
|
||||
policyLen: len(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName)),
|
||||
accessKey: credentials.AccessKey,
|
||||
@@ -271,7 +271,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
|
||||
// should result in 404 StatusNotFound
|
||||
{
|
||||
bucketName: "non-existent-bucket",
|
||||
bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, "non-existent-bucket", "non-existent-bucket"))),
|
||||
bucketPolicyReader: bytes.NewReader(fmt.Appendf(nil, bucketPolicyTemplate, "non-existent-bucket", "non-existent-bucket")),
|
||||
|
||||
policyLen: len(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName)),
|
||||
accessKey: credentials.AccessKey,
|
||||
@@ -284,7 +284,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
|
||||
// should result in 404 StatusNotFound
|
||||
{
|
||||
bucketName: ".invalid-bucket",
|
||||
bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, ".invalid-bucket", ".invalid-bucket"))),
|
||||
bucketPolicyReader: bytes.NewReader(fmt.Appendf(nil, bucketPolicyTemplate, ".invalid-bucket", ".invalid-bucket")),
|
||||
|
||||
policyLen: len(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName)),
|
||||
accessKey: credentials.AccessKey,
|
||||
@@ -297,7 +297,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
|
||||
// should result in 400 StatusBadRequest.
|
||||
{
|
||||
bucketName: bucketName,
|
||||
bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplateWithoutVersion, bucketName, bucketName))),
|
||||
bucketPolicyReader: bytes.NewReader(fmt.Appendf(nil, bucketPolicyTemplateWithoutVersion, bucketName, bucketName)),
|
||||
|
||||
policyLen: len(fmt.Sprintf(bucketPolicyTemplateWithoutVersion, bucketName, bucketName)),
|
||||
accessKey: credentials.AccessKey,
|
||||
|
||||
@@ -19,6 +19,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
@@ -187,9 +188,7 @@ func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[s
|
||||
}
|
||||
|
||||
cloneURLValues := make(url.Values, len(r.Form))
|
||||
for k, v := range r.Form {
|
||||
cloneURLValues[k] = v
|
||||
}
|
||||
maps.Copy(cloneURLValues, r.Form)
|
||||
|
||||
for _, objLock := range []string{
|
||||
xhttp.AmzObjectLockMode,
|
||||
@@ -224,7 +223,7 @@ func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[s
|
||||
// Add groups claim which could be a list. This will ensure that the claim
|
||||
// `jwt:groups` works.
|
||||
if grpsVal, ok := claims["groups"]; ok {
|
||||
if grpsIs, ok := grpsVal.([]interface{}); ok {
|
||||
if grpsIs, ok := grpsVal.([]any); ok {
|
||||
grps := []string{}
|
||||
for _, gI := range grpsIs {
|
||||
if g, ok := gI.(string); ok {
|
||||
|
||||
+1
-1
@@ -92,7 +92,7 @@ func parseBucketQuota(bucket string, data []byte) (quotaCfg *madmin.BucketQuota,
|
||||
}
|
||||
if !quotaCfg.IsValid() {
|
||||
if quotaCfg.Type == "fifo" {
|
||||
internalLogIf(GlobalContext, errors.New("Detected older 'fifo' quota config, 'fifo' feature is removed and not supported anymore. Please clear your quota configs using 'mc admin bucket quota alias/bucket --clear' and use 'mc ilm add' for expiration of objects"), logger.WarningKind)
|
||||
internalLogIf(GlobalContext, errors.New("Detected older 'fifo' quota config, 'fifo' feature is removed and not supported anymore. Please clear your quota configs using 'mc quota clear alias/bucket' and use 'mc ilm add' for expiration of objects"), logger.WarningKind)
|
||||
return quotaCfg, fmt.Errorf("invalid quota type 'fifo'")
|
||||
}
|
||||
return quotaCfg, fmt.Errorf("Invalid quota config %#v", quotaCfg)
|
||||
|
||||
@@ -112,7 +112,6 @@ func (r *ReplicationStats) collectWorkerMetrics(ctx context.Context) {
|
||||
r.wlock.Lock()
|
||||
r.workers.update()
|
||||
r.wlock.Unlock()
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
@@ -125,16 +126,16 @@ func (ri replicatedInfos) VersionPurgeStatus() VersionPurgeStatusType {
|
||||
completed := 0
|
||||
for _, v := range ri.Targets {
|
||||
switch v.VersionPurgeStatus {
|
||||
case Failed:
|
||||
return Failed
|
||||
case Complete:
|
||||
case replication.VersionPurgeFailed:
|
||||
return replication.VersionPurgeFailed
|
||||
case replication.VersionPurgeComplete:
|
||||
completed++
|
||||
}
|
||||
}
|
||||
if completed == len(ri.Targets) {
|
||||
return Complete
|
||||
return replication.VersionPurgeComplete
|
||||
}
|
||||
return Pending
|
||||
return replication.VersionPurgePending
|
||||
}
|
||||
|
||||
func (ri replicatedInfos) VersionPurgeStatusInternal() string {
|
||||
@@ -311,7 +312,7 @@ func parseReplicateDecision(ctx context.Context, bucket, s string) (r ReplicateD
|
||||
if len(s) == 0 {
|
||||
return
|
||||
}
|
||||
for _, p := range strings.Split(s, ",") {
|
||||
for p := range strings.SplitSeq(s, ",") {
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
@@ -380,7 +381,7 @@ func (rs *ReplicationState) CompositeReplicationStatus() (st replication.StatusT
|
||||
// CompositeVersionPurgeStatus returns overall replication purge status for the permanent delete being replicated.
|
||||
func (rs *ReplicationState) CompositeVersionPurgeStatus() VersionPurgeStatusType {
|
||||
switch VersionPurgeStatusType(rs.VersionPurgeStatusInternal) {
|
||||
case Pending, Complete, Failed: // for backward compatibility
|
||||
case replication.VersionPurgePending, replication.VersionPurgeComplete, replication.VersionPurgeFailed: // for backward compatibility
|
||||
return VersionPurgeStatusType(rs.VersionPurgeStatusInternal)
|
||||
default:
|
||||
return getCompositeVersionPurgeStatus(rs.PurgeTargets)
|
||||
@@ -478,16 +479,16 @@ func getCompositeVersionPurgeStatus(m map[string]VersionPurgeStatusType) Version
|
||||
completed := 0
|
||||
for _, v := range m {
|
||||
switch v {
|
||||
case Failed:
|
||||
return Failed
|
||||
case Complete:
|
||||
case replication.VersionPurgeFailed:
|
||||
return replication.VersionPurgeFailed
|
||||
case replication.VersionPurgeComplete:
|
||||
completed++
|
||||
}
|
||||
}
|
||||
if completed == len(m) {
|
||||
return Complete
|
||||
return replication.VersionPurgeComplete
|
||||
}
|
||||
return Pending
|
||||
return replication.VersionPurgePending
|
||||
}
|
||||
|
||||
// getHealReplicateObjectInfo returns info needed by heal replication in ReplicateObjectInfo
|
||||
@@ -635,28 +636,7 @@ type ResyncTarget struct {
|
||||
}
|
||||
|
||||
// VersionPurgeStatusType represents status of a versioned delete or permanent delete w.r.t bucket replication
|
||||
type VersionPurgeStatusType string
|
||||
|
||||
const (
|
||||
// Pending - versioned delete replication is pending.
|
||||
Pending VersionPurgeStatusType = "PENDING"
|
||||
|
||||
// Complete - versioned delete replication is now complete, erase version on disk.
|
||||
Complete VersionPurgeStatusType = "COMPLETE"
|
||||
|
||||
// Failed - versioned delete replication failed.
|
||||
Failed VersionPurgeStatusType = "FAILED"
|
||||
)
|
||||
|
||||
// Empty returns true if purge status was not set.
|
||||
func (v VersionPurgeStatusType) Empty() bool {
|
||||
return string(v) == ""
|
||||
}
|
||||
|
||||
// Pending returns true if the version is pending purge.
|
||||
func (v VersionPurgeStatusType) Pending() bool {
|
||||
return v == Pending || v == Failed
|
||||
}
|
||||
type VersionPurgeStatusType = replication.VersionPurgeStatusType
|
||||
|
||||
type replicationResyncer struct {
|
||||
// map of bucket to their resync status
|
||||
@@ -756,9 +736,7 @@ type BucketReplicationResyncStatus struct {
|
||||
|
||||
func (rs *BucketReplicationResyncStatus) cloneTgtStats() (m map[string]TargetReplicationResyncStatus) {
|
||||
m = make(map[string]TargetReplicationResyncStatus)
|
||||
for arn, st := range rs.TargetsMap {
|
||||
m[arn] = st
|
||||
}
|
||||
maps.Copy(m, rs.TargetsMap)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -915,33 +915,29 @@ func (z *ReplicationState) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
err = msgp.WrapError(err, "PurgeTargets")
|
||||
return
|
||||
}
|
||||
{
|
||||
var zb0004 string
|
||||
zb0004, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "PurgeTargets", za0003)
|
||||
return
|
||||
}
|
||||
za0004 = VersionPurgeStatusType(zb0004)
|
||||
err = za0004.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "PurgeTargets", za0003)
|
||||
return
|
||||
}
|
||||
z.PurgeTargets[za0003] = za0004
|
||||
}
|
||||
case "ResetStatusesMap":
|
||||
var zb0005 uint32
|
||||
zb0005, err = dc.ReadMapHeader()
|
||||
var zb0004 uint32
|
||||
zb0004, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ResetStatusesMap")
|
||||
return
|
||||
}
|
||||
if z.ResetStatusesMap == nil {
|
||||
z.ResetStatusesMap = make(map[string]string, zb0005)
|
||||
z.ResetStatusesMap = make(map[string]string, zb0004)
|
||||
} else if len(z.ResetStatusesMap) > 0 {
|
||||
for key := range z.ResetStatusesMap {
|
||||
delete(z.ResetStatusesMap, key)
|
||||
}
|
||||
}
|
||||
for zb0005 > 0 {
|
||||
zb0005--
|
||||
for zb0004 > 0 {
|
||||
zb0004--
|
||||
var za0005 string
|
||||
var za0006 string
|
||||
za0005, err = dc.ReadString()
|
||||
@@ -1078,7 +1074,7 @@ func (z *ReplicationState) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
err = msgp.WrapError(err, "PurgeTargets")
|
||||
return
|
||||
}
|
||||
err = en.WriteString(string(za0004))
|
||||
err = za0004.EncodeMsg(en)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "PurgeTargets", za0003)
|
||||
return
|
||||
@@ -1154,7 +1150,11 @@ func (z *ReplicationState) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.AppendMapHeader(o, uint32(len(z.PurgeTargets)))
|
||||
for za0003, za0004 := range z.PurgeTargets {
|
||||
o = msgp.AppendString(o, za0003)
|
||||
o = msgp.AppendString(o, string(za0004))
|
||||
o, err = za0004.MarshalMsg(o)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "PurgeTargets", za0003)
|
||||
return
|
||||
}
|
||||
}
|
||||
// string "ResetStatusesMap"
|
||||
o = append(o, 0xb0, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x4d, 0x61, 0x70)
|
||||
@@ -1279,35 +1279,31 @@ func (z *ReplicationState) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
err = msgp.WrapError(err, "PurgeTargets")
|
||||
return
|
||||
}
|
||||
{
|
||||
var zb0004 string
|
||||
zb0004, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "PurgeTargets", za0003)
|
||||
return
|
||||
}
|
||||
za0004 = VersionPurgeStatusType(zb0004)
|
||||
bts, err = za0004.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "PurgeTargets", za0003)
|
||||
return
|
||||
}
|
||||
z.PurgeTargets[za0003] = za0004
|
||||
}
|
||||
case "ResetStatusesMap":
|
||||
var zb0005 uint32
|
||||
zb0005, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
var zb0004 uint32
|
||||
zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ResetStatusesMap")
|
||||
return
|
||||
}
|
||||
if z.ResetStatusesMap == nil {
|
||||
z.ResetStatusesMap = make(map[string]string, zb0005)
|
||||
z.ResetStatusesMap = make(map[string]string, zb0004)
|
||||
} else if len(z.ResetStatusesMap) > 0 {
|
||||
for key := range z.ResetStatusesMap {
|
||||
delete(z.ResetStatusesMap, key)
|
||||
}
|
||||
}
|
||||
for zb0005 > 0 {
|
||||
for zb0004 > 0 {
|
||||
var za0005 string
|
||||
var za0006 string
|
||||
zb0005--
|
||||
zb0004--
|
||||
za0005, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ResetStatusesMap")
|
||||
@@ -1345,7 +1341,7 @@ func (z *ReplicationState) Msgsize() (s int) {
|
||||
if z.PurgeTargets != nil {
|
||||
for za0003, za0004 := range z.PurgeTargets {
|
||||
_ = za0004
|
||||
s += msgp.StringPrefixSize + len(za0003) + msgp.StringPrefixSize + len(string(za0004))
|
||||
s += msgp.StringPrefixSize + len(za0003) + za0004.Msgsize()
|
||||
}
|
||||
}
|
||||
s += 17 + msgp.MapHeaderSize
|
||||
@@ -2507,55 +2503,3 @@ func (z *TargetReplicationResyncStatus) Msgsize() (s int) {
|
||||
s = 1 + 3 + msgp.TimeSize + 4 + msgp.TimeSize + 3 + msgp.StringPrefixSize + len(z.ResyncID) + 4 + msgp.TimeSize + 4 + msgp.IntSize + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.StringPrefixSize + len(z.Bucket) + 4 + msgp.StringPrefixSize + len(z.Object)
|
||||
return
|
||||
}
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *VersionPurgeStatusType) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
{
|
||||
var zb0001 string
|
||||
zb0001, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
(*z) = VersionPurgeStatusType(zb0001)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z VersionPurgeStatusType) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
err = en.WriteString(string(z))
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z VersionPurgeStatusType) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
o = msgp.AppendString(o, string(z))
|
||||
return
|
||||
}
|
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (z *VersionPurgeStatusType) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
{
|
||||
var zb0001 string
|
||||
zb0001, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
(*z) = VersionPurgeStatusType(zb0001)
|
||||
}
|
||||
o = bts
|
||||
return
|
||||
}
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z VersionPurgeStatusType) Msgsize() (s int) {
|
||||
s = msgp.StringPrefixSize + len(string(z))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
@@ -184,7 +183,7 @@ var parseReplicationDecisionTest = []struct {
|
||||
|
||||
func TestParseReplicateDecision(t *testing.T) {
|
||||
for i, test := range parseReplicationDecisionTest {
|
||||
dsc, err := parseReplicateDecision(context.Background(), "bucket", test.expDsc.String())
|
||||
dsc, err := parseReplicateDecision(t.Context(), "bucket", test.expDsc.String())
|
||||
if err != nil {
|
||||
if test.expErr != err {
|
||||
t.Errorf("Test%d (%s): Expected parse error got %t , want %t", i+1, test.name, err, test.expErr)
|
||||
|
||||
+109
-54
@@ -24,6 +24,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -49,12 +50,11 @@ import (
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/once"
|
||||
"github.com/tinylib/msgp/msgp"
|
||||
"github.com/zeebo/xxh3"
|
||||
"golang.org/x/exp/maps"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -147,7 +147,7 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
|
||||
if errInt == nil {
|
||||
err = nil
|
||||
} else {
|
||||
err = errInt.(error)
|
||||
err, _ = errInt.(error)
|
||||
}
|
||||
}
|
||||
switch err.(type) {
|
||||
@@ -391,7 +391,7 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
|
||||
// can be the case that other cluster is down and duplicate `mc rm --vid`
|
||||
// is issued - this still needs to be replicated back to the other target
|
||||
if !oi.VersionPurgeStatus.Empty() {
|
||||
replicate = oi.VersionPurgeStatus == Pending || oi.VersionPurgeStatus == Failed
|
||||
replicate = oi.VersionPurgeStatus == replication.VersionPurgePending || oi.VersionPurgeStatus == replication.VersionPurgeFailed
|
||||
dsc.Set(newReplicateTargetDecision(tgtArn, replicate, sync))
|
||||
}
|
||||
continue
|
||||
@@ -619,7 +619,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
|
||||
rinfo.ReplicationStatus = rinfo.PrevReplicationStatus
|
||||
return
|
||||
}
|
||||
if dobj.VersionID != "" && rinfo.VersionPurgeStatus == Complete {
|
||||
if dobj.VersionID != "" && rinfo.VersionPurgeStatus == replication.VersionPurgeComplete {
|
||||
return
|
||||
}
|
||||
if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
|
||||
@@ -639,7 +639,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
|
||||
if dobj.VersionID == "" {
|
||||
rinfo.ReplicationStatus = replication.Failed
|
||||
} else {
|
||||
rinfo.VersionPurgeStatus = Failed
|
||||
rinfo.VersionPurgeStatus = replication.VersionPurgeFailed
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -663,7 +663,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
|
||||
case isErrObjectNotFound(serr), isErrVersionNotFound(serr):
|
||||
// version being purged is already not found on target.
|
||||
if !rinfo.VersionPurgeStatus.Empty() {
|
||||
rinfo.VersionPurgeStatus = Complete
|
||||
rinfo.VersionPurgeStatus = replication.VersionPurgeComplete
|
||||
return
|
||||
}
|
||||
case isErrReadQuorum(serr), isErrWriteQuorum(serr):
|
||||
@@ -696,7 +696,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
|
||||
if dobj.VersionID == "" {
|
||||
rinfo.ReplicationStatus = replication.Failed
|
||||
} else {
|
||||
rinfo.VersionPurgeStatus = Failed
|
||||
rinfo.VersionPurgeStatus = replication.VersionPurgeFailed
|
||||
}
|
||||
replLogIf(ctx, fmt.Errorf("unable to replicate delete marker to %s: %s/%s(%s): %w", tgt.EndpointURL(), tgt.Bucket, dobj.ObjectName, versionID, rmErr))
|
||||
if rmErr != nil && minio.IsNetworkOrHostDown(rmErr, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
|
||||
@@ -706,7 +706,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
|
||||
if dobj.VersionID == "" {
|
||||
rinfo.ReplicationStatus = replication.Completed
|
||||
} else {
|
||||
rinfo.VersionPurgeStatus = Complete
|
||||
rinfo.VersionPurgeStatus = replication.VersionPurgeComplete
|
||||
}
|
||||
}
|
||||
return
|
||||
@@ -775,13 +775,14 @@ func (m caseInsensitiveMap) Lookup(key string) (string, bool) {
|
||||
return "", false
|
||||
}
|
||||
|
||||
func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, partNum int) (putOpts minio.PutObjectOptions, err error) {
|
||||
func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts minio.PutObjectOptions, isMP bool, err error) {
|
||||
meta := make(map[string]string)
|
||||
isSSEC := crypto.SSEC.IsEncrypted(objInfo.UserDefined)
|
||||
|
||||
for k, v := range objInfo.UserDefined {
|
||||
_, isValidSSEHeader := validSSEReplicationHeaders[k]
|
||||
// In case of SSE-C objects copy the allowed internal headers as well
|
||||
if !isSSEC || !slices.Contains(maps.Keys(validSSEReplicationHeaders), k) {
|
||||
if !isSSEC || !isValidSSEHeader {
|
||||
if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
|
||||
continue
|
||||
}
|
||||
@@ -789,24 +790,26 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
|
||||
continue
|
||||
}
|
||||
}
|
||||
if slices.Contains(maps.Keys(validSSEReplicationHeaders), k) {
|
||||
if isValidSSEHeader {
|
||||
meta[validSSEReplicationHeaders[k]] = v
|
||||
} else {
|
||||
meta[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
isMP = objInfo.isMultipart()
|
||||
if len(objInfo.Checksum) > 0 {
|
||||
// Add encrypted CRC to metadata for SSE-C objects.
|
||||
if isSSEC {
|
||||
meta[ReplicationSsecChecksumHeader] = base64.StdEncoding.EncodeToString(objInfo.Checksum)
|
||||
} else {
|
||||
for _, pi := range objInfo.Parts {
|
||||
if pi.Number == partNum {
|
||||
for k, v := range pi.Checksums {
|
||||
meta[k] = v
|
||||
}
|
||||
}
|
||||
cs, mp := getCRCMeta(objInfo, 0, nil)
|
||||
// Set object checksum.
|
||||
maps.Copy(meta, cs)
|
||||
isMP = mp
|
||||
if !objInfo.isMultipart() && cs[xhttp.AmzChecksumType] == xhttp.AmzChecksumTypeFullObject {
|
||||
// For objects where checksum is full object, it will be the same.
|
||||
// Therefore, we use the cheaper PutObject replication.
|
||||
isMP = false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -837,7 +840,7 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
|
||||
if tagTmstampStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]; ok {
|
||||
tagTimestamp, err = time.Parse(time.RFC3339Nano, tagTmstampStr)
|
||||
if err != nil {
|
||||
return putOpts, err
|
||||
return putOpts, false, err
|
||||
}
|
||||
}
|
||||
putOpts.Internal.TaggingTimestamp = tagTimestamp
|
||||
@@ -861,7 +864,7 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
|
||||
if retainDateStr, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok {
|
||||
rdate, err := amztime.ISO8601Parse(retainDateStr)
|
||||
if err != nil {
|
||||
return putOpts, err
|
||||
return putOpts, false, err
|
||||
}
|
||||
putOpts.RetainUntilDate = rdate
|
||||
// set retention timestamp in opts
|
||||
@@ -869,7 +872,7 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
|
||||
if retainTmstampStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp]; ok {
|
||||
retTimestamp, err = time.Parse(time.RFC3339Nano, retainTmstampStr)
|
||||
if err != nil {
|
||||
return putOpts, err
|
||||
return putOpts, false, err
|
||||
}
|
||||
}
|
||||
putOpts.Internal.RetentionTimestamp = retTimestamp
|
||||
@@ -881,7 +884,7 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
|
||||
if lholdTmstampStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp]; ok {
|
||||
lholdTimestamp, err = time.Parse(time.RFC3339Nano, lholdTmstampStr)
|
||||
if err != nil {
|
||||
return putOpts, err
|
||||
return putOpts, false, err
|
||||
}
|
||||
}
|
||||
putOpts.Internal.LegalholdTimestamp = lholdTimestamp
|
||||
@@ -889,6 +892,24 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
|
||||
if crypto.S3.IsEncrypted(objInfo.UserDefined) {
|
||||
putOpts.ServerSideEncryption = encrypt.NewSSE()
|
||||
}
|
||||
|
||||
if crypto.S3KMS.IsEncrypted(objInfo.UserDefined) {
|
||||
// If KMS key ID replication is enabled (as by default)
|
||||
// we include the object's KMS key ID. In any case, we
|
||||
// always set the SSE-KMS header. If no KMS key ID is
|
||||
// specified, MinIO is supposed to use whatever default
|
||||
// config applies on the site or bucket.
|
||||
var keyID string
|
||||
if kms.ReplicateKeyID() {
|
||||
keyID = objInfo.KMSKeyID()
|
||||
}
|
||||
|
||||
sseEnc, err := encrypt.NewSSEKMS(keyID, nil)
|
||||
if err != nil {
|
||||
return putOpts, false, err
|
||||
}
|
||||
putOpts.ServerSideEncryption = sseEnc
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -946,7 +967,9 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
|
||||
}
|
||||
|
||||
t, _ := tags.ParseObjectTags(oi1.UserTags)
|
||||
if (oi2.UserTagCount > 0 && !reflect.DeepEqual(oi2.UserTags, t.ToMap())) || (oi2.UserTagCount != len(t.ToMap())) {
|
||||
oi2Map := make(map[string]string)
|
||||
maps.Copy(oi2Map, oi2.UserTags)
|
||||
if (oi2.UserTagCount > 0 && !reflect.DeepEqual(oi2Map, t.ToMap())) || (oi2.UserTagCount != len(t.ToMap())) {
|
||||
return replicateMetadata
|
||||
}
|
||||
|
||||
@@ -1274,7 +1297,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
|
||||
// use core client to avoid doing multipart on PUT
|
||||
c := &minio.Core{Client: tgt.Client}
|
||||
|
||||
putOpts, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo, 0)
|
||||
putOpts, isMP, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
|
||||
if err != nil {
|
||||
replLogIf(ctx, fmt.Errorf("failure setting options for replication bucket:%s err:%w", bucket, err))
|
||||
sendEvent(eventArgs{
|
||||
@@ -1306,7 +1329,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
|
||||
defer cancel()
|
||||
}
|
||||
r := bandwidth.NewMonitoredReader(newCtx, globalBucketMonitor, gr, opts)
|
||||
if objInfo.isMultipart() {
|
||||
if isMP {
|
||||
rinfo.Err = replicateObjectWithMultipart(ctx, c, tgt.Bucket, object, r, objInfo, putOpts)
|
||||
} else {
|
||||
_, rinfo.Err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts)
|
||||
@@ -1435,13 +1458,14 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
|
||||
}
|
||||
rinfo.Duration = time.Since(startTime)
|
||||
}()
|
||||
|
||||
oi, cerr := tgt.StatObject(ctx, tgt.Bucket, object, minio.StatObjectOptions{
|
||||
sOpts := minio.StatObjectOptions{
|
||||
VersionID: objInfo.VersionID,
|
||||
Internal: minio.AdvancedGetOptions{
|
||||
ReplicationProxyRequest: "false",
|
||||
},
|
||||
})
|
||||
}
|
||||
sOpts.Set(xhttp.AmzTagDirective, "ACCESS")
|
||||
oi, cerr := tgt.StatObject(ctx, tgt.Bucket, object, sOpts)
|
||||
if cerr == nil {
|
||||
rAction = getReplicationAction(objInfo, oi, ri.OpType)
|
||||
rinfo.ReplicationStatus = replication.Completed
|
||||
@@ -1526,19 +1550,30 @@ applyAction:
|
||||
ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside
|
||||
},
|
||||
}
|
||||
if tagTmStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]; ok {
|
||||
// default timestamps to ModTime unless present in metadata
|
||||
lkMap := caseInsensitiveMap(objInfo.UserDefined)
|
||||
if _, ok := lkMap.Lookup(xhttp.AmzObjectLockLegalHold); ok {
|
||||
dstOpts.Internal.LegalholdTimestamp = objInfo.ModTime
|
||||
}
|
||||
if _, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok {
|
||||
dstOpts.Internal.RetentionTimestamp = objInfo.ModTime
|
||||
}
|
||||
if objInfo.UserTags != "" {
|
||||
dstOpts.Internal.TaggingTimestamp = objInfo.ModTime
|
||||
}
|
||||
if tagTmStr, ok := lkMap.Lookup(ReservedMetadataPrefixLower + TaggingTimestamp); ok {
|
||||
ondiskTimestamp, err := time.Parse(time.RFC3339, tagTmStr)
|
||||
if err == nil {
|
||||
dstOpts.Internal.TaggingTimestamp = ondiskTimestamp
|
||||
}
|
||||
}
|
||||
if retTmStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp]; ok {
|
||||
if retTmStr, ok := lkMap.Lookup(ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp); ok {
|
||||
ondiskTimestamp, err := time.Parse(time.RFC3339, retTmStr)
|
||||
if err == nil {
|
||||
dstOpts.Internal.RetentionTimestamp = ondiskTimestamp
|
||||
}
|
||||
}
|
||||
if lholdTmStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp]; ok {
|
||||
if lholdTmStr, ok := lkMap.Lookup(ReservedMetadataPrefixLower + ObjectLockLegalHoldTimestamp); ok {
|
||||
ondiskTimestamp, err := time.Parse(time.RFC3339, lholdTmStr)
|
||||
if err == nil {
|
||||
dstOpts.Internal.LegalholdTimestamp = ondiskTimestamp
|
||||
@@ -1549,8 +1584,7 @@ applyAction:
|
||||
replLogIf(ctx, fmt.Errorf("unable to replicate metadata for object %s/%s(%s) to target %s: %w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err))
|
||||
}
|
||||
} else {
|
||||
var putOpts minio.PutObjectOptions
|
||||
putOpts, err = putReplicationOpts(ctx, tgt.StorageClass, objInfo, 0)
|
||||
putOpts, isMP, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
|
||||
if err != nil {
|
||||
replLogIf(ctx, fmt.Errorf("failed to set replicate options for object %s/%s(%s) (target %s) err:%w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), err))
|
||||
sendEvent(eventArgs{
|
||||
@@ -1581,7 +1615,7 @@ applyAction:
|
||||
defer cancel()
|
||||
}
|
||||
r := bandwidth.NewMonitoredReader(newCtx, globalBucketMonitor, gr, opts)
|
||||
if objInfo.isMultipart() {
|
||||
if isMP {
|
||||
rinfo.Err = replicateObjectWithMultipart(ctx, c, tgt.Bucket, object, r, objInfo, putOpts)
|
||||
} else {
|
||||
_, rinfo.Err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts)
|
||||
@@ -1659,7 +1693,8 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
|
||||
cHeader := http.Header{}
|
||||
cHeader.Add(xhttp.MinIOSourceReplicationRequest, "true")
|
||||
if !isSSEC {
|
||||
for k, v := range partInfo.Checksums {
|
||||
cs, _ := getCRCMeta(objInfo, partInfo.Number, nil)
|
||||
for k, v := range cs {
|
||||
cHeader.Add(k, v)
|
||||
}
|
||||
}
|
||||
@@ -1683,12 +1718,13 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
|
||||
return fmt.Errorf("ssec(%t): Part size mismatch: got %d, want %d", isSSEC, pInfo.Size, size)
|
||||
}
|
||||
uploadedParts = append(uploadedParts, minio.CompletePart{
|
||||
PartNumber: pInfo.PartNumber,
|
||||
ETag: pInfo.ETag,
|
||||
ChecksumCRC32: pInfo.ChecksumCRC32,
|
||||
ChecksumCRC32C: pInfo.ChecksumCRC32C,
|
||||
ChecksumSHA1: pInfo.ChecksumSHA1,
|
||||
ChecksumSHA256: pInfo.ChecksumSHA256,
|
||||
PartNumber: pInfo.PartNumber,
|
||||
ETag: pInfo.ETag,
|
||||
ChecksumCRC32: pInfo.ChecksumCRC32,
|
||||
ChecksumCRC32C: pInfo.ChecksumCRC32C,
|
||||
ChecksumSHA1: pInfo.ChecksumSHA1,
|
||||
ChecksumSHA256: pInfo.ChecksumSHA256,
|
||||
ChecksumCRC64NVME: pInfo.ChecksumCRC64NVME,
|
||||
})
|
||||
}
|
||||
userMeta := map[string]string{
|
||||
@@ -1701,6 +1737,13 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
|
||||
// really big value but its okay on heavily loaded systems. This is just tail end timeout.
|
||||
cctx, ccancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||
defer ccancel()
|
||||
|
||||
if len(objInfo.Checksum) > 0 {
|
||||
cs, _ := getCRCMeta(objInfo, 0, nil)
|
||||
for k, v := range cs {
|
||||
userMeta[k] = strings.Split(v, "-")[0]
|
||||
}
|
||||
}
|
||||
_, err = c.CompleteMultipartUpload(cctx, bucket, object, uploadID, uploadedParts, minio.PutObjectOptions{
|
||||
UserMetadata: userMeta,
|
||||
Internal: minio.AdvancedPutOptions{
|
||||
@@ -1724,9 +1767,7 @@ func filterReplicationStatusMetadata(metadata map[string]string) map[string]stri
|
||||
}
|
||||
if !copied {
|
||||
dst = make(map[string]string, len(metadata))
|
||||
for k, v := range metadata {
|
||||
dst[k] = v
|
||||
}
|
||||
maps.Copy(dst, metadata)
|
||||
copied = true
|
||||
}
|
||||
delete(dst, key)
|
||||
@@ -2908,7 +2949,7 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
}()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < resyncParallelRoutines; i++ {
|
||||
for i := range resyncParallelRoutines {
|
||||
wg.Add(1)
|
||||
workers[i] = make(chan ReplicateObjectInfo, 100)
|
||||
i := i
|
||||
@@ -3017,7 +3058,7 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
workers[h%uint64(resyncParallelRoutines)] <- roi
|
||||
}
|
||||
}
|
||||
for i := 0; i < resyncParallelRoutines; i++ {
|
||||
for i := range resyncParallelRoutines {
|
||||
xioutil.SafeClose(workers[i])
|
||||
}
|
||||
wg.Wait()
|
||||
@@ -3147,11 +3188,9 @@ func (p *ReplicationPool) startResyncRoutine(ctx context.Context, buckets []stri
|
||||
<-ctx.Done()
|
||||
return
|
||||
}
|
||||
duration := time.Duration(r.Float64() * float64(time.Minute))
|
||||
if duration < time.Second {
|
||||
duration := max(time.Duration(r.Float64()*float64(time.Minute)),
|
||||
// Make sure to sleep at least a second to avoid high CPU ticks.
|
||||
duration = time.Second
|
||||
}
|
||||
time.Second)
|
||||
time.Sleep(duration)
|
||||
}
|
||||
}
|
||||
@@ -3317,7 +3356,7 @@ func getReplicationDiff(ctx context.Context, objAPI ObjectLayer, bucket string,
|
||||
}
|
||||
for arn, st := range roi.TargetPurgeStatuses {
|
||||
if opts.ARN == "" || opts.ARN == arn {
|
||||
if !opts.Verbose && st == Complete {
|
||||
if !opts.Verbose && st == replication.VersionPurgeComplete {
|
||||
continue
|
||||
}
|
||||
t, ok := tgtsMap[arn]
|
||||
@@ -3416,7 +3455,7 @@ func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcf
|
||||
// heal delete marker replication failure or versioned delete replication failure
|
||||
if roi.ReplicationStatus == replication.Pending ||
|
||||
roi.ReplicationStatus == replication.Failed ||
|
||||
roi.VersionPurgeStatus == Failed || roi.VersionPurgeStatus == Pending {
|
||||
roi.VersionPurgeStatus == replication.VersionPurgeFailed || roi.VersionPurgeStatus == replication.VersionPurgePending {
|
||||
globalReplicationPool.Get().queueReplicaDeleteTask(dv)
|
||||
return
|
||||
}
|
||||
@@ -3704,7 +3743,7 @@ func (p *ReplicationPool) queueMRFHeal() error {
|
||||
}
|
||||
|
||||
func (p *ReplicationPool) initialized() bool {
|
||||
return !(p == nil || p.objLayer == nil)
|
||||
return p != nil && p.objLayer != nil
|
||||
}
|
||||
|
||||
// getMRF returns MRF entries for this node.
|
||||
@@ -3746,3 +3785,19 @@ type validateReplicationDestinationOptions struct {
|
||||
|
||||
checkReadyErr sync.Map
|
||||
}
|
||||
|
||||
func getCRCMeta(oi ObjectInfo, partNum int, h http.Header) (cs map[string]string, isMP bool) {
|
||||
meta := make(map[string]string)
|
||||
cs, isMP = oi.decryptChecksums(partNum, h)
|
||||
for k, v := range cs {
|
||||
if k == xhttp.AmzChecksumType {
|
||||
continue
|
||||
}
|
||||
cktype := hash.ChecksumStringToType(k)
|
||||
if cktype.IsSet() {
|
||||
meta[cktype.Key()] = v
|
||||
meta[xhttp.AmzChecksumAlgo] = cktype.String()
|
||||
}
|
||||
}
|
||||
return meta, isMP
|
||||
}
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
@@ -86,7 +85,7 @@ var replicationConfigTests = []struct {
|
||||
}
|
||||
|
||||
func TestReplicationResync(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctx := t.Context()
|
||||
for i, test := range replicationConfigTests {
|
||||
if sync := test.rcfg.Resync(ctx, test.info, test.dsc, test.tgtStatuses); sync.mustResync() != test.expectedSync {
|
||||
t.Errorf("Test%d (%s): Resync got %t , want %t", i+1, test.name, sync.mustResync(), test.expectedSync)
|
||||
|
||||
+9
-10
@@ -19,6 +19,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"math"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
@@ -108,18 +109,18 @@ func (l ReplicationLastHour) merge(o ReplicationLastHour) (merged ReplicationLas
|
||||
|
||||
// Add a new duration data
|
||||
func (l *ReplicationLastHour) addsize(sz int64) {
|
||||
min := time.Now().Unix() / 60
|
||||
l.forwardTo(min)
|
||||
winIdx := min % 60
|
||||
l.Totals[winIdx].merge(AccElem{Total: min, Size: sz, N: 1})
|
||||
l.LastMin = min
|
||||
minutes := time.Now().Unix() / 60
|
||||
l.forwardTo(minutes)
|
||||
winIdx := minutes % 60
|
||||
l.Totals[winIdx].merge(AccElem{Total: minutes, Size: sz, N: 1})
|
||||
l.LastMin = minutes
|
||||
}
|
||||
|
||||
// Merge all recorded counts of last hour into one
|
||||
func (l *ReplicationLastHour) getTotal() AccElem {
|
||||
var res AccElem
|
||||
min := time.Now().Unix() / 60
|
||||
l.forwardTo(min)
|
||||
minutes := time.Now().Unix() / 60
|
||||
l.forwardTo(minutes)
|
||||
for _, elem := range l.Totals[:] {
|
||||
res.merge(elem)
|
||||
}
|
||||
@@ -221,9 +222,7 @@ func (brs BucketReplicationStats) Clone() (c BucketReplicationStats) {
|
||||
}
|
||||
if s.Failed.ErrCounts == nil {
|
||||
s.Failed.ErrCounts = make(map[string]int)
|
||||
for k, v := range st.Failed.ErrCounts {
|
||||
s.Failed.ErrCounts[k] = v
|
||||
}
|
||||
maps.Copy(s.Failed.ErrCounts, st.Failed.ErrCounts)
|
||||
}
|
||||
c.Stats[arn] = &s
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"maps"
|
||||
"net/url"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -236,9 +237,7 @@ func (sys *BucketTargetSys) healthStats() map[string]epHealth {
|
||||
sys.hMutex.RLock()
|
||||
defer sys.hMutex.RUnlock()
|
||||
m := make(map[string]epHealth, len(sys.hc))
|
||||
for k, v := range sys.hc {
|
||||
m[k] = v
|
||||
}
|
||||
maps.Copy(m, sys.hc)
|
||||
return m
|
||||
}
|
||||
|
||||
|
||||
+2
-4
@@ -57,11 +57,9 @@ func initCallhome(ctx context.Context, objAPI ObjectLayer) {
|
||||
|
||||
// callhome running on a different node.
|
||||
// sleep for some time and try again.
|
||||
duration := time.Duration(r.Float64() * float64(globalCallhomeConfig.FrequencyDur()))
|
||||
if duration < time.Second {
|
||||
duration := max(time.Duration(r.Float64()*float64(globalCallhomeConfig.FrequencyDur())),
|
||||
// Make sure to sleep at least a second to avoid high CPU ticks.
|
||||
duration = time.Second
|
||||
}
|
||||
time.Second)
|
||||
time.Sleep(duration)
|
||||
}
|
||||
}()
|
||||
|
||||
+69
-44
@@ -47,6 +47,7 @@ import (
|
||||
"github.com/minio/console/api/operations"
|
||||
consoleoauth2 "github.com/minio/console/pkg/auth/idp/oauth2"
|
||||
consoleCerts "github.com/minio/console/pkg/certs"
|
||||
"github.com/minio/kms-go/kes"
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
@@ -91,9 +92,6 @@ func init() {
|
||||
logger.Init(GOPATH, GOROOT)
|
||||
logger.RegisterError(config.FmtError)
|
||||
|
||||
globalBatchJobsMetrics = batchJobMetrics{metrics: make(map[string]*batchJobInfo)}
|
||||
go globalBatchJobsMetrics.purgeJobMetrics()
|
||||
|
||||
t, _ := minioVersionToReleaseTime(Version)
|
||||
if !t.IsZero() {
|
||||
globalVersionUnix = uint64(t.Unix())
|
||||
@@ -107,7 +105,7 @@ func init() {
|
||||
gob.Register(madmin.TimeInfo{})
|
||||
gob.Register(madmin.XFSErrorConfigs{})
|
||||
gob.Register(map[string]string{})
|
||||
gob.Register(map[string]interface{}{})
|
||||
gob.Register(map[string]any{})
|
||||
|
||||
// All minio-go and madmin-go API operations shall be performed only once,
|
||||
// another way to look at this is we are turning off retries.
|
||||
@@ -138,6 +136,9 @@ func minioConfigToConsoleFeatures() {
|
||||
if value := env.Get(config.EnvBrowserRedirectURL, ""); value != "" {
|
||||
os.Setenv("CONSOLE_BROWSER_REDIRECT_URL", value)
|
||||
}
|
||||
if value := env.Get(config.EnvConsoleDebugLogLevel, ""); value != "" {
|
||||
os.Setenv("CONSOLE_DEBUG_LOGLEVEL", value)
|
||||
}
|
||||
// pass the console subpath configuration
|
||||
if globalBrowserRedirectURL != nil {
|
||||
subPath := path.Clean(pathJoin(strings.TrimSpace(globalBrowserRedirectURL.Path), SlashSeparator))
|
||||
@@ -257,7 +258,7 @@ func initConsoleServer() (*consoleapi.Server, error) {
|
||||
|
||||
if !serverDebugLog {
|
||||
// Disable console logging if server debug log is not enabled
|
||||
noLog := func(string, ...interface{}) {}
|
||||
noLog := func(string, ...any) {}
|
||||
|
||||
consoleapi.LogInfo = noLog
|
||||
consoleapi.LogError = noLog
|
||||
@@ -382,7 +383,7 @@ func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) {
|
||||
}
|
||||
|
||||
// Check "no-compat" flag from command line argument.
|
||||
ctxt.StrictS3Compat = !(ctx.IsSet("no-compat") || ctx.GlobalIsSet("no-compat"))
|
||||
ctxt.StrictS3Compat = !ctx.IsSet("no-compat") && !ctx.GlobalIsSet("no-compat")
|
||||
|
||||
switch {
|
||||
case ctx.IsSet("config-dir"):
|
||||
@@ -717,9 +718,7 @@ func serverHandleEnvVars() {
|
||||
logger.Fatal(err, "Invalid MINIO_BROWSER_REDIRECT_URL value in environment variable")
|
||||
}
|
||||
// Look for if URL has invalid values and return error.
|
||||
if !((u.Scheme == "http" || u.Scheme == "https") &&
|
||||
u.Opaque == "" &&
|
||||
!u.ForceQuery && u.RawQuery == "" && u.Fragment == "") {
|
||||
if !isValidURLEndpoint((*url.URL)(u)) {
|
||||
err := fmt.Errorf("URL contains unexpected resources, expected URL to be one of http(s)://console.example.com or as a subpath via API endpoint http(s)://minio.example.com/minio format: %v", u)
|
||||
logger.Fatal(err, "Invalid MINIO_BROWSER_REDIRECT_URL value is environment variable")
|
||||
}
|
||||
@@ -734,9 +733,7 @@ func serverHandleEnvVars() {
|
||||
logger.Fatal(err, "Invalid MINIO_SERVER_URL value in environment variable")
|
||||
}
|
||||
// Look for if URL has invalid values and return error.
|
||||
if !((u.Scheme == "http" || u.Scheme == "https") &&
|
||||
(u.Path == "/" || u.Path == "") && u.Opaque == "" &&
|
||||
!u.ForceQuery && u.RawQuery == "" && u.Fragment == "") {
|
||||
if !isValidURLEndpoint((*url.URL)(u)) {
|
||||
err := fmt.Errorf("URL contains unexpected resources, expected URL to be of http(s)://minio.example.com format: %v", u)
|
||||
logger.Fatal(err, "Invalid MINIO_SERVER_URL value is environment variable")
|
||||
}
|
||||
@@ -764,7 +761,7 @@ func serverHandleEnvVars() {
|
||||
|
||||
domains := env.Get(config.EnvDomain, "")
|
||||
if len(domains) != 0 {
|
||||
for _, domainName := range strings.Split(domains, config.ValueSeparator) {
|
||||
for domainName := range strings.SplitSeq(domains, config.ValueSeparator) {
|
||||
if _, ok := dns2.IsDomainName(domainName); !ok {
|
||||
logger.Fatal(config.ErrInvalidDomainValue(nil).Msgf("Unknown value `%s`", domainName),
|
||||
"Invalid MINIO_DOMAIN value in environment variable")
|
||||
@@ -835,55 +832,83 @@ func serverHandleEnvVars() {
|
||||
globalEnableSyncBoot = env.Get("MINIO_SYNC_BOOT", config.EnableOff) == config.EnableOn
|
||||
}
|
||||
|
||||
func loadRootCredentials() {
|
||||
func loadRootCredentials() auth.Credentials {
|
||||
// At this point, either both environment variables
|
||||
// are defined or both are not defined.
|
||||
// Check both cases and authenticate them if correctly defined
|
||||
var user, password string
|
||||
var hasCredentials bool
|
||||
var legacyCredentials bool
|
||||
//nolint:gocritic
|
||||
if env.IsSet(config.EnvRootUser) && env.IsSet(config.EnvRootPassword) {
|
||||
user = env.Get(config.EnvRootUser, "")
|
||||
password = env.Get(config.EnvRootPassword, "")
|
||||
hasCredentials = true
|
||||
} else if env.IsSet(config.EnvAccessKey) && env.IsSet(config.EnvSecretKey) {
|
||||
user = env.Get(config.EnvAccessKey, "")
|
||||
password = env.Get(config.EnvSecretKey, "")
|
||||
legacyCredentials = true
|
||||
hasCredentials = true
|
||||
} else if globalServerCtxt.RootUser != "" && globalServerCtxt.RootPwd != "" {
|
||||
user, password = globalServerCtxt.RootUser, globalServerCtxt.RootPwd
|
||||
hasCredentials = true
|
||||
}
|
||||
if hasCredentials {
|
||||
cred, err := auth.CreateCredentials(user, password)
|
||||
if err != nil {
|
||||
if legacyCredentials {
|
||||
logger.Fatal(config.ErrInvalidCredentials(err),
|
||||
"Unable to validate credentials inherited from the shell environment")
|
||||
} else {
|
||||
logger.Fatal(config.ErrInvalidRootUserCredentials(err),
|
||||
"Unable to validate credentials inherited from the shell environment")
|
||||
}
|
||||
if user == "" || password == "" {
|
||||
return auth.Credentials{}
|
||||
}
|
||||
cred, err := auth.CreateCredentials(user, password)
|
||||
if err != nil {
|
||||
if legacyCredentials {
|
||||
logger.Fatal(config.ErrInvalidCredentials(err),
|
||||
"Unable to validate credentials inherited from the shell environment")
|
||||
} else {
|
||||
logger.Fatal(config.ErrInvalidRootUserCredentials(err),
|
||||
"Unable to validate credentials inherited from the shell environment")
|
||||
}
|
||||
if env.IsSet(config.EnvAccessKey) && env.IsSet(config.EnvSecretKey) {
|
||||
msg := fmt.Sprintf("WARNING: %s and %s are deprecated.\n"+
|
||||
" Please use %s and %s",
|
||||
config.EnvAccessKey, config.EnvSecretKey,
|
||||
config.EnvRootUser, config.EnvRootPassword)
|
||||
logger.Info(color.RedBold(msg))
|
||||
}
|
||||
globalActiveCred = cred
|
||||
globalCredViaEnv = true
|
||||
} else {
|
||||
globalActiveCred = auth.DefaultCredentials
|
||||
}
|
||||
if env.IsSet(config.EnvAccessKey) && env.IsSet(config.EnvSecretKey) {
|
||||
msg := fmt.Sprintf("WARNING: %s and %s are deprecated.\n"+
|
||||
" Please use %s and %s",
|
||||
config.EnvAccessKey, config.EnvSecretKey,
|
||||
config.EnvRootUser, config.EnvRootPassword)
|
||||
logger.Info(color.RedBold(msg))
|
||||
}
|
||||
globalCredViaEnv = true
|
||||
return cred
|
||||
}
|
||||
|
||||
// autoGenerateRootCredentials generates root credentials deterministically if
|
||||
// a KMS is configured, no manual credentials have been specified and if root
|
||||
// access is disabled.
|
||||
func autoGenerateRootCredentials() auth.Credentials {
|
||||
if GlobalKMS == nil {
|
||||
return globalActiveCred
|
||||
}
|
||||
|
||||
var err error
|
||||
globalNodeAuthToken, err = authenticateNode(globalActiveCred.AccessKey, globalActiveCred.SecretKey)
|
||||
aKey, err := GlobalKMS.MAC(GlobalContext, &kms.MACRequest{Message: []byte("root access key")})
|
||||
if IsErrIgnored(err, kes.ErrNotAllowed, kms.ErrNotSupported, errors.ErrUnsupported, kms.ErrPermission) {
|
||||
// If we don't have permission to compute the HMAC, don't change the cred.
|
||||
return globalActiveCred
|
||||
}
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to generate internode credentials")
|
||||
logger.Fatal(err, "Unable to generate root access key using KMS")
|
||||
}
|
||||
|
||||
sKey, err := GlobalKMS.MAC(GlobalContext, &kms.MACRequest{Message: []byte("root secret key")})
|
||||
if err != nil {
|
||||
// Here, we must have permission. Otherwise, we would have failed earlier.
|
||||
logger.Fatal(err, "Unable to generate root secret key using KMS")
|
||||
}
|
||||
|
||||
accessKey, err := auth.GenerateAccessKey(20, bytes.NewReader(aKey))
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to generate root access key")
|
||||
}
|
||||
secretKey, err := auth.GenerateSecretKey(32, bytes.NewReader(sKey))
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to generate root secret key")
|
||||
}
|
||||
|
||||
logger.Info("Automatically generated root access key and secret key with the KMS")
|
||||
return auth.Credentials{
|
||||
AccessKey: accessKey,
|
||||
SecretKey: secretKey,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -915,7 +940,7 @@ func handleKMSConfig() {
|
||||
}
|
||||
|
||||
func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secureConn bool, err error) {
|
||||
if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
|
||||
if !isFile(getPublicCertFile()) || !isFile(getPrivateKeyFile()) {
|
||||
return nil, nil, false, nil
|
||||
}
|
||||
|
||||
@@ -1034,6 +1059,6 @@ func (a bgCtx) Deadline() (deadline time.Time, ok bool) {
|
||||
return time.Time{}, false
|
||||
}
|
||||
|
||||
func (a bgCtx) Value(key interface{}) interface{} {
|
||||
func (a bgCtx) Value(key any) any {
|
||||
return a.parent.Value(key)
|
||||
}
|
||||
|
||||
@@ -43,9 +43,8 @@ func Test_readFromSecret(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
testCase := testCase
|
||||
t.Run("", func(t *testing.T) {
|
||||
tmpfile, err := os.CreateTemp("", "testfile")
|
||||
tmpfile, err := os.CreateTemp(t.TempDir(), "testfile")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
@@ -155,9 +154,8 @@ MINIO_ROOT_PASSWORD=minio123`,
|
||||
},
|
||||
}
|
||||
for _, testCase := range testCases {
|
||||
testCase := testCase
|
||||
t.Run("", func(t *testing.T) {
|
||||
tmpfile, err := os.CreateTemp("", "testfile")
|
||||
tmpfile, err := os.CreateTemp(t.TempDir(), "testfile")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
+6
-53
@@ -18,17 +18,14 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/minio/kms-go/kes"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/config/browser"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/config"
|
||||
@@ -82,12 +79,8 @@ func initHelp() {
|
||||
config.BatchSubSys: batch.DefaultKVS,
|
||||
config.BrowserSubSys: browser.DefaultKVS,
|
||||
}
|
||||
for k, v := range notify.DefaultNotificationKVS {
|
||||
kvs[k] = v
|
||||
}
|
||||
for k, v := range lambda.DefaultLambdaKVS {
|
||||
kvs[k] = v
|
||||
}
|
||||
maps.Copy(kvs, notify.DefaultNotificationKVS)
|
||||
maps.Copy(kvs, lambda.DefaultLambdaKVS)
|
||||
if globalIsErasure {
|
||||
kvs[config.StorageClassSubSys] = storageclass.DefaultKVS
|
||||
kvs[config.HealSubSys] = heal.DefaultKVS
|
||||
@@ -359,7 +352,9 @@ func validateSubSysConfig(ctx context.Context, s config.Config, subSys string, o
|
||||
}
|
||||
case config.IdentityOpenIDSubSys:
|
||||
if _, err := openid.LookupConfig(s,
|
||||
NewHTTPTransport(), xhttp.DrainBody, globalSite.Region()); err != nil {
|
||||
xhttp.WithUserAgent(NewHTTPTransport(), func() string {
|
||||
return getUserAgent(getMinioMode())
|
||||
}), xhttp.DrainBody, globalSite.Region()); err != nil {
|
||||
return err
|
||||
}
|
||||
case config.IdentityLDAPSubSys:
|
||||
@@ -570,7 +565,6 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
|
||||
}
|
||||
|
||||
globalAPIConfig.init(apiConfig, setDriveCounts, objAPI.Legacy())
|
||||
autoGenerateRootCredentials() // Generate the KMS root credentials here since we don't know whether API root access is disabled until now.
|
||||
setRemoteInstanceTransport(NewHTTPTransportWithTimeout(apiConfig.RemoteTransportDeadline))
|
||||
case config.CompressionSubSys:
|
||||
cmpCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
|
||||
@@ -729,47 +723,6 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
|
||||
return nil
|
||||
}
|
||||
|
||||
// autoGenerateRootCredentials generates root credentials deterministically if
|
||||
// a KMS is configured, no manual credentials have been specified and if root
|
||||
// access is disabled.
|
||||
func autoGenerateRootCredentials() {
|
||||
if GlobalKMS == nil {
|
||||
return
|
||||
}
|
||||
if globalAPIConfig.permitRootAccess() || !globalActiveCred.Equal(auth.DefaultCredentials) {
|
||||
return
|
||||
}
|
||||
|
||||
aKey, err := GlobalKMS.MAC(GlobalContext, &kms.MACRequest{Message: []byte("root access key")})
|
||||
if errors.Is(err, kes.ErrNotAllowed) || errors.Is(err, errors.ErrUnsupported) {
|
||||
return // If we don't have permission to compute the HMAC, don't change the cred.
|
||||
}
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to generate root access key using KMS")
|
||||
}
|
||||
|
||||
sKey, err := GlobalKMS.MAC(GlobalContext, &kms.MACRequest{Message: []byte("root secret key")})
|
||||
if err != nil {
|
||||
// Here, we must have permission. Otherwise, we would have failed earlier.
|
||||
logger.Fatal(err, "Unable to generate root secret key using KMS")
|
||||
}
|
||||
|
||||
accessKey, err := auth.GenerateAccessKey(20, bytes.NewReader(aKey))
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to generate root access key")
|
||||
}
|
||||
secretKey, err := auth.GenerateSecretKey(32, bytes.NewReader(sKey))
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to generate root secret key")
|
||||
}
|
||||
|
||||
logger.Info("Automatically generated root access key and secret key with the KMS")
|
||||
globalActiveCred = auth.Credentials{
|
||||
AccessKey: accessKey,
|
||||
SecretKey: secretKey,
|
||||
}
|
||||
}
|
||||
|
||||
// applyDynamicConfig will apply dynamic config values.
|
||||
// Dynamic systems should be in config.SubSystemsDynamic as well.
|
||||
func applyDynamicConfig(ctx context.Context, objAPI ObjectLayer, s config.Config) error {
|
||||
|
||||
@@ -26,7 +26,7 @@ import (
|
||||
)
|
||||
|
||||
func TestServerConfig(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
|
||||
objLayer, fsDir, err := prepareFS(ctx)
|
||||
@@ -56,7 +56,7 @@ func TestServerConfig(t *testing.T) {
|
||||
t.Errorf("Expecting region `us-west-1` found %s", globalSite.Region())
|
||||
}
|
||||
|
||||
if err := saveServerConfig(context.Background(), objLayer, globalServerConfig); err != nil {
|
||||
if err := saveServerConfig(t.Context(), objLayer, globalServerConfig); err != nil {
|
||||
t.Fatalf("Unable to save updated config file %s", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -38,12 +38,12 @@ import (
|
||||
)
|
||||
|
||||
// Save config file to corresponding backend
|
||||
func Save(configFile string, data interface{}) error {
|
||||
func Save(configFile string, data any) error {
|
||||
return quick.SaveConfig(data, configFile, globalEtcdClient)
|
||||
}
|
||||
|
||||
// Load config from backend
|
||||
func Load(configFile string, data interface{}) (quick.Config, error) {
|
||||
func Load(configFile string, data any) (quick.Config, error) {
|
||||
return quick.LoadConfig(configFile, globalEtcdClient, data)
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -129,7 +129,7 @@ func saveServerConfigHistory(ctx context.Context, objAPI ObjectLayer, kv []byte)
|
||||
return saveConfig(ctx, objAPI, historyFile, kv)
|
||||
}
|
||||
|
||||
func saveServerConfig(ctx context.Context, objAPI ObjectLayer, cfg interface{}) error {
|
||||
func saveServerConfig(ctx context.Context, objAPI ObjectLayer, cfg any) error {
|
||||
data, err := json.Marshal(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -25,11 +25,11 @@ import (
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/madmin-go/v3/logger/log"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/logger/target/console"
|
||||
"github.com/minio/minio/internal/logger/target/types"
|
||||
types "github.com/minio/minio/internal/logger/target/loggertypes"
|
||||
"github.com/minio/minio/internal/pubsub"
|
||||
"github.com/minio/pkg/v3/logger/message/log"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
)
|
||||
|
||||
@@ -101,7 +101,7 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan log.Info, doneCh <-chan st
|
||||
|
||||
lastN = make([]log.Info, last)
|
||||
sys.RLock()
|
||||
sys.logBuf.Do(func(p interface{}) {
|
||||
sys.logBuf.Do(func(p any) {
|
||||
if p != nil {
|
||||
lg, ok := p.(log.Info)
|
||||
if ok && lg.SendLog(node, logKind) {
|
||||
@@ -113,7 +113,7 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan log.Info, doneCh <-chan st
|
||||
sys.RUnlock()
|
||||
// send last n console log messages in order filtered by node
|
||||
if cnt > 0 {
|
||||
for i := 0; i < last; i++ {
|
||||
for i := range last {
|
||||
entry := lastN[(cnt+i)%last]
|
||||
if (entry == log.Info{}) {
|
||||
continue
|
||||
@@ -155,7 +155,7 @@ func (sys *HTTPConsoleLoggerSys) Stats() types.TargetStats {
|
||||
// Content returns the console stdout log
|
||||
func (sys *HTTPConsoleLoggerSys) Content() (logs []log.Entry) {
|
||||
sys.RLock()
|
||||
sys.logBuf.Do(func(p interface{}) {
|
||||
sys.logBuf.Do(func(p any) {
|
||||
if p != nil {
|
||||
lg, ok := p.(log.Info)
|
||||
if ok {
|
||||
@@ -181,7 +181,7 @@ func (sys *HTTPConsoleLoggerSys) Type() types.TargetType {
|
||||
|
||||
// Send log message 'e' to console and publish to console
|
||||
// log pubsub system
|
||||
func (sys *HTTPConsoleLoggerSys) Send(ctx context.Context, entry interface{}) error {
|
||||
func (sys *HTTPConsoleLoggerSys) Send(ctx context.Context, entry any) error {
|
||||
var lg log.Info
|
||||
switch e := entry.(type) {
|
||||
case log.Entry:
|
||||
|
||||
@@ -104,6 +104,22 @@ func (p *scannerMetrics) log(s scannerMetric, paths ...string) func(custom map[s
|
||||
}
|
||||
}
|
||||
|
||||
// time n scanner actions.
|
||||
// Use for s < scannerMetricLastRealtime
|
||||
func (p *scannerMetrics) timeN(s scannerMetric) func(n int) func() {
|
||||
startTime := time.Now()
|
||||
return func(n int) func() {
|
||||
return func() {
|
||||
duration := time.Since(startTime)
|
||||
|
||||
atomic.AddUint64(&p.operations[s], uint64(n))
|
||||
if s < scannerMetricLastRealtime {
|
||||
p.latency[s].add(duration)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// time a scanner action.
|
||||
// Use for s < scannerMetricLastRealtime
|
||||
func (p *scannerMetrics) time(s scannerMetric) func() {
|
||||
@@ -182,7 +198,7 @@ func (p *scannerMetrics) currentPathUpdater(disk, initial string) (update func(p
|
||||
func (p *scannerMetrics) getCurrentPaths() []string {
|
||||
var res []string
|
||||
prefix := globalLocalNodeName + "/"
|
||||
p.currentPaths.Range(func(key, value interface{}) bool {
|
||||
p.currentPaths.Range(func(key, value any) bool {
|
||||
// We are a bit paranoid, but better miss an entry than crash.
|
||||
name, ok := key.(string)
|
||||
if !ok {
|
||||
@@ -205,7 +221,7 @@ func (p *scannerMetrics) getCurrentPaths() []string {
|
||||
// (since this is concurrent it may not be 100% reliable)
|
||||
func (p *scannerMetrics) activeDrives() int {
|
||||
var i int
|
||||
p.currentPaths.Range(func(k, v interface{}) bool {
|
||||
p.currentPaths.Range(func(k, v any) bool {
|
||||
i++
|
||||
return true
|
||||
})
|
||||
@@ -283,7 +299,7 @@ func (p *scannerMetrics) report() madmin.ScannerMetrics {
|
||||
m.CollectedAt = time.Now()
|
||||
m.ActivePaths = p.getCurrentPaths()
|
||||
m.LifeTimeOps = make(map[string]uint64, scannerMetricLast)
|
||||
for i := scannerMetric(0); i < scannerMetricLast; i++ {
|
||||
for i := range scannerMetricLast {
|
||||
if n := atomic.LoadUint64(&p.operations[i]); n > 0 {
|
||||
m.LifeTimeOps[i.String()] = n
|
||||
}
|
||||
@@ -293,7 +309,7 @@ func (p *scannerMetrics) report() madmin.ScannerMetrics {
|
||||
}
|
||||
|
||||
m.LastMinute.Actions = make(map[string]madmin.TimedAction, scannerMetricLastRealtime)
|
||||
for i := scannerMetric(0); i < scannerMetricLastRealtime; i++ {
|
||||
for i := range scannerMetricLastRealtime {
|
||||
lm := p.lastMinute(i)
|
||||
if lm.N > 0 {
|
||||
m.LastMinute.Actions[i.String()] = lm.asTimedAction()
|
||||
|
||||
+135
-208
@@ -37,9 +37,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
"github.com/minio/minio/internal/bucket/object/lock"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
"github.com/minio/minio/internal/color"
|
||||
"github.com/minio/minio/internal/config/heal"
|
||||
"github.com/minio/minio/internal/event"
|
||||
@@ -80,11 +78,9 @@ func initDataScanner(ctx context.Context, objAPI ObjectLayer) {
|
||||
// Run the data scanner in a loop
|
||||
for {
|
||||
runDataScanner(ctx, objAPI)
|
||||
duration := time.Duration(r.Float64() * float64(scannerCycle.Load()))
|
||||
if duration < time.Second {
|
||||
duration := max(time.Duration(r.Float64()*float64(scannerCycle.Load())),
|
||||
// Make sure to sleep at least a second to avoid high CPU ticks.
|
||||
duration = time.Second
|
||||
}
|
||||
time.Second)
|
||||
time.Sleep(duration)
|
||||
}
|
||||
}()
|
||||
@@ -334,7 +330,7 @@ func scanDataFolder(ctx context.Context, disks []StorageAPI, drive *xlStorage, c
|
||||
}
|
||||
|
||||
var skipHeal atomic.Bool
|
||||
if globalIsErasure || cache.Info.SkipHealing {
|
||||
if !globalIsErasure || cache.Info.SkipHealing {
|
||||
skipHeal.Store(true)
|
||||
}
|
||||
|
||||
@@ -664,6 +660,12 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
|
||||
into.addChild(h)
|
||||
continue
|
||||
}
|
||||
// Adjust the probability of healing.
|
||||
// This first removes lowest x from the mod check and makes it x times more likely.
|
||||
// So if duudc = 10 and we want heal check every 50 cycles, we check
|
||||
// if (cycle/10) % (50/10) == 0, which would make heal checks run once every 50 cycles,
|
||||
// if the objects are pre-selected as 1:10.
|
||||
folder.objectHealProbDiv = dataUsageUpdateDirCycles
|
||||
}
|
||||
f.updateCurrentPath(folder.name)
|
||||
stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, folder.name)
|
||||
@@ -858,8 +860,8 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if compact {
|
||||
stop := globalScannerMetrics.log(scannerMetricCompactFolder, folder.name)
|
||||
f.newCache.deleteRecursive(thisHash)
|
||||
@@ -873,7 +875,6 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
|
||||
}
|
||||
stop(total)
|
||||
}
|
||||
|
||||
}
|
||||
// Compact if too many children...
|
||||
if !into.Compacted {
|
||||
@@ -946,10 +947,7 @@ func (i *scannerItem) transformMetaDir() {
|
||||
i.objectName = split[len(split)-1]
|
||||
}
|
||||
|
||||
var (
|
||||
applyActionsLogPrefix = color.Green("applyActions:")
|
||||
applyVersionActionsLogPrefix = color.Green("applyVersionActions:")
|
||||
)
|
||||
var applyActionsLogPrefix = color.Green("applyActions:")
|
||||
|
||||
func (i *scannerItem) applyHealing(ctx context.Context, o ObjectLayer, oi ObjectInfo) (size int64) {
|
||||
if i.debug {
|
||||
@@ -974,153 +972,8 @@ func (i *scannerItem) applyHealing(ctx context.Context, o ObjectLayer, oi Object
|
||||
return 0
|
||||
}
|
||||
|
||||
func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi ObjectInfo) (action lifecycle.Action, size int64) {
|
||||
size, err := oi.GetActualSize()
|
||||
if i.debug {
|
||||
scannerLogIf(ctx, err)
|
||||
}
|
||||
if i.lifeCycle == nil {
|
||||
return action, size
|
||||
}
|
||||
|
||||
versionID := oi.VersionID
|
||||
|
||||
var vc *versioning.Versioning
|
||||
var lr objectlock.Retention
|
||||
var rcfg *replication.Config
|
||||
if !isMinioMetaBucketName(i.bucket) {
|
||||
vc, err = globalBucketVersioningSys.Get(i.bucket)
|
||||
if err != nil {
|
||||
scannerLogOnceIf(ctx, err, i.bucket)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if bucket is object locked.
|
||||
lr, err = globalBucketObjectLockSys.Get(i.bucket)
|
||||
if err != nil {
|
||||
scannerLogOnceIf(ctx, err, i.bucket)
|
||||
return
|
||||
}
|
||||
|
||||
rcfg, err = getReplicationConfig(ctx, i.bucket)
|
||||
if err != nil {
|
||||
scannerLogOnceIf(ctx, err, i.bucket)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
lcEvt := evalActionFromLifecycle(ctx, *i.lifeCycle, lr, rcfg, oi)
|
||||
if i.debug {
|
||||
if versionID != "" {
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, lcEvt.Action)
|
||||
} else {
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), lcEvt.Action)
|
||||
}
|
||||
}
|
||||
|
||||
switch lcEvt.Action {
|
||||
// This version doesn't contribute towards sizeS only when it is permanently deleted.
|
||||
// This can happen when,
|
||||
// - ExpireObjectAllVersions flag is enabled
|
||||
// - NoncurrentVersionExpiration is applicable
|
||||
case lifecycle.DeleteVersionAction, lifecycle.DeleteAllVersionsAction, lifecycle.DelMarkerDeleteAllVersionsAction:
|
||||
size = 0
|
||||
case lifecycle.DeleteAction:
|
||||
// On a non-versioned bucket, DeleteObject removes the only version permanently.
|
||||
if !vc.PrefixEnabled(oi.Name) {
|
||||
size = 0
|
||||
}
|
||||
}
|
||||
|
||||
applyLifecycleAction(lcEvt, lcEventSrc_Scanner, oi)
|
||||
return lcEvt.Action, size
|
||||
}
|
||||
|
||||
// applyNewerNoncurrentVersionLimit removes noncurrent versions older than the most recent NewerNoncurrentVersions configured.
|
||||
// Note: This function doesn't update sizeSummary since it always removes versions that it doesn't return.
|
||||
func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ ObjectLayer, fivs []FileInfo, expState *expiryState) ([]ObjectInfo, error) {
|
||||
done := globalScannerMetrics.time(scannerMetricApplyNonCurrent)
|
||||
defer done()
|
||||
|
||||
rcfg, _ := globalBucketObjectLockSys.Get(i.bucket)
|
||||
vcfg, _ := globalBucketVersioningSys.Get(i.bucket)
|
||||
|
||||
versioned := vcfg != nil && vcfg.Versioned(i.objectPath())
|
||||
|
||||
objectInfos := make([]ObjectInfo, 0, len(fivs))
|
||||
|
||||
if i.lifeCycle == nil {
|
||||
for _, fi := range fivs {
|
||||
objectInfos = append(objectInfos, fi.ToObjectInfo(i.bucket, i.objectPath(), versioned))
|
||||
}
|
||||
return objectInfos, nil
|
||||
}
|
||||
|
||||
event := i.lifeCycle.NoncurrentVersionsExpirationLimit(lifecycle.ObjectOpts{Name: i.objectPath()})
|
||||
lim := event.NewerNoncurrentVersions
|
||||
if lim == 0 || len(fivs) <= lim+1 { // fewer than lim _noncurrent_ versions
|
||||
for _, fi := range fivs {
|
||||
objectInfos = append(objectInfos, fi.ToObjectInfo(i.bucket, i.objectPath(), versioned))
|
||||
}
|
||||
return objectInfos, nil
|
||||
}
|
||||
|
||||
overflowVersions := fivs[lim+1:]
|
||||
// Retain the current version + most recent lim noncurrent versions
|
||||
for _, fi := range fivs[:lim+1] {
|
||||
objectInfos = append(objectInfos, fi.ToObjectInfo(i.bucket, i.objectPath(), versioned))
|
||||
}
|
||||
|
||||
toDel := make([]ObjectToDelete, 0, len(overflowVersions))
|
||||
for _, fi := range overflowVersions {
|
||||
obj := fi.ToObjectInfo(i.bucket, i.objectPath(), versioned)
|
||||
// skip versions with object locking enabled
|
||||
if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
|
||||
if i.debug {
|
||||
if obj.VersionID != "" {
|
||||
console.Debugf(applyVersionActionsLogPrefix+" lifecycle: %s v(%s) is locked, not deleting\n", obj.Name, obj.VersionID)
|
||||
} else {
|
||||
console.Debugf(applyVersionActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", obj.Name)
|
||||
}
|
||||
}
|
||||
// add this version back to remaining versions for
|
||||
// subsequent lifecycle policy applications
|
||||
objectInfos = append(objectInfos, obj)
|
||||
continue
|
||||
}
|
||||
|
||||
// NoncurrentDays not passed yet.
|
||||
if time.Now().UTC().Before(lifecycle.ExpectedExpiryTime(obj.SuccessorModTime, event.NoncurrentDays)) {
|
||||
// add this version back to remaining versions for
|
||||
// subsequent lifecycle policy applications
|
||||
objectInfos = append(objectInfos, obj)
|
||||
continue
|
||||
}
|
||||
|
||||
toDel = append(toDel, ObjectToDelete{
|
||||
ObjectV: ObjectV{
|
||||
ObjectName: obj.Name,
|
||||
VersionID: obj.VersionID,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
if len(toDel) > 0 {
|
||||
expState.enqueueByNewerNoncurrent(i.bucket, toDel, event)
|
||||
}
|
||||
return objectInfos, nil
|
||||
}
|
||||
|
||||
// applyVersionActions will apply lifecycle checks on all versions of a scanned item. Returns versions that remain
|
||||
// after applying lifecycle checks configured.
|
||||
func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fivs []FileInfo, expState *expiryState) ([]ObjectInfo, error) {
|
||||
objInfos, err := i.applyNewerNoncurrentVersionLimit(ctx, o, fivs, expState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check if we have many versions after applyNewerNoncurrentVersionLimit.
|
||||
if len(objInfos) >= int(scannerExcessObjectVersions.Load()) {
|
||||
func (i *scannerItem) alertExcessiveVersions(remainingVersions int, cumulativeSize int64) {
|
||||
if remainingVersions >= int(scannerExcessObjectVersions.Load()) {
|
||||
// Notify object accessed via a GET request.
|
||||
sendEvent(eventArgs{
|
||||
EventName: event.ObjectManyVersions,
|
||||
@@ -1130,7 +983,7 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
|
||||
},
|
||||
UserAgent: "Scanner",
|
||||
Host: globalLocalNodeName,
|
||||
RespElements: map[string]string{"x-minio-versions": strconv.Itoa(len(objInfos))},
|
||||
RespElements: map[string]string{"x-minio-versions": strconv.Itoa(remainingVersions)},
|
||||
})
|
||||
|
||||
auditLogInternal(context.Background(), AuditLogOptions{
|
||||
@@ -1139,15 +992,11 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
|
||||
Bucket: i.bucket,
|
||||
Object: i.objectPath(),
|
||||
Tags: map[string]string{
|
||||
"x-minio-versions": strconv.Itoa(len(objInfos)),
|
||||
"x-minio-versions": strconv.Itoa(remainingVersions),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
cumulativeSize := int64(0)
|
||||
for _, objInfo := range objInfos {
|
||||
cumulativeSize += objInfo.Size
|
||||
}
|
||||
// Check if the cumulative size of all versions of this object is high.
|
||||
if cumulativeSize >= scannerExcessObjectVersionsTotalSize.Load() {
|
||||
// Notify object accessed via a GET request.
|
||||
@@ -1160,7 +1009,7 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
|
||||
UserAgent: "Scanner",
|
||||
Host: globalLocalNodeName,
|
||||
RespElements: map[string]string{
|
||||
"x-minio-versions-count": strconv.Itoa(len(objInfos)),
|
||||
"x-minio-versions-count": strconv.Itoa(remainingVersions),
|
||||
"x-minio-versions-size": strconv.FormatInt(cumulativeSize, 10),
|
||||
},
|
||||
})
|
||||
@@ -1171,43 +1020,33 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
|
||||
Bucket: i.bucket,
|
||||
Object: i.objectPath(),
|
||||
Tags: map[string]string{
|
||||
"x-minio-versions-count": strconv.Itoa(len(objInfos)),
|
||||
"x-minio-versions-count": strconv.Itoa(remainingVersions),
|
||||
"x-minio-versions-size": strconv.FormatInt(cumulativeSize, 10),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
return objInfos, nil
|
||||
}
|
||||
|
||||
type actionsAccountingFn func(oi ObjectInfo, sz, actualSz int64, sizeS *sizeSummary)
|
||||
|
||||
// applyActions will apply lifecycle checks on to a scanned item.
|
||||
// The resulting size on disk will always be returned.
|
||||
// The metadata will be compared to consensus on the object layer before any changes are applied.
|
||||
// If no metadata is supplied, -1 is returned if no action is taken.
|
||||
func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) (objDeleted bool, size int64) {
|
||||
done := globalScannerMetrics.time(scannerMetricILM)
|
||||
var action lifecycle.Action
|
||||
action, size = i.applyLifecycle(ctx, o, oi)
|
||||
done()
|
||||
|
||||
// Note: objDeleted is true if and only if action ==
|
||||
// lifecycle.DeleteAllVersionsAction
|
||||
if action.DeleteAll() {
|
||||
return true, 0
|
||||
func (i *scannerItem) applyActions(ctx context.Context, objAPI ObjectLayer, objInfos []ObjectInfo, lr lock.Retention, sizeS *sizeSummary, fn actionsAccountingFn) {
|
||||
if len(objInfos) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// For instance, an applied lifecycle means we remove/transitioned an object
|
||||
// from the current deployment, which means we don't have to call healing
|
||||
// routine even if we are asked to do via heal flag.
|
||||
if action == lifecycle.NoneAction {
|
||||
healActions := func(oi ObjectInfo, actualSz int64) int64 {
|
||||
size := actualSz
|
||||
if i.heal.enabled {
|
||||
done := globalScannerMetrics.time(scannerMetricHealCheck)
|
||||
size = i.applyHealing(ctx, o, oi)
|
||||
size = i.applyHealing(ctx, objAPI, oi)
|
||||
done()
|
||||
|
||||
if healDeleteDangling {
|
||||
done := globalScannerMetrics.time(scannerMetricCleanAbandoned)
|
||||
err := o.CheckAbandonedParts(ctx, i.bucket, i.objectPath(), madmin.HealOpts{Remove: healDeleteDangling})
|
||||
err := objAPI.CheckAbandonedParts(ctx, i.bucket, i.objectPath(), madmin.HealOpts{Remove: healDeleteDangling})
|
||||
done()
|
||||
if err != nil {
|
||||
healingLogIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", i.bucket, i.objectPath(), err), i.objectPath())
|
||||
@@ -1217,10 +1056,109 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
|
||||
|
||||
// replicate only if lifecycle rules are not applied.
|
||||
done := globalScannerMetrics.time(scannerMetricCheckReplication)
|
||||
i.healReplication(ctx, o, oi.Clone(), sizeS)
|
||||
i.healReplication(ctx, oi.Clone(), sizeS)
|
||||
done()
|
||||
return size
|
||||
}
|
||||
return false, size
|
||||
|
||||
vc, err := globalBucketVersioningSys.Get(i.bucket)
|
||||
if err != nil {
|
||||
scannerLogOnceIf(ctx, err, i.bucket)
|
||||
return
|
||||
}
|
||||
|
||||
// start ILM check timer
|
||||
done := globalScannerMetrics.timeN(scannerMetricILM)
|
||||
if i.lifeCycle == nil { // no ILM configured, apply healing and replication checks
|
||||
var cumulativeSize int64
|
||||
for _, oi := range objInfos {
|
||||
actualSz, err := oi.GetActualSize()
|
||||
if err != nil {
|
||||
scannerLogIf(ctx, err)
|
||||
continue
|
||||
}
|
||||
size := healActions(oi, actualSz)
|
||||
if fn != nil { // call accountingfn
|
||||
fn(oi, size, actualSz, sizeS)
|
||||
}
|
||||
cumulativeSize += size
|
||||
}
|
||||
// end ILM check timer
|
||||
done(len(objInfos))
|
||||
i.alertExcessiveVersions(len(objInfos), cumulativeSize)
|
||||
return
|
||||
}
|
||||
objOpts := make([]lifecycle.ObjectOpts, len(objInfos))
|
||||
for i, oi := range objInfos {
|
||||
objOpts[i] = oi.ToLifecycleOpts()
|
||||
}
|
||||
evaluator := lifecycle.NewEvaluator(*i.lifeCycle).WithLockRetention(&lr).WithReplicationConfig(i.replication.Config)
|
||||
events, err := evaluator.Eval(objOpts)
|
||||
if err != nil {
|
||||
// This error indicates that the objOpts passed to Eval is invalid.
|
||||
bugLogIf(ctx, err, i.bucket, i.objectPath())
|
||||
done(len(objInfos)) // end ILM check timer
|
||||
return
|
||||
}
|
||||
done(len(objInfos)) // end ILM check timer
|
||||
|
||||
var (
|
||||
toDel []ObjectToDelete
|
||||
noncurrentEvents []lifecycle.Event
|
||||
cumulativeSize int64
|
||||
)
|
||||
remainingVersions := len(objInfos)
|
||||
eventLoop:
|
||||
for idx, event := range events {
|
||||
oi := objInfos[idx]
|
||||
actualSz, err := oi.GetActualSize()
|
||||
if i.debug {
|
||||
scannerLogIf(ctx, err)
|
||||
}
|
||||
size := actualSz
|
||||
switch event.Action {
|
||||
case lifecycle.DeleteAllVersionsAction, lifecycle.DelMarkerDeleteAllVersionsAction:
|
||||
remainingVersions = 0
|
||||
applyExpiryRule(event, lcEventSrc_Scanner, oi)
|
||||
break eventLoop
|
||||
|
||||
case lifecycle.DeleteAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
|
||||
if !vc.PrefixEnabled(i.objectPath()) && event.Action == lifecycle.DeleteAction {
|
||||
remainingVersions--
|
||||
size = 0
|
||||
}
|
||||
applyExpiryRule(event, lcEventSrc_Scanner, oi)
|
||||
|
||||
case lifecycle.DeleteVersionAction: // noncurrent versions expiration
|
||||
opts := objOpts[idx]
|
||||
remainingVersions--
|
||||
size = 0
|
||||
toDel = append(toDel, ObjectToDelete{
|
||||
ObjectV: ObjectV{
|
||||
ObjectName: opts.Name,
|
||||
VersionID: opts.VersionID,
|
||||
},
|
||||
})
|
||||
noncurrentEvents = append(noncurrentEvents, event)
|
||||
|
||||
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
|
||||
applyTransitionRule(event, lcEventSrc_Scanner, oi)
|
||||
|
||||
case lifecycle.NoneAction:
|
||||
size = healActions(oi, actualSz)
|
||||
}
|
||||
// NB fn must be called for every object version except if it is
|
||||
// expired or was a dangling object.
|
||||
if fn != nil {
|
||||
fn(oi, size, actualSz, sizeS)
|
||||
}
|
||||
cumulativeSize += size
|
||||
}
|
||||
|
||||
if len(toDel) > 0 {
|
||||
globalExpiryState.enqueueNoncurrentVersions(i.bucket, toDel, noncurrentEvents)
|
||||
}
|
||||
i.alertExcessiveVersions(remainingVersions, cumulativeSize)
|
||||
}
|
||||
|
||||
func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, obj ObjectInfo) lifecycle.Event {
|
||||
@@ -1325,10 +1263,13 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
|
||||
dobj, err = objLayer.DeleteObject(ctx, obj.Bucket, encodeDirObject(obj.Name), opts)
|
||||
if err != nil {
|
||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||
traceFn(ILMExpiry, nil, nil)
|
||||
return false
|
||||
}
|
||||
// Assume it is still there.
|
||||
ilmLogOnceIf(ctx, fmt.Errorf("DeleteObject(%s, %s): %w", obj.Bucket, obj.Name, err), "non-transition-expiry"+obj.Name)
|
||||
err := fmt.Errorf("DeleteObject(%s, %s): %w", obj.Bucket, obj.Name, err)
|
||||
ilmLogOnceIf(ctx, err, "non-transition-expiry"+obj.Name)
|
||||
traceFn(ILMExpiry, nil, err)
|
||||
return false
|
||||
}
|
||||
if dobj.Name == "" {
|
||||
@@ -1364,22 +1305,8 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
|
||||
}
|
||||
|
||||
// Apply object, object version, restored object or restored object version action on the given object
|
||||
func applyExpiryRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) bool {
|
||||
func applyExpiryRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) {
|
||||
globalExpiryState.enqueueByDays(obj, event, src)
|
||||
return true
|
||||
}
|
||||
|
||||
// Perform actions (removal or transitioning of objects), return true the action is successfully performed
|
||||
func applyLifecycleAction(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) (success bool) {
|
||||
switch action := event.Action; action {
|
||||
case lifecycle.DeleteVersionAction, lifecycle.DeleteAction,
|
||||
lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction,
|
||||
lifecycle.DeleteAllVersionsAction, lifecycle.DelMarkerDeleteAllVersionsAction:
|
||||
success = applyExpiryRule(event, src, obj)
|
||||
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
|
||||
success = applyTransitionRule(event, src, obj)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// objectPath returns the prefix and object name.
|
||||
@@ -1388,7 +1315,7 @@ func (i *scannerItem) objectPath() string {
|
||||
}
|
||||
|
||||
// healReplication will heal a scanned item that has failed replication.
|
||||
func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) {
|
||||
func (i *scannerItem) healReplication(ctx context.Context, oi ObjectInfo, sizeS *sizeSummary) {
|
||||
if oi.VersionID == "" {
|
||||
return
|
||||
}
|
||||
@@ -1549,7 +1476,7 @@ const (
|
||||
ILMTransition = " ilm:transition"
|
||||
)
|
||||
|
||||
func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags map[string]string, traceFn func(event string, metadata map[string]string)) {
|
||||
func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags map[string]string, traceFn func(event string, metadata map[string]string, err error)) {
|
||||
var apiName string
|
||||
switch event {
|
||||
case ILMExpiry:
|
||||
@@ -1567,5 +1494,5 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags ma
|
||||
VersionID: oi.VersionID,
|
||||
Tags: tags,
|
||||
})
|
||||
traceFn(event, tags)
|
||||
traceFn(event, tags, nil)
|
||||
}
|
||||
|
||||
+239
-71
@@ -18,61 +18,67 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/minio/minio/internal/amztime"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
"github.com/minio/minio/internal/bucket/object/lock"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
func TestApplyNewerNoncurrentVersionsLimit(t *testing.T) {
|
||||
objAPI, disks, err := prepareErasure(context.Background(), 8)
|
||||
// Prepare object layer
|
||||
objAPI, disks, err := prepareErasure(t.Context(), 8)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to initialize object layer: %v", err)
|
||||
}
|
||||
defer removeRoots(disks)
|
||||
setObjectLayer(objAPI)
|
||||
|
||||
// Prepare bucket metadata
|
||||
globalBucketMetadataSys = NewBucketMetadataSys()
|
||||
globalBucketObjectLockSys = &BucketObjectLockSys{}
|
||||
globalBucketVersioningSys = &BucketVersioningSys{}
|
||||
es := newExpiryState(context.Background(), objAPI, 0)
|
||||
workers := []chan expiryOp{make(chan expiryOp)}
|
||||
es.workers.Store(&workers)
|
||||
globalExpiryState = es
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
expired := make([]ObjectToDelete, 0, 5)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
workers := globalExpiryState.workers.Load()
|
||||
for t := range (*workers)[0] {
|
||||
if t, ok := t.(newerNoncurrentTask); ok {
|
||||
expired = append(expired, t.versions...)
|
||||
}
|
||||
}
|
||||
}()
|
||||
lc := lifecycle.Lifecycle{
|
||||
Rules: []lifecycle.Rule{
|
||||
{
|
||||
ID: "max-versions",
|
||||
Status: "Enabled",
|
||||
NoncurrentVersionExpiration: lifecycle.NoncurrentVersionExpiration{
|
||||
NewerNoncurrentVersions: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
lcXML, err := xml.Marshal(lc)
|
||||
|
||||
lcXML := `
|
||||
<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>max-versions</ID>
|
||||
<Status>Enabled</Status>
|
||||
<NoncurrentVersionExpiration>
|
||||
<NewerNoncurrentVersions>2</NewerNoncurrentVersions>
|
||||
</NoncurrentVersionExpiration>
|
||||
</Rule>
|
||||
<Rule>
|
||||
<ID>delete-all-versions</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Filter>
|
||||
<Tag>
|
||||
<Key>del-all</Key>
|
||||
<Value>true</Value>
|
||||
</Tag>
|
||||
</Filter>
|
||||
<Expiration>
|
||||
<Days>1</Days>
|
||||
<ExpiredObjectAllVersions>true</ExpiredObjectAllVersions>
|
||||
</Expiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>
|
||||
`
|
||||
lc, err := lifecycle.ParseLifecycleConfig(strings.NewReader(lcXML))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to marshal lifecycle config: %v", err)
|
||||
t.Fatalf("Failed to unmarshal lifecycle config: %v", err)
|
||||
}
|
||||
|
||||
vcfg := versioning.Versioning{
|
||||
Status: "Enabled",
|
||||
}
|
||||
@@ -82,34 +88,46 @@ func TestApplyNewerNoncurrentVersionsLimit(t *testing.T) {
|
||||
}
|
||||
|
||||
bucket := "bucket"
|
||||
obj := "obj-1"
|
||||
now := time.Now()
|
||||
meta := BucketMetadata{
|
||||
Name: bucket,
|
||||
Created: now,
|
||||
LifecycleConfigXML: lcXML,
|
||||
LifecycleConfigXML: []byte(lcXML),
|
||||
VersioningConfigXML: vcfgXML,
|
||||
VersioningConfigUpdatedAt: now,
|
||||
LifecycleConfigUpdatedAt: now,
|
||||
lifecycleConfig: &lc,
|
||||
lifecycleConfig: lc,
|
||||
versioningConfig: &vcfg,
|
||||
}
|
||||
globalBucketMetadataSys.Set(bucket, meta)
|
||||
item := scannerItem{
|
||||
Path: obj,
|
||||
bucket: bucket,
|
||||
prefix: "",
|
||||
objectName: obj,
|
||||
lifeCycle: &lc,
|
||||
}
|
||||
// Prepare lifecycle expiration workers
|
||||
es := newExpiryState(t.Context(), objAPI, 0)
|
||||
globalExpiryState = es
|
||||
|
||||
modTime := time.Now()
|
||||
// Prepare object versions
|
||||
obj := "obj-1"
|
||||
// Simulate objects uploaded 30 hours ago
|
||||
modTime := now.Add(-48 * time.Hour)
|
||||
uuids := make([]uuid.UUID, 5)
|
||||
for i := range uuids {
|
||||
uuids[i] = uuid.UUID([16]byte{15: uint8(i + 1)})
|
||||
}
|
||||
fivs := make([]FileInfo, 5)
|
||||
for i := 0; i < 5; i++ {
|
||||
objInfos := make([]ObjectInfo, 5)
|
||||
objRetentionMeta := make(map[string]string)
|
||||
objRetentionMeta[strings.ToLower(xhttp.AmzObjectLockMode)] = string(objectlock.RetCompliance)
|
||||
// Set retain until date 12 hours into the future
|
||||
objRetentionMeta[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(now.Add(12 * time.Hour))
|
||||
/*
|
||||
objInfos:
|
||||
version stack for obj-1
|
||||
v5 uuid-5 modTime
|
||||
v4 uuid-4 modTime -1m
|
||||
v3 uuid-3 modTime -2m
|
||||
v2 uuid-2 modTime -3m
|
||||
v1 uuid-1 modTime -4m
|
||||
*/
|
||||
for i := range 5 {
|
||||
fivs[i] = FileInfo{
|
||||
Volume: bucket,
|
||||
Name: obj,
|
||||
@@ -119,41 +137,189 @@ func TestApplyNewerNoncurrentVersionsLimit(t *testing.T) {
|
||||
Size: 1 << 10,
|
||||
NumVersions: 5,
|
||||
}
|
||||
objInfos[i] = fivs[i].ToObjectInfo(bucket, obj, true)
|
||||
}
|
||||
versioned := vcfg.Status == "Enabled"
|
||||
wants := make([]ObjectInfo, 2)
|
||||
for i, fi := range fivs[:2] {
|
||||
wants[i] = fi.ToObjectInfo(bucket, obj, versioned)
|
||||
/*
|
||||
lrObjInfos: objInfos with following modifications
|
||||
version stack for obj-1
|
||||
v2 uuid-2 modTime -3m objRetentionMeta
|
||||
*/
|
||||
lrObjInfos := slices.Clone(objInfos)
|
||||
lrObjInfos[3].UserDefined = objRetentionMeta
|
||||
var lrWants []ObjectInfo
|
||||
lrWants = append(lrWants, lrObjInfos[:4]...)
|
||||
|
||||
/*
|
||||
replObjInfos: objInfos with following modifications
|
||||
version stack for obj-1
|
||||
v1 uuid-1 modTime -4m "VersionPurgeStatus: replication.VersionPurgePending"
|
||||
*/
|
||||
replObjInfos := slices.Clone(objInfos)
|
||||
replObjInfos[4].VersionPurgeStatus = replication.VersionPurgePending
|
||||
var replWants []ObjectInfo
|
||||
replWants = append(replWants, replObjInfos[:3]...)
|
||||
replWants = append(replWants, replObjInfos[4])
|
||||
|
||||
allVersExpObjInfos := slices.Clone(objInfos)
|
||||
allVersExpObjInfos[0].UserTags = "del-all=true"
|
||||
|
||||
replCfg := replication.Config{
|
||||
Rules: []replication.Rule{
|
||||
{
|
||||
ID: "",
|
||||
Status: "Enabled",
|
||||
Priority: 1,
|
||||
Destination: replication.Destination{
|
||||
ARN: "arn:minio:replication:::dest-bucket",
|
||||
Bucket: "dest-bucket",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
gots, err := item.applyNewerNoncurrentVersionLimit(context.TODO(), objAPI, fivs, es)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed with err: %v", err)
|
||||
}
|
||||
if len(gots) != len(wants) {
|
||||
t.Fatalf("Expected %d objects but got %d", len(wants), len(gots))
|
||||
lr := objectlock.Retention{
|
||||
Mode: objectlock.RetCompliance,
|
||||
Validity: 12 * time.Hour,
|
||||
LockEnabled: true,
|
||||
}
|
||||
|
||||
// Close expiry state's channel to inspect object versions enqueued for expiration
|
||||
close(workers[0])
|
||||
wg.Wait()
|
||||
for _, obj := range expired {
|
||||
switch obj.ObjectV.VersionID {
|
||||
case uuids[2].String(), uuids[3].String(), uuids[4].String():
|
||||
default:
|
||||
t.Errorf("Unexpected versionID being expired: %#v\n", obj)
|
||||
expiryWorker := func(wg *sync.WaitGroup, readyCh chan<- struct{}, taskCh <-chan expiryOp, gotExpired *[]ObjectToDelete) {
|
||||
defer wg.Done()
|
||||
// signal the calling goroutine that the worker is ready tor receive tasks
|
||||
close(readyCh)
|
||||
var expired []ObjectToDelete
|
||||
for t := range taskCh {
|
||||
switch v := t.(type) {
|
||||
case noncurrentVersionsTask:
|
||||
expired = append(expired, v.versions...)
|
||||
case expiryTask:
|
||||
expired = append(expired, ObjectToDelete{
|
||||
ObjectV: ObjectV{
|
||||
ObjectName: v.objInfo.Name,
|
||||
VersionID: v.objInfo.VersionID,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
if len(expired) > 0 {
|
||||
*gotExpired = expired
|
||||
}
|
||||
}
|
||||
tests := []struct {
|
||||
replCfg replicationConfig
|
||||
lr objectlock.Retention
|
||||
objInfos []ObjectInfo
|
||||
wants []ObjectInfo
|
||||
wantExpired []ObjectToDelete
|
||||
}{
|
||||
{
|
||||
// With replication configured, version(s) with PENDING purge status
|
||||
replCfg: replicationConfig{Config: &replCfg},
|
||||
objInfos: replObjInfos,
|
||||
wants: replWants,
|
||||
wantExpired: []ObjectToDelete{
|
||||
{ObjectV: ObjectV{ObjectName: obj, VersionID: objInfos[3].VersionID}},
|
||||
},
|
||||
},
|
||||
{
|
||||
// With lock retention configured and version(s) with retention metadata
|
||||
lr: lr,
|
||||
objInfos: lrObjInfos,
|
||||
wants: lrWants,
|
||||
wantExpired: []ObjectToDelete{
|
||||
{ObjectV: ObjectV{ObjectName: obj, VersionID: objInfos[4].VersionID}},
|
||||
},
|
||||
},
|
||||
{
|
||||
// With replication configured, but no versions with PENDING purge status
|
||||
replCfg: replicationConfig{Config: &replCfg},
|
||||
objInfos: objInfos,
|
||||
wants: objInfos[:3],
|
||||
wantExpired: []ObjectToDelete{
|
||||
{ObjectV: ObjectV{ObjectName: obj, VersionID: objInfos[3].VersionID}},
|
||||
{ObjectV: ObjectV{ObjectName: obj, VersionID: objInfos[4].VersionID}},
|
||||
},
|
||||
},
|
||||
{
|
||||
objInfos: allVersExpObjInfos,
|
||||
wants: nil,
|
||||
wantExpired: []ObjectToDelete{{ObjectV: ObjectV{ObjectName: obj, VersionID: allVersExpObjInfos[0].VersionID}}},
|
||||
},
|
||||
{
|
||||
// When no versions are present, in practice this could be an object with only free versions
|
||||
objInfos: nil,
|
||||
wants: nil,
|
||||
wantExpired: nil,
|
||||
},
|
||||
}
|
||||
for i, test := range tests {
|
||||
t.Run(fmt.Sprintf("TestApplyNewerNoncurrentVersionsLimit-%d", i), func(t *testing.T) {
|
||||
workers := []chan expiryOp{make(chan expiryOp)}
|
||||
es.workers.Store(&workers)
|
||||
workerReady := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
var gotExpired []ObjectToDelete
|
||||
go expiryWorker(&wg, workerReady, workers[0], &gotExpired)
|
||||
<-workerReady
|
||||
|
||||
item := scannerItem{
|
||||
Path: obj,
|
||||
bucket: bucket,
|
||||
prefix: "",
|
||||
objectName: obj,
|
||||
lifeCycle: lc,
|
||||
replication: test.replCfg,
|
||||
}
|
||||
|
||||
var (
|
||||
sizeS sizeSummary
|
||||
gots []ObjectInfo
|
||||
)
|
||||
item.applyActions(t.Context(), objAPI, test.objInfos, test.lr, &sizeS, func(oi ObjectInfo, sz, _ int64, _ *sizeSummary) {
|
||||
if sz != 0 {
|
||||
gots = append(gots, oi)
|
||||
}
|
||||
})
|
||||
|
||||
if len(gots) != len(test.wants) {
|
||||
t.Fatalf("Expected %d objects but got %d", len(test.wants), len(gots))
|
||||
}
|
||||
if slices.CompareFunc(gots, test.wants, func(g, w ObjectInfo) int {
|
||||
if g.VersionID == w.VersionID {
|
||||
return 0
|
||||
}
|
||||
return -1
|
||||
}) != 0 {
|
||||
t.Fatalf("Expected %v but got %v", test.wants, gots)
|
||||
}
|
||||
// verify the objects to be deleted
|
||||
close(workers[0])
|
||||
wg.Wait()
|
||||
if len(gotExpired) != len(test.wantExpired) {
|
||||
t.Fatalf("Expected expiry of %d objects but got %d", len(test.wantExpired), len(gotExpired))
|
||||
}
|
||||
if slices.CompareFunc(gotExpired, test.wantExpired, func(g, w ObjectToDelete) int {
|
||||
if g.VersionID == w.VersionID {
|
||||
return 0
|
||||
}
|
||||
return -1
|
||||
}) != 0 {
|
||||
t.Fatalf("Expected %v but got %v", test.wantExpired, gotExpired)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvalActionFromLifecycle(t *testing.T) {
|
||||
// Tests cover only ExpiredObjectDeleteAllVersions and DelMarkerExpiration actions
|
||||
numVersions := 4
|
||||
obj := ObjectInfo{
|
||||
Name: "foo",
|
||||
ModTime: time.Now().Add(-31 * 24 * time.Hour),
|
||||
Size: 100 << 20,
|
||||
VersionID: uuid.New().String(),
|
||||
IsLatest: true,
|
||||
NumVersions: 4,
|
||||
NumVersions: numVersions,
|
||||
}
|
||||
delMarker := ObjectInfo{
|
||||
Name: "foo-deleted",
|
||||
@@ -162,8 +328,9 @@ func TestEvalActionFromLifecycle(t *testing.T) {
|
||||
VersionID: uuid.New().String(),
|
||||
IsLatest: true,
|
||||
DeleteMarker: true,
|
||||
NumVersions: 4,
|
||||
NumVersions: numVersions,
|
||||
}
|
||||
|
||||
deleteAllILM := `<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<Expiration>
|
||||
@@ -195,35 +362,35 @@ func TestEvalActionFromLifecycle(t *testing.T) {
|
||||
}
|
||||
tests := []struct {
|
||||
ilm lifecycle.Lifecycle
|
||||
retention lock.Retention
|
||||
retention *objectlock.Retention
|
||||
obj ObjectInfo
|
||||
want lifecycle.Action
|
||||
}{
|
||||
{
|
||||
// with object locking
|
||||
ilm: *deleteAllLc,
|
||||
retention: lock.Retention{LockEnabled: true},
|
||||
retention: &objectlock.Retention{LockEnabled: true},
|
||||
obj: obj,
|
||||
want: lifecycle.NoneAction,
|
||||
},
|
||||
{
|
||||
// without object locking
|
||||
ilm: *deleteAllLc,
|
||||
retention: lock.Retention{},
|
||||
retention: &objectlock.Retention{},
|
||||
obj: obj,
|
||||
want: lifecycle.DeleteAllVersionsAction,
|
||||
},
|
||||
{
|
||||
// with object locking
|
||||
ilm: *delMarkerLc,
|
||||
retention: lock.Retention{LockEnabled: true},
|
||||
retention: &objectlock.Retention{LockEnabled: true},
|
||||
obj: delMarker,
|
||||
want: lifecycle.NoneAction,
|
||||
},
|
||||
{
|
||||
// without object locking
|
||||
ilm: *delMarkerLc,
|
||||
retention: lock.Retention{},
|
||||
retention: &objectlock.Retention{},
|
||||
obj: delMarker,
|
||||
want: lifecycle.DelMarkerDeleteAllVersionsAction,
|
||||
},
|
||||
@@ -231,8 +398,9 @@ func TestEvalActionFromLifecycle(t *testing.T) {
|
||||
|
||||
for i, test := range tests {
|
||||
t.Run(fmt.Sprintf("TestEvalAction-%d", i), func(t *testing.T) {
|
||||
if got := evalActionFromLifecycle(context.TODO(), test.ilm, test.retention, nil, test.obj); got.Action != test.want {
|
||||
t.Fatalf("Expected %v but got %v", test.want, got)
|
||||
gotEvent := evalActionFromLifecycle(t.Context(), test.ilm, *test.retention, nil, test.obj)
|
||||
if gotEvent.Action != test.want {
|
||||
t.Fatalf("Expected %v but got %v", test.want, gotEvent.Action)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+51
-290
@@ -22,6 +22,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"path"
|
||||
@@ -39,6 +40,8 @@ import (
|
||||
"github.com/valyala/bytebufferpool"
|
||||
)
|
||||
|
||||
//msgp:clearomitted
|
||||
|
||||
//go:generate msgp -file $GOFILE -unexported
|
||||
|
||||
// dataUsageHash is the hash type used.
|
||||
@@ -57,15 +60,14 @@ type versionsHistogram [dataUsageVersionLen]uint64
|
||||
type dataUsageEntry struct {
|
||||
Children dataUsageHashMap `msg:"ch"`
|
||||
// These fields do no include any children.
|
||||
Size int64 `msg:"sz"`
|
||||
Objects uint64 `msg:"os"`
|
||||
Versions uint64 `msg:"vs"` // Versions that are not delete markers.
|
||||
DeleteMarkers uint64 `msg:"dms"`
|
||||
ObjSizes sizeHistogram `msg:"szs"`
|
||||
ObjVersions versionsHistogram `msg:"vh"`
|
||||
ReplicationStats *replicationAllStats `msg:"rs,omitempty"`
|
||||
AllTierStats *allTierStats `msg:"ats,omitempty"`
|
||||
Compacted bool `msg:"c"`
|
||||
Size int64 `msg:"sz"`
|
||||
Objects uint64 `msg:"os"`
|
||||
Versions uint64 `msg:"vs"` // Versions that are not delete markers.
|
||||
DeleteMarkers uint64 `msg:"dms"`
|
||||
ObjSizes sizeHistogram `msg:"szs"`
|
||||
ObjVersions versionsHistogram `msg:"vh"`
|
||||
AllTierStats *allTierStats `msg:"ats,omitempty"`
|
||||
Compacted bool `msg:"c"`
|
||||
}
|
||||
|
||||
// allTierStats is a collection of per-tier stats across all configured remote
|
||||
@@ -98,9 +100,7 @@ func (ats *allTierStats) clone() *allTierStats {
|
||||
}
|
||||
dst := *ats
|
||||
dst.Tiers = make(map[string]tierStats, len(ats.Tiers))
|
||||
for tier, st := range ats.Tiers {
|
||||
dst.Tiers[tier] = st
|
||||
}
|
||||
maps.Copy(dst.Tiers, ats.Tiers)
|
||||
return &dst
|
||||
}
|
||||
|
||||
@@ -117,7 +117,6 @@ func (ats *allTierStats) populateStats(stats map[string]madmin.TierStats) {
|
||||
NumObjects: st.NumObjects,
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// tierStats holds per-tier stats of a remote tier.
|
||||
@@ -135,93 +134,6 @@ func (ts tierStats) add(u tierStats) tierStats {
|
||||
}
|
||||
}
|
||||
|
||||
//msgp:tuple replicationStatsV1
|
||||
type replicationStatsV1 struct {
|
||||
PendingSize uint64
|
||||
ReplicatedSize uint64
|
||||
FailedSize uint64
|
||||
ReplicaSize uint64
|
||||
FailedCount uint64
|
||||
PendingCount uint64
|
||||
MissedThresholdSize uint64
|
||||
AfterThresholdSize uint64
|
||||
MissedThresholdCount uint64
|
||||
AfterThresholdCount uint64
|
||||
}
|
||||
|
||||
func (rsv1 replicationStatsV1) Empty() bool {
|
||||
return rsv1.ReplicatedSize == 0 &&
|
||||
rsv1.FailedSize == 0 &&
|
||||
rsv1.FailedCount == 0
|
||||
}
|
||||
|
||||
//msgp:tuple replicationStats
|
||||
type replicationStats struct {
|
||||
PendingSize uint64
|
||||
ReplicatedSize uint64
|
||||
FailedSize uint64
|
||||
FailedCount uint64
|
||||
PendingCount uint64
|
||||
MissedThresholdSize uint64
|
||||
AfterThresholdSize uint64
|
||||
MissedThresholdCount uint64
|
||||
AfterThresholdCount uint64
|
||||
ReplicatedCount uint64
|
||||
}
|
||||
|
||||
func (rs replicationStats) Empty() bool {
|
||||
return rs.ReplicatedSize == 0 &&
|
||||
rs.FailedSize == 0 &&
|
||||
rs.FailedCount == 0
|
||||
}
|
||||
|
||||
type replicationAllStats struct {
|
||||
Targets map[string]replicationStats `msg:"t,omitempty"`
|
||||
ReplicaSize uint64 `msg:"r,omitempty"`
|
||||
ReplicaCount uint64 `msg:"rc,omitempty"`
|
||||
}
|
||||
|
||||
//msgp:tuple replicationAllStatsV1
|
||||
type replicationAllStatsV1 struct {
|
||||
Targets map[string]replicationStats
|
||||
ReplicaSize uint64 `msg:"ReplicaSize,omitempty"`
|
||||
ReplicaCount uint64 `msg:"ReplicaCount,omitempty"`
|
||||
}
|
||||
|
||||
// empty returns true if the replicationAllStats is empty (contains no entries).
|
||||
func (r *replicationAllStats) empty() bool {
|
||||
if r == nil {
|
||||
return true
|
||||
}
|
||||
if r.ReplicaSize != 0 || r.ReplicaCount != 0 {
|
||||
return false
|
||||
}
|
||||
for _, v := range r.Targets {
|
||||
if !v.Empty() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// clone creates a deep-copy clone.
|
||||
func (r *replicationAllStats) clone() *replicationAllStats {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shallow copy
|
||||
dst := *r
|
||||
|
||||
// Copy individual targets.
|
||||
dst.Targets = make(map[string]replicationStats, len(r.Targets))
|
||||
for k, v := range r.Targets {
|
||||
dst.Targets[k] = v
|
||||
}
|
||||
|
||||
return &dst
|
||||
}
|
||||
|
||||
//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7
|
||||
//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7
|
||||
|
||||
@@ -237,62 +149,54 @@ type dataUsageEntryV2 struct {
|
||||
//msgp:tuple dataUsageEntryV3
|
||||
type dataUsageEntryV3 struct {
|
||||
// These fields do no include any children.
|
||||
Size int64
|
||||
ReplicatedSize uint64
|
||||
ReplicationPendingSize uint64
|
||||
ReplicationFailedSize uint64
|
||||
ReplicaSize uint64
|
||||
Objects uint64
|
||||
ObjSizes sizeHistogram
|
||||
Children dataUsageHashMap
|
||||
Size int64
|
||||
Objects uint64
|
||||
ObjSizes sizeHistogram
|
||||
Children dataUsageHashMap
|
||||
}
|
||||
|
||||
//msgp:tuple dataUsageEntryV4
|
||||
type dataUsageEntryV4 struct {
|
||||
Children dataUsageHashMap
|
||||
// These fields do no include any children.
|
||||
Size int64
|
||||
Objects uint64
|
||||
ObjSizes sizeHistogram
|
||||
ReplicationStats replicationStatsV1
|
||||
Size int64
|
||||
Objects uint64
|
||||
ObjSizes sizeHistogram
|
||||
}
|
||||
|
||||
//msgp:tuple dataUsageEntryV5
|
||||
type dataUsageEntryV5 struct {
|
||||
Children dataUsageHashMap
|
||||
// These fields do no include any children.
|
||||
Size int64
|
||||
Objects uint64
|
||||
Versions uint64 // Versions that are not delete markers.
|
||||
ObjSizes sizeHistogram
|
||||
ReplicationStats *replicationStatsV1
|
||||
Compacted bool
|
||||
Size int64
|
||||
Objects uint64
|
||||
Versions uint64 // Versions that are not delete markers.
|
||||
ObjSizes sizeHistogram
|
||||
Compacted bool
|
||||
}
|
||||
|
||||
//msgp:tuple dataUsageEntryV6
|
||||
type dataUsageEntryV6 struct {
|
||||
Children dataUsageHashMap
|
||||
// These fields do no include any children.
|
||||
Size int64
|
||||
Objects uint64
|
||||
Versions uint64 // Versions that are not delete markers.
|
||||
ObjSizes sizeHistogram
|
||||
ReplicationStats *replicationAllStatsV1
|
||||
Compacted bool
|
||||
Size int64
|
||||
Objects uint64
|
||||
Versions uint64 // Versions that are not delete markers.
|
||||
ObjSizes sizeHistogram
|
||||
Compacted bool
|
||||
}
|
||||
|
||||
type dataUsageEntryV7 struct {
|
||||
Children dataUsageHashMap `msg:"ch"`
|
||||
// These fields do no include any children.
|
||||
Size int64 `msg:"sz"`
|
||||
Objects uint64 `msg:"os"`
|
||||
Versions uint64 `msg:"vs"` // Versions that are not delete markers.
|
||||
DeleteMarkers uint64 `msg:"dms"`
|
||||
ObjSizes sizeHistogramV1 `msg:"szs"`
|
||||
ObjVersions versionsHistogram `msg:"vh"`
|
||||
ReplicationStats *replicationAllStats `msg:"rs,omitempty"`
|
||||
AllTierStats *allTierStats `msg:"ats,omitempty"`
|
||||
Compacted bool `msg:"c"`
|
||||
Size int64 `msg:"sz"`
|
||||
Objects uint64 `msg:"os"`
|
||||
Versions uint64 `msg:"vs"` // Versions that are not delete markers.
|
||||
DeleteMarkers uint64 `msg:"dms"`
|
||||
ObjSizes sizeHistogramV1 `msg:"szs"`
|
||||
ObjVersions versionsHistogram `msg:"vh"`
|
||||
AllTierStats *allTierStats `msg:"ats,omitempty"`
|
||||
Compacted bool `msg:"c"`
|
||||
}
|
||||
|
||||
// dataUsageCache contains a cache of data usage entries latest version.
|
||||
@@ -373,29 +277,6 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) {
|
||||
e.ObjSizes.add(summary.totalSize)
|
||||
e.ObjVersions.add(summary.versions)
|
||||
|
||||
if e.ReplicationStats == nil {
|
||||
e.ReplicationStats = &replicationAllStats{
|
||||
Targets: make(map[string]replicationStats),
|
||||
}
|
||||
} else if e.ReplicationStats.Targets == nil {
|
||||
e.ReplicationStats.Targets = make(map[string]replicationStats)
|
||||
}
|
||||
e.ReplicationStats.ReplicaSize += uint64(summary.replicaSize)
|
||||
e.ReplicationStats.ReplicaCount += uint64(summary.replicaCount)
|
||||
|
||||
for arn, st := range summary.replTargetStats {
|
||||
tgtStat, ok := e.ReplicationStats.Targets[arn]
|
||||
if !ok {
|
||||
tgtStat = replicationStats{}
|
||||
}
|
||||
tgtStat.PendingSize += uint64(st.pendingSize)
|
||||
tgtStat.FailedSize += uint64(st.failedSize)
|
||||
tgtStat.ReplicatedSize += uint64(st.replicatedSize)
|
||||
tgtStat.ReplicatedCount += uint64(st.replicatedCount)
|
||||
tgtStat.FailedCount += st.failedCount
|
||||
tgtStat.PendingCount += st.pendingCount
|
||||
e.ReplicationStats.Targets[arn] = tgtStat
|
||||
}
|
||||
if len(summary.tiers) != 0 {
|
||||
if e.AllTierStats == nil {
|
||||
e.AllTierStats = newAllTierStats()
|
||||
@@ -410,26 +291,6 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
|
||||
e.Versions += other.Versions
|
||||
e.DeleteMarkers += other.DeleteMarkers
|
||||
e.Size += other.Size
|
||||
if other.ReplicationStats != nil {
|
||||
if e.ReplicationStats == nil {
|
||||
e.ReplicationStats = &replicationAllStats{Targets: make(map[string]replicationStats)}
|
||||
} else if e.ReplicationStats.Targets == nil {
|
||||
e.ReplicationStats.Targets = make(map[string]replicationStats)
|
||||
}
|
||||
e.ReplicationStats.ReplicaSize += other.ReplicationStats.ReplicaSize
|
||||
e.ReplicationStats.ReplicaCount += other.ReplicationStats.ReplicaCount
|
||||
for arn, stat := range other.ReplicationStats.Targets {
|
||||
st := e.ReplicationStats.Targets[arn]
|
||||
e.ReplicationStats.Targets[arn] = replicationStats{
|
||||
PendingSize: stat.PendingSize + st.PendingSize,
|
||||
FailedSize: stat.FailedSize + st.FailedSize,
|
||||
ReplicatedSize: stat.ReplicatedSize + st.ReplicatedSize,
|
||||
PendingCount: stat.PendingCount + st.PendingCount,
|
||||
FailedCount: stat.FailedCount + st.FailedCount,
|
||||
ReplicatedCount: stat.ReplicatedCount + st.ReplicatedCount,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for i, v := range other.ObjSizes[:] {
|
||||
e.ObjSizes[i] += v
|
||||
@@ -485,15 +346,10 @@ func (e dataUsageEntry) clone() dataUsageEntry {
|
||||
// We operate on a copy from the receiver.
|
||||
if e.Children != nil {
|
||||
ch := make(dataUsageHashMap, len(e.Children))
|
||||
for k, v := range e.Children {
|
||||
ch[k] = v
|
||||
}
|
||||
maps.Copy(ch, e.Children)
|
||||
e.Children = ch
|
||||
}
|
||||
if e.ReplicationStats != nil {
|
||||
// Clone ReplicationStats
|
||||
e.ReplicationStats = e.ReplicationStats.clone()
|
||||
}
|
||||
|
||||
if e.AllTierStats != nil {
|
||||
e.AllTierStats = e.AllTierStats.clone()
|
||||
}
|
||||
@@ -931,22 +787,6 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
|
||||
ObjectSizesHistogram: flat.ObjSizes.toMap(),
|
||||
ObjectVersionsHistogram: flat.ObjVersions.toMap(),
|
||||
}
|
||||
if flat.ReplicationStats != nil {
|
||||
bui.ReplicaSize = flat.ReplicationStats.ReplicaSize
|
||||
bui.ReplicaCount = flat.ReplicationStats.ReplicaCount
|
||||
|
||||
bui.ReplicationInfo = make(map[string]BucketTargetUsageInfo, len(flat.ReplicationStats.Targets))
|
||||
for arn, stat := range flat.ReplicationStats.Targets {
|
||||
bui.ReplicationInfo[arn] = BucketTargetUsageInfo{
|
||||
ReplicationPendingSize: stat.PendingSize,
|
||||
ReplicatedSize: stat.ReplicatedSize,
|
||||
ReplicationFailedSize: stat.FailedSize,
|
||||
ReplicationPendingCount: stat.PendingCount,
|
||||
ReplicationFailedCount: stat.FailedCount,
|
||||
ReplicatedCount: stat.ReplicatedCount,
|
||||
}
|
||||
}
|
||||
}
|
||||
dst[bucket.Name] = bui
|
||||
}
|
||||
return dst
|
||||
@@ -959,9 +799,6 @@ func (d *dataUsageCache) sizeRecursive(path string) *dataUsageEntry {
|
||||
return root
|
||||
}
|
||||
flat := d.flatten(*root)
|
||||
if flat.ReplicationStats.empty() {
|
||||
flat.ReplicationStats = nil
|
||||
}
|
||||
return &flat
|
||||
}
|
||||
|
||||
@@ -1238,20 +1075,6 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
|
||||
ObjSizes: v.ObjSizes,
|
||||
Children: v.Children,
|
||||
}
|
||||
if v.ReplicatedSize > 0 || v.ReplicaSize > 0 || v.ReplicationFailedSize > 0 || v.ReplicationPendingSize > 0 {
|
||||
cfg, _ := getReplicationConfig(GlobalContext, d.Info.Name)
|
||||
if cfg != nil && cfg.RoleArn != "" {
|
||||
due.ReplicationStats = &replicationAllStats{
|
||||
Targets: make(map[string]replicationStats),
|
||||
}
|
||||
due.ReplicationStats.ReplicaSize = v.ReplicaSize
|
||||
due.ReplicationStats.Targets[cfg.RoleArn] = replicationStats{
|
||||
ReplicatedSize: v.ReplicatedSize,
|
||||
FailedSize: v.ReplicationFailedSize,
|
||||
PendingSize: v.ReplicationPendingSize,
|
||||
}
|
||||
}
|
||||
}
|
||||
due.Compacted = len(due.Children) == 0 && k != d.Info.Name
|
||||
|
||||
d.Cache[k] = due
|
||||
@@ -1277,36 +1100,10 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
|
||||
ObjSizes: v.ObjSizes,
|
||||
Children: v.Children,
|
||||
}
|
||||
empty := replicationStatsV1{}
|
||||
|
||||
if v.ReplicationStats != empty {
|
||||
cfg, _ := getReplicationConfig(GlobalContext, d.Info.Name)
|
||||
if cfg != nil && cfg.RoleArn != "" {
|
||||
due.ReplicationStats = &replicationAllStats{
|
||||
Targets: make(map[string]replicationStats),
|
||||
}
|
||||
due.ReplicationStats.Targets[cfg.RoleArn] = replicationStats{
|
||||
ReplicatedSize: v.ReplicationStats.ReplicatedSize,
|
||||
FailedSize: v.ReplicationStats.FailedSize,
|
||||
FailedCount: v.ReplicationStats.FailedCount,
|
||||
PendingSize: v.ReplicationStats.PendingSize,
|
||||
PendingCount: v.ReplicationStats.PendingCount,
|
||||
}
|
||||
due.ReplicationStats.ReplicaSize = v.ReplicationStats.ReplicaSize
|
||||
}
|
||||
}
|
||||
due.Compacted = len(due.Children) == 0 && k != d.Info.Name
|
||||
|
||||
d.Cache[k] = due
|
||||
}
|
||||
|
||||
// Populate compacted value and remove unneeded replica stats.
|
||||
for k, e := range d.Cache {
|
||||
if e.ReplicationStats != nil && len(e.ReplicationStats.Targets) == 0 {
|
||||
e.ReplicationStats = nil
|
||||
}
|
||||
d.Cache[k] = e
|
||||
}
|
||||
return nil
|
||||
case dataUsageCacheVerV5:
|
||||
// Zstd compressed.
|
||||
@@ -1328,36 +1125,10 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
|
||||
ObjSizes: v.ObjSizes,
|
||||
Children: v.Children,
|
||||
}
|
||||
if v.ReplicationStats != nil && !v.ReplicationStats.Empty() {
|
||||
cfg, _ := getReplicationConfig(GlobalContext, d.Info.Name)
|
||||
if cfg != nil && cfg.RoleArn != "" {
|
||||
due.ReplicationStats = &replicationAllStats{
|
||||
Targets: make(map[string]replicationStats),
|
||||
}
|
||||
d.Info.replication = replicationConfig{Config: cfg}
|
||||
|
||||
due.ReplicationStats.Targets[cfg.RoleArn] = replicationStats{
|
||||
ReplicatedSize: v.ReplicationStats.ReplicatedSize,
|
||||
FailedSize: v.ReplicationStats.FailedSize,
|
||||
FailedCount: v.ReplicationStats.FailedCount,
|
||||
PendingSize: v.ReplicationStats.PendingSize,
|
||||
PendingCount: v.ReplicationStats.PendingCount,
|
||||
}
|
||||
due.ReplicationStats.ReplicaSize = v.ReplicationStats.ReplicaSize
|
||||
}
|
||||
}
|
||||
due.Compacted = len(due.Children) == 0 && k != d.Info.Name
|
||||
|
||||
d.Cache[k] = due
|
||||
}
|
||||
|
||||
// Populate compacted value and remove unneeded replica stats.
|
||||
for k, e := range d.Cache {
|
||||
if e.ReplicationStats != nil && len(e.ReplicationStats.Targets) == 0 {
|
||||
e.ReplicationStats = nil
|
||||
}
|
||||
d.Cache[k] = e
|
||||
}
|
||||
return nil
|
||||
case dataUsageCacheVerV6:
|
||||
// Zstd compressed.
|
||||
@@ -1373,22 +1144,13 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
|
||||
d.Info = dold.Info
|
||||
d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
|
||||
for k, v := range dold.Cache {
|
||||
var replicationStats *replicationAllStats
|
||||
if v.ReplicationStats != nil {
|
||||
replicationStats = &replicationAllStats{
|
||||
Targets: v.ReplicationStats.Targets,
|
||||
ReplicaSize: v.ReplicationStats.ReplicaSize,
|
||||
ReplicaCount: v.ReplicationStats.ReplicaCount,
|
||||
}
|
||||
}
|
||||
due := dataUsageEntry{
|
||||
Children: v.Children,
|
||||
Size: v.Size,
|
||||
Objects: v.Objects,
|
||||
Versions: v.Versions,
|
||||
ObjSizes: v.ObjSizes,
|
||||
ReplicationStats: replicationStats,
|
||||
Compacted: v.Compacted,
|
||||
Children: v.Children,
|
||||
Size: v.Size,
|
||||
Objects: v.Objects,
|
||||
Versions: v.Versions,
|
||||
ObjSizes: v.ObjSizes,
|
||||
Compacted: v.Compacted,
|
||||
}
|
||||
d.Cache[k] = due
|
||||
}
|
||||
@@ -1410,13 +1172,12 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
|
||||
var szHist sizeHistogram
|
||||
szHist.mergeV1(v.ObjSizes)
|
||||
d.Cache[k] = dataUsageEntry{
|
||||
Children: v.Children,
|
||||
Size: v.Size,
|
||||
Objects: v.Objects,
|
||||
Versions: v.Versions,
|
||||
ObjSizes: szHist,
|
||||
ReplicationStats: v.ReplicationStats,
|
||||
Compacted: v.Compacted,
|
||||
Children: v.Children,
|
||||
Size: v.Size,
|
||||
Objects: v.Objects,
|
||||
Versions: v.Versions,
|
||||
ObjSizes: szHist,
|
||||
Compacted: v.Compacted,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+615
-1330
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user