mirror of
https://github.com/pgsty/minio.git
synced 2026-08-02 08:35:58 +03:00
Compare commits
329 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 752e18e795 | |||
| 76d822bf1e | |||
| ddeca9f12a | |||
| 19d0340ddf | |||
| 21251d8c22 | |||
| 1f3db03bf0 | |||
| 944c62daf4 | |||
| 9547b7d0e9 | |||
| 76c4ea7682 | |||
| 808ecfe0f2 | |||
| 2894dd4d1a | |||
| 797fa7f97b | |||
| fd8750e959 | |||
| 7be65f66b8 | |||
| 4f5d38a4b1 | |||
| 7e73fc2870 | |||
| d2c9a9e395 | |||
| 0d49b365ff | |||
| 7721595aa9 | |||
| fd6f6fc8df | |||
| 4fb47cd568 | |||
| ecc932d5dd | |||
| b57fbff7c1 | |||
| 4892a766a8 | |||
| 0303cd8625 | |||
| d765b89a63 | |||
| 6e4acf0504 | |||
| 71954faa3a | |||
| 6d22e74d11 | |||
| dc92bb4646 | |||
| 0f0e154315 | |||
| 136d41775f | |||
| ec77d28e62 | |||
| 86420a1f46 | |||
| 7dd8b6c8ed | |||
| 8afa6fefd8 | |||
| 533c9d4fe3 | |||
| a35ef155fc | |||
| 8dd3c41b2a | |||
| 4523da6543 | |||
| ce8456a1a9 | |||
| 1673778633 | |||
| 9ce1884732 | |||
| 23b329b9df | |||
| 0c34e51a75 | |||
| 1633b30979 | |||
| 630dabf4b9 | |||
| fc6c794972 | |||
| 2e33b99c6b | |||
| 3b7292b637 | |||
| e4f469ae7a | |||
| c921dc75c7 | |||
| 86d543d0f6 | |||
| e4e90b53c1 | |||
| 58d776daa0 | |||
| f6b2e89109 | |||
| ac85c2af76 | |||
| 5aba2aedb3 | |||
| bd77f1df4c | |||
| a8332efa94 | |||
| 3dbef72dc7 | |||
| 2d16e74f38 | |||
| de5070446d | |||
| 374abd1e7d | |||
| 0506d9e83d | |||
| bd3dfad8b9 | |||
| 9fff315555 | |||
| 07b6dce1a5 | |||
| 18fb86b7be | |||
| 58a8275e84 | |||
| 196fab6834 | |||
| 85fc7cea97 | |||
| 328d660106 | |||
| c68910005b | |||
| c79bcc8838 | |||
| 0fe58dbb34 | |||
| 6cb2f56395 | |||
| 59e33b3b21 | |||
| 0e3c92c027 | |||
| db7a9b2c37 | |||
| 44097faec1 | |||
| ff5fca76ab | |||
| bf3da5081f | |||
| 5532982857 | |||
| 783dd875f7 | |||
| 97112c69be | |||
| b0b573052a | |||
| 2939000342 | |||
| 41e1654f9a | |||
| e3cb0278ce | |||
| 0c81f1bdb3 | |||
| 6220875803 | |||
| afd4279cd8 | |||
| 0c8dd8046a | |||
| 3c4ef4338f | |||
| 64cf887b28 | |||
| 927a879052 | |||
| dfe0c96b87 | |||
| e856e10ac2 | |||
| 6d6a731d6d | |||
| 928feb0889 | |||
| b3febe2d24 | |||
| b6b26dba87 | |||
| 5c034e26bd | |||
| cef0fb1434 | |||
| 158d0e26a2 | |||
| 78385bfbeb | |||
| 2a13cc28f2 | |||
| 4d761fda81 | |||
| 4bdf41a6c7 | |||
| 3c605c93fe | |||
| 121f18a443 | |||
| 538aeef27a | |||
| be0d2537b7 | |||
| 3307aa1260 | |||
| 57cfdfd8fb | |||
| dc6733dacc | |||
| f696a221af | |||
| ed5b67720c | |||
| 2aac50571d | |||
| 45edd27ad7 | |||
| c302d1cfc8 | |||
| 6287e8c571 | |||
| f69a98ce49 | |||
| d44f3526dc | |||
| 41b633f5ea | |||
| 4f1ff9c4d9 | |||
| 86bb48792c | |||
| 94dbb4a427 | |||
| 048a46ec2a | |||
| 1480340830 | |||
| 877bd95fa3 | |||
| 8ea6fb368d | |||
| 5fd5ddea23 | |||
| b04c0697e1 | |||
| 50a8ba6a6f | |||
| 334f1ed45a | |||
| 20c89ebbb3 | |||
| 6f56ba80b3 | |||
| 6e84283c66 | |||
| 9d6fddcfdf | |||
| a83105df9d | |||
| 9528b55c25 | |||
| 749ce107ee | |||
| b2a67834ac | |||
| aec2aa3497 | |||
| c7dcbfd6c1 | |||
| ff12080ff5 | |||
| 0b6175b742 | |||
| cf49da387b | |||
| ac714e7e3d | |||
| 79fb79b71c | |||
| 98874c3baf | |||
| d89f6af6c4 | |||
| d4bca00df9 | |||
| 2c68a19dfd | |||
| 9e5853ecc0 | |||
| 124544d834 | |||
| b910904fa6 | |||
| eee1ce305c | |||
| 5c61c3ccdc | |||
| fb8d512f58 | |||
| a0fb0c1835 | |||
| e152b2a975 | |||
| a71629d4dd | |||
| c22f3ca7a8 | |||
| 4a92134235 | |||
| 6b9fd256e1 | |||
| d6132b854f | |||
| ff9a74b91f | |||
| b579163802 | |||
| 87f0c8e7e8 | |||
| bb855499e1 | |||
| 96bfa77856 | |||
| 8e997eba4a | |||
| 228c6686f8 | |||
| 52861d3aea | |||
| cc26911c46 | |||
| 7776d064cf | |||
| 2d9b5a65f1 | |||
| c240da6568 | |||
| 157272dc5b | |||
| 9065274d02 | |||
| f4c56026a2 | |||
| 37e3f5de10 | |||
| 5ea629beb2 | |||
| 240164560f | |||
| cf52691959 | |||
| 10e75116ef | |||
| f649968c69 | |||
| 5ce1448049 | |||
| bcedc2b0d9 | |||
| 8e4a45ec41 | |||
| dec942beb6 | |||
| d4e0f13bb3 | |||
| 1f28a3bb80 | |||
| 3a1d3a7952 | |||
| a9f1ad7924 | |||
| 929b9e164e | |||
| 97376f6e8f | |||
| 92a0a59de2 | |||
| cd18599e7b | |||
| 1f22a16b15 | |||
| 433b6fa8fe | |||
| d7cd857c7c | |||
| 99fbfe2421 | |||
| b1b6264bea | |||
| 1fd72d5aea | |||
| 18dffb26e7 | |||
| edba7c987b | |||
| b737c83a66 | |||
| 97a6322de1 | |||
| 037fe4afdc | |||
| afbb63a197 | |||
| 8902561f3c | |||
| a67116b5bc | |||
| 0f7aa4125f | |||
| b62a5c954c | |||
| b8cdf060c8 | |||
| 9fb937986e | |||
| 2c48f6a02b | |||
| 4155c5b695 | |||
| 471467d310 | |||
| c54c13831a | |||
| ae4ee95d25 | |||
| a2e037f0ec | |||
| e9055e9ef7 | |||
| d350b666ff | |||
| 21831b3fe2 | |||
| 895357607a | |||
| ac240a8477 | |||
| bf38c0c0d1 | |||
| 67cf15d036 | |||
| 701a82642b | |||
| 21fe14201f | |||
| 48640b1de2 | |||
| 5682685c80 | |||
| 9c025b8cce | |||
| eef9f13360 | |||
| fa9b361a3d | |||
| 49862ba347 | |||
| ee2afcf70b | |||
| c7d535c648 | |||
| 9986e103cf | |||
| c5b3666089 | |||
| d265fe7f9e | |||
| 91e6af4470 | |||
| b940fe8fca | |||
| 73fe2e95fe | |||
| 316c492842 | |||
| 74418b542a | |||
| 172e63dbb6 | |||
| 21bf5b4db7 | |||
| a406bb0288 | |||
| 1823ab6808 | |||
| 8eec49304d | |||
| ecdc2f2f5f | |||
| 6a6c772ff2 | |||
| e178c55bc3 | |||
| 2c137c0d04 | |||
| 1d35f2b58f | |||
| 638c57e466 | |||
| 5e4213b3be | |||
| 102295f58a | |||
| a0d14f8ff7 | |||
| e0b0a351c6 | |||
| fcd4b3ba9b | |||
| 1d2ff46a89 | |||
| 8f7c739328 | |||
| 1beea3daba | |||
| 1ffd063939 | |||
| 3d94c38ec4 | |||
| f4af2d3cdc | |||
| b57e7321e7 | |||
| e93867488b | |||
| c08790edd2 | |||
| a46baddbc4 | |||
| 3bd9615d0e | |||
| 2871cb5775 | |||
| a6e0ec4e6f | |||
| e956369c4e | |||
| 76f950c663 | |||
| d774a3309b | |||
| b3edb25377 | |||
| 026b87e39b | |||
| 53a816b17a | |||
| 043aaa792d | |||
| aad9cb208a | |||
| edf081c6a2 | |||
| fd349103e8 | |||
| 10b49eb4fb | |||
| 3856d078d2 | |||
| 6b4cb35f4f | |||
| e6eab2091f | |||
| 3cdb609cca | |||
| d6a7f62ff5 | |||
| 72f170f5d2 | |||
| 824d52a82b | |||
| 067ebab9d8 | |||
| 6be6c0d2e3 | |||
| aa874010e2 | |||
| 8ec888d13d | |||
| 916f274c83 | |||
| 7ac53c07af | |||
| bc72e4226e | |||
| 5e0776e96a | |||
| 2f1ef02d35 | |||
| db8442584e | |||
| d46cf50760 | |||
| 0357121d17 | |||
| aff236e20e | |||
| cbd70d26b5 | |||
| 5e763b71dc | |||
| 7e4e7a66af | |||
| 906947a285 | |||
| bfc70bc74e | |||
| 6b4f833a12 | |||
| 426c902b87 | |||
| e4b51235f8 | |||
| 4c6498d726 | |||
| 0a3b1ad4eb | |||
| f23f442d33 | |||
| e465c3587b | |||
| 7109b6d414 | |||
| 8c97f3e9bc | |||
| 3795b2c8ba | |||
| 7725425e05 | |||
| b2f4948bbe | |||
| f802d2ba83 |
@@ -15,5 +15,6 @@
|
||||
|
||||
## Checklist:
|
||||
- [ ] Fixes a regression (If yes, please add `commit-id` or `PR #` here)
|
||||
- [ ] Documentation updated
|
||||
- [ ] Unit tests added/updated
|
||||
- [ ] Internal documentation updated
|
||||
- [ ] Create a documentation update request [here](https://github.com/minio/docs/issues/new?label=doc-change,title=Doc+Updated+Needed+For+PR+github.com%2fminio%2fminio%2fpull%2fNNNNN)
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
name: 'Dependency Review'
|
||||
on: [pull_request]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
dependency-review:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@v3
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@v1
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.17.x, 1.18.x]
|
||||
go-version: [1.18.x, 1.19.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.17.11b7, 1.18.3b7]
|
||||
go-version: [1.18.5b7]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.17.x]
|
||||
go-version: [1.18.x, 1.19.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
@@ -28,14 +28,6 @@ jobs:
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ matrix.go-version }}-go-
|
||||
- name: Build on ${{ matrix.os }}
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
env:
|
||||
@@ -49,3 +41,4 @@ jobs:
|
||||
make verify-healing
|
||||
make verify-healing-inconsistent-versions
|
||||
make verify-healing-with-root-disks
|
||||
make verify-healing-with-rewrite
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.17.x, 1.18.x]
|
||||
go-version: [1.18.x, 1.19.x]
|
||||
os: [ubuntu-latest, windows-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
@@ -28,30 +28,13 @@ jobs:
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- uses: actions/cache@v2
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ matrix.go-version }}-go-
|
||||
- uses: actions/cache@v2
|
||||
if: matrix.os == 'windows-latest'
|
||||
with:
|
||||
path: |
|
||||
%LocalAppData%\go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ matrix.go-version }}-go-
|
||||
- name: Build on ${{ matrix.os }}
|
||||
if: matrix.os == 'windows-latest'
|
||||
env:
|
||||
CGO_ENABLED: 0
|
||||
GO111MODULE: on
|
||||
run: |
|
||||
netsh int ipv4 set dynamicport tcp start=60000 num=61000
|
||||
go build --ldflags="-s -w" -o %GOPATH%\bin\minio.exe
|
||||
go test -v --timeout 50m ./...
|
||||
- name: Build on ${{ matrix.os }}
|
||||
@@ -63,9 +46,6 @@ jobs:
|
||||
sudo apt install jq -y
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
nancy_version=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/sonatype-nexus-community/nancy/releases/latest | sed "s/https:\/\/github.com\/sonatype-nexus-community\/nancy\/releases\/tag\///")
|
||||
curl -L -o nancy https://github.com/sonatype-nexus-community/nancy/releases/download/${nancy_version}/nancy-${nancy_version}-linux-amd64 && chmod +x nancy
|
||||
go list -deps -json ./... | jq -s 'unique_by(.Module.Path)|.[]|select(has("Module"))|.Module' | ./nancy sleuth
|
||||
make
|
||||
make test
|
||||
make test-race
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.17.x, 1.18.x]
|
||||
go-version: [1.18.x, 1.19.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
@@ -28,23 +28,12 @@ jobs:
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ matrix.go-version }}-go-
|
||||
- name: Build on ${{ matrix.os }}
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
env:
|
||||
CGO_ENABLED: 0
|
||||
GO111MODULE: on
|
||||
MINIO_KMS_KES_CERT_FILE: /home/runner/work/minio/minio/.github/workflows/root.cert
|
||||
MINIO_KMS_KES_KEY_FILE: /home/runner/work/minio/minio/.github/workflows/root.key
|
||||
MINIO_KMS_KES_ENDPOINT: "https://play.min.io:7373"
|
||||
MINIO_KMS_KES_KEY_NAME: "my-minio-key"
|
||||
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
|
||||
MINIO_KMS_AUTO_ENCRYPTION: on
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
# are turned off - i.e. if ldap="", then ldap server is not enabled for
|
||||
# the tests.
|
||||
matrix:
|
||||
go-version: [1.17.x]
|
||||
go-version: [1.18.x]
|
||||
ldap: ["", "localhost:389"]
|
||||
etcd: ["", "http://localhost:2379"]
|
||||
openid: ["", "http://127.0.0.1:5556/dex"]
|
||||
@@ -80,14 +80,6 @@ jobs:
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ matrix.go-version }}-go-
|
||||
- name: Test LDAP/OpenID/Etcd combo
|
||||
env:
|
||||
LDAP_TEST_SERVER: ${{ matrix.ldap }}
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.17.x]
|
||||
go-version: [1.18.x, 1.19.x]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
@@ -29,14 +29,6 @@ jobs:
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ matrix.go-version }}-go-
|
||||
- name: Test Decom
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.17.x]
|
||||
go-version: [1.18.x, 1.19.x]
|
||||
os: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
name: VulnCheck
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
jobs:
|
||||
vulncheck:
|
||||
name: Analysis
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v3
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: 1.19.x
|
||||
check-latest: true
|
||||
- name: Get official govulncheck
|
||||
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
shell: bash
|
||||
- name: Run govulncheck
|
||||
run: govulncheck ./...
|
||||
shell: bash
|
||||
+1
-1
@@ -25,7 +25,7 @@ mc.*
|
||||
s3-check-md5*
|
||||
xl-meta*
|
||||
healing-*
|
||||
inspect*
|
||||
inspect*.zip
|
||||
200M*
|
||||
hash-set
|
||||
minio.RELEASE*
|
||||
|
||||
+3
-16
@@ -1,6 +1,6 @@
|
||||
linters-settings:
|
||||
golint:
|
||||
min-confidence: 0
|
||||
gofumpt:
|
||||
lang-version: "1.18"
|
||||
|
||||
misspell:
|
||||
locale: US
|
||||
@@ -14,28 +14,15 @@ linters:
|
||||
- govet
|
||||
- revive
|
||||
- ineffassign
|
||||
- gosimple
|
||||
- deadcode
|
||||
- structcheck
|
||||
- gomodguard
|
||||
- gofmt
|
||||
- unused
|
||||
- structcheck
|
||||
- unconvert
|
||||
- varcheck
|
||||
- unused
|
||||
- gocritic
|
||||
- gofumpt
|
||||
- tenv
|
||||
- durationcheck
|
||||
|
||||
linters-settings:
|
||||
gofumpt:
|
||||
lang-version: "1.17"
|
||||
|
||||
# Choose whether or not to use the extra rules that are disabled
|
||||
# by default
|
||||
extra-rules: false
|
||||
|
||||
issues:
|
||||
exclude-use-default: false
|
||||
exclude:
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
CVE-2020-26160
|
||||
CVE-2020-15136
|
||||
CVE-2020-15115
|
||||
CVE-2020-15114
|
||||
@@ -15,12 +15,12 @@ checks: ## check dependencies
|
||||
@(env bash $(PWD)/buildscripts/checkdeps.sh)
|
||||
|
||||
help: ## print this help
|
||||
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' Makefile | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
|
||||
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' Makefile | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-40s\033[0m %s\n", $$1, $$2}'
|
||||
|
||||
getdeps: ## fetch necessary dependencies
|
||||
@mkdir -p ${GOPATH}/bin
|
||||
@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.45.2
|
||||
@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7-0.20211026165309-e818a1881b0e
|
||||
@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin
|
||||
@echo "Installing msgp" && go install -v github.com/tinylib/msgp@f3635b96e4838a6c773babb65ef35297fe5fe2f9
|
||||
@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
|
||||
|
||||
crosscompile: ## cross compile minio
|
||||
@@ -89,12 +89,18 @@ verify-healing: ## verify healing and replacing disks with minio binary
|
||||
@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
||||
@(env bash $(PWD)/buildscripts/verify-healing.sh)
|
||||
@(env bash $(PWD)/buildscripts/unaligned-healing.sh)
|
||||
@(env bash $(PWD)/buildscripts/heal-inconsistent-versions.sh)
|
||||
|
||||
verify-healing-with-root-disks:
|
||||
@echo "Verify healing with root disks"
|
||||
verify-healing-with-root-disks: ## verify healing root disks
|
||||
@echo "Verify healing with root drives"
|
||||
@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
||||
@(env bash $(PWD)/buildscripts/verify-healing-with-root-disks.sh)
|
||||
|
||||
verify-healing-with-rewrite: ## verify healing to rewrite old xl.meta -> new xl.meta
|
||||
@echo "Verify healing with rewrite"
|
||||
@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
||||
@(env bash $(PWD)/buildscripts/rewrite-old-new.sh)
|
||||
|
||||
verify-healing-inconsistent-versions: ## verify resolving inconsistent versions
|
||||
@echo "Verify resolving inconsistent versions build with race"
|
||||
@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
||||
|
||||
@@ -14,7 +14,7 @@ Use the following commands to run a standalone MinIO server as a container.
|
||||
|
||||
Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication
|
||||
require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically,
|
||||
with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html)
|
||||
with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html)
|
||||
for more complete documentation.
|
||||
|
||||
### Stable
|
||||
@@ -32,7 +32,7 @@ root credentials. You can use the Browser to create buckets, upload objects, and
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See
|
||||
[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
|
||||
see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
|
||||
see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
|
||||
|
||||
> NOTE: To deploy MinIO on with persistent storage, you must map local persistent directories from the host OS to the container using the `podman -v` option. For example, `-v /mnt/data:/data` maps the host OS drive at `/mnt/data` to `/data` on the container.
|
||||
|
||||
@@ -40,7 +40,7 @@ see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to vi
|
||||
|
||||
Use the following commands to run a standalone MinIO server on macOS.
|
||||
|
||||
Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) for more complete documentation.
|
||||
Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html) for more complete documentation.
|
||||
|
||||
### Homebrew (recommended)
|
||||
|
||||
@@ -60,7 +60,7 @@ brew install minio/stable/minio
|
||||
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html/> to view MinIO SDKs for supported languages.
|
||||
|
||||
### Binary Download
|
||||
|
||||
@@ -74,7 +74,7 @@ chmod +x minio
|
||||
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
|
||||
|
||||
## GNU/Linux
|
||||
|
||||
@@ -99,9 +99,9 @@ The following table lists supported architectures. Replace the `wget` URL with t
|
||||
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
|
||||
|
||||
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) for more complete documentation.
|
||||
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html#) for more complete documentation.
|
||||
|
||||
## Microsoft Windows
|
||||
|
||||
@@ -119,23 +119,23 @@ minio.exe server D:\
|
||||
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
|
||||
|
||||
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) for more complete documentation.
|
||||
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html#) for more complete documentation.
|
||||
|
||||
## Install from Source
|
||||
|
||||
Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.17](https://golang.org/dl/#stable)
|
||||
Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.18](https://golang.org/dl/#stable)
|
||||
|
||||
```sh
|
||||
GO111MODULE=on go install github.com/minio/minio@latest
|
||||
go install github.com/minio/minio@latest
|
||||
```
|
||||
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
|
||||
|
||||
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) for more complete documentation.
|
||||
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html) for more complete documentation.
|
||||
|
||||
MinIO strongly recommends *against* using compiled-from-source MinIO servers for production environments.
|
||||
|
||||
@@ -222,7 +222,7 @@ For example: `export MINIO_SERVER_URL="https://minio.example.net"`
|
||||
|
||||
## Test using MinIO Client `mc`
|
||||
|
||||
`mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the MinIO Client [Quickstart Guide](https://docs.min.io/docs/minio-client-quickstart-guide) for further instructions.
|
||||
`mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the MinIO Client [Quickstart Guide](https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart) for further instructions.
|
||||
|
||||
## Upgrading MinIO
|
||||
|
||||
@@ -230,7 +230,7 @@ Upgrades require zero downtime in MinIO, all upgrades are non-disruptive, all tr
|
||||
|
||||
> NOTE: requires internet access to update directly from <https://dl.min.io>, optionally you can host any mirrors at <https://my-artifactory.example.com/minio/>
|
||||
|
||||
- For deployments that installed the MinIO server binary by hand, use [`mc admin update`](https://docs.min.io/minio/baremetal/reference/minio-mc-admin/mc-admin-update.html)
|
||||
- For deployments that installed the MinIO server binary by hand, use [`mc admin update`](https://min.io/docs/minio/linux/reference/minio-mc-admin/mc-admin-update.html)
|
||||
|
||||
```sh
|
||||
mc admin update <minio alias, e.g., myminio>
|
||||
@@ -250,12 +250,10 @@ mc admin update <minio alias, e.g., myminio>
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [MinIO Erasure Code QuickStart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)
|
||||
- [Use `mc` with MinIO Server](https://docs.min.io/docs/minio-client-quickstart-guide)
|
||||
- [Use `aws-cli` with MinIO Server](https://docs.min.io/docs/aws-cli-with-minio)
|
||||
- [Use `s3cmd` with MinIO Server](https://docs.min.io/docs/s3cmd-with-minio)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/docs/golang-client-quickstart-guide)
|
||||
- [The MinIO documentation website](https://docs.min.io)
|
||||
- [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html)
|
||||
- [Use `mc` with MinIO Server](https://min.io/docs/minio/linux/reference/minio-mc.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://min.io/docs/minio/linux/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://min.io/docs/minio/linux/index.html)
|
||||
|
||||
## Contribute to MinIO Project
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ function _init() {
|
||||
export CGO_ENABLED=0
|
||||
|
||||
## List of architectures and OS to test coss compilation.
|
||||
SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64"
|
||||
SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64"
|
||||
}
|
||||
|
||||
function _build() {
|
||||
|
||||
Executable
+93
@@ -0,0 +1,93 @@
|
||||
#!/bin/bash -e
|
||||
|
||||
set -E
|
||||
set -o pipefail
|
||||
set -x
|
||||
|
||||
WORK_DIR="$PWD/.verify-$RANDOM"
|
||||
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
|
||||
MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
|
||||
|
||||
if [ ! -x "$PWD/minio" ]; then
|
||||
echo "minio executable binary not found in current directory"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -x "$PWD/minio" ]; then
|
||||
echo "minio executable binary not found in current directory"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
function start_minio_4drive() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ROOT_USER=minio
|
||||
export MINIO_ROOT_PASSWORD=minio123
|
||||
export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
|
||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
mkdir ${WORK_DIR}
|
||||
C_PWD=${PWD}
|
||||
if [ ! -x "$PWD/mc" ]; then
|
||||
MC_BUILD_DIR="mc-$RANDOM"
|
||||
if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
|
||||
echo "failed to download https://github.com/minio/mc"
|
||||
purge "${MC_BUILD_DIR}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
(cd "${MC_BUILD_DIR}" && go build -o "$C_PWD/mc")
|
||||
|
||||
# remove mc source.
|
||||
purge "${MC_BUILD_DIR}"
|
||||
fi
|
||||
|
||||
"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/disk{1...4}" > "${WORK_DIR}/server1.log" 2>&1 &
|
||||
pid=$!
|
||||
disown $pid
|
||||
sleep 5
|
||||
|
||||
if ! ps -p ${pid} 1>&2 >/dev/null; then
|
||||
echo "server1 log:"
|
||||
cat "${WORK_DIR}/server1.log"
|
||||
echo "FAILED"
|
||||
purge "$WORK_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"${PWD}/mc" mb --with-versioning minio/bucket
|
||||
|
||||
for i in $(seq 1 4); do
|
||||
"${PWD}/mc" cp /etc/hosts minio/bucket/testobj
|
||||
|
||||
sudo chown -R root. "${WORK_DIR}/disk${i}"
|
||||
|
||||
"${PWD}/mc" cp /etc/hosts minio/bucket/testobj
|
||||
|
||||
sudo chown -R ${USER}. "${WORK_DIR}/disk${i}"
|
||||
done
|
||||
|
||||
for vid in $("${PWD}/mc" ls --json --versions minio/bucket/testobj | jq -r .versionId); do
|
||||
"${PWD}/mc" cat --vid "${vid}" minio/bucket/testobj | md5sum
|
||||
done
|
||||
|
||||
pkill minio
|
||||
sleep 3
|
||||
}
|
||||
|
||||
function main() {
|
||||
start_port=$(shuf -i 10000-65000 -n 1)
|
||||
|
||||
start_minio_4drive ${start_port}
|
||||
}
|
||||
|
||||
function purge()
|
||||
{
|
||||
rm -rf "$1"
|
||||
}
|
||||
|
||||
( main "$@" )
|
||||
rv=$?
|
||||
purge "$WORK_DIR"
|
||||
exit "$rv"
|
||||
@@ -0,0 +1,87 @@
|
||||
//go:build ignore
|
||||
// +build ignore
|
||||
|
||||
//
|
||||
// MinIO Object Storage (c) 2022 MinIO, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
|
||||
// dummy values, please replace them with original values.
|
||||
|
||||
// API requests are secure (HTTPS) if secure=true and insecure (HTTP) otherwise.
|
||||
// New returns an MinIO Admin client object.
|
||||
madmClnt, err := madmin.New(os.Args[1], os.Args[2], os.Args[3], false)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
opts := madmin.HealOpts{
|
||||
Recursive: true, // recursively heal all objects at 'prefix'
|
||||
Remove: true, // remove content that has lost quorum and not recoverable
|
||||
Recreate: true, // rewrite all old non-inlined xl.meta to new xl.meta
|
||||
ScanMode: madmin.HealNormalScan, // by default do not do 'deep' scanning
|
||||
}
|
||||
|
||||
start, _, err := madmClnt.Heal(context.Background(), "healing-rewrite-bucket", "", opts, "", false, false)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
fmt.Println("Healstart sequence ===")
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
if err = enc.Encode(&start); err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
for {
|
||||
_, status, err := madmClnt.Heal(context.Background(), "healing-rewrite-bucket", "", opts, start.ClientToken, false, false)
|
||||
if status.Summary == "finished" {
|
||||
fmt.Println("Healstatus on items ===")
|
||||
for _, item := range status.Items {
|
||||
if err = enc.Encode(&item); err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
if status.Summary == "stopped" {
|
||||
fmt.Println("Healstatus on items ===")
|
||||
fmt.Println("Heal failed with", status.FailureDetail)
|
||||
break
|
||||
}
|
||||
|
||||
for _, item := range status.Items {
|
||||
if err = enc.Encode(&item); err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
}
|
||||
@@ -66,7 +66,7 @@ __init__() {
|
||||
mc mb minio/minio-test/
|
||||
mc cp ./minio minio/minio-test/to-read/
|
||||
mc cp /etc/hosts minio/minio-test/to-read/hosts
|
||||
mc policy set download minio/minio-test
|
||||
mc anonymous set download minio/minio-test
|
||||
|
||||
verify_checksum_mc ./minio minio/minio-test/to-read/minio
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ function start_minio_5drive() {
|
||||
"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/cicd-corpus/disk{1...5}" > "${WORK_DIR}/server1.log" 2>&1 &
|
||||
pid=$!
|
||||
disown $pid
|
||||
sleep 30
|
||||
sleep 5
|
||||
|
||||
if ! ps -p ${pid} 1>&2 >/dev/null; then
|
||||
echo "server1 log:"
|
||||
|
||||
Executable
+151
@@ -0,0 +1,151 @@
|
||||
#!/bin/bash -e
|
||||
|
||||
set -E
|
||||
set -o pipefail
|
||||
set -x
|
||||
|
||||
WORK_DIR="$PWD/.verify-$RANDOM"
|
||||
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
|
||||
MINIO_OLD=( "$PWD/minio.RELEASE.2020-10-28T08-16-50Z" --config-dir "$MINIO_CONFIG_DIR" server )
|
||||
MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
|
||||
|
||||
if [ ! -x "$PWD/minio" ]; then
|
||||
echo "minio executable binary not found in current directory"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
function download_old_release() {
|
||||
if [ ! -f minio.RELEASE.2020-10-28T08-16-50Z ]; then
|
||||
curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2020-10-28T08-16-50Z
|
||||
chmod a+x minio.RELEASE.2020-10-28T08-16-50Z
|
||||
fi
|
||||
}
|
||||
|
||||
function verify_rewrite() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ACCESS_KEY=minio
|
||||
export MINIO_SECRET_KEY=minio123
|
||||
export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
|
||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
MC_BUILD_DIR="mc-$RANDOM"
|
||||
if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
|
||||
echo "failed to download https://github.com/minio/mc"
|
||||
purge "${MC_BUILD_DIR}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
(cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
|
||||
|
||||
# remove mc source.
|
||||
purge "${MC_BUILD_DIR}"
|
||||
|
||||
"${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
|
||||
pid=$!
|
||||
disown $pid
|
||||
sleep 10
|
||||
|
||||
if ! ps -p ${pid} 1>&2 >/dev/null; then
|
||||
echo "server1 log:"
|
||||
cat "${WORK_DIR}/server1.log"
|
||||
echo "FAILED"
|
||||
purge "$WORK_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"${WORK_DIR}/mc" mb minio/healing-rewrite-bucket --quiet --with-lock
|
||||
"${WORK_DIR}/mc" cp \
|
||||
buildscripts/verify-build.sh \
|
||||
minio/healing-rewrite-bucket/ \
|
||||
--disable-multipart --quiet
|
||||
|
||||
"${WORK_DIR}/mc" cp \
|
||||
buildscripts/verify-build.sh \
|
||||
minio/healing-rewrite-bucket/ \
|
||||
--disable-multipart --quiet
|
||||
|
||||
"${WORK_DIR}/mc" cp \
|
||||
buildscripts/verify-build.sh \
|
||||
minio/healing-rewrite-bucket/ \
|
||||
--disable-multipart --quiet
|
||||
|
||||
kill ${pid}
|
||||
sleep 3
|
||||
|
||||
"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
|
||||
pid=$!
|
||||
disown $pid
|
||||
sleep 10
|
||||
|
||||
if ! ps -p ${pid} 1>&2 >/dev/null; then
|
||||
echo "server1 log:"
|
||||
cat "${WORK_DIR}/server1.log"
|
||||
echo "FAILED"
|
||||
purge "$WORK_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
go build ./docs/debugging/s3-check-md5/
|
||||
if ! ./s3-check-md5 \
|
||||
-debug \
|
||||
-versions \
|
||||
-access-key minio \
|
||||
-secret-key minio123 \
|
||||
-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
|
||||
echo "server1 log:"
|
||||
cat "${WORK_DIR}/server1.log"
|
||||
echo "FAILED"
|
||||
mkdir -p inspects
|
||||
(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**)
|
||||
|
||||
"${WORK_DIR}/mc" mb play/inspects
|
||||
"${WORK_DIR}/mc" mirror inspects play/inspects
|
||||
|
||||
purge "$WORK_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "minio" "minio123"
|
||||
sleep 1
|
||||
|
||||
if ! ./s3-check-md5 \
|
||||
-debug \
|
||||
-versions \
|
||||
-access-key minio \
|
||||
-secret-key minio123 \
|
||||
-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
|
||||
echo "server1 log:"
|
||||
cat "${WORK_DIR}/server1.log"
|
||||
echo "FAILED"
|
||||
mkdir -p inspects
|
||||
(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**)
|
||||
|
||||
"${WORK_DIR}/mc" mb play/inspects
|
||||
"${WORK_DIR}/mc" mirror inspects play/inspects
|
||||
|
||||
purge "$WORK_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
kill ${pid}
|
||||
}
|
||||
|
||||
function main() {
|
||||
download_old_release
|
||||
|
||||
start_port=$(shuf -i 10000-65000 -n 1)
|
||||
|
||||
verify_rewrite ${start_port}
|
||||
}
|
||||
|
||||
function purge()
|
||||
{
|
||||
rm -rf "$1"
|
||||
}
|
||||
|
||||
( main "$@" )
|
||||
rv=$?
|
||||
purge "$WORK_DIR"
|
||||
exit "$rv"
|
||||
@@ -120,7 +120,7 @@ function start_minio_16drive() {
|
||||
cat "${WORK_DIR}/server1.log"
|
||||
echo "FAILED"
|
||||
mkdir -p inspects
|
||||
(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-shard-bucket/unaligned/**)
|
||||
(cd inspects; "${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**)
|
||||
|
||||
"${WORK_DIR}/mc" mb play/inspects
|
||||
"${WORK_DIR}/mc" mirror inspects play/inspects
|
||||
@@ -140,7 +140,7 @@ function start_minio_16drive() {
|
||||
cat "${WORK_DIR}/server1.log"
|
||||
echo "FAILED"
|
||||
mkdir -p inspects
|
||||
(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-shard-bucket/unaligned/**)
|
||||
(cd inspects; "${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**)
|
||||
|
||||
"${WORK_DIR}/mc" mb play/inspects
|
||||
"${WORK_DIR}/mc" mirror inspects play/inspects
|
||||
|
||||
+2
-2
@@ -80,7 +80,7 @@ func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.
|
||||
}
|
||||
|
||||
// Before proceeding validate if bucket exists.
|
||||
_, err := objAPI.GetBucketInfo(ctx, bucket)
|
||||
_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -142,7 +142,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
|
||||
}
|
||||
|
||||
// Before proceeding validate if bucket exists.
|
||||
_, err := objAPI.GetBucketInfo(ctx, bucket)
|
||||
_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
|
||||
+183
-196
@@ -25,7 +25,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -69,12 +68,12 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
|
||||
vars := mux.Vars(r)
|
||||
bucket := pathClean(vars["bucket"])
|
||||
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
data, err := ioutil.ReadAll(r.Body)
|
||||
data, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||
return
|
||||
@@ -131,7 +130,7 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
|
||||
vars := mux.Vars(r)
|
||||
bucket := pathClean(vars["bucket"])
|
||||
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -161,11 +160,6 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
|
||||
bucket := pathClean(vars["bucket"])
|
||||
update := r.Form.Get("update") == "true"
|
||||
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketTargetAction)
|
||||
if objectAPI == nil {
|
||||
@@ -173,7 +167,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -249,7 +243,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
|
||||
}
|
||||
if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, update); err != nil {
|
||||
switch err.(type) {
|
||||
case BucketRemoteConnectionErr:
|
||||
case RemoteTargetConnectionErr:
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationRemoteConnectionError, err), r.URL)
|
||||
default:
|
||||
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
@@ -290,10 +284,6 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
|
||||
bucket := pathClean(vars["bucket"])
|
||||
arnType := vars["type"]
|
||||
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetBucketTargetAction)
|
||||
if objectAPI == nil {
|
||||
@@ -301,7 +291,7 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
|
||||
}
|
||||
if bucket != "" {
|
||||
// Check if bucket exists.
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -329,10 +319,6 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
|
||||
bucket := pathClean(vars["bucket"])
|
||||
arn := vars["arn"]
|
||||
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketTargetAction)
|
||||
if objectAPI == nil {
|
||||
@@ -340,7 +326,7 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -374,10 +360,6 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
bucket := pathClean(r.Form.Get("bucket"))
|
||||
if !globalIsErasure {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ExportBucketMetadataAction)
|
||||
if objectAPI == nil {
|
||||
@@ -390,13 +372,13 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
)
|
||||
if bucket != "" {
|
||||
// Check if bucket exists.
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
buckets = append(buckets, BucketInfo{Name: bucket})
|
||||
} else {
|
||||
buckets, err = objectAPI.ListBuckets(ctx)
|
||||
buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -619,6 +601,39 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
}
|
||||
}
|
||||
|
||||
type importMetaReport struct {
|
||||
madmin.BucketMetaImportErrs
|
||||
}
|
||||
|
||||
func (i *importMetaReport) SetStatus(bucket, fname string, err error) {
|
||||
st := i.Buckets[bucket]
|
||||
var errMsg string
|
||||
if err != nil {
|
||||
errMsg = err.Error()
|
||||
}
|
||||
switch fname {
|
||||
case bucketPolicyConfig:
|
||||
st.Policy = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case bucketNotificationConfig:
|
||||
st.Notification = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case bucketLifecycleConfig:
|
||||
st.Lifecycle = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case bucketSSEConfig:
|
||||
st.SSEConfig = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case bucketTaggingConfig:
|
||||
st.Tagging = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case bucketQuotaConfigFile:
|
||||
st.Quota = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case objectLockConfig:
|
||||
st.ObjectLock = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case bucketVersioningConfig:
|
||||
st.Versioning = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
default:
|
||||
st.Err = errMsg
|
||||
}
|
||||
i.Buckets[bucket] = st
|
||||
}
|
||||
|
||||
// ImportBucketMetadataHandler - imports all bucket metadata from a zipped file and overwrite bucket metadata config
|
||||
// There are some caveats regarding the following:
|
||||
// 1. object lock config - object lock should have been specified at time of bucket creation. Only default retention settings are imported here.
|
||||
@@ -628,18 +643,13 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
ctx := newContext(r, w, "ImportBucketMetadata")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
bucket := pathClean(r.Form.Get("bucket"))
|
||||
|
||||
if !globalIsErasure {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ImportBucketMetadataAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
data, err := ioutil.ReadAll(r.Body)
|
||||
data, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||
return
|
||||
@@ -651,47 +661,46 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
return
|
||||
}
|
||||
bucketMap := make(map[string]struct{}, 1)
|
||||
|
||||
rpt := importMetaReport{
|
||||
madmin.BucketMetaImportErrs{
|
||||
Buckets: make(map[string]madmin.BucketStatus, len(zr.File)),
|
||||
},
|
||||
}
|
||||
// import object lock config if any - order of import matters here.
|
||||
for _, file := range zr.File {
|
||||
slc := strings.Split(file.Name, slashSeparator)
|
||||
if len(slc) != 2 { // expecting bucket/configfile in the zipfile
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||
return
|
||||
}
|
||||
b, fileName := slc[0], slc[1]
|
||||
if bucket == "" { // use bucket requested in query parameters if specified. Otherwise default bucket name to directory name within zip
|
||||
bucket = b
|
||||
rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/<config.json>"))
|
||||
continue
|
||||
}
|
||||
bucket, fileName := slc[0], slc[1]
|
||||
switch fileName {
|
||||
case objectLockConfig:
|
||||
reader, err := file.Open()
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
config, err := objectlock.ParseObjectLockConfig(reader)
|
||||
if err != nil {
|
||||
apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
|
||||
apiErr.Description = err.Error()
|
||||
writeErrorResponse(ctx, w, apiErr, r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
|
||||
continue
|
||||
}
|
||||
|
||||
configData, err := xml.Marshal(config)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
if _, ok := bucketMap[bucket]; !ok {
|
||||
opts := BucketOptions{
|
||||
opts := MakeBucketOptions{
|
||||
LockEnabled: config.ObjectLockEnabled == "Enabled",
|
||||
}
|
||||
err = objectAPI.MakeBucketWithLocation(ctx, bucket, opts)
|
||||
if err != nil {
|
||||
if _, ok := err.(BucketExists); !ok {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
bucketMap[bucket] = struct{}{}
|
||||
@@ -699,15 +708,16 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
// Deny object locking configuration settings on existing buckets without object lock enabled.
|
||||
if _, _, err = globalBucketMetadataSys.GetObjectLockConfig(bucket); err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
|
||||
// Call site replication hook.
|
||||
//
|
||||
@@ -720,8 +730,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
ObjectLockConfig: &cfgStr,
|
||||
UpdatedAt: updatedAt,
|
||||
}); err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -730,162 +740,145 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
for _, file := range zr.File {
|
||||
slc := strings.Split(file.Name, slashSeparator)
|
||||
if len(slc) != 2 { // expecting bucket/configfile in the zipfile
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||
return
|
||||
}
|
||||
b, fileName := slc[0], slc[1]
|
||||
if bucket == "" { // use bucket requested in query parameters if specified. Otherwise default bucket name to directory name within zip
|
||||
bucket = b
|
||||
rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/<config.json>"))
|
||||
continue
|
||||
}
|
||||
bucket, fileName := slc[0], slc[1]
|
||||
switch fileName {
|
||||
case bucketVersioningConfig:
|
||||
reader, err := file.Open()
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
v, err := versioning.ParseConfig(io.LimitReader(reader, maxBucketVersioningConfigSize))
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
if _, ok := bucketMap[bucket]; !ok {
|
||||
err = objectAPI.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
|
||||
if err != nil {
|
||||
if err = objectAPI.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
if _, ok := err.(BucketExists); !ok {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
bucketMap[bucket] = struct{}{}
|
||||
}
|
||||
|
||||
if globalSiteReplicationSys.isEnabled() && v.Suspended() {
|
||||
writeErrorResponse(ctx, w, APIError{
|
||||
Code: "InvalidBucketState",
|
||||
Description: "Cluster replication is enabled for this site, so the versioning state cannot be changed.",
|
||||
HTTPStatusCode: http.StatusConflict,
|
||||
}, r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, fmt.Errorf("Cluster replication is enabled for this site, so the versioning state cannot be suspended."))
|
||||
continue
|
||||
}
|
||||
|
||||
if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && v.Suspended() {
|
||||
writeErrorResponse(ctx, w, APIError{
|
||||
Code: "InvalidBucketState",
|
||||
Description: "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.",
|
||||
HTTPStatusCode: http.StatusConflict,
|
||||
}, r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, fmt.Errorf("An Object Lock configuration is present on this bucket, so the versioning state cannot be suspended."))
|
||||
continue
|
||||
}
|
||||
if _, err := getReplicationConfig(ctx, bucket); err == nil && v.Suspended() {
|
||||
writeErrorResponse(ctx, w, APIError{
|
||||
Code: "InvalidBucketState",
|
||||
Description: "A replication configuration is present on this bucket, so the versioning state cannot be changed.",
|
||||
HTTPStatusCode: http.StatusConflict,
|
||||
}, r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, fmt.Errorf("A replication configuration is present on this bucket, so the versioning state cannot be suspended."))
|
||||
continue
|
||||
}
|
||||
|
||||
configData, err := xml.Marshal(v)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData); err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
}
|
||||
}
|
||||
|
||||
for _, file := range zr.File {
|
||||
reader, err := file.Open()
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, ""), r.URL)
|
||||
return
|
||||
rpt.SetStatus(file.Name, "", err)
|
||||
continue
|
||||
}
|
||||
sz := file.FileInfo().Size()
|
||||
slc := strings.Split(file.Name, slashSeparator)
|
||||
if len(slc) != 2 { // expecting bucket/configfile in the zipfile
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||
return
|
||||
}
|
||||
b, fileName := slc[0], slc[1]
|
||||
if bucket == "" { // use bucket requested in query parameters if specified. Otherwise default bucket name to directory name within zip
|
||||
bucket = b
|
||||
rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/<config.json>"))
|
||||
continue
|
||||
}
|
||||
bucket, fileName := slc[0], slc[1]
|
||||
// create bucket if it does not exist yet.
|
||||
if _, ok := bucketMap[bucket]; !ok {
|
||||
err = objectAPI.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
|
||||
err = objectAPI.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
if _, ok := err.(BucketExists); !ok {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, "", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
bucketMap[bucket] = struct{}{}
|
||||
}
|
||||
if _, ok := bucketMap[bucket]; !ok {
|
||||
continue
|
||||
}
|
||||
switch fileName {
|
||||
case bucketNotificationConfig:
|
||||
config, err := event.ParseConfig(io.LimitReader(reader, sz), globalSite.Region, globalNotificationSys.targetList)
|
||||
config, err := event.ParseConfig(io.LimitReader(reader, sz), globalSite.Region, globalEventNotifier.targetList)
|
||||
if err != nil {
|
||||
apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
|
||||
if event.IsEventError(err) {
|
||||
apiErr = importError(ctx, err, file.Name, bucket)
|
||||
}
|
||||
writeErrorResponse(ctx, w, apiErr, r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
|
||||
continue
|
||||
}
|
||||
|
||||
configData, err := xml.Marshal(config)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketNotificationConfig, configData); err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
rulesMap := config.ToRulesMap()
|
||||
globalNotificationSys.AddRulesMap(bucket, rulesMap)
|
||||
globalEventNotifier.AddRulesMap(bucket, rulesMap)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketPolicyConfig:
|
||||
// Error out if Content-Length is beyond allowed size.
|
||||
if sz > maxBucketPolicySize {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPolicyTooLarge), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, fmt.Errorf(ErrPolicyTooLarge.String()))
|
||||
continue
|
||||
}
|
||||
|
||||
bucketPolicyBytes, err := ioutil.ReadAll(io.LimitReader(reader, sz))
|
||||
bucketPolicyBytes, err := io.ReadAll(io.LimitReader(reader, sz))
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
bucketPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyBytes), bucket)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Version in policy must not be empty
|
||||
if bucketPolicy.Version == "" {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, fmt.Errorf(ErrMalformedPolicy.String()))
|
||||
continue
|
||||
}
|
||||
|
||||
configData, err := json.Marshal(bucketPolicy)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
// Call site replication hook.
|
||||
if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypePolicy,
|
||||
@@ -893,56 +886,51 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
Policy: bucketPolicyBytes,
|
||||
UpdatedAt: updatedAt,
|
||||
}); err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
case bucketLifecycleConfig:
|
||||
bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(reader, sz))
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Validate the received bucket policy document
|
||||
if err = bucketLifecycle.Validate(); err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Validate the transition storage ARNs
|
||||
if err = validateTransitionTier(bucketLifecycle); err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
configData, err := xml.Marshal(bucketLifecycle)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketSSEConfig:
|
||||
// Parse bucket encryption xml
|
||||
encConfig, err := validateBucketSSEConfig(io.LimitReader(reader, maxBucketSSEConfigSize))
|
||||
if err != nil {
|
||||
apiErr := APIError{
|
||||
Code: "MalformedXML",
|
||||
Description: fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description, err),
|
||||
HTTPStatusCode: errorCodes[ErrMalformedXML].HTTPStatusCode,
|
||||
}
|
||||
writeErrorResponse(ctx, w, apiErr, r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
|
||||
continue
|
||||
}
|
||||
|
||||
// Return error if KMS is not initialized
|
||||
if GlobalKMS == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, fmt.Errorf("%s", errorCodes[ErrKMSNotConfigured].Description))
|
||||
continue
|
||||
}
|
||||
kmsKey := encConfig.KeyID()
|
||||
if kmsKey != "" {
|
||||
@@ -950,26 +938,27 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
_, err := GlobalKMS.GenerateKey(ctx, kmsKey, kmsContext)
|
||||
if err != nil {
|
||||
if errors.Is(err, kes.ErrKeyNotFound) {
|
||||
writeErrorResponse(ctx, w, importError(ctx, errKMSKeyNotFound, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, errKMSKeyNotFound)
|
||||
continue
|
||||
}
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
configData, err := xml.Marshal(encConfig)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Store the bucket encryption configuration in the object layer
|
||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
|
||||
// Call site replication hook.
|
||||
//
|
||||
@@ -982,29 +971,30 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
SSEConfig: &cfgStr,
|
||||
UpdatedAt: updatedAt,
|
||||
}); err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
case bucketTaggingConfig:
|
||||
tags, err := tags.ParseBucketXML(io.LimitReader(reader, sz))
|
||||
if err != nil {
|
||||
apiErr := errorCodes.ToAPIErrWithErr(ErrMalformedXML, fmt.Errorf("error importing %s with %w", file.Name, err))
|
||||
writeErrorResponse(ctx, w, apiErr, r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
|
||||
continue
|
||||
}
|
||||
|
||||
configData, err := xml.Marshal(tags)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
|
||||
// Call site replication hook.
|
||||
//
|
||||
// We encode the xml bytes as base64 to ensure there are no encoding
|
||||
@@ -1016,32 +1006,33 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
Tags: &cfgStr,
|
||||
UpdatedAt: updatedAt,
|
||||
}); err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
case bucketQuotaConfigFile:
|
||||
data, err := ioutil.ReadAll(reader)
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
quotaConfig, err := parseBucketQuota(bucket, data)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if quotaConfig.Type == "fifo" {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, fmt.Errorf("Detected older 'fifo' quota config, 'fifo' feature is removed and not supported anymore"))
|
||||
continue
|
||||
}
|
||||
|
||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
|
||||
bucketMeta := madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypeQuotaConfig,
|
||||
@@ -1055,15 +1046,23 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
// Call site replication hook.
|
||||
if err = globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta); err != nil {
|
||||
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
|
||||
return
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rptData, err := json.Marshal(rpt.BucketMetaImportErrs)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseJSON(w, rptData)
|
||||
}
|
||||
|
||||
// ReplicationDiffHandler - POST returns info on unreplicated versions for a remote target ARN
|
||||
// to the connected HTTP client. This is a MinIO only extension
|
||||
// to the connected HTTP client.
|
||||
func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ReplicationDiff")
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
@@ -1071,25 +1070,13 @@ func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ReplicationDiff)
|
||||
if objectAPI == nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
// check if user has permissions to perform this operation
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketVersionsAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -124,6 +124,18 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
|
||||
Description: err.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
case errors.Is(err, errDecommissionRebalanceAlreadyRunning):
|
||||
apiErr = APIError{
|
||||
Code: "XMinioDecommissionNotAllowed",
|
||||
Description: err.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
case errors.Is(err, errRebalanceDecommissionAlreadyRunning):
|
||||
apiErr = APIError{
|
||||
Code: "XMinioRebalanceNotAllowed",
|
||||
Description: err.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
case errors.Is(err, errConfigNotFound):
|
||||
apiErr = APIError{
|
||||
Code: "XMinioConfigError",
|
||||
|
||||
@@ -81,6 +81,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err = validateConfig(cfg, subSys); err != nil {
|
||||
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
|
||||
return
|
||||
@@ -91,6 +92,12 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
// freshly retrieve the config so that default values are loaded for reset config
|
||||
if cfg, err = getValidConfig(objectAPI); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
dynamic := config.SubSystemsDynamic.Contains(subSys)
|
||||
if dynamic {
|
||||
applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
|
||||
@@ -173,10 +180,16 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
||||
if dynamic {
|
||||
applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
|
||||
}
|
||||
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
// GetConfigKVHandler - GET /minio/admin/v3/get-config-kv?key={key}
|
||||
//
|
||||
// `key` can be one of three forms:
|
||||
// 1. `subsys:target` -> request for config of a single subsystem and target pair.
|
||||
// 2. `subsys:` -> request for config of a single subsystem and the default target.
|
||||
// 3. `subsys` -> request for config of all targets for the given subsystem.
|
||||
func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetConfigKV")
|
||||
|
||||
@@ -189,15 +202,34 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
||||
|
||||
cfg := globalServerConfig.Clone()
|
||||
vars := mux.Vars(r)
|
||||
buf := &bytes.Buffer{}
|
||||
cw := config.NewConfigWriteTo(cfg, vars["key"])
|
||||
if _, err := cw.WriteTo(buf); err != nil {
|
||||
key := vars["key"]
|
||||
|
||||
var subSys, target string
|
||||
{
|
||||
ws := strings.SplitN(key, madmin.SubSystemSeparator, 2)
|
||||
subSys = ws[0]
|
||||
if len(ws) == 2 {
|
||||
if ws[1] == "" {
|
||||
target = madmin.Default
|
||||
} else {
|
||||
target = ws[1]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
subSysConfigs, err := cfg.GetSubsysInfo(subSys, target)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var s strings.Builder
|
||||
for _, subSysConfig := range subSysConfigs {
|
||||
subSysConfig.WriteTo(&s, false)
|
||||
}
|
||||
|
||||
password := cred.SecretKey
|
||||
econfigData, err := madmin.EncryptData(password, buf.Bytes())
|
||||
econfigData, err := madmin.EncryptData(password, []byte(s.String()))
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -422,47 +454,31 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
|
||||
|
||||
var s strings.Builder
|
||||
hkvs := config.HelpSubSysMap[""]
|
||||
var count int
|
||||
for _, hkv := range hkvs {
|
||||
count += len(cfg[hkv.Key])
|
||||
}
|
||||
for _, hkv := range hkvs {
|
||||
v := cfg[hkv.Key]
|
||||
for target, kv := range v {
|
||||
off := kv.Get(config.Enable) == config.EnableOff
|
||||
// We ignore the error below, as we cannot get one.
|
||||
cfgSubsysItems, _ := cfg.GetSubsysInfo(hkv.Key, "")
|
||||
|
||||
for _, item := range cfgSubsysItems {
|
||||
off := item.Config.Get(config.Enable) == config.EnableOff
|
||||
switch hkv.Key {
|
||||
case config.EtcdSubSys:
|
||||
off = !etcd.Enabled(kv)
|
||||
off = !etcd.Enabled(item.Config)
|
||||
case config.CacheSubSys:
|
||||
off = !cache.Enabled(kv)
|
||||
off = !cache.Enabled(item.Config)
|
||||
case config.StorageClassSubSys:
|
||||
off = !storageclass.Enabled(kv)
|
||||
off = !storageclass.Enabled(item.Config)
|
||||
case config.PolicyPluginSubSys:
|
||||
off = !polplugin.Enabled(kv)
|
||||
off = !polplugin.Enabled(item.Config)
|
||||
case config.IdentityOpenIDSubSys:
|
||||
off = !openid.Enabled(kv)
|
||||
off = !openid.Enabled(item.Config)
|
||||
case config.IdentityLDAPSubSys:
|
||||
off = !xldap.Enabled(kv)
|
||||
off = !xldap.Enabled(item.Config)
|
||||
case config.IdentityTLSSubSys:
|
||||
off = !globalSTSTLSConfig.Enabled
|
||||
case config.IdentityPluginSubSys:
|
||||
off = !idplugin.Enabled(kv)
|
||||
}
|
||||
if off {
|
||||
s.WriteString(config.KvComment)
|
||||
s.WriteString(config.KvSpaceSeparator)
|
||||
}
|
||||
s.WriteString(hkv.Key)
|
||||
if target != config.Default {
|
||||
s.WriteString(config.SubSystemSeparator)
|
||||
s.WriteString(target)
|
||||
}
|
||||
s.WriteString(config.KvSpaceSeparator)
|
||||
s.WriteString(kv.String())
|
||||
count--
|
||||
if count > 0 {
|
||||
s.WriteString(config.KvNewline)
|
||||
off = !idplugin.Enabled(item.Config)
|
||||
}
|
||||
item.WriteTo(&s, off)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+245
-128
@@ -20,6 +20,7 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -29,22 +30,14 @@ import (
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio/internal/config"
|
||||
cfgldap "github.com/minio/minio/internal/config/identity/ldap"
|
||||
"github.com/minio/minio/internal/config/identity/openid"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
"github.com/minio/pkg/ldap"
|
||||
)
|
||||
|
||||
// List of implemented ID config types.
|
||||
var idCfgTypes = set.CreateStringSet("openid")
|
||||
|
||||
// SetIdentityProviderCfg:
|
||||
//
|
||||
// PUT <admin-prefix>/id-cfg?type=openid&name=dex1
|
||||
func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "SetIdentityCfg")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
func (a adminAPIHandlers) addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
@@ -56,6 +49,14 @@ func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
|
||||
// Ensure body content type is opaque to ensure that request body has not
|
||||
// been interpreted as form data.
|
||||
contentType := r.Header.Get("Content-Type")
|
||||
if contentType != "application/octet-stream" {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
password := cred.SecretKey
|
||||
reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
|
||||
if err != nil {
|
||||
@@ -64,42 +65,48 @@ func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
|
||||
cfgType := mux.Vars(r)["type"]
|
||||
if !idCfgTypes.Contains(cfgType) {
|
||||
// TODO: change this to invalid type error when implementation
|
||||
// is complete.
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
idpCfgType := mux.Vars(r)["type"]
|
||||
if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var cfgDataBuilder strings.Builder
|
||||
switch cfgType {
|
||||
case "openid":
|
||||
fmt.Fprintf(&cfgDataBuilder, "identity_openid")
|
||||
var subSys string
|
||||
switch idpCfgType {
|
||||
case madmin.OpenidIDPCfg:
|
||||
subSys = madmin.IdentityOpenIDSubSys
|
||||
case madmin.LDAPIDPCfg:
|
||||
subSys = madmin.IdentityLDAPSubSys
|
||||
}
|
||||
|
||||
// Ensure body content type is opaque.
|
||||
contentType := r.Header.Get("Content-Type")
|
||||
if contentType != "application/octet-stream" {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Subsystem configuration name could be empty.
|
||||
cfgName := mux.Vars(r)["name"]
|
||||
cfgTarget := madmin.Default
|
||||
if cfgName != "" {
|
||||
fmt.Fprintf(&cfgDataBuilder, "%s%s", config.SubSystemSeparator, cfgName)
|
||||
cfgTarget = cfgName
|
||||
if idpCfgType == madmin.LDAPIDPCfg && cfgName != madmin.Default {
|
||||
// LDAP does not support multiple configurations. So cfgName must be
|
||||
// empty or `madmin.Default`.
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Fprintf(&cfgDataBuilder, "%s%s", config.KvSpaceSeparator, string(reqBytes))
|
||||
|
||||
cfgData := cfgDataBuilder.String()
|
||||
subSys, _, _, err := config.GetSubSys(cfgData)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
// Check that this is a valid Create vs Update API call.
|
||||
s := globalServerConfig.Clone()
|
||||
if apiErrCode := handleCreateUpdateValidation(s, subSys, cfgTarget, isUpdate); apiErrCode != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
cfgData := ""
|
||||
{
|
||||
tgtSuffix := ""
|
||||
if cfgTarget != madmin.Default {
|
||||
tgtSuffix = config.SubSystemSeparator + cfgTarget
|
||||
}
|
||||
cfgData = subSys + tgtSuffix + config.KvSpaceSeparator + string(reqBytes)
|
||||
}
|
||||
|
||||
cfg, err := readServerConfig(ctx, objectAPI)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
@@ -119,6 +126,16 @@ func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.
|
||||
}
|
||||
|
||||
if err = validateConfig(cfg, subSys); err != nil {
|
||||
|
||||
var validationErr ldap.Validation
|
||||
if errors.As(err, &validationErr) {
|
||||
// If we got an LDAP validation error, we need to send appropriate
|
||||
// error message back to client (likely mc).
|
||||
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigLDAPValidation),
|
||||
validationErr.FormatError(), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -138,86 +155,101 @@ func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
// GetIdentityProviderCfg:
|
||||
//
|
||||
// GET <admin-prefix>/id-cfg?type=openid&name=dex_test
|
||||
//
|
||||
// GetIdentityProviderCfg returns a list of configured IDPs on the server if
|
||||
// name is empty. If name is non-empty, returns the configuration details for
|
||||
// the IDP of the given type and configuration name. The configuration name for
|
||||
// the default ("un-named") configuration target is `_`.
|
||||
func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetIdentityProviderCfg")
|
||||
func handleCreateUpdateValidation(s config.Config, subSys, cfgTarget string, isUpdate bool) APIErrorCode {
|
||||
if cfgTarget != madmin.Default {
|
||||
// This cannot give an error at this point.
|
||||
subSysTargets, _ := s.GetAvailableTargets(subSys)
|
||||
subSysTargetsSet := set.CreateStringSet(subSysTargets...)
|
||||
if isUpdate && !subSysTargetsSet.Contains(cfgTarget) {
|
||||
return ErrAdminConfigIDPCfgNameDoesNotExist
|
||||
}
|
||||
if !isUpdate && subSysTargetsSet.Contains(cfgTarget) {
|
||||
return ErrAdminConfigIDPCfgNameAlreadyExists
|
||||
}
|
||||
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
// For the default configuration name, since it will always be an available
|
||||
// target, we need to check if a configuration value has been set previously
|
||||
// to figure out if this is a valid create or update API call.
|
||||
|
||||
// This cannot really error (FIXME: improve the type for GetConfigInfo)
|
||||
var cfgInfos []madmin.IDPCfgInfo
|
||||
switch subSys {
|
||||
case madmin.IdentityOpenIDSubSys:
|
||||
cfgInfos, _ = globalOpenIDConfig.GetConfigInfo(s, cfgTarget)
|
||||
case madmin.IdentityLDAPSubSys:
|
||||
cfgInfos, _ = globalLDAPConfig.GetConfigInfo(s, cfgTarget)
|
||||
}
|
||||
|
||||
if len(cfgInfos) > 0 && !isUpdate {
|
||||
return ErrAdminConfigIDPCfgNameAlreadyExists
|
||||
}
|
||||
if len(cfgInfos) == 0 && isUpdate {
|
||||
return ErrAdminConfigIDPCfgNameDoesNotExist
|
||||
}
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
// AddIdentityProviderCfg: adds a new IDP config for openid/ldap.
|
||||
//
|
||||
// PUT <admin-prefix>/idp-cfg/openid/dex1 -> create named config `dex1`
|
||||
//
|
||||
// PUT <admin-prefix>/idp-cfg/openid/_ -> create (default) named config `_`
|
||||
func (a adminAPIHandlers) AddIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "AddIdentityProviderCfg")
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
a.addOrUpdateIDPHandler(ctx, w, r, false)
|
||||
}
|
||||
|
||||
// UpdateIdentityProviderCfg: updates an existing IDP config for openid/ldap.
|
||||
//
|
||||
// PATCH <admin-prefix>/idp-cfg/openid/dex1 -> update named config `dex1`
|
||||
//
|
||||
// PATCH <admin-prefix>/idp-cfg/openid/_ -> update (default) named config `_`
|
||||
func (a adminAPIHandlers) UpdateIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "UpdateIdentityProviderCfg")
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
a.addOrUpdateIDPHandler(ctx, w, r, true)
|
||||
}
|
||||
|
||||
// ListIdentityProviderCfg:
|
||||
//
|
||||
// GET <admin-prefix>/idp-cfg/openid -> lists openid provider configs.
|
||||
func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListIdentityProviderCfg")
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
cfgType := mux.Vars(r)["type"]
|
||||
cfgName := r.Form.Get("name")
|
||||
password := cred.SecretKey
|
||||
|
||||
if !idCfgTypes.Contains(cfgType) {
|
||||
// TODO: change this to invalid type error when implementation
|
||||
// is complete.
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
idpCfgType := mux.Vars(r)["type"]
|
||||
if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// If no cfgName is provided, we list.
|
||||
if cfgName == "" {
|
||||
a.listIdentityProviders(ctx, w, r, cfgType, password)
|
||||
return
|
||||
}
|
||||
var cfgList []madmin.IDPListItem
|
||||
var err error
|
||||
switch idpCfgType {
|
||||
case madmin.OpenidIDPCfg:
|
||||
cfg := globalServerConfig.Clone()
|
||||
cfgList, err = globalOpenIDConfig.GetConfigList(cfg)
|
||||
case madmin.LDAPIDPCfg:
|
||||
cfg := globalServerConfig.Clone()
|
||||
cfgList, err = globalLDAPConfig.GetConfigList(cfg)
|
||||
|
||||
cfg := globalServerConfig.Clone()
|
||||
|
||||
cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
|
||||
if err != nil {
|
||||
if err == openid.ErrProviderConfigNotFound {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
res := madmin.IDPConfig{
|
||||
Type: cfgType,
|
||||
Name: cfgName,
|
||||
Info: cfgInfos,
|
||||
}
|
||||
data, err := json.Marshal(res)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
econfigData, err := madmin.EncryptData(password, data)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseJSON(w, econfigData)
|
||||
}
|
||||
|
||||
func (a adminAPIHandlers) listIdentityProviders(ctx context.Context, w http.ResponseWriter, r *http.Request, cfgType, password string) {
|
||||
// var subSys string
|
||||
switch cfgType {
|
||||
case "openid":
|
||||
// subSys = config.IdentityOpenIDSubSys
|
||||
default:
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
cfg := globalServerConfig.Clone()
|
||||
cfgList, err := globalOpenIDConfig.GetConfigList(cfg)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -238,9 +270,69 @@ func (a adminAPIHandlers) listIdentityProviders(ctx context.Context, w http.Resp
|
||||
writeSuccessResponseJSON(w, econfigData)
|
||||
}
|
||||
|
||||
// GetIdentityProviderCfg:
|
||||
//
|
||||
// GET <admin-prefix>/idp-cfg/openid/dex_test
|
||||
func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetIdentityProviderCfg")
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
idpCfgType := mux.Vars(r)["type"]
|
||||
cfgName := mux.Vars(r)["name"]
|
||||
password := cred.SecretKey
|
||||
|
||||
if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
cfg := globalServerConfig.Clone()
|
||||
var cfgInfos []madmin.IDPCfgInfo
|
||||
var err error
|
||||
switch idpCfgType {
|
||||
case madmin.OpenidIDPCfg:
|
||||
cfgInfos, err = globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
|
||||
case madmin.LDAPIDPCfg:
|
||||
cfgInfos, err = globalLDAPConfig.GetConfigInfo(cfg, cfgName)
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, openid.ErrProviderConfigNotFound) || errors.Is(err, cfgldap.ErrProviderConfigNotFound) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
res := madmin.IDPConfig{
|
||||
Type: idpCfgType,
|
||||
Name: cfgName,
|
||||
Info: cfgInfos,
|
||||
}
|
||||
data, err := json.Marshal(res)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
econfigData, err := madmin.EncryptData(password, data)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseJSON(w, econfigData)
|
||||
}
|
||||
|
||||
// DeleteIdentityProviderCfg:
|
||||
//
|
||||
// DELETE <admin-prefix>/id-cfg?type=openid&name=dex_test
|
||||
// DELETE <admin-prefix>/idp-cfg/openid/dex_test
|
||||
func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "DeleteIdentityProviderCfg")
|
||||
|
||||
@@ -251,57 +343,82 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
|
||||
return
|
||||
}
|
||||
|
||||
cfgType := mux.Vars(r)["type"]
|
||||
idpCfgType := mux.Vars(r)["type"]
|
||||
cfgName := mux.Vars(r)["name"]
|
||||
if !idCfgTypes.Contains(cfgType) {
|
||||
// TODO: change this to invalid type error when implementation
|
||||
// is complete.
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
cfg := globalServerConfig.Clone()
|
||||
cfgCopy := globalServerConfig.Clone()
|
||||
var subSys string
|
||||
switch idpCfgType {
|
||||
case madmin.OpenidIDPCfg:
|
||||
subSys = config.IdentityOpenIDSubSys
|
||||
cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfgCopy, cfgName)
|
||||
if err != nil {
|
||||
if errors.Is(err, openid.ErrProviderConfigNotFound) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
|
||||
if err != nil {
|
||||
if err == openid.ErrProviderConfigNotFound {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
hasEnv := false
|
||||
for _, ci := range cfgInfos {
|
||||
if ci.IsCfg && ci.IsEnv {
|
||||
hasEnv = true
|
||||
break
|
||||
hasEnv := false
|
||||
for _, ci := range cfgInfos {
|
||||
if ci.IsCfg && ci.IsEnv {
|
||||
hasEnv = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if hasEnv {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigEnvOverridden), r.URL)
|
||||
return
|
||||
}
|
||||
if hasEnv {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigEnvOverridden), r.URL)
|
||||
return
|
||||
}
|
||||
case madmin.LDAPIDPCfg:
|
||||
subSys = config.IdentityLDAPSubSys
|
||||
cfgInfos, err := globalLDAPConfig.GetConfigInfo(cfgCopy, cfgName)
|
||||
if err != nil {
|
||||
if errors.Is(err, openid.ErrProviderConfigNotFound) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var subSys string
|
||||
switch cfgType {
|
||||
case "openid":
|
||||
subSys = config.IdentityOpenIDSubSys
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
hasEnv := false
|
||||
for _, ci := range cfgInfos {
|
||||
if ci.IsCfg && ci.IsEnv {
|
||||
hasEnv = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if hasEnv {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigEnvOverridden), r.URL)
|
||||
return
|
||||
}
|
||||
default:
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
cfg, err = readServerConfig(ctx, objectAPI)
|
||||
cfg, err := readServerConfig(ctx, objectAPI)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err = cfg.DelKVS(fmt.Sprintf("%s:%s", subSys, cfgName)); err != nil {
|
||||
cfgKey := fmt.Sprintf("%s:%s", subSys, cfgName)
|
||||
if cfgName == madmin.Default {
|
||||
cfgKey = subSys
|
||||
}
|
||||
if err = cfg.DelKVS(cfgKey); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
// Copyright (c) 2015-2022 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
)
|
||||
|
||||
// ListLDAPPolicyMappingEntities lists users/groups mapped to given/all policies.
|
||||
//
|
||||
// GET <admin-prefix>/idp/ldap/policy-entities?[query-params]
|
||||
//
|
||||
// Query params:
|
||||
//
|
||||
// user=... -> repeatable query parameter, specifying users to query for
|
||||
// policy mapping
|
||||
//
|
||||
// group=... -> repeatable query parameter, specifying groups to query for
|
||||
// policy mapping
|
||||
//
|
||||
// policy=... -> repeatable query parameter, specifying policy to query for
|
||||
// user/group mapping
|
||||
//
|
||||
// When all query parameters are omitted, returns mappings for all policies.
|
||||
func (a adminAPIHandlers) ListLDAPPolicyMappingEntities(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListLDAPPolicyMappingEntities")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
// Check authorization.
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r,
|
||||
iampolicy.ListGroupsAdminAction, iampolicy.ListUsersAdminAction, iampolicy.ListUserPoliciesAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Validate API arguments.
|
||||
|
||||
q := madmin.PolicyEntitiesQuery{
|
||||
Users: r.Form["user"],
|
||||
Groups: r.Form["group"],
|
||||
Policy: r.Form["policy"],
|
||||
}
|
||||
|
||||
// Query IAM
|
||||
|
||||
res, err := globalIAMSys.QueryLDAPPolicyEntities(r.Context(), q)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Encode result and send response.
|
||||
|
||||
data, err := json.Marshal(res)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
password := cred.SecretKey
|
||||
econfigData, err := madmin.EncryptData(password, data)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
writeSuccessResponseJSON(w, econfigData)
|
||||
}
|
||||
@@ -19,6 +19,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
@@ -27,6 +28,11 @@ import (
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
)
|
||||
|
||||
var (
|
||||
errRebalanceDecommissionAlreadyRunning = errors.New("Rebalance cannot be started, decommission is aleady in progress")
|
||||
errDecommissionRebalanceAlreadyRunning = errors.New("Decommission cannot be started, rebalance is already in progress")
|
||||
)
|
||||
|
||||
func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "StartDecommission")
|
||||
|
||||
@@ -49,6 +55,11 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
|
||||
return
|
||||
}
|
||||
|
||||
if pools.IsRebalanceStarted() {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errDecommissionRebalanceAlreadyRunning), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
v := vars["pool"]
|
||||
|
||||
@@ -200,3 +211,137 @@ func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
logger.LogIf(r.Context(), json.NewEncoder(w).Encode(poolsStatus))
|
||||
}
|
||||
|
||||
func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "RebalanceStart")
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// NB rebalance-start admin API is always coordinated from first pool's
|
||||
// first node. The following is required to serialize (the effects of)
|
||||
// concurrent rebalance-start commands.
|
||||
if ep := globalEndpoints[0].Endpoints[0]; !ep.IsLocal {
|
||||
for nodeIdx, proxyEp := range globalProxyEndpoints {
|
||||
if proxyEp.Endpoint.Host == ep.Host {
|
||||
if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pools, ok := objectAPI.(*erasureServerPools)
|
||||
if !ok || len(pools.serverPools) == 1 {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if pools.IsDecommissionRunning() {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errRebalanceDecommissionAlreadyRunning), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if pools.IsRebalanceStarted() {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminRebalanceAlreadyStarted), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
bucketInfos, err := objectAPI.ListBuckets(ctx, BucketOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
buckets := make([]string, 0, len(bucketInfos))
|
||||
for _, bInfo := range bucketInfos {
|
||||
buckets = append(buckets, bInfo.Name)
|
||||
}
|
||||
|
||||
var id string
|
||||
if id, err = pools.initRebalanceMeta(ctx, buckets); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Rebalance routine is run on the first node of any pool participating in rebalance.
|
||||
pools.StartRebalance()
|
||||
|
||||
b, err := json.Marshal(struct {
|
||||
ID string `json:"id"`
|
||||
}{ID: id})
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseJSON(w, b)
|
||||
// Notify peers to load rebalance.bin and start rebalance routine if they happen to be
|
||||
// participating pool's leader node
|
||||
globalNotificationSys.LoadRebalanceMeta(ctx, true)
|
||||
}
|
||||
|
||||
func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "RebalanceStatus")
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Proxy rebalance-status to first pool first node, so that users see a
|
||||
// consistent view of rebalance progress even though different rebalancing
|
||||
// pools may temporarily have out of date info on the others.
|
||||
if ep := globalEndpoints[0].Endpoints[0]; !ep.IsLocal {
|
||||
for nodeIdx, proxyEp := range globalProxyEndpoints {
|
||||
if proxyEp.Endpoint.Host == ep.Host {
|
||||
if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pools, ok := objectAPI.(*erasureServerPools)
|
||||
if !ok {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
rs, err := rebalanceStatus(ctx, pools)
|
||||
if err != nil {
|
||||
if errors.Is(err, errRebalanceNotStarted) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminRebalanceNotStarted), r.URL)
|
||||
return
|
||||
}
|
||||
logger.LogIf(ctx, fmt.Errorf("failed to fetch rebalance status: %w", err))
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
logger.LogIf(r.Context(), json.NewEncoder(w).Encode(rs))
|
||||
}
|
||||
|
||||
func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "RebalanceStop")
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
pools, ok := objectAPI.(*erasureServerPools)
|
||||
if !ok {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Cancel any ongoing rebalance operation
|
||||
globalNotificationSys.StopRebalance(r.Context())
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
logger.LogIf(ctx, pools.saveRebalanceStats(GlobalContext, 0, rebalSaveStoppedAt))
|
||||
}
|
||||
|
||||
@@ -22,8 +22,9 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/madmin-go"
|
||||
@@ -116,19 +117,38 @@ func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request
|
||||
_, isLockEnabled := r.Form["lockEnabled"]
|
||||
_, isVersioningEnabled := r.Form["versioningEnabled"]
|
||||
_, isForceCreate := r.Form["forceCreate"]
|
||||
opts := BucketOptions{
|
||||
createdAtStr := strings.TrimSpace(r.Form.Get("createdAt"))
|
||||
createdAt, cerr := time.Parse(time.RFC3339Nano, createdAtStr)
|
||||
if cerr != nil {
|
||||
createdAt = timeSentinel
|
||||
}
|
||||
|
||||
opts := MakeBucketOptions{
|
||||
Location: r.Form.Get("location"),
|
||||
LockEnabled: isLockEnabled,
|
||||
VersioningEnabled: isVersioningEnabled,
|
||||
ForceCreate: isForceCreate,
|
||||
CreatedAt: createdAt,
|
||||
}
|
||||
err = globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts)
|
||||
case madmin.ConfigureReplBktOp:
|
||||
err = globalSiteReplicationSys.PeerBucketConfigureReplHandler(ctx, bucket)
|
||||
case madmin.DeleteBucketBktOp:
|
||||
err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, false)
|
||||
_, noRecreate := r.Form["noRecreate"]
|
||||
err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{
|
||||
Force: false,
|
||||
NoRecreate: noRecreate,
|
||||
SRDeleteOp: getSRBucketDeleteOp(true),
|
||||
})
|
||||
case madmin.ForceDeleteBucketBktOp:
|
||||
err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, true)
|
||||
_, noRecreate := r.Form["noRecreate"]
|
||||
err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{
|
||||
Force: true,
|
||||
NoRecreate: noRecreate,
|
||||
SRDeleteOp: getSRBucketDeleteOp(true),
|
||||
})
|
||||
case madmin.PurgeDeletedBucketOp:
|
||||
globalSiteReplicationSys.purgeDeletedBucket(ctx, objectAPI, bucket)
|
||||
}
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
@@ -298,7 +318,7 @@ func (a adminAPIHandlers) SRPeerGetIDPSettings(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
|
||||
func parseJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptionKey string) error {
|
||||
data, err := ioutil.ReadAll(body)
|
||||
data, err := io.ReadAll(body)
|
||||
if err != nil {
|
||||
return SRError{
|
||||
Cause: err,
|
||||
@@ -436,6 +456,7 @@ func getSRStatusOptions(r *http.Request) (opts madmin.SRStatusOptions) {
|
||||
opts.Users = q.Get("users") == "true"
|
||||
opts.Entity = madmin.GetSREntityType(q.Get("entity"))
|
||||
opts.EntityValue = q.Get("entityvalue")
|
||||
opts.ShowDeleted = q.Get("showDeleted") == "true"
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
+79
-34
@@ -22,7 +22,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
@@ -136,7 +135,7 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// Add ldap users which have mapped policies if in LDAP mode
|
||||
// FIXME(vadmeste): move this to policy info in the future
|
||||
ldapUsers, err := globalIAMSys.ListLDAPUsers()
|
||||
ldapUsers, err := globalIAMSys.ListLDAPUsers(ctx)
|
||||
if err != nil && err != errIAMActionNotAllowed {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -229,7 +228,7 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
data, err := ioutil.ReadAll(r.Body)
|
||||
data, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||
return
|
||||
@@ -658,7 +657,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
// In case of LDAP we need to resolve the targetUser to a DN and
|
||||
// query their groups:
|
||||
if globalLDAPConfig.Enabled {
|
||||
if globalLDAPConfig.Enabled() {
|
||||
opts.claims[ldapUserN] = targetUser // simple username
|
||||
targetUser, targetGroups, err = globalLDAPConfig.LookupUserDN(targetUser)
|
||||
if err != nil {
|
||||
@@ -1150,15 +1149,12 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
return rd, wr
|
||||
}
|
||||
|
||||
var dataUsageInfo DataUsageInfo
|
||||
var err error
|
||||
if !globalIsGateway {
|
||||
// Load the latest calculated data usage
|
||||
dataUsageInfo, _ = loadDataUsageFromBackend(ctx, objectAPI)
|
||||
}
|
||||
// Load the latest calculated data usage
|
||||
dataUsageInfo, _ := loadDataUsageFromBackend(ctx, objectAPI)
|
||||
|
||||
// If etcd, dns federation configured list buckets from etcd.
|
||||
var buckets []BucketInfo
|
||||
var err error
|
||||
if globalDNSConfig != nil && globalBucketFederation {
|
||||
dnsBuckets, err := globalDNSConfig.List()
|
||||
if err != nil && !IsErrIgnored(err,
|
||||
@@ -1177,7 +1173,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
return buckets[i].Name < buckets[j].Name
|
||||
})
|
||||
} else {
|
||||
buckets, err = objectAPI.ListBuckets(ctx)
|
||||
buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -1189,14 +1185,40 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
// For derived credentials, check the parent user's permissions.
|
||||
accountName = cred.ParentUser
|
||||
}
|
||||
policies, err := globalIAMSys.PolicyDBGet(accountName, false, cred.Groups...)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
buf, err := json.MarshalIndent(globalIAMSys.GetCombinedPolicy(policies...), "", " ")
|
||||
roleArn := iampolicy.Args{Claims: claims}.GetRoleArn()
|
||||
var effectivePolicy iampolicy.Policy
|
||||
|
||||
var buf []byte
|
||||
switch {
|
||||
case accountName == globalActiveCred.AccessKey:
|
||||
for _, policy := range iampolicy.DefaultPolicies {
|
||||
if policy.Name == "consoleAdmin" {
|
||||
effectivePolicy = policy.Definition
|
||||
break
|
||||
}
|
||||
}
|
||||
case roleArn != "":
|
||||
_, policy, err := globalIAMSys.GetRolePolicy(roleArn)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
policySlice := newMappedPolicy(policy).toSlice()
|
||||
effectivePolicy = globalIAMSys.GetCombinedPolicy(policySlice...)
|
||||
|
||||
default:
|
||||
policies, err := globalIAMSys.PolicyDBGet(accountName, false, cred.Groups...)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
effectivePolicy = globalIAMSys.GetCombinedPolicy(policies...)
|
||||
|
||||
}
|
||||
buf, err = json.MarshalIndent(effectivePolicy, "", " ")
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -1449,7 +1471,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
|
||||
return
|
||||
}
|
||||
|
||||
iamPolicyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, r.ContentLength))
|
||||
iamPolicyBytes, err := io.ReadAll(io.LimitReader(r.Body, r.ContentLength))
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -1514,7 +1536,29 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
|
||||
}
|
||||
}
|
||||
|
||||
updatedAt, err := globalIAMSys.PolicyDBSet(ctx, entityName, policyName, isGroup)
|
||||
// Validate that user or group exists.
|
||||
if !isGroup {
|
||||
if globalIAMSys.GetUsersSysType() == MinIOUsersSysType {
|
||||
_, ok := globalIAMSys.GetUser(ctx, entityName)
|
||||
if !ok {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
} else {
|
||||
_, err := globalIAMSys.GetGroupDescription(entityName)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
userType := regUser
|
||||
if globalIAMSys.GetUsersSysType() == LDAPUsersSysType {
|
||||
userType = stsUser
|
||||
}
|
||||
|
||||
updatedAt, err := globalIAMSys.PolicyDBSet(ctx, entityName, policyName, userType, isGroup)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -1524,6 +1568,7 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
|
||||
Type: madmin.SRIAMItemPolicyMapping,
|
||||
PolicyMapping: &madmin.SRPolicyMapping{
|
||||
UserOrGroup: entityName,
|
||||
UserType: int(userType),
|
||||
IsGroup: isGroup,
|
||||
Policy: policyName,
|
||||
},
|
||||
@@ -1810,7 +1855,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
data, err := ioutil.ReadAll(r.Body)
|
||||
data, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||
return
|
||||
@@ -1833,7 +1878,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
default:
|
||||
defer f.Close()
|
||||
var allPolicies map[string]iampolicy.Policy
|
||||
data, err = ioutil.ReadAll(f)
|
||||
data, err = io.ReadAll(f)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allPoliciesFile, ""), r.URL)
|
||||
return
|
||||
@@ -1868,7 +1913,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
default:
|
||||
defer f.Close()
|
||||
var userAccts map[string]madmin.AddOrUpdateUserReq
|
||||
data, err := ioutil.ReadAll(f)
|
||||
data, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allUsersFile, ""), r.URL)
|
||||
return
|
||||
@@ -1945,7 +1990,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
default:
|
||||
defer f.Close()
|
||||
var grpInfos map[string]GroupInfo
|
||||
data, err := ioutil.ReadAll(f)
|
||||
data, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allGroupsFile, ""), r.URL)
|
||||
return
|
||||
@@ -1983,7 +2028,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
default:
|
||||
defer f.Close()
|
||||
var serviceAcctReqs map[string]madmin.SRSvcAccCreate
|
||||
data, err := ioutil.ReadAll(f)
|
||||
data, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allSvcAcctsFile, ""), r.URL)
|
||||
return
|
||||
@@ -2049,7 +2094,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// In case of LDAP we need to resolve the targetUser to a DN and
|
||||
// query their groups:
|
||||
if globalLDAPConfig.Enabled {
|
||||
if globalLDAPConfig.Enabled() {
|
||||
opts.claims[ldapUserN] = svcAcctReq.AccessKey // simple username
|
||||
targetUser, _, err := globalLDAPConfig.LookupUserDN(svcAcctReq.AccessKey)
|
||||
if err != nil {
|
||||
@@ -2079,7 +2124,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
default:
|
||||
defer f.Close()
|
||||
var userPolicyMap map[string]MappedPolicy
|
||||
data, err := ioutil.ReadAll(f)
|
||||
data, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, userPolicyMappingsFile, ""), r.URL)
|
||||
return
|
||||
@@ -2099,7 +2144,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrorResponseJSON(ctx, w, importError(ctx, errIAMActionNotAllowed, userPolicyMappingsFile, u), r.URL)
|
||||
return
|
||||
}
|
||||
if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, false); err != nil {
|
||||
if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, regUser, false); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, importError(ctx, err, userPolicyMappingsFile, u), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -2118,7 +2163,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
default:
|
||||
defer f.Close()
|
||||
var grpPolicyMap map[string]MappedPolicy
|
||||
data, err := ioutil.ReadAll(f)
|
||||
data, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, groupPolicyMappingsFile, ""), r.URL)
|
||||
return
|
||||
@@ -2128,7 +2173,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
for g, pm := range grpPolicyMap {
|
||||
if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, true); err != nil {
|
||||
if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, unknownIAMUserType, true); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, importError(ctx, err, groupPolicyMappingsFile, g), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -2147,7 +2192,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
default:
|
||||
defer f.Close()
|
||||
var userPolicyMap map[string]MappedPolicy
|
||||
data, err := ioutil.ReadAll(f)
|
||||
data, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, stsUserPolicyMappingsFile, ""), r.URL)
|
||||
return
|
||||
@@ -2167,7 +2212,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrorResponseJSON(ctx, w, importError(ctx, errIAMActionNotAllowed, stsUserPolicyMappingsFile, u), r.URL)
|
||||
return
|
||||
}
|
||||
if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, false); err != nil {
|
||||
if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, stsUser, false); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, importError(ctx, err, stsUserPolicyMappingsFile, u), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -2186,7 +2231,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
default:
|
||||
defer f.Close()
|
||||
var grpPolicyMap map[string]MappedPolicy
|
||||
data, err := ioutil.ReadAll(f)
|
||||
data, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, stsGroupPolicyMappingsFile, ""), r.URL)
|
||||
return
|
||||
@@ -2196,7 +2241,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
for g, pm := range grpPolicyMap {
|
||||
if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, true); err != nil {
|
||||
if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, unknownIAMUserType, true); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, importError(ctx, err, stsGroupPolicyMappingsFile, g), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -34,7 +34,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
minio "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
cr "github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio-go/v7/pkg/s3utils"
|
||||
@@ -388,7 +388,7 @@ func (s *TestSuiteIAM) TestUserPolicyEscalationBug(c *check) {
|
||||
req.ContentLength = int64(len(buf))
|
||||
sum := sha256.Sum256(buf)
|
||||
req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum[:]))
|
||||
req.Body = ioutil.NopCloser(bytes.NewReader(buf))
|
||||
req.Body = io.NopCloser(bytes.NewReader(buf))
|
||||
req = signer.SignV4(*req, accessKey, secretKey, "", "")
|
||||
|
||||
// 3.1 Execute the request.
|
||||
@@ -1141,6 +1141,7 @@ func (s *TestSuiteIAM) TestAccMgmtPlugin(c *check) {
|
||||
}
|
||||
|
||||
func (c *check) mustCreateIAMUser(ctx context.Context, admClnt *madmin.AdminClient) madmin.Credentials {
|
||||
c.Helper()
|
||||
randUser := mustGetUUID()
|
||||
randPass := mustGetUUID()
|
||||
err := admClnt.AddUser(ctx, randUser, randPass)
|
||||
@@ -1154,6 +1155,7 @@ func (c *check) mustCreateIAMUser(ctx context.Context, admClnt *madmin.AdminClie
|
||||
}
|
||||
|
||||
func (c *check) mustGetIAMUserInfo(ctx context.Context, admClnt *madmin.AdminClient, accessKey string) madmin.UserInfo {
|
||||
c.Helper()
|
||||
ui, err := admClnt.GetUserInfo(ctx, accessKey)
|
||||
if err != nil {
|
||||
c.Fatalf("should be able to get user info: %v", err)
|
||||
@@ -1162,6 +1164,7 @@ func (c *check) mustGetIAMUserInfo(ctx context.Context, admClnt *madmin.AdminCli
|
||||
}
|
||||
|
||||
func (c *check) mustNotCreateIAMUser(ctx context.Context, admClnt *madmin.AdminClient) {
|
||||
c.Helper()
|
||||
randUser := mustGetUUID()
|
||||
randPass := mustGetUUID()
|
||||
err := admClnt.AddUser(ctx, randUser, randPass)
|
||||
@@ -1171,6 +1174,7 @@ func (c *check) mustNotCreateIAMUser(ctx context.Context, admClnt *madmin.AdminC
|
||||
}
|
||||
|
||||
func (c *check) mustCreateSvcAccount(ctx context.Context, tgtUser string, admClnt *madmin.AdminClient) madmin.Credentials {
|
||||
c.Helper()
|
||||
cr, err := admClnt.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
|
||||
TargetUser: tgtUser,
|
||||
})
|
||||
@@ -1181,6 +1185,7 @@ func (c *check) mustCreateSvcAccount(ctx context.Context, tgtUser string, admCln
|
||||
}
|
||||
|
||||
func (c *check) mustNotCreateSvcAccount(ctx context.Context, tgtUser string, admClnt *madmin.AdminClient) {
|
||||
c.Helper()
|
||||
_, err := admClnt.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
|
||||
TargetUser: tgtUser,
|
||||
})
|
||||
@@ -1190,6 +1195,7 @@ func (c *check) mustNotCreateSvcAccount(ctx context.Context, tgtUser string, adm
|
||||
}
|
||||
|
||||
func (c *check) mustNotListObjects(ctx context.Context, client *minio.Client, bucket string) {
|
||||
c.Helper()
|
||||
res := client.ListObjects(ctx, bucket, minio.ListObjectsOptions{})
|
||||
v, ok := <-res
|
||||
if !ok || v.Err == nil {
|
||||
@@ -1197,7 +1203,49 @@ func (c *check) mustNotListObjects(ctx context.Context, client *minio.Client, bu
|
||||
}
|
||||
}
|
||||
|
||||
func (c *check) mustPutObjectWithTags(ctx context.Context, client *minio.Client, bucket, object string) {
|
||||
c.Helper()
|
||||
_, err := client.PutObject(ctx, bucket, object, bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{
|
||||
UserTags: map[string]string{
|
||||
"security": "public",
|
||||
"virus": "true",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
c.Fatalf("user was unable to upload the object: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *check) mustGetObject(ctx context.Context, client *minio.Client, bucket, object string) {
|
||||
c.Helper()
|
||||
|
||||
r, err := client.GetObject(ctx, bucket, object, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
c.Fatalf("user was unable to download the object: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
_, err = io.Copy(io.Discard, r)
|
||||
if err != nil {
|
||||
c.Fatalf("user was unable to download the object: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *check) mustHeadObject(ctx context.Context, client *minio.Client, bucket, object string, tagCount int) {
|
||||
c.Helper()
|
||||
|
||||
oinfo, err := client.StatObject(ctx, bucket, object, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
c.Fatalf("user was unable to download the object: %v", err)
|
||||
}
|
||||
|
||||
if oinfo.UserTagCount != tagCount {
|
||||
c.Fatalf("expected tagCount: %d, got %d", tagCount, oinfo.UserTagCount)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *check) mustListObjects(ctx context.Context, client *minio.Client, bucket string) {
|
||||
c.Helper()
|
||||
res := client.ListObjects(ctx, bucket, minio.ListObjectsOptions{})
|
||||
v, ok := <-res
|
||||
if ok && v.Err != nil {
|
||||
@@ -1205,7 +1253,16 @@ func (c *check) mustListObjects(ctx context.Context, client *minio.Client, bucke
|
||||
}
|
||||
}
|
||||
|
||||
func (c *check) mustListBuckets(ctx context.Context, client *minio.Client) {
|
||||
c.Helper()
|
||||
_, err := client.ListBuckets(ctx)
|
||||
if err != nil {
|
||||
c.Fatalf("user was unable to list buckets: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *check) mustNotUpload(ctx context.Context, client *minio.Client, bucket string) {
|
||||
c.Helper()
|
||||
_, err := client.PutObject(ctx, bucket, "some-object", bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{})
|
||||
if e, ok := err.(minio.ErrorResponse); ok {
|
||||
if e.Code == "AccessDenied" {
|
||||
@@ -1221,6 +1278,7 @@ func (c *check) assertSvcAccS3Access(ctx context.Context, s *TestSuiteIAM, cr ma
|
||||
}
|
||||
|
||||
func (c *check) assertSvcAccAppearsInListing(ctx context.Context, madmClient *madmin.AdminClient, parentAK, svcAK string) {
|
||||
c.Helper()
|
||||
listResp, err := madmClient.ListServiceAccounts(ctx, parentAK)
|
||||
if err != nil {
|
||||
c.Fatalf("unable to list svc accounts: %v", err)
|
||||
@@ -1246,6 +1304,7 @@ func (c *check) assertSvcAccInfoQueryable(ctx context.Context, madmClient *madmi
|
||||
// bucket. It creates a session policy that restricts listing on the bucket and
|
||||
// then enables it again in a session policy update call.
|
||||
func (c *check) assertSvcAccSessionPolicyUpdate(ctx context.Context, s *TestSuiteIAM, madmClient *madmin.AdminClient, accessKey, bucket string) {
|
||||
c.Helper()
|
||||
svcAK, svcSK := mustGenerateCredentials(c)
|
||||
|
||||
// This policy does not allow listing objects.
|
||||
@@ -1301,6 +1360,7 @@ func (c *check) assertSvcAccSessionPolicyUpdate(ctx context.Context, s *TestSuit
|
||||
}
|
||||
|
||||
func (c *check) assertSvcAccSecretKeyAndStatusUpdate(ctx context.Context, s *TestSuiteIAM, madmClient *madmin.AdminClient, accessKey, bucket string) {
|
||||
c.Helper()
|
||||
svcAK, svcSK := mustGenerateCredentials(c)
|
||||
cr, err := madmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
|
||||
TargetUser: accessKey,
|
||||
@@ -1337,6 +1397,7 @@ func (c *check) assertSvcAccSecretKeyAndStatusUpdate(ctx context.Context, s *Tes
|
||||
}
|
||||
|
||||
func (c *check) assertSvcAccDeletion(ctx context.Context, s *TestSuiteIAM, madmClient *madmin.AdminClient, accessKey, bucket string) {
|
||||
c.Helper()
|
||||
svcAK, svcSK := mustGenerateCredentials(c)
|
||||
cr, err := madmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
|
||||
TargetUser: accessKey,
|
||||
@@ -1357,6 +1418,7 @@ func (c *check) assertSvcAccDeletion(ctx context.Context, s *TestSuiteIAM, madmC
|
||||
}
|
||||
|
||||
func mustGenerateCredentials(c *check) (string, string) {
|
||||
c.Helper()
|
||||
ak, sk, err := auth.GenerateCredentials()
|
||||
if err != nil {
|
||||
c.Fatalf("unable to generate credentials: %v", err)
|
||||
|
||||
+318
-231
@@ -21,13 +21,16 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
crand "crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/subtle"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
@@ -45,13 +48,13 @@ import (
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/klauspost/compress/zip"
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/estream"
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/logger/message/log"
|
||||
"github.com/minio/minio/internal/pubsub"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
xnet "github.com/minio/pkg/net"
|
||||
"github.com/secure-io/sio-go"
|
||||
@@ -71,16 +74,6 @@ const (
|
||||
mgmtForceStop = "forceStop"
|
||||
)
|
||||
|
||||
func updateServer(u *url.URL, sha256Sum []byte, lrTime time.Time, releaseInfo string, mode string) (us madmin.ServerUpdateStatus, err error) {
|
||||
if err = doUpdate(u, lrTime, sha256Sum, releaseInfo, mode); err != nil {
|
||||
return us, err
|
||||
}
|
||||
|
||||
us.CurrentVersion = Version
|
||||
us.UpdatedVersion = lrTime.Format(minioReleaseTagTimeLayout)
|
||||
return us, nil
|
||||
}
|
||||
|
||||
// ServerUpdateHandler - POST /minio/admin/v3/update?updateURL={updateURL}
|
||||
// ----------
|
||||
// updates all minio servers and restarts them gracefully.
|
||||
@@ -152,7 +145,16 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
|
||||
return
|
||||
}
|
||||
|
||||
for _, nerr := range globalNotificationSys.ServerUpdate(ctx, u, sha256Sum, lrTime, releaseInfo) {
|
||||
// Download Binary Once
|
||||
reader, err := downloadBinary(u, mode)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Push binary to other servers
|
||||
for _, nerr := range globalNotificationSys.VerifyBinary(ctx, u, sha256Sum, releaseInfo, reader) {
|
||||
if nerr.Err != nil {
|
||||
err := AdminError{
|
||||
Code: AdminUpdateApplyFailure,
|
||||
@@ -166,13 +168,39 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
}
|
||||
|
||||
updateStatus, err := updateServer(u, sha256Sum, lrTime, releaseInfo, mode)
|
||||
err = verifyBinary(u, sha256Sum, releaseInfo, mode, reader)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
for _, nerr := range globalNotificationSys.CommitBinary(ctx) {
|
||||
if nerr.Err != nil {
|
||||
err := AdminError{
|
||||
Code: AdminUpdateApplyFailure,
|
||||
Message: nerr.Err.Error(),
|
||||
StatusCode: http.StatusInternalServerError,
|
||||
}
|
||||
logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
|
||||
logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
err = commitBinary()
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
updateStatus := madmin.ServerUpdateStatus{
|
||||
CurrentVersion: Version,
|
||||
UpdatedVersion: lrTime.Format(minioReleaseTagTimeLayout),
|
||||
}
|
||||
|
||||
// Marshal API response
|
||||
jsonBytes, err := json.Marshal(updateStatus)
|
||||
if err != nil {
|
||||
@@ -363,18 +391,34 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
|
||||
if err != nil || interval < time.Second {
|
||||
interval = defaultMetricsInterval
|
||||
}
|
||||
|
||||
n, err := strconv.Atoi(r.Form.Get("n"))
|
||||
if err != nil || n <= 0 {
|
||||
n = math.MaxInt32
|
||||
}
|
||||
|
||||
var types madmin.MetricType
|
||||
if t, _ := strconv.ParseUint(r.Form.Get("types"), 10, 64); t != 0 {
|
||||
types = madmin.MetricType(t)
|
||||
} else {
|
||||
types = madmin.MetricsAll
|
||||
}
|
||||
|
||||
disks := strings.Split(r.Form.Get("disks"), ",")
|
||||
byDisk := strings.EqualFold(r.Form.Get("by-disk"), "true")
|
||||
var diskMap map[string]struct{}
|
||||
if len(disks) > 0 && disks[0] != "" {
|
||||
diskMap = make(map[string]struct{}, len(disks))
|
||||
for _, k := range disks {
|
||||
if k != "" {
|
||||
diskMap[k] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
jobID := r.Form.Get("by-jobID")
|
||||
|
||||
hosts := strings.Split(r.Form.Get("hosts"), ",")
|
||||
byhost := strings.EqualFold(r.Form.Get("by-host"), "true")
|
||||
byHost := strings.EqualFold(r.Form.Get("by-host"), "true")
|
||||
var hostMap map[string]struct{}
|
||||
if len(hosts) > 0 && hosts[0] != "" {
|
||||
hostMap = make(map[string]struct{}, len(hosts))
|
||||
@@ -384,34 +428,42 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
done := ctx.Done()
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
w.Header().Set(xhttp.ContentType, string(mimeJSON))
|
||||
|
||||
enc := json.NewEncoder(w)
|
||||
for n > 0 {
|
||||
var m madmin.RealtimeMetrics
|
||||
mLocal := collectLocalMetrics(types, hostMap)
|
||||
mLocal := collectLocalMetrics(types, collectMetricsOpts{
|
||||
hosts: hostMap,
|
||||
disks: diskMap,
|
||||
jobID: jobID,
|
||||
})
|
||||
m.Merge(&mLocal)
|
||||
|
||||
// Allow half the interval for collecting remote...
|
||||
cctx, cancel := context.WithTimeout(ctx, interval/2)
|
||||
mRemote := collectRemoteMetrics(cctx, types, hostMap)
|
||||
mRemote := collectRemoteMetrics(cctx, types, collectMetricsOpts{
|
||||
hosts: hostMap,
|
||||
disks: diskMap,
|
||||
jobID: jobID,
|
||||
})
|
||||
cancel()
|
||||
m.Merge(&mRemote)
|
||||
if !byhost {
|
||||
if !byHost {
|
||||
m.ByHost = nil
|
||||
}
|
||||
if !byDisk {
|
||||
m.ByDisk = nil
|
||||
}
|
||||
|
||||
m.Final = n <= 1
|
||||
// Marshal API response
|
||||
jsonBytes, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
_, err = w.Write(jsonBytes)
|
||||
if err != nil {
|
||||
|
||||
// Marshal API reesponse
|
||||
if err := enc.Encode(&m); err != nil {
|
||||
n = 0
|
||||
}
|
||||
|
||||
@@ -718,7 +770,7 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
}
|
||||
// read request body
|
||||
io.CopyN(ioutil.Discard, r.Body, 1)
|
||||
io.CopyN(io.Discard, r.Body, 1)
|
||||
|
||||
globalProfilerMu.Lock()
|
||||
|
||||
@@ -896,11 +948,6 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
hip, errCode := extractHealInitParams(mux.Vars(r), r.Form, r.Body)
|
||||
if errCode != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
|
||||
@@ -1087,12 +1134,6 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
|
||||
return
|
||||
}
|
||||
|
||||
// Check if this setup has an erasure coded backend.
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
aggregateHealStateResult, err := getAggregatedBackgroundHealState(r.Context(), objectAPI)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
@@ -1168,11 +1209,6 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
sizeStr := r.Form.Get(peerRESTSize)
|
||||
durationStr := r.Form.Get(peerRESTDuration)
|
||||
concurrentStr := r.Form.Get(peerRESTConcurrent)
|
||||
@@ -1190,16 +1226,14 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
|
||||
concurrent = 32
|
||||
}
|
||||
|
||||
if runtime.GOMAXPROCS(0) < concurrent {
|
||||
concurrent = runtime.GOMAXPROCS(0)
|
||||
}
|
||||
|
||||
duration, err := time.ParseDuration(durationStr)
|
||||
if err != nil {
|
||||
duration = time.Second * 10
|
||||
}
|
||||
|
||||
sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, objectAPI, concurrent, size, autotune)
|
||||
storageInfo, _ := objectAPI.StorageInfo(ctx)
|
||||
|
||||
sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, storageInfo, concurrent, size, autotune)
|
||||
if !sufficientCapacity {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, AdminError{
|
||||
Code: "XMinioSpeedtestInsufficientCapacity",
|
||||
@@ -1249,14 +1283,24 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
|
||||
storageClass: storageClass,
|
||||
bucketName: customBucket,
|
||||
})
|
||||
var prevResult madmin.SpeedTestResult
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-keepAliveTicker.C:
|
||||
// Write a blank entry to prevent client from disconnecting
|
||||
if err := enc.Encode(madmin.SpeedTestResult{}); err != nil {
|
||||
return
|
||||
// if previous result is set keep writing the
|
||||
// previous result back to the client
|
||||
if prevResult.Version != "" {
|
||||
if err := enc.Encode(prevResult); err != nil {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
// first result is not yet obtained, keep writing
|
||||
// empty entry to prevent client from disconnecting.
|
||||
if err := enc.Encode(madmin.SpeedTestResult{}); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
case result, ok := <-ch:
|
||||
@@ -1266,13 +1310,14 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
|
||||
if err := enc.Encode(result); err != nil {
|
||||
return
|
||||
}
|
||||
prevResult = result
|
||||
w.(http.Flusher).Flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer, bucketName string) (bucketExists bool, err error) {
|
||||
if err = objectAPI.MakeBucketWithLocation(ctx, bucketName, BucketOptions{}); err != nil {
|
||||
if err = objectAPI.MakeBucketWithLocation(ctx, bucketName, MakeBucketOptions{}); err != nil {
|
||||
if _, ok := err.(BucketExists); !ok {
|
||||
// Only BucketExists error can be ignored.
|
||||
return false, err
|
||||
@@ -1286,11 +1331,11 @@ func deleteObjectPerfBucket(objectAPI ObjectLayer) {
|
||||
objectAPI.DeleteBucket(context.Background(), globalObjectPerfBucket, DeleteBucketOptions{
|
||||
Force: true,
|
||||
NoRecreate: true,
|
||||
SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
|
||||
})
|
||||
}
|
||||
|
||||
func validateObjPerfOptions(ctx context.Context, objectAPI ObjectLayer, concurrent int, size int, autotune bool) (sufficientCapacity bool, canAutotune bool, capacityErrMsg string) {
|
||||
storageInfo, _ := objectAPI.StorageInfo(ctx)
|
||||
func validateObjPerfOptions(ctx context.Context, storageInfo madmin.StorageInfo, concurrent int, size int, autotune bool) (bool, bool, string) {
|
||||
capacityNeeded := uint64(concurrent * size)
|
||||
capacity := GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo)
|
||||
|
||||
@@ -1332,11 +1377,6 @@ func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.R
|
||||
return
|
||||
}
|
||||
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Freeze all incoming S3 API calls before running speedtest.
|
||||
globalNotificationSys.ServiceFreeze(ctx, true)
|
||||
|
||||
@@ -1468,16 +1508,12 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
|
||||
// Use buffered channel to take care of burst sends or slow w.Write()
|
||||
traceCh := make(chan pubsub.Maskable, 4000)
|
||||
traceCh := make(chan madmin.TraceInfo, 4000)
|
||||
|
||||
peers, _ := newPeerRestClients(globalEndpoints)
|
||||
mask := pubsub.MaskFromMaskable(traceOpts.TraceTypes())
|
||||
|
||||
err = globalTrace.Subscribe(mask, traceCh, ctx.Done(), func(entry pubsub.Maskable) bool {
|
||||
if e, ok := entry.(madmin.TraceInfo); ok {
|
||||
return shouldTrace(e, traceOpts)
|
||||
}
|
||||
return false
|
||||
err = globalTrace.Subscribe(traceOpts.TraceTypes(), traceCh, ctx.Done(), func(entry madmin.TraceInfo) bool {
|
||||
return shouldTrace(entry, traceOpts)
|
||||
})
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrSlowDown), r.URL)
|
||||
@@ -1549,7 +1585,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
setEventStreamHeaders(w)
|
||||
|
||||
logCh := make(chan pubsub.Maskable, 4000)
|
||||
logCh := make(chan log.Info, 4000)
|
||||
|
||||
peers, _ := newPeerRestClients(globalEndpoints)
|
||||
|
||||
@@ -1575,9 +1611,11 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
for {
|
||||
select {
|
||||
case entry := <-logCh:
|
||||
log, ok := entry.(log.Info)
|
||||
if ok && log.SendLog(node, logKind) {
|
||||
case log, ok := <-logCh:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if log.SendLog(node, logKind) {
|
||||
if err := enc.Encode(log); err != nil {
|
||||
return
|
||||
}
|
||||
@@ -1741,8 +1779,8 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
|
||||
kmsStat := fetchKMSStatus()
|
||||
|
||||
ldap := madmin.LDAP{}
|
||||
if globalLDAPConfig.Enabled {
|
||||
ldapConn, err := globalLDAPConfig.Connect()
|
||||
if globalLDAPConfig.Enabled() {
|
||||
ldapConn, err := globalLDAPConfig.LDAP.Connect()
|
||||
//nolint:gocritic
|
||||
if err != nil {
|
||||
ldap.Status = string(madmin.ItemOffline)
|
||||
@@ -1828,7 +1866,7 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
|
||||
Mode: string(mode),
|
||||
Domain: domain,
|
||||
Region: globalSite.Region,
|
||||
SQSARN: globalNotificationSys.GetARNList(false),
|
||||
SQSARN: globalEventNotifier.GetARNList(false),
|
||||
DeploymentID: globalDeploymentID,
|
||||
Buckets: buckets,
|
||||
Objects: objects,
|
||||
@@ -1853,7 +1891,7 @@ func getKubernetesInfo(dctx context.Context) madmin.KubernetesInfo {
|
||||
}
|
||||
|
||||
client := &http.Client{
|
||||
Transport: NewGatewayHTTPTransport(),
|
||||
Transport: NewHTTPTransport(),
|
||||
Timeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
@@ -1911,11 +1949,6 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
|
||||
deadline := 10 * time.Second // Default deadline is 10secs for health diagnostics.
|
||||
if query.Get(string(madmin.HealthDataTypePerfNet)) != "" ||
|
||||
query.Get(string(madmin.HealthDataTypePerfDrive)) != "" ||
|
||||
query.Get(string(madmin.HealthDataTypePerfObj)) != "" {
|
||||
deadline = 1 * time.Hour
|
||||
}
|
||||
if dstr := r.Form.Get("deadline"); dstr != "" {
|
||||
var err error
|
||||
deadline, err = time.ParseDuration(dstr)
|
||||
@@ -1936,12 +1969,6 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
healthCtx, healthCancel := context.WithTimeout(lkctx.Context(), deadline)
|
||||
defer healthCancel()
|
||||
|
||||
// Freeze all incoming S3 API calls before running speedtest.
|
||||
globalNotificationSys.ServiceFreeze(ctx, true)
|
||||
|
||||
// unfreeze all incoming S3 API calls after speedtest.
|
||||
defer globalNotificationSys.ServiceFreeze(ctx, false)
|
||||
|
||||
hostAnonymizer := createHostAnonymizer()
|
||||
// anonAddr - Anonymizes hosts in given input string.
|
||||
anonAddr := func(addr string) string {
|
||||
@@ -2087,7 +2114,10 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
if !globalIsDistErasure {
|
||||
// FS mode - single server - hard code to `server1`
|
||||
anonCmdLine := strings.ReplaceAll(cmdLine, globalLocalNodeName, "server1")
|
||||
return strings.ReplaceAll(anonCmdLine, globalMinioConsoleHost, "server1")
|
||||
if len(globalMinioConsoleHost) > 0 {
|
||||
anonCmdLine = strings.ReplaceAll(anonCmdLine, globalMinioConsoleHost, "server1")
|
||||
}
|
||||
return anonCmdLine
|
||||
}
|
||||
|
||||
// Server start command regex groups:
|
||||
@@ -2177,89 +2207,6 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
}
|
||||
|
||||
getAndWriteDrivePerfInfo := func() {
|
||||
if query.Get(string(madmin.HealthDataTypePerfDrive)) == "true" {
|
||||
opts := madmin.DriveSpeedTestOpts{
|
||||
Serial: false,
|
||||
BlockSize: 4 * humanize.MiByte,
|
||||
FileSize: 1 * humanize.GiByte,
|
||||
}
|
||||
|
||||
localDPI := driveSpeedTest(ctx, opts)
|
||||
healthInfo.Perf.DrivePerf = append(healthInfo.Perf.DrivePerf, localDPI)
|
||||
|
||||
perfCh := globalNotificationSys.DriveSpeedTest(ctx, opts)
|
||||
for perfInfo := range perfCh {
|
||||
healthInfo.Perf.DrivePerf = append(healthInfo.Perf.DrivePerf, perfInfo)
|
||||
}
|
||||
partialWrite(healthInfo)
|
||||
}
|
||||
}
|
||||
|
||||
getAndWriteObjPerfInfo := func() {
|
||||
if query.Get(string(madmin.HealthDataTypePerfObj)) == "true" {
|
||||
concurrent := 32
|
||||
if runtime.GOMAXPROCS(0) < concurrent {
|
||||
concurrent = runtime.GOMAXPROCS(0)
|
||||
}
|
||||
|
||||
size := 64 * humanize.MiByte
|
||||
autotune := true
|
||||
|
||||
sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, objectAPI, concurrent, size, autotune)
|
||||
|
||||
if !sufficientCapacity {
|
||||
healthInfo.Perf.Error = capacityErrMsg
|
||||
partialWrite(healthInfo)
|
||||
return
|
||||
}
|
||||
|
||||
if !canAutotune {
|
||||
autotune = false
|
||||
}
|
||||
|
||||
bucketExists, err := makeObjectPerfBucket(ctx, objectAPI, globalObjectPerfBucket)
|
||||
if err != nil {
|
||||
healthInfo.Perf.Error = "Unable to create bucket: " + err.Error()
|
||||
partialWrite(healthInfo)
|
||||
return
|
||||
}
|
||||
|
||||
if !bucketExists {
|
||||
defer deleteObjectPerfBucket(objectAPI)
|
||||
}
|
||||
|
||||
opts := speedTestOpts{
|
||||
objectSize: size,
|
||||
concurrencyStart: concurrent,
|
||||
duration: 10 * time.Second,
|
||||
autotune: autotune,
|
||||
}
|
||||
|
||||
perfCh := objectSpeedTest(ctx, opts)
|
||||
for perfInfo := range perfCh {
|
||||
healthInfo.Perf.ObjPerf = append(healthInfo.Perf.ObjPerf, perfInfo)
|
||||
}
|
||||
partialWrite(healthInfo)
|
||||
}
|
||||
}
|
||||
|
||||
getAndWriteNetPerfInfo := func() {
|
||||
if query.Get(string(madmin.HealthDataTypePerfObj)) == "true" {
|
||||
if !globalIsDistErasure {
|
||||
return
|
||||
}
|
||||
|
||||
netPerf := globalNotificationSys.Netperf(ctx, time.Second*10)
|
||||
for _, np := range netPerf {
|
||||
np.Endpoint = anonAddr(np.Endpoint)
|
||||
healthInfo.Perf.NetPerf = append(healthInfo.Perf.NetPerf, np)
|
||||
}
|
||||
|
||||
partialWrite(healthInfo)
|
||||
}
|
||||
}
|
||||
|
||||
anonymizeNetwork := func(network map[string]string) map[string]string {
|
||||
anonNetwork := map[string]string{}
|
||||
for endpoint, status := range network {
|
||||
@@ -2289,16 +2236,13 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
getAndWriteMemInfo()
|
||||
getAndWriteProcInfo()
|
||||
getAndWriteMinioConfig()
|
||||
getAndWriteDrivePerfInfo()
|
||||
getAndWriteObjPerfInfo()
|
||||
getAndWriteNetPerfInfo()
|
||||
getAndWriteSysErrors()
|
||||
getAndWriteSysServices()
|
||||
getAndWriteSysConfig()
|
||||
|
||||
if query.Get("minioinfo") == "true" {
|
||||
infoMessage := getServerInfo(ctx, r)
|
||||
servers := []madmin.ServerInfo{}
|
||||
servers := make([]madmin.ServerInfo, 0, len(infoMessage.Servers))
|
||||
for _, server := range infoMessage.Servers {
|
||||
anonEndpoint := anonAddr(server.Endpoint)
|
||||
servers = append(servers, madmin.ServerInfo{
|
||||
@@ -2317,6 +2261,11 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
Frees: server.MemStats.Frees,
|
||||
HeapAlloc: server.MemStats.HeapAlloc,
|
||||
},
|
||||
GoMaxProcs: server.GoMaxProcs,
|
||||
NumCPU: server.NumCPU,
|
||||
RuntimeVersion: server.RuntimeVersion,
|
||||
GCStats: server.GCStats,
|
||||
MinioEnvVars: server.MinioEnvVars,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2509,20 +2458,6 @@ func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
|
||||
lambdaMap[targetID.Name] = list
|
||||
}
|
||||
|
||||
for _, tgt := range globalEnvTargetList.Targets() {
|
||||
targetIDStatus := make(map[string]madmin.Status)
|
||||
active, _ := tgt.IsActive()
|
||||
targetID := tgt.ID()
|
||||
if active {
|
||||
targetIDStatus[targetID.ID] = madmin.Status{Status: string(madmin.ItemOnline)}
|
||||
} else {
|
||||
targetIDStatus[targetID.ID] = madmin.Status{Status: string(madmin.ItemOffline)}
|
||||
}
|
||||
list := lambdaMap[targetID.Name]
|
||||
list = append(list, targetIDStatus)
|
||||
lambdaMap[targetID.Name] = list
|
||||
}
|
||||
|
||||
notify := make([]map[string][]madmin.TargetIDStatus, len(lambdaMap))
|
||||
counter := 0
|
||||
for key, value := range lambdaMap {
|
||||
@@ -2636,6 +2571,94 @@ func checkConnection(endpointStr string, timeout time.Duration) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func embedFileInZip(zipWriter *zip.Writer, name string, data []byte) error {
|
||||
// Send profiling data to zip as file
|
||||
header, zerr := zip.FileInfoHeader(dummyFileInfo{
|
||||
name: name,
|
||||
size: int64(len(data)),
|
||||
mode: 0o600,
|
||||
modTime: UTCNow(),
|
||||
isDir: false,
|
||||
sys: nil,
|
||||
})
|
||||
if zerr != nil {
|
||||
return zerr
|
||||
}
|
||||
header.Method = zip.Deflate
|
||||
zwriter, zerr := zipWriter.CreateHeader(header)
|
||||
if zerr != nil {
|
||||
return zerr
|
||||
}
|
||||
_, err := io.Copy(zwriter, bytes.NewReader(data))
|
||||
return err
|
||||
}
|
||||
|
||||
// getClusterMetaInfo gets information of the current cluster and
|
||||
// returns it.
|
||||
// This is not a critical function, and it is allowed
|
||||
// to fail with a ten seconds timeout, returning nil.
|
||||
func getClusterMetaInfo(ctx context.Context) []byte {
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Add a ten seconds timeout because getting profiling data
|
||||
// is critical for debugging, in contrary to getting cluster info
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
resultCh := make(chan madmin.ClusterRegistrationInfo)
|
||||
go func() {
|
||||
ci := madmin.ClusterRegistrationInfo{}
|
||||
ci.Info.NoOfServerPools = len(globalEndpoints)
|
||||
ci.Info.NoOfServers = len(globalEndpoints.Hostnames())
|
||||
ci.Info.MinioVersion = Version
|
||||
|
||||
si, _ := objectAPI.StorageInfo(ctx)
|
||||
|
||||
ci.Info.NoOfDrives = len(si.Disks)
|
||||
for _, disk := range si.Disks {
|
||||
ci.Info.TotalDriveSpace += disk.TotalSpace
|
||||
ci.Info.UsedDriveSpace += disk.UsedSpace
|
||||
}
|
||||
|
||||
dataUsageInfo, _ := loadDataUsageFromBackend(ctx, objectAPI)
|
||||
|
||||
ci.UsedCapacity = dataUsageInfo.ObjectsTotalSize
|
||||
ci.Info.NoOfBuckets = dataUsageInfo.BucketsCount
|
||||
ci.Info.NoOfObjects = dataUsageInfo.ObjectsTotalCount
|
||||
|
||||
ci.DeploymentID = globalDeploymentID
|
||||
ci.ClusterName = fmt.Sprintf("%d-servers-%d-disks-%s", ci.Info.NoOfServers, ci.Info.NoOfDrives, ci.Info.MinioVersion)
|
||||
resultCh <- ci
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case ci := <-resultCh:
|
||||
out, err := json.MarshalIndent(ci, "", " ")
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
func bytesToPublicKey(pub []byte) (*rsa.PublicKey, error) {
|
||||
block, _ := pem.Decode(pub)
|
||||
if block != nil {
|
||||
pub = block.Bytes
|
||||
}
|
||||
key, err := x509.ParsePKCS1PublicKey(pub)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// getRawDataer provides an interface for getting raw FS files.
|
||||
type getRawDataer interface {
|
||||
GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, info StatInfo) error) error
|
||||
@@ -2655,18 +2678,24 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
o, ok := newObjectLayerFn().(getRawDataer)
|
||||
objLayer := newObjectLayerFn()
|
||||
o, ok := objLayer.(getRawDataer)
|
||||
if !ok {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err := parseForm(r); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
volume := r.Form.Get("volume")
|
||||
file := r.Form.Get("file")
|
||||
if len(volume) == 0 {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketName), r.URL)
|
||||
return
|
||||
}
|
||||
file := r.Form.Get("file")
|
||||
if len(file) == 0 {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||
return
|
||||
@@ -2679,41 +2708,102 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
var key [32]byte
|
||||
// MUST use crypto/rand
|
||||
n, err := crand.Read(key[:])
|
||||
if err != nil || n != len(key) {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
|
||||
return
|
||||
}
|
||||
stream, err := sio.AES_256_GCM.Stream(key[:])
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
|
||||
return
|
||||
}
|
||||
// Zero nonce, we only use each key once, and 32 bytes is plenty.
|
||||
nonce := make([]byte, stream.NonceSize())
|
||||
encw := stream.EncryptWriter(w, nonce, nil)
|
||||
var publicKey *rsa.PublicKey
|
||||
|
||||
defer encw.Close()
|
||||
publicKeyB64 := r.Form.Get("public-key")
|
||||
if publicKeyB64 != "" {
|
||||
publicKeyBytes, err := base64.StdEncoding.DecodeString(publicKeyB64)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
publicKey, err = bytesToPublicKey(publicKeyBytes)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Write a version for making *incompatible* changes.
|
||||
// The AdminClient will reject any version it does not know.
|
||||
w.Write([]byte{1})
|
||||
var inspectZipW *zip.Writer
|
||||
if publicKey != nil {
|
||||
w.WriteHeader(200)
|
||||
stream := estream.NewWriter(w)
|
||||
defer stream.Close()
|
||||
|
||||
// Write key first (without encryption)
|
||||
_, err = w.Write(key[:])
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
|
||||
return
|
||||
clusterKey, err := bytesToPublicKey(subnetAdminPublicKey)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, stream.AddError(err.Error()))
|
||||
return
|
||||
}
|
||||
err = stream.AddKeyEncrypted(clusterKey)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, stream.AddError(err.Error()))
|
||||
return
|
||||
}
|
||||
if b := getClusterMetaInfo(ctx); len(b) > 0 {
|
||||
w, err := stream.AddEncryptedStream("cluster.info", nil)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return
|
||||
}
|
||||
w.Write(b)
|
||||
w.Close()
|
||||
}
|
||||
|
||||
// Add new key for inspect data.
|
||||
if err := stream.AddKeyEncrypted(publicKey); err != nil {
|
||||
logger.LogIf(ctx, stream.AddError(err.Error()))
|
||||
return
|
||||
}
|
||||
encStream, err := stream.AddEncryptedStream("inspect.zip", nil)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, stream.AddError(err.Error()))
|
||||
return
|
||||
}
|
||||
defer encStream.Close()
|
||||
|
||||
inspectZipW = zip.NewWriter(encStream)
|
||||
defer inspectZipW.Close()
|
||||
} else {
|
||||
// Legacy: Remove if we stop supporting inspection without public key.
|
||||
var key [32]byte
|
||||
// MUST use crypto/rand
|
||||
n, err := crand.Read(key[:])
|
||||
if err != nil || n != len(key) {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Write a version for making *incompatible* changes.
|
||||
// The AdminClient will reject any version it does not know.
|
||||
if publicKey == nil {
|
||||
w.Write([]byte{1})
|
||||
w.Write(key[:])
|
||||
}
|
||||
|
||||
stream, err := sio.AES_256_GCM.Stream(key[:])
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return
|
||||
}
|
||||
// Zero nonce, we only use each key once, and 32 bytes is plenty.
|
||||
nonce := make([]byte, stream.NonceSize())
|
||||
encw := stream.EncryptWriter(w, nonce, nil)
|
||||
defer encw.Close()
|
||||
|
||||
// Initialize a zip writer which will provide a zipped content
|
||||
// of profiling data of all nodes
|
||||
inspectZipW = zip.NewWriter(encw)
|
||||
defer inspectZipW.Close()
|
||||
|
||||
if b := getClusterMetaInfo(ctx); len(b) > 0 {
|
||||
logger.LogIf(ctx, embedFileInZip(inspectZipW, "cluster.info", b))
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize a zip writer which will provide a zipped content
|
||||
// of profiling data of all nodes
|
||||
zipWriter := zip.NewWriter(encw)
|
||||
defer zipWriter.Close()
|
||||
rawDataFn := func(r io.Reader, host, disk, filename string, si StatInfo) error {
|
||||
// Prefix host+disk
|
||||
filename = path.Join(host, disk, filename)
|
||||
@@ -2742,17 +2832,17 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
|
||||
return nil
|
||||
}
|
||||
header.Method = zip.Deflate
|
||||
zwriter, zerr := zipWriter.CreateHeader(header)
|
||||
zwriter, zerr := inspectZipW.CreateHeader(header)
|
||||
if zerr != nil {
|
||||
logger.LogIf(ctx, zerr)
|
||||
return nil
|
||||
}
|
||||
if _, err = io.Copy(zwriter, r); err != nil {
|
||||
if _, err := io.Copy(zwriter, r); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
err = o.GetRawData(ctx, volume, file, rawDataFn)
|
||||
err := o.GetRawData(ctx, volume, file, rawDataFn)
|
||||
if !errors.Is(err, errFileNotFound) {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
@@ -2764,20 +2854,17 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
|
||||
if !errors.Is(err, errFileNotFound) {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
|
||||
// save args passed to inspect command
|
||||
inspectArgs := []string{fmt.Sprintf(" Inspect path: %s%s%s\n", volume, slashSeparator, file)}
|
||||
cmdLine := []string{"Server command line args: "}
|
||||
var sb bytes.Buffer
|
||||
fmt.Fprintf(&sb, "Inspect path: %s%s%s\n", volume, slashSeparator, file)
|
||||
sb.WriteString("Server command line args:")
|
||||
for _, pool := range globalEndpoints {
|
||||
cmdLine = append(cmdLine, pool.CmdLine)
|
||||
}
|
||||
cmdLine = append(cmdLine, "\n")
|
||||
inspectArgs = append(inspectArgs, cmdLine...)
|
||||
inspectArgsBytes := []byte(strings.Join(inspectArgs, " "))
|
||||
if err = rawDataFn(bytes.NewReader(inspectArgsBytes), "", "", "inspect-input.txt", StatInfo{
|
||||
Size: int64(len(inspectArgsBytes)),
|
||||
}); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
sb.WriteString(" ")
|
||||
sb.WriteString(pool.CmdLine)
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
logger.LogIf(ctx, embedFileInZip(inspectZipW, "inspect-input.txt", sb.Bytes()))
|
||||
}
|
||||
|
||||
func createHostAnonymizerForFSMode() map[string]string {
|
||||
|
||||
@@ -22,7 +22,6 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
@@ -74,7 +73,7 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
|
||||
|
||||
globalEndpoints = mustGetPoolEndpoints(erasureDirs...)
|
||||
|
||||
initAllSubsystems()
|
||||
initAllSubsystems(ctx)
|
||||
|
||||
initConfigSubsystem(ctx, objLayer)
|
||||
|
||||
@@ -220,7 +219,7 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
|
||||
adminTestBed.router.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
resp, _ := ioutil.ReadAll(rec.Body)
|
||||
resp, _ := io.ReadAll(rec.Body)
|
||||
t.Errorf("Expected to receive %d status code but received %d. Body (%s)",
|
||||
http.StatusOK, rec.Code, string(resp))
|
||||
}
|
||||
|
||||
@@ -100,14 +100,14 @@ type allHealState struct {
|
||||
}
|
||||
|
||||
// newHealState - initialize global heal state management
|
||||
func newHealState(cleanup bool) *allHealState {
|
||||
func newHealState(ctx context.Context, cleanup bool) *allHealState {
|
||||
hstate := &allHealState{
|
||||
healSeqMap: make(map[string]*healSequence),
|
||||
healLocalDisks: make(map[Endpoint]bool),
|
||||
healStatus: make(map[string]healingTracker),
|
||||
}
|
||||
if cleanup {
|
||||
go hstate.periodicHealSeqsClean(GlobalContext)
|
||||
go hstate.periodicHealSeqsClean(ctx)
|
||||
}
|
||||
return hstate
|
||||
}
|
||||
@@ -822,7 +822,7 @@ func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
|
||||
return h.healBucket(objAPI, h.bucket, bucketsOnly)
|
||||
}
|
||||
|
||||
buckets, err := objAPI.ListBuckets(h.ctx)
|
||||
buckets, err := objAPI.ListBuckets(h.ctx, BucketOptions{})
|
||||
if err != nil {
|
||||
return errFnHealFromAPIErr(h.ctx, err)
|
||||
}
|
||||
|
||||
+32
-14
@@ -60,7 +60,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
|
||||
// Info operations
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/info").HandlerFunc(gz(httpTraceAll(adminAPI.ServerInfoHandler)))
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/inspect-data").HandlerFunc(httpTraceHdrs(adminAPI.InspectDataHandler)).Queries("volume", "{volume:.*}", "file", "{file:.*}")
|
||||
adminRouter.Methods(http.MethodGet, http.MethodPost).Path(adminVersion + "/inspect-data").HandlerFunc(httpTraceAll(adminAPI.InspectDataHandler))
|
||||
|
||||
// StorageInfo operations
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.StorageInfoHandler)))
|
||||
@@ -84,6 +84,11 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/decommission").HandlerFunc(gz(httpTraceAll(adminAPI.StartDecommission))).Queries("pool", "{pool:.*}")
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/cancel").HandlerFunc(gz(httpTraceAll(adminAPI.CancelDecommission))).Queries("pool", "{pool:.*}")
|
||||
|
||||
// Rebalance operations
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/rebalance/start").HandlerFunc(gz(httpTraceAll(adminAPI.RebalanceStart)))
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/rebalance/status").HandlerFunc(gz(httpTraceAll(adminAPI.RebalanceStatus)))
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/rebalance/stop").HandlerFunc(gz(httpTraceAll(adminAPI.RebalanceStop)))
|
||||
}
|
||||
|
||||
// Profiling operations - deprecated API
|
||||
@@ -179,9 +184,14 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam").HandlerFunc(httpTraceHdrs(adminAPI.ImportIAM))
|
||||
|
||||
// IDentity Provider configuration APIs
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetIdentityProviderCfg))).Queries("type", "{type:.*}").Queries("name", "{name:.*}")
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetIdentityProviderCfg))).Queries("type", "{type:.*}")
|
||||
adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.DeleteIdentityProviderCfg))).Queries("type", "{type:.*}").Queries("name", "{name:.*}")
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddIdentityProviderCfg)))
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.UpdateIdentityProviderCfg)))
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListIdentityProviderCfg)))
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetIdentityProviderCfg)))
|
||||
adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.DeleteIdentityProviderCfg)))
|
||||
|
||||
// LDAP IAM operations
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/ldap/policy-entities").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListLDAPPolicyMappingEntities)))
|
||||
|
||||
// -- END IAM APIs --
|
||||
|
||||
@@ -206,6 +216,16 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/replication/diff").HandlerFunc(
|
||||
gz(httpTraceHdrs(adminAPI.ReplicationDiffHandler))).Queries("bucket", "{bucket:.*}")
|
||||
|
||||
// Batch job operations
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/start-job").HandlerFunc(
|
||||
gz(httpTraceHdrs(adminAPI.StartBatchJob)))
|
||||
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-jobs").HandlerFunc(
|
||||
gz(httpTraceHdrs(adminAPI.ListBatchJobs)))
|
||||
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/describe-job").HandlerFunc(
|
||||
gz(httpTraceHdrs(adminAPI.DescribeBatchJob)))
|
||||
|
||||
// Bucket migration operations
|
||||
// ExportBucketMetaHandler
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/export-bucket-metadata").HandlerFunc(
|
||||
@@ -264,16 +284,14 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/kms/key/create").HandlerFunc(gz(httpTraceAll(adminAPI.KMSCreateKeyHandler))).Queries("key-id", "{key-id:.*}")
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(gz(httpTraceAll(adminAPI.KMSKeyStatusHandler)))
|
||||
|
||||
if !globalIsGateway {
|
||||
// Keep obdinfo for backward compatibility with mc
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
|
||||
HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
|
||||
// -- Health API --
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/healthinfo").
|
||||
HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/bandwidth").
|
||||
HandlerFunc(gz(httpTraceHdrs(adminAPI.BandwidthMonitorHandler)))
|
||||
}
|
||||
// Keep obdinfo for backward compatibility with mc
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
|
||||
HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
|
||||
// -- Health API --
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/healthinfo").
|
||||
HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/bandwidth").
|
||||
HandlerFunc(gz(httpTraceHdrs(adminAPI.BandwidthMonitorHandler)))
|
||||
}
|
||||
|
||||
// If none of the routes match add default error handler routes
|
||||
|
||||
@@ -20,10 +20,14 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
)
|
||||
|
||||
@@ -67,6 +71,22 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
|
||||
var memstats runtime.MemStats
|
||||
runtime.ReadMemStats(&memstats)
|
||||
|
||||
gcStats := debug.GCStats{
|
||||
// If stats.PauseQuantiles is non-empty, ReadGCStats fills
|
||||
// it with quantiles summarizing the distribution of pause time.
|
||||
// For example, if len(stats.PauseQuantiles) is 5, it will be
|
||||
// filled with the minimum, 25%, 50%, 75%, and maximum pause times.
|
||||
PauseQuantiles: make([]time.Duration, 5),
|
||||
}
|
||||
debug.ReadGCStats(&gcStats)
|
||||
// Truncate GC stats to max 5 entries.
|
||||
if len(gcStats.PauseEnd) > 5 {
|
||||
gcStats.PauseEnd = gcStats.PauseEnd[len(gcStats.PauseEnd)-5:]
|
||||
}
|
||||
if len(gcStats.Pause) > 5 {
|
||||
gcStats.Pause = gcStats.Pause[len(gcStats.Pause)-5:]
|
||||
}
|
||||
|
||||
props := madmin.ServerProperties{
|
||||
State: string(madmin.ItemInitializing),
|
||||
Endpoint: addr,
|
||||
@@ -81,10 +101,48 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
|
||||
Frees: memstats.Frees,
|
||||
HeapAlloc: memstats.HeapAlloc,
|
||||
},
|
||||
GoMaxProcs: runtime.GOMAXPROCS(0),
|
||||
NumCPU: runtime.NumCPU(),
|
||||
RuntimeVersion: runtime.Version(),
|
||||
GCStats: &madmin.GCStats{
|
||||
LastGC: gcStats.LastGC,
|
||||
NumGC: gcStats.NumGC,
|
||||
PauseTotal: gcStats.PauseTotal,
|
||||
Pause: gcStats.Pause,
|
||||
PauseEnd: gcStats.PauseEnd,
|
||||
},
|
||||
MinioEnvVars: make(map[string]string, 10),
|
||||
}
|
||||
|
||||
sensitive := map[string]struct{}{
|
||||
config.EnvAccessKey: {},
|
||||
config.EnvSecretKey: {},
|
||||
config.EnvRootUser: {},
|
||||
config.EnvRootPassword: {},
|
||||
config.EnvMinIOSubnetAPIKey: {},
|
||||
config.EnvKMSSecretKey: {},
|
||||
}
|
||||
for _, v := range os.Environ() {
|
||||
if !strings.HasPrefix(v, "MINIO") && !strings.HasPrefix(v, "_MINIO") {
|
||||
continue
|
||||
}
|
||||
split := strings.SplitN(v, "=", 2)
|
||||
key := split[0]
|
||||
value := ""
|
||||
if len(split) > 1 {
|
||||
value = split[1]
|
||||
}
|
||||
|
||||
// Do not send sensitive creds.
|
||||
if _, ok := sensitive[key]; ok || strings.Contains(strings.ToLower(key), "password") || strings.HasSuffix(strings.ToLower(key), "key") {
|
||||
props.MinioEnvVars[key] = "*** EXISTS, REDACTED ***"
|
||||
continue
|
||||
}
|
||||
props.MinioEnvVars[key] = value
|
||||
}
|
||||
|
||||
objLayer := newObjectLayerFn()
|
||||
if objLayer != nil && !globalIsGateway {
|
||||
if objLayer != nil {
|
||||
// only need Disks information in server mode.
|
||||
storageInfo, _ := objLayer.LocalStorageInfo(GlobalContext)
|
||||
props.State = string(madmin.ItemOnline)
|
||||
|
||||
+90
-9
@@ -30,7 +30,8 @@ import (
|
||||
"github.com/Azure/azure-storage-blob-go/azblob"
|
||||
"google.golang.org/api/googleapi"
|
||||
|
||||
minio "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
@@ -176,6 +177,7 @@ const (
|
||||
ErrBucketAlreadyOwnedByYou
|
||||
ErrInvalidDuration
|
||||
ErrBucketAlreadyExists
|
||||
ErrTooManyBuckets
|
||||
ErrMetadataTooLarge
|
||||
ErrUnsupportedMetadata
|
||||
ErrMaximumExpires
|
||||
@@ -196,8 +198,9 @@ const (
|
||||
ErrInvalidTagDirective
|
||||
// Add new error codes here.
|
||||
|
||||
// SSE-S3 related API errors
|
||||
// SSE-S3/SSE-KMS related API errors
|
||||
ErrInvalidEncryptionMethod
|
||||
ErrInvalidEncryptionKeyID
|
||||
|
||||
// Server-Side-Encryption (with Customer provided key) related API errors.
|
||||
ErrInsecureSSECustomerRequest
|
||||
@@ -231,6 +234,7 @@ const (
|
||||
|
||||
// S3 extended errors.
|
||||
ErrContentSHA256Mismatch
|
||||
ErrContentChecksumMismatch
|
||||
|
||||
// Add new extended error codes here.
|
||||
|
||||
@@ -260,6 +264,7 @@ const (
|
||||
ErrAdminNoSuchUser
|
||||
ErrAdminNoSuchGroup
|
||||
ErrAdminGroupNotEmpty
|
||||
ErrAdminNoSuchJob
|
||||
ErrAdminNoSuchPolicy
|
||||
ErrAdminInvalidArgument
|
||||
ErrAdminInvalidAccessKey
|
||||
@@ -270,6 +275,10 @@ const (
|
||||
ErrAdminNoSuchConfigTarget
|
||||
ErrAdminConfigEnvOverridden
|
||||
ErrAdminConfigDuplicateKeys
|
||||
ErrAdminConfigInvalidIDPType
|
||||
ErrAdminConfigLDAPValidation
|
||||
ErrAdminConfigIDPCfgNameAlreadyExists
|
||||
ErrAdminConfigIDPCfgNameDoesNotExist
|
||||
ErrAdminCredentialsMismatch
|
||||
ErrInsecureClientRequest
|
||||
ErrObjectTampered
|
||||
@@ -284,6 +293,10 @@ const (
|
||||
ErrSiteReplicationIAMError
|
||||
ErrSiteReplicationConfigMissing
|
||||
|
||||
// Pool rebalance errors
|
||||
ErrAdminRebalanceAlreadyStarted
|
||||
ErrAdminRebalanceNotStarted
|
||||
|
||||
// Bucket Quota error codes
|
||||
ErrAdminBucketQuotaExceeded
|
||||
ErrAdminNoSuchQuotaConfiguration
|
||||
@@ -391,6 +404,8 @@ const (
|
||||
ErrAccountNotEligible
|
||||
ErrAdminServiceAccountNotFound
|
||||
ErrPostPolicyConditionInvalidFormat
|
||||
|
||||
ErrInvalidChecksum
|
||||
)
|
||||
|
||||
type errorCodeMap map[APIErrorCode]APIError
|
||||
@@ -680,6 +695,11 @@ var errorCodes = errorCodeMap{
|
||||
Description: "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrTooManyBuckets: {
|
||||
Code: "TooManyBuckets",
|
||||
Description: "You have attempted to create more buckets than allowed",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrBucketNotEmpty: {
|
||||
Code: "BucketNotEmpty",
|
||||
Description: "The bucket you tried to delete is not empty",
|
||||
@@ -887,7 +907,7 @@ var errorCodes = errorCodeMap{
|
||||
},
|
||||
ErrReplicationRemoteConnectionError: {
|
||||
Code: "XMinioAdminReplicationRemoteConnectionError",
|
||||
Description: "Remote service connection error - please check remote service credentials and target bucket",
|
||||
Description: "Remote service connection error",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrReplicationBandwidthLimitError: {
|
||||
@@ -1072,6 +1092,11 @@ var errorCodes = errorCodeMap{
|
||||
Description: "The encryption method specified is not supported",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidEncryptionKeyID: {
|
||||
Code: "InvalidRequest",
|
||||
Description: "The specified KMS KeyID contains unsupported characters",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInsecureSSECustomerRequest: {
|
||||
Code: "InvalidRequest",
|
||||
Description: "Requests specifying Server Side Encryption with Customer provided keys must be made over a secure connection.",
|
||||
@@ -1154,11 +1179,16 @@ var errorCodes = errorCodeMap{
|
||||
Description: "The provided 'x-amz-content-sha256' header does not match what was computed.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrContentChecksumMismatch: {
|
||||
Code: "XAmzContentChecksumMismatch",
|
||||
Description: "The provided 'x-amz-checksum' header does not match what was computed.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
|
||||
// MinIO extensions.
|
||||
ErrStorageFull: {
|
||||
Code: "XMinioStorageFull",
|
||||
Description: "Storage backend has reached its minimum free disk threshold. Please delete a few objects to proceed.",
|
||||
Description: "Storage backend has reached its minimum free drive threshold. Please delete a few objects to proceed.",
|
||||
HTTPStatusCode: http.StatusInsufficientStorage,
|
||||
},
|
||||
ErrRequestBodyParse: {
|
||||
@@ -1206,6 +1236,11 @@ var errorCodes = errorCodeMap{
|
||||
Description: "The specified group does not exist.",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrAdminNoSuchJob: {
|
||||
Code: "XMinioAdminNoSuchJob",
|
||||
Description: "The specified job does not exist.",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrAdminGroupNotEmpty: {
|
||||
Code: "XMinioAdminGroupNotEmpty",
|
||||
Description: "The specified group is not empty - cannot remove it.",
|
||||
@@ -1262,6 +1297,26 @@ var errorCodes = errorCodeMap{
|
||||
Description: "JSON configuration provided has objects with duplicate keys",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminConfigInvalidIDPType: {
|
||||
Code: "XMinioAdminConfigInvalidIDPType",
|
||||
Description: fmt.Sprintf("Invalid IDP configuration type - must be one of %v", madmin.ValidIDPConfigTypes),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminConfigLDAPValidation: {
|
||||
Code: "XMinioAdminConfigLDAPValidation",
|
||||
Description: "LDAP Configuration validation failed",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminConfigIDPCfgNameAlreadyExists: {
|
||||
Code: "XMinioAdminConfigIDPCfgNameAlreadyExists",
|
||||
Description: "An IDP configuration with the given name aleady exists",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminConfigIDPCfgNameDoesNotExist: {
|
||||
Code: "XMinioAdminConfigIDPCfgNameDoesNotExist",
|
||||
Description: "No such IDP configuration exists",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminConfigNotificationTargetsFailed: {
|
||||
Code: "XMinioAdminNotificationTargetsTestFailed",
|
||||
Description: "Configuration update failed due an unsuccessful attempt to connect to one or more notification servers",
|
||||
@@ -1358,6 +1413,16 @@ var errorCodes = errorCodeMap{
|
||||
Description: "Site not found in site replication configuration",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminRebalanceAlreadyStarted: {
|
||||
Code: "XMinioAdminRebalanceAlreadyStarted",
|
||||
Description: "Pool rebalance is already started",
|
||||
HTTPStatusCode: http.StatusConflict,
|
||||
},
|
||||
ErrAdminRebalanceNotStarted: {
|
||||
Code: "XMinioAdminRebalanceNotStarted",
|
||||
Description: "Pool rebalance is not started",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrMaximumExpires: {
|
||||
Code: "AuthorizationQueryParametersError",
|
||||
Description: "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
|
||||
@@ -1868,6 +1933,11 @@ var errorCodes = errorCodeMap{
|
||||
Description: "Invalid according to Policy: Policy Condition failed",
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
},
|
||||
ErrInvalidChecksum: {
|
||||
Code: "InvalidArgument",
|
||||
Description: "Invalid checksum provided.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
// Add your error structure here.
|
||||
}
|
||||
|
||||
@@ -1898,6 +1968,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrAdminNoSuchGroup
|
||||
case errGroupNotEmpty:
|
||||
apiErr = ErrAdminGroupNotEmpty
|
||||
case errNoSuchJob:
|
||||
apiErr = ErrAdminNoSuchJob
|
||||
case errNoSuchPolicy:
|
||||
apiErr = ErrAdminNoSuchPolicy
|
||||
case errSignatureMismatch:
|
||||
@@ -1921,6 +1993,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrInvalidEncryptionParameters
|
||||
case crypto.ErrInvalidEncryptionMethod:
|
||||
apiErr = ErrInvalidEncryptionMethod
|
||||
case crypto.ErrInvalidEncryptionKeyID:
|
||||
apiErr = ErrInvalidEncryptionKeyID
|
||||
case crypto.ErrInvalidCustomerAlgorithm:
|
||||
apiErr = ErrInvalidSSECustomerAlgorithm
|
||||
case crypto.ErrMissingCustomerKey:
|
||||
@@ -2038,6 +2112,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrSignatureDoesNotMatch
|
||||
case hash.SHA256Mismatch:
|
||||
apiErr = ErrContentSHA256Mismatch
|
||||
case hash.ChecksumMismatch:
|
||||
apiErr = ErrContentChecksumMismatch
|
||||
case ObjectTooLarge:
|
||||
apiErr = ErrEntityTooLarge
|
||||
case ObjectTooSmall:
|
||||
@@ -2066,7 +2142,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrRemoteDestinationNotFoundError
|
||||
case BucketRemoteTargetNotFound:
|
||||
apiErr = ErrRemoteTargetNotFoundError
|
||||
case BucketRemoteConnectionErr:
|
||||
case RemoteTargetConnectionErr:
|
||||
apiErr = ErrReplicationRemoteConnectionError
|
||||
case BucketRemoteAlreadyExists:
|
||||
apiErr = ErrBucketRemoteAlreadyExists
|
||||
@@ -2188,9 +2264,14 @@ func toAPIError(ctx context.Context, err error) APIError {
|
||||
if apiErr.Code == "InternalError" {
|
||||
// If we see an internal error try to interpret
|
||||
// any underlying errors if possible depending on
|
||||
// their internal error types. This code is only
|
||||
// useful with gateway implementations.
|
||||
// their internal error types.
|
||||
switch e := err.(type) {
|
||||
case batchReplicationJobError:
|
||||
apiErr = APIError{
|
||||
Code: e.Code,
|
||||
Description: e.Description,
|
||||
HTTPStatusCode: e.HTTPStatusCode,
|
||||
}
|
||||
case InvalidArgument:
|
||||
apiErr = APIError{
|
||||
Code: "InvalidArgument",
|
||||
@@ -2253,7 +2334,7 @@ func toAPIError(ctx context.Context, err error) APIError {
|
||||
Description: e.Message,
|
||||
HTTPStatusCode: e.StatusCode,
|
||||
}
|
||||
if globalIsGateway && strings.Contains(e.Message, "KMS is not configured") {
|
||||
if strings.Contains(e.Message, "KMS is not configured") {
|
||||
apiErr = APIError{
|
||||
Code: "NotImplemented",
|
||||
Description: e.Message,
|
||||
@@ -2277,7 +2358,7 @@ func toAPIError(ctx context.Context, err error) APIError {
|
||||
Description: e.Error(),
|
||||
HTTPStatusCode: e.Response().StatusCode,
|
||||
}
|
||||
// Add more Gateway SDKs here if any in future.
|
||||
// Add more other SDK related errors here if any in future.
|
||||
default:
|
||||
//nolint:gocritic
|
||||
if errors.Is(err, errMalformedEncoding) {
|
||||
|
||||
+16
-4
@@ -20,7 +20,6 @@ package cmd
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -30,6 +29,8 @@ import (
|
||||
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
xxml "github.com/minio/xxml"
|
||||
)
|
||||
|
||||
// Returns a hexadecimal representation of time at the
|
||||
@@ -64,9 +65,13 @@ func setCommonHeaders(w http.ResponseWriter) {
|
||||
// Encodes the response headers into XML format.
|
||||
func encodeResponse(response interface{}) []byte {
|
||||
var bytesBuffer bytes.Buffer
|
||||
bytesBuffer.WriteString(xml.Header)
|
||||
e := xml.NewEncoder(&bytesBuffer)
|
||||
e.Encode(response)
|
||||
bytesBuffer.WriteString(xxml.Header)
|
||||
buf, err := xxml.Marshal(response)
|
||||
if err != nil {
|
||||
logger.LogIf(GlobalContext, err)
|
||||
return nil
|
||||
}
|
||||
bytesBuffer.Write(buf)
|
||||
return bytesBuffer.Bytes()
|
||||
}
|
||||
|
||||
@@ -199,5 +204,12 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
|
||||
lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts())
|
||||
}
|
||||
|
||||
if v, ok := objInfo.UserDefined[ReservedMetadataPrefix+"compression"]; ok {
|
||||
if i := strings.LastIndexByte(v, '/'); i >= 0 {
|
||||
v = v[i+1:]
|
||||
}
|
||||
w.Header()[xhttp.MinIOCompressed] = []string{v}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
+101
-32
@@ -31,8 +31,10 @@ import (
|
||||
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
xxml "github.com/minio/xxml"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -162,6 +164,12 @@ type Part struct {
|
||||
LastModified string
|
||||
ETag string
|
||||
Size int64
|
||||
|
||||
// Checksum values
|
||||
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
|
||||
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
|
||||
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
|
||||
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
|
||||
}
|
||||
|
||||
// ListPartsResponse - format for list parts response.
|
||||
@@ -183,6 +191,7 @@ type ListPartsResponse struct {
|
||||
MaxParts int
|
||||
IsTruncated bool
|
||||
|
||||
ChecksumAlgorithm string
|
||||
// List of parts.
|
||||
Parts []Part `xml:"Part"`
|
||||
}
|
||||
@@ -252,45 +261,88 @@ type ObjectVersion struct {
|
||||
}
|
||||
|
||||
// MarshalXML - marshal ObjectVersion
|
||||
func (o ObjectVersion) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
func (o ObjectVersion) MarshalXML(e *xxml.Encoder, start xxml.StartElement) error {
|
||||
if o.isDeleteMarker {
|
||||
start.Name.Local = "DeleteMarker"
|
||||
} else {
|
||||
start.Name.Local = "Version"
|
||||
}
|
||||
|
||||
type objectVersionWrapper ObjectVersion
|
||||
return e.EncodeElement(objectVersionWrapper(o), start)
|
||||
}
|
||||
|
||||
// StringMap is a map[string]string
|
||||
type StringMap map[string]string
|
||||
// DeleteMarkerVersion container for delete marker metadata
|
||||
type DeleteMarkerVersion struct {
|
||||
Key string
|
||||
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
||||
|
||||
// Owner of the object.
|
||||
Owner Owner
|
||||
|
||||
IsLatest bool
|
||||
VersionID string `xml:"VersionId"`
|
||||
}
|
||||
|
||||
// Metadata metadata items implemented to ensure XML marshaling works.
|
||||
type Metadata struct {
|
||||
Items []struct {
|
||||
Key string
|
||||
Value string
|
||||
}
|
||||
}
|
||||
|
||||
// Set add items, duplicate items get replaced.
|
||||
func (s *Metadata) Set(k, v string) {
|
||||
for i, item := range s.Items {
|
||||
if item.Key == k {
|
||||
s.Items[i] = struct {
|
||||
Key string
|
||||
Value string
|
||||
}{
|
||||
Key: k,
|
||||
Value: v,
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
s.Items = append(s.Items, struct {
|
||||
Key string
|
||||
Value string
|
||||
}{
|
||||
Key: k,
|
||||
Value: v,
|
||||
})
|
||||
}
|
||||
|
||||
type xmlKeyEntry struct {
|
||||
XMLName xml.Name
|
||||
Value string `xml:",chardata"`
|
||||
}
|
||||
|
||||
// MarshalXML - StringMap marshals into XML.
|
||||
func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
tokens := []xml.Token{start}
|
||||
|
||||
for key, value := range s {
|
||||
t := xml.StartElement{}
|
||||
t.Name = xml.Name{
|
||||
Space: "",
|
||||
Local: key,
|
||||
}
|
||||
tokens = append(tokens, t, xml.CharData(value), xml.EndElement{Name: t.Name})
|
||||
func (s *Metadata) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
tokens = append(tokens, xml.EndElement{
|
||||
Name: start.Name,
|
||||
})
|
||||
if len(s.Items) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, t := range tokens {
|
||||
if err := e.EncodeToken(t); err != nil {
|
||||
if err := e.EncodeToken(start); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, item := range s.Items {
|
||||
if err := e.Encode(xmlKeyEntry{
|
||||
XMLName: xml.Name{Local: item.Key},
|
||||
Value: item.Value,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// flush to ensure tokens are written
|
||||
return e.Flush()
|
||||
return e.EncodeToken(start.End())
|
||||
}
|
||||
|
||||
// Object container for object metadata
|
||||
@@ -307,7 +359,7 @@ type Object struct {
|
||||
StorageClass string
|
||||
|
||||
// UserMetadata user-defined metadata
|
||||
UserMetadata StringMap `xml:"UserMetadata,omitempty"`
|
||||
UserMetadata *Metadata `xml:"UserMetadata,omitempty"`
|
||||
}
|
||||
|
||||
// CopyObjectResponse container returns ETag and LastModified of the successfully copied object
|
||||
@@ -350,6 +402,11 @@ type CompleteMultipartUploadResponse struct {
|
||||
Bucket string
|
||||
Key string
|
||||
ETag string
|
||||
|
||||
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
|
||||
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
|
||||
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
|
||||
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
|
||||
}
|
||||
|
||||
// DeleteError structure.
|
||||
@@ -438,6 +495,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
|
||||
// generates an ListBucketVersions response for the said bucket with other enumerated options.
|
||||
func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo) ListVersionsResponse {
|
||||
versions := make([]ObjectVersion, 0, len(resp.Objects))
|
||||
|
||||
owner := Owner{
|
||||
ID: globalMinioDefaultOwnerID,
|
||||
DisplayName: "minio",
|
||||
@@ -445,10 +503,10 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
|
||||
data := ListVersionsResponse{}
|
||||
|
||||
for _, object := range resp.Objects {
|
||||
content := ObjectVersion{}
|
||||
if object.Name == "" {
|
||||
continue
|
||||
}
|
||||
content := ObjectVersion{}
|
||||
content.Key = s3EncodeName(object.Name, encodingType)
|
||||
content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
|
||||
if object.ETag != "" {
|
||||
@@ -569,16 +627,16 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
|
||||
}
|
||||
content.Owner = owner
|
||||
if metadata {
|
||||
content.UserMetadata = make(StringMap)
|
||||
content.UserMetadata = &Metadata{}
|
||||
switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
|
||||
case crypto.S3:
|
||||
content.UserMetadata[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
|
||||
content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
|
||||
case crypto.S3KMS:
|
||||
content.UserMetadata[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
|
||||
content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
|
||||
case crypto.SSEC:
|
||||
content.UserMetadata[xhttp.AmzServerSideEncryptionCustomerAlgorithm] = xhttp.AmzEncryptionAES
|
||||
content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
|
||||
}
|
||||
for k, v := range CleanMinioInternalMetadataKeys(object.UserDefined) {
|
||||
for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
|
||||
if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
|
||||
// Do not need to send any internal metadata
|
||||
// values to client.
|
||||
@@ -588,7 +646,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
|
||||
if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
|
||||
continue
|
||||
}
|
||||
content.UserMetadata[k] = v
|
||||
content.UserMetadata.Set(k, v)
|
||||
}
|
||||
}
|
||||
contents = append(contents, content)
|
||||
@@ -642,14 +700,20 @@ func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) Initi
|
||||
}
|
||||
|
||||
// generates CompleteMultipartUploadResponse for given bucket, key, location and ETag.
|
||||
func generateCompleteMultpartUploadResponse(bucket, key, location, etag string) CompleteMultipartUploadResponse {
|
||||
return CompleteMultipartUploadResponse{
|
||||
func generateCompleteMultpartUploadResponse(bucket, key, location string, oi ObjectInfo) CompleteMultipartUploadResponse {
|
||||
cs := oi.decryptChecksums()
|
||||
c := CompleteMultipartUploadResponse{
|
||||
Location: location,
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
// AWS S3 quotes the ETag in XML, make sure we are compatible here.
|
||||
ETag: "\"" + etag + "\"",
|
||||
ETag: "\"" + oi.ETag + "\"",
|
||||
ChecksumSHA1: cs[hash.ChecksumSHA1.String()],
|
||||
ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
|
||||
ChecksumCRC32: cs[hash.ChecksumCRC32.String()],
|
||||
ChecksumCRC32C: cs[hash.ChecksumCRC32C.String()],
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// generates ListPartsResponse from ListPartsInfo.
|
||||
@@ -674,6 +738,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
|
||||
listPartsResponse.PartNumberMarker = partsInfo.PartNumberMarker
|
||||
listPartsResponse.IsTruncated = partsInfo.IsTruncated
|
||||
listPartsResponse.NextPartNumberMarker = partsInfo.NextPartNumberMarker
|
||||
listPartsResponse.ChecksumAlgorithm = partsInfo.ChecksumAlgorithm
|
||||
|
||||
listPartsResponse.Parts = make([]Part, len(partsInfo.Parts))
|
||||
for index, part := range partsInfo.Parts {
|
||||
@@ -682,6 +747,10 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
|
||||
newPart.ETag = "\"" + part.ETag + "\""
|
||||
newPart.Size = part.Size
|
||||
newPart.LastModified = part.LastModified.UTC().Format(iso8601TimeFormat)
|
||||
newPart.ChecksumCRC32 = part.ChecksumCRC32
|
||||
newPart.ChecksumCRC32C = part.ChecksumCRC32C
|
||||
newPart.ChecksumSHA1 = part.ChecksumSHA1
|
||||
newPart.ChecksumSHA256 = part.ChecksumSHA256
|
||||
listPartsResponse.Parts[index] = newPart
|
||||
}
|
||||
return listPartsResponse
|
||||
|
||||
+2
-2
@@ -35,8 +35,8 @@ func shouldEscape(c byte) bool {
|
||||
|
||||
// s3URLEncode is based on Golang's url.QueryEscape() code,
|
||||
// while considering some S3 exceptions:
|
||||
// - Avoid encoding '/' and '*'
|
||||
// - Force encoding of '~'
|
||||
// - Avoid encoding '/' and '*'
|
||||
// - Force encoding of '~'
|
||||
func s3URLEncode(s string) string {
|
||||
spaceCount, hexCount := 0, 0
|
||||
for i := 0; i < len(s); i++ {
|
||||
|
||||
+201
-190
File diff suppressed because one or more lines are too long
+96
-52
@@ -25,7 +25,6 @@ import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
@@ -85,6 +84,8 @@ func isRequestSignStreamingV4(r *http.Request) bool {
|
||||
}
|
||||
|
||||
// Authorization type.
|
||||
//
|
||||
//go:generate stringer -type=authType -trimprefix=authType $GOFILE
|
||||
type authType int
|
||||
|
||||
// List of all supported auth types.
|
||||
@@ -288,28 +289,33 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
|
||||
}
|
||||
|
||||
// Check request auth type verifies the incoming http request
|
||||
// - validates the request signature
|
||||
// - validates the policy action if anonymous tests bucket policies if any,
|
||||
// for authenticated requests validates IAM policies.
|
||||
// - validates the request signature
|
||||
// - validates the policy action if anonymous tests bucket policies if any,
|
||||
// for authenticated requests validates IAM policies.
|
||||
//
|
||||
// returns APIErrorCode if any to be replied to the client.
|
||||
func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (s3Err APIErrorCode) {
|
||||
_, _, s3Err = checkRequestAuthTypeCredential(ctx, r, action, bucketName, objectName)
|
||||
logger.GetReqInfo(ctx).BucketName = bucketName
|
||||
logger.GetReqInfo(ctx).ObjectName = objectName
|
||||
|
||||
_, _, s3Err = checkRequestAuthTypeCredential(ctx, r, action)
|
||||
return s3Err
|
||||
}
|
||||
|
||||
// Check request auth type verifies the incoming http request
|
||||
// - validates the request signature
|
||||
// - validates the policy action if anonymous tests bucket policies if any,
|
||||
// for authenticated requests validates IAM policies.
|
||||
// returns APIErrorCode if any to be replied to the client.
|
||||
// Additionally returns the accessKey used in the request, and if this request is by an admin.
|
||||
func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (cred auth.Credentials, owner bool, s3Err APIErrorCode) {
|
||||
func authenticateRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
|
||||
if logger.GetReqInfo(ctx) == nil {
|
||||
logger.LogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.Minio)
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
var cred auth.Credentials
|
||||
var owner bool
|
||||
switch getRequestAuthType(r) {
|
||||
case authTypeUnknown, authTypeStreamingSigned:
|
||||
return cred, owner, ErrSignatureVersionNotSupported
|
||||
return ErrSignatureVersionNotSupported
|
||||
case authTypePresignedV2, authTypeSignedV2:
|
||||
if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone {
|
||||
return cred, owner, s3Err
|
||||
return s3Err
|
||||
}
|
||||
cred, owner, s3Err = getReqAccessKeyV2(r)
|
||||
case authTypeSigned, authTypePresigned:
|
||||
@@ -319,52 +325,67 @@ func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action
|
||||
region = ""
|
||||
}
|
||||
if s3Err = isReqAuthenticated(ctx, r, region, serviceS3); s3Err != ErrNone {
|
||||
return cred, owner, s3Err
|
||||
return s3Err
|
||||
}
|
||||
cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
|
||||
}
|
||||
if s3Err != ErrNone {
|
||||
return cred, owner, s3Err
|
||||
return s3Err
|
||||
}
|
||||
|
||||
// LocationConstraint is valid only for CreateBucketAction.
|
||||
var locationConstraint string
|
||||
logger.GetReqInfo(ctx).Cred = cred
|
||||
logger.GetReqInfo(ctx).Owner = owner
|
||||
|
||||
// region is valid only for CreateBucketAction.
|
||||
var region string
|
||||
if action == policy.CreateBucketAction {
|
||||
// To extract region from XML in request body, get copy of request body.
|
||||
payload, err := ioutil.ReadAll(io.LimitReader(r.Body, maxLocationConstraintSize))
|
||||
payload, err := io.ReadAll(io.LimitReader(r.Body, maxLocationConstraintSize))
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err, logger.Application)
|
||||
return cred, owner, ErrMalformedXML
|
||||
return ErrMalformedXML
|
||||
}
|
||||
|
||||
// Populate payload to extract location constraint.
|
||||
r.Body = ioutil.NopCloser(bytes.NewReader(payload))
|
||||
|
||||
var s3Error APIErrorCode
|
||||
locationConstraint, s3Error = parseLocationConstraint(r)
|
||||
if s3Error != ErrNone {
|
||||
return cred, owner, s3Error
|
||||
r.Body = io.NopCloser(bytes.NewReader(payload))
|
||||
region, s3Err = parseLocationConstraint(r)
|
||||
if s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
|
||||
// Populate payload again to handle it in HTTP handler.
|
||||
r.Body = ioutil.NopCloser(bytes.NewReader(payload))
|
||||
}
|
||||
if cred.AccessKey != "" {
|
||||
logger.GetReqInfo(ctx).AccessKey = cred.AccessKey
|
||||
r.Body = io.NopCloser(bytes.NewReader(payload))
|
||||
}
|
||||
|
||||
logger.GetReqInfo(ctx).Region = region
|
||||
|
||||
return s3Err
|
||||
}
|
||||
|
||||
func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
if reqInfo == nil {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
cred := reqInfo.Cred
|
||||
owner := reqInfo.Owner
|
||||
region := reqInfo.Region
|
||||
bucket := reqInfo.BucketName
|
||||
object := reqInfo.ObjectName
|
||||
|
||||
if action != policy.ListAllMyBucketsAction && cred.AccessKey == "" {
|
||||
// Anonymous checks are not meant for ListBuckets action
|
||||
// Anonymous checks are not meant for ListAllBuckets action
|
||||
if globalPolicySys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Action: action,
|
||||
BucketName: bucketName,
|
||||
ConditionValues: getConditionValues(r, locationConstraint, "", nil),
|
||||
BucketName: bucket,
|
||||
ConditionValues: getConditionValues(r, region, "", nil),
|
||||
IsOwner: false,
|
||||
ObjectName: objectName,
|
||||
ObjectName: object,
|
||||
}) {
|
||||
// Request is allowed return the appropriate access key.
|
||||
return cred, owner, ErrNone
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
if action == policy.ListBucketVersionsAction {
|
||||
@@ -373,31 +394,31 @@ func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action
|
||||
if globalPolicySys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Action: policy.ListBucketAction,
|
||||
BucketName: bucketName,
|
||||
ConditionValues: getConditionValues(r, locationConstraint, "", nil),
|
||||
BucketName: bucket,
|
||||
ConditionValues: getConditionValues(r, region, "", nil),
|
||||
IsOwner: false,
|
||||
ObjectName: objectName,
|
||||
ObjectName: object,
|
||||
}) {
|
||||
// Request is allowed return the appropriate access key.
|
||||
return cred, owner, ErrNone
|
||||
return ErrNone
|
||||
}
|
||||
}
|
||||
|
||||
return cred, owner, ErrAccessDenied
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
if globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.Action(action),
|
||||
BucketName: bucketName,
|
||||
BucketName: bucket,
|
||||
ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
|
||||
ObjectName: objectName,
|
||||
ObjectName: object,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
// Request is allowed return the appropriate access key.
|
||||
return cred, owner, ErrNone
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
if action == policy.ListBucketVersionsAction {
|
||||
@@ -407,18 +428,41 @@ func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.ListBucketAction,
|
||||
BucketName: bucketName,
|
||||
BucketName: bucket,
|
||||
ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
|
||||
ObjectName: objectName,
|
||||
ObjectName: object,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
// Request is allowed return the appropriate access key.
|
||||
return cred, owner, ErrNone
|
||||
return ErrNone
|
||||
}
|
||||
}
|
||||
|
||||
return cred, owner, ErrAccessDenied
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
// Check request auth type verifies the incoming http request
|
||||
// - validates the request signature
|
||||
// - validates the policy action if anonymous tests bucket policies if any,
|
||||
// for authenticated requests validates IAM policies.
|
||||
//
|
||||
// returns APIErrorCode if any to be replied to the client.
|
||||
// Additionally returns the accessKey used in the request, and if this request is by an admin.
|
||||
func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action policy.Action) (cred auth.Credentials, owner bool, s3Err APIErrorCode) {
|
||||
s3Err = authenticateRequest(ctx, r, action)
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
if reqInfo == nil {
|
||||
return cred, owner, ErrAccessDenied
|
||||
}
|
||||
|
||||
cred = reqInfo.Cred
|
||||
owner = reqInfo.Owner
|
||||
if s3Err != ErrNone {
|
||||
return cred, owner, s3Err
|
||||
}
|
||||
|
||||
return cred, owner, authorizeRequest(ctx, r, action)
|
||||
}
|
||||
|
||||
// Verify if request has valid AWS Signature Version '2'.
|
||||
@@ -582,7 +626,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
|
||||
|
||||
conditions := getConditionValues(r, "", cred.AccessKey, cred.Claims)
|
||||
conditions["object-lock-mode"] = []string{string(retMode)}
|
||||
conditions["object-lock-retain-until-date"] = []string{retDate.Format(time.RFC3339)}
|
||||
conditions["object-lock-retain-until-date"] = []string{retDate.UTC().Format(time.RFC3339)}
|
||||
if retDays > 0 {
|
||||
conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(retDays)}
|
||||
}
|
||||
@@ -622,22 +666,22 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
|
||||
func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action iampolicy.Action) (s3Err APIErrorCode) {
|
||||
var cred auth.Credentials
|
||||
var owner bool
|
||||
region := globalSite.Region
|
||||
switch atype {
|
||||
case authTypeUnknown:
|
||||
return ErrSignatureVersionNotSupported
|
||||
case authTypeSignedV2, authTypePresignedV2:
|
||||
cred, owner, s3Err = getReqAccessKeyV2(r)
|
||||
case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
|
||||
region := globalSite.Region
|
||||
cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
|
||||
}
|
||||
if s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
|
||||
if cred.AccessKey != "" {
|
||||
logger.GetReqInfo(ctx).AccessKey = cred.AccessKey
|
||||
}
|
||||
logger.GetReqInfo(ctx).Cred = cred
|
||||
logger.GetReqInfo(ctx).Owner = owner
|
||||
logger.GetReqInfo(ctx).Region = region
|
||||
|
||||
// Do not check for PutObjectRetentionAction permission,
|
||||
// if mode and retain until date are not set.
|
||||
|
||||
+13
-13
@@ -21,7 +21,6 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -44,7 +43,7 @@ func TestGetRequestAuthType(t *testing.T) {
|
||||
req *http.Request
|
||||
authT authType
|
||||
}
|
||||
nopCloser := ioutil.NopCloser(io.LimitReader(&nullReader{}, 1024))
|
||||
nopCloser := io.NopCloser(io.LimitReader(&nullReader{}, 1024))
|
||||
testCases := []testCase{
|
||||
// Test case - 1
|
||||
// Check for generic signature v4 header.
|
||||
@@ -360,7 +359,10 @@ func mustNewSignedBadMD5Request(method string, urlStr string, contentLength int6
|
||||
|
||||
// Tests is requested authenticated function, tests replies for s3 errors.
|
||||
func TestIsReqAuthenticated(t *testing.T) {
|
||||
objLayer, fsDir, err := prepareFS()
|
||||
ctx, cancel := context.WithCancel(GlobalContext)
|
||||
defer cancel()
|
||||
|
||||
objLayer, fsDir, err := prepareFS(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -369,10 +371,7 @@ func TestIsReqAuthenticated(t *testing.T) {
|
||||
t.Fatalf("unable initialize config file, %s", err)
|
||||
}
|
||||
|
||||
initAllSubsystems()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
initAllSubsystems(ctx)
|
||||
|
||||
initConfigSubsystem(ctx, objLayer)
|
||||
|
||||
@@ -406,7 +405,7 @@ func TestIsReqAuthenticated(t *testing.T) {
|
||||
for i, testCase := range testCases {
|
||||
s3Error := isReqAuthenticated(ctx, testCase.req, globalSite.Region, serviceS3)
|
||||
if s3Error != testCase.s3Error {
|
||||
if _, err := ioutil.ReadAll(testCase.req.Body); toAPIErrorCode(ctx, err) != testCase.s3Error {
|
||||
if _, err := io.ReadAll(testCase.req.Body); toAPIErrorCode(ctx, err) != testCase.s3Error {
|
||||
t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %s)", i, testCase.s3Error, s3Error, toAPIError(ctx, err).Code)
|
||||
}
|
||||
}
|
||||
@@ -414,7 +413,10 @@ func TestIsReqAuthenticated(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCheckAdminRequestAuthType(t *testing.T) {
|
||||
objLayer, fsDir, err := prepareFS()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
objLayer, fsDir, err := prepareFS(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -440,7 +442,6 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
|
||||
{Request: mustNewPresignedV2Request(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
|
||||
{Request: mustNewPresignedRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
|
||||
}
|
||||
ctx := context.Background()
|
||||
for i, testCase := range testCases {
|
||||
if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, iampolicy.AllAdminActions, globalSite.Region); s3Error != testCase.ErrCode {
|
||||
t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
|
||||
@@ -452,7 +453,7 @@ func TestValidateAdminSignature(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
objLayer, fsDir, err := prepareFS()
|
||||
objLayer, fsDir, err := prepareFS(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -462,8 +463,7 @@ func TestValidateAdminSignature(t *testing.T) {
|
||||
t.Fatalf("unable initialize config file, %s", err)
|
||||
}
|
||||
|
||||
initAllSubsystems()
|
||||
|
||||
initAllSubsystems(ctx)
|
||||
initConfigSubsystem(ctx, objLayer)
|
||||
|
||||
globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
// Code generated by "stringer -type=authType -trimprefix=authType auth-handler.go"; DO NOT EDIT.
|
||||
|
||||
package cmd
|
||||
|
||||
import "strconv"
|
||||
|
||||
func _() {
|
||||
// An "invalid array index" compiler error signifies that the constant values have changed.
|
||||
// Re-run the stringer command to generate them again.
|
||||
var x [1]struct{}
|
||||
_ = x[authTypeUnknown-0]
|
||||
_ = x[authTypeAnonymous-1]
|
||||
_ = x[authTypePresigned-2]
|
||||
_ = x[authTypePresignedV2-3]
|
||||
_ = x[authTypePostPolicy-4]
|
||||
_ = x[authTypeStreamingSigned-5]
|
||||
_ = x[authTypeSigned-6]
|
||||
_ = x[authTypeSignedV2-7]
|
||||
_ = x[authTypeJWT-8]
|
||||
_ = x[authTypeSTS-9]
|
||||
}
|
||||
|
||||
const _authType_name = "UnknownAnonymousPresignedPresignedV2PostPolicyStreamingSignedSignedSignedV2JWTSTS"
|
||||
|
||||
var _authType_index = [...]uint8{0, 7, 16, 25, 36, 46, 61, 67, 75, 78, 81}
|
||||
|
||||
func (i authType) String() string {
|
||||
if i < 0 || i >= authType(len(_authType_index)-1) {
|
||||
return "authType(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _authType_name[_authType_index[i]:_authType_index[i+1]]
|
||||
}
|
||||
@@ -22,13 +22,13 @@ import (
|
||||
"runtime"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/minio/internal/pubsub"
|
||||
)
|
||||
|
||||
// healTask represents what to heal along with options
|
||||
// path: '/' => Heal disk formats along with metadata
|
||||
// path: 'bucket/' or '/bucket/' => Heal bucket
|
||||
// path: 'bucket/object' => Heal object
|
||||
//
|
||||
// path: '/' => Heal disk formats along with metadata
|
||||
// path: 'bucket/' or '/bucket/' => Heal bucket
|
||||
// path: 'bucket/object' => Heal object
|
||||
type healTask struct {
|
||||
bucket string
|
||||
object string
|
||||
@@ -53,7 +53,7 @@ type healRoutine struct {
|
||||
func activeListeners() int {
|
||||
// Bucket notification and http trace are not costly, it is okay to ignore them
|
||||
// while counting the number of concurrent connections
|
||||
return int(globalHTTPListen.NumSubscribers(pubsub.MaskAll)) + int(globalTrace.NumSubscribers(pubsub.MaskAll))
|
||||
return int(globalHTTPListen.Subscribers()) + int(globalTrace.Subscribers())
|
||||
}
|
||||
|
||||
func waitForLowHTTPReq() {
|
||||
|
||||
@@ -87,7 +87,7 @@ type healingTracker struct {
|
||||
// The disk ID will be validated against the loaded one.
|
||||
func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker, error) {
|
||||
if disk == nil {
|
||||
return nil, errors.New("loadHealingTracker: nil disk given")
|
||||
return nil, errors.New("loadHealingTracker: nil drive given")
|
||||
}
|
||||
diskID, err := disk.GetDiskID()
|
||||
if err != nil {
|
||||
@@ -104,7 +104,7 @@ func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker,
|
||||
return nil, err
|
||||
}
|
||||
if h.ID != diskID && h.ID != "" {
|
||||
return nil, fmt.Errorf("loadHealingTracker: disk id mismatch expected %s, got %s", h.ID, diskID)
|
||||
return nil, fmt.Errorf("loadHealingTracker: drive id mismatch expected %s, got %s", h.ID, diskID)
|
||||
}
|
||||
h.disk = disk
|
||||
h.ID = diskID
|
||||
@@ -129,7 +129,7 @@ func newHealingTracker(disk StorageAPI) *healingTracker {
|
||||
// If the tracker has been deleted an error is returned.
|
||||
func (h *healingTracker) update(ctx context.Context) error {
|
||||
if h.disk.Healing() == nil {
|
||||
return fmt.Errorf("healingTracker: disk %q is not marked as healing", h.ID)
|
||||
return fmt.Errorf("healingTracker: drive %q is not marked as healing", h.ID)
|
||||
}
|
||||
if h.ID == "" || h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 {
|
||||
h.ID, _ = h.disk.GetDiskID()
|
||||
@@ -310,7 +310,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
|
||||
}
|
||||
|
||||
// Prevent parallel erasure set healing
|
||||
locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-disk-healing/%s/%d/%d", endpoint, poolIdx, setIdx))
|
||||
locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-drive-healing/%d/%d", poolIdx, setIdx))
|
||||
lkctx, err := locker.GetLock(ctx, newDiskHealingTimeout)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -318,8 +318,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
|
||||
ctx = lkctx.Context()
|
||||
defer locker.Unlock(lkctx.Cancel)
|
||||
|
||||
buckets, _ := z.ListBuckets(ctx)
|
||||
|
||||
buckets, _ := z.ListBuckets(ctx, BucketOptions{})
|
||||
// Buckets data are dispersed in multiple zones/sets, make
|
||||
// sure to heal all bucket metadata configuration.
|
||||
buckets = append(buckets, BucketInfo{
|
||||
@@ -338,14 +337,14 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
|
||||
})
|
||||
|
||||
if serverDebugLog {
|
||||
logger.Info("Healing disk '%v' on %s pool", disk, humanize.Ordinal(poolIdx+1))
|
||||
logger.Info("Healing drive '%v' on %s pool", disk, humanize.Ordinal(poolIdx+1))
|
||||
}
|
||||
|
||||
// Load healing tracker in this disk
|
||||
tracker, err := loadHealingTracker(ctx, disk)
|
||||
if err != nil {
|
||||
// So someone changed the drives underneath, healing tracker missing.
|
||||
logger.LogIf(ctx, fmt.Errorf("Healing tracker missing on '%s', disk was swapped again on %s pool: %w",
|
||||
logger.LogIf(ctx, fmt.Errorf("Healing tracker missing on '%s', drive was swapped again on %s pool: %w",
|
||||
disk, humanize.Ordinal(poolIdx+1), err))
|
||||
tracker = newHealingTracker(disk)
|
||||
}
|
||||
@@ -370,9 +369,9 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
|
||||
}
|
||||
|
||||
if tracker.ItemsFailed > 0 {
|
||||
logger.Info("Healing disk '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
|
||||
logger.Info("Healing drive '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
|
||||
} else {
|
||||
logger.Info("Healing disk '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
|
||||
logger.Info("Healing drive '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
|
||||
}
|
||||
|
||||
if serverDebugLog {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,23 @@
|
||||
// Code generated by "stringer -type=batchReplicationMetric -trimprefix=batchReplicationMetric batch-handlers.go"; DO NOT EDIT.
|
||||
|
||||
package cmd
|
||||
|
||||
import "strconv"
|
||||
|
||||
func _() {
|
||||
// An "invalid array index" compiler error signifies that the constant values have changed.
|
||||
// Re-run the stringer command to generate them again.
|
||||
var x [1]struct{}
|
||||
_ = x[batchReplicationMetricObject-0]
|
||||
}
|
||||
|
||||
const _batchReplicationMetric_name = "Object"
|
||||
|
||||
var _batchReplicationMetric_index = [...]uint8{0, 6}
|
||||
|
||||
func (i batchReplicationMetric) String() string {
|
||||
if i >= batchReplicationMetric(len(_batchReplicationMetric_index)-1) {
|
||||
return "batchReplicationMetric(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _batchReplicationMetric_name[_batchReplicationMetric_index[i]:_batchReplicationMetric_index[i+1]]
|
||||
}
|
||||
@@ -25,7 +25,7 @@ import (
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/dustin/go-humanize"
|
||||
)
|
||||
|
||||
// Benchmark utility functions for ObjectLayer.PutObject().
|
||||
@@ -35,7 +35,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// obtains random bucket name.
|
||||
bucket := getRandomBucketName()
|
||||
// create bucket.
|
||||
err = obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
|
||||
err = obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -76,7 +76,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
|
||||
object := getRandomObjectName()
|
||||
|
||||
// create bucket.
|
||||
err = obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
|
||||
err = obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -85,12 +85,12 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
|
||||
|
||||
// PutObjectPart returns etag of the object inserted.
|
||||
// etag variable is assigned with that value.
|
||||
var etag, uploadID string
|
||||
var etag string
|
||||
// get text data generated for number of bytes equal to object size.
|
||||
textData := generateBytesData(objSize)
|
||||
// generate md5sum for the generated data.
|
||||
// md5sum of the data to written is required as input for NewMultipartUpload.
|
||||
uploadID, err = obj.NewMultipartUpload(context.Background(), bucket, object, ObjectOptions{})
|
||||
res, err := obj.NewMultipartUpload(context.Background(), bucket, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -113,7 +113,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
|
||||
}
|
||||
md5hex := getMD5Hash(textPartData)
|
||||
var partInfo PartInfo
|
||||
partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, j,
|
||||
partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, res.UploadID, j,
|
||||
mustGetPutObjReader(b, bytes.NewReader(textPartData), int64(len(textPartData)), md5hex, sha256hex), ObjectOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
@@ -196,7 +196,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// obtains random bucket name.
|
||||
bucket := getRandomBucketName()
|
||||
// create bucket.
|
||||
err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
|
||||
err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
+18
-5
@@ -24,6 +24,7 @@ import (
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
@@ -146,16 +147,28 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
|
||||
// Can never happen unless there are programmer bugs
|
||||
return 0, errUnexpected
|
||||
}
|
||||
ignoredErrs := []error{
|
||||
errDiskNotFound,
|
||||
}
|
||||
if strings.HasPrefix(b.volume, minioMetaBucket) {
|
||||
ignoredErrs = append(ignoredErrs,
|
||||
errFileNotFound,
|
||||
errVolumeNotFound,
|
||||
errFileVersionNotFound,
|
||||
)
|
||||
}
|
||||
if b.rc == nil {
|
||||
// For the first ReadAt() call we need to open the stream for reading.
|
||||
b.currOffset = offset
|
||||
streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
|
||||
if len(b.data) == 0 && b.tillOffset != streamOffset {
|
||||
b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
|
||||
if err != nil && err != errDiskNotFound {
|
||||
logger.LogIf(GlobalContext,
|
||||
fmt.Errorf("Reading erasure shards at (%s: %s/%s) returned '%w', will attempt to reconstruct if we have quorum",
|
||||
b.disk, b.volume, b.filePath, err))
|
||||
if err != nil {
|
||||
if !IsErr(err, ignoredErrs...) {
|
||||
logger.LogIf(GlobalContext,
|
||||
fmt.Errorf("Reading erasure shards at (%s: %s/%s) returned '%w', will attempt to reconstruct if we have quorum",
|
||||
b.disk, b.volume, b.filePath, err))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
b.rc = io.NewSectionReader(bytes.NewReader(b.data), streamOffset, b.tillOffset-streamOffset)
|
||||
@@ -180,7 +193,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
|
||||
b.h.Write(buf)
|
||||
|
||||
if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
|
||||
logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s -> %s/%s - content hash does not match - expected %s, got %s",
|
||||
logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s -> %s/%s - content hash does not match - expected %s, got %s",
|
||||
b.disk, b.volume, b.filePath, hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))))
|
||||
return 0, errFileCorrupt
|
||||
}
|
||||
|
||||
+4
-4
@@ -38,12 +38,12 @@ type wholeBitrotWriter struct {
|
||||
func (b *wholeBitrotWriter) Write(p []byte) (int, error) {
|
||||
err := b.disk.AppendFile(context.TODO(), b.volume, b.filePath, p)
|
||||
if err != nil {
|
||||
logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
|
||||
logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s returned %w", b.disk, err))
|
||||
return 0, err
|
||||
}
|
||||
_, err = b.Hash.Write(p)
|
||||
if err != nil {
|
||||
logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
|
||||
logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s returned %w", b.disk, err))
|
||||
return 0, err
|
||||
}
|
||||
return len(p), nil
|
||||
@@ -72,12 +72,12 @@ func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error)
|
||||
if b.buf == nil {
|
||||
b.buf = make([]byte, b.tillOffset-offset)
|
||||
if _, err := b.disk.ReadFile(context.TODO(), b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
|
||||
logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, err))
|
||||
logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, err))
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if len(b.buf) < len(buf) {
|
||||
logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, errLessData))
|
||||
logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, errLessData))
|
||||
return 0, errLessData
|
||||
}
|
||||
n = copy(buf, b.buf)
|
||||
|
||||
+7
-8
@@ -20,17 +20,11 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
|
||||
tmpDir, err := ioutil.TempDir("", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
volume := "testvol"
|
||||
filePath := "testfile"
|
||||
@@ -60,7 +54,9 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writer.(io.Closer).Close()
|
||||
if bw, ok := writer.(io.Closer); ok {
|
||||
bw.Close()
|
||||
}
|
||||
|
||||
reader := newBitrotReader(disk, nil, volume, filePath, 35, bitrotAlgo, bitrotWriterSum(writer), 10)
|
||||
b := make([]byte, 10)
|
||||
@@ -76,6 +72,9 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
|
||||
if _, err = reader.ReadAt(b[:5], 30); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if br, ok := reader.(io.Closer); ok {
|
||||
br.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllBitrotAlgorithms(t *testing.T) {
|
||||
|
||||
@@ -65,7 +65,7 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -155,7 +155,7 @@ func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r
|
||||
|
||||
// Check if bucket exists
|
||||
var err error
|
||||
if _, err = objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err = objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -198,13 +198,13 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
|
||||
|
||||
// Check if bucket exists
|
||||
var err error
|
||||
if _, err = objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err = objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Delete bucket encryption config from object layer
|
||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, nil)
|
||||
updatedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketSSEConfig)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
|
||||
@@ -34,15 +34,6 @@ func NewBucketSSEConfigSys() *BucketSSEConfigSys {
|
||||
|
||||
// Get - gets bucket encryption config for the given bucket.
|
||||
func (sys *BucketSSEConfigSys) Get(bucket string) (*sse.BucketSSEConfig, error) {
|
||||
if globalIsGateway {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return nil, errServerNotInitialized
|
||||
}
|
||||
|
||||
return nil, BucketSSEConfigNotFound{Bucket: bucket}
|
||||
}
|
||||
|
||||
sseCfg, _, err := globalBucketMetadataSys.GetSSEConfig(bucket)
|
||||
return sseCfg, err
|
||||
}
|
||||
|
||||
+55
-26
@@ -67,11 +67,16 @@ const (
|
||||
// - Check if a bucket has an entry in etcd backend
|
||||
// -- If no, make an entry
|
||||
// -- If yes, check if the entry matches local IP check if we
|
||||
// need to update the entry then proceed to update
|
||||
//
|
||||
// need to update the entry then proceed to update
|
||||
//
|
||||
// -- If yes, check if the IP of entry matches local IP.
|
||||
// This means entry is for this instance.
|
||||
//
|
||||
// This means entry is for this instance.
|
||||
//
|
||||
// -- If IP of the entry doesn't match, this means entry is
|
||||
// for another instance. Log an error to console.
|
||||
//
|
||||
// for another instance. Log an error to console.
|
||||
func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
|
||||
if len(buckets) == 0 {
|
||||
return
|
||||
@@ -201,7 +206,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
|
||||
|
||||
getBucketInfo := objectAPI.GetBucketInfo
|
||||
|
||||
if _, err := getBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := getBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -227,7 +232,6 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
|
||||
// using the Initiate Multipart Upload request, but has not yet been
|
||||
// completed or aborted. This operation returns at most 1,000 multipart
|
||||
// uploads in the response.
|
||||
//
|
||||
func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListMultipartUploads")
|
||||
|
||||
@@ -296,7 +300,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
|
||||
listBuckets := objectAPI.ListBuckets
|
||||
|
||||
cred, owner, s3Error := checkRequestAuthTypeCredential(ctx, r, policy.ListAllMyBucketsAction, "", "")
|
||||
cred, owner, s3Error := checkRequestAuthTypeCredential(ctx, r, policy.ListAllMyBucketsAction)
|
||||
if s3Error != ErrNone && s3Error != ErrAccessDenied {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
@@ -332,7 +336,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
} else {
|
||||
// Invoke the list buckets.
|
||||
var err error
|
||||
bucketsInfo, err = listBuckets(ctx)
|
||||
bucketsInfo, err = listBuckets(ctx, BucketOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -447,7 +451,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "")
|
||||
|
||||
// Before proceeding validate if bucket exists.
|
||||
_, err := objectAPI.GetBucketInfo(ctx, bucket)
|
||||
_, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -727,11 +731,30 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.CreateBucketAction, bucket, ""); s3Error != ErrNone {
|
||||
cred, owner, s3Error := checkRequestAuthTypeCredential(ctx, r, policy.CreateBucketAction)
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if objectLockEnabled {
|
||||
// Creating a bucket with locking requires the user having more permissions
|
||||
for _, action := range []iampolicy.Action{iampolicy.PutBucketObjectLockConfigurationAction, iampolicy.PutBucketVersioningAction} {
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: action,
|
||||
ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
|
||||
BucketName: bucket,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Parse incoming location constraint.
|
||||
location, s3Error := parseLocationConstraint(r)
|
||||
if s3Error != ErrNone {
|
||||
@@ -746,7 +769,15 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
return
|
||||
}
|
||||
|
||||
opts := BucketOptions{
|
||||
// check if client is attempting to create more buckets than allowed maximum.
|
||||
if currBuckets := globalBucketMetadataSys.Count(); currBuckets+1 > maxBuckets {
|
||||
apiErr := errorCodes.ToAPIErr(ErrTooManyBuckets)
|
||||
apiErr.Description = fmt.Sprintf("You have attempted to create %d buckets than allowed %d", currBuckets+1, maxBuckets)
|
||||
writeErrorResponse(ctx, w, apiErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
opts := MakeBucketOptions{
|
||||
Location: location,
|
||||
LockEnabled: objectLockEnabled,
|
||||
ForceCreate: forceCreate,
|
||||
@@ -765,7 +796,11 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
|
||||
if err = globalDNSConfig.Put(bucket); err != nil {
|
||||
objectAPI.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{Force: false, NoRecreate: true})
|
||||
objectAPI.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{
|
||||
Force: false,
|
||||
NoRecreate: true,
|
||||
SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
|
||||
})
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -823,10 +858,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
|
||||
|
||||
// Call site replication hook
|
||||
if err := globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts)
|
||||
|
||||
// Make sure to add Location information here only for bucket
|
||||
if cp := pathClean(r.URL.Path); cp != "" {
|
||||
@@ -1030,10 +1062,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
sseConfig, _ := globalBucketSSEConfigSys.Get(bucket)
|
||||
sseConfig.Apply(r.Header, sse.ApplyOptions{
|
||||
AutoEncrypt: globalAutoEncryption,
|
||||
Passthrough: globalIsGateway && globalGatewayName == S3BackendGateway,
|
||||
})
|
||||
|
||||
// get gateway encryption options
|
||||
var opts ObjectOptions
|
||||
opts, err = putOpts(ctx, r, bucket, object, metadata)
|
||||
if err != nil {
|
||||
@@ -1165,7 +1195,7 @@ func (api objectAPIHandlers) GetBucketPolicyStatusHandler(w http.ResponseWriter,
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -1228,7 +1258,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
|
||||
|
||||
getBucketInfo := objectAPI.GetBucketInfo
|
||||
|
||||
if _, err := getBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := getBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
|
||||
return
|
||||
}
|
||||
@@ -1305,7 +1335,10 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
|
||||
deleteBucket := objectAPI.DeleteBucket
|
||||
|
||||
// Attempt to delete bucket.
|
||||
if err := deleteBucket(ctx, bucket, DeleteBucketOptions{Force: forceDelete}); err != nil {
|
||||
if err := deleteBucket(ctx, bucket, DeleteBucketOptions{
|
||||
Force: forceDelete,
|
||||
SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
|
||||
}); err != nil {
|
||||
apiErr := toAPIError(ctx, err)
|
||||
if _, ok := err.(BucketNotEmpty); ok {
|
||||
if globalBucketVersioningSys.Enabled(bucket) || globalBucketVersioningSys.Suspended(bucket) {
|
||||
@@ -1360,10 +1393,6 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketObjectLockConfigurationAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
@@ -1472,7 +1501,7 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -1580,7 +1609,7 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, nil)
|
||||
updatedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketTaggingConfig)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
|
||||
@@ -21,7 +21,7 @@ import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
@@ -885,7 +885,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
|
||||
}
|
||||
|
||||
// read the response body.
|
||||
actualContent, err = ioutil.ReadAll(rec.Body)
|
||||
actualContent, err = io.ReadAll(rec.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d : MinIO %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
|
||||
}
|
||||
|
||||
@@ -62,12 +62,12 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(r.Body, r.ContentLength))
|
||||
bucketLifecycle, err := lifecycle.ParseLifecycleConfigWithID(io.LimitReader(r.Body, r.ContentLength))
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -121,7 +121,7 @@ func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -163,12 +163,12 @@ func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter,
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, nil); err != nil {
|
||||
if _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketLifecycleConfig); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
+56
-34
@@ -31,6 +31,7 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/amztime"
|
||||
sse "github.com/minio/minio/internal/bucket/encryption"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
"github.com/minio/minio/internal/event"
|
||||
@@ -57,15 +58,6 @@ type LifecycleSys struct{}
|
||||
|
||||
// Get - gets lifecycle config associated to a given bucket name.
|
||||
func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, err error) {
|
||||
if globalIsGateway {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return nil, errServerNotInitialized
|
||||
}
|
||||
|
||||
return nil, BucketLifecycleNotFound{Bucket: bucketName}
|
||||
}
|
||||
|
||||
return globalBucketMetadataSys.GetLifecycleConfig(bucketName)
|
||||
}
|
||||
|
||||
@@ -154,9 +146,14 @@ type newerNoncurrentTask struct {
|
||||
versions []ObjectToDelete
|
||||
}
|
||||
|
||||
type transitionTask struct {
|
||||
tier string
|
||||
objInfo ObjectInfo
|
||||
}
|
||||
|
||||
type transitionState struct {
|
||||
once sync.Once
|
||||
transitionCh chan ObjectInfo
|
||||
transitionCh chan transitionTask
|
||||
|
||||
ctx context.Context
|
||||
objAPI ObjectLayer
|
||||
@@ -170,13 +167,13 @@ type transitionState struct {
|
||||
lastDayStats map[string]*lastDayTierStats
|
||||
}
|
||||
|
||||
func (t *transitionState) queueTransitionTask(oi ObjectInfo) {
|
||||
func (t *transitionState) queueTransitionTask(oi ObjectInfo, sc string) {
|
||||
select {
|
||||
case <-GlobalContext.Done():
|
||||
t.once.Do(func() {
|
||||
close(t.transitionCh)
|
||||
})
|
||||
case t.transitionCh <- oi:
|
||||
case t.transitionCh <- transitionTask{objInfo: oi, tier: sc}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
@@ -185,7 +182,7 @@ var globalTransitionState *transitionState
|
||||
|
||||
func newTransitionState(ctx context.Context, objAPI ObjectLayer) *transitionState {
|
||||
return &transitionState{
|
||||
transitionCh: make(chan ObjectInfo, 10000),
|
||||
transitionCh: make(chan transitionTask, 10000),
|
||||
ctx: ctx,
|
||||
objAPI: objAPI,
|
||||
killCh: make(chan struct{}),
|
||||
@@ -212,27 +209,25 @@ func (t *transitionState) worker(ctx context.Context, objectAPI ObjectLayer) {
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case oi, ok := <-t.transitionCh:
|
||||
case task, ok := <-t.transitionCh:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
atomic.AddInt32(&t.activeTasks, 1)
|
||||
var tier string
|
||||
var err error
|
||||
if tier, err = transitionObject(ctx, objectAPI, oi); err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w", oi.Bucket, oi.Name, oi.VersionID, err))
|
||||
if err := transitionObject(ctx, objectAPI, task.objInfo, task.tier); err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w",
|
||||
task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
|
||||
} else {
|
||||
ts := tierStats{
|
||||
TotalSize: uint64(oi.Size),
|
||||
TotalSize: uint64(task.objInfo.Size),
|
||||
NumVersions: 1,
|
||||
}
|
||||
if oi.IsLatest {
|
||||
if task.objInfo.IsLatest {
|
||||
ts.NumObjects = 1
|
||||
}
|
||||
t.addLastDayStats(tier, ts)
|
||||
t.addLastDayStats(task.tier, ts)
|
||||
}
|
||||
atomic.AddInt32(&t.activeTasks, -1)
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -303,9 +298,10 @@ func validateTransitionTier(lc *lifecycle.Lifecycle) error {
|
||||
// This is to be called after a successful upload of an object (version).
|
||||
func enqueueTransitionImmediate(obj ObjectInfo) {
|
||||
if lc, err := globalLifecycleSys.Get(obj.Bucket); err == nil {
|
||||
switch lc.ComputeAction(obj.ToLifecycleOpts()) {
|
||||
event := lc.Eval(obj.ToLifecycleOpts(), time.Now())
|
||||
switch event.Action {
|
||||
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
|
||||
globalTransitionState.queueTransitionTask(obj)
|
||||
globalTransitionState.queueTransitionTask(obj, event.StorageClass)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -402,12 +398,7 @@ func genTransitionObjName(bucket string) (string, error) {
|
||||
// storage specified by the transition ARN, the metadata is left behind on source cluster and original content
|
||||
// is moved to the transition tier. Note that in the case of encrypted objects, entire encrypted stream is moved
|
||||
// to the transition tier without decrypting or re-encrypting.
|
||||
func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo) (string, error) {
|
||||
lc, err := globalLifecycleSys.Get(oi.Bucket)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
tier := lc.TransitionTier(oi.ToLifecycleOpts())
|
||||
func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo, tier string) error {
|
||||
opts := ObjectOptions{
|
||||
Transition: TransitionOptions{
|
||||
Status: lifecycle.TransitionPending,
|
||||
@@ -419,7 +410,37 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo)
|
||||
VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
|
||||
MTime: oi.ModTime,
|
||||
}
|
||||
return tier, objectAPI.TransitionObject(ctx, oi.Bucket, oi.Name, opts)
|
||||
return objectAPI.TransitionObject(ctx, oi.Bucket, oi.Name, opts)
|
||||
}
|
||||
|
||||
type auditTierOp struct {
|
||||
Tier string `json:"tier"`
|
||||
TimeToResponseNS int64 `json:"timeToResponseNS"`
|
||||
OutputBytes int64 `json:"tx,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func auditTierActions(ctx context.Context, tier string, bytes int64) func(err error) {
|
||||
startTime := time.Now()
|
||||
return func(err error) {
|
||||
// Record only when audit targets configured.
|
||||
if len(logger.AuditTargets()) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
op := auditTierOp{
|
||||
Tier: tier,
|
||||
OutputBytes: bytes,
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
op.TimeToResponseNS = time.Since(startTime).Nanoseconds()
|
||||
} else {
|
||||
op.Error = err.Error()
|
||||
}
|
||||
|
||||
logger.GetReqInfo(ctx).AppendTags("tierStats", op)
|
||||
}
|
||||
}
|
||||
|
||||
// getTransitionedObjectReader returns a reader from the transitioned tier.
|
||||
@@ -441,12 +462,13 @@ func getTransitionedObjectReader(ctx context.Context, bucket, object string, rs
|
||||
gopts.length = length
|
||||
}
|
||||
|
||||
timeTierAction := auditTierActions(ctx, oi.TransitionedObject.Tier, length)
|
||||
reader, err := tgtClient.Get(ctx, oi.TransitionedObject.Name, remoteVersionID(oi.TransitionedObject.VersionID), gopts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
closer := func() {
|
||||
reader.Close()
|
||||
timeTierAction(reader.Close())
|
||||
}
|
||||
return fn(reader, h, closer)
|
||||
}
|
||||
@@ -560,7 +582,7 @@ func (r *RestoreObjectRequest) validate(ctx context.Context, objAPI ObjectLayer)
|
||||
}
|
||||
// Check if bucket exists.
|
||||
if !r.OutputLocation.IsEmpty() {
|
||||
if _, err := objAPI.GetBucketInfo(ctx, r.OutputLocation.S3.BucketName); err != nil {
|
||||
if _, err := objAPI.GetBucketInfo(ctx, r.OutputLocation.S3.BucketName, BucketOptions{}); err != nil {
|
||||
return err
|
||||
}
|
||||
if r.OutputLocation.S3.Prefix == "" {
|
||||
@@ -755,7 +777,7 @@ func parseRestoreObjStatus(restoreHdr string) (restoreObjStatus, error) {
|
||||
if strings.TrimSpace(expiryTokens[0]) != "expiry-date" {
|
||||
return restoreObjStatus{}, errRestoreHDRMalformed
|
||||
}
|
||||
expiry, err := time.Parse(http.TimeFormat, strings.Trim(expiryTokens[1], `"`))
|
||||
expiry, err := amztime.ParseHeader(strings.Trim(expiryTokens[1], `"`))
|
||||
if err != nil {
|
||||
return restoreObjStatus{}, errRestoreHDRMalformed
|
||||
}
|
||||
|
||||
@@ -32,9 +32,9 @@ import (
|
||||
// Validate all the ListObjects query arguments, returns an APIErrorCode
|
||||
// if one of the args do not meet the required conditions.
|
||||
// Special conditions required by MinIO server are as below
|
||||
// - delimiter if set should be equal to '/', otherwise the request is rejected.
|
||||
// - marker if set should have a common prefix with 'prefix' param, otherwise
|
||||
// the request is rejected.
|
||||
// - delimiter if set should be equal to '/', otherwise the request is rejected.
|
||||
// - marker if set should have a common prefix with 'prefix' param, otherwise
|
||||
// the request is rejected.
|
||||
func validateListObjectsArgs(marker, delimiter, encodingType string, maxKeys int) APIErrorCode {
|
||||
// Max keys cannot be negative.
|
||||
if maxKeys < 0 {
|
||||
@@ -298,7 +298,6 @@ func proxyRequestByNodeIndex(ctx context.Context, w http.ResponseWriter, r *http
|
||||
// This implementation of the GET operation returns some or all (up to 1000)
|
||||
// of the objects in a bucket. You can use the request parameters as selection
|
||||
// criteria to return a subset of the objects in a bucket.
|
||||
//
|
||||
func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListObjectsV1")
|
||||
|
||||
|
||||
+24
-76
@@ -18,7 +18,6 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -45,11 +44,16 @@ type BucketMetadataSys struct {
|
||||
metadataMap map[string]BucketMetadata
|
||||
}
|
||||
|
||||
// Count returns number of bucket metadata map entries.
|
||||
func (sys *BucketMetadataSys) Count() int {
|
||||
sys.RLock()
|
||||
defer sys.RUnlock()
|
||||
|
||||
return len(sys.metadataMap)
|
||||
}
|
||||
|
||||
// Remove bucket metadata from memory.
|
||||
func (sys *BucketMetadataSys) Remove(bucket string) {
|
||||
if globalIsGateway {
|
||||
return
|
||||
}
|
||||
sys.Lock()
|
||||
delete(sys.metadataMap, bucket)
|
||||
globalBucketMonitor.DeleteBucket(bucket)
|
||||
@@ -62,10 +66,6 @@ func (sys *BucketMetadataSys) Remove(bucket string) {
|
||||
// so they should be replaced atomically and not appended to, etc.
|
||||
// Data is not persisted to disk.
|
||||
func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
|
||||
if globalIsGateway {
|
||||
return
|
||||
}
|
||||
|
||||
if bucket != minioMetaBucket {
|
||||
sys.Lock()
|
||||
sys.metadataMap[bucket] = meta
|
||||
@@ -73,33 +73,17 @@ func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
|
||||
}
|
||||
}
|
||||
|
||||
// Update update bucket metadata for the specified config file.
|
||||
// The configData data should not be modified after being sent here.
|
||||
func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) (updatedAt time.Time, err error) {
|
||||
func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string, configFile string, configData []byte, parse bool) (updatedAt time.Time, err error) {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return updatedAt, errServerNotInitialized
|
||||
}
|
||||
|
||||
if globalIsGateway && globalGatewayName != NASBackendGateway {
|
||||
if configFile == bucketPolicyConfig {
|
||||
if configData == nil {
|
||||
return updatedAt, objAPI.DeleteBucketPolicy(ctx, bucket)
|
||||
}
|
||||
config, err := policy.ParseConfig(bytes.NewReader(configData), bucket)
|
||||
if err != nil {
|
||||
return updatedAt, err
|
||||
}
|
||||
return updatedAt, objAPI.SetBucketPolicy(ctx, bucket, config)
|
||||
}
|
||||
return updatedAt, NotImplemented{}
|
||||
}
|
||||
|
||||
if bucket == minioMetaBucket {
|
||||
return updatedAt, errInvalidArgument
|
||||
}
|
||||
|
||||
meta, err := loadBucketMetadata(ctx, objAPI, bucket)
|
||||
meta, err := loadBucketMetadataParse(ctx, objAPI, bucket, parse)
|
||||
if err != nil {
|
||||
if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
|
||||
// Only single drive mode needs this fallback.
|
||||
@@ -157,6 +141,18 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
|
||||
return updatedAt, nil
|
||||
}
|
||||
|
||||
// Delete delete the bucket metadata for the specified bucket.
|
||||
// must be used by all callers instead of using Update() with nil configData.
|
||||
func (sys *BucketMetadataSys) Delete(ctx context.Context, bucket string, configFile string) (updatedAt time.Time, err error) {
|
||||
return sys.updateAndParse(ctx, bucket, configFile, nil, false)
|
||||
}
|
||||
|
||||
// Update update bucket metadata for the specified bucket.
|
||||
// The configData data should not be modified after being sent here.
|
||||
func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) (updatedAt time.Time, err error) {
|
||||
return sys.updateAndParse(ctx, bucket, configFile, configData, true)
|
||||
}
|
||||
|
||||
// Get metadata for a bucket.
|
||||
// If no metadata exists errConfigNotFound is returned and a new metadata is returned.
|
||||
// Only a shallow copy is returned, so referenced data should not be modified,
|
||||
@@ -168,7 +164,7 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
|
||||
// For all other bucket specific metadata, use the relevant
|
||||
// calls implemented specifically for each of those features.
|
||||
func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
|
||||
if globalIsGateway || bucket == minioMetaBucket {
|
||||
if bucket == minioMetaBucket {
|
||||
return newBucketMetadata(bucket), errConfigNotFound
|
||||
}
|
||||
|
||||
@@ -231,22 +227,6 @@ func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Co
|
||||
// GetLifecycleConfig returns configured lifecycle config
|
||||
// The returned object may not be modified.
|
||||
func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Lifecycle, error) {
|
||||
if globalIsGateway && globalGatewayName == NASBackendGateway {
|
||||
// Only needed in case of NAS gateway.
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return nil, errServerNotInitialized
|
||||
}
|
||||
meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if meta.lifecycleConfig == nil {
|
||||
return nil, BucketLifecycleNotFound{Bucket: bucket}
|
||||
}
|
||||
return meta.lifecycleConfig, nil
|
||||
}
|
||||
|
||||
meta, err := sys.GetConfig(GlobalContext, bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, errConfigNotFound) {
|
||||
@@ -263,19 +243,6 @@ func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Life
|
||||
// GetNotificationConfig returns configured notification config
|
||||
// The returned object may not be modified.
|
||||
func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Config, error) {
|
||||
if globalIsGateway && globalGatewayName == NASBackendGateway {
|
||||
// Only needed in case of NAS gateway.
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return nil, errServerNotInitialized
|
||||
}
|
||||
meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return meta.notificationConfig, nil
|
||||
}
|
||||
|
||||
meta, err := sys.GetConfig(GlobalContext, bucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -311,15 +278,6 @@ func (sys *BucketMetadataSys) CreatedAt(bucket string) (time.Time, error) {
|
||||
// GetPolicyConfig returns configured bucket policy
|
||||
// The returned object may not be modified.
|
||||
func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, time.Time, error) {
|
||||
if globalIsGateway {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return nil, time.Time{}, errServerNotInitialized
|
||||
}
|
||||
p, err := objAPI.GetBucketPolicy(GlobalContext, bucket)
|
||||
return p, UTCNow(), err
|
||||
}
|
||||
|
||||
meta, err := sys.GetConfig(GlobalContext, bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, errConfigNotFound) {
|
||||
@@ -387,10 +345,6 @@ func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (Buc
|
||||
return newBucketMetadata(bucket), errServerNotInitialized
|
||||
}
|
||||
|
||||
if globalIsGateway {
|
||||
return newBucketMetadata(bucket), NotImplemented{}
|
||||
}
|
||||
|
||||
if bucket == minioMetaBucket {
|
||||
return newBucketMetadata(bucket), errInvalidArgument
|
||||
}
|
||||
@@ -418,12 +372,6 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
|
||||
return errServerNotInitialized
|
||||
}
|
||||
|
||||
// In gateway mode, we don't need to load bucket metadata except
|
||||
// NAS gateway backend.
|
||||
if globalIsGateway && !objAPI.IsNotificationSupported() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Load bucket metadata sys in background
|
||||
go sys.load(ctx, buckets, objAPI)
|
||||
return nil
|
||||
@@ -452,7 +400,7 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
|
||||
sys.metadataMap[buckets[index].Name] = meta
|
||||
sys.Unlock()
|
||||
|
||||
globalNotificationSys.set(buckets[index], meta) // set notification targets
|
||||
globalEventNotifier.set(buckets[index], meta) // set notification targets
|
||||
|
||||
globalBucketTargetSys.set(buckets[index], meta) // set remote replication targets
|
||||
|
||||
|
||||
+45
-33
@@ -105,10 +105,8 @@ type BucketMetadata struct {
|
||||
|
||||
// newBucketMetadata creates BucketMetadata with the supplied name and Created to Now.
|
||||
func newBucketMetadata(name string) BucketMetadata {
|
||||
now := UTCNow()
|
||||
return BucketMetadata{
|
||||
Name: name,
|
||||
Created: now,
|
||||
Name: name,
|
||||
notificationConfig: &event.Config{
|
||||
XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
|
||||
},
|
||||
@@ -121,6 +119,17 @@ func newBucketMetadata(name string) BucketMetadata {
|
||||
}
|
||||
}
|
||||
|
||||
// SetCreatedAt preserves the CreatedAt time for bucket across sites in site replication. It defaults to
|
||||
// creation time of bucket on this cluster in all other cases.
|
||||
func (b *BucketMetadata) SetCreatedAt(createdAt time.Time) {
|
||||
if b.Created.IsZero() {
|
||||
b.Created = UTCNow()
|
||||
}
|
||||
if !createdAt.IsZero() {
|
||||
b.Created = createdAt.UTC()
|
||||
}
|
||||
}
|
||||
|
||||
// Load - loads the metadata of bucket by name from ObjectLayer api.
|
||||
// If an error is returned the returned metadata will be default initialized.
|
||||
func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string) error {
|
||||
@@ -153,27 +162,47 @@ func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string)
|
||||
return err
|
||||
}
|
||||
|
||||
// loadBucketMetadata loads and migrates to bucket metadata.
|
||||
func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
|
||||
func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket string, parse bool) (BucketMetadata, error) {
|
||||
b := newBucketMetadata(bucket)
|
||||
err := b.Load(ctx, objectAPI, b.Name)
|
||||
if err != nil && !errors.Is(err, errConfigNotFound) {
|
||||
return b, err
|
||||
}
|
||||
if err == nil {
|
||||
b.defaultTimestamps()
|
||||
}
|
||||
|
||||
// Old bucket without bucket metadata. Hence we migrate existing settings.
|
||||
if err := b.convertLegacyConfigs(ctx, objectAPI); err != nil {
|
||||
configs, err := b.getAllLegacyConfigs(ctx, objectAPI)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
|
||||
if len(configs) == 0 {
|
||||
if parse {
|
||||
// nothing to update, parse and proceed.
|
||||
err = b.parseAllConfigs(ctx, objectAPI)
|
||||
}
|
||||
} else {
|
||||
// Old bucket without bucket metadata. Hence we migrate existing settings.
|
||||
err = b.convertLegacyConfigs(ctx, objectAPI, configs)
|
||||
}
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
|
||||
// migrate unencrypted remote targets
|
||||
if err = b.migrateTargetConfig(ctx, objectAPI); err != nil {
|
||||
if err := b.migrateTargetConfig(ctx, objectAPI); err != nil {
|
||||
return b, err
|
||||
}
|
||||
b.defaultTimestamps()
|
||||
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// loadBucketMetadata loads and migrates to bucket metadata.
|
||||
func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
|
||||
return loadBucketMetadataParse(ctx, objectAPI, bucket, true)
|
||||
}
|
||||
|
||||
// parseAllConfigs will parse all configs and populate the private fields.
|
||||
// The first error encountered is returned.
|
||||
func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLayer) (err error) {
|
||||
@@ -266,7 +295,7 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
func (b *BucketMetadata) getAllLegacyConfigs(ctx context.Context, objectAPI ObjectLayer) (map[string][]byte, error) {
|
||||
legacyConfigs := []string{
|
||||
legacyBucketObjectLockEnabledConfigFile,
|
||||
bucketPolicyConfig,
|
||||
@@ -280,7 +309,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
|
||||
objectLockConfig,
|
||||
}
|
||||
|
||||
configs := make(map[string][]byte)
|
||||
configs := make(map[string][]byte, len(legacyConfigs))
|
||||
|
||||
// Handle migration from lockEnabled to newer format.
|
||||
if b.LockEnabled {
|
||||
@@ -305,16 +334,15 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
|
||||
continue
|
||||
}
|
||||
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
configs[legacyFile] = configData
|
||||
}
|
||||
|
||||
if len(configs) == 0 {
|
||||
// nothing to update, return right away.
|
||||
return b.parseAllConfigs(ctx, objectAPI)
|
||||
}
|
||||
return configs, nil
|
||||
}
|
||||
|
||||
func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer, configs map[string][]byte) error {
|
||||
for legacyFile, configData := range configs {
|
||||
switch legacyFile {
|
||||
case legacyBucketObjectLockEnabledConfigFile:
|
||||
@@ -345,6 +373,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
|
||||
b.BucketTargetsConfigJSON = configData
|
||||
}
|
||||
}
|
||||
b.defaultTimestamps()
|
||||
|
||||
if err := b.Save(ctx, objectAPI); err != nil {
|
||||
return err
|
||||
@@ -413,23 +442,6 @@ func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
|
||||
return saveConfig(ctx, api, configFile, data)
|
||||
}
|
||||
|
||||
// deleteBucketMetadata deletes bucket metadata
|
||||
// If config does not exist no error is returned.
|
||||
func deleteBucketMetadata(ctx context.Context, obj objectDeleter, bucket string) error {
|
||||
metadataFiles := []string{
|
||||
dataUsageCacheName,
|
||||
bucketMetadataFile,
|
||||
path.Join(replicationDir, resyncFileName),
|
||||
}
|
||||
for _, metaFile := range metadataFiles {
|
||||
configFile := path.Join(bucketMetaPrefix, bucket, metaFile)
|
||||
if err := deleteConfig(ctx, obj, configFile); err != nil && err != errConfigNotFound {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrate config for remote targets by encrypting data if currently unencrypted and kms is configured.
|
||||
func (b *BucketMetadata) migrateTargetConfig(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
var err error
|
||||
|
||||
@@ -60,7 +60,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
|
||||
return
|
||||
}
|
||||
|
||||
_, err := objAPI.GetBucketInfo(ctx, bucketName)
|
||||
_, err := objAPI.GetBucketInfo(ctx, bucketName, BucketOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -72,7 +72,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
|
||||
return
|
||||
}
|
||||
config.SetRegion(globalSite.Region)
|
||||
if err = config.Validate(globalSite.Region, globalNotificationSys.targetList); err != nil {
|
||||
if err = config.Validate(globalSite.Region, globalEventNotifier.targetList); err != nil {
|
||||
arnErr, ok := err.(*event.ErrARNNotFound)
|
||||
if ok {
|
||||
for i, queue := range config.QueueList {
|
||||
@@ -132,7 +132,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
|
||||
return
|
||||
}
|
||||
|
||||
_, err := objectAPI.GetBucketInfo(ctx, bucketName)
|
||||
_, err := objectAPI.GetBucketInfo(ctx, bucketName, BucketOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -144,7 +144,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
|
||||
return
|
||||
}
|
||||
|
||||
config, err := event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalSite.Region, globalNotificationSys.targetList)
|
||||
config, err := event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalSite.Region, globalEventNotifier.targetList)
|
||||
if err != nil {
|
||||
apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
|
||||
if event.IsEventError(err) {
|
||||
@@ -166,7 +166,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
|
||||
}
|
||||
|
||||
rulesMap := config.ToRulesMap()
|
||||
globalNotificationSys.AddRulesMap(bucketName, rulesMap)
|
||||
globalEventNotifier.AddRulesMap(bucketName, rulesMap)
|
||||
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"net/http"
|
||||
|
||||
@@ -35,22 +36,16 @@ type BucketObjectLockSys struct{}
|
||||
|
||||
// Get - Get retention configuration.
|
||||
func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention, err error) {
|
||||
if globalIsGateway {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return r, errServerNotInitialized
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucketName)
|
||||
if err != nil {
|
||||
if _, ok := err.(BucketObjectLockConfigNotFound); ok {
|
||||
if errors.Is(err, BucketObjectLockConfigNotFound{Bucket: bucketName}) {
|
||||
return r, nil
|
||||
}
|
||||
if errors.Is(err, errInvalidArgument) {
|
||||
return r, err
|
||||
}
|
||||
logger.CriticalIf(context.Background(), err)
|
||||
return r, err
|
||||
|
||||
}
|
||||
return config.ToRetention(), nil
|
||||
}
|
||||
|
||||
@@ -21,7 +21,6 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
@@ -61,7 +60,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -79,7 +78,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
return
|
||||
}
|
||||
|
||||
bucketPolicyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, r.ContentLength))
|
||||
bucketPolicyBytes, err := io.ReadAll(io.LimitReader(r.Body, r.ContentLength))
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -145,12 +144,12 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, nil)
|
||||
updatedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketPolicyConfig)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -191,7 +190,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -21,7 +21,6 @@ import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
@@ -128,7 +127,7 @@ func testCreateBucket(obj ObjectLayer, instanceType, bucketName string, apiRoute
|
||||
defer wg.Done()
|
||||
// Sync start.
|
||||
<-start
|
||||
if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, BucketOptions{}); err != nil {
|
||||
if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil {
|
||||
if _, ok := err.(BucketExists); !ok {
|
||||
t.Logf("unexpected error: %T: %v", err, err)
|
||||
return
|
||||
@@ -163,7 +162,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
bucketName1 := fmt.Sprintf("%s-1", bucketName)
|
||||
if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, BucketOptions{}); err != nil {
|
||||
if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -487,7 +486,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
|
||||
t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, recV4.Code)
|
||||
}
|
||||
// read the response body.
|
||||
bucketPolicyReadBuf, err := ioutil.ReadAll(recV4.Body)
|
||||
bucketPolicyReadBuf, err := io.ReadAll(recV4.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
|
||||
}
|
||||
@@ -525,7 +524,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
|
||||
t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, recV2.Code)
|
||||
}
|
||||
// read the response body.
|
||||
bucketPolicyReadBuf, err = ioutil.ReadAll(recV2.Body)
|
||||
bucketPolicyReadBuf, err = io.ReadAll(recV2.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
miniogopolicy "github.com/minio/minio-go/v7/pkg/policy"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
@@ -125,6 +126,20 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
|
||||
|
||||
cloneHeader := r.Header.Clone()
|
||||
|
||||
if userTags := cloneHeader.Get(xhttp.AmzObjectTagging); userTags != "" {
|
||||
tag, _ := tags.ParseObjectTags(userTags)
|
||||
if tag != nil {
|
||||
tagMap := tag.ToMap()
|
||||
keys := make([]string, 0, len(tagMap))
|
||||
for k, v := range tagMap {
|
||||
args[pathJoin("ExistingObjectTag", k)] = []string{v}
|
||||
args[pathJoin("RequestObjectTag", k)] = []string{v}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
args["RequestObjectTagKeys"] = keys
|
||||
}
|
||||
}
|
||||
|
||||
for _, objLock := range []string{
|
||||
xhttp.AmzObjectLockMode,
|
||||
xhttp.AmzObjectLockLegalHold,
|
||||
@@ -137,6 +152,9 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
|
||||
}
|
||||
|
||||
for key, values := range cloneHeader {
|
||||
if strings.EqualFold(key, xhttp.AmzObjectTagging) {
|
||||
continue
|
||||
}
|
||||
if existingValues, found := args[key]; found {
|
||||
args[key] = append(existingValues, values...)
|
||||
} else {
|
||||
@@ -169,6 +187,8 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
|
||||
}
|
||||
|
||||
// JWT specific values
|
||||
//
|
||||
// Add all string claims
|
||||
for k, v := range claims {
|
||||
vStr, ok := v.(string)
|
||||
if ok {
|
||||
@@ -183,6 +203,21 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
|
||||
}
|
||||
}
|
||||
}
|
||||
// Add groups claim which could be a list. This will ensure that the claim
|
||||
// `jwt:groups` works.
|
||||
if grpsVal, ok := claims["groups"]; ok {
|
||||
if grpsIs, ok := grpsVal.([]interface{}); ok {
|
||||
grps := []string{}
|
||||
for _, gI := range grpsIs {
|
||||
if g, ok := gI.(string); ok {
|
||||
grps = append(grps, g)
|
||||
}
|
||||
}
|
||||
if len(grps) > 0 {
|
||||
args["groups"] = grps
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
@@ -35,13 +35,6 @@ type BucketQuotaSys struct {
|
||||
|
||||
// Get - Get quota configuration.
|
||||
func (sys *BucketQuotaSys) Get(ctx context.Context, bucketName string) (*madmin.BucketQuota, error) {
|
||||
if globalIsGateway {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return nil, errServerNotInitialized
|
||||
}
|
||||
return &madmin.BucketQuota{}, nil
|
||||
}
|
||||
qCfg, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucketName)
|
||||
return qCfg, err
|
||||
}
|
||||
|
||||
@@ -46,20 +46,19 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.PutReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
// Check if bucket exists.
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if globalSiteReplicationSys.isEnabled() && logger.GetReqInfo(ctx).Cred.AccessKey != globalActiveCred.AccessKey {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
|
||||
return
|
||||
}
|
||||
if versioned := globalBucketVersioningSys.Enabled(bucket); !versioned {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNeedsVersioningError), r.URL)
|
||||
return
|
||||
@@ -118,7 +117,7 @@ func (api objectAPIHandlers) GetBucketReplicationConfigHandler(w http.ResponseWr
|
||||
return
|
||||
}
|
||||
// Check if bucket exists.
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -157,7 +156,7 @@ func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.Respons
|
||||
return
|
||||
}
|
||||
// Check if bucket exists.
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -165,7 +164,7 @@ func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.Respons
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
|
||||
return
|
||||
}
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, nil); err != nil {
|
||||
if _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketReplicationConfig); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -198,7 +197,7 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -217,7 +216,7 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
|
||||
w.Header().Set(xhttp.ContentType, string(mimeJSON))
|
||||
|
||||
enc := json.NewEncoder(w)
|
||||
if err = enc.Encode(getLatestReplicationStats(bucket, usageInfo)); err != nil {
|
||||
if err = enc.Encode(globalReplicationStats.getLatestReplicationStats(bucket, usageInfo)); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -267,7 +266,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -277,7 +276,13 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if !config.HasExistingObjectReplication(arn) {
|
||||
hasARN, hasExistingObjEnabled := config.HasExistingObjectReplication(arn)
|
||||
if !hasARN {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrRemoteTargetNotFoundError), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if !hasExistingObjEnabled {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNoExistingObjects), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -310,7 +315,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
|
||||
rinfo.Targets = append(rinfo.Targets, ResyncTarget{Arn: tgtArns[0], ResetID: target.ResetID})
|
||||
if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, true); err != nil {
|
||||
switch err.(type) {
|
||||
case BucketRemoteConnectionErr:
|
||||
case RemoteTargetConnectionErr:
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationRemoteConnectionError, err), r.URL)
|
||||
default:
|
||||
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
@@ -356,7 +361,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.Response
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
+175
-14
@@ -19,6 +19,8 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -36,10 +38,12 @@ func (b *BucketReplicationStats) hasReplicationUsage() bool {
|
||||
|
||||
// ReplicationStats holds the global in-memory replication stats
|
||||
type ReplicationStats struct {
|
||||
Cache map[string]*BucketReplicationStats
|
||||
UsageCache map[string]*BucketReplicationStats
|
||||
sync.RWMutex
|
||||
ulock sync.RWMutex
|
||||
Cache map[string]*BucketReplicationStats
|
||||
UsageCache map[string]*BucketReplicationStats
|
||||
mostRecentStats BucketStatsMap
|
||||
sync.RWMutex // mutex for Cache
|
||||
ulock sync.RWMutex // mutex for UsageCache
|
||||
mostRecentStatsMu sync.Mutex // mutex for mostRecentStats
|
||||
}
|
||||
|
||||
// Delete deletes in-memory replication statistics for a bucket.
|
||||
@@ -92,14 +96,23 @@ func (r *ReplicationStats) Update(bucket string, arn string, n int64, duration t
|
||||
bs.Stats[arn] = b
|
||||
}
|
||||
switch status {
|
||||
case replication.Pending:
|
||||
if opType.IsDataReplication() && prevStatus != status {
|
||||
b.PendingSize += n
|
||||
b.PendingCount++
|
||||
}
|
||||
case replication.Completed:
|
||||
switch prevStatus { // adjust counters based on previous state
|
||||
case replication.Pending:
|
||||
b.PendingCount--
|
||||
case replication.Failed:
|
||||
b.FailedCount--
|
||||
}
|
||||
if opType == replication.ObjectReplicationType {
|
||||
if opType.IsDataReplication() {
|
||||
b.ReplicatedSize += n
|
||||
switch prevStatus {
|
||||
case replication.Pending:
|
||||
b.PendingSize -= n
|
||||
case replication.Failed:
|
||||
b.FailedSize -= n
|
||||
}
|
||||
@@ -108,10 +121,12 @@ func (r *ReplicationStats) Update(bucket string, arn string, n int64, duration t
|
||||
}
|
||||
}
|
||||
case replication.Failed:
|
||||
if opType == replication.ObjectReplicationType {
|
||||
if opType.IsDataReplication() {
|
||||
if prevStatus == replication.Pending {
|
||||
b.FailedSize += n
|
||||
b.FailedCount++
|
||||
b.PendingSize -= n
|
||||
b.PendingCount--
|
||||
}
|
||||
}
|
||||
case replication.Replica:
|
||||
@@ -176,9 +191,20 @@ func NewReplicationStats(ctx context.Context, objectAPI ObjectLayer) *Replicatio
|
||||
}
|
||||
}
|
||||
|
||||
// load replication metrics at cluster start from initial data usage
|
||||
// load replication metrics at cluster start from latest replication stats saved in .minio.sys/buckets/replication/node-name.stats
|
||||
// fallback to replication stats in data usage to be backward compatible
|
||||
func (r *ReplicationStats) loadInitialReplicationMetrics(ctx context.Context) {
|
||||
rTimer := time.NewTimer(time.Minute)
|
||||
m := make(map[string]*BucketReplicationStats)
|
||||
if stats, err := globalReplicationPool.loadStatsFromDisk(); err == nil {
|
||||
for b, st := range stats {
|
||||
m[b] = &st
|
||||
}
|
||||
r.ulock.Lock()
|
||||
r.UsageCache = m
|
||||
r.ulock.Unlock()
|
||||
return
|
||||
}
|
||||
rTimer := time.NewTimer(time.Second * 5)
|
||||
defer rTimer.Stop()
|
||||
var (
|
||||
dui DataUsageInfo
|
||||
@@ -192,15 +218,12 @@ outer:
|
||||
case <-rTimer.C:
|
||||
dui, err = loadDataUsageFromBackend(GlobalContext, newObjectLayerFn())
|
||||
// If LastUpdate is set, data usage is available.
|
||||
if err == nil && !dui.LastUpdate.IsZero() {
|
||||
if err == nil {
|
||||
break outer
|
||||
}
|
||||
|
||||
rTimer.Reset(time.Minute)
|
||||
rTimer.Reset(time.Second * 5)
|
||||
}
|
||||
}
|
||||
|
||||
m := make(map[string]*BucketReplicationStats)
|
||||
for bucket, usage := range dui.BucketsUsage {
|
||||
b := &BucketReplicationStats{
|
||||
Stats: make(map[string]*BucketReplicationStat, len(usage.ReplicationInfo)),
|
||||
@@ -219,6 +242,144 @@ outer:
|
||||
}
|
||||
}
|
||||
r.ulock.Lock()
|
||||
defer r.ulock.Unlock()
|
||||
r.UsageCache = m
|
||||
r.ulock.Unlock()
|
||||
}
|
||||
|
||||
// serializeStats will serialize the current stats.
|
||||
// Will return (nil, nil) if no data.
|
||||
func (r *ReplicationStats) serializeStats() ([]byte, error) {
|
||||
if r == nil {
|
||||
return nil, nil
|
||||
}
|
||||
r.mostRecentStatsMu.Lock()
|
||||
defer r.mostRecentStatsMu.Unlock()
|
||||
bsm := r.mostRecentStats
|
||||
if len(bsm.Stats) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
data := make([]byte, 4, 4+bsm.Msgsize())
|
||||
// Add the replication stats meta header.
|
||||
binary.LittleEndian.PutUint16(data[0:2], replStatsMetaFormat)
|
||||
binary.LittleEndian.PutUint16(data[2:4], replStatsVersion)
|
||||
// Add data
|
||||
return r.mostRecentStats.MarshalMsg(data)
|
||||
}
|
||||
|
||||
func (r *ReplicationStats) getAllLatest(bucketsUsage map[string]BucketUsageInfo) (bucketsReplicationStats map[string]BucketReplicationStats) {
|
||||
peerBucketStatsList := globalNotificationSys.GetClusterAllBucketStats(GlobalContext)
|
||||
bucketsReplicationStats = make(map[string]BucketReplicationStats, len(bucketsUsage))
|
||||
|
||||
for bucket, u := range bucketsUsage {
|
||||
bucketStats := make([]BucketStats, len(peerBucketStatsList))
|
||||
for i, peerBucketStats := range peerBucketStatsList {
|
||||
bucketStat, ok := peerBucketStats.Stats[bucket]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
bucketStats[i] = bucketStat
|
||||
}
|
||||
bucketsReplicationStats[bucket] = r.calculateBucketReplicationStats(bucket, u, bucketStats)
|
||||
}
|
||||
return bucketsReplicationStats
|
||||
}
|
||||
|
||||
func (r *ReplicationStats) calculateBucketReplicationStats(bucket string, u BucketUsageInfo, bucketStats []BucketStats) (s BucketReplicationStats) {
|
||||
if r == nil {
|
||||
s = BucketReplicationStats{
|
||||
Stats: make(map[string]*BucketReplicationStat),
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// accumulate cluster bucket stats
|
||||
stats := make(map[string]*BucketReplicationStat)
|
||||
var totReplicaSize int64
|
||||
for _, bucketStat := range bucketStats {
|
||||
totReplicaSize += bucketStat.ReplicationStats.ReplicaSize
|
||||
for arn, stat := range bucketStat.ReplicationStats.Stats {
|
||||
oldst := stats[arn]
|
||||
if oldst == nil {
|
||||
oldst = &BucketReplicationStat{}
|
||||
}
|
||||
stats[arn] = &BucketReplicationStat{
|
||||
FailedCount: stat.FailedCount + oldst.FailedCount,
|
||||
FailedSize: stat.FailedSize + oldst.FailedSize,
|
||||
ReplicatedSize: stat.ReplicatedSize + oldst.ReplicatedSize,
|
||||
Latency: stat.Latency.merge(oldst.Latency),
|
||||
PendingCount: stat.PendingCount + oldst.PendingCount,
|
||||
PendingSize: stat.PendingSize + oldst.PendingSize,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// add initial usage stat to cluster stats
|
||||
usageStat := globalReplicationStats.GetInitialUsage(bucket)
|
||||
|
||||
totReplicaSize += usageStat.ReplicaSize
|
||||
for arn, stat := range usageStat.Stats {
|
||||
st, ok := stats[arn]
|
||||
if !ok {
|
||||
st = &BucketReplicationStat{}
|
||||
stats[arn] = st
|
||||
}
|
||||
st.ReplicatedSize += stat.ReplicatedSize
|
||||
st.FailedSize += stat.FailedSize
|
||||
st.FailedCount += stat.FailedCount
|
||||
st.PendingSize += stat.PendingSize
|
||||
st.PendingCount += stat.PendingCount
|
||||
}
|
||||
|
||||
s = BucketReplicationStats{
|
||||
Stats: make(map[string]*BucketReplicationStat, len(stats)),
|
||||
}
|
||||
var latestTotReplicatedSize int64
|
||||
for _, st := range u.ReplicationInfo {
|
||||
latestTotReplicatedSize += int64(st.ReplicatedSize)
|
||||
}
|
||||
|
||||
// normalize computed real time stats with latest usage stat
|
||||
for arn, tgtstat := range stats {
|
||||
st := BucketReplicationStat{}
|
||||
bu, ok := u.ReplicationInfo[arn]
|
||||
if !ok {
|
||||
bu = BucketTargetUsageInfo{}
|
||||
}
|
||||
// use in memory replication stats if it is ahead of usage info.
|
||||
st.ReplicatedSize = int64(bu.ReplicatedSize)
|
||||
if tgtstat.ReplicatedSize >= int64(bu.ReplicatedSize) {
|
||||
st.ReplicatedSize = tgtstat.ReplicatedSize
|
||||
}
|
||||
s.ReplicatedSize += st.ReplicatedSize
|
||||
// Reset FailedSize and FailedCount to 0 for negative overflows which can
|
||||
// happen since data usage picture can lag behind actual usage state at the time of cluster start
|
||||
st.FailedSize = int64(math.Max(float64(tgtstat.FailedSize), 0))
|
||||
st.FailedCount = int64(math.Max(float64(tgtstat.FailedCount), 0))
|
||||
st.PendingSize = int64(math.Max(float64(tgtstat.PendingSize), 0))
|
||||
st.PendingCount = int64(math.Max(float64(tgtstat.PendingCount), 0))
|
||||
st.Latency = tgtstat.Latency
|
||||
|
||||
s.Stats[arn] = &st
|
||||
s.FailedSize += st.FailedSize
|
||||
s.FailedCount += st.FailedCount
|
||||
s.PendingCount += st.PendingCount
|
||||
s.PendingSize += st.PendingSize
|
||||
}
|
||||
// normalize overall stats
|
||||
s.ReplicaSize = int64(math.Max(float64(totReplicaSize), float64(u.ReplicaSize)))
|
||||
s.ReplicatedSize = int64(math.Max(float64(s.ReplicatedSize), float64(latestTotReplicatedSize)))
|
||||
r.mostRecentStatsMu.Lock()
|
||||
if len(r.mostRecentStats.Stats) == 0 {
|
||||
r.mostRecentStats = BucketStatsMap{Stats: make(map[string]BucketStats, 1), Timestamp: UTCNow()}
|
||||
}
|
||||
r.mostRecentStats.Stats[bucket] = BucketStats{ReplicationStats: s}
|
||||
r.mostRecentStats.Timestamp = UTCNow()
|
||||
r.mostRecentStatsMu.Unlock()
|
||||
return s
|
||||
}
|
||||
|
||||
// get the most current of in-memory replication stats and data usage info from crawler.
|
||||
func (r *ReplicationStats) getLatestReplicationStats(bucket string, u BucketUsageInfo) (s BucketReplicationStats) {
|
||||
bucketStats := globalNotificationSys.GetClusterBucketStats(GlobalContext, bucket)
|
||||
return r.calculateBucketReplicationStats(bucket, u, bucketStats)
|
||||
}
|
||||
|
||||
@@ -737,3 +737,44 @@ func extractReplicateDiffOpts(q url.Values) (opts madmin.ReplDiffOpts) {
|
||||
opts.Prefix = q.Get("prefix")
|
||||
return
|
||||
}
|
||||
|
||||
const (
|
||||
replicationMRFDir = bucketMetaPrefix + SlashSeparator + replicationDir + SlashSeparator + "mrf"
|
||||
mrfMetaFormat = 1
|
||||
mrfMetaVersionV1 = 1
|
||||
mrfMetaVersion = mrfMetaVersionV1
|
||||
)
|
||||
|
||||
// MRFReplicateEntry mrf entry to save to disk
|
||||
type MRFReplicateEntry struct {
|
||||
Bucket string `json:"bucket" msg:"b"`
|
||||
Object string `json:"object" msg:"o"`
|
||||
versionID string `json:"-"`
|
||||
}
|
||||
|
||||
// MRFReplicateEntries has the map of MRF entries to save to disk
|
||||
type MRFReplicateEntries struct {
|
||||
Entries map[string]MRFReplicateEntry `json:"entries" msg:"e"`
|
||||
Version int `json:"version" msg:"v"`
|
||||
}
|
||||
|
||||
// ToMRFEntry returns the relevant info needed by MRF
|
||||
func (ri ReplicateObjectInfo) ToMRFEntry() MRFReplicateEntry {
|
||||
return MRFReplicateEntry{
|
||||
Bucket: ri.Bucket,
|
||||
Object: ri.Name,
|
||||
versionID: ri.VersionID,
|
||||
}
|
||||
}
|
||||
|
||||
func getReplicationStatsPath(nodeName string) string {
|
||||
nodeStr := strings.ReplaceAll(nodeName, ":", "_")
|
||||
return bucketMetaPrefix + SlashSeparator + replicationDir + SlashSeparator + nodeStr + ".stats"
|
||||
}
|
||||
|
||||
const (
|
||||
replStatsMetaFormat = 1
|
||||
replStatsVersionV1 = 1
|
||||
replStatsVersion = replStatsVersionV1
|
||||
replStatsSaveInterval = time.Minute * 5
|
||||
)
|
||||
|
||||
@@ -260,6 +260,413 @@ func (z *BucketReplicationResyncStatus) Msgsize() (s int) {
|
||||
return
|
||||
}
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *MRFReplicateEntries) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, err = dc.ReadMapKeyPtr()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "e":
|
||||
var zb0002 uint32
|
||||
zb0002, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Entries")
|
||||
return
|
||||
}
|
||||
if z.Entries == nil {
|
||||
z.Entries = make(map[string]MRFReplicateEntry, zb0002)
|
||||
} else if len(z.Entries) > 0 {
|
||||
for key := range z.Entries {
|
||||
delete(z.Entries, key)
|
||||
}
|
||||
}
|
||||
for zb0002 > 0 {
|
||||
zb0002--
|
||||
var za0001 string
|
||||
var za0002 MRFReplicateEntry
|
||||
za0001, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Entries")
|
||||
return
|
||||
}
|
||||
var zb0003 uint32
|
||||
zb0003, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Entries", za0001)
|
||||
return
|
||||
}
|
||||
for zb0003 > 0 {
|
||||
zb0003--
|
||||
field, err = dc.ReadMapKeyPtr()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Entries", za0001)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "b":
|
||||
za0002.Bucket, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Entries", za0001, "Bucket")
|
||||
return
|
||||
}
|
||||
case "o":
|
||||
za0002.Object, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Entries", za0001, "Object")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Entries", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
z.Entries[za0001] = za0002
|
||||
}
|
||||
case "v":
|
||||
z.Version, err = dc.ReadInt()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Version")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *MRFReplicateEntries) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// map header, size 2
|
||||
// write "e"
|
||||
err = en.Append(0x82, 0xa1, 0x65)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteMapHeader(uint32(len(z.Entries)))
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Entries")
|
||||
return
|
||||
}
|
||||
for za0001, za0002 := range z.Entries {
|
||||
err = en.WriteString(za0001)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Entries")
|
||||
return
|
||||
}
|
||||
// map header, size 2
|
||||
// write "b"
|
||||
err = en.Append(0x82, 0xa1, 0x62)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(za0002.Bucket)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Entries", za0001, "Bucket")
|
||||
return
|
||||
}
|
||||
// write "o"
|
||||
err = en.Append(0xa1, 0x6f)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(za0002.Object)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Entries", za0001, "Object")
|
||||
return
|
||||
}
|
||||
}
|
||||
// write "v"
|
||||
err = en.Append(0xa1, 0x76)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteInt(z.Version)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Version")
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *MRFReplicateEntries) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// map header, size 2
|
||||
// string "e"
|
||||
o = append(o, 0x82, 0xa1, 0x65)
|
||||
o = msgp.AppendMapHeader(o, uint32(len(z.Entries)))
|
||||
for za0001, za0002 := range z.Entries {
|
||||
o = msgp.AppendString(o, za0001)
|
||||
// map header, size 2
|
||||
// string "b"
|
||||
o = append(o, 0x82, 0xa1, 0x62)
|
||||
o = msgp.AppendString(o, za0002.Bucket)
|
||||
// string "o"
|
||||
o = append(o, 0xa1, 0x6f)
|
||||
o = msgp.AppendString(o, za0002.Object)
|
||||
}
|
||||
// string "v"
|
||||
o = append(o, 0xa1, 0x76)
|
||||
o = msgp.AppendInt(o, z.Version)
|
||||
return
|
||||
}
|
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (z *MRFReplicateEntries) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "e":
|
||||
var zb0002 uint32
|
||||
zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Entries")
|
||||
return
|
||||
}
|
||||
if z.Entries == nil {
|
||||
z.Entries = make(map[string]MRFReplicateEntry, zb0002)
|
||||
} else if len(z.Entries) > 0 {
|
||||
for key := range z.Entries {
|
||||
delete(z.Entries, key)
|
||||
}
|
||||
}
|
||||
for zb0002 > 0 {
|
||||
var za0001 string
|
||||
var za0002 MRFReplicateEntry
|
||||
zb0002--
|
||||
za0001, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Entries")
|
||||
return
|
||||
}
|
||||
var zb0003 uint32
|
||||
zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Entries", za0001)
|
||||
return
|
||||
}
|
||||
for zb0003 > 0 {
|
||||
zb0003--
|
||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Entries", za0001)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "b":
|
||||
za0002.Bucket, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Entries", za0001, "Bucket")
|
||||
return
|
||||
}
|
||||
case "o":
|
||||
za0002.Object, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Entries", za0001, "Object")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Entries", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
z.Entries[za0001] = za0002
|
||||
}
|
||||
case "v":
|
||||
z.Version, bts, err = msgp.ReadIntBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Version")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
o = bts
|
||||
return
|
||||
}
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *MRFReplicateEntries) Msgsize() (s int) {
|
||||
s = 1 + 2 + msgp.MapHeaderSize
|
||||
if z.Entries != nil {
|
||||
for za0001, za0002 := range z.Entries {
|
||||
_ = za0002
|
||||
s += msgp.StringPrefixSize + len(za0001) + 1 + 2 + msgp.StringPrefixSize + len(za0002.Bucket) + 2 + msgp.StringPrefixSize + len(za0002.Object)
|
||||
}
|
||||
}
|
||||
s += 2 + msgp.IntSize
|
||||
return
|
||||
}
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *MRFReplicateEntry) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, err = dc.ReadMapKeyPtr()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "b":
|
||||
z.Bucket, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Bucket")
|
||||
return
|
||||
}
|
||||
case "o":
|
||||
z.Object, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Object")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z MRFReplicateEntry) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// map header, size 2
|
||||
// write "b"
|
||||
err = en.Append(0x82, 0xa1, 0x62)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.Bucket)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Bucket")
|
||||
return
|
||||
}
|
||||
// write "o"
|
||||
err = en.Append(0xa1, 0x6f)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.Object)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Object")
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z MRFReplicateEntry) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// map header, size 2
|
||||
// string "b"
|
||||
o = append(o, 0x82, 0xa1, 0x62)
|
||||
o = msgp.AppendString(o, z.Bucket)
|
||||
// string "o"
|
||||
o = append(o, 0xa1, 0x6f)
|
||||
o = msgp.AppendString(o, z.Object)
|
||||
return
|
||||
}
|
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (z *MRFReplicateEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "b":
|
||||
z.Bucket, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Bucket")
|
||||
return
|
||||
}
|
||||
case "o":
|
||||
z.Object, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Object")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
o = bts
|
||||
return
|
||||
}
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z MRFReplicateEntry) Msgsize() (s int) {
|
||||
s = 1 + 2 + msgp.StringPrefixSize + len(z.Bucket) + 2 + msgp.StringPrefixSize + len(z.Object)
|
||||
return
|
||||
}
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *ReplicateDecision) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
var field []byte
|
||||
|
||||
@@ -122,6 +122,232 @@ func BenchmarkDecodeBucketReplicationResyncStatus(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalMRFReplicateEntries(t *testing.T) {
|
||||
v := MRFReplicateEntries{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgMRFReplicateEntries(b *testing.B) {
|
||||
v := MRFReplicateEntries{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgMRFReplicateEntries(b *testing.B) {
|
||||
v := MRFReplicateEntries{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalMRFReplicateEntries(b *testing.B) {
|
||||
v := MRFReplicateEntries{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeMRFReplicateEntries(t *testing.T) {
|
||||
v := MRFReplicateEntries{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeMRFReplicateEntries Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := MRFReplicateEntries{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeMRFReplicateEntries(b *testing.B) {
|
||||
v := MRFReplicateEntries{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeMRFReplicateEntries(b *testing.B) {
|
||||
v := MRFReplicateEntries{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalMRFReplicateEntry(t *testing.T) {
|
||||
v := MRFReplicateEntry{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgMRFReplicateEntry(b *testing.B) {
|
||||
v := MRFReplicateEntry{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgMRFReplicateEntry(b *testing.B) {
|
||||
v := MRFReplicateEntry{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalMRFReplicateEntry(b *testing.B) {
|
||||
v := MRFReplicateEntry{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeMRFReplicateEntry(t *testing.T) {
|
||||
v := MRFReplicateEntry{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeMRFReplicateEntry Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := MRFReplicateEntry{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeMRFReplicateEntry(b *testing.B) {
|
||||
v := MRFReplicateEntry{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeMRFReplicateEntry(b *testing.B) {
|
||||
v := MRFReplicateEntry{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalReplicateDecision(t *testing.T) {
|
||||
v := ReplicateDecision{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
|
||||
+779
-208
File diff suppressed because it is too large
Load Diff
+14
-2
@@ -18,6 +18,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -52,7 +53,10 @@ func (rl *ReplicationLatency) update(size int64, duration time.Duration) {
|
||||
}
|
||||
|
||||
// BucketStatsMap captures bucket statistics for all buckets
|
||||
type BucketStatsMap map[string]BucketStats
|
||||
type BucketStatsMap struct {
|
||||
Stats map[string]BucketStats
|
||||
Timestamp time.Time
|
||||
}
|
||||
|
||||
// BucketStats bucket statistics
|
||||
type BucketStats struct {
|
||||
@@ -62,7 +66,7 @@ type BucketStats struct {
|
||||
// BucketReplicationStats represents inline replication statistics
|
||||
// such as pending, failed and completed bytes in total for a bucket
|
||||
type BucketReplicationStats struct {
|
||||
Stats map[string]*BucketReplicationStat
|
||||
Stats map[string]*BucketReplicationStat `json:",omitempty"`
|
||||
// Pending size in bytes
|
||||
PendingSize int64 `json:"pendingReplicationSize"`
|
||||
// Completed size in bytes
|
||||
@@ -126,3 +130,11 @@ func (bs *BucketReplicationStat) hasReplicationUsage() bool {
|
||||
bs.PendingCount > 0 ||
|
||||
bs.PendingSize > 0
|
||||
}
|
||||
|
||||
func (brs BucketReplicationStats) String() string {
|
||||
s := "ReplicatedSize=" + fmt.Sprintf("%d", brs.ReplicatedSize) + "+\n ReplicaSize=" + fmt.Sprintf("%d", brs.ReplicaSize)
|
||||
for arn, st := range brs.Stats {
|
||||
s += "\n arn: " + arn + " ReplicatedSize=" + fmt.Sprintf("%d", st.ReplicatedSize) + " +::ReplicaSize=" + fmt.Sprintf("%d", st.ReplicaSize)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
+179
-98
@@ -794,74 +794,109 @@ func (z *BucketStats) Msgsize() (s int) {
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *BucketStatsMap) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
var zb0003 uint32
|
||||
zb0003, err = dc.ReadMapHeader()
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
if (*z) == nil {
|
||||
(*z) = make(BucketStatsMap, zb0003)
|
||||
} else if len((*z)) > 0 {
|
||||
for key := range *z {
|
||||
delete((*z), key)
|
||||
}
|
||||
}
|
||||
for zb0003 > 0 {
|
||||
zb0003--
|
||||
var zb0001 string
|
||||
var zb0002 BucketStats
|
||||
zb0001, err = dc.ReadString()
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, err = dc.ReadMapKeyPtr()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0004 uint32
|
||||
zb0004, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, zb0001)
|
||||
return
|
||||
}
|
||||
for zb0004 > 0 {
|
||||
zb0004--
|
||||
field, err = dc.ReadMapKeyPtr()
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "Stats":
|
||||
var zb0002 uint32
|
||||
zb0002, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, zb0001)
|
||||
err = msgp.WrapError(err, "Stats")
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "ReplicationStats":
|
||||
err = zb0002.ReplicationStats.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, zb0001, "ReplicationStats")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, zb0001)
|
||||
return
|
||||
if z.Stats == nil {
|
||||
z.Stats = make(map[string]BucketStats, zb0002)
|
||||
} else if len(z.Stats) > 0 {
|
||||
for key := range z.Stats {
|
||||
delete(z.Stats, key)
|
||||
}
|
||||
}
|
||||
for zb0002 > 0 {
|
||||
zb0002--
|
||||
var za0001 string
|
||||
var za0002 BucketStats
|
||||
za0001, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Stats")
|
||||
return
|
||||
}
|
||||
var zb0003 uint32
|
||||
zb0003, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Stats", za0001)
|
||||
return
|
||||
}
|
||||
for zb0003 > 0 {
|
||||
zb0003--
|
||||
field, err = dc.ReadMapKeyPtr()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Stats", za0001)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "ReplicationStats":
|
||||
err = za0002.ReplicationStats.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Stats", za0001, "ReplicationStats")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Stats", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
z.Stats[za0001] = za0002
|
||||
}
|
||||
case "Timestamp":
|
||||
z.Timestamp, err = dc.ReadTime()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Timestamp")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
(*z)[zb0001] = zb0002
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z BucketStatsMap) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
err = en.WriteMapHeader(uint32(len(z)))
|
||||
func (z *BucketStatsMap) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// map header, size 2
|
||||
// write "Stats"
|
||||
err = en.Append(0x82, 0xa5, 0x53, 0x74, 0x61, 0x74, 0x73)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
for zb0005, zb0006 := range z {
|
||||
err = en.WriteString(zb0005)
|
||||
err = en.WriteMapHeader(uint32(len(z.Stats)))
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Stats")
|
||||
return
|
||||
}
|
||||
for za0001, za0002 := range z.Stats {
|
||||
err = en.WriteString(za0001)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
err = msgp.WrapError(err, "Stats")
|
||||
return
|
||||
}
|
||||
// map header, size 1
|
||||
@@ -870,102 +905,148 @@ func (z BucketStatsMap) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = zb0006.ReplicationStats.EncodeMsg(en)
|
||||
err = za0002.ReplicationStats.EncodeMsg(en)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, zb0005, "ReplicationStats")
|
||||
err = msgp.WrapError(err, "Stats", za0001, "ReplicationStats")
|
||||
return
|
||||
}
|
||||
}
|
||||
// write "Timestamp"
|
||||
err = en.Append(0xa9, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteTime(z.Timestamp)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Timestamp")
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z BucketStatsMap) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
func (z *BucketStatsMap) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
o = msgp.AppendMapHeader(o, uint32(len(z)))
|
||||
for zb0005, zb0006 := range z {
|
||||
o = msgp.AppendString(o, zb0005)
|
||||
// map header, size 2
|
||||
// string "Stats"
|
||||
o = append(o, 0x82, 0xa5, 0x53, 0x74, 0x61, 0x74, 0x73)
|
||||
o = msgp.AppendMapHeader(o, uint32(len(z.Stats)))
|
||||
for za0001, za0002 := range z.Stats {
|
||||
o = msgp.AppendString(o, za0001)
|
||||
// map header, size 1
|
||||
// string "ReplicationStats"
|
||||
o = append(o, 0x81, 0xb0, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73)
|
||||
o, err = zb0006.ReplicationStats.MarshalMsg(o)
|
||||
o, err = za0002.ReplicationStats.MarshalMsg(o)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, zb0005, "ReplicationStats")
|
||||
err = msgp.WrapError(err, "Stats", za0001, "ReplicationStats")
|
||||
return
|
||||
}
|
||||
}
|
||||
// string "Timestamp"
|
||||
o = append(o, 0xa9, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70)
|
||||
o = msgp.AppendTime(o, z.Timestamp)
|
||||
return
|
||||
}
|
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (z *BucketStatsMap) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
var zb0003 uint32
|
||||
zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
if (*z) == nil {
|
||||
(*z) = make(BucketStatsMap, zb0003)
|
||||
} else if len((*z)) > 0 {
|
||||
for key := range *z {
|
||||
delete((*z), key)
|
||||
}
|
||||
}
|
||||
for zb0003 > 0 {
|
||||
var zb0001 string
|
||||
var zb0002 BucketStats
|
||||
zb0003--
|
||||
zb0001, bts, err = msgp.ReadStringBytes(bts)
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0004 uint32
|
||||
zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, zb0001)
|
||||
return
|
||||
}
|
||||
for zb0004 > 0 {
|
||||
zb0004--
|
||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "Stats":
|
||||
var zb0002 uint32
|
||||
zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, zb0001)
|
||||
err = msgp.WrapError(err, "Stats")
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "ReplicationStats":
|
||||
bts, err = zb0002.ReplicationStats.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, zb0001, "ReplicationStats")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, zb0001)
|
||||
return
|
||||
if z.Stats == nil {
|
||||
z.Stats = make(map[string]BucketStats, zb0002)
|
||||
} else if len(z.Stats) > 0 {
|
||||
for key := range z.Stats {
|
||||
delete(z.Stats, key)
|
||||
}
|
||||
}
|
||||
for zb0002 > 0 {
|
||||
var za0001 string
|
||||
var za0002 BucketStats
|
||||
zb0002--
|
||||
za0001, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Stats")
|
||||
return
|
||||
}
|
||||
var zb0003 uint32
|
||||
zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Stats", za0001)
|
||||
return
|
||||
}
|
||||
for zb0003 > 0 {
|
||||
zb0003--
|
||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Stats", za0001)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "ReplicationStats":
|
||||
bts, err = za0002.ReplicationStats.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Stats", za0001, "ReplicationStats")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Stats", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
z.Stats[za0001] = za0002
|
||||
}
|
||||
case "Timestamp":
|
||||
z.Timestamp, bts, err = msgp.ReadTimeBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Timestamp")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
(*z)[zb0001] = zb0002
|
||||
}
|
||||
o = bts
|
||||
return
|
||||
}
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z BucketStatsMap) Msgsize() (s int) {
|
||||
s = msgp.MapHeaderSize
|
||||
if z != nil {
|
||||
for zb0005, zb0006 := range z {
|
||||
_ = zb0006
|
||||
s += msgp.StringPrefixSize + len(zb0005) + 1 + 17 + zb0006.ReplicationStats.Msgsize()
|
||||
func (z *BucketStatsMap) Msgsize() (s int) {
|
||||
s = 1 + 6 + msgp.MapHeaderSize
|
||||
if z.Stats != nil {
|
||||
for za0001, za0002 := range z.Stats {
|
||||
_ = za0002
|
||||
s += msgp.StringPrefixSize + len(za0001) + 1 + 17 + za0002.ReplicationStats.Msgsize()
|
||||
}
|
||||
}
|
||||
s += 10 + msgp.TimeSize
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
+146
-38
@@ -19,6 +19,8 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -28,14 +30,15 @@ import (
|
||||
miniogo "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultHealthCheckDuration = 30 * time.Second
|
||||
defaultHealthCheckDuration = 5 * time.Second
|
||||
// default interval for reload of all remote target endpoints
|
||||
defaultHealthCheckReloadDuration = 30 * time.Minute
|
||||
)
|
||||
|
||||
// BucketTargetSys represents bucket targets subsystem
|
||||
@@ -43,6 +46,113 @@ type BucketTargetSys struct {
|
||||
sync.RWMutex
|
||||
arnRemotesMap map[string]*TargetClient
|
||||
targetsMap map[string][]madmin.BucketTarget
|
||||
hMutex sync.RWMutex
|
||||
hc map[string]epHealth
|
||||
hcClient *madmin.AnonymousClient
|
||||
}
|
||||
|
||||
// epHealth struct represents health of a replication target endpoint.
|
||||
type epHealth struct {
|
||||
Endpoint string
|
||||
Scheme string
|
||||
Online bool
|
||||
}
|
||||
|
||||
// isOffline returns current liveness result of remote target. Add endpoint to
|
||||
// healthcheck map if missing and default to online status
|
||||
func (sys *BucketTargetSys) isOffline(ep *url.URL) bool {
|
||||
sys.hMutex.RLock()
|
||||
defer sys.hMutex.RUnlock()
|
||||
if h, ok := sys.hc[ep.Host]; ok {
|
||||
return !h.Online
|
||||
}
|
||||
go sys.initHC(ep)
|
||||
return false
|
||||
}
|
||||
|
||||
func (sys *BucketTargetSys) initHC(ep *url.URL) {
|
||||
sys.hMutex.Lock()
|
||||
sys.hc[ep.Host] = epHealth{
|
||||
Endpoint: ep.Host,
|
||||
Scheme: ep.Scheme,
|
||||
Online: true,
|
||||
}
|
||||
sys.hMutex.Unlock()
|
||||
}
|
||||
|
||||
// newHCClient initializes an anonymous client for performing health check on the remote endpoints
|
||||
func newHCClient() *madmin.AnonymousClient {
|
||||
clnt, e := madmin.NewAnonymousClientNoEndpoint()
|
||||
if e != nil {
|
||||
logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to initialize health check client"), string(replicationSubsystem))
|
||||
return nil
|
||||
}
|
||||
clnt.SetCustomTransport(globalRemoteTargetTransport)
|
||||
return clnt
|
||||
}
|
||||
|
||||
// heartBeat performs liveness check on remote endpoints.
|
||||
func (sys *BucketTargetSys) heartBeat(ctx context.Context) {
|
||||
hcTimer := time.NewTimer(defaultHealthCheckDuration)
|
||||
defer hcTimer.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-hcTimer.C:
|
||||
sys.hMutex.RLock()
|
||||
var eps []madmin.ServerProperties
|
||||
for _, ep := range sys.hc {
|
||||
eps = append(eps, madmin.ServerProperties{Endpoint: ep.Endpoint, Scheme: ep.Scheme})
|
||||
}
|
||||
sys.hMutex.RUnlock()
|
||||
|
||||
if len(eps) > 0 {
|
||||
cctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
m := map[string]epHealth{}
|
||||
for result := range sys.hcClient.Alive(cctx, madmin.AliveOpts{}, eps...) {
|
||||
var online bool
|
||||
if result.Error == nil {
|
||||
online = result.Online
|
||||
}
|
||||
m[result.Endpoint.Host] = epHealth{
|
||||
Endpoint: result.Endpoint.Host,
|
||||
Scheme: result.Endpoint.Scheme,
|
||||
Online: online,
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
sys.hMutex.Lock()
|
||||
sys.hc = m
|
||||
sys.hMutex.Unlock()
|
||||
}
|
||||
hcTimer.Reset(defaultHealthCheckDuration)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// periodically rebuild the healthcheck map from list of targets to clear
|
||||
// out stale endpoints
|
||||
func (sys *BucketTargetSys) reloadHealthCheckers(ctx context.Context) {
|
||||
m := make(map[string]epHealth)
|
||||
tgts := sys.ListTargets(ctx, "", "")
|
||||
for _, t := range tgts {
|
||||
if _, ok := m[t.Endpoint]; !ok {
|
||||
scheme := "http"
|
||||
if t.Secure {
|
||||
scheme = "https"
|
||||
}
|
||||
m[t.Endpoint] = epHealth{
|
||||
Online: true,
|
||||
Endpoint: t.Endpoint,
|
||||
Scheme: scheme,
|
||||
}
|
||||
}
|
||||
}
|
||||
sys.hMutex.Lock()
|
||||
// swap out the map
|
||||
sys.hc = m
|
||||
sys.hMutex.Unlock()
|
||||
}
|
||||
|
||||
// ListTargets lists bucket targets across tenant or for individual bucket, and returns
|
||||
@@ -91,9 +201,6 @@ func (sys *BucketTargetSys) Delete(bucket string) {
|
||||
return
|
||||
}
|
||||
for _, t := range tgts {
|
||||
if tgt, ok := sys.arnRemotesMap[t.Arn]; ok && tgt.healthCancelFn != nil {
|
||||
tgt.healthCancelFn()
|
||||
}
|
||||
delete(sys.arnRemotesMap, t.Arn)
|
||||
}
|
||||
delete(sys.targetsMap, bucket)
|
||||
@@ -101,9 +208,6 @@ func (sys *BucketTargetSys) Delete(bucket string) {
|
||||
|
||||
// SetTarget - sets a new minio-go client target for this bucket.
|
||||
func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *madmin.BucketTarget, update bool) error {
|
||||
if globalIsGateway {
|
||||
return nil
|
||||
}
|
||||
if !tgt.Type.IsValid() && !update {
|
||||
return BucketRemoteArnTypeInvalid{Bucket: bucket}
|
||||
}
|
||||
@@ -112,11 +216,15 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
|
||||
return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
|
||||
}
|
||||
// validate if target credentials are ok
|
||||
if _, err = clnt.BucketExists(ctx, tgt.TargetBucket); err != nil {
|
||||
exists, err := clnt.BucketExists(ctx, tgt.TargetBucket)
|
||||
if err != nil {
|
||||
if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
|
||||
return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
|
||||
}
|
||||
return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket, Err: err}
|
||||
return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: err}
|
||||
}
|
||||
if !exists {
|
||||
return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
|
||||
}
|
||||
if tgt.Type == madmin.ReplicationService {
|
||||
if !globalBucketVersioningSys.Enabled(bucket) {
|
||||
@@ -124,9 +232,9 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
|
||||
}
|
||||
vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
|
||||
if err != nil {
|
||||
return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket, Err: err}
|
||||
return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: err}
|
||||
}
|
||||
if vcfg.Status != string(versioning.Enabled) {
|
||||
if !vcfg.Enabled() {
|
||||
return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
|
||||
}
|
||||
}
|
||||
@@ -152,10 +260,7 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
|
||||
if !found && !update {
|
||||
newtgts = append(newtgts, *tgt)
|
||||
}
|
||||
// cancel health check for previous target client to avoid leak.
|
||||
if prevClnt, ok := sys.arnRemotesMap[tgt.Arn]; ok && prevClnt.healthCancelFn != nil {
|
||||
prevClnt.healthCancelFn()
|
||||
}
|
||||
|
||||
sys.targetsMap[bucket] = newtgts
|
||||
sys.arnRemotesMap[tgt.Arn] = clnt
|
||||
sys.updateBandwidthLimit(bucket, tgt.BandwidthLimit)
|
||||
@@ -163,9 +268,6 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
|
||||
}
|
||||
|
||||
func (sys *BucketTargetSys) updateBandwidthLimit(bucket string, limit int64) {
|
||||
if globalIsGateway {
|
||||
return
|
||||
}
|
||||
if limit == 0 {
|
||||
globalBucketMonitor.DeleteBucket(bucket)
|
||||
return
|
||||
@@ -177,10 +279,6 @@ func (sys *BucketTargetSys) updateBandwidthLimit(bucket string, limit int64) {
|
||||
|
||||
// RemoveTarget - removes a remote bucket target for this source bucket.
|
||||
func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr string) error {
|
||||
if globalIsGateway {
|
||||
return nil
|
||||
}
|
||||
|
||||
if arnStr == "" {
|
||||
return BucketRemoteArnInvalid{Bucket: bucket}
|
||||
}
|
||||
@@ -227,9 +325,6 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
|
||||
return BucketRemoteTargetNotFound{Bucket: bucket}
|
||||
}
|
||||
sys.targetsMap[bucket] = targets
|
||||
if tgt, ok := sys.arnRemotesMap[arnStr]; ok && tgt.healthCancelFn != nil {
|
||||
tgt.healthCancelFn()
|
||||
}
|
||||
delete(sys.arnRemotesMap, arnStr)
|
||||
sys.updateBandwidthLimit(bucket, 0)
|
||||
return nil
|
||||
@@ -258,11 +353,29 @@ func (sys *BucketTargetSys) GetRemoteBucketTargetByArn(ctx context.Context, buck
|
||||
}
|
||||
|
||||
// NewBucketTargetSys - creates new replication system.
|
||||
func NewBucketTargetSys() *BucketTargetSys {
|
||||
return &BucketTargetSys{
|
||||
func NewBucketTargetSys(ctx context.Context) *BucketTargetSys {
|
||||
sys := &BucketTargetSys{
|
||||
arnRemotesMap: make(map[string]*TargetClient),
|
||||
targetsMap: make(map[string][]madmin.BucketTarget),
|
||||
hc: make(map[string]epHealth),
|
||||
hcClient: newHCClient(),
|
||||
}
|
||||
// reload healthcheck endpoints map periodically to remove stale endpoints from the map.
|
||||
go func() {
|
||||
rTimer := time.NewTimer(defaultHealthCheckReloadDuration)
|
||||
defer rTimer.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-rTimer.C:
|
||||
sys.reloadHealthCheckers(ctx)
|
||||
rTimer.Reset(defaultHealthCheckReloadDuration)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
go sys.heartBeat(ctx)
|
||||
return sys
|
||||
}
|
||||
|
||||
// UpdateAllTargets updates target to reflect metadata updates
|
||||
@@ -276,9 +389,6 @@ func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketT
|
||||
// Remove existingtarget and arn association
|
||||
if tgts, ok := sys.targetsMap[bucket]; ok {
|
||||
for _, t := range tgts {
|
||||
if tgt, ok := sys.arnRemotesMap[t.Arn]; ok && tgt.healthCancelFn != nil {
|
||||
tgt.healthCancelFn()
|
||||
}
|
||||
delete(sys.arnRemotesMap, t.Arn)
|
||||
}
|
||||
delete(sys.targetsMap, bucket)
|
||||
@@ -345,10 +455,6 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
|
||||
if tcfg.HealthCheckDuration >= 1 { // require minimum health check duration of 1 sec.
|
||||
hcDuration = tcfg.HealthCheckDuration
|
||||
}
|
||||
cancelFn, err := api.HealthCheck(hcDuration)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tc := &TargetClient{
|
||||
Client: api,
|
||||
healthCheckDuration: hcDuration,
|
||||
@@ -356,9 +462,10 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
|
||||
Bucket: tcfg.TargetBucket,
|
||||
StorageClass: tcfg.StorageClass,
|
||||
disableProxy: tcfg.DisableProxy,
|
||||
healthCancelFn: cancelFn,
|
||||
ARN: tcfg.Arn,
|
||||
ResetID: tcfg.ResetID,
|
||||
Endpoint: tcfg.Endpoint,
|
||||
Secure: tcfg.Secure,
|
||||
}
|
||||
return tc, nil
|
||||
}
|
||||
@@ -432,7 +539,8 @@ type TargetClient struct {
|
||||
replicateSync bool
|
||||
StorageClass string // storage class on remote
|
||||
disableProxy bool
|
||||
healthCancelFn context.CancelFunc // cancellation function for client healthcheck
|
||||
ARN string // ARN to uniquely identify remote target
|
||||
ARN string // ARN to uniquely identify remote target
|
||||
ResetID string
|
||||
Endpoint string
|
||||
Secure bool
|
||||
}
|
||||
|
||||
@@ -143,7 +143,7 @@ func (api objectAPIHandlers) GetBucketVersioningHandler(w http.ResponseWriter, r
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -69,11 +69,6 @@ func (sys *BucketVersioningSys) PrefixSuspended(bucket, prefix string) bool {
|
||||
|
||||
// Get returns stored bucket policy
|
||||
func (sys *BucketVersioningSys) Get(bucket string) (*versioning.Versioning, error) {
|
||||
if globalIsGateway {
|
||||
// Gateway does not implement versioning.
|
||||
return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, nil
|
||||
}
|
||||
|
||||
if bucket == minioMetaBucket || strings.HasPrefix(bucket, minioMetaBucket) {
|
||||
return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, nil
|
||||
}
|
||||
|
||||
+28
-35
@@ -25,7 +25,6 @@ import (
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
uatomic "go.uber.org/atomic"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -34,9 +33,6 @@ const (
|
||||
|
||||
// callhomeSchemaVersion is current callhome schema version.
|
||||
callhomeSchemaVersion = callhomeSchemaVersion1
|
||||
|
||||
// callhomeCycleDefault is the default interval between two callhome cycles (24hrs)
|
||||
callhomeCycleDefault = 24 * time.Hour
|
||||
)
|
||||
|
||||
// CallhomeInfo - Contains callhome information
|
||||
@@ -45,26 +41,14 @@ type CallhomeInfo struct {
|
||||
AdminInfo madmin.InfoMessage `json:"admin_info"`
|
||||
}
|
||||
|
||||
var (
|
||||
enableCallhome = uatomic.NewBool(false)
|
||||
callhomeLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
|
||||
callhomeFreq = uatomic.NewDuration(callhomeCycleDefault)
|
||||
)
|
||||
|
||||
func updateCallhomeParams(ctx context.Context, objAPI ObjectLayer) {
|
||||
alreadyEnabled := enableCallhome.Load()
|
||||
enableCallhome.Store(globalCallhomeConfig.Enable)
|
||||
callhomeFreq.Store(globalCallhomeConfig.Frequency)
|
||||
|
||||
// If callhome was disabled earlier and has now been enabled,
|
||||
// initialize the callhome process again.
|
||||
if !alreadyEnabled && enableCallhome.Load() {
|
||||
initCallhome(ctx, objAPI)
|
||||
}
|
||||
}
|
||||
var callhomeLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
|
||||
|
||||
// initCallhome will start the callhome task in the background.
|
||||
func initCallhome(ctx context.Context, objAPI ObjectLayer) {
|
||||
if !globalCallhomeConfig.Enabled() {
|
||||
return
|
||||
}
|
||||
|
||||
go func() {
|
||||
r := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
// Leader node (that successfully acquires the lock inside runCallhome)
|
||||
@@ -72,54 +56,63 @@ func initCallhome(ctx context.Context, objAPI ObjectLayer) {
|
||||
// the lock will be released and another node will acquire it and take over
|
||||
// because of this loop.
|
||||
for {
|
||||
runCallhome(ctx, objAPI)
|
||||
if !enableCallhome.Load() {
|
||||
if !globalCallhomeConfig.Enabled() {
|
||||
return
|
||||
}
|
||||
|
||||
if !runCallhome(ctx, objAPI) {
|
||||
// callhome was disabled or context was canceled
|
||||
return
|
||||
}
|
||||
|
||||
// callhome running on a different node.
|
||||
// sleep for some time and try again.
|
||||
duration := time.Duration(r.Float64() * float64(callhomeFreq.Load()))
|
||||
duration := time.Duration(r.Float64() * float64(globalCallhomeConfig.FrequencyDur()))
|
||||
if duration < time.Second {
|
||||
// Make sure to sleep atleast a second to avoid high CPU ticks.
|
||||
duration = time.Second
|
||||
}
|
||||
time.Sleep(duration)
|
||||
|
||||
if !enableCallhome.Load() {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func runCallhome(ctx context.Context, objAPI ObjectLayer) {
|
||||
func runCallhome(ctx context.Context, objAPI ObjectLayer) bool {
|
||||
// Make sure only 1 callhome is running on the cluster.
|
||||
locker := objAPI.NewNSLock(minioMetaBucket, "callhome/runCallhome.lock")
|
||||
lkctx, err := locker.GetLock(ctx, callhomeLeaderLockTimeout)
|
||||
if err != nil {
|
||||
return
|
||||
// lock timedout means some other node is the leader,
|
||||
// cycle back return 'true'
|
||||
return true
|
||||
}
|
||||
|
||||
ctx = lkctx.Context()
|
||||
defer locker.Unlock(lkctx.Cancel)
|
||||
|
||||
callhomeTimer := time.NewTimer(callhomeFreq.Load())
|
||||
callhomeTimer := time.NewTimer(globalCallhomeConfig.FrequencyDur())
|
||||
defer callhomeTimer.Stop()
|
||||
|
||||
for {
|
||||
if !globalCallhomeConfig.Enabled() {
|
||||
// Stop the processing as callhome got disabled
|
||||
return false
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
// indicates that we do not need to run callhome anymore
|
||||
return false
|
||||
case <-callhomeTimer.C:
|
||||
if !enableCallhome.Load() {
|
||||
if !globalCallhomeConfig.Enabled() {
|
||||
// Stop the processing as callhome got disabled
|
||||
return
|
||||
return false
|
||||
}
|
||||
|
||||
performCallhome(ctx)
|
||||
|
||||
// Reset the timer for next cycle.
|
||||
callhomeTimer.Reset(callhomeFreq.Load())
|
||||
callhomeTimer.Reset(globalCallhomeConfig.FrequencyDur())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+64
-67
@@ -27,7 +27,6 @@ import (
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -135,7 +134,7 @@ func init() {
|
||||
|
||||
globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
|
||||
PassHost: true,
|
||||
RoundTripper: newGatewayHTTPTransport(1 * time.Hour),
|
||||
RoundTripper: newHTTPTransport(1 * time.Hour),
|
||||
Logger: func(err error) {
|
||||
if err != nil && !errors.Is(err, context.Canceled) {
|
||||
logger.LogIf(GlobalContext, err)
|
||||
@@ -152,7 +151,7 @@ func init() {
|
||||
defaultAWSCredProvider = []credentials.Provider{
|
||||
&credentials.IAM{
|
||||
Client: &http.Client{
|
||||
Transport: NewGatewayHTTPTransport(),
|
||||
Transport: NewHTTPTransport(),
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -176,7 +175,9 @@ func minioConfigToConsoleFeatures() {
|
||||
if globalMinioEndpoint != "" {
|
||||
os.Setenv("CONSOLE_MINIO_SERVER", globalMinioEndpoint)
|
||||
} else {
|
||||
os.Setenv("CONSOLE_MINIO_SERVER", getAPIEndpoints()[0])
|
||||
// Explicitly set 127.0.0.1 so Console will automatically bypass TLS verification to the local S3 API.
|
||||
// This will save users from providing a certificate with IP or FQDN SAN that points to the local host.
|
||||
os.Setenv("CONSOLE_MINIO_SERVER", fmt.Sprintf("%s://127.0.0.1:%s", getURLScheme(globalIsTLS), globalMinioPort))
|
||||
}
|
||||
if value := env.Get("MINIO_LOG_QUERY_URL", ""); value != "" {
|
||||
os.Setenv("CONSOLE_LOG_QUERY_URL", value)
|
||||
@@ -203,20 +204,13 @@ func minioConfigToConsoleFeatures() {
|
||||
}
|
||||
}
|
||||
// Enable if LDAP is enabled.
|
||||
if globalLDAPConfig.Enabled {
|
||||
if globalLDAPConfig.Enabled() {
|
||||
os.Setenv("CONSOLE_LDAP_ENABLED", config.EnableOn)
|
||||
}
|
||||
os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
|
||||
os.Setenv("CONSOLE_CERT_PASSWD", env.Get("MINIO_CERT_PASSWD", ""))
|
||||
if globalSubnetConfig.License != "" {
|
||||
os.Setenv("CONSOLE_SUBNET_LICENSE", globalSubnetConfig.License)
|
||||
}
|
||||
if globalSubnetConfig.APIKey != "" {
|
||||
os.Setenv("CONSOLE_SUBNET_API_KEY", globalSubnetConfig.APIKey)
|
||||
}
|
||||
if globalSubnetConfig.ProxyURL != nil {
|
||||
os.Setenv("CONSOLE_SUBNET_PROXY", globalSubnetConfig.ProxyURL.String())
|
||||
}
|
||||
|
||||
globalSubnetConfig.ApplyEnv()
|
||||
}
|
||||
|
||||
func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
|
||||
@@ -237,6 +231,7 @@ func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
|
||||
Userinfo: cfg.ClaimUserinfo,
|
||||
RedirectCallbackDynamic: cfg.RedirectURIDynamic,
|
||||
RedirectCallback: callback,
|
||||
RoleArn: cfg.GetRoleArn(),
|
||||
}
|
||||
}
|
||||
return m
|
||||
@@ -264,7 +259,7 @@ func initConsoleServer() (*restapi.Server, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
api := operations.NewConsoleAPI(swaggerSpec, buildOpenIDConsoleConfig())
|
||||
api := operations.NewConsoleAPI(swaggerSpec)
|
||||
|
||||
if !serverDebugLog {
|
||||
// Disable console logging if server debug log is not enabled
|
||||
@@ -275,6 +270,12 @@ func initConsoleServer() (*restapi.Server, error) {
|
||||
api.Logger = noLog
|
||||
}
|
||||
|
||||
// Pass in console application config. This needs to happen before the
|
||||
// ConfigureAPI() call.
|
||||
restapi.GlobalMinIOConfig = restapi.MinIOConfig{
|
||||
OpenIDProviders: buildOpenIDConsoleConfig(),
|
||||
}
|
||||
|
||||
server := restapi.NewServer(api)
|
||||
// register all APIs
|
||||
server.ConfigureAPI()
|
||||
@@ -300,22 +301,6 @@ func initConsoleServer() (*restapi.Server, error) {
|
||||
return server, nil
|
||||
}
|
||||
|
||||
func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
|
||||
if strings.HasPrefix(name, "gateway") {
|
||||
if GlobalGatewaySSE.IsSet() && GlobalKMS == nil {
|
||||
uiErr := config.ErrInvalidGWSSEEnvValue(nil).Msg("MINIO_GATEWAY_SSE set but KMS is not configured")
|
||||
logger.Fatal(uiErr, "Unable to start gateway with SSE")
|
||||
}
|
||||
}
|
||||
|
||||
globalCompressConfigMu.Lock()
|
||||
if globalCompressConfig.Enabled && !objAPI.IsCompressionSupported() {
|
||||
logger.Fatal(errInvalidArgument,
|
||||
"Compression support is requested but '%s' does not support compression", name)
|
||||
}
|
||||
globalCompressConfigMu.Unlock()
|
||||
}
|
||||
|
||||
// Check for updates and print a notification message
|
||||
func checkUpdate(mode string) {
|
||||
updateURL := minioReleaseInfoURL
|
||||
@@ -502,8 +487,7 @@ func parsEnvEntry(envEntry string) (envKV, error) {
|
||||
Skip: true,
|
||||
}, nil
|
||||
}
|
||||
const envSeparator = "="
|
||||
envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), envSeparator, 2)
|
||||
envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), config.EnvSeparator, 2)
|
||||
if len(envTokens) != 2 {
|
||||
return envKV{}, fmt.Errorf("envEntry malformed; %s, expected to be of form 'KEY=value'", envEntry)
|
||||
}
|
||||
@@ -529,7 +513,7 @@ func parsEnvEntry(envEntry string) (envKV, error) {
|
||||
// the environment values from a file, in the form "key, value".
|
||||
// in a structured form.
|
||||
func minioEnvironFromFile(envConfigFile string) ([]envKV, error) {
|
||||
f, err := os.Open(envConfigFile)
|
||||
f, err := Open(envConfigFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -559,7 +543,7 @@ func readFromSecret(sp string) (string, error) {
|
||||
if isFile(pathJoin("/run/secrets/", sp)) {
|
||||
sp = pathJoin("/run/secrets/", sp)
|
||||
}
|
||||
credBuf, err := ioutil.ReadFile(sp)
|
||||
credBuf, err := os.ReadFile(sp)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) { // ignore if file doesn't exist.
|
||||
return "", nil
|
||||
@@ -824,43 +808,56 @@ func handleKMSConfig() {
|
||||
endpoints = append(endpoints, strings.Join(lbls, ""))
|
||||
}
|
||||
}
|
||||
// Manually load the certificate and private key into memory.
|
||||
// We need to check whether the private key is encrypted, and
|
||||
// if so, decrypt it using the user-provided password.
|
||||
certBytes, err := os.ReadFile(env.Get(config.EnvKESClientCert, ""))
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to load KES client certificate as specified by the shell environment")
|
||||
}
|
||||
keyBytes, err := os.ReadFile(env.Get(config.EnvKESClientKey, ""))
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to load KES client private key as specified by the shell environment")
|
||||
}
|
||||
privateKeyPEM, rest := pem.Decode(bytes.TrimSpace(keyBytes))
|
||||
if len(rest) != 0 {
|
||||
logger.Fatal(errors.New("private key contains additional data"), "Unable to load KES client private key as specified by the shell environment")
|
||||
}
|
||||
if x509.IsEncryptedPEMBlock(privateKeyPEM) {
|
||||
keyBytes, err = x509.DecryptPEMBlock(privateKeyPEM, []byte(env.Get(config.EnvKESClientPassword, "")))
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to decrypt KES client private key as specified by the shell environment")
|
||||
}
|
||||
keyBytes = pem.EncodeToMemory(&pem.Block{Type: privateKeyPEM.Type, Bytes: keyBytes})
|
||||
}
|
||||
certificate, err := tls.X509KeyPair(certBytes, keyBytes)
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to load KES client certificate as specified by the shell environment")
|
||||
}
|
||||
rootCAs, err := certs.GetRootCAs(env.Get(config.EnvKESServerCA, globalCertsCADir.Get()))
|
||||
if err != nil {
|
||||
logger.Fatal(err, fmt.Sprintf("Unable to load X.509 root CAs for KES from %q", env.Get(config.EnvKESServerCA, globalCertsCADir.Get())))
|
||||
}
|
||||
|
||||
loadX509KeyPair := func(certFile, keyFile string) (tls.Certificate, error) {
|
||||
// Manually load the certificate and private key into memory.
|
||||
// We need to check whether the private key is encrypted, and
|
||||
// if so, decrypt it using the user-provided password.
|
||||
certBytes, err := os.ReadFile(certFile)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
|
||||
}
|
||||
keyBytes, err := os.ReadFile(keyFile)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, fmt.Errorf("Unable to load KES client private key as specified by the shell environment: %v", err)
|
||||
}
|
||||
privateKeyPEM, rest := pem.Decode(bytes.TrimSpace(keyBytes))
|
||||
if len(rest) != 0 {
|
||||
return tls.Certificate{}, errors.New("Unable to load KES client private key as specified by the shell environment: private key contains additional data")
|
||||
}
|
||||
if x509.IsEncryptedPEMBlock(privateKeyPEM) {
|
||||
keyBytes, err = x509.DecryptPEMBlock(privateKeyPEM, []byte(env.Get(config.EnvKESClientPassword, "")))
|
||||
if err != nil {
|
||||
return tls.Certificate{}, fmt.Errorf("Unable to decrypt KES client private key as specified by the shell environment: %v", err)
|
||||
}
|
||||
keyBytes = pem.EncodeToMemory(&pem.Block{Type: privateKeyPEM.Type, Bytes: keyBytes})
|
||||
}
|
||||
certificate, err := tls.X509KeyPair(certBytes, keyBytes)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
|
||||
}
|
||||
return certificate, nil
|
||||
}
|
||||
|
||||
reloadCertEvents := make(chan tls.Certificate, 1)
|
||||
certificate, err := certs.NewCertificate(env.Get(config.EnvKESClientCert, ""), env.Get(config.EnvKESClientKey, ""), loadX509KeyPair)
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Failed to load KES client certificate")
|
||||
}
|
||||
certificate.Watch(context.Background(), 15*time.Minute, syscall.SIGHUP)
|
||||
certificate.Notify(reloadCertEvents)
|
||||
|
||||
defaultKeyID := env.Get(config.EnvKESKeyName, "")
|
||||
KMS, err := kms.NewWithConfig(kms.Config{
|
||||
Endpoints: endpoints,
|
||||
DefaultKeyID: defaultKeyID,
|
||||
Certificate: certificate,
|
||||
RootCAs: rootCAs,
|
||||
Endpoints: endpoints,
|
||||
DefaultKeyID: defaultKeyID,
|
||||
Certificate: certificate,
|
||||
ReloadCertEvents: reloadCertEvents,
|
||||
RootCAs: rootCAs,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
|
||||
@@ -910,7 +907,7 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
|
||||
// Therefore, we read all filenames in the cert directory and check
|
||||
// for each directory whether it contains a public.crt and private.key.
|
||||
// If so, we try to add it to certificate manager.
|
||||
root, err := os.Open(globalCertsDir.Get())
|
||||
root, err := Open(globalCertsDir.Get())
|
||||
if err != nil {
|
||||
return nil, nil, false, err
|
||||
}
|
||||
@@ -929,7 +926,7 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
|
||||
continue
|
||||
}
|
||||
if file.Mode()&os.ModeSymlink == os.ModeSymlink {
|
||||
file, err = os.Stat(filepath.Join(root.Name(), file.Name()))
|
||||
file, err = Stat(filepath.Join(root.Name(), file.Name()))
|
||||
if err != nil {
|
||||
// not accessible ignore
|
||||
continue
|
||||
|
||||
@@ -19,7 +19,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
@@ -45,7 +45,7 @@ func Test_readFromSecret(t *testing.T) {
|
||||
for _, testCase := range testCases {
|
||||
testCase := testCase
|
||||
t.Run("", func(t *testing.T) {
|
||||
tmpfile, err := ioutil.TempFile("", "testfile")
|
||||
tmpfile, err := os.CreateTemp("", "testfile")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
@@ -157,7 +157,7 @@ MINIO_ROOT_PASSWORD=minio123`,
|
||||
for _, testCase := range testCases {
|
||||
testCase := testCase
|
||||
t.Run("", func(t *testing.T) {
|
||||
tmpfile, err := ioutil.TempFile("", "testfile")
|
||||
tmpfile, err := os.CreateTemp("", "testfile")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
+16
-8
@@ -21,7 +21,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/minio/minio/internal/hash"
|
||||
@@ -29,10 +29,14 @@ import (
|
||||
|
||||
var errConfigNotFound = errors.New("config file not found")
|
||||
|
||||
func readConfigWithMetadata(ctx context.Context, store objectIO, configFile string) ([]byte, ObjectInfo, error) {
|
||||
r, err := store.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, readLock, ObjectOptions{})
|
||||
func readConfigWithMetadata(ctx context.Context, store objectIO, configFile string, opts ObjectOptions) ([]byte, ObjectInfo, error) {
|
||||
lockType := readLock
|
||||
if opts.NoLock {
|
||||
lockType = noLock // erasureObjects.GetObjectNInfo honors lockType argument but not opts.NoLock.
|
||||
}
|
||||
|
||||
r, err := store.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, lockType, opts)
|
||||
if err != nil {
|
||||
// Treat object not found as config not found.
|
||||
if isErrObjectNotFound(err) {
|
||||
return nil, ObjectInfo{}, errConfigNotFound
|
||||
}
|
||||
@@ -41,7 +45,7 @@ func readConfigWithMetadata(ctx context.Context, store objectIO, configFile stri
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
buf, err := ioutil.ReadAll(r)
|
||||
buf, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, ObjectInfo{}, err
|
||||
}
|
||||
@@ -52,7 +56,7 @@ func readConfigWithMetadata(ctx context.Context, store objectIO, configFile stri
|
||||
}
|
||||
|
||||
func readConfig(ctx context.Context, store objectIO, configFile string) ([]byte, error) {
|
||||
buf, _, err := readConfigWithMetadata(ctx, store, configFile)
|
||||
buf, _, err := readConfigWithMetadata(ctx, store, configFile, ObjectOptions{})
|
||||
return buf, err
|
||||
}
|
||||
|
||||
@@ -70,16 +74,20 @@ func deleteConfig(ctx context.Context, objAPI objectDeleter, configFile string)
|
||||
return err
|
||||
}
|
||||
|
||||
func saveConfig(ctx context.Context, store objectIO, configFile string, data []byte) error {
|
||||
func saveConfigWithOpts(ctx context.Context, store objectIO, configFile string, data []byte, opts ObjectOptions) error {
|
||||
hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = store.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader), ObjectOptions{MaxParity: true})
|
||||
_, err = store.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader), opts)
|
||||
return err
|
||||
}
|
||||
|
||||
func saveConfig(ctx context.Context, store objectIO, configFile string, data []byte) error {
|
||||
return saveConfigWithOpts(ctx, store, configFile, data, ObjectOptions{MaxParity: true})
|
||||
}
|
||||
|
||||
func checkConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
|
||||
if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
|
||||
// Treat object not found as config not found.
|
||||
|
||||
+89
-162
@@ -25,6 +25,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/config/api"
|
||||
"github.com/minio/minio/internal/config/cache"
|
||||
@@ -68,7 +69,6 @@ func initHelp() {
|
||||
config.LoggerWebhookSubSys: logger.DefaultLoggerWebhookKVS,
|
||||
config.AuditWebhookSubSys: logger.DefaultAuditWebhookKVS,
|
||||
config.AuditKafkaSubSys: logger.DefaultAuditKafkaKVS,
|
||||
config.HealSubSys: heal.DefaultKVS,
|
||||
config.ScannerSubSys: scanner.DefaultKVS,
|
||||
config.SubnetSubSys: subnet.DefaultKVS,
|
||||
config.CallhomeSubSys: callhome.DefaultKVS,
|
||||
@@ -78,6 +78,7 @@ func initHelp() {
|
||||
}
|
||||
if globalIsErasure {
|
||||
kvs[config.StorageClassSubSys] = storageclass.DefaultKVS
|
||||
kvs[config.HealSubSys] = heal.DefaultKVS
|
||||
}
|
||||
config.RegisterDefaultKVS(kvs)
|
||||
|
||||
@@ -124,10 +125,6 @@ func initHelp() {
|
||||
Key: config.APISubSys,
|
||||
Description: "manage global HTTP API call specific features, such as throttling, authentication types, etc.",
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: config.HealSubSys,
|
||||
Description: "manage object healing frequency and bitrot verification checks",
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: config.ScannerSubSys,
|
||||
Description: "manage namespace scanning for usage calculation, lifecycle, healing and more",
|
||||
@@ -212,12 +209,13 @@ func initHelp() {
|
||||
}
|
||||
|
||||
if globalIsErasure {
|
||||
helpSubSys = append(helpSubSys, config.HelpKV{})
|
||||
copy(helpSubSys[2:], helpSubSys[1:])
|
||||
helpSubSys[1] = config.HelpKV{
|
||||
helpSubSys = append(helpSubSys, config.HelpKV{
|
||||
Key: config.StorageClassSubSys,
|
||||
Description: "define object level redundancy",
|
||||
}
|
||||
}, config.HelpKV{
|
||||
Key: config.HealSubSys,
|
||||
Description: "manage object healing frequency and bitrot verification checks",
|
||||
})
|
||||
}
|
||||
|
||||
helpMap := map[string]config.HelpKVS{
|
||||
@@ -279,10 +277,6 @@ var (
|
||||
|
||||
func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) error {
|
||||
switch subSys {
|
||||
case config.CredentialsSubSys:
|
||||
if _, err := config.LookupCreds(s[config.CredentialsSubSys][config.Default]); err != nil {
|
||||
return err
|
||||
}
|
||||
case config.SiteSubSys:
|
||||
if _, err := config.LookupSite(s[config.SiteSubSys][config.Default], s[config.RegionSubSys][config.Default]); err != nil {
|
||||
return err
|
||||
@@ -292,14 +286,12 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
|
||||
return err
|
||||
}
|
||||
case config.StorageClassSubSys:
|
||||
if globalIsErasure {
|
||||
if objAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
for _, setDriveCount := range objAPI.SetDriveCounts() {
|
||||
if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount); err != nil {
|
||||
return err
|
||||
}
|
||||
if objAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
for _, setDriveCount := range objAPI.SetDriveCounts() {
|
||||
if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case config.CacheSubSys:
|
||||
@@ -339,16 +331,16 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
|
||||
}
|
||||
case config.IdentityOpenIDSubSys:
|
||||
if _, err := openid.LookupConfig(s,
|
||||
NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
|
||||
NewHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
|
||||
return err
|
||||
}
|
||||
case config.IdentityLDAPSubSys:
|
||||
cfg, err := xldap.Lookup(s[config.IdentityLDAPSubSys][config.Default], globalRootCAs)
|
||||
cfg, err := xldap.Lookup(s, globalRootCAs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.Enabled {
|
||||
conn, cerr := cfg.Connect()
|
||||
if cfg.Enabled() {
|
||||
conn, cerr := cfg.LDAP.Connect()
|
||||
if cerr != nil {
|
||||
return cerr
|
||||
}
|
||||
@@ -360,7 +352,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
|
||||
}
|
||||
case config.IdentityPluginSubSys:
|
||||
if _, err := idplugin.LookupConfig(s[config.IdentityPluginSubSys][config.Default],
|
||||
NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
|
||||
NewHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
|
||||
return err
|
||||
}
|
||||
case config.SubnetSubSys:
|
||||
@@ -368,9 +360,14 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
|
||||
return err
|
||||
}
|
||||
case config.CallhomeSubSys:
|
||||
if _, err := callhome.LookupConfig(s[config.CallhomeSubSys][config.Default]); err != nil {
|
||||
cfg, err := callhome.LookupConfig(s[config.CallhomeSubSys][config.Default])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// callhome cannot be enabled if license is not registered yet, throw an error.
|
||||
if cfg.Enabled() && !globalSubnetConfig.Registered() {
|
||||
return errors.New("Deployment is not registered with SUBNET. Please register the deployment via 'mc license register ALIAS'")
|
||||
}
|
||||
case config.PolicyOPASubSys:
|
||||
// In case legacy OPA config is being set, we treat it as if the
|
||||
// AuthZPlugin is being set.
|
||||
@@ -378,12 +375,12 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
|
||||
fallthrough
|
||||
case config.PolicyPluginSubSys:
|
||||
if ppargs, err := polplugin.LookupConfig(s[config.PolicyPluginSubSys][config.Default],
|
||||
NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
|
||||
NewHTTPTransport(), xhttp.DrainBody); err != nil {
|
||||
return err
|
||||
} else if ppargs.URL == nil {
|
||||
// Check if legacy opa is configured.
|
||||
if _, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
|
||||
NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
|
||||
NewHTTPTransport(), xhttp.DrainBody); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -396,7 +393,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
|
||||
}
|
||||
|
||||
if config.NotifySubSystems.Contains(subSys) {
|
||||
if err := notify.TestSubSysNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), globalNotificationSys.ConfiguredTargetIDs(), subSys); err != nil {
|
||||
if err := notify.TestSubSysNotificationTargets(GlobalContext, s, subSys, NewHTTPTransport()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -442,51 +439,33 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
|
||||
|
||||
dnsURL, dnsUser, dnsPass, err := env.LookupEnv(config.EnvDNSWebhook)
|
||||
if err != nil {
|
||||
if globalIsGateway {
|
||||
logger.FatalIf(err, "Unable to initialize remote webhook DNS config")
|
||||
} else {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
|
||||
}
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
|
||||
}
|
||||
if err == nil && dnsURL != "" {
|
||||
globalDNSConfig, err = dns.NewOperatorDNS(dnsURL,
|
||||
dns.Authentication(dnsUser, dnsPass),
|
||||
dns.RootCAs(globalRootCAs))
|
||||
if err != nil {
|
||||
if globalIsGateway {
|
||||
logger.FatalIf(err, "Unable to initialize remote webhook DNS config")
|
||||
} else {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
|
||||
}
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
|
||||
if err != nil {
|
||||
if globalIsGateway {
|
||||
logger.FatalIf(err, "Unable to initialize etcd config")
|
||||
} else {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
|
||||
}
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
|
||||
}
|
||||
|
||||
if etcdCfg.Enabled {
|
||||
if globalEtcdClient == nil {
|
||||
globalEtcdClient, err = etcd.New(etcdCfg)
|
||||
if err != nil {
|
||||
if globalIsGateway {
|
||||
logger.FatalIf(err, "Unable to initialize etcd config")
|
||||
} else {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
|
||||
}
|
||||
}
|
||||
globalEtcdClient, err = etcd.New(etcdCfg)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
|
||||
}
|
||||
|
||||
if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
|
||||
if globalDNSConfig != nil {
|
||||
// if global DNS is already configured, indicate with a warning, incase
|
||||
// users are confused.
|
||||
logger.LogIf(ctx, fmt.Errorf("DNS store is already configured with %s, not using etcd for DNS store", globalDNSConfig))
|
||||
logger.LogIf(ctx, fmt.Errorf("DNS store is already configured with %s, etcd is not used for DNS store", globalDNSConfig))
|
||||
} else {
|
||||
globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
|
||||
dns.DomainNames(globalDomainNames),
|
||||
@@ -495,12 +474,8 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
|
||||
dns.CoreDNSPath(etcdCfg.CoreDNSPath),
|
||||
)
|
||||
if err != nil {
|
||||
if globalIsGateway {
|
||||
logger.FatalIf(err, "Unable to initialize DNS config")
|
||||
} else {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
|
||||
globalDomainNames, err))
|
||||
}
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
|
||||
globalDomainNames, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -518,23 +493,9 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
|
||||
logger.LogIf(ctx, fmt.Errorf("Invalid site configuration: %w", err))
|
||||
}
|
||||
|
||||
apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
|
||||
}
|
||||
|
||||
// Initialize remote instance transport once.
|
||||
getRemoteInstanceTransportOnce.Do(func() {
|
||||
getRemoteInstanceTransport = newGatewayHTTPTransport(apiConfig.RemoteTransportDeadline)
|
||||
})
|
||||
|
||||
globalCacheConfig, err = cache.LookupConfig(s[config.CacheSubSys][config.Default])
|
||||
if err != nil {
|
||||
if globalIsGateway {
|
||||
logger.FatalIf(err, "Unable to setup cache")
|
||||
} else {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
|
||||
}
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
|
||||
}
|
||||
|
||||
if globalCacheConfig.Enabled {
|
||||
@@ -560,92 +521,46 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
|
||||
logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
|
||||
}
|
||||
|
||||
globalOpenIDConfig, err = openid.LookupConfig(s,
|
||||
NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err))
|
||||
}
|
||||
transport := NewHTTPTransport()
|
||||
|
||||
globalLDAPConfig, err = xldap.Lookup(s[config.IdentityLDAPSubSys][config.Default],
|
||||
globalRootCAs)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to parse LDAP configuration: %w", err))
|
||||
}
|
||||
|
||||
authNPluginCfg, err := idplugin.LookupConfig(s[config.IdentityPluginSubSys][config.Default],
|
||||
NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthNPlugin: %w", err))
|
||||
}
|
||||
globalAuthNPlugin = idplugin.New(authNPluginCfg)
|
||||
|
||||
authZPluginCfg, err := polplugin.LookupConfig(s[config.PolicyPluginSubSys][config.Default],
|
||||
NewGatewayHTTPTransport(), xhttp.DrainBody)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin: %w", err))
|
||||
}
|
||||
if authZPluginCfg.URL == nil {
|
||||
opaCfg, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
|
||||
NewGatewayHTTPTransport(), xhttp.DrainBody)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin from legacy OPA config: %w", err))
|
||||
} else {
|
||||
authZPluginCfg.URL = opaCfg.URL
|
||||
authZPluginCfg.AuthToken = opaCfg.AuthToken
|
||||
authZPluginCfg.Transport = opaCfg.Transport
|
||||
authZPluginCfg.CloseRespFn = opaCfg.CloseRespFn
|
||||
}
|
||||
}
|
||||
|
||||
setGlobalAuthZPlugin(polplugin.New(authZPluginCfg))
|
||||
|
||||
globalSubnetConfig, err = subnet.LookupConfig(s[config.SubnetSubSys][config.Default], globalProxyTransport)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
|
||||
}
|
||||
|
||||
globalConfigTargetList, err = notify.GetNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), false)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
|
||||
}
|
||||
|
||||
globalEnvTargetList, err = notify.GetNotificationTargets(GlobalContext, newServerConfig(), NewGatewayHTTPTransport(), true)
|
||||
globalConfigTargetList, err = notify.FetchEnabledTargets(GlobalContext, s, transport)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
|
||||
}
|
||||
|
||||
// Apply dynamic config values
|
||||
if err := applyDynamicConfig(ctx, objAPI, s); err != nil {
|
||||
if globalIsGateway {
|
||||
logger.FatalIf(err, "Unable to initialize dynamic configuration")
|
||||
} else {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}
|
||||
|
||||
func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s config.Config, subSys string) error {
|
||||
if objAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
|
||||
setDriveCounts := objAPI.SetDriveCounts()
|
||||
switch subSys {
|
||||
case config.APISubSys:
|
||||
apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
|
||||
}
|
||||
var setDriveCounts []int
|
||||
if objAPI != nil {
|
||||
setDriveCounts = objAPI.SetDriveCounts()
|
||||
}
|
||||
|
||||
globalAPIConfig.init(apiConfig, setDriveCounts)
|
||||
|
||||
// Initialize remote instance transport once.
|
||||
getRemoteInstanceTransportOnce.Do(func() {
|
||||
getRemoteInstanceTransport = newHTTPTransport(apiConfig.RemoteTransportDeadline)
|
||||
})
|
||||
case config.CompressionSubSys:
|
||||
cmpCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
|
||||
if err != nil {
|
||||
return fmt.Errorf("Unable to setup Compression: %w", err)
|
||||
}
|
||||
// Validate if the object layer supports compression.
|
||||
if objAPI != nil {
|
||||
if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
|
||||
return fmt.Errorf("Backend does not support compression")
|
||||
}
|
||||
if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
|
||||
return fmt.Errorf("Backend does not support compression")
|
||||
}
|
||||
globalCompressConfigMu.Lock()
|
||||
globalCompressConfig = cmpCfg
|
||||
@@ -674,12 +589,11 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
|
||||
if l.Enabled {
|
||||
l.LogOnce = logger.LogOnceConsoleIf
|
||||
l.UserAgent = userAgent
|
||||
l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
|
||||
l.Transport = NewHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
|
||||
loggerCfg.HTTP[n] = l
|
||||
}
|
||||
}
|
||||
err = logger.UpdateSystemTargets(loggerCfg)
|
||||
if err != nil {
|
||||
if err = logger.UpdateSystemTargets(loggerCfg); err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %w", err))
|
||||
}
|
||||
case config.AuditWebhookSubSys:
|
||||
@@ -692,13 +606,12 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
|
||||
if l.Enabled {
|
||||
l.LogOnce = logger.LogOnceConsoleIf
|
||||
l.UserAgent = userAgent
|
||||
l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
|
||||
l.Transport = NewHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
|
||||
loggerCfg.AuditWebhook[n] = l
|
||||
}
|
||||
}
|
||||
|
||||
err = logger.UpdateAuditWebhookTargets(loggerCfg)
|
||||
if err != nil {
|
||||
if err = logger.UpdateAuditWebhookTargets(loggerCfg); err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %w", err))
|
||||
}
|
||||
case config.AuditKafkaSubSys:
|
||||
@@ -712,33 +625,40 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
|
||||
loggerCfg.AuditKafka[n] = l
|
||||
}
|
||||
}
|
||||
err = logger.UpdateAuditKafkaTargets(loggerCfg)
|
||||
if err != nil {
|
||||
if err = logger.UpdateAuditKafkaTargets(loggerCfg); err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to update audit kafka targets: %w", err))
|
||||
}
|
||||
case config.StorageClassSubSys:
|
||||
if globalIsErasure && objAPI != nil {
|
||||
setDriveCounts := objAPI.SetDriveCounts()
|
||||
for i, setDriveCount := range setDriveCounts {
|
||||
sc, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
|
||||
break
|
||||
}
|
||||
// if we validated all setDriveCounts and it was successful
|
||||
// proceed to store the correct storage class globally.
|
||||
if i == len(setDriveCounts)-1 {
|
||||
globalStorageClass.Update(sc)
|
||||
}
|
||||
for i, setDriveCount := range setDriveCounts {
|
||||
sc, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
|
||||
break
|
||||
}
|
||||
// if we validated all setDriveCounts and it was successful
|
||||
// proceed to store the correct storage class globally.
|
||||
if i == len(setDriveCounts)-1 {
|
||||
globalStorageClass.Update(sc)
|
||||
}
|
||||
}
|
||||
case config.SubnetSubSys:
|
||||
subnetConfig, err := subnet.LookupConfig(s[config.SubnetSubSys][config.Default], globalProxyTransport)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
|
||||
} else {
|
||||
globalSubnetConfig.Update(subnetConfig)
|
||||
globalSubnetConfig.ApplyEnv() // update environment settings for Console UI
|
||||
}
|
||||
case config.CallhomeSubSys:
|
||||
callhomeCfg, err := callhome.LookupConfig(s[config.CallhomeSubSys][config.Default])
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to load callhome config: %w", err))
|
||||
} else {
|
||||
globalCallhomeConfig = callhomeCfg
|
||||
updateCallhomeParams(ctx, objAPI)
|
||||
enable := callhomeCfg.Enable && !globalCallhomeConfig.Enabled()
|
||||
globalCallhomeConfig.Update(callhomeCfg)
|
||||
if enable {
|
||||
initCallhome(ctx, objAPI)
|
||||
}
|
||||
}
|
||||
}
|
||||
globalServerConfigMu.Lock()
|
||||
@@ -846,13 +766,20 @@ func newSrvConfig(objAPI ObjectLayer) error {
|
||||
// Initialize server config.
|
||||
srvCfg := newServerConfig()
|
||||
|
||||
if globalActiveCred.IsValid() && !globalActiveCred.Equal(auth.DefaultCredentials) {
|
||||
kvs := srvCfg[config.CredentialsSubSys][config.Default]
|
||||
kvs.Set(config.AccessKey, globalActiveCred.AccessKey)
|
||||
kvs.Set(config.SecretKey, globalActiveCred.SecretKey)
|
||||
srvCfg[config.CredentialsSubSys][config.Default] = kvs
|
||||
}
|
||||
|
||||
// hold the mutex lock before a new config is assigned.
|
||||
globalServerConfigMu.Lock()
|
||||
globalServerConfig = srvCfg
|
||||
globalServerConfigMu.Unlock()
|
||||
|
||||
// Save config into file.
|
||||
return saveServerConfig(GlobalContext, objAPI, globalServerConfig)
|
||||
return saveServerConfig(GlobalContext, objAPI, srvCfg)
|
||||
}
|
||||
|
||||
func getValidConfig(objAPI ObjectLayer) (config.Config, error) {
|
||||
|
||||
@@ -26,7 +26,10 @@ import (
|
||||
)
|
||||
|
||||
func TestServerConfig(t *testing.T) {
|
||||
objLayer, fsDir, err := prepareFS()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
objLayer, fsDir, err := prepareFS(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
+47
-26
@@ -69,6 +69,12 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
|
||||
return err
|
||||
}
|
||||
|
||||
// If backend doesn't have this file means we have already
|
||||
// attempted then migration
|
||||
if !encrypted {
|
||||
return nil
|
||||
}
|
||||
|
||||
if encrypted && GlobalKMS != nil {
|
||||
stat, err := GlobalKMS.Stat(ctx)
|
||||
if err != nil {
|
||||
@@ -109,6 +115,8 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
|
||||
return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
|
||||
}
|
||||
}
|
||||
data = pdata
|
||||
@@ -131,6 +139,7 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
|
||||
if encrypted && GlobalKMS != nil {
|
||||
logger.Info("Migration of encrypted IAM config data completed. All data is now encrypted on etcd.")
|
||||
}
|
||||
|
||||
return deleteKeyEtcd(ctx, client, backendEncryptedFile)
|
||||
}
|
||||
|
||||
@@ -146,44 +155,56 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, encrypted bool) error {
|
||||
logger.Info(fmt.Sprintf("Attempting to re-encrypt config, IAM users and policies on MinIO with %q (%s)", stat.DefaultKey, stat.Name))
|
||||
}
|
||||
|
||||
var marker string
|
||||
for {
|
||||
res, err := objAPI.ListObjects(GlobalContext, minioMetaBucket, minioConfigPrefix, marker, "", maxObjectList)
|
||||
results := make(chan ObjectInfo)
|
||||
if err := objAPI.Walk(GlobalContext, minioMetaBucket, minioConfigPrefix, results, ObjectOptions{}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for obj := range results {
|
||||
data, err := readConfig(GlobalContext, objAPI, obj.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, obj := range res.Objects {
|
||||
data, err := readConfig(GlobalContext, objAPI, obj.Name)
|
||||
|
||||
if !utf8.Valid(data) {
|
||||
pdata, err := madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
|
||||
if err != nil {
|
||||
if GlobalKMS != nil {
|
||||
pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
|
||||
minioMetaBucket: path.Join(minioMetaBucket, obj.Name),
|
||||
})
|
||||
if err != nil {
|
||||
pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
|
||||
minioMetaBucket: obj.Name,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
|
||||
}
|
||||
}
|
||||
data = pdata
|
||||
}
|
||||
|
||||
if GlobalKMS != nil {
|
||||
data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
|
||||
obj.Bucket: path.Join(obj.Bucket, obj.Name),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !utf8.Valid(data) {
|
||||
data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return fmt.Errorf("Decrypting config failed %w, possibly credentials are incorrect", err)
|
||||
}
|
||||
}
|
||||
if GlobalKMS != nil {
|
||||
data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
|
||||
obj.Bucket: path.Join(obj.Bucket, obj.Name),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err = saveConfig(GlobalContext, objAPI, obj.Name, data); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if !res.IsTruncated {
|
||||
break
|
||||
if err = saveConfig(GlobalContext, objAPI, obj.Name, data); err != nil {
|
||||
return err
|
||||
}
|
||||
marker = res.NextMarker
|
||||
}
|
||||
|
||||
if encrypted && GlobalKMS != nil {
|
||||
logger.Info("Migration of encrypted config data completed. All config data is now encrypted.")
|
||||
}
|
||||
|
||||
return deleteConfig(GlobalContext, objAPI, backendEncryptedFile)
|
||||
}
|
||||
|
||||
+21
-38
@@ -18,8 +18,8 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
@@ -28,7 +28,10 @@ import (
|
||||
|
||||
// Test if config v1 is purged
|
||||
func TestServerConfigMigrateV1(t *testing.T) {
|
||||
objLayer, fsDir, err := prepareFS()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
objLayer, fsDir, err := prepareFS(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -37,11 +40,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Init Test config failed")
|
||||
}
|
||||
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(rootPath)
|
||||
rootPath := t.TempDir()
|
||||
globalConfigDir = &ConfigDir{path: rootPath}
|
||||
|
||||
globalObjLayerMutex.Lock()
|
||||
@@ -51,7 +50,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
|
||||
// Create a V1 config json file and store it
|
||||
configJSON := "{ \"version\":\"1\", \"accessKeyId\":\"abcde\", \"secretAccessKey\":\"abcdefgh\"}"
|
||||
configPath := rootPath + "/fsUsers.json"
|
||||
if err := ioutil.WriteFile(configPath, []byte(configJSON), 0o644); err != nil {
|
||||
if err := os.WriteFile(configPath, []byte(configJSON), 0o644); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
@@ -74,13 +73,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
|
||||
// Test if all migrate code returns nil when config file does not
|
||||
// exist
|
||||
func TestServerConfigMigrateInexistentConfig(t *testing.T) {
|
||||
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(rootPath)
|
||||
|
||||
globalConfigDir = &ConfigDir{path: rootPath}
|
||||
globalConfigDir = &ConfigDir{path: t.TempDir()}
|
||||
|
||||
if err := migrateV2ToV3(); err != nil {
|
||||
t.Fatal("migrate v2 to v3 should succeed when no config file is found")
|
||||
@@ -164,15 +157,13 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
|
||||
|
||||
// Test if a config migration from v2 to v33 is successfully done
|
||||
func TestServerConfigMigrateV2toV33(t *testing.T) {
|
||||
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(rootPath)
|
||||
rootPath := t.TempDir()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
globalConfigDir = &ConfigDir{path: rootPath}
|
||||
|
||||
objLayer, fsDir, err := prepareFS()
|
||||
objLayer, fsDir, err := prepareFS(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -181,7 +172,7 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
|
||||
configPath := rootPath + SlashSeparator + minioConfigFile
|
||||
|
||||
// Create a corrupted config file
|
||||
if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\","), 0o644); err != nil {
|
||||
if err := os.WriteFile(configPath, []byte("{ \"version\":\"2\","), 0o644); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
// Fire a migrateConfig()
|
||||
@@ -194,7 +185,7 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
|
||||
|
||||
// Create a V2 config json file and store it
|
||||
configJSON := "{ \"version\":\"2\", \"credentials\": {\"accessKeyId\":\"" + accessKey + "\", \"secretAccessKey\":\"" + secretKey + "\", \"region\":\"us-east-1\"}, \"mongoLogger\":{\"addr\":\"127.0.0.1:3543\", \"db\":\"foodb\", \"collection\":\"foo\"}, \"syslogLogger\":{\"network\":\"127.0.0.1:543\", \"addr\":\"addr\"}, \"fileLogger\":{\"filename\":\"log.out\"}}"
|
||||
if err := ioutil.WriteFile(configPath, []byte(configJSON), 0o644); err != nil {
|
||||
if err := os.WriteFile(configPath, []byte(configJSON), 0o644); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
@@ -234,17 +225,13 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
|
||||
|
||||
// Test if all migrate code returns error with corrupted config files
|
||||
func TestServerConfigMigrateFaultyConfig(t *testing.T) {
|
||||
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(rootPath)
|
||||
rootPath := t.TempDir()
|
||||
|
||||
globalConfigDir = &ConfigDir{path: rootPath}
|
||||
configPath := rootPath + SlashSeparator + minioConfigFile
|
||||
|
||||
// Create a corrupted config file
|
||||
if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"test\":"), 0o644); err != nil {
|
||||
if err := os.WriteFile(configPath, []byte("{ \"version\":\"2\", \"test\":"), 0o644); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
@@ -331,35 +318,31 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
|
||||
|
||||
// Test if all migrate code returns error with corrupted config files
|
||||
func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
|
||||
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(rootPath)
|
||||
rootPath := t.TempDir()
|
||||
|
||||
globalConfigDir = &ConfigDir{path: rootPath}
|
||||
configPath := rootPath + SlashSeparator + minioConfigFile
|
||||
|
||||
for i := 3; i <= 17; i++ {
|
||||
// Create a corrupted config file
|
||||
if err = ioutil.WriteFile(configPath, []byte(fmt.Sprintf("{ \"version\":\"%d\", \"credential\": { \"accessKey\": 1 } }", i)),
|
||||
if err := os.WriteFile(configPath, []byte(fmt.Sprintf("{ \"version\":\"%d\", \"credential\": { \"accessKey\": 1 } }", i)),
|
||||
0o644); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
// Test different migrate versions and be sure they are returning an error
|
||||
if err = migrateConfig(); err == nil {
|
||||
if err := migrateConfig(); err == nil {
|
||||
t.Fatal("migrateConfig() should fail with a corrupted json")
|
||||
}
|
||||
}
|
||||
|
||||
// Create a corrupted config file for version '2'.
|
||||
if err = ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"credentials\": { \"accessKeyId\": 1 } }"), 0o644); err != nil {
|
||||
if err := os.WriteFile(configPath, []byte("{ \"version\":\"2\", \"credentials\": { \"accessKeyId\": 1 } }"), 0o644); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
// Test different migrate versions and be sure they are returning an error
|
||||
if err = migrateConfig(); err == nil {
|
||||
if err := migrateConfig(); err == nil {
|
||||
t.Fatal("migrateConfig() should fail with a corrupted json")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -855,5 +855,5 @@ type serverConfigV33 struct {
|
||||
// Add new external policy enforcements here.
|
||||
} `json:"policy"`
|
||||
|
||||
LDAPServerConfig xldap.Config `json:"ldapserverconfig"`
|
||||
LDAPServerConfig xldap.LegacyConfig `json:"ldapserverconfig"`
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ const defaultLogBufferCount = 10000
|
||||
// HTTPConsoleLoggerSys holds global console logger state
|
||||
type HTTPConsoleLoggerSys struct {
|
||||
sync.RWMutex
|
||||
pubsub *pubsub.PubSub
|
||||
pubsub *pubsub.PubSub[log.Info, madmin.LogMask]
|
||||
console *console.Target
|
||||
nodeName string
|
||||
logBuf *ring.Ring
|
||||
@@ -46,9 +46,8 @@ type HTTPConsoleLoggerSys struct {
|
||||
// NewConsoleLogger - creates new HTTPConsoleLoggerSys with all nodes subscribed to
|
||||
// the console logging pub sub system
|
||||
func NewConsoleLogger(ctx context.Context) *HTTPConsoleLoggerSys {
|
||||
ps := pubsub.New(8)
|
||||
return &HTTPConsoleLoggerSys{
|
||||
pubsub: ps,
|
||||
pubsub: pubsub.New[log.Info, madmin.LogMask](8),
|
||||
console: console.New(),
|
||||
logBuf: ring.New(defaultLogBufferCount),
|
||||
}
|
||||
@@ -72,11 +71,11 @@ func (sys *HTTPConsoleLoggerSys) SetNodeName(nodeName string) {
|
||||
// HasLogListeners returns true if console log listeners are registered
|
||||
// for this node or peers
|
||||
func (sys *HTTPConsoleLoggerSys) HasLogListeners() bool {
|
||||
return sys != nil && sys.pubsub.NumSubscribers(madmin.LogMaskAll) > 0
|
||||
return sys != nil && sys.pubsub.Subscribers() > 0
|
||||
}
|
||||
|
||||
// Subscribe starts console logging for this node.
|
||||
func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan pubsub.Maskable, doneCh <-chan struct{}, node string, last int, logKind madmin.LogMask, filter func(entry pubsub.Maskable) bool) error {
|
||||
func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan log.Info, doneCh <-chan struct{}, node string, last int, logKind madmin.LogMask, filter func(entry log.Info) bool) error {
|
||||
// Enable console logging for remote client.
|
||||
if !sys.HasLogListeners() {
|
||||
logger.AddSystemTarget(sys)
|
||||
@@ -116,7 +115,7 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan pubsub.Maskable, doneCh <-
|
||||
}
|
||||
}
|
||||
}
|
||||
return sys.pubsub.Subscribe(pubsub.MaskFromMaskable(madmin.LogMaskAll), subCh, doneCh, filter)
|
||||
return sys.pubsub.Subscribe(madmin.LogMaskAll, subCh, doneCh, filter)
|
||||
}
|
||||
|
||||
// Init if HTTPConsoleLoggerSys is valid, always returns nil right now
|
||||
|
||||
@@ -119,13 +119,6 @@ func (p *scannerMetrics) incTime(s scannerMetric, d time.Duration) {
|
||||
}
|
||||
}
|
||||
|
||||
func (p *scannerMetrics) incNoTime(s scannerMetric) {
|
||||
atomic.AddUint64(&p.operations[s], 1)
|
||||
if s < scannerMetricLastRealtime {
|
||||
p.latency[s].add(0)
|
||||
}
|
||||
}
|
||||
|
||||
// timeILM times an ILM action.
|
||||
// lifecycle.NoneAction is ignored.
|
||||
// Use for s < scannerMetricLastRealtime
|
||||
|
||||
+48
-89
@@ -64,7 +64,11 @@ const (
|
||||
var (
|
||||
globalHealConfig heal.Config
|
||||
|
||||
dataScannerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
|
||||
dataScannerLeaderLockTimeout = newDynamicTimeoutWithOpts(dynamicTimeoutOpts{
|
||||
timeout: 30 * time.Second,
|
||||
minimum: 10 * time.Second,
|
||||
retryInterval: time.Second,
|
||||
})
|
||||
// Sleeper values are updated when config is loaded.
|
||||
scannerSleeper = newDynamicSleeper(10, 10*time.Second, true)
|
||||
scannerCycle = uatomic.NewDuration(dataScannerStartDelay)
|
||||
@@ -114,6 +118,10 @@ type backgroundHealInfo struct {
|
||||
}
|
||||
|
||||
func readBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer) backgroundHealInfo {
|
||||
if globalIsErasureSD {
|
||||
return backgroundHealInfo{}
|
||||
}
|
||||
|
||||
// Get last healing information
|
||||
buf, err := readConfig(ctx, objAPI, backgroundHealInfoPath)
|
||||
if err != nil {
|
||||
@@ -123,15 +131,17 @@ func readBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer) backgroundH
|
||||
return backgroundHealInfo{}
|
||||
}
|
||||
var info backgroundHealInfo
|
||||
err = json.Unmarshal(buf, &info)
|
||||
if err != nil {
|
||||
if err = json.Unmarshal(buf, &info); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return backgroundHealInfo{}
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
func saveBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer, info backgroundHealInfo) {
|
||||
if globalIsErasureSD {
|
||||
return
|
||||
}
|
||||
|
||||
b, err := json.Marshal(info)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
@@ -934,7 +944,10 @@ func (i *scannerItem) applyHealing(ctx context.Context, o ObjectLayer, oi Object
|
||||
ScanMode: scanMode,
|
||||
}
|
||||
res, err := o.HealObject(ctx, i.bucket, i.objectPath(), oi.VersionID, healOpts)
|
||||
if err != nil && !errors.Is(err, NotImplemented{}) {
|
||||
if err != nil {
|
||||
if errors.Is(err, NotImplemented{}) || isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||
err = nil
|
||||
}
|
||||
logger.LogIf(ctx, err)
|
||||
return 0
|
||||
}
|
||||
@@ -956,21 +969,21 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
|
||||
|
||||
versionID := oi.VersionID
|
||||
rCfg, _ := globalBucketObjectLockSys.Get(i.bucket)
|
||||
action := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, oi, false)
|
||||
lcEvt := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, oi)
|
||||
if i.debug {
|
||||
if versionID != "" {
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, action)
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, lcEvt.Action)
|
||||
} else {
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), action)
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), lcEvt.Action)
|
||||
}
|
||||
}
|
||||
defer globalScannerMetrics.timeILM(action)
|
||||
defer globalScannerMetrics.timeILM(lcEvt.Action)
|
||||
|
||||
switch action {
|
||||
switch lcEvt.Action {
|
||||
case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
|
||||
return applyLifecycleAction(action, oi), 0
|
||||
return applyLifecycleAction(lcEvt.Action, oi, ""), 0
|
||||
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
|
||||
return applyLifecycleAction(action, oi), size
|
||||
return applyLifecycleAction(lcEvt.Action, oi, lcEvt.StorageClass), size
|
||||
default:
|
||||
// No action.
|
||||
return false, size
|
||||
@@ -1102,42 +1115,42 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
|
||||
return size
|
||||
}
|
||||
|
||||
func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, obj ObjectInfo, debug bool) (action lifecycle.Action) {
|
||||
action = lc.ComputeAction(obj.ToLifecycleOpts())
|
||||
if debug {
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", action)
|
||||
func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, obj ObjectInfo) lifecycle.Event {
|
||||
event := lc.Eval(obj.ToLifecycleOpts(), time.Now().UTC())
|
||||
if serverDebugLog {
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", event.Action)
|
||||
}
|
||||
|
||||
if action == lifecycle.NoneAction {
|
||||
return action
|
||||
if event.Action == lifecycle.NoneAction {
|
||||
return event
|
||||
}
|
||||
|
||||
switch action {
|
||||
switch event.Action {
|
||||
case lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredVersionAction:
|
||||
// Defensive code, should never happen
|
||||
if obj.VersionID == "" {
|
||||
return lifecycle.NoneAction
|
||||
return lifecycle.Event{Action: lifecycle.NoneAction}
|
||||
}
|
||||
if lr.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
|
||||
if debug {
|
||||
if serverDebugLog {
|
||||
if obj.VersionID != "" {
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: %s v(%s) is locked, not deleting\n", obj.Name, obj.VersionID)
|
||||
} else {
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", obj.Name)
|
||||
}
|
||||
}
|
||||
return lifecycle.NoneAction
|
||||
return lifecycle.Event{Action: lifecycle.NoneAction}
|
||||
}
|
||||
}
|
||||
|
||||
return action
|
||||
return event
|
||||
}
|
||||
|
||||
func applyTransitionRule(obj ObjectInfo) bool {
|
||||
func applyTransitionRule(obj ObjectInfo, storageClass string) bool {
|
||||
if obj.DeleteMarker {
|
||||
return false
|
||||
}
|
||||
globalTransitionState.queueTransitionTask(obj)
|
||||
globalTransitionState.queueTransitionTask(obj, storageClass)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -1206,14 +1219,14 @@ func applyExpiryRule(obj ObjectInfo, restoredObject, applyOnVersion bool) bool {
|
||||
}
|
||||
|
||||
// Perform actions (removal or transitioning of objects), return true the action is successfully performed
|
||||
func applyLifecycleAction(action lifecycle.Action, obj ObjectInfo) (success bool) {
|
||||
func applyLifecycleAction(action lifecycle.Action, obj ObjectInfo, storageClass string) (success bool) {
|
||||
switch action {
|
||||
case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
|
||||
success = applyExpiryRule(obj, false, action == lifecycle.DeleteVersionAction)
|
||||
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
|
||||
success = applyExpiryRule(obj, true, action == lifecycle.DeleteRestoredVersionAction)
|
||||
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
|
||||
success = applyTransitionRule(obj)
|
||||
success = applyTransitionRule(obj, storageClass)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -1225,29 +1238,16 @@ func (i *scannerItem) objectPath() string {
|
||||
|
||||
// healReplication will heal a scanned item that has failed replication.
|
||||
func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) {
|
||||
roi := getHealReplicateObjectInfo(oi, i.replication)
|
||||
if !roi.Dsc.ReplicateAny() {
|
||||
if oi.VersionID == "" {
|
||||
return
|
||||
}
|
||||
|
||||
if i.replication.Config == nil {
|
||||
return
|
||||
}
|
||||
roi := queueReplicationHeal(ctx, oi.Bucket, oi, i.replication)
|
||||
if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
|
||||
// heal delete marker replication failure or versioned delete replication failure
|
||||
if oi.ReplicationStatus == replication.Pending ||
|
||||
oi.ReplicationStatus == replication.Failed ||
|
||||
oi.VersionPurgeStatus == Failed || oi.VersionPurgeStatus == Pending {
|
||||
i.healReplicationDeletes(ctx, o, roi)
|
||||
return
|
||||
}
|
||||
// if replication status is Complete on DeleteMarker and existing object resync required
|
||||
if roi.ExistingObjResync.mustResync() && (oi.ReplicationStatus == replication.Completed || oi.ReplicationStatus.Empty()) {
|
||||
i.healReplicationDeletes(ctx, o, roi)
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
if roi.ExistingObjResync.mustResync() {
|
||||
roi.OpType = replication.ExistingObjectReplicationType
|
||||
}
|
||||
|
||||
if sizeS.replTargetStats == nil && len(roi.TargetStatuses) > 0 {
|
||||
sizeS.replTargetStats = make(map[string]replTargetSizeSummary)
|
||||
@@ -1277,52 +1277,9 @@ func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi Obj
|
||||
}
|
||||
|
||||
switch oi.ReplicationStatus {
|
||||
case replication.Pending, replication.Failed:
|
||||
roi.EventType = ReplicateHeal
|
||||
globalReplicationPool.queueReplicaTask(roi)
|
||||
return
|
||||
case replication.Replica:
|
||||
sizeS.replicaSize += oi.Size
|
||||
}
|
||||
if roi.ExistingObjResync.mustResync() {
|
||||
roi.EventType = ReplicateExisting
|
||||
globalReplicationPool.queueReplicaTask(roi)
|
||||
}
|
||||
}
|
||||
|
||||
// healReplicationDeletes will heal a scanned deleted item that failed to replicate deletes.
|
||||
func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, roi ReplicateObjectInfo) {
|
||||
// handle soft delete and permanent delete failures here.
|
||||
if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
|
||||
versionID := ""
|
||||
dmVersionID := ""
|
||||
if roi.VersionPurgeStatus.Empty() {
|
||||
dmVersionID = roi.VersionID
|
||||
} else {
|
||||
versionID = roi.VersionID
|
||||
}
|
||||
|
||||
doi := DeletedObjectReplicationInfo{
|
||||
DeletedObject: DeletedObject{
|
||||
ObjectName: roi.Name,
|
||||
DeleteMarkerVersionID: dmVersionID,
|
||||
VersionID: versionID,
|
||||
ReplicationState: roi.getReplicationState(roi.Dsc.String(), versionID, true),
|
||||
DeleteMarkerMTime: DeleteMarkerMTime{roi.ModTime},
|
||||
DeleteMarker: roi.DeleteMarker,
|
||||
},
|
||||
Bucket: roi.Bucket,
|
||||
OpType: replication.HealReplicationType,
|
||||
EventType: ReplicateHealDelete,
|
||||
}
|
||||
if roi.ExistingObjResync.mustResync() {
|
||||
doi.OpType = replication.ExistingObjectReplicationType
|
||||
doi.EventType = ReplicateExistingDelete
|
||||
queueReplicateDeletesWrapper(doi, roi.ExistingObjResync)
|
||||
return
|
||||
}
|
||||
globalReplicationPool.queueReplicaDeleteTask(doi)
|
||||
}
|
||||
}
|
||||
|
||||
type dynamicSleeper struct {
|
||||
@@ -1490,9 +1447,11 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string) {
|
||||
case ILMTransition:
|
||||
apiName = "ILMTransition"
|
||||
}
|
||||
auditLogInternal(ctx, oi.Bucket, oi.Name, AuditLogOptions{
|
||||
auditLogInternal(ctx, AuditLogOptions{
|
||||
Event: event,
|
||||
APIName: apiName,
|
||||
Bucket: oi.Bucket,
|
||||
Object: oi.Name,
|
||||
VersionID: oi.VersionID,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -24,7 +24,6 @@ import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
@@ -227,7 +226,7 @@ func (d *dataUpdateTracker) load(ctx context.Context, drives ...string) {
|
||||
for _, drive := range drives {
|
||||
|
||||
cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
|
||||
f, err := os.Open(cacheFormatPath)
|
||||
f, err := OpenFile(cacheFormatPath, readMode, 0o666)
|
||||
if err != nil {
|
||||
if osIsNotExist(err) {
|
||||
continue
|
||||
@@ -296,7 +295,7 @@ func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Durati
|
||||
}
|
||||
for _, drive := range drives {
|
||||
cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
|
||||
err := ioutil.WriteFile(cacheFormatPath, buf.Bytes(), os.ModePerm)
|
||||
err := os.WriteFile(cacheFormatPath, buf.Bytes(), os.ModePerm)
|
||||
if err != nil {
|
||||
if osIsNotExist(err) {
|
||||
continue
|
||||
|
||||
@@ -21,7 +21,6 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path"
|
||||
@@ -100,15 +99,11 @@ func TestDataUpdateTracker(t *testing.T) {
|
||||
|
||||
dut.Current.bf = dut.newBloomFilter()
|
||||
|
||||
tmpDir, err := ioutil.TempDir("", "TestDataUpdateTracker")
|
||||
tmpDir := t.TempDir()
|
||||
err := os.MkdirAll(filepath.Dir(filepath.Join(tmpDir, dataUpdateTrackerFilename)), os.ModePerm)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = os.MkdirAll(filepath.Dir(filepath.Join(tmpDir, dataUpdateTrackerFilename)), os.ModePerm)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
dut.start(ctx, tmpDir)
|
||||
@@ -205,11 +200,13 @@ func TestDataUpdateTracker(t *testing.T) {
|
||||
}
|
||||
|
||||
ctx, cancel = context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// Reload...
|
||||
dut = newDataUpdateTracker()
|
||||
dut.start(ctx, tmpDir)
|
||||
defer func() {
|
||||
cancel()
|
||||
<-dut.saveExited
|
||||
}()
|
||||
|
||||
if dut.current() != 2 {
|
||||
t.Fatal("current idx after load not preserved. want 2, got:", dut.current())
|
||||
|
||||
@@ -392,13 +392,6 @@ func (e *dataUsageEntry) addChild(hash dataUsageHash) {
|
||||
e.Children[hash.Key()] = struct{}{}
|
||||
}
|
||||
|
||||
// removeChild will remove a child based on its hash.
|
||||
func (e *dataUsageEntry) removeChild(hash dataUsageHash) {
|
||||
if len(e.Children) > 0 {
|
||||
delete(e.Children, hash.Key())
|
||||
}
|
||||
}
|
||||
|
||||
// Create a clone of the entry.
|
||||
func (e dataUsageEntry) clone() dataUsageEntry {
|
||||
// We operate on a copy from the receiver.
|
||||
@@ -488,43 +481,6 @@ func (d *dataUsageCache) deleteRecursive(h dataUsageHash) {
|
||||
}
|
||||
}
|
||||
|
||||
// keepBuckets will keep only the buckets specified specified by delete all others.
|
||||
func (d *dataUsageCache) keepBuckets(b []BucketInfo) {
|
||||
lu := make(map[dataUsageHash]struct{})
|
||||
for _, v := range b {
|
||||
lu[hashPath(v.Name)] = struct{}{}
|
||||
}
|
||||
d.keepRootChildren(lu)
|
||||
}
|
||||
|
||||
// keepRootChildren will keep the root children specified by delete all others.
|
||||
func (d *dataUsageCache) keepRootChildren(list map[dataUsageHash]struct{}) {
|
||||
root := d.root()
|
||||
if root == nil {
|
||||
return
|
||||
}
|
||||
rh := d.rootHash()
|
||||
for k := range d.Cache {
|
||||
h := dataUsageHash(k)
|
||||
if h == rh {
|
||||
continue
|
||||
}
|
||||
if _, ok := list[h]; !ok {
|
||||
delete(d.Cache, k)
|
||||
d.deleteRecursive(h)
|
||||
root.removeChild(h)
|
||||
}
|
||||
}
|
||||
// Clean up abandoned children.
|
||||
for k := range root.Children {
|
||||
h := dataUsageHash(k)
|
||||
if _, ok := list[h]; !ok {
|
||||
delete(root.Children, k)
|
||||
}
|
||||
}
|
||||
d.Cache[rh.Key()] = *root
|
||||
}
|
||||
|
||||
// dui converts the flattened version of the path to madmin.DataUsageInfo.
|
||||
// As a side effect d will be flattened, use a clone if this is not ok.
|
||||
func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
|
||||
|
||||
@@ -651,6 +651,7 @@ func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// omitempty: check for empty values
|
||||
zb0001Len := uint32(5)
|
||||
var zb0001Mask uint8 /* 5 bits */
|
||||
_ = zb0001Mask
|
||||
if z.BloomFilter == nil {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x10
|
||||
@@ -724,6 +725,7 @@ func (z *dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
// omitempty: check for empty values
|
||||
zb0001Len := uint32(5)
|
||||
var zb0001Mask uint8 /* 5 bits */
|
||||
_ = zb0001Mask
|
||||
if z.BloomFilter == nil {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x10
|
||||
@@ -1652,6 +1654,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// omitempty: check for empty values
|
||||
zb0001Len := uint32(8)
|
||||
var zb0001Mask uint8 /* 8 bits */
|
||||
_ = zb0001Mask
|
||||
if z.ReplicationStats == nil {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x20
|
||||
@@ -1782,6 +1785,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
// omitempty: check for empty values
|
||||
zb0001Len := uint32(8)
|
||||
var zb0001Mask uint8 /* 8 bits */
|
||||
_ = zb0001Mask
|
||||
if z.ReplicationStats == nil {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x20
|
||||
@@ -2774,6 +2778,7 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// omitempty: check for empty values
|
||||
zb0001Len := uint32(2)
|
||||
var zb0001Mask uint8 /* 2 bits */
|
||||
_ = zb0001Mask
|
||||
if z.Targets == nil {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x1
|
||||
@@ -2835,6 +2840,7 @@ func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
// omitempty: check for empty values
|
||||
zb0001Len := uint32(2)
|
||||
var zb0001Mask uint8 /* 2 bits */
|
||||
_ = zb0001Mask
|
||||
if z.Targets == nil {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x1
|
||||
|
||||
+2
-1
@@ -56,7 +56,8 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan
|
||||
}
|
||||
|
||||
// loadPrefixUsageFromBackend returns prefix usages found in passed buckets
|
||||
// e.g.: /testbucket/prefix => 355601334
|
||||
//
|
||||
// e.g.: /testbucket/prefix => 355601334
|
||||
func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket string) (map[string]uint64, error) {
|
||||
z, ok := objAPI.(*erasureServerPools)
|
||||
if !ok {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user