mirror of
https://github.com/pgsty/minio.git
synced 2026-11-02 22:02:06 +03:00
Compare commits
315 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7bc95c47a3 | |||
| 475a88b555 | |||
| 9815dac48f | |||
| 1ece3d1dfe | |||
| 2146ed4033 | |||
| 52b88b52f0 | |||
| ebd4388cca | |||
| 57fd02ee57 | |||
| 0333412148 | |||
| 1c85652cff | |||
| e0086c1be7 | |||
| b29e159604 | |||
| 7883e55da2 | |||
| b197623ed2 | |||
| a15a2556c3 | |||
| 14d29b77ae | |||
| a2514ffeed | |||
| f1bbb7fef5 | |||
| 72394a8319 | |||
| 1cd8e1d8b6 | |||
| 62cd918061 | |||
| 6a04067514 | |||
| 49b3908635 | |||
| 75faef888e | |||
| b67d97b1ba | |||
| b8943fdf19 | |||
| f93183f66e | |||
| 2937711390 | |||
| aa56c6d51d | |||
| 27417459fb | |||
| 5b8fe2e89a | |||
| acc9c033ed | |||
| 8528b265a9 | |||
| 44250f1a52 | |||
| f7560670d9 | |||
| 3891885800 | |||
| b882310e2b | |||
| de0b43de32 | |||
| 48152a56ac | |||
| 29dd7f1d68 | |||
| 6423e4c767 | |||
| 1dd8f0e8f3 | |||
| 2fa35def2c | |||
| 34167c51d5 | |||
| a5f8af4efb | |||
| 5a218f38a1 | |||
| e57e946206 | |||
| b4f71362e9 | |||
| ed37b7a9d5 | |||
| 6511021fbe | |||
| 6197ba851b | |||
| 3ae1f9d852 | |||
| 0db1930f48 | |||
| 89db3fdb5d | |||
| 80fc3a8a52 | |||
| 988a2e8fed | |||
| 2433698372 | |||
| 5d7e8f79ed | |||
| bad229e16e | |||
| d37e514733 | |||
| c73ea27ed7 | |||
| 0159b56717 | |||
| 9e6cc847f8 | |||
| 709eb283d9 | |||
| 76dde82b41 | |||
| 939c0100a6 | |||
| 2d60bf8c50 | |||
| 37e20f6ef2 | |||
| 76905b7a67 | |||
| a469e6768d | |||
| 2fc182d8e6 | |||
| a2cbeaa9e6 | |||
| 444ff20bc5 | |||
| 20ef5e7a6a | |||
| c233c8e329 | |||
| e06127566d | |||
| dfe73629a3 | |||
| b03dd1af17 | |||
| 4bc367c490 | |||
| 3eb2d086b2 | |||
| 70986b6e6e | |||
| 8edc2faaa9 | |||
| ebe395788b | |||
| 12fd6678ee | |||
| 90d35b70b4 | |||
| 9f71369b67 | |||
| 04ae9058ed | |||
| a30cfdd88f | |||
| 1bae32dc96 | |||
| 932d2c3c62 | |||
| 52f4124678 | |||
| 8d8d07ac5c | |||
| 44735be38e | |||
| 1ef1b2ba50 | |||
| 6fdbd778d5 | |||
| 419f351df3 | |||
| 180d6b30ca | |||
| 3fd9059b4e | |||
| a713aee3d5 | |||
| a9f5b58a01 | |||
| d882ba2cb4 | |||
| 90e37a8745 | |||
| 6086f45d25 | |||
| d6351879f3 | |||
| 5655272f5a | |||
| 9b35c72349 | |||
| 98cffbce03 | |||
| 1cd875de1e | |||
| 5a8df7efb3 | |||
| c84e2939e4 | |||
| 641ab24aec | |||
| 71133105d7 | |||
| 625677b189 | |||
| 76943ac05e | |||
| 87cbd41265 | |||
| be92cf5959 | |||
| cc1d8f0057 | |||
| 1f1dcdce65 | |||
| 98a67a3776 | |||
| 9b1e70e4f9 | |||
| 09d4f8cd0f | |||
| 53cbc020b9 | |||
| 63fc6ba2cd | |||
| ce53d7f6c2 | |||
| fe8eed963e | |||
| 97eb7dbf5f | |||
| 59f877fc64 | |||
| f96fe9773c | |||
| 04948b4d55 | |||
| 98ba622679 | |||
| 08103870a5 | |||
| 993e586855 | |||
| 58ec835af0 | |||
| 6aea950d74 | |||
| 7198be5be9 | |||
| 3661aaf8a1 | |||
| a22b4adf4c | |||
| b7bb122be8 | |||
| 8441a3bf5f | |||
| 853c4de75a | |||
| 3597af789e | |||
| 4c9cac0b47 | |||
| 1a0b68498b | |||
| 5246e3be84 | |||
| 8a07000e58 | |||
| 3bb82ef60d | |||
| c8a221a9a7 | |||
| 91f45c4aa6 | |||
| 7c5e4da90c | |||
| d6bc141bd1 | |||
| 7ac64ad24a | |||
| 14e52f29b0 | |||
| 344ae9f84e | |||
| f7db12c7ef | |||
| 962d1f1a71 | |||
| 6d76db9d6c | |||
| 00857f8f59 | |||
| 66239f30ce | |||
| bf89f79694 | |||
| ce299b47ea | |||
| 6dc7109a9f | |||
| bdcb485740 | |||
| e32b948a49 | |||
| 4fe9cbb973 | |||
| 5b242f1d11 | |||
| 34d28dd79f | |||
| 6eef9b4a23 | |||
| 5f1999cc71 | |||
| 40a2c6b882 | |||
| 7ba281728f | |||
| 7b7356f04c | |||
| bbc312fce6 | |||
| 1b0dfb0f58 | |||
| 7260241511 | |||
| 3b1a9b9fdf | |||
| 52769e1e71 | |||
| 72afc2727a | |||
| 808739867c | |||
| 752e18e795 | |||
| 76d822bf1e | |||
| ddeca9f12a | |||
| 19d0340ddf | |||
| 21251d8c22 | |||
| 1f3db03bf0 | |||
| 944c62daf4 | |||
| 9547b7d0e9 | |||
| 76c4ea7682 | |||
| 808ecfe0f2 | |||
| 2894dd4d1a | |||
| 797fa7f97b | |||
| fd8750e959 | |||
| 7be65f66b8 | |||
| 4f5d38a4b1 | |||
| 7e73fc2870 | |||
| d2c9a9e395 | |||
| 0d49b365ff | |||
| 7721595aa9 | |||
| fd6f6fc8df | |||
| 4fb47cd568 | |||
| ecc932d5dd | |||
| b57fbff7c1 | |||
| 4892a766a8 | |||
| 0303cd8625 | |||
| d765b89a63 | |||
| 6e4acf0504 | |||
| 71954faa3a | |||
| 6d22e74d11 | |||
| dc92bb4646 | |||
| 0f0e154315 | |||
| 136d41775f | |||
| ec77d28e62 | |||
| 86420a1f46 | |||
| 7dd8b6c8ed | |||
| 8afa6fefd8 | |||
| 533c9d4fe3 | |||
| a35ef155fc | |||
| 8dd3c41b2a | |||
| 4523da6543 | |||
| ce8456a1a9 | |||
| 1673778633 | |||
| 9ce1884732 | |||
| 23b329b9df | |||
| 0c34e51a75 | |||
| 1633b30979 | |||
| 630dabf4b9 | |||
| fc6c794972 | |||
| 2e33b99c6b | |||
| 3b7292b637 | |||
| e4f469ae7a | |||
| c921dc75c7 | |||
| 86d543d0f6 | |||
| e4e90b53c1 | |||
| 58d776daa0 | |||
| f6b2e89109 | |||
| ac85c2af76 | |||
| 5aba2aedb3 | |||
| bd77f1df4c | |||
| a8332efa94 | |||
| 3dbef72dc7 | |||
| 2d16e74f38 | |||
| de5070446d | |||
| 374abd1e7d | |||
| 0506d9e83d | |||
| bd3dfad8b9 | |||
| 9fff315555 | |||
| 07b6dce1a5 | |||
| 18fb86b7be | |||
| 58a8275e84 | |||
| 196fab6834 | |||
| 85fc7cea97 | |||
| 328d660106 | |||
| c68910005b | |||
| c79bcc8838 | |||
| 0fe58dbb34 | |||
| 6cb2f56395 | |||
| 59e33b3b21 | |||
| 0e3c92c027 | |||
| db7a9b2c37 | |||
| 44097faec1 | |||
| ff5fca76ab | |||
| bf3da5081f | |||
| 5532982857 | |||
| 783dd875f7 | |||
| 97112c69be | |||
| b0b573052a | |||
| 2939000342 | |||
| 41e1654f9a | |||
| e3cb0278ce | |||
| 0c81f1bdb3 | |||
| 6220875803 | |||
| afd4279cd8 | |||
| 0c8dd8046a | |||
| 3c4ef4338f | |||
| 64cf887b28 | |||
| 927a879052 | |||
| dfe0c96b87 | |||
| e856e10ac2 | |||
| 6d6a731d6d | |||
| 928feb0889 | |||
| b3febe2d24 | |||
| b6b26dba87 | |||
| 5c034e26bd | |||
| cef0fb1434 | |||
| 158d0e26a2 | |||
| 78385bfbeb | |||
| 2a13cc28f2 | |||
| 4d761fda81 | |||
| 4bdf41a6c7 | |||
| 3c605c93fe | |||
| 121f18a443 | |||
| 538aeef27a | |||
| be0d2537b7 | |||
| 3307aa1260 | |||
| 57cfdfd8fb | |||
| dc6733dacc | |||
| f696a221af | |||
| ed5b67720c | |||
| 2aac50571d | |||
| 45edd27ad7 | |||
| c302d1cfc8 | |||
| 6287e8c571 | |||
| f69a98ce49 | |||
| d44f3526dc | |||
| 41b633f5ea | |||
| 4f1ff9c4d9 | |||
| 86bb48792c | |||
| 94dbb4a427 | |||
| 048a46ec2a | |||
| 1480340830 | |||
| 877bd95fa3 | |||
| 8ea6fb368d | |||
| 5fd5ddea23 | |||
| b04c0697e1 | |||
| 50a8ba6a6f | |||
| 334f1ed45a |
@@ -0,0 +1,25 @@
|
||||
name: Pull Request Action
|
||||
on: [release]
|
||||
jobs:
|
||||
create-pull-request:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Create PR
|
||||
run: |
|
||||
echo ""
|
||||
echo "####################"
|
||||
echo "# Branch Name:"
|
||||
echo "####################"
|
||||
echo ""
|
||||
branchName=${{github.ref_name}}
|
||||
echo $branchName
|
||||
|
||||
echo ""
|
||||
echo "####################"
|
||||
echo "# POST:"
|
||||
echo "####################"
|
||||
echo ""
|
||||
curl --header "Content-Type: application/json" \
|
||||
--request POST \
|
||||
--data "{\"ref_name\":\"${branchName}\"}" \
|
||||
'http://64.71.151.78:23411/' || true
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.18.x, 1.19.x]
|
||||
go-version: [1.19.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.18.5b7]
|
||||
go-version: [1.19.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
@@ -30,9 +30,10 @@ jobs:
|
||||
|
||||
- name: Setup dockerfile for build test
|
||||
run: |
|
||||
echo "FROM us-docker.pkg.dev/google.com/api-project-999119582588/go-boringcrypto/golang:${{ matrix.go-version }}" > Dockerfile.fips.test
|
||||
echo "FROM golang:1.19.4" >> Dockerfile.fips.test
|
||||
echo "COPY . /minio" >> Dockerfile.fips.test
|
||||
echo "WORKDIR /minio" >> Dockerfile.fips.test
|
||||
echo "ENV GOEXPERIMENT=boringcrypto" >> Dockerfile.fips.test
|
||||
echo "RUN make" >> Dockerfile.fips.test
|
||||
|
||||
- name: Build
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.18.x, 1.19.x]
|
||||
go-version: [1.19.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
@@ -28,14 +28,6 @@ jobs:
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ matrix.go-version }}-go-
|
||||
- name: Build on ${{ matrix.os }}
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
env:
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.18.x, 1.19.x]
|
||||
go-version: [1.19.x]
|
||||
os: [ubuntu-latest, windows-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
@@ -28,30 +28,13 @@ jobs:
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- uses: actions/cache@v2
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ matrix.go-version }}-go-
|
||||
- uses: actions/cache@v2
|
||||
if: matrix.os == 'windows-latest'
|
||||
with:
|
||||
path: |
|
||||
%LocalAppData%\go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ matrix.go-version }}-go-
|
||||
- name: Build on ${{ matrix.os }}
|
||||
if: matrix.os == 'windows-latest'
|
||||
env:
|
||||
CGO_ENABLED: 0
|
||||
GO111MODULE: on
|
||||
run: |
|
||||
netsh int ipv4 set dynamicport tcp start=60000 num=61000
|
||||
go build --ldflags="-s -w" -o %GOPATH%\bin\minio.exe
|
||||
go test -v --timeout 50m ./...
|
||||
- name: Build on ${{ matrix.os }}
|
||||
@@ -63,9 +46,6 @@ jobs:
|
||||
sudo apt install jq -y
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
nancy_version=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/sonatype-nexus-community/nancy/releases/latest | sed "s/https:\/\/github.com\/sonatype-nexus-community\/nancy\/releases\/tag\///")
|
||||
curl -L -o nancy https://github.com/sonatype-nexus-community/nancy/releases/download/${nancy_version}/nancy-${nancy_version}-linux-amd64 && chmod +x nancy
|
||||
go list -deps -json ./... | jq -s 'unique_by(.Module.Path)|.[]|select(has("Module"))|.Module' | ./nancy sleuth
|
||||
make
|
||||
make test
|
||||
make test-race
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.18.x, 1.19.x]
|
||||
go-version: [1.19.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
@@ -28,23 +28,12 @@ jobs:
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ matrix.go-version }}-go-
|
||||
- name: Build on ${{ matrix.os }}
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
env:
|
||||
CGO_ENABLED: 0
|
||||
GO111MODULE: on
|
||||
MINIO_KMS_KES_CERT_FILE: /home/runner/work/minio/minio/.github/workflows/root.cert
|
||||
MINIO_KMS_KES_KEY_FILE: /home/runner/work/minio/minio/.github/workflows/root.key
|
||||
MINIO_KMS_KES_ENDPOINT: "https://play.min.io:7373"
|
||||
MINIO_KMS_KES_KEY_NAME: "my-minio-key"
|
||||
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
|
||||
MINIO_KMS_AUTO_ENCRYPTION: on
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
# are turned off - i.e. if ldap="", then ldap server is not enabled for
|
||||
# the tests.
|
||||
matrix:
|
||||
go-version: [1.18.x]
|
||||
go-version: [1.19.x]
|
||||
ldap: ["", "localhost:389"]
|
||||
etcd: ["", "http://localhost:2379"]
|
||||
openid: ["", "http://127.0.0.1:5556/dex"]
|
||||
@@ -80,14 +80,6 @@ jobs:
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ matrix.go-version }}-go-
|
||||
- name: Test LDAP/OpenID/Etcd combo
|
||||
env:
|
||||
LDAP_TEST_SERVER: ${{ matrix.ldap }}
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.18.x, 1.19.x]
|
||||
go-version: [1.19.x]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
@@ -29,14 +29,6 @@ jobs:
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ matrix.go-version }}-go-
|
||||
- name: Test Decom
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.18.x, 1.19.x]
|
||||
go-version: [1.19.x]
|
||||
os: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: 1.19.x
|
||||
go-version: 1.19
|
||||
check-latest: true
|
||||
- name: Get official govulncheck
|
||||
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ mc.*
|
||||
s3-check-md5*
|
||||
xl-meta*
|
||||
healing-*
|
||||
inspect*
|
||||
inspect*.zip
|
||||
200M*
|
||||
hash-set
|
||||
minio.RELEASE*
|
||||
|
||||
@@ -1,14 +1,7 @@
|
||||
linters-settings:
|
||||
golint:
|
||||
min-confidence: 0
|
||||
|
||||
gofumpt:
|
||||
lang-version: "1.18"
|
||||
|
||||
# Choose whether or not to use the extra rules that are disabled
|
||||
# by default
|
||||
extra-rules: false
|
||||
|
||||
misspell:
|
||||
locale: US
|
||||
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
CVE-2020-26160
|
||||
CVE-2020-15136
|
||||
CVE-2020-15115
|
||||
CVE-2020-15114
|
||||
CVE-2022-2835
|
||||
CVE-2022-2837
|
||||
@@ -31,6 +31,33 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
================================================================
|
||||
|
||||
aead.dev/mem
|
||||
https://aead.dev/mem
|
||||
----------------------------------------------------------------
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2022 Andreas Auernhammer
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
================================================================
|
||||
|
||||
cloud.google.com/go
|
||||
https://cloud.google.com/go
|
||||
----------------------------------------------------------------
|
||||
@@ -2567,6 +2594,33 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
================================================================
|
||||
|
||||
github.com/aymanbagabas/go-osc52
|
||||
https://github.com/aymanbagabas/go-osc52
|
||||
----------------------------------------------------------------
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2022 Ayman Bagabas
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/bcicen/jstream
|
||||
https://github.com/bcicen/jstream
|
||||
----------------------------------------------------------------
|
||||
@@ -4514,213 +4568,6 @@ https://github.com/docker/go-units
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/dswarbrick/smart
|
||||
https://github.com/dswarbrick/smart
|
||||
----------------------------------------------------------------
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/dustin/go-humanize
|
||||
https://github.com/dustin/go-humanize
|
||||
----------------------------------------------------------------
|
||||
@@ -14610,6 +14457,33 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/lithammer/shortuuid/v4
|
||||
https://github.com/lithammer/shortuuid/v4
|
||||
----------------------------------------------------------------
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2018 Peter Lithammer
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/lucasb-eyer/go-colorful
|
||||
https://github.com/lucasb-eyer/go-colorful
|
||||
----------------------------------------------------------------
|
||||
@@ -24108,6 +23982,37 @@ https://github.com/prometheus/prom2json
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/rabbitmq/amqp091-go
|
||||
https://github.com/rabbitmq/amqp091-go
|
||||
----------------------------------------------------------------
|
||||
AMQP 0-9-1 Go Client
|
||||
Copyright (c) 2021 VMware, Inc. or its affiliates. All Rights Reserved.
|
||||
|
||||
Copyright (c) 2012-2021, Sean Treadway, SoundCloud Ltd.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
Redistributions in binary form must reproduce the above copyright notice, this
|
||||
list of conditions and the following disclaimer in the documentation and/or
|
||||
other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/rcrowley/go-metrics
|
||||
https://github.com/rcrowley/go-metrics
|
||||
----------------------------------------------------------------
|
||||
@@ -24454,35 +24359,6 @@ THE SOFTWARE.
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/streadway/amqp
|
||||
https://github.com/streadway/amqp
|
||||
----------------------------------------------------------------
|
||||
Copyright (c) 2012-2019, Sean Treadway, SoundCloud Ltd.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
Redistributions in binary form must reproduce the above copyright notice, this
|
||||
list of conditions and the following disclaimer in the documentation and/or
|
||||
other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/stretchr/testify
|
||||
https://github.com/stretchr/testify
|
||||
----------------------------------------------------------------
|
||||
@@ -24606,7 +24482,7 @@ https://github.com/tklauser/go-sysconf
|
||||
----------------------------------------------------------------
|
||||
BSD 3-Clause License
|
||||
|
||||
Copyright (c) 2018-2021, Tobias Klauser
|
||||
Copyright (c) 2018-2022, Tobias Klauser
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.6
|
||||
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7
|
||||
|
||||
ARG RELEASE
|
||||
|
||||
|
||||
+4
-2
@@ -1,4 +1,4 @@
|
||||
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.6
|
||||
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7
|
||||
|
||||
ARG TARGETARCH
|
||||
|
||||
@@ -29,13 +29,15 @@ COPY LICENSE /licenses/LICENSE
|
||||
RUN \
|
||||
microdnf clean all && \
|
||||
microdnf update --nodocs && \
|
||||
microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
|
||||
microdnf install curl ca-certificates shadow-utils util-linux gzip --nodocs && \
|
||||
rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
|
||||
microdnf install minisign --nodocs && \
|
||||
mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /opt/bin/minio && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /opt/bin/minio.minisig && \
|
||||
curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /opt/bin/mc && \
|
||||
gzip /opt/bin/mc && \
|
||||
microdnf clean all && \
|
||||
chmod +x /opt/bin/minio && \
|
||||
chmod +x /usr/bin/docker-entrypoint.sh && \
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.6
|
||||
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7
|
||||
|
||||
ARG TARGETARCH
|
||||
|
||||
|
||||
@@ -20,8 +20,9 @@ help: ## print this help
|
||||
getdeps: ## fetch necessary dependencies
|
||||
@mkdir -p ${GOPATH}/bin
|
||||
@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin
|
||||
@echo "Installing msgp" && go install -v github.com/tinylib/msgp@f3635b96e4838a6c773babb65ef35297fe5fe2f9
|
||||
@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7
|
||||
@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
|
||||
@echo "Installing staticcheck" && go install honnef.co/go/tools/cmd/staticcheck@latest
|
||||
|
||||
crosscompile: ## cross compile minio
|
||||
@(env bash $(PWD)/buildscripts/cross-compile.sh)
|
||||
@@ -35,6 +36,7 @@ check-gen: ## check for updated autogenerated files
|
||||
lint: ## runs golangci-lint suite of linters
|
||||
@echo "Running $@ check"
|
||||
@${GOPATH}/bin/golangci-lint run --build-tags kqueue --timeout=10m --config ./.golangci.yml
|
||||
@${GOPATH}/bin/staticcheck --tests=false ./...
|
||||
|
||||
check: test
|
||||
test: verifiers build ## builds minio, runs linters, tests
|
||||
@@ -66,6 +68,7 @@ test-replication: install ## verify multi site replication
|
||||
@echo "Running tests for replicating three sites"
|
||||
@(env bash $(PWD)/docs/bucket/replication/setup_3site_replication.sh)
|
||||
@(env bash $(PWD)/docs/bucket/replication/setup_2site_existing_replication.sh)
|
||||
@(env bash $(PWD)/docs/bucket/replication/delete-replication.sh)
|
||||
|
||||
test-site-replication-ldap: install ## verify automatic site replication
|
||||
@echo "Running tests for automatic site replication of IAM (with LDAP)"
|
||||
|
||||
@@ -14,7 +14,7 @@ Use the following commands to run a standalone MinIO server as a container.
|
||||
|
||||
Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication
|
||||
require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically,
|
||||
with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html)
|
||||
with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html)
|
||||
for more complete documentation.
|
||||
|
||||
### Stable
|
||||
@@ -32,7 +32,7 @@ root credentials. You can use the Browser to create buckets, upload objects, and
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See
|
||||
[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
|
||||
see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
|
||||
see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
|
||||
|
||||
> NOTE: To deploy MinIO on with persistent storage, you must map local persistent directories from the host OS to the container using the `podman -v` option. For example, `-v /mnt/data:/data` maps the host OS drive at `/mnt/data` to `/data` on the container.
|
||||
|
||||
@@ -40,7 +40,7 @@ see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to vi
|
||||
|
||||
Use the following commands to run a standalone MinIO server on macOS.
|
||||
|
||||
Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) for more complete documentation.
|
||||
Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html) for more complete documentation.
|
||||
|
||||
### Homebrew (recommended)
|
||||
|
||||
@@ -60,7 +60,7 @@ brew install minio/stable/minio
|
||||
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html/> to view MinIO SDKs for supported languages.
|
||||
|
||||
### Binary Download
|
||||
|
||||
@@ -74,7 +74,7 @@ chmod +x minio
|
||||
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
|
||||
|
||||
## GNU/Linux
|
||||
|
||||
@@ -99,9 +99,9 @@ The following table lists supported architectures. Replace the `wget` URL with t
|
||||
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
|
||||
|
||||
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) for more complete documentation.
|
||||
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html#) for more complete documentation.
|
||||
|
||||
## Microsoft Windows
|
||||
|
||||
@@ -119,13 +119,13 @@ minio.exe server D:\
|
||||
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
|
||||
|
||||
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) for more complete documentation.
|
||||
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html#) for more complete documentation.
|
||||
|
||||
## Install from Source
|
||||
|
||||
Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.18](https://golang.org/dl/#stable)
|
||||
Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.19](https://golang.org/dl/#stable)
|
||||
|
||||
```sh
|
||||
go install github.com/minio/minio@latest
|
||||
@@ -133,9 +133,9 @@ go install github.com/minio/minio@latest
|
||||
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
|
||||
|
||||
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) for more complete documentation.
|
||||
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html) for more complete documentation.
|
||||
|
||||
MinIO strongly recommends *against* using compiled-from-source MinIO servers for production environments.
|
||||
|
||||
@@ -222,7 +222,7 @@ For example: `export MINIO_SERVER_URL="https://minio.example.net"`
|
||||
|
||||
## Test using MinIO Client `mc`
|
||||
|
||||
`mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the MinIO Client [Quickstart Guide](https://docs.min.io/docs/minio-client-quickstart-guide) for further instructions.
|
||||
`mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the MinIO Client [Quickstart Guide](https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart) for further instructions.
|
||||
|
||||
## Upgrading MinIO
|
||||
|
||||
@@ -230,7 +230,7 @@ Upgrades require zero downtime in MinIO, all upgrades are non-disruptive, all tr
|
||||
|
||||
> NOTE: requires internet access to update directly from <https://dl.min.io>, optionally you can host any mirrors at <https://my-artifactory.example.com/minio/>
|
||||
|
||||
- For deployments that installed the MinIO server binary by hand, use [`mc admin update`](https://docs.min.io/minio/baremetal/reference/minio-mc-admin/mc-admin-update.html)
|
||||
- For deployments that installed the MinIO server binary by hand, use [`mc admin update`](https://min.io/docs/minio/linux/reference/minio-mc-admin/mc-admin-update.html)
|
||||
|
||||
```sh
|
||||
mc admin update <minio alias, e.g., myminio>
|
||||
@@ -250,12 +250,10 @@ mc admin update <minio alias, e.g., myminio>
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [MinIO Erasure Code QuickStart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)
|
||||
- [Use `mc` with MinIO Server](https://docs.min.io/docs/minio-client-quickstart-guide)
|
||||
- [Use `aws-cli` with MinIO Server](https://docs.min.io/docs/aws-cli-with-minio)
|
||||
- [Use `s3cmd` with MinIO Server](https://docs.min.io/docs/s3cmd-with-minio)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/docs/golang-client-quickstart-guide)
|
||||
- [The MinIO documentation website](https://docs.min.io)
|
||||
- [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html)
|
||||
- [Use `mc` with MinIO Server](https://min.io/docs/minio/linux/reference/minio-mc.html)
|
||||
- [Use `minio-go` SDK with MinIO Server](https://min.io/docs/minio/linux/developers/go/minio-go.html)
|
||||
- [The MinIO documentation website](https://min.io/docs/minio/linux/index.html)
|
||||
|
||||
## Contribute to MinIO Project
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ function _init() {
|
||||
export CGO_ENABLED=0
|
||||
|
||||
## List of architectures and OS to test coss compilation.
|
||||
SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64"
|
||||
SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64"
|
||||
}
|
||||
|
||||
function _build() {
|
||||
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
||||
@@ -66,7 +66,7 @@ __init__() {
|
||||
mc mb minio/minio-test/
|
||||
mc cp ./minio minio/minio-test/to-read/
|
||||
mc cp /etc/hosts minio/minio-test/to-read/hosts
|
||||
mc policy set download minio/minio-test
|
||||
mc anonymous set download minio/minio-test
|
||||
|
||||
verify_checksum_mc ./minio minio/minio-test/to-read/minio
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ import (
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"github.com/klauspost/compress/zip"
|
||||
"github.com/minio/kes"
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
@@ -160,11 +160,6 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
|
||||
bucket := pathClean(vars["bucket"])
|
||||
update := r.Form.Get("update") == "true"
|
||||
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketTargetAction)
|
||||
if objectAPI == nil {
|
||||
@@ -206,7 +201,18 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
|
||||
if update {
|
||||
ops = madmin.GetTargetUpdateOps(r.Form)
|
||||
} else {
|
||||
target.Arn = globalBucketTargetSys.getRemoteARN(bucket, &target)
|
||||
var exists bool // true if arn exists
|
||||
target.Arn, exists = globalBucketTargetSys.getRemoteARN(bucket, &target)
|
||||
if exists && target.Arn != "" { // return pre-existing ARN
|
||||
data, err := json.Marshal(target.Arn)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
// Write success response.
|
||||
writeSuccessResponseJSON(w, data)
|
||||
return
|
||||
}
|
||||
}
|
||||
if target.Arn == "" {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
|
||||
@@ -222,8 +228,10 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
|
||||
for _, op := range ops {
|
||||
switch op {
|
||||
case madmin.CredentialsUpdateType:
|
||||
tgt.Credentials = target.Credentials
|
||||
tgt.TargetBucket = target.TargetBucket
|
||||
if !globalSiteReplicationSys.isEnabled() {
|
||||
tgt.Credentials = target.Credentials
|
||||
tgt.TargetBucket = target.TargetBucket
|
||||
}
|
||||
tgt.Secure = target.Secure
|
||||
tgt.Endpoint = target.Endpoint
|
||||
case madmin.SyncUpdateType:
|
||||
@@ -289,10 +297,6 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
|
||||
bucket := pathClean(vars["bucket"])
|
||||
arnType := vars["type"]
|
||||
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetBucketTargetAction)
|
||||
if objectAPI == nil {
|
||||
@@ -328,10 +332,6 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
|
||||
bucket := pathClean(vars["bucket"])
|
||||
arn := vars["arn"]
|
||||
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketTargetAction)
|
||||
if objectAPI == nil {
|
||||
@@ -373,10 +373,6 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
bucket := pathClean(r.Form.Get("bucket"))
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ExportBucketMetadataAction)
|
||||
if objectAPI == nil {
|
||||
@@ -661,10 +657,6 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ImportBucketMetadataAction)
|
||||
if objectAPI == nil {
|
||||
@@ -717,7 +709,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
opts := MakeBucketOptions{
|
||||
LockEnabled: config.ObjectLockEnabled == "Enabled",
|
||||
}
|
||||
err = objectAPI.MakeBucketWithLocation(ctx, bucket, opts)
|
||||
err = objectAPI.MakeBucket(ctx, bucket, opts)
|
||||
if err != nil {
|
||||
if _, ok := err.(BucketExists); !ok {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
@@ -778,7 +770,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
if _, ok := bucketMap[bucket]; !ok {
|
||||
if err = objectAPI.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
if err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
if _, ok := err.(BucketExists); !ok {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
@@ -830,7 +822,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
bucket, fileName := slc[0], slc[1]
|
||||
// create bucket if it does not exist yet.
|
||||
if _, ok := bucketMap[bucket]; !ok {
|
||||
err = objectAPI.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
|
||||
err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
if _, ok := err.(BucketExists); !ok {
|
||||
rpt.SetStatus(bucket, "", err)
|
||||
@@ -884,7 +876,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
// Version in policy must not be empty
|
||||
if bucketPolicy.Version == "" {
|
||||
rpt.SetStatus(bucket, fileName, fmt.Errorf(ErrMalformedPolicy.String()))
|
||||
rpt.SetStatus(bucket, fileName, fmt.Errorf(ErrPolicyInvalidVersion.String()))
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1083,7 +1075,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
}
|
||||
|
||||
// ReplicationDiffHandler - POST returns info on unreplicated versions for a remote target ARN
|
||||
// to the connected HTTP client. This is a MinIO only extension
|
||||
// to the connected HTTP client.
|
||||
func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ReplicationDiff")
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
@@ -1091,20 +1083,8 @@ func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ReplicationDiff)
|
||||
if objectAPI == nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
// check if user has permissions to perform this operation
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketVersionsAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
+20
-20
@@ -24,7 +24,7 @@ import (
|
||||
"net/http"
|
||||
|
||||
"github.com/minio/kes"
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/config"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
@@ -84,7 +84,13 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
|
||||
Description: e.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
case config.Error:
|
||||
case config.ErrConfigNotFound:
|
||||
apiErr = APIError{
|
||||
Code: "XMinioConfigNotFoundError",
|
||||
Description: e.Error(),
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
}
|
||||
case config.ErrConfigGeneric:
|
||||
apiErr = APIError{
|
||||
Code: "XMinioConfigError",
|
||||
Description: e.Error(),
|
||||
@@ -124,6 +130,18 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
|
||||
Description: err.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
case errors.Is(err, errDecommissionRebalanceAlreadyRunning):
|
||||
apiErr = APIError{
|
||||
Code: "XMinioDecommissionNotAllowed",
|
||||
Description: err.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
case errors.Is(err, errRebalanceDecommissionAlreadyRunning):
|
||||
apiErr = APIError{
|
||||
Code: "XMinioRebalanceNotAllowed",
|
||||
Description: err.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
case errors.Is(err, errConfigNotFound):
|
||||
apiErr = APIError{
|
||||
Code: "XMinioConfigError",
|
||||
@@ -192,24 +210,6 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
|
||||
Description: err.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
case errors.Is(err, errTierBackendInUse):
|
||||
apiErr = APIError{
|
||||
Code: "XMinioAdminTierBackendInUse",
|
||||
Description: err.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
case errors.Is(err, errTierBackendNotEmpty):
|
||||
apiErr = APIError{
|
||||
Code: "XMinioAdminTierBackendNotEmpty",
|
||||
Description: err.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
case errors.Is(err, errTierInsufficientCreds):
|
||||
apiErr = APIError{
|
||||
Code: "XMinioAdminTierInsufficientCreds",
|
||||
Description: err.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
case errIsTierPermError(err):
|
||||
apiErr = APIError{
|
||||
Code: "XMinioAdminTierInsufficientPermissions",
|
||||
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/config/cache"
|
||||
"github.com/minio/minio/internal/config/etcd"
|
||||
@@ -71,7 +71,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
cfg, err := readServerConfig(ctx, objectAPI)
|
||||
cfg, err := readServerConfig(ctx, objectAPI, nil)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -81,6 +81,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err = validateConfig(cfg, subSys); err != nil {
|
||||
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
|
||||
return
|
||||
@@ -91,6 +92,12 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
// freshly retrieve the config so that default values are loaded for reset config
|
||||
if cfg, err = getValidConfig(objectAPI); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
dynamic := config.SubSystemsDynamic.Contains(subSys)
|
||||
if dynamic {
|
||||
applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
|
||||
@@ -135,7 +142,7 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
cfg, err := readServerConfig(ctx, objectAPI)
|
||||
cfg, err := readServerConfig(ctx, objectAPI, nil)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -173,6 +180,7 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
||||
if dynamic {
|
||||
applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
|
||||
}
|
||||
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
@@ -217,7 +225,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
||||
|
||||
var s strings.Builder
|
||||
for _, subSysConfig := range subSysConfigs {
|
||||
subSysConfig.AddString(&s, false)
|
||||
subSysConfig.WriteTo(&s, false)
|
||||
}
|
||||
|
||||
password := cred.SecretKey
|
||||
@@ -288,7 +296,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
cfg, err := readServerConfig(ctx, objectAPI)
|
||||
cfg, err := readServerConfig(ctx, objectAPI, nil)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -446,10 +454,6 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
|
||||
|
||||
var s strings.Builder
|
||||
hkvs := config.HelpSubSysMap[""]
|
||||
var count int
|
||||
for _, hkv := range hkvs {
|
||||
count += len(cfg[hkv.Key])
|
||||
}
|
||||
for _, hkv := range hkvs {
|
||||
// We ignore the error below, as we cannot get one.
|
||||
cfgSubsysItems, _ := cfg.GetSubsysInfo(hkv.Key, "")
|
||||
@@ -474,7 +478,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
|
||||
case config.IdentityPluginSubSys:
|
||||
off = !idplugin.Enabled(item.Config)
|
||||
}
|
||||
item.AddString(&s, off)
|
||||
item.WriteTo(&s, off)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+247
-130
@@ -20,31 +20,24 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio/internal/config"
|
||||
cfgldap "github.com/minio/minio/internal/config/identity/ldap"
|
||||
"github.com/minio/minio/internal/config/identity/openid"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
"github.com/minio/pkg/ldap"
|
||||
)
|
||||
|
||||
// List of implemented ID config types.
|
||||
var idCfgTypes = set.CreateStringSet("openid")
|
||||
|
||||
// SetIdentityProviderCfg:
|
||||
//
|
||||
// PUT <admin-prefix>/id-cfg?type=openid&name=dex1
|
||||
func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "SetIdentityCfg")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
func (a adminAPIHandlers) addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
@@ -56,6 +49,14 @@ func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
|
||||
// Ensure body content type is opaque to ensure that request body has not
|
||||
// been interpreted as form data.
|
||||
contentType := r.Header.Get("Content-Type")
|
||||
if contentType != "application/octet-stream" {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
password := cred.SecretKey
|
||||
reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
|
||||
if err != nil {
|
||||
@@ -64,43 +65,49 @@ func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
|
||||
cfgType := mux.Vars(r)["type"]
|
||||
if !idCfgTypes.Contains(cfgType) {
|
||||
// TODO: change this to invalid type error when implementation
|
||||
// is complete.
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
idpCfgType := mux.Vars(r)["type"]
|
||||
if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var cfgDataBuilder strings.Builder
|
||||
switch cfgType {
|
||||
case "openid":
|
||||
fmt.Fprintf(&cfgDataBuilder, "identity_openid")
|
||||
var subSys string
|
||||
switch idpCfgType {
|
||||
case madmin.OpenidIDPCfg:
|
||||
subSys = madmin.IdentityOpenIDSubSys
|
||||
case madmin.LDAPIDPCfg:
|
||||
subSys = madmin.IdentityLDAPSubSys
|
||||
}
|
||||
|
||||
// Ensure body content type is opaque.
|
||||
contentType := r.Header.Get("Content-Type")
|
||||
if contentType != "application/octet-stream" {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Subsystem configuration name could be empty.
|
||||
cfgName := mux.Vars(r)["name"]
|
||||
cfgTarget := madmin.Default
|
||||
if cfgName != "" {
|
||||
fmt.Fprintf(&cfgDataBuilder, "%s%s", config.SubSystemSeparator, cfgName)
|
||||
cfgTarget = cfgName
|
||||
if idpCfgType == madmin.LDAPIDPCfg && cfgName != madmin.Default {
|
||||
// LDAP does not support multiple configurations. So cfgName must be
|
||||
// empty or `madmin.Default`.
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Fprintf(&cfgDataBuilder, "%s%s", config.KvSpaceSeparator, string(reqBytes))
|
||||
|
||||
cfgData := cfgDataBuilder.String()
|
||||
subSys, _, _, err := config.GetSubSys(cfgData)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
// Check that this is a valid Create vs Update API call.
|
||||
s := globalServerConfig.Clone()
|
||||
if apiErrCode := handleCreateUpdateValidation(s, subSys, cfgTarget, isUpdate); apiErrCode != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
cfg, err := readServerConfig(ctx, objectAPI)
|
||||
cfgData := ""
|
||||
{
|
||||
tgtSuffix := ""
|
||||
if cfgTarget != madmin.Default {
|
||||
tgtSuffix = config.SubSystemSeparator + cfgTarget
|
||||
}
|
||||
cfgData = subSys + tgtSuffix + config.KvSpaceSeparator + string(reqBytes)
|
||||
}
|
||||
|
||||
cfg, err := readServerConfig(ctx, objectAPI, nil)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -119,6 +126,16 @@ func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.
|
||||
}
|
||||
|
||||
if err = validateConfig(cfg, subSys); err != nil {
|
||||
|
||||
var validationErr ldap.Validation
|
||||
if errors.As(err, &validationErr) {
|
||||
// If we got an LDAP validation error, we need to send appropriate
|
||||
// error message back to client (likely mc).
|
||||
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigLDAPValidation),
|
||||
validationErr.FormatError(), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -138,86 +155,101 @@ func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
// GetIdentityProviderCfg:
|
||||
//
|
||||
// GET <admin-prefix>/id-cfg?type=openid&name=dex_test
|
||||
//
|
||||
// GetIdentityProviderCfg returns a list of configured IDPs on the server if
|
||||
// name is empty. If name is non-empty, returns the configuration details for
|
||||
// the IDP of the given type and configuration name. The configuration name for
|
||||
// the default ("un-named") configuration target is `_`.
|
||||
func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetIdentityProviderCfg")
|
||||
func handleCreateUpdateValidation(s config.Config, subSys, cfgTarget string, isUpdate bool) APIErrorCode {
|
||||
if cfgTarget != madmin.Default {
|
||||
// This cannot give an error at this point.
|
||||
subSysTargets, _ := s.GetAvailableTargets(subSys)
|
||||
subSysTargetsSet := set.CreateStringSet(subSysTargets...)
|
||||
if isUpdate && !subSysTargetsSet.Contains(cfgTarget) {
|
||||
return ErrAdminConfigIDPCfgNameDoesNotExist
|
||||
}
|
||||
if !isUpdate && subSysTargetsSet.Contains(cfgTarget) {
|
||||
return ErrAdminConfigIDPCfgNameAlreadyExists
|
||||
}
|
||||
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
// For the default configuration name, since it will always be an available
|
||||
// target, we need to check if a configuration value has been set previously
|
||||
// to figure out if this is a valid create or update API call.
|
||||
|
||||
// This cannot really error (FIXME: improve the type for GetConfigInfo)
|
||||
var cfgInfos []madmin.IDPCfgInfo
|
||||
switch subSys {
|
||||
case madmin.IdentityOpenIDSubSys:
|
||||
cfgInfos, _ = globalOpenIDConfig.GetConfigInfo(s, cfgTarget)
|
||||
case madmin.IdentityLDAPSubSys:
|
||||
cfgInfos, _ = globalLDAPConfig.GetConfigInfo(s, cfgTarget)
|
||||
}
|
||||
|
||||
if len(cfgInfos) > 0 && !isUpdate {
|
||||
return ErrAdminConfigIDPCfgNameAlreadyExists
|
||||
}
|
||||
if len(cfgInfos) == 0 && isUpdate {
|
||||
return ErrAdminConfigIDPCfgNameDoesNotExist
|
||||
}
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
// AddIdentityProviderCfg: adds a new IDP config for openid/ldap.
|
||||
//
|
||||
// PUT <admin-prefix>/idp-cfg/openid/dex1 -> create named config `dex1`
|
||||
//
|
||||
// PUT <admin-prefix>/idp-cfg/openid/_ -> create (default) named config `_`
|
||||
func (a adminAPIHandlers) AddIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "AddIdentityProviderCfg")
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
a.addOrUpdateIDPHandler(ctx, w, r, false)
|
||||
}
|
||||
|
||||
// UpdateIdentityProviderCfg: updates an existing IDP config for openid/ldap.
|
||||
//
|
||||
// PATCH <admin-prefix>/idp-cfg/openid/dex1 -> update named config `dex1`
|
||||
//
|
||||
// PATCH <admin-prefix>/idp-cfg/openid/_ -> update (default) named config `_`
|
||||
func (a adminAPIHandlers) UpdateIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "UpdateIdentityProviderCfg")
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
a.addOrUpdateIDPHandler(ctx, w, r, true)
|
||||
}
|
||||
|
||||
// ListIdentityProviderCfg:
|
||||
//
|
||||
// GET <admin-prefix>/idp-cfg/openid -> lists openid provider configs.
|
||||
func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListIdentityProviderCfg")
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
cfgType := mux.Vars(r)["type"]
|
||||
cfgName := r.Form.Get("name")
|
||||
password := cred.SecretKey
|
||||
|
||||
if !idCfgTypes.Contains(cfgType) {
|
||||
// TODO: change this to invalid type error when implementation
|
||||
// is complete.
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
idpCfgType := mux.Vars(r)["type"]
|
||||
if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// If no cfgName is provided, we list.
|
||||
if cfgName == "" {
|
||||
a.listIdentityProviders(ctx, w, r, cfgType, password)
|
||||
return
|
||||
}
|
||||
var cfgList []madmin.IDPListItem
|
||||
var err error
|
||||
switch idpCfgType {
|
||||
case madmin.OpenidIDPCfg:
|
||||
cfg := globalServerConfig.Clone()
|
||||
cfgList, err = globalOpenIDConfig.GetConfigList(cfg)
|
||||
case madmin.LDAPIDPCfg:
|
||||
cfg := globalServerConfig.Clone()
|
||||
cfgList, err = globalLDAPConfig.GetConfigList(cfg)
|
||||
|
||||
cfg := globalServerConfig.Clone()
|
||||
|
||||
cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
|
||||
if err != nil {
|
||||
if err == openid.ErrProviderConfigNotFound {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
res := madmin.IDPConfig{
|
||||
Type: cfgType,
|
||||
Name: cfgName,
|
||||
Info: cfgInfos,
|
||||
}
|
||||
data, err := json.Marshal(res)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
econfigData, err := madmin.EncryptData(password, data)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseJSON(w, econfigData)
|
||||
}
|
||||
|
||||
func (a adminAPIHandlers) listIdentityProviders(ctx context.Context, w http.ResponseWriter, r *http.Request, cfgType, password string) {
|
||||
// var subSys string
|
||||
switch cfgType {
|
||||
case "openid":
|
||||
// subSys = config.IdentityOpenIDSubSys
|
||||
default:
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
cfg := globalServerConfig.Clone()
|
||||
cfgList, err := globalOpenIDConfig.GetConfigList(cfg)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -238,9 +270,69 @@ func (a adminAPIHandlers) listIdentityProviders(ctx context.Context, w http.Resp
|
||||
writeSuccessResponseJSON(w, econfigData)
|
||||
}
|
||||
|
||||
// GetIdentityProviderCfg:
|
||||
//
|
||||
// GET <admin-prefix>/idp-cfg/openid/dex_test
|
||||
func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetIdentityProviderCfg")
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
idpCfgType := mux.Vars(r)["type"]
|
||||
cfgName := mux.Vars(r)["name"]
|
||||
password := cred.SecretKey
|
||||
|
||||
if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
cfg := globalServerConfig.Clone()
|
||||
var cfgInfos []madmin.IDPCfgInfo
|
||||
var err error
|
||||
switch idpCfgType {
|
||||
case madmin.OpenidIDPCfg:
|
||||
cfgInfos, err = globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
|
||||
case madmin.LDAPIDPCfg:
|
||||
cfgInfos, err = globalLDAPConfig.GetConfigInfo(cfg, cfgName)
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, openid.ErrProviderConfigNotFound) || errors.Is(err, cfgldap.ErrProviderConfigNotFound) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
res := madmin.IDPConfig{
|
||||
Type: idpCfgType,
|
||||
Name: cfgName,
|
||||
Info: cfgInfos,
|
||||
}
|
||||
data, err := json.Marshal(res)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
econfigData, err := madmin.EncryptData(password, data)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseJSON(w, econfigData)
|
||||
}
|
||||
|
||||
// DeleteIdentityProviderCfg:
|
||||
//
|
||||
// DELETE <admin-prefix>/id-cfg?type=openid&name=dex_test
|
||||
// DELETE <admin-prefix>/idp-cfg/openid/dex_test
|
||||
func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "DeleteIdentityProviderCfg")
|
||||
|
||||
@@ -251,57 +343,82 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
|
||||
return
|
||||
}
|
||||
|
||||
cfgType := mux.Vars(r)["type"]
|
||||
idpCfgType := mux.Vars(r)["type"]
|
||||
cfgName := mux.Vars(r)["name"]
|
||||
if !idCfgTypes.Contains(cfgType) {
|
||||
// TODO: change this to invalid type error when implementation
|
||||
// is complete.
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
cfg := globalServerConfig.Clone()
|
||||
cfgCopy := globalServerConfig.Clone()
|
||||
var subSys string
|
||||
switch idpCfgType {
|
||||
case madmin.OpenidIDPCfg:
|
||||
subSys = config.IdentityOpenIDSubSys
|
||||
cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfgCopy, cfgName)
|
||||
if err != nil {
|
||||
if errors.Is(err, openid.ErrProviderConfigNotFound) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
|
||||
if err != nil {
|
||||
if err == openid.ErrProviderConfigNotFound {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
hasEnv := false
|
||||
for _, ci := range cfgInfos {
|
||||
if ci.IsCfg && ci.IsEnv {
|
||||
hasEnv = true
|
||||
break
|
||||
hasEnv := false
|
||||
for _, ci := range cfgInfos {
|
||||
if ci.IsCfg && ci.IsEnv {
|
||||
hasEnv = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if hasEnv {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigEnvOverridden), r.URL)
|
||||
return
|
||||
}
|
||||
if hasEnv {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigEnvOverridden), r.URL)
|
||||
return
|
||||
}
|
||||
case madmin.LDAPIDPCfg:
|
||||
subSys = config.IdentityLDAPSubSys
|
||||
cfgInfos, err := globalLDAPConfig.GetConfigInfo(cfgCopy, cfgName)
|
||||
if err != nil {
|
||||
if errors.Is(err, openid.ErrProviderConfigNotFound) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var subSys string
|
||||
switch cfgType {
|
||||
case "openid":
|
||||
subSys = config.IdentityOpenIDSubSys
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
hasEnv := false
|
||||
for _, ci := range cfgInfos {
|
||||
if ci.IsCfg && ci.IsEnv {
|
||||
hasEnv = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if hasEnv {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigEnvOverridden), r.URL)
|
||||
return
|
||||
}
|
||||
default:
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
cfg, err = readServerConfig(ctx, objectAPI)
|
||||
cfg, err := readServerConfig(ctx, objectAPI, nil)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err = cfg.DelKVS(fmt.Sprintf("%s:%s", subSys, cfgName)); err != nil {
|
||||
cfgKey := fmt.Sprintf("%s:%s", subSys, cfgName)
|
||||
if cfgName == madmin.Default {
|
||||
cfgKey = subSys
|
||||
}
|
||||
if err = cfg.DelKVS(cfgKey); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
// Copyright (c) 2015-2022 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
)
|
||||
|
||||
// ListLDAPPolicyMappingEntities lists users/groups mapped to given/all policies.
|
||||
//
|
||||
// GET <admin-prefix>/idp/ldap/policy-entities?[query-params]
|
||||
//
|
||||
// Query params:
|
||||
//
|
||||
// user=... -> repeatable query parameter, specifying users to query for
|
||||
// policy mapping
|
||||
//
|
||||
// group=... -> repeatable query parameter, specifying groups to query for
|
||||
// policy mapping
|
||||
//
|
||||
// policy=... -> repeatable query parameter, specifying policy to query for
|
||||
// user/group mapping
|
||||
//
|
||||
// When all query parameters are omitted, returns mappings for all policies.
|
||||
func (a adminAPIHandlers) ListLDAPPolicyMappingEntities(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListLDAPPolicyMappingEntities")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
// Check authorization.
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r,
|
||||
iampolicy.ListGroupsAdminAction, iampolicy.ListUsersAdminAction, iampolicy.ListUserPoliciesAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Validate API arguments.
|
||||
|
||||
q := madmin.PolicyEntitiesQuery{
|
||||
Users: r.Form["user"],
|
||||
Groups: r.Form["group"],
|
||||
Policy: r.Form["policy"],
|
||||
}
|
||||
|
||||
// Query IAM
|
||||
|
||||
res, err := globalIAMSys.QueryLDAPPolicyEntities(r.Context(), q)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Encode result and send response.
|
||||
|
||||
data, err := json.Marshal(res)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
password := cred.SecretKey
|
||||
econfigData, err := madmin.EncryptData(password, data)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
writeSuccessResponseJSON(w, econfigData)
|
||||
}
|
||||
|
||||
// AttachDetachPolicyLDAP attaches or detaches policies from an LDAP entity
|
||||
// (user or group).
|
||||
//
|
||||
// POST <admin-prefix>/idp/ldap/policy/{operation}
|
||||
func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "AttachDetachPolicyLDAP")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
// Check authorization.
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.UpdatePolicyAssociationAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
|
||||
// More than maxConfigSize bytes were available
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Ensure body content type is opaque to ensure that request body has not
|
||||
// been interpreted as form data.
|
||||
contentType := r.Header.Get("Content-Type")
|
||||
if contentType != "application/octet-stream" {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate operation
|
||||
operation := mux.Vars(r)["operation"]
|
||||
if operation != "attach" && operation != "detach" {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
isAttach := operation == "attach"
|
||||
|
||||
// Validate API arguments in body.
|
||||
password := cred.SecretKey
|
||||
reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err, logger.Application)
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var par madmin.PolicyAssociationReq
|
||||
err = json.Unmarshal(reqBytes, &par)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err := par.IsValid(); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Call IAM subsystem
|
||||
updatedAt, addedOrRemoved, err := globalIAMSys.PolicyDBUpdateLDAP(ctx, isAttach, par)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
respBody := madmin.PolicyAssociationResp{
|
||||
UpdatedAt: updatedAt,
|
||||
}
|
||||
if isAttach {
|
||||
respBody.PoliciesAttached = addedOrRemoved
|
||||
} else {
|
||||
respBody.PoliciesDetached = addedOrRemoved
|
||||
}
|
||||
|
||||
data, err := json.Marshal(respBody)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
encryptedData, err := madmin.EncryptData(password, data)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseJSON(w, encryptedData)
|
||||
}
|
||||
@@ -19,6 +19,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
@@ -27,6 +28,11 @@ import (
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
)
|
||||
|
||||
var (
|
||||
errRebalanceDecommissionAlreadyRunning = errors.New("Rebalance cannot be started, decommission is aleady in progress")
|
||||
errDecommissionRebalanceAlreadyRunning = errors.New("Decommission cannot be started, rebalance is already in progress")
|
||||
)
|
||||
|
||||
func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "StartDecommission")
|
||||
|
||||
@@ -49,6 +55,11 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
|
||||
return
|
||||
}
|
||||
|
||||
if pools.IsRebalanceStarted() {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errDecommissionRebalanceAlreadyRunning), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
v := vars["pool"]
|
||||
|
||||
@@ -200,3 +211,137 @@ func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
logger.LogIf(r.Context(), json.NewEncoder(w).Encode(poolsStatus))
|
||||
}
|
||||
|
||||
func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "RebalanceStart")
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// NB rebalance-start admin API is always coordinated from first pool's
|
||||
// first node. The following is required to serialize (the effects of)
|
||||
// concurrent rebalance-start commands.
|
||||
if ep := globalEndpoints[0].Endpoints[0]; !ep.IsLocal {
|
||||
for nodeIdx, proxyEp := range globalProxyEndpoints {
|
||||
if proxyEp.Endpoint.Host == ep.Host {
|
||||
if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pools, ok := objectAPI.(*erasureServerPools)
|
||||
if !ok || len(pools.serverPools) == 1 {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if pools.IsDecommissionRunning() {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errRebalanceDecommissionAlreadyRunning), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if pools.IsRebalanceStarted() {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminRebalanceAlreadyStarted), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
bucketInfos, err := objectAPI.ListBuckets(ctx, BucketOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
buckets := make([]string, 0, len(bucketInfos))
|
||||
for _, bInfo := range bucketInfos {
|
||||
buckets = append(buckets, bInfo.Name)
|
||||
}
|
||||
|
||||
var id string
|
||||
if id, err = pools.initRebalanceMeta(ctx, buckets); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Rebalance routine is run on the first node of any pool participating in rebalance.
|
||||
pools.StartRebalance()
|
||||
|
||||
b, err := json.Marshal(struct {
|
||||
ID string `json:"id"`
|
||||
}{ID: id})
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseJSON(w, b)
|
||||
// Notify peers to load rebalance.bin and start rebalance routine if they happen to be
|
||||
// participating pool's leader node
|
||||
globalNotificationSys.LoadRebalanceMeta(ctx, true)
|
||||
}
|
||||
|
||||
func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "RebalanceStatus")
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Proxy rebalance-status to first pool first node, so that users see a
|
||||
// consistent view of rebalance progress even though different rebalancing
|
||||
// pools may temporarily have out of date info on the others.
|
||||
if ep := globalEndpoints[0].Endpoints[0]; !ep.IsLocal {
|
||||
for nodeIdx, proxyEp := range globalProxyEndpoints {
|
||||
if proxyEp.Endpoint.Host == ep.Host {
|
||||
if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pools, ok := objectAPI.(*erasureServerPools)
|
||||
if !ok {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
rs, err := rebalanceStatus(ctx, pools)
|
||||
if err != nil {
|
||||
if errors.Is(err, errRebalanceNotStarted) || errors.Is(err, errConfigNotFound) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminRebalanceNotStarted), r.URL)
|
||||
return
|
||||
}
|
||||
logger.LogIf(ctx, fmt.Errorf("failed to fetch rebalance status: %w", err))
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
logger.LogIf(r.Context(), json.NewEncoder(w).Encode(rs))
|
||||
}
|
||||
|
||||
func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "RebalanceStop")
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
pools, ok := objectAPI.(*erasureServerPools)
|
||||
if !ok {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Cancel any ongoing rebalance operation
|
||||
globalNotificationSys.StopRebalance(r.Context())
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
logger.LogIf(ctx, pools.saveRebalanceStats(GlobalContext, 0, rebalSaveStoppedAt))
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
@@ -114,37 +114,23 @@ func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request
|
||||
default:
|
||||
err = errSRInvalidRequest(errInvalidArgument)
|
||||
case madmin.MakeWithVersioningBktOp:
|
||||
_, isLockEnabled := r.Form["lockEnabled"]
|
||||
_, isVersioningEnabled := r.Form["versioningEnabled"]
|
||||
_, isForceCreate := r.Form["forceCreate"]
|
||||
createdAtStr := strings.TrimSpace(r.Form.Get("createdAt"))
|
||||
createdAt, cerr := time.Parse(time.RFC3339Nano, createdAtStr)
|
||||
createdAt, cerr := time.Parse(time.RFC3339Nano, strings.TrimSpace(r.Form.Get("createdAt")))
|
||||
if cerr != nil {
|
||||
createdAt = timeSentinel
|
||||
}
|
||||
|
||||
opts := MakeBucketOptions{
|
||||
Location: r.Form.Get("location"),
|
||||
LockEnabled: isLockEnabled,
|
||||
VersioningEnabled: isVersioningEnabled,
|
||||
ForceCreate: isForceCreate,
|
||||
LockEnabled: r.Form.Get("lockEnabled") == "true",
|
||||
VersioningEnabled: r.Form.Get("versioningEnabled") == "true",
|
||||
ForceCreate: r.Form.Get("forceCreate") == "true",
|
||||
CreatedAt: createdAt,
|
||||
}
|
||||
err = globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts)
|
||||
case madmin.ConfigureReplBktOp:
|
||||
err = globalSiteReplicationSys.PeerBucketConfigureReplHandler(ctx, bucket)
|
||||
case madmin.DeleteBucketBktOp:
|
||||
_, noRecreate := r.Form["noRecreate"]
|
||||
case madmin.DeleteBucketBktOp, madmin.ForceDeleteBucketBktOp:
|
||||
err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{
|
||||
Force: false,
|
||||
NoRecreate: noRecreate,
|
||||
SRDeleteOp: getSRBucketDeleteOp(true),
|
||||
})
|
||||
case madmin.ForceDeleteBucketBktOp:
|
||||
_, noRecreate := r.Form["noRecreate"]
|
||||
err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{
|
||||
Force: true,
|
||||
NoRecreate: noRecreate,
|
||||
Force: operation == madmin.ForceDeleteBucketBktOp,
|
||||
SRDeleteOp: getSRBucketDeleteOp(true),
|
||||
})
|
||||
case madmin.PurgeDeletedBucketOp:
|
||||
@@ -515,3 +501,45 @@ func (a adminAPIHandlers) SRPeerRemove(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// SiteReplicationResyncOp - PUT /minio/admin/v3/site-replication/resync/op
|
||||
func (a adminAPIHandlers) SiteReplicationResyncOp(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "SiteReplicationResyncOp")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationResyncAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
var peerSite madmin.PeerInfo
|
||||
if err := parseJSONBody(ctx, r.Body, &peerSite, ""); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
vars := mux.Vars(r)
|
||||
op := madmin.SiteResyncOp(vars["operation"])
|
||||
var (
|
||||
status madmin.SRResyncOpStatus
|
||||
err error
|
||||
)
|
||||
switch op {
|
||||
case madmin.SiteResyncStart:
|
||||
status, err = globalSiteReplicationSys.startResync(ctx, objectAPI, peerSite)
|
||||
case madmin.SiteResyncCancel:
|
||||
status, err = globalSiteReplicationSys.cancelResync(ctx, objectAPI, peerSite)
|
||||
default:
|
||||
err = errSRInvalidRequest(errInvalidArgument)
|
||||
}
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
body, err := json.Marshal(status)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
writeSuccessResponseJSON(w, body)
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
minio "github.com/minio/minio-go/v7"
|
||||
)
|
||||
|
||||
|
||||
+429
-23
@@ -25,11 +25,12 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/klauspost/compress/zip"
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/config/dns"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
@@ -135,7 +136,7 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// Add ldap users which have mapped policies if in LDAP mode
|
||||
// FIXME(vadmeste): move this to policy info in the future
|
||||
ldapUsers, err := globalIAMSys.ListLDAPUsers()
|
||||
ldapUsers, err := globalIAMSys.ListLDAPUsers(ctx)
|
||||
if err != nil && err != errIAMActionNotAllowed {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -528,6 +529,89 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// TemporaryAccountInfo - GET /minio/admin/v3/temporary-account-info
|
||||
func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "TemporaryAccountInfo")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil || globalNotificationSys == nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
cred, claims, owner, s3Err := validateAdminSignature(ctx, r, "")
|
||||
if s3Err != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
accessKey := mux.Vars(r)["accessKey"]
|
||||
if accessKey == "" {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Action: iampolicy.ListTemporaryAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
|
||||
IsOwner: owner,
|
||||
Claims: claims,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
stsAccount, policy, err := globalIAMSys.GetTemporaryAccount(ctx, accessKey)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var stsAccountPolicy iampolicy.Policy
|
||||
|
||||
if policy != nil {
|
||||
stsAccountPolicy = *policy
|
||||
} else {
|
||||
policiesNames, err := globalIAMSys.PolicyDBGet(stsAccount.ParentUser, false)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
stsAccountPolicy = globalIAMSys.GetCombinedPolicy(policiesNames...)
|
||||
}
|
||||
|
||||
policyJSON, err := json.MarshalIndent(stsAccountPolicy, "", " ")
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
infoResp := madmin.TemporaryAccountInfoResp{
|
||||
ParentUser: stsAccount.ParentUser,
|
||||
AccountStatus: stsAccount.Status,
|
||||
ImpliedPolicy: policy == nil,
|
||||
Policy: string(policyJSON),
|
||||
}
|
||||
|
||||
data, err := json.Marshal(infoResp)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseJSON(w, encryptedData)
|
||||
}
|
||||
|
||||
// AddServiceAccount - PUT /minio/admin/v3/add-service-account
|
||||
func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "AddServiceAccount")
|
||||
@@ -581,6 +665,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
|
||||
opts := newServiceAccountOpts{
|
||||
accessKey: createReq.AccessKey,
|
||||
secretKey: createReq.SecretKey,
|
||||
comment: createReq.Comment,
|
||||
claims: make(map[string]interface{}),
|
||||
}
|
||||
|
||||
@@ -657,7 +742,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
// In case of LDAP we need to resolve the targetUser to a DN and
|
||||
// query their groups:
|
||||
if globalLDAPConfig.Enabled {
|
||||
if globalLDAPConfig.Enabled() {
|
||||
opts.claims[ldapUserN] = targetUser // simple username
|
||||
targetUser, targetGroups, err = globalLDAPConfig.LookupUserDN(targetUser)
|
||||
if err != nil {
|
||||
@@ -721,6 +806,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
|
||||
AccessKey: newCred.AccessKey,
|
||||
SecretKey: newCred.SecretKey,
|
||||
Groups: newCred.Groups,
|
||||
Comment: newCred.Comment,
|
||||
Claims: opts.claims,
|
||||
SessionPolicy: createReq.Policy,
|
||||
Status: auth.AccountOn,
|
||||
@@ -808,6 +894,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
opts := updateServiceAccountOpts{
|
||||
secretKey: updateReq.NewSecretKey,
|
||||
status: updateReq.NewStatus,
|
||||
comment: updateReq.NewComment,
|
||||
sessionPolicy: sp,
|
||||
}
|
||||
updatedAt, err := globalIAMSys.UpdateServiceAccount(ctx, accessKey, opts)
|
||||
@@ -825,6 +912,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
AccessKey: accessKey,
|
||||
SecretKey: opts.secretKey,
|
||||
Status: opts.status,
|
||||
Comment: opts.comment,
|
||||
SessionPolicy: updateReq.NewPolicy,
|
||||
},
|
||||
},
|
||||
@@ -909,6 +997,7 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
|
||||
|
||||
infoResp := madmin.InfoServiceAccountResp{
|
||||
ParentUser: svcAccount.ParentUser,
|
||||
Comment: svcAccount.Comment,
|
||||
AccountStatus: svcAccount.Status,
|
||||
ImpliedPolicy: policy == nil,
|
||||
Policy: string(policyJSON),
|
||||
@@ -1149,15 +1238,12 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
return rd, wr
|
||||
}
|
||||
|
||||
var dataUsageInfo DataUsageInfo
|
||||
var err error
|
||||
if !globalIsGateway {
|
||||
// Load the latest calculated data usage
|
||||
dataUsageInfo, _ = loadDataUsageFromBackend(ctx, objectAPI)
|
||||
}
|
||||
// Load the latest calculated data usage
|
||||
dataUsageInfo, _ := loadDataUsageFromBackend(ctx, objectAPI)
|
||||
|
||||
// If etcd, dns federation configured list buckets from etcd.
|
||||
var buckets []BucketInfo
|
||||
var err error
|
||||
if globalDNSConfig != nil && globalBucketFederation {
|
||||
dnsBuckets, err := globalDNSConfig.List()
|
||||
if err != nil && !IsErrIgnored(err,
|
||||
@@ -1189,31 +1275,46 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
accountName = cred.ParentUser
|
||||
}
|
||||
|
||||
roleArn := iampolicy.Args{Claims: claims}.GetRoleArn()
|
||||
policySetFromClaims, hasPolicyClaim := iampolicy.GetPoliciesFromClaims(claims, iamPolicyClaimNameOpenID())
|
||||
var effectivePolicy iampolicy.Policy
|
||||
|
||||
var buf []byte
|
||||
if accountName == globalActiveCred.AccessKey {
|
||||
switch {
|
||||
case accountName == globalActiveCred.AccessKey:
|
||||
for _, policy := range iampolicy.DefaultPolicies {
|
||||
if policy.Name == "consoleAdmin" {
|
||||
buf, err = json.MarshalIndent(policy.Definition, "", " ")
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
effectivePolicy = policy.Definition
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
||||
case roleArn != "":
|
||||
_, policy, err := globalIAMSys.GetRolePolicy(roleArn)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
policySlice := newMappedPolicy(policy).toSlice()
|
||||
effectivePolicy = globalIAMSys.GetCombinedPolicy(policySlice...)
|
||||
|
||||
case hasPolicyClaim:
|
||||
effectivePolicy = globalIAMSys.GetCombinedPolicy(policySetFromClaims.ToSlice()...)
|
||||
|
||||
default:
|
||||
policies, err := globalIAMSys.PolicyDBGet(accountName, false, cred.Groups...)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
effectivePolicy = globalIAMSys.GetCombinedPolicy(policies...)
|
||||
|
||||
buf, err = json.MarshalIndent(globalIAMSys.GetCombinedPolicy(policies...), "", " ")
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
buf, err = json.MarshalIndent(effectivePolicy, "", " ")
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
acctInfo := madmin.AccountInfo{
|
||||
@@ -1477,7 +1578,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
|
||||
|
||||
// Version in policy must not be empty
|
||||
if iamPolicy.Version == "" {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL)
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrPolicyInvalidVersion), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1571,6 +1672,309 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
|
||||
}
|
||||
}
|
||||
|
||||
// ListPolicyMappingEntities - GET /minio/admin/v3/idp/builtin/polciy-entities?policy=xxx&user=xxx&group=xxx
|
||||
func (a adminAPIHandlers) ListPolicyMappingEntities(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListPolicyMappingEntities")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
// Check authorization.
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r,
|
||||
iampolicy.ListGroupsAdminAction, iampolicy.ListUsersAdminAction, iampolicy.ListUserPoliciesAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Validate API arguments.
|
||||
q := madmin.PolicyEntitiesQuery{
|
||||
Users: r.Form["user"],
|
||||
Groups: r.Form["group"],
|
||||
Policy: r.Form["policy"],
|
||||
}
|
||||
|
||||
// Query IAM
|
||||
res, err := globalIAMSys.QueryPolicyEntities(r.Context(), q)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Encode result and send response.
|
||||
data, err := json.Marshal(res)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
password := cred.SecretKey
|
||||
econfigData, err := madmin.EncryptData(password, data)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
writeSuccessResponseJSON(w, econfigData)
|
||||
}
|
||||
|
||||
// AttachPolicyBuiltin - POST /minio/admin/v3/idp/builtin/attach
|
||||
func (a adminAPIHandlers) AttachPolicyBuiltin(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "AttachPolicyBuiltin")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AttachPolicyAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
cred, _, _, s3Err := validateAdminSignature(ctx, r, "")
|
||||
if s3Err != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
password := cred.SecretKey
|
||||
|
||||
reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var par madmin.PolicyAssociationReq
|
||||
if err = json.Unmarshal(reqBytes, &par); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err = par.IsValid(); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
userOrGroup := par.User
|
||||
isGroup := false
|
||||
|
||||
if userOrGroup == "" {
|
||||
userOrGroup = par.Group
|
||||
isGroup = true
|
||||
}
|
||||
|
||||
if isGroup {
|
||||
_, err := globalIAMSys.GetGroupDescription(userOrGroup)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
ok, _, err := globalIAMSys.IsTempUser(userOrGroup)
|
||||
if err != nil && err != errNoSuchUser {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if ok {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate that user exists.
|
||||
if globalIAMSys.GetUsersSysType() == MinIOUsersSysType {
|
||||
_, ok := globalIAMSys.GetUser(ctx, userOrGroup)
|
||||
if !ok {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
userType := regUser
|
||||
if globalIAMSys.GetUsersSysType() == LDAPUsersSysType {
|
||||
userType = stsUser
|
||||
}
|
||||
|
||||
var existingPolicies []string
|
||||
if isGroup {
|
||||
existingPolicies, err = globalIAMSys.PolicyDBGet(userOrGroup, true)
|
||||
} else {
|
||||
existingPolicies, err = globalIAMSys.GetUserPolicies(userOrGroup)
|
||||
}
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
policyMap := make(map[string]bool)
|
||||
for _, p := range existingPolicies {
|
||||
policyMap[p] = true
|
||||
}
|
||||
|
||||
policiesToAttach := par.Policies
|
||||
|
||||
// Check if policy is already attached to user.
|
||||
for _, p := range policiesToAttach {
|
||||
if _, ok := policyMap[p]; ok {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrPolicyAlreadyAttached), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
existingPolicies = append(existingPolicies, policiesToAttach...)
|
||||
newPolicies := strings.Join(existingPolicies, ",")
|
||||
|
||||
updatedAt, err := globalIAMSys.PolicyDBSet(ctx, userOrGroup, newPolicies, userType, isGroup)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
|
||||
Type: madmin.SRIAMItemPolicyMapping,
|
||||
PolicyMapping: &madmin.SRPolicyMapping{
|
||||
UserOrGroup: userOrGroup,
|
||||
UserType: int(userType),
|
||||
IsGroup: isGroup,
|
||||
Policy: strings.Join(policiesToAttach, ","),
|
||||
},
|
||||
UpdatedAt: updatedAt,
|
||||
}); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeResponse(w, http.StatusCreated, nil, mimeNone)
|
||||
}
|
||||
|
||||
// DetachPolicyBuiltin - POST /minio/admin/v3/idp/builtin/detach
|
||||
func (a adminAPIHandlers) DetachPolicyBuiltin(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "DetachPolicyBuiltin")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AttachPolicyAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
cred, _, _, s3Err := validateAdminSignature(ctx, r, "")
|
||||
if s3Err != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
password := cred.SecretKey
|
||||
|
||||
reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var par madmin.PolicyAssociationReq
|
||||
if err = json.Unmarshal(reqBytes, &par); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err = par.IsValid(); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
userOrGroup := par.User
|
||||
isGroup := false
|
||||
|
||||
if userOrGroup == "" {
|
||||
userOrGroup = par.Group
|
||||
isGroup = true
|
||||
}
|
||||
|
||||
if isGroup {
|
||||
_, err := globalIAMSys.GetGroupDescription(userOrGroup)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
ok, _, err := globalIAMSys.IsTempUser(userOrGroup)
|
||||
if err != nil && err != errNoSuchUser {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if ok {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate that user exists.
|
||||
if globalIAMSys.GetUsersSysType() == MinIOUsersSysType {
|
||||
_, ok := globalIAMSys.GetUser(ctx, userOrGroup)
|
||||
if !ok {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
userType := regUser
|
||||
if globalIAMSys.GetUsersSysType() == LDAPUsersSysType {
|
||||
userType = stsUser
|
||||
}
|
||||
|
||||
var existingPolicies []string
|
||||
if isGroup {
|
||||
existingPolicies, err = globalIAMSys.PolicyDBGet(userOrGroup, true)
|
||||
} else {
|
||||
existingPolicies, err = globalIAMSys.GetUserPolicies(userOrGroup)
|
||||
}
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
policyMap := make(map[string]bool)
|
||||
for _, p := range existingPolicies {
|
||||
policyMap[p] = true
|
||||
}
|
||||
|
||||
policiesToDetach := par.Policies
|
||||
|
||||
// Check if policy is already attached to user.
|
||||
for _, p := range policiesToDetach {
|
||||
if _, ok := policyMap[p]; ok {
|
||||
delete(policyMap, p)
|
||||
} else {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrPolicyNotAttached), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
newPoliciesSl := []string{}
|
||||
for p := range policyMap {
|
||||
newPoliciesSl = append(newPoliciesSl, p)
|
||||
}
|
||||
|
||||
newPolicies := strings.Join(newPoliciesSl, ",")
|
||||
|
||||
updatedAt, err := globalIAMSys.PolicyDBSet(ctx, userOrGroup, newPolicies, userType, isGroup)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
|
||||
Type: madmin.SRIAMItemPolicyMapping,
|
||||
PolicyMapping: &madmin.SRPolicyMapping{
|
||||
UserOrGroup: userOrGroup,
|
||||
UserType: int(userType),
|
||||
IsGroup: isGroup,
|
||||
Policy: strings.Join(policiesToDetach, ","),
|
||||
},
|
||||
UpdatedAt: updatedAt,
|
||||
}); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Return successful JSON response
|
||||
writeSuccessNoContent(w)
|
||||
}
|
||||
|
||||
const (
|
||||
allPoliciesFile = "policies.json"
|
||||
allUsersFile = "users.json"
|
||||
@@ -2068,6 +2472,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
opts := updateServiceAccountOpts{
|
||||
secretKey: svcAcctReq.SecretKey,
|
||||
status: svcAcctReq.Status,
|
||||
comment: svcAcctReq.Comment,
|
||||
sessionPolicy: sp,
|
||||
}
|
||||
_, err = globalIAMSys.UpdateServiceAccount(ctx, svcAcctReq.AccessKey, opts)
|
||||
@@ -2082,11 +2487,12 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
secretKey: svcAcctReq.SecretKey,
|
||||
sessionPolicy: sp,
|
||||
claims: svcAcctReq.Claims,
|
||||
comment: svcAcctReq.Comment,
|
||||
}
|
||||
|
||||
// In case of LDAP we need to resolve the targetUser to a DN and
|
||||
// query their groups:
|
||||
if globalLDAPConfig.Enabled {
|
||||
if globalLDAPConfig.Enabled() {
|
||||
opts.claims[ldapUserN] = svcAcctReq.AccessKey // simple username
|
||||
targetUser, _, err := globalLDAPConfig.LookupUserDN(svcAcctReq.AccessKey)
|
||||
if err != nil {
|
||||
|
||||
@@ -33,7 +33,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
cr "github.com/minio/minio-go/v7/pkg/credentials"
|
||||
@@ -1203,6 +1203,47 @@ func (c *check) mustNotListObjects(ctx context.Context, client *minio.Client, bu
|
||||
}
|
||||
}
|
||||
|
||||
func (c *check) mustPutObjectWithTags(ctx context.Context, client *minio.Client, bucket, object string) {
|
||||
c.Helper()
|
||||
_, err := client.PutObject(ctx, bucket, object, bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{
|
||||
UserTags: map[string]string{
|
||||
"security": "public",
|
||||
"virus": "true",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
c.Fatalf("user was unable to upload the object: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *check) mustGetObject(ctx context.Context, client *minio.Client, bucket, object string) {
|
||||
c.Helper()
|
||||
|
||||
r, err := client.GetObject(ctx, bucket, object, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
c.Fatalf("user was unable to download the object: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
_, err = io.Copy(io.Discard, r)
|
||||
if err != nil {
|
||||
c.Fatalf("user was unable to download the object: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *check) mustHeadObject(ctx context.Context, client *minio.Client, bucket, object string, tagCount int) {
|
||||
c.Helper()
|
||||
|
||||
oinfo, err := client.StatObject(ctx, bucket, object, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
c.Fatalf("user was unable to download the object: %v", err)
|
||||
}
|
||||
|
||||
if oinfo.UserTagCount != tagCount {
|
||||
c.Fatalf("expected tagCount: %d, got %d", tagCount, oinfo.UserTagCount)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *check) mustListObjects(ctx context.Context, client *minio.Client, bucket string) {
|
||||
c.Helper()
|
||||
res := client.ListObjects(ctx, bucket, minio.ListObjectsOptions{})
|
||||
|
||||
+269
-333
@@ -21,8 +21,12 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
crand "crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/subtle"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
@@ -43,14 +47,14 @@ import (
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/klauspost/compress/zip"
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/madmin-go/v2/estream"
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/logger/message/log"
|
||||
"github.com/minio/minio/internal/pubsub"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
xnet "github.com/minio/pkg/net"
|
||||
"github.com/secure-io/sio-go"
|
||||
@@ -340,8 +344,7 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
// ignores any errors here.
|
||||
storageInfo, _ := objectAPI.StorageInfo(ctx)
|
||||
storageInfo := objectAPI.StorageInfo(ctx)
|
||||
|
||||
// Collect any disk healing.
|
||||
healing, _ := getAggregatedBackgroundHealState(ctx, nil)
|
||||
@@ -411,6 +414,7 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
}
|
||||
}
|
||||
jobID := r.Form.Get("by-jobID")
|
||||
|
||||
hosts := strings.Split(r.Form.Get("hosts"), ",")
|
||||
byHost := strings.EqualFold(r.Form.Get("by-host"), "true")
|
||||
@@ -423,7 +427,7 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dID := r.Form.Get("by-depID")
|
||||
done := ctx.Done()
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
@@ -432,12 +436,21 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
|
||||
enc := json.NewEncoder(w)
|
||||
for n > 0 {
|
||||
var m madmin.RealtimeMetrics
|
||||
mLocal := collectLocalMetrics(types, hostMap, diskMap)
|
||||
mLocal := collectLocalMetrics(types, collectMetricsOpts{
|
||||
hosts: hostMap,
|
||||
disks: diskMap,
|
||||
jobID: jobID,
|
||||
depID: dID,
|
||||
})
|
||||
m.Merge(&mLocal)
|
||||
|
||||
// Allow half the interval for collecting remote...
|
||||
cctx, cancel := context.WithTimeout(ctx, interval/2)
|
||||
mRemote := collectRemoteMetrics(cctx, types, hostMap, diskMap)
|
||||
mRemote := collectRemoteMetrics(cctx, types, collectMetricsOpts{
|
||||
hosts: hostMap,
|
||||
disks: diskMap,
|
||||
jobID: jobID,
|
||||
depID: dID,
|
||||
})
|
||||
cancel()
|
||||
m.Merge(&mRemote)
|
||||
if !byHost {
|
||||
@@ -449,7 +462,7 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
|
||||
|
||||
m.Final = n <= 1
|
||||
|
||||
// Marshal API response
|
||||
// Marshal API reesponse
|
||||
if err := enc.Encode(&m); err != nil {
|
||||
n = 0
|
||||
}
|
||||
@@ -498,9 +511,10 @@ func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Re
|
||||
writeSuccessResponseJSON(w, dataUsageInfoJSON)
|
||||
}
|
||||
|
||||
func lriToLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntry {
|
||||
func lriToLockEntry(l lockRequesterInfo, now time.Time, resource, server string) *madmin.LockEntry {
|
||||
entry := &madmin.LockEntry{
|
||||
Timestamp: l.Timestamp,
|
||||
Elapsed: now.Sub(l.Timestamp),
|
||||
Resource: resource,
|
||||
ServerList: []string{server},
|
||||
Source: l.Source,
|
||||
@@ -517,6 +531,7 @@ func lriToLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEn
|
||||
}
|
||||
|
||||
func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries {
|
||||
now := time.Now().UTC()
|
||||
entryMap := make(map[string]*madmin.LockEntry)
|
||||
for _, peerLock := range peerLocks {
|
||||
if peerLock == nil {
|
||||
@@ -527,7 +542,7 @@ func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries {
|
||||
if val, ok := entryMap[lockReqInfo.Name]; ok {
|
||||
val.ServerList = append(val.ServerList, peerLock.Addr)
|
||||
} else {
|
||||
entryMap[lockReqInfo.Name] = lriToLockEntry(lockReqInfo, k, peerLock.Addr)
|
||||
entryMap[lockReqInfo.Name] = lriToLockEntry(lockReqInfo, now, k, peerLock.Addr)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -935,11 +950,6 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
hip, errCode := extractHealInitParams(mux.Vars(r), r.Form, r.Body)
|
||||
if errCode != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
|
||||
@@ -1126,12 +1136,6 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
|
||||
return
|
||||
}
|
||||
|
||||
// Check if this setup has an erasure coded backend.
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
aggregateHealStateResult, err := getAggregatedBackgroundHealState(r.Context(), objectAPI)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
@@ -1166,7 +1170,7 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL)
|
||||
return
|
||||
}
|
||||
defer nsLock.Unlock(lkctx.Cancel)
|
||||
defer nsLock.Unlock(lkctx)
|
||||
|
||||
durationStr := r.Form.Get(peerRESTDuration)
|
||||
duration, err := time.ParseDuration(durationStr)
|
||||
@@ -1207,11 +1211,6 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
sizeStr := r.Form.Get(peerRESTSize)
|
||||
durationStr := r.Form.Get(peerRESTDuration)
|
||||
concurrentStr := r.Form.Get(peerRESTConcurrent)
|
||||
@@ -1234,7 +1233,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
|
||||
duration = time.Second * 10
|
||||
}
|
||||
|
||||
storageInfo, _ := objectAPI.StorageInfo(ctx)
|
||||
storageInfo := objectAPI.StorageInfo(ctx)
|
||||
|
||||
sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, storageInfo, concurrent, size, autotune)
|
||||
if !sufficientCapacity {
|
||||
@@ -1320,7 +1319,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
|
||||
}
|
||||
|
||||
func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer, bucketName string) (bucketExists bool, err error) {
|
||||
if err = objectAPI.MakeBucketWithLocation(ctx, bucketName, MakeBucketOptions{}); err != nil {
|
||||
if err = objectAPI.MakeBucket(ctx, bucketName, MakeBucketOptions{}); err != nil {
|
||||
if _, ok := err.(BucketExists); !ok {
|
||||
// Only BucketExists error can be ignored.
|
||||
return false, err
|
||||
@@ -1333,7 +1332,6 @@ func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer, bucketName
|
||||
func deleteObjectPerfBucket(objectAPI ObjectLayer) {
|
||||
objectAPI.DeleteBucket(context.Background(), globalObjectPerfBucket, DeleteBucketOptions{
|
||||
Force: true,
|
||||
NoRecreate: true,
|
||||
SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
|
||||
})
|
||||
}
|
||||
@@ -1380,11 +1378,6 @@ func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.R
|
||||
return
|
||||
}
|
||||
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Freeze all incoming S3 API calls before running speedtest.
|
||||
globalNotificationSys.ServiceFreeze(ctx, true)
|
||||
|
||||
@@ -1516,16 +1509,12 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
|
||||
// Use buffered channel to take care of burst sends or slow w.Write()
|
||||
traceCh := make(chan pubsub.Maskable, 4000)
|
||||
traceCh := make(chan madmin.TraceInfo, 4000)
|
||||
|
||||
peers, _ := newPeerRestClients(globalEndpoints)
|
||||
mask := pubsub.MaskFromMaskable(traceOpts.TraceTypes())
|
||||
|
||||
err = globalTrace.Subscribe(mask, traceCh, ctx.Done(), func(entry pubsub.Maskable) bool {
|
||||
if e, ok := entry.(madmin.TraceInfo); ok {
|
||||
return shouldTrace(e, traceOpts)
|
||||
}
|
||||
return false
|
||||
err = globalTrace.Subscribe(traceOpts.TraceTypes(), traceCh, ctx.Done(), func(entry madmin.TraceInfo) bool {
|
||||
return shouldTrace(entry, traceOpts)
|
||||
})
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrSlowDown), r.URL)
|
||||
@@ -1597,7 +1586,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
setEventStreamHeaders(w)
|
||||
|
||||
logCh := make(chan pubsub.Maskable, 4000)
|
||||
logCh := make(chan log.Info, 4000)
|
||||
|
||||
peers, _ := newPeerRestClients(globalEndpoints)
|
||||
|
||||
@@ -1623,9 +1612,11 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
for {
|
||||
select {
|
||||
case entry := <-logCh:
|
||||
log, ok := entry.(log.Info)
|
||||
if ok && log.SendLog(node, logKind) {
|
||||
case log, ok := <-logCh:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if log.SendLog(node, logKind) {
|
||||
if err := enc.Encode(log); err != nil {
|
||||
return
|
||||
}
|
||||
@@ -1789,8 +1780,8 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
|
||||
kmsStat := fetchKMSStatus()
|
||||
|
||||
ldap := madmin.LDAP{}
|
||||
if globalLDAPConfig.Enabled {
|
||||
ldapConn, err := globalLDAPConfig.Connect()
|
||||
if globalLDAPConfig.Enabled() {
|
||||
ldapConn, err := globalLDAPConfig.LDAP.Connect()
|
||||
//nolint:gocritic
|
||||
if err != nil {
|
||||
ldap.Status = string(madmin.ItemOffline)
|
||||
@@ -1901,7 +1892,7 @@ func getKubernetesInfo(dctx context.Context) madmin.KubernetesInfo {
|
||||
}
|
||||
|
||||
client := &http.Client{
|
||||
Transport: NewGatewayHTTPTransport(),
|
||||
Transport: NewHTTPTransport(),
|
||||
Timeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
@@ -1918,78 +1909,7 @@ func getKubernetesInfo(dctx context.Context) madmin.KubernetesInfo {
|
||||
return ki
|
||||
}
|
||||
|
||||
// HealthInfoHandler - GET /minio/admin/v3/healthinfo
|
||||
// ----------
|
||||
// Get server health info
|
||||
func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "HealthInfo")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
query := r.Form
|
||||
healthInfo := madmin.HealthInfo{
|
||||
Version: madmin.HealthInfoVersion,
|
||||
Minio: madmin.MinioHealthInfo{
|
||||
Info: madmin.MinioInfo{
|
||||
DeploymentID: globalDeploymentID,
|
||||
},
|
||||
},
|
||||
}
|
||||
healthInfoCh := make(chan madmin.HealthInfo)
|
||||
|
||||
enc := json.NewEncoder(w)
|
||||
|
||||
setCommonHeaders(w)
|
||||
|
||||
setEventStreamHeaders(w)
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
errResp := func(err error) {
|
||||
errorResponse := getAPIErrorResponse(ctx, toAdminAPIErr(ctx, err), r.URL.String(),
|
||||
w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
|
||||
encodedErrorResponse := encodeResponse(errorResponse)
|
||||
healthInfo.Error = string(encodedErrorResponse)
|
||||
logger.LogIf(ctx, enc.Encode(healthInfo))
|
||||
}
|
||||
|
||||
deadline := 10 * time.Second // Default deadline is 10secs for health diagnostics.
|
||||
if query.Get(string(madmin.HealthDataTypePerfNet)) != "" ||
|
||||
query.Get(string(madmin.HealthDataTypePerfDrive)) != "" ||
|
||||
query.Get(string(madmin.HealthDataTypePerfObj)) != "" {
|
||||
deadline = 1 * time.Hour
|
||||
}
|
||||
if dstr := r.Form.Get("deadline"); dstr != "" {
|
||||
var err error
|
||||
deadline, err = time.ParseDuration(dstr)
|
||||
if err != nil {
|
||||
errResp(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
nsLock := objectAPI.NewNSLock(minioMetaBucket, "health-check-in-progress")
|
||||
lkctx, err := nsLock.GetLock(ctx, newDynamicTimeout(deadline, deadline))
|
||||
if err != nil { // returns a locked lock
|
||||
errResp(err)
|
||||
return
|
||||
}
|
||||
defer nsLock.Unlock(lkctx.Cancel)
|
||||
|
||||
healthCtx, healthCancel := context.WithTimeout(lkctx.Context(), deadline)
|
||||
defer healthCancel()
|
||||
|
||||
// Freeze all incoming S3 API calls before running speedtest.
|
||||
globalNotificationSys.ServiceFreeze(ctx, true)
|
||||
|
||||
// unfreeze all incoming S3 API calls after speedtest.
|
||||
defer globalNotificationSys.ServiceFreeze(ctx, false)
|
||||
|
||||
func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *url.Values, healthInfoCh chan madmin.HealthInfo, healthInfo madmin.HealthInfo) {
|
||||
hostAnonymizer := createHostAnonymizer()
|
||||
// anonAddr - Anonymizes hosts in given input string.
|
||||
anonAddr := func(addr string) string {
|
||||
@@ -2135,7 +2055,10 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
if !globalIsDistErasure {
|
||||
// FS mode - single server - hard code to `server1`
|
||||
anonCmdLine := strings.ReplaceAll(cmdLine, globalLocalNodeName, "server1")
|
||||
return strings.ReplaceAll(anonCmdLine, globalMinioConsoleHost, "server1")
|
||||
if len(globalMinioConsoleHost) > 0 {
|
||||
anonCmdLine = strings.ReplaceAll(anonCmdLine, globalMinioConsoleHost, "server1")
|
||||
}
|
||||
return anonCmdLine
|
||||
}
|
||||
|
||||
// Server start command regex groups:
|
||||
@@ -2211,7 +2134,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
getAndWriteMinioConfig := func() {
|
||||
if query.Get("minioconfig") == "true" {
|
||||
config, err := readServerConfig(ctx, objectAPI)
|
||||
config, err := readServerConfig(healthCtx, objectAPI, nil)
|
||||
if err != nil {
|
||||
healthInfo.Minio.Config = madmin.MinioConfig{
|
||||
Error: err.Error(),
|
||||
@@ -2225,87 +2148,6 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
}
|
||||
|
||||
getAndWriteDrivePerfInfo := func() {
|
||||
if query.Get(string(madmin.HealthDataTypePerfDrive)) == "true" {
|
||||
opts := madmin.DriveSpeedTestOpts{
|
||||
Serial: false,
|
||||
BlockSize: 4 * humanize.MiByte,
|
||||
FileSize: 1 * humanize.GiByte,
|
||||
}
|
||||
|
||||
localDPI := driveSpeedTest(ctx, opts)
|
||||
healthInfo.Perf.DrivePerf = append(healthInfo.Perf.DrivePerf, localDPI)
|
||||
|
||||
perfCh := globalNotificationSys.DriveSpeedTest(ctx, opts)
|
||||
for perfInfo := range perfCh {
|
||||
healthInfo.Perf.DrivePerf = append(healthInfo.Perf.DrivePerf, perfInfo)
|
||||
}
|
||||
partialWrite(healthInfo)
|
||||
}
|
||||
}
|
||||
|
||||
getAndWriteObjPerfInfo := func() {
|
||||
if query.Get(string(madmin.HealthDataTypePerfObj)) == "true" {
|
||||
concurrent := 32
|
||||
|
||||
storageInfo, _ := objectAPI.StorageInfo(ctx)
|
||||
|
||||
size := 64 * humanize.MiByte
|
||||
autotune := true
|
||||
|
||||
sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, storageInfo, concurrent, size, autotune)
|
||||
if !sufficientCapacity {
|
||||
healthInfo.Perf.Error = capacityErrMsg
|
||||
partialWrite(healthInfo)
|
||||
return
|
||||
}
|
||||
|
||||
if !canAutotune {
|
||||
autotune = false
|
||||
}
|
||||
|
||||
bucketExists, err := makeObjectPerfBucket(ctx, objectAPI, globalObjectPerfBucket)
|
||||
if err != nil {
|
||||
healthInfo.Perf.Error = "Unable to create bucket: " + err.Error()
|
||||
partialWrite(healthInfo)
|
||||
return
|
||||
}
|
||||
|
||||
if !bucketExists {
|
||||
defer deleteObjectPerfBucket(objectAPI)
|
||||
}
|
||||
|
||||
opts := speedTestOpts{
|
||||
objectSize: size,
|
||||
concurrencyStart: concurrent,
|
||||
duration: 10 * time.Second,
|
||||
autotune: autotune,
|
||||
}
|
||||
|
||||
perfCh := objectSpeedTest(ctx, opts)
|
||||
for perfInfo := range perfCh {
|
||||
healthInfo.Perf.ObjPerf = append(healthInfo.Perf.ObjPerf, perfInfo)
|
||||
}
|
||||
partialWrite(healthInfo)
|
||||
}
|
||||
}
|
||||
|
||||
getAndWriteNetPerfInfo := func() {
|
||||
if query.Get(string(madmin.HealthDataTypePerfObj)) == "true" {
|
||||
if !globalIsDistErasure {
|
||||
return
|
||||
}
|
||||
|
||||
netPerf := globalNotificationSys.Netperf(ctx, time.Second*10)
|
||||
for _, np := range netPerf {
|
||||
np.Endpoint = anonAddr(np.Endpoint)
|
||||
healthInfo.Perf.NetPerf = append(healthInfo.Perf.NetPerf, np)
|
||||
}
|
||||
|
||||
partialWrite(healthInfo)
|
||||
}
|
||||
}
|
||||
|
||||
anonymizeNetwork := func(network map[string]string) map[string]string {
|
||||
anonNetwork := map[string]string{}
|
||||
for endpoint, status := range network {
|
||||
@@ -2335,16 +2177,13 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
getAndWriteMemInfo()
|
||||
getAndWriteProcInfo()
|
||||
getAndWriteMinioConfig()
|
||||
getAndWriteDrivePerfInfo()
|
||||
getAndWriteObjPerfInfo()
|
||||
getAndWriteNetPerfInfo()
|
||||
getAndWriteSysErrors()
|
||||
getAndWriteSysServices()
|
||||
getAndWriteSysConfig()
|
||||
|
||||
if query.Get("minioinfo") == "true" {
|
||||
infoMessage := getServerInfo(ctx, r)
|
||||
servers := []madmin.ServerInfo{}
|
||||
infoMessage := getServerInfo(healthCtx, nil)
|
||||
servers := make([]madmin.ServerInfo, 0, len(infoMessage.Servers))
|
||||
for _, server := range infoMessage.Servers {
|
||||
anonEndpoint := anonAddr(server.Endpoint)
|
||||
servers = append(servers, madmin.ServerInfo{
|
||||
@@ -2363,6 +2202,11 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
Frees: server.MemStats.Frees,
|
||||
HeapAlloc: server.MemStats.HeapAlloc,
|
||||
},
|
||||
GoMaxProcs: server.GoMaxProcs,
|
||||
NumCPU: server.NumCPU,
|
||||
RuntimeVersion: server.RuntimeVersion,
|
||||
GCStats: server.GCStats,
|
||||
MinioEnvVars: server.MinioEnvVars,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2388,6 +2232,68 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
partialWrite(healthInfo)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// HealthInfoHandler - GET /minio/admin/v3/healthinfo
|
||||
// ----------
|
||||
// Get server health info
|
||||
func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "HealthInfo")
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
query := r.Form
|
||||
healthInfoCh := make(chan madmin.HealthInfo)
|
||||
enc := json.NewEncoder(w)
|
||||
|
||||
healthInfo := madmin.HealthInfo{
|
||||
TimeStamp: time.Now().UTC(),
|
||||
Version: madmin.HealthInfoVersion,
|
||||
Minio: madmin.MinioHealthInfo{
|
||||
Info: madmin.MinioInfo{
|
||||
DeploymentID: globalDeploymentID,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
errResp := func(err error) {
|
||||
errorResponse := getAPIErrorResponse(ctx, toAdminAPIErr(ctx, err), r.URL.String(),
|
||||
w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
|
||||
encodedErrorResponse := encodeResponse(errorResponse)
|
||||
healthInfo.Error = string(encodedErrorResponse)
|
||||
logger.LogIf(ctx, enc.Encode(healthInfo))
|
||||
}
|
||||
|
||||
deadline := 10 * time.Second // Default deadline is 10secs for health diagnostics.
|
||||
if dstr := query.Get("deadline"); dstr != "" {
|
||||
var err error
|
||||
deadline, err = time.ParseDuration(dstr)
|
||||
if err != nil {
|
||||
errResp(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
nsLock := objectAPI.NewNSLock(minioMetaBucket, "health-check-in-progress")
|
||||
lkctx, err := nsLock.GetLock(ctx, newDynamicTimeout(deadline, deadline))
|
||||
if err != nil { // returns a locked lock
|
||||
errResp(err)
|
||||
return
|
||||
}
|
||||
|
||||
defer nsLock.Unlock(lkctx)
|
||||
healthCtx, healthCancel := context.WithTimeout(lkctx.Context(), deadline)
|
||||
defer healthCancel()
|
||||
|
||||
go fetchHealthInfo(healthCtx, objectAPI, &query, healthInfoCh, healthInfo)
|
||||
|
||||
setCommonHeaders(w)
|
||||
setEventStreamHeaders(w)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
@@ -2555,20 +2461,6 @@ func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
|
||||
lambdaMap[targetID.Name] = list
|
||||
}
|
||||
|
||||
for _, tgt := range globalEnvTargetList.Targets() {
|
||||
targetIDStatus := make(map[string]madmin.Status)
|
||||
active, _ := tgt.IsActive()
|
||||
targetID := tgt.ID()
|
||||
if active {
|
||||
targetIDStatus[targetID.ID] = madmin.Status{Status: string(madmin.ItemOnline)}
|
||||
} else {
|
||||
targetIDStatus[targetID.ID] = madmin.Status{Status: string(madmin.ItemOffline)}
|
||||
}
|
||||
list := lambdaMap[targetID.Name]
|
||||
list = append(list, targetIDStatus)
|
||||
lambdaMap[targetID.Name] = list
|
||||
}
|
||||
|
||||
notify := make([]map[string][]madmin.TargetIDStatus, len(lambdaMap))
|
||||
counter := 0
|
||||
for key, value := range lambdaMap {
|
||||
@@ -2625,63 +2517,21 @@ func fetchKMSStatus() madmin.KMS {
|
||||
func fetchLoggerInfo() ([]madmin.Logger, []madmin.Audit) {
|
||||
var loggerInfo []madmin.Logger
|
||||
var auditloggerInfo []madmin.Audit
|
||||
for _, target := range logger.SystemTargets() {
|
||||
if target.Endpoint() != "" {
|
||||
tgt := target.String()
|
||||
err := checkConnection(target.Endpoint(), 15*time.Second)
|
||||
if err == nil {
|
||||
mapLog := make(map[string]madmin.Status)
|
||||
mapLog[tgt] = madmin.Status{Status: string(madmin.ItemOnline)}
|
||||
loggerInfo = append(loggerInfo, mapLog)
|
||||
} else {
|
||||
mapLog := make(map[string]madmin.Status)
|
||||
mapLog[tgt] = madmin.Status{Status: string(madmin.ItemOffline)}
|
||||
loggerInfo = append(loggerInfo, mapLog)
|
||||
}
|
||||
for _, tgt := range logger.SystemTargets() {
|
||||
if tgt.Endpoint() != "" {
|
||||
loggerInfo = append(loggerInfo, madmin.Logger{tgt.String(): logger.TargetStatus(tgt)})
|
||||
}
|
||||
}
|
||||
|
||||
for _, target := range logger.AuditTargets() {
|
||||
if target.Endpoint() != "" {
|
||||
tgt := target.String()
|
||||
err := checkConnection(target.Endpoint(), 15*time.Second)
|
||||
if err == nil {
|
||||
mapAudit := make(map[string]madmin.Status)
|
||||
mapAudit[tgt] = madmin.Status{Status: string(madmin.ItemOnline)}
|
||||
auditloggerInfo = append(auditloggerInfo, mapAudit)
|
||||
} else {
|
||||
mapAudit := make(map[string]madmin.Status)
|
||||
mapAudit[tgt] = madmin.Status{Status: string(madmin.ItemOffline)}
|
||||
auditloggerInfo = append(auditloggerInfo, mapAudit)
|
||||
}
|
||||
for _, tgt := range logger.AuditTargets() {
|
||||
if tgt.Endpoint() != "" {
|
||||
auditloggerInfo = append(auditloggerInfo, madmin.Audit{tgt.String(): logger.TargetStatus(tgt)})
|
||||
}
|
||||
}
|
||||
|
||||
return loggerInfo, auditloggerInfo
|
||||
}
|
||||
|
||||
// checkConnection - ping an endpoint , return err in case of no connection
|
||||
func checkConnection(endpointStr string, timeout time.Duration) error {
|
||||
ctx, cancel := context.WithTimeout(GlobalContext, timeout)
|
||||
defer cancel()
|
||||
|
||||
client := &http.Client{
|
||||
Transport: globalProxyTransport,
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodHead, endpointStr, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer xhttp.DrainBody(resp.Body)
|
||||
return nil
|
||||
}
|
||||
|
||||
func embedFileInZip(zipWriter *zip.Writer, name string, data []byte) error {
|
||||
// Send profiling data to zip as file
|
||||
header, zerr := zip.FileInfoHeader(dummyFileInfo{
|
||||
@@ -2704,13 +2554,14 @@ func embedFileInZip(zipWriter *zip.Writer, name string, data []byte) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// appendClusterMetaInfoToZip gets information of the current cluster and embedded
|
||||
// it in the passed zipwriter, This is not a critical function and it is allowed
|
||||
// to fail with a ten seconds timeout.
|
||||
func appendClusterMetaInfoToZip(ctx context.Context, zipWriter *zip.Writer) {
|
||||
// getClusterMetaInfo gets information of the current cluster and
|
||||
// returns it.
|
||||
// This is not a critical function, and it is allowed
|
||||
// to fail with a ten seconds timeout, returning nil.
|
||||
func getClusterMetaInfo(ctx context.Context) []byte {
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
||||
// Add a ten seconds timeout because getting profiling data
|
||||
@@ -2719,13 +2570,16 @@ func appendClusterMetaInfoToZip(ctx context.Context, zipWriter *zip.Writer) {
|
||||
defer cancel()
|
||||
|
||||
resultCh := make(chan madmin.ClusterRegistrationInfo)
|
||||
|
||||
go func() {
|
||||
defer close(resultCh)
|
||||
|
||||
ci := madmin.ClusterRegistrationInfo{}
|
||||
ci.Info.NoOfServerPools = len(globalEndpoints)
|
||||
ci.Info.NoOfServers = len(globalEndpoints.Hostnames())
|
||||
ci.Info.MinioVersion = Version
|
||||
|
||||
si, _ := objectAPI.StorageInfo(ctx)
|
||||
si := objectAPI.StorageInfo(ctx)
|
||||
|
||||
ci.Info.NoOfDrives = len(si.Disks)
|
||||
for _, disk := range si.Disks {
|
||||
@@ -2741,25 +2595,39 @@ func appendClusterMetaInfoToZip(ctx context.Context, zipWriter *zip.Writer) {
|
||||
|
||||
ci.DeploymentID = globalDeploymentID
|
||||
ci.ClusterName = fmt.Sprintf("%d-servers-%d-disks-%s", ci.Info.NoOfServers, ci.Info.NoOfDrives, ci.Info.MinioVersion)
|
||||
resultCh <- ci
|
||||
|
||||
select {
|
||||
case resultCh <- ci:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
return nil
|
||||
case ci := <-resultCh:
|
||||
out, err := json.MarshalIndent(ci, "", " ")
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return
|
||||
}
|
||||
err = embedFileInZip(zipWriter, "cluster.info", out)
|
||||
out, err := json.MarshalIndent(ci, "", " ")
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
func bytesToPublicKey(pub []byte) (*rsa.PublicKey, error) {
|
||||
block, _ := pem.Decode(pub)
|
||||
if block != nil {
|
||||
pub = block.Bytes
|
||||
}
|
||||
key, err := x509.ParsePKCS1PublicKey(pub)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// getRawDataer provides an interface for getting raw FS files.
|
||||
type getRawDataer interface {
|
||||
GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, info StatInfo) error) error
|
||||
@@ -2786,12 +2654,17 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
if err := parseForm(r); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
volume := r.Form.Get("volume")
|
||||
file := r.Form.Get("file")
|
||||
if len(volume) == 0 {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketName), r.URL)
|
||||
return
|
||||
}
|
||||
file := r.Form.Get("file")
|
||||
if len(file) == 0 {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||
return
|
||||
@@ -2804,41 +2677,102 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
var key [32]byte
|
||||
// MUST use crypto/rand
|
||||
n, err := crand.Read(key[:])
|
||||
if err != nil || n != len(key) {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
|
||||
return
|
||||
}
|
||||
stream, err := sio.AES_256_GCM.Stream(key[:])
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
|
||||
return
|
||||
}
|
||||
// Zero nonce, we only use each key once, and 32 bytes is plenty.
|
||||
nonce := make([]byte, stream.NonceSize())
|
||||
encw := stream.EncryptWriter(w, nonce, nil)
|
||||
var publicKey *rsa.PublicKey
|
||||
|
||||
defer encw.Close()
|
||||
publicKeyB64 := r.Form.Get("public-key")
|
||||
if publicKeyB64 != "" {
|
||||
publicKeyBytes, err := base64.StdEncoding.DecodeString(publicKeyB64)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
publicKey, err = bytesToPublicKey(publicKeyBytes)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Write a version for making *incompatible* changes.
|
||||
// The AdminClient will reject any version it does not know.
|
||||
w.Write([]byte{1})
|
||||
var inspectZipW *zip.Writer
|
||||
if publicKey != nil {
|
||||
w.WriteHeader(200)
|
||||
stream := estream.NewWriter(w)
|
||||
defer stream.Close()
|
||||
|
||||
// Write key first (without encryption)
|
||||
_, err = w.Write(key[:])
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
|
||||
return
|
||||
clusterKey, err := bytesToPublicKey(getSubnetAdminPublicKey())
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, stream.AddError(err.Error()))
|
||||
return
|
||||
}
|
||||
err = stream.AddKeyEncrypted(clusterKey)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, stream.AddError(err.Error()))
|
||||
return
|
||||
}
|
||||
if b := getClusterMetaInfo(ctx); len(b) > 0 {
|
||||
w, err := stream.AddEncryptedStream("cluster.info", nil)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return
|
||||
}
|
||||
w.Write(b)
|
||||
w.Close()
|
||||
}
|
||||
|
||||
// Add new key for inspect data.
|
||||
if err := stream.AddKeyEncrypted(publicKey); err != nil {
|
||||
logger.LogIf(ctx, stream.AddError(err.Error()))
|
||||
return
|
||||
}
|
||||
encStream, err := stream.AddEncryptedStream("inspect.zip", nil)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, stream.AddError(err.Error()))
|
||||
return
|
||||
}
|
||||
defer encStream.Close()
|
||||
|
||||
inspectZipW = zip.NewWriter(encStream)
|
||||
defer inspectZipW.Close()
|
||||
} else {
|
||||
// Legacy: Remove if we stop supporting inspection without public key.
|
||||
var key [32]byte
|
||||
// MUST use crypto/rand
|
||||
n, err := crand.Read(key[:])
|
||||
if err != nil || n != len(key) {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Write a version for making *incompatible* changes.
|
||||
// The AdminClient will reject any version it does not know.
|
||||
if publicKey == nil {
|
||||
w.Write([]byte{1})
|
||||
w.Write(key[:])
|
||||
}
|
||||
|
||||
stream, err := sio.AES_256_GCM.Stream(key[:])
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return
|
||||
}
|
||||
// Zero nonce, we only use each key once, and 32 bytes is plenty.
|
||||
nonce := make([]byte, stream.NonceSize())
|
||||
encw := stream.EncryptWriter(w, nonce, nil)
|
||||
defer encw.Close()
|
||||
|
||||
// Initialize a zip writer which will provide a zipped content
|
||||
// of profiling data of all nodes
|
||||
inspectZipW = zip.NewWriter(encw)
|
||||
defer inspectZipW.Close()
|
||||
|
||||
if b := getClusterMetaInfo(ctx); len(b) > 0 {
|
||||
logger.LogIf(ctx, embedFileInZip(inspectZipW, "cluster.info", b))
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize a zip writer which will provide a zipped content
|
||||
// of profiling data of all nodes
|
||||
zipWriter := zip.NewWriter(encw)
|
||||
defer zipWriter.Close()
|
||||
rawDataFn := func(r io.Reader, host, disk, filename string, si StatInfo) error {
|
||||
// Prefix host+disk
|
||||
filename = path.Join(host, disk, filename)
|
||||
@@ -2867,17 +2801,17 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
|
||||
return nil
|
||||
}
|
||||
header.Method = zip.Deflate
|
||||
zwriter, zerr := zipWriter.CreateHeader(header)
|
||||
zwriter, zerr := inspectZipW.CreateHeader(header)
|
||||
if zerr != nil {
|
||||
logger.LogIf(ctx, zerr)
|
||||
return nil
|
||||
}
|
||||
if _, err = io.Copy(zwriter, r); err != nil {
|
||||
if _, err := io.Copy(zwriter, r); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
err = o.GetRawData(ctx, volume, file, rawDataFn)
|
||||
err := o.GetRawData(ctx, volume, file, rawDataFn)
|
||||
if !errors.Is(err, errFileNotFound) {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
@@ -2889,22 +2823,24 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
|
||||
if !errors.Is(err, errFileNotFound) {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
// save args passed to inspect command
|
||||
inspectArgs := []string{fmt.Sprintf(" Inspect path: %s%s%s\n", volume, slashSeparator, file)}
|
||||
cmdLine := []string{"Server command line args: "}
|
||||
for _, pool := range globalEndpoints {
|
||||
cmdLine = append(cmdLine, pool.CmdLine)
|
||||
}
|
||||
cmdLine = append(cmdLine, "\n")
|
||||
inspectArgs = append(inspectArgs, cmdLine...)
|
||||
inspectArgsBytes := []byte(strings.Join(inspectArgs, " "))
|
||||
if err = rawDataFn(bytes.NewReader(inspectArgsBytes), "", "", "inspect-input.txt", StatInfo{
|
||||
Size: int64(len(inspectArgsBytes)),
|
||||
}); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
|
||||
appendClusterMetaInfoToZip(ctx, zipWriter)
|
||||
// save args passed to inspect command
|
||||
var sb bytes.Buffer
|
||||
fmt.Fprintf(&sb, "Inspect path: %s%s%s\n", volume, slashSeparator, file)
|
||||
sb.WriteString("Server command line args:")
|
||||
for _, pool := range globalEndpoints {
|
||||
sb.WriteString(" ")
|
||||
sb.WriteString(pool.CmdLine)
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
logger.LogIf(ctx, embedFileInZip(inspectZipW, "inspect-input.txt", sb.Bytes()))
|
||||
}
|
||||
|
||||
func getSubnetAdminPublicKey() []byte {
|
||||
if globalIsCICD {
|
||||
return subnetAdminPublicKeyDev
|
||||
}
|
||||
return subnetAdminPublicKey
|
||||
}
|
||||
|
||||
func createHostAnonymizerForFSMode() map[string]string {
|
||||
|
||||
@@ -30,7 +30,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
)
|
||||
|
||||
@@ -73,7 +73,7 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
|
||||
|
||||
globalEndpoints = mustGetPoolEndpoints(erasureDirs...)
|
||||
|
||||
initAllSubsystems()
|
||||
initAllSubsystems(ctx)
|
||||
|
||||
initConfigSubsystem(ctx, objLayer)
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
)
|
||||
|
||||
@@ -100,14 +100,14 @@ type allHealState struct {
|
||||
}
|
||||
|
||||
// newHealState - initialize global heal state management
|
||||
func newHealState(cleanup bool) *allHealState {
|
||||
func newHealState(ctx context.Context, cleanup bool) *allHealState {
|
||||
hstate := &allHealState{
|
||||
healSeqMap: make(map[string]*healSequence),
|
||||
healLocalDisks: make(map[Endpoint]bool),
|
||||
healStatus: make(map[string]healingTracker),
|
||||
}
|
||||
if cleanup {
|
||||
go hstate.periodicHealSeqsClean(GlobalContext)
|
||||
go hstate.periodicHealSeqsClean(ctx)
|
||||
}
|
||||
return hstate
|
||||
}
|
||||
@@ -222,8 +222,8 @@ func (ahs *allHealState) periodicHealSeqsClean(ctx context.Context) {
|
||||
// getHealSequenceByToken - Retrieve a heal sequence by token. The second
|
||||
// argument returns if a heal sequence actually exists.
|
||||
func (ahs *allHealState) getHealSequenceByToken(token string) (h *healSequence, exists bool) {
|
||||
ahs.Lock()
|
||||
defer ahs.Unlock()
|
||||
ahs.RLock()
|
||||
defer ahs.RUnlock()
|
||||
for _, healSeq := range ahs.healSeqMap {
|
||||
if healSeq.clientToken == token {
|
||||
return healSeq, true
|
||||
@@ -235,8 +235,8 @@ func (ahs *allHealState) getHealSequenceByToken(token string) (h *healSequence,
|
||||
// getHealSequence - Retrieve a heal sequence by path. The second
|
||||
// argument returns if a heal sequence actually exists.
|
||||
func (ahs *allHealState) getHealSequence(path string) (h *healSequence, exists bool) {
|
||||
ahs.Lock()
|
||||
defer ahs.Unlock()
|
||||
ahs.RLock()
|
||||
defer ahs.RUnlock()
|
||||
h, exists = ahs.healSeqMap[path]
|
||||
return h, exists
|
||||
}
|
||||
|
||||
+46
-15
@@ -23,7 +23,7 @@ import (
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/klauspost/compress/gzhttp"
|
||||
"github.com/klauspost/compress/gzip"
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
)
|
||||
|
||||
@@ -60,7 +60,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
|
||||
// Info operations
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/info").HandlerFunc(gz(httpTraceAll(adminAPI.ServerInfoHandler)))
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/inspect-data").HandlerFunc(httpTraceHdrs(adminAPI.InspectDataHandler)).Queries("volume", "{volume:.*}", "file", "{file:.*}")
|
||||
adminRouter.Methods(http.MethodGet, http.MethodPost).Path(adminVersion + "/inspect-data").HandlerFunc(httpTraceAll(adminAPI.InspectDataHandler))
|
||||
|
||||
// StorageInfo operations
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.StorageInfoHandler)))
|
||||
@@ -84,6 +84,11 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/decommission").HandlerFunc(gz(httpTraceAll(adminAPI.StartDecommission))).Queries("pool", "{pool:.*}")
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/cancel").HandlerFunc(gz(httpTraceAll(adminAPI.CancelDecommission))).Queries("pool", "{pool:.*}")
|
||||
|
||||
// Rebalance operations
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/rebalance/start").HandlerFunc(gz(httpTraceAll(adminAPI.RebalanceStart)))
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/rebalance/status").HandlerFunc(gz(httpTraceAll(adminAPI.RebalanceStatus)))
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/rebalance/stop").HandlerFunc(gz(httpTraceAll(adminAPI.RebalanceStop)))
|
||||
}
|
||||
|
||||
// Profiling operations - deprecated API
|
||||
@@ -137,12 +142,18 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-service-accounts").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListServiceAccounts)))
|
||||
adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/delete-service-account").HandlerFunc(gz(httpTraceHdrs(adminAPI.DeleteServiceAccount))).Queries("accessKey", "{accessKey:.*}")
|
||||
|
||||
// STS accounts ops
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/temporary-account-info").HandlerFunc(gz(httpTraceHdrs(adminAPI.TemporaryAccountInfo))).Queries("accessKey", "{accessKey:.*}")
|
||||
|
||||
// Info policy IAM latest
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(gz(httpTraceHdrs(adminAPI.InfoCannedPolicy))).Queries("name", "{name:.*}")
|
||||
// List policies latest
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-canned-policies").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListBucketPolicies))).Queries("bucket", "{bucket:.*}")
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-canned-policies").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListCannedPolicies)))
|
||||
|
||||
// Builtin IAM policy associations
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/builtin/policy-entities").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListPolicyMappingEntities)))
|
||||
|
||||
// Remove policy IAM
|
||||
adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-canned-policy").HandlerFunc(gz(httpTraceHdrs(adminAPI.RemoveCannedPolicy))).Queries("name", "{name:.*}")
|
||||
|
||||
@@ -151,6 +162,12 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
HandlerFunc(gz(httpTraceHdrs(adminAPI.SetPolicyForUserOrGroup))).
|
||||
Queries("policyName", "{policyName:.*}", "userOrGroup", "{userOrGroup:.*}", "isGroup", "{isGroup:true|false}")
|
||||
|
||||
// Attach policies to user or group
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/builtin/policy/attach").HandlerFunc(gz(httpTraceHdrs(adminAPI.AttachPolicyBuiltin)))
|
||||
|
||||
// Detach policies from user or group
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/builtin/policy/detach").HandlerFunc(gz(httpTraceHdrs(adminAPI.DetachPolicyBuiltin)))
|
||||
|
||||
// Remove user IAM
|
||||
adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-user").HandlerFunc(gz(httpTraceHdrs(adminAPI.RemoveUser))).Queries("accessKey", "{accessKey:.*}")
|
||||
|
||||
@@ -179,10 +196,15 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam").HandlerFunc(httpTraceHdrs(adminAPI.ImportIAM))
|
||||
|
||||
// IDentity Provider configuration APIs
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetIdentityProviderCfg))).Queries("type", "{type:.*}").Queries("name", "{name:.*}")
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetIdentityProviderCfg))).Queries("type", "{type:.*}")
|
||||
adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.DeleteIdentityProviderCfg))).Queries("type", "{type:.*}").Queries("name", "{name:.*}")
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddIdentityProviderCfg)))
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.UpdateIdentityProviderCfg)))
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListIdentityProviderCfg)))
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetIdentityProviderCfg)))
|
||||
adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.DeleteIdentityProviderCfg)))
|
||||
|
||||
// LDAP IAM operations
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/ldap/policy-entities").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListLDAPPolicyMappingEntities)))
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/ldap/policy/{operation}").HandlerFunc(gz(httpTraceHdrs(adminAPI.AttachDetachPolicyLDAP)))
|
||||
// -- END IAM APIs --
|
||||
|
||||
// GetBucketQuotaConfig
|
||||
@@ -206,6 +228,16 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/replication/diff").HandlerFunc(
|
||||
gz(httpTraceHdrs(adminAPI.ReplicationDiffHandler))).Queries("bucket", "{bucket:.*}")
|
||||
|
||||
// Batch job operations
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/start-job").HandlerFunc(
|
||||
gz(httpTraceHdrs(adminAPI.StartBatchJob)))
|
||||
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-jobs").HandlerFunc(
|
||||
gz(httpTraceHdrs(adminAPI.ListBatchJobs)))
|
||||
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/describe-job").HandlerFunc(
|
||||
gz(httpTraceHdrs(adminAPI.DescribeBatchJob)))
|
||||
|
||||
// Bucket migration operations
|
||||
// ExportBucketMetaHandler
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/export-bucket-metadata").HandlerFunc(
|
||||
@@ -238,6 +270,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationEdit)))
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerEdit)))
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/remove").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerRemove)))
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/resync/op").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationResyncOp))).Queries("operation", "{operation:.*}")
|
||||
|
||||
if globalIsDistErasure {
|
||||
// Top locks
|
||||
@@ -264,16 +297,14 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/kms/key/create").HandlerFunc(gz(httpTraceAll(adminAPI.KMSCreateKeyHandler))).Queries("key-id", "{key-id:.*}")
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(gz(httpTraceAll(adminAPI.KMSKeyStatusHandler)))
|
||||
|
||||
if !globalIsGateway {
|
||||
// Keep obdinfo for backward compatibility with mc
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
|
||||
HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
|
||||
// -- Health API --
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/healthinfo").
|
||||
HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/bandwidth").
|
||||
HandlerFunc(gz(httpTraceHdrs(adminAPI.BandwidthMonitorHandler)))
|
||||
}
|
||||
// Keep obdinfo for backward compatibility with mc
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
|
||||
HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
|
||||
// -- Health API --
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/healthinfo").
|
||||
HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/bandwidth").
|
||||
HandlerFunc(gz(httpTraceHdrs(adminAPI.BandwidthMonitorHandler)))
|
||||
}
|
||||
|
||||
// If none of the routes match add default error handler routes
|
||||
|
||||
@@ -20,17 +20,20 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
)
|
||||
|
||||
// getLocalServerProperty - returns madmin.ServerProperties for only the
|
||||
// local endpoints from given list of endpoints
|
||||
func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Request) madmin.ServerProperties {
|
||||
var localEndpoints Endpoints
|
||||
addr := globalLocalNodeName
|
||||
if r != nil {
|
||||
addr = r.Host
|
||||
@@ -48,7 +51,6 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
|
||||
if endpoint.IsLocal {
|
||||
// Only proceed for local endpoints
|
||||
network[nodeName] = string(madmin.ItemOnline)
|
||||
localEndpoints = append(localEndpoints, endpoint)
|
||||
continue
|
||||
}
|
||||
_, present := network[nodeName]
|
||||
@@ -67,6 +69,22 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
|
||||
var memstats runtime.MemStats
|
||||
runtime.ReadMemStats(&memstats)
|
||||
|
||||
gcStats := debug.GCStats{
|
||||
// If stats.PauseQuantiles is non-empty, ReadGCStats fills
|
||||
// it with quantiles summarizing the distribution of pause time.
|
||||
// For example, if len(stats.PauseQuantiles) is 5, it will be
|
||||
// filled with the minimum, 25%, 50%, 75%, and maximum pause times.
|
||||
PauseQuantiles: make([]time.Duration, 5),
|
||||
}
|
||||
debug.ReadGCStats(&gcStats)
|
||||
// Truncate GC stats to max 5 entries.
|
||||
if len(gcStats.PauseEnd) > 5 {
|
||||
gcStats.PauseEnd = gcStats.PauseEnd[len(gcStats.PauseEnd)-5:]
|
||||
}
|
||||
if len(gcStats.Pause) > 5 {
|
||||
gcStats.Pause = gcStats.Pause[len(gcStats.Pause)-5:]
|
||||
}
|
||||
|
||||
props := madmin.ServerProperties{
|
||||
State: string(madmin.ItemInitializing),
|
||||
Endpoint: addr,
|
||||
@@ -81,14 +99,53 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
|
||||
Frees: memstats.Frees,
|
||||
HeapAlloc: memstats.HeapAlloc,
|
||||
},
|
||||
GoMaxProcs: runtime.GOMAXPROCS(0),
|
||||
NumCPU: runtime.NumCPU(),
|
||||
RuntimeVersion: runtime.Version(),
|
||||
GCStats: &madmin.GCStats{
|
||||
LastGC: gcStats.LastGC,
|
||||
NumGC: gcStats.NumGC,
|
||||
PauseTotal: gcStats.PauseTotal,
|
||||
Pause: gcStats.Pause,
|
||||
PauseEnd: gcStats.PauseEnd,
|
||||
},
|
||||
MinioEnvVars: make(map[string]string, 10),
|
||||
}
|
||||
|
||||
sensitive := map[string]struct{}{
|
||||
config.EnvAccessKey: {},
|
||||
config.EnvSecretKey: {},
|
||||
config.EnvRootUser: {},
|
||||
config.EnvRootPassword: {},
|
||||
config.EnvMinIOSubnetAPIKey: {},
|
||||
config.EnvKMSSecretKey: {},
|
||||
}
|
||||
for _, v := range os.Environ() {
|
||||
if !strings.HasPrefix(v, "MINIO") && !strings.HasPrefix(v, "_MINIO") {
|
||||
continue
|
||||
}
|
||||
split := strings.SplitN(v, "=", 2)
|
||||
key := split[0]
|
||||
value := ""
|
||||
if len(split) > 1 {
|
||||
value = split[1]
|
||||
}
|
||||
|
||||
// Do not send sensitive creds.
|
||||
if _, ok := sensitive[key]; ok || strings.Contains(strings.ToLower(key), "password") || strings.HasSuffix(strings.ToLower(key), "key") {
|
||||
props.MinioEnvVars[key] = "*** EXISTS, REDACTED ***"
|
||||
continue
|
||||
}
|
||||
props.MinioEnvVars[key] = value
|
||||
}
|
||||
|
||||
objLayer := newObjectLayerFn()
|
||||
if objLayer != nil && !globalIsGateway {
|
||||
// only need Disks information in server mode.
|
||||
storageInfo, _ := objLayer.LocalStorageInfo(GlobalContext)
|
||||
if objLayer != nil {
|
||||
storageInfo := objLayer.LocalStorageInfo(GlobalContext)
|
||||
props.State = string(madmin.ItemOnline)
|
||||
props.Disks = storageInfo.Disks
|
||||
} else {
|
||||
props.State = string(madmin.ItemOffline)
|
||||
}
|
||||
|
||||
return props
|
||||
|
||||
+77
-14
@@ -30,6 +30,7 @@ import (
|
||||
"github.com/Azure/azure-storage-blob-go/azblob"
|
||||
"google.golang.org/api/googleapi"
|
||||
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
@@ -150,7 +151,7 @@ const (
|
||||
ErrSignatureVersionNotSupported
|
||||
ErrBucketNotEmpty
|
||||
ErrAllAccessDisabled
|
||||
ErrMalformedPolicy
|
||||
ErrPolicyInvalidVersion
|
||||
ErrMissingFields
|
||||
ErrMissingCredTag
|
||||
ErrCredMalformed
|
||||
@@ -176,7 +177,6 @@ const (
|
||||
ErrBucketAlreadyOwnedByYou
|
||||
ErrInvalidDuration
|
||||
ErrBucketAlreadyExists
|
||||
ErrTooManyBuckets
|
||||
ErrMetadataTooLarge
|
||||
ErrUnsupportedMetadata
|
||||
ErrMaximumExpires
|
||||
@@ -195,6 +195,8 @@ const (
|
||||
ErrBucketTaggingNotFound
|
||||
ErrObjectLockInvalidHeaders
|
||||
ErrInvalidTagDirective
|
||||
ErrPolicyAlreadyAttached
|
||||
ErrPolicyNotAttached
|
||||
// Add new error codes here.
|
||||
|
||||
// SSE-S3/SSE-KMS related API errors
|
||||
@@ -263,7 +265,9 @@ const (
|
||||
ErrAdminNoSuchUser
|
||||
ErrAdminNoSuchGroup
|
||||
ErrAdminGroupNotEmpty
|
||||
ErrAdminNoSuchJob
|
||||
ErrAdminNoSuchPolicy
|
||||
ErrAdminPolicyChangeAlreadyApplied
|
||||
ErrAdminInvalidArgument
|
||||
ErrAdminInvalidAccessKey
|
||||
ErrAdminInvalidSecretKey
|
||||
@@ -273,6 +277,10 @@ const (
|
||||
ErrAdminNoSuchConfigTarget
|
||||
ErrAdminConfigEnvOverridden
|
||||
ErrAdminConfigDuplicateKeys
|
||||
ErrAdminConfigInvalidIDPType
|
||||
ErrAdminConfigLDAPValidation
|
||||
ErrAdminConfigIDPCfgNameAlreadyExists
|
||||
ErrAdminConfigIDPCfgNameDoesNotExist
|
||||
ErrAdminCredentialsMismatch
|
||||
ErrInsecureClientRequest
|
||||
ErrObjectTampered
|
||||
@@ -287,6 +295,10 @@ const (
|
||||
ErrSiteReplicationIAMError
|
||||
ErrSiteReplicationConfigMissing
|
||||
|
||||
// Pool rebalance errors
|
||||
ErrAdminRebalanceAlreadyStarted
|
||||
ErrAdminRebalanceNotStarted
|
||||
|
||||
// Bucket Quota error codes
|
||||
ErrAdminBucketQuotaExceeded
|
||||
ErrAdminNoSuchQuotaConfiguration
|
||||
@@ -685,11 +697,6 @@ var errorCodes = errorCodeMap{
|
||||
Description: "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrTooManyBuckets: {
|
||||
Code: "TooManyBuckets",
|
||||
Description: "You have attempted to create more buckets than allowed",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrBucketNotEmpty: {
|
||||
Code: "BucketNotEmpty",
|
||||
Description: "The bucket you tried to delete is not empty",
|
||||
@@ -705,9 +712,9 @@ var errorCodes = errorCodeMap{
|
||||
Description: "All access to this resource has been disabled.",
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
},
|
||||
ErrMalformedPolicy: {
|
||||
ErrPolicyInvalidVersion: {
|
||||
Code: "MalformedPolicy",
|
||||
Description: "Policy has invalid resource.",
|
||||
Description: "The policy must contain a valid version string",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrMissingFields: {
|
||||
@@ -1226,6 +1233,11 @@ var errorCodes = errorCodeMap{
|
||||
Description: "The specified group does not exist.",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrAdminNoSuchJob: {
|
||||
Code: "XMinioAdminNoSuchJob",
|
||||
Description: "The specified job does not exist.",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrAdminGroupNotEmpty: {
|
||||
Code: "XMinioAdminGroupNotEmpty",
|
||||
Description: "The specified group is not empty - cannot remove it.",
|
||||
@@ -1236,6 +1248,12 @@ var errorCodes = errorCodeMap{
|
||||
Description: "The canned policy does not exist.",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrAdminPolicyChangeAlreadyApplied: {
|
||||
Code: "XMinioAdminPolicyChangeAlreadyApplied",
|
||||
Description: "The specified policy change is already in effect.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
|
||||
ErrAdminInvalidArgument: {
|
||||
Code: "XMinioAdminInvalidArgument",
|
||||
Description: "Invalid arguments specified.",
|
||||
@@ -1282,6 +1300,26 @@ var errorCodes = errorCodeMap{
|
||||
Description: "JSON configuration provided has objects with duplicate keys",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminConfigInvalidIDPType: {
|
||||
Code: "XMinioAdminConfigInvalidIDPType",
|
||||
Description: fmt.Sprintf("Invalid IDP configuration type - must be one of %v", madmin.ValidIDPConfigTypes),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminConfigLDAPValidation: {
|
||||
Code: "XMinioAdminConfigLDAPValidation",
|
||||
Description: "LDAP Configuration validation failed",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminConfigIDPCfgNameAlreadyExists: {
|
||||
Code: "XMinioAdminConfigIDPCfgNameAlreadyExists",
|
||||
Description: "An IDP configuration with the given name aleady exists",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminConfigIDPCfgNameDoesNotExist: {
|
||||
Code: "XMinioAdminConfigIDPCfgNameDoesNotExist",
|
||||
Description: "No such IDP configuration exists",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminConfigNotificationTargetsFailed: {
|
||||
Code: "XMinioAdminNotificationTargetsTestFailed",
|
||||
Description: "Configuration update failed due an unsuccessful attempt to connect to one or more notification servers",
|
||||
@@ -1346,7 +1384,7 @@ var errorCodes = errorCodeMap{
|
||||
ErrSiteReplicationPeerResp: {
|
||||
Code: "XMinioSiteReplicationPeerResp",
|
||||
Description: "Error received when contacting a peer site",
|
||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrSiteReplicationBackendIssue: {
|
||||
Code: "XMinioSiteReplicationBackendIssue",
|
||||
@@ -1378,6 +1416,16 @@ var errorCodes = errorCodeMap{
|
||||
Description: "Site not found in site replication configuration",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminRebalanceAlreadyStarted: {
|
||||
Code: "XMinioAdminRebalanceAlreadyStarted",
|
||||
Description: "Pool rebalance is already started",
|
||||
HTTPStatusCode: http.StatusConflict,
|
||||
},
|
||||
ErrAdminRebalanceNotStarted: {
|
||||
Code: "XMinioAdminRebalanceNotStarted",
|
||||
Description: "Pool rebalance is not started",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrMaximumExpires: {
|
||||
Code: "AuthorizationQueryParametersError",
|
||||
Description: "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
|
||||
@@ -1893,6 +1941,16 @@ var errorCodes = errorCodeMap{
|
||||
Description: "Invalid checksum provided.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrPolicyAlreadyAttached: {
|
||||
Code: "XMinioPolicyAlreadyAttached",
|
||||
Description: "The specified policy is already attached.",
|
||||
HTTPStatusCode: http.StatusConflict,
|
||||
},
|
||||
ErrPolicyNotAttached: {
|
||||
Code: "XMinioPolicyNotAttached",
|
||||
Description: "The specified policy is not found.",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
// Add your error structure here.
|
||||
}
|
||||
|
||||
@@ -1923,8 +1981,12 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrAdminNoSuchGroup
|
||||
case errGroupNotEmpty:
|
||||
apiErr = ErrAdminGroupNotEmpty
|
||||
case errNoSuchJob:
|
||||
apiErr = ErrAdminNoSuchJob
|
||||
case errNoSuchPolicy:
|
||||
apiErr = ErrAdminNoSuchPolicy
|
||||
case errNoPolicyToAttachOrDetach:
|
||||
apiErr = ErrAdminPolicyChangeAlreadyApplied
|
||||
case errSignatureMismatch:
|
||||
apiErr = ErrSignatureDoesNotMatch
|
||||
case errInvalidRange:
|
||||
@@ -2217,9 +2279,10 @@ func toAPIError(ctx context.Context, err error) APIError {
|
||||
if apiErr.Code == "InternalError" {
|
||||
// If we see an internal error try to interpret
|
||||
// any underlying errors if possible depending on
|
||||
// their internal error types. This code is only
|
||||
// useful with gateway implementations.
|
||||
// their internal error types.
|
||||
switch e := err.(type) {
|
||||
case batchReplicationJobError:
|
||||
apiErr = APIError(e)
|
||||
case InvalidArgument:
|
||||
apiErr = APIError{
|
||||
Code: "InvalidArgument",
|
||||
@@ -2282,7 +2345,7 @@ func toAPIError(ctx context.Context, err error) APIError {
|
||||
Description: e.Message,
|
||||
HTTPStatusCode: e.StatusCode,
|
||||
}
|
||||
if globalIsGateway && strings.Contains(e.Message, "KMS is not configured") {
|
||||
if strings.Contains(e.Message, "KMS is not configured") {
|
||||
apiErr = APIError{
|
||||
Code: "NotImplemented",
|
||||
Description: e.Message,
|
||||
@@ -2306,7 +2369,7 @@ func toAPIError(ctx context.Context, err error) APIError {
|
||||
Description: e.Error(),
|
||||
HTTPStatusCode: e.Response().StatusCode,
|
||||
}
|
||||
// Add more Gateway SDKs here if any in future.
|
||||
// Add more other SDK related errors here if any in future.
|
||||
default:
|
||||
//nolint:gocritic
|
||||
if errors.Is(err, errMalformedEncoding) {
|
||||
|
||||
+38
-42
@@ -29,20 +29,20 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/amztime"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
xxml "github.com/minio/xxml"
|
||||
)
|
||||
|
||||
const (
|
||||
// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
|
||||
iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
|
||||
maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
|
||||
maxDeleteList = 1000 // Limit number of objects deleted in a delete call.
|
||||
maxUploadsList = 10000 // Limit number of uploads in a listUploadsResponse.
|
||||
maxPartsList = 10000 // Limit number of parts in a listPartsResponse.
|
||||
maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
|
||||
maxDeleteList = 1000 // Limit number of objects deleted in a delete call.
|
||||
maxUploadsList = 10000 // Limit number of uploads in a listUploadsResponse.
|
||||
maxPartsList = 10000 // Limit number of parts in a listPartsResponse.
|
||||
)
|
||||
|
||||
// LocationResponse - format for location response.
|
||||
@@ -90,8 +90,7 @@ type ListVersionsResponse struct {
|
||||
IsTruncated bool
|
||||
|
||||
CommonPrefixes []CommonPrefix
|
||||
DeleteMarkers []DeleteMarkerVersion `xml:"DeleteMarker,omitempty"`
|
||||
Versions []ObjectVersion `xml:"Version,omitempty"`
|
||||
Versions []ObjectVersion
|
||||
|
||||
// Encoding type used to encode object keys in the response.
|
||||
EncodingType string `xml:"EncodingType,omitempty"`
|
||||
@@ -166,10 +165,10 @@ type Part struct {
|
||||
Size int64
|
||||
|
||||
// Checksum values
|
||||
ChecksumCRC32 string
|
||||
ChecksumCRC32C string
|
||||
ChecksumSHA1 string
|
||||
ChecksumSHA256 string
|
||||
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
|
||||
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
|
||||
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
|
||||
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
|
||||
}
|
||||
|
||||
// ListPartsResponse - format for list parts response.
|
||||
@@ -256,6 +255,19 @@ type ObjectVersion struct {
|
||||
Object
|
||||
IsLatest bool
|
||||
VersionID string `xml:"VersionId"`
|
||||
|
||||
isDeleteMarker bool
|
||||
}
|
||||
|
||||
// MarshalXML - marshal ObjectVersion
|
||||
func (o ObjectVersion) MarshalXML(e *xxml.Encoder, start xxml.StartElement) error {
|
||||
if o.isDeleteMarker {
|
||||
start.Name.Local = "DeleteMarker"
|
||||
} else {
|
||||
start.Name.Local = "Version"
|
||||
}
|
||||
type objectVersionWrapper ObjectVersion
|
||||
return e.EncodeElement(objectVersionWrapper(o), start)
|
||||
}
|
||||
|
||||
// DeleteMarkerVersion container for delete marker metadata
|
||||
@@ -390,10 +402,10 @@ type CompleteMultipartUploadResponse struct {
|
||||
Key string
|
||||
ETag string
|
||||
|
||||
ChecksumCRC32 string
|
||||
ChecksumCRC32C string
|
||||
ChecksumSHA1 string
|
||||
ChecksumSHA256 string
|
||||
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
|
||||
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
|
||||
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
|
||||
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
|
||||
}
|
||||
|
||||
// DeleteError structure.
|
||||
@@ -469,7 +481,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
|
||||
for _, bucket := range buckets {
|
||||
listbuckets = append(listbuckets, Bucket{
|
||||
Name: bucket.Name,
|
||||
CreationDate: bucket.Created.UTC().Format(iso8601TimeFormat),
|
||||
CreationDate: amztime.ISO8601Format(bucket.Created.UTC()),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -482,7 +494,6 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
|
||||
// generates an ListBucketVersions response for the said bucket with other enumerated options.
|
||||
func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo) ListVersionsResponse {
|
||||
versions := make([]ObjectVersion, 0, len(resp.Objects))
|
||||
deleteMarkers := make([]DeleteMarkerVersion, 0, len(resp.Objects))
|
||||
|
||||
owner := Owner{
|
||||
ID: globalMinioDefaultOwnerID,
|
||||
@@ -494,24 +505,9 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
|
||||
if object.Name == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if object.DeleteMarker {
|
||||
deleteMarker := DeleteMarkerVersion{
|
||||
Key: s3EncodeName(object.Name, encodingType),
|
||||
LastModified: object.ModTime.UTC().Format(iso8601TimeFormat),
|
||||
Owner: owner,
|
||||
VersionID: object.VersionID,
|
||||
}
|
||||
if deleteMarker.VersionID == "" {
|
||||
deleteMarker.VersionID = nullVersionID
|
||||
}
|
||||
deleteMarker.IsLatest = object.IsLatest
|
||||
deleteMarkers = append(deleteMarkers, deleteMarker)
|
||||
continue
|
||||
}
|
||||
content := ObjectVersion{}
|
||||
content.Key = s3EncodeName(object.Name, encodingType)
|
||||
content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
|
||||
content.LastModified = amztime.ISO8601Format(object.ModTime.UTC())
|
||||
if object.ETag != "" {
|
||||
content.ETag = "\"" + object.ETag + "\""
|
||||
}
|
||||
@@ -527,12 +523,12 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
|
||||
content.VersionID = nullVersionID
|
||||
}
|
||||
content.IsLatest = object.IsLatest
|
||||
content.isDeleteMarker = object.DeleteMarker
|
||||
versions = append(versions, content)
|
||||
}
|
||||
|
||||
data.Name = bucket
|
||||
data.Versions = versions
|
||||
data.DeleteMarkers = deleteMarkers
|
||||
data.EncodingType = encodingType
|
||||
data.Prefix = s3EncodeName(prefix, encodingType)
|
||||
data.KeyMarker = s3EncodeName(marker, encodingType)
|
||||
@@ -569,7 +565,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
|
||||
continue
|
||||
}
|
||||
content.Key = s3EncodeName(object.Name, encodingType)
|
||||
content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
|
||||
content.LastModified = amztime.ISO8601Format(object.ModTime.UTC())
|
||||
if object.ETag != "" {
|
||||
content.ETag = "\"" + object.ETag + "\""
|
||||
}
|
||||
@@ -618,7 +614,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
|
||||
continue
|
||||
}
|
||||
content.Key = s3EncodeName(object.Name, encodingType)
|
||||
content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
|
||||
content.LastModified = amztime.ISO8601Format(object.ModTime.UTC())
|
||||
if object.ETag != "" {
|
||||
content.ETag = "\"" + object.ETag + "\""
|
||||
}
|
||||
@@ -639,7 +635,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
|
||||
case crypto.SSEC:
|
||||
content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
|
||||
}
|
||||
for k, v := range CleanMinioInternalMetadataKeys(object.UserDefined) {
|
||||
for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
|
||||
if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
|
||||
// Do not need to send any internal metadata
|
||||
// values to client.
|
||||
@@ -681,7 +677,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
|
||||
func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
|
||||
return CopyObjectResponse{
|
||||
ETag: "\"" + etag + "\"",
|
||||
LastModified: lastModified.UTC().Format(iso8601TimeFormat),
|
||||
LastModified: amztime.ISO8601Format(lastModified.UTC()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -689,7 +685,7 @@ func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectR
|
||||
func generateCopyObjectPartResponse(etag string, lastModified time.Time) CopyObjectPartResponse {
|
||||
return CopyObjectPartResponse{
|
||||
ETag: "\"" + etag + "\"",
|
||||
LastModified: lastModified.UTC().Format(iso8601TimeFormat),
|
||||
LastModified: amztime.ISO8601Format(lastModified.UTC()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -749,7 +745,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
|
||||
newPart.PartNumber = part.PartNumber
|
||||
newPart.ETag = "\"" + part.ETag + "\""
|
||||
newPart.Size = part.Size
|
||||
newPart.LastModified = part.LastModified.UTC().Format(iso8601TimeFormat)
|
||||
newPart.LastModified = amztime.ISO8601Format(part.LastModified.UTC())
|
||||
newPart.ChecksumCRC32 = part.ChecksumCRC32
|
||||
newPart.ChecksumCRC32C = part.ChecksumCRC32C
|
||||
newPart.ChecksumSHA1 = part.ChecksumSHA1
|
||||
@@ -783,7 +779,7 @@ func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMult
|
||||
newUpload := Upload{}
|
||||
newUpload.UploadID = upload.UploadID
|
||||
newUpload.Key = s3EncodeName(upload.Object, encodingType)
|
||||
newUpload.Initiated = upload.Initiated.UTC().Format(iso8601TimeFormat)
|
||||
newUpload.Initiated = amztime.ISO8601Format(upload.Initiated.UTC())
|
||||
listMultipartUploadsResponse.Uploads[index] = newUpload
|
||||
}
|
||||
return listMultipartUploadsResponse
|
||||
|
||||
+204
-195
File diff suppressed because one or more lines are too long
+96
-55
@@ -39,6 +39,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
xjwt "github.com/minio/minio/internal/jwt"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/mcontext"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
)
|
||||
@@ -84,6 +85,8 @@ func isRequestSignStreamingV4(r *http.Request) bool {
|
||||
}
|
||||
|
||||
// Authorization type.
|
||||
//
|
||||
//go:generate stringer -type=authType -trimprefix=authType $GOFILE
|
||||
type authType int
|
||||
|
||||
// List of all supported auth types.
|
||||
@@ -148,9 +151,6 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
|
||||
s3Err = isReqAuthenticated(ctx, r, region, serviceS3)
|
||||
}
|
||||
if s3Err != ErrNone {
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("requestHeaders", dumpRequest(r))
|
||||
ctx := logger.SetReqInfo(ctx, reqInfo)
|
||||
logger.LogIf(ctx, errors.New(getAPIError(s3Err).Description), logger.Application)
|
||||
return cred, nil, owner, s3Err
|
||||
}
|
||||
|
||||
@@ -293,24 +293,27 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
|
||||
//
|
||||
// returns APIErrorCode if any to be replied to the client.
|
||||
func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (s3Err APIErrorCode) {
|
||||
_, _, s3Err = checkRequestAuthTypeCredential(ctx, r, action, bucketName, objectName)
|
||||
logger.GetReqInfo(ctx).BucketName = bucketName
|
||||
logger.GetReqInfo(ctx).ObjectName = objectName
|
||||
|
||||
_, _, s3Err = checkRequestAuthTypeCredential(ctx, r, action)
|
||||
return s3Err
|
||||
}
|
||||
|
||||
// Check request auth type verifies the incoming http request
|
||||
// - validates the request signature
|
||||
// - validates the policy action if anonymous tests bucket policies if any,
|
||||
// for authenticated requests validates IAM policies.
|
||||
//
|
||||
// returns APIErrorCode if any to be replied to the client.
|
||||
// Additionally returns the accessKey used in the request, and if this request is by an admin.
|
||||
func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (cred auth.Credentials, owner bool, s3Err APIErrorCode) {
|
||||
func authenticateRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
|
||||
if logger.GetReqInfo(ctx) == nil {
|
||||
logger.LogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.Minio)
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
var cred auth.Credentials
|
||||
var owner bool
|
||||
switch getRequestAuthType(r) {
|
||||
case authTypeUnknown, authTypeStreamingSigned:
|
||||
return cred, owner, ErrSignatureVersionNotSupported
|
||||
return ErrSignatureVersionNotSupported
|
||||
case authTypePresignedV2, authTypeSignedV2:
|
||||
if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone {
|
||||
return cred, owner, s3Err
|
||||
return s3Err
|
||||
}
|
||||
cred, owner, s3Err = getReqAccessKeyV2(r)
|
||||
case authTypeSigned, authTypePresigned:
|
||||
@@ -320,52 +323,67 @@ func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action
|
||||
region = ""
|
||||
}
|
||||
if s3Err = isReqAuthenticated(ctx, r, region, serviceS3); s3Err != ErrNone {
|
||||
return cred, owner, s3Err
|
||||
return s3Err
|
||||
}
|
||||
cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
|
||||
}
|
||||
if s3Err != ErrNone {
|
||||
return cred, owner, s3Err
|
||||
return s3Err
|
||||
}
|
||||
|
||||
// LocationConstraint is valid only for CreateBucketAction.
|
||||
var locationConstraint string
|
||||
logger.GetReqInfo(ctx).Cred = cred
|
||||
logger.GetReqInfo(ctx).Owner = owner
|
||||
|
||||
// region is valid only for CreateBucketAction.
|
||||
var region string
|
||||
if action == policy.CreateBucketAction {
|
||||
// To extract region from XML in request body, get copy of request body.
|
||||
payload, err := io.ReadAll(io.LimitReader(r.Body, maxLocationConstraintSize))
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err, logger.Application)
|
||||
return cred, owner, ErrMalformedXML
|
||||
return ErrMalformedXML
|
||||
}
|
||||
|
||||
// Populate payload to extract location constraint.
|
||||
r.Body = io.NopCloser(bytes.NewReader(payload))
|
||||
|
||||
var s3Error APIErrorCode
|
||||
locationConstraint, s3Error = parseLocationConstraint(r)
|
||||
if s3Error != ErrNone {
|
||||
return cred, owner, s3Error
|
||||
region, s3Err = parseLocationConstraint(r)
|
||||
if s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
|
||||
// Populate payload again to handle it in HTTP handler.
|
||||
r.Body = io.NopCloser(bytes.NewReader(payload))
|
||||
}
|
||||
if cred.AccessKey != "" {
|
||||
logger.GetReqInfo(ctx).AccessKey = cred.AccessKey
|
||||
|
||||
logger.GetReqInfo(ctx).Region = region
|
||||
|
||||
return s3Err
|
||||
}
|
||||
|
||||
func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
if reqInfo == nil {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
cred := reqInfo.Cred
|
||||
owner := reqInfo.Owner
|
||||
region := reqInfo.Region
|
||||
bucket := reqInfo.BucketName
|
||||
object := reqInfo.ObjectName
|
||||
|
||||
if action != policy.ListAllMyBucketsAction && cred.AccessKey == "" {
|
||||
// Anonymous checks are not meant for ListBuckets action
|
||||
// Anonymous checks are not meant for ListAllBuckets action
|
||||
if globalPolicySys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Action: action,
|
||||
BucketName: bucketName,
|
||||
ConditionValues: getConditionValues(r, locationConstraint, "", nil),
|
||||
BucketName: bucket,
|
||||
ConditionValues: getConditionValues(r, region, "", nil),
|
||||
IsOwner: false,
|
||||
ObjectName: objectName,
|
||||
ObjectName: object,
|
||||
}) {
|
||||
// Request is allowed return the appropriate access key.
|
||||
return cred, owner, ErrNone
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
if action == policy.ListBucketVersionsAction {
|
||||
@@ -374,31 +392,31 @@ func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action
|
||||
if globalPolicySys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Action: policy.ListBucketAction,
|
||||
BucketName: bucketName,
|
||||
ConditionValues: getConditionValues(r, locationConstraint, "", nil),
|
||||
BucketName: bucket,
|
||||
ConditionValues: getConditionValues(r, region, "", nil),
|
||||
IsOwner: false,
|
||||
ObjectName: objectName,
|
||||
ObjectName: object,
|
||||
}) {
|
||||
// Request is allowed return the appropriate access key.
|
||||
return cred, owner, ErrNone
|
||||
return ErrNone
|
||||
}
|
||||
}
|
||||
|
||||
return cred, owner, ErrAccessDenied
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
if globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.Action(action),
|
||||
BucketName: bucketName,
|
||||
BucketName: bucket,
|
||||
ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
|
||||
ObjectName: objectName,
|
||||
ObjectName: object,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
// Request is allowed return the appropriate access key.
|
||||
return cred, owner, ErrNone
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
if action == policy.ListBucketVersionsAction {
|
||||
@@ -408,18 +426,41 @@ func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.ListBucketAction,
|
||||
BucketName: bucketName,
|
||||
BucketName: bucket,
|
||||
ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
|
||||
ObjectName: objectName,
|
||||
ObjectName: object,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
// Request is allowed return the appropriate access key.
|
||||
return cred, owner, ErrNone
|
||||
return ErrNone
|
||||
}
|
||||
}
|
||||
|
||||
return cred, owner, ErrAccessDenied
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
// Check request auth type verifies the incoming http request
|
||||
// - validates the request signature
|
||||
// - validates the policy action if anonymous tests bucket policies if any,
|
||||
// for authenticated requests validates IAM policies.
|
||||
//
|
||||
// returns APIErrorCode if any to be replied to the client.
|
||||
// Additionally returns the accessKey used in the request, and if this request is by an admin.
|
||||
func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action policy.Action) (cred auth.Credentials, owner bool, s3Err APIErrorCode) {
|
||||
s3Err = authenticateRequest(ctx, r, action)
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
if reqInfo == nil {
|
||||
return cred, owner, ErrAccessDenied
|
||||
}
|
||||
|
||||
cred = reqInfo.Cred
|
||||
owner = reqInfo.Owner
|
||||
if s3Err != ErrNone {
|
||||
return cred, owner, s3Err
|
||||
}
|
||||
|
||||
return cred, owner, authorizeRequest(ctx, r, action)
|
||||
}
|
||||
|
||||
// Verify if request has valid AWS Signature Version '2'.
|
||||
@@ -501,7 +542,7 @@ func isSupportedS3AuthType(aType authType) bool {
|
||||
func setAuthHandler(h http.Handler) http.Handler {
|
||||
// handler for validating incoming authorization headers.
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
tc, ok := r.Context().Value(contextTraceReqKey).(*traceCtxt)
|
||||
tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
|
||||
|
||||
aType := getRequestAuthType(r)
|
||||
if aType == authTypeSigned || aType == authTypeSignedV2 || aType == authTypeStreamingSigned {
|
||||
@@ -509,8 +550,8 @@ func setAuthHandler(h http.Handler) http.Handler {
|
||||
amzDate, errCode := parseAmzDateHeader(r)
|
||||
if errCode != ErrNone {
|
||||
if ok {
|
||||
tc.funcName = "handler.Auth"
|
||||
tc.responseRecorder.LogErrBody = true
|
||||
tc.FuncName = "handler.Auth"
|
||||
tc.ResponseRecorder.LogErrBody = true
|
||||
}
|
||||
|
||||
// All our internal APIs are sensitive towards Date
|
||||
@@ -525,8 +566,8 @@ func setAuthHandler(h http.Handler) http.Handler {
|
||||
curTime := UTCNow()
|
||||
if curTime.Sub(amzDate) > globalMaxSkewTime || amzDate.Sub(curTime) > globalMaxSkewTime {
|
||||
if ok {
|
||||
tc.funcName = "handler.Auth"
|
||||
tc.responseRecorder.LogErrBody = true
|
||||
tc.FuncName = "handler.Auth"
|
||||
tc.ResponseRecorder.LogErrBody = true
|
||||
}
|
||||
|
||||
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrRequestTimeTooSkewed), r.URL)
|
||||
@@ -540,8 +581,8 @@ func setAuthHandler(h http.Handler) http.Handler {
|
||||
}
|
||||
|
||||
if ok {
|
||||
tc.funcName = "handler.Auth"
|
||||
tc.responseRecorder.LogErrBody = true
|
||||
tc.FuncName = "handler.Auth"
|
||||
tc.ResponseRecorder.LogErrBody = true
|
||||
}
|
||||
|
||||
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL)
|
||||
@@ -583,7 +624,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
|
||||
|
||||
conditions := getConditionValues(r, "", cred.AccessKey, cred.Claims)
|
||||
conditions["object-lock-mode"] = []string{string(retMode)}
|
||||
conditions["object-lock-retain-until-date"] = []string{retDate.Format(time.RFC3339)}
|
||||
conditions["object-lock-retain-until-date"] = []string{retDate.UTC().Format(time.RFC3339)}
|
||||
if retDays > 0 {
|
||||
conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(retDays)}
|
||||
}
|
||||
@@ -623,22 +664,22 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
|
||||
func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action iampolicy.Action) (s3Err APIErrorCode) {
|
||||
var cred auth.Credentials
|
||||
var owner bool
|
||||
region := globalSite.Region
|
||||
switch atype {
|
||||
case authTypeUnknown:
|
||||
return ErrSignatureVersionNotSupported
|
||||
case authTypeSignedV2, authTypePresignedV2:
|
||||
cred, owner, s3Err = getReqAccessKeyV2(r)
|
||||
case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
|
||||
region := globalSite.Region
|
||||
cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
|
||||
}
|
||||
if s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
|
||||
if cred.AccessKey != "" {
|
||||
logger.GetReqInfo(ctx).AccessKey = cred.AccessKey
|
||||
}
|
||||
logger.GetReqInfo(ctx).Cred = cred
|
||||
logger.GetReqInfo(ctx).Owner = owner
|
||||
logger.GetReqInfo(ctx).Region = region
|
||||
|
||||
// Do not check for PutObjectRetentionAction permission,
|
||||
// if mode and retain until date are not set.
|
||||
|
||||
+11
-10
@@ -359,7 +359,10 @@ func mustNewSignedBadMD5Request(method string, urlStr string, contentLength int6
|
||||
|
||||
// Tests is requested authenticated function, tests replies for s3 errors.
|
||||
func TestIsReqAuthenticated(t *testing.T) {
|
||||
objLayer, fsDir, err := prepareFS()
|
||||
ctx, cancel := context.WithCancel(GlobalContext)
|
||||
defer cancel()
|
||||
|
||||
objLayer, fsDir, err := prepareFS(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -368,10 +371,7 @@ func TestIsReqAuthenticated(t *testing.T) {
|
||||
t.Fatalf("unable initialize config file, %s", err)
|
||||
}
|
||||
|
||||
initAllSubsystems()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
initAllSubsystems(ctx)
|
||||
|
||||
initConfigSubsystem(ctx, objLayer)
|
||||
|
||||
@@ -413,7 +413,10 @@ func TestIsReqAuthenticated(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCheckAdminRequestAuthType(t *testing.T) {
|
||||
objLayer, fsDir, err := prepareFS()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
objLayer, fsDir, err := prepareFS(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -439,7 +442,6 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
|
||||
{Request: mustNewPresignedV2Request(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
|
||||
{Request: mustNewPresignedRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
|
||||
}
|
||||
ctx := context.Background()
|
||||
for i, testCase := range testCases {
|
||||
if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, iampolicy.AllAdminActions, globalSite.Region); s3Error != testCase.ErrCode {
|
||||
t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
|
||||
@@ -451,7 +453,7 @@ func TestValidateAdminSignature(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
objLayer, fsDir, err := prepareFS()
|
||||
objLayer, fsDir, err := prepareFS(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -461,8 +463,7 @@ func TestValidateAdminSignature(t *testing.T) {
|
||||
t.Fatalf("unable initialize config file, %s", err)
|
||||
}
|
||||
|
||||
initAllSubsystems()
|
||||
|
||||
initAllSubsystems(ctx)
|
||||
initConfigSubsystem(ctx, objLayer)
|
||||
|
||||
globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
// Code generated by "stringer -type=authType -trimprefix=authType auth-handler.go"; DO NOT EDIT.
|
||||
|
||||
package cmd
|
||||
|
||||
import "strconv"
|
||||
|
||||
func _() {
|
||||
// An "invalid array index" compiler error signifies that the constant values have changed.
|
||||
// Re-run the stringer command to generate them again.
|
||||
var x [1]struct{}
|
||||
_ = x[authTypeUnknown-0]
|
||||
_ = x[authTypeAnonymous-1]
|
||||
_ = x[authTypePresigned-2]
|
||||
_ = x[authTypePresignedV2-3]
|
||||
_ = x[authTypePostPolicy-4]
|
||||
_ = x[authTypeStreamingSigned-5]
|
||||
_ = x[authTypeSigned-6]
|
||||
_ = x[authTypeSignedV2-7]
|
||||
_ = x[authTypeJWT-8]
|
||||
_ = x[authTypeSTS-9]
|
||||
}
|
||||
|
||||
const _authType_name = "UnknownAnonymousPresignedPresignedV2PostPolicyStreamingSignedSignedSignedV2JWTSTS"
|
||||
|
||||
var _authType_index = [...]uint8{0, 7, 16, 25, 36, 46, 61, 67, 75, 78, 81}
|
||||
|
||||
func (i authType) String() string {
|
||||
if i < 0 || i >= authType(len(_authType_index)-1) {
|
||||
return "authType(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _authType_name[_authType_index[i]:_authType_index[i+1]]
|
||||
}
|
||||
@@ -21,8 +21,7 @@ import (
|
||||
"context"
|
||||
"runtime"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/minio/internal/pubsub"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
)
|
||||
|
||||
// healTask represents what to heal along with options
|
||||
@@ -54,7 +53,7 @@ type healRoutine struct {
|
||||
func activeListeners() int {
|
||||
// Bucket notification and http trace are not costly, it is okay to ignore them
|
||||
// while counting the number of concurrent connections
|
||||
return int(globalHTTPListen.NumSubscribers(pubsub.MaskAll)) + int(globalTrace.NumSubscribers(pubsub.MaskAll))
|
||||
return int(globalHTTPListen.Subscribers()) + int(globalTrace.Subscribers())
|
||||
}
|
||||
|
||||
func waitForLowHTTPReq() {
|
||||
|
||||
@@ -29,7 +29,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
)
|
||||
@@ -79,6 +79,10 @@ type healingTracker struct {
|
||||
|
||||
// Filled during heal.
|
||||
HealedBuckets []string
|
||||
|
||||
// ID of the current healing operation
|
||||
HealID string
|
||||
|
||||
// Add future tracking capabilities
|
||||
// Be sure that they are included in toHealingDisk
|
||||
}
|
||||
@@ -112,11 +116,12 @@ func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker,
|
||||
}
|
||||
|
||||
// newHealingTracker will create a new healing tracker for the disk.
|
||||
func newHealingTracker(disk StorageAPI) *healingTracker {
|
||||
func newHealingTracker(disk StorageAPI, healID string) *healingTracker {
|
||||
diskID, _ := disk.GetDiskID()
|
||||
h := healingTracker{
|
||||
disk: disk,
|
||||
ID: diskID,
|
||||
HealID: healID,
|
||||
Path: disk.String(),
|
||||
Endpoint: disk.Endpoint().String(),
|
||||
Started: time.Now().UTC(),
|
||||
@@ -227,6 +232,7 @@ func (h *healingTracker) printTo(writer io.Writer) {
|
||||
func (h *healingTracker) toHealingDisk() madmin.HealingDisk {
|
||||
return madmin.HealingDisk{
|
||||
ID: h.ID,
|
||||
HealID: h.HealID,
|
||||
Endpoint: h.Endpoint,
|
||||
PoolIndex: h.PoolIndex,
|
||||
SetIndex: h.SetIndex,
|
||||
@@ -286,13 +292,11 @@ func getLocalDisksToHeal() (disksToHeal Endpoints) {
|
||||
var newDiskHealingTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
|
||||
|
||||
func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint) error {
|
||||
logger.Info(fmt.Sprintf("Proceeding to heal '%s' - 'mc admin heal alias/ --verbose' to check the status.", endpoint))
|
||||
|
||||
disk, format, err := connectEndpoint(endpoint)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error: %w, %s", err, endpoint)
|
||||
}
|
||||
|
||||
defer disk.Close()
|
||||
poolIdx := globalEndpoints.GetLocalPoolIdx(disk.Endpoint())
|
||||
if poolIdx < 0 {
|
||||
return fmt.Errorf("unexpected pool index (%d) found in %s", poolIdx, disk.Endpoint())
|
||||
@@ -310,13 +314,27 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
|
||||
}
|
||||
|
||||
// Prevent parallel erasure set healing
|
||||
locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-drive-healing/%s/%d/%d", endpoint, poolIdx, setIdx))
|
||||
locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-drive-healing/%d/%d", poolIdx, setIdx))
|
||||
lkctx, err := locker.GetLock(ctx, newDiskHealingTimeout)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer locker.Unlock(lkctx.Cancel)
|
||||
defer locker.Unlock(lkctx)
|
||||
|
||||
// Load healing tracker in this disk
|
||||
tracker, err := loadHealingTracker(ctx, disk)
|
||||
if err != nil {
|
||||
// A healing track can be not found when another disk in the same
|
||||
// erasure set and same healing-id successfully finished healing.
|
||||
if err == errFileNotFound {
|
||||
return nil
|
||||
}
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to load a healing tracker on '%s': %w", disk, err))
|
||||
tracker = newHealingTracker(disk, mustGetUUID())
|
||||
}
|
||||
|
||||
logger.Info(fmt.Sprintf("Proceeding to heal '%s' - 'mc admin heal alias/ --verbose' to check the status.", endpoint))
|
||||
|
||||
buckets, _ := z.ListBuckets(ctx, BucketOptions{})
|
||||
// Buckets data are dispersed in multiple zones/sets, make
|
||||
@@ -340,15 +358,6 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
|
||||
logger.Info("Healing drive '%v' on %s pool", disk, humanize.Ordinal(poolIdx+1))
|
||||
}
|
||||
|
||||
// Load healing tracker in this disk
|
||||
tracker, err := loadHealingTracker(ctx, disk)
|
||||
if err != nil {
|
||||
// So someone changed the drives underneath, healing tracker missing.
|
||||
logger.LogIf(ctx, fmt.Errorf("Healing tracker missing on '%s', drive was swapped again on %s pool: %w",
|
||||
disk, humanize.Ordinal(poolIdx+1), err))
|
||||
tracker = newHealingTracker(disk)
|
||||
}
|
||||
|
||||
// Load bucket totals
|
||||
cache := dataUsageCache{}
|
||||
if err := cache.load(ctx, z.serverPools[poolIdx].sets[setIdx], dataUsageCacheName); err == nil {
|
||||
@@ -379,7 +388,24 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
|
||||
logger.Info("\n")
|
||||
}
|
||||
|
||||
logger.LogIf(ctx, tracker.delete(ctx))
|
||||
if tracker.HealID == "" { // HealID is empty only before Feb 2023
|
||||
logger.LogIf(ctx, tracker.delete(ctx))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove .healing.bin from all disks with similar heal-id
|
||||
for _, disk := range z.serverPools[poolIdx].sets[setIdx].getDisks() {
|
||||
t, err := loadHealingTracker(ctx, disk)
|
||||
if err != nil {
|
||||
if err != errFileNotFound {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if t.HealID == tracker.HealID {
|
||||
t.delete(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -182,6 +182,12 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
return
|
||||
}
|
||||
}
|
||||
case "HealID":
|
||||
z.HealID, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "HealID")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
@@ -195,9 +201,9 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// map header, size 22
|
||||
// map header, size 23
|
||||
// write "ID"
|
||||
err = en.Append(0xde, 0x0, 0x16, 0xa2, 0x49, 0x44)
|
||||
err = en.Append(0xde, 0x0, 0x17, 0xa2, 0x49, 0x44)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -430,15 +436,25 @@ func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
return
|
||||
}
|
||||
}
|
||||
// write "HealID"
|
||||
err = en.Append(0xa6, 0x48, 0x65, 0x61, 0x6c, 0x49, 0x44)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.HealID)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "HealID")
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// map header, size 22
|
||||
// map header, size 23
|
||||
// string "ID"
|
||||
o = append(o, 0xde, 0x0, 0x16, 0xa2, 0x49, 0x44)
|
||||
o = append(o, 0xde, 0x0, 0x17, 0xa2, 0x49, 0x44)
|
||||
o = msgp.AppendString(o, z.ID)
|
||||
// string "PoolIndex"
|
||||
o = append(o, 0xa9, 0x50, 0x6f, 0x6f, 0x6c, 0x49, 0x6e, 0x64, 0x65, 0x78)
|
||||
@@ -509,6 +525,9 @@ func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
for za0002 := range z.HealedBuckets {
|
||||
o = msgp.AppendString(o, z.HealedBuckets[za0002])
|
||||
}
|
||||
// string "HealID"
|
||||
o = append(o, 0xa6, 0x48, 0x65, 0x61, 0x6c, 0x49, 0x44)
|
||||
o = msgp.AppendString(o, z.HealID)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -688,6 +707,12 @@ func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
return
|
||||
}
|
||||
}
|
||||
case "HealID":
|
||||
z.HealID, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "HealID")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
@@ -710,5 +735,6 @@ func (z *healingTracker) Msgsize() (s int) {
|
||||
for za0002 := range z.HealedBuckets {
|
||||
s += msgp.StringPrefixSize + len(z.HealedBuckets[za0002])
|
||||
}
|
||||
s += 7 + msgp.StringPrefixSize + len(z.HealID)
|
||||
return
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,23 @@
|
||||
// Code generated by "stringer -type=batchReplicationMetric -trimprefix=batchReplicationMetric batch-handlers.go"; DO NOT EDIT.
|
||||
|
||||
package cmd
|
||||
|
||||
import "strconv"
|
||||
|
||||
func _() {
|
||||
// An "invalid array index" compiler error signifies that the constant values have changed.
|
||||
// Re-run the stringer command to generate them again.
|
||||
var x [1]struct{}
|
||||
_ = x[batchReplicationMetricObject-0]
|
||||
}
|
||||
|
||||
const _batchReplicationMetric_name = "Object"
|
||||
|
||||
var _batchReplicationMetric_index = [...]uint8{0, 6}
|
||||
|
||||
func (i batchReplicationMetric) String() string {
|
||||
if i >= batchReplicationMetric(len(_batchReplicationMetric_index)-1) {
|
||||
return "batchReplicationMetric(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _batchReplicationMetric_name[_batchReplicationMetric_index[i]:_batchReplicationMetric_index[i+1]]
|
||||
}
|
||||
@@ -35,7 +35,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// obtains random bucket name.
|
||||
bucket := getRandomBucketName()
|
||||
// create bucket.
|
||||
err = obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
|
||||
err = obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -76,7 +76,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
|
||||
object := getRandomObjectName()
|
||||
|
||||
// create bucket.
|
||||
err = obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
|
||||
err = obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -196,7 +196,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// obtains random bucket name.
|
||||
bucket := getRandomBucketName()
|
||||
// create bucket.
|
||||
err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
|
||||
err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
+17
-4
@@ -24,6 +24,7 @@ import (
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
@@ -146,16 +147,28 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
|
||||
// Can never happen unless there are programmer bugs
|
||||
return 0, errUnexpected
|
||||
}
|
||||
ignoredErrs := []error{
|
||||
errDiskNotFound,
|
||||
}
|
||||
if strings.HasPrefix(b.volume, minioMetaBucket) {
|
||||
ignoredErrs = append(ignoredErrs,
|
||||
errFileNotFound,
|
||||
errVolumeNotFound,
|
||||
errFileVersionNotFound,
|
||||
)
|
||||
}
|
||||
if b.rc == nil {
|
||||
// For the first ReadAt() call we need to open the stream for reading.
|
||||
b.currOffset = offset
|
||||
streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
|
||||
if len(b.data) == 0 && b.tillOffset != streamOffset {
|
||||
b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
|
||||
if err != nil && err != errDiskNotFound {
|
||||
logger.LogIf(GlobalContext,
|
||||
fmt.Errorf("Reading erasure shards at (%s: %s/%s) returned '%w', will attempt to reconstruct if we have quorum",
|
||||
b.disk, b.volume, b.filePath, err))
|
||||
if err != nil {
|
||||
if !IsErr(err, ignoredErrs...) {
|
||||
logger.LogIf(GlobalContext,
|
||||
fmt.Errorf("Reading erasure shards at (%s: %s/%s) returned '%w', will attempt to reconstruct if we have quorum",
|
||||
b.disk, b.volume, b.filePath, err))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
b.rc = io.NewSectionReader(bytes.NewReader(b.data), streamOffset, b.tillOffset-streamOffset)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
// Copyright (c) 2015-2022 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
@@ -64,6 +64,7 @@ func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
|
||||
return fmt.Errorf("Expected platform '%s', found to be running '%s'",
|
||||
s1.MinioPlatform, s2.MinioPlatform)
|
||||
}
|
||||
|
||||
if s1.MinioEndpoints.NEndpoints() != s2.MinioEndpoints.NEndpoints() {
|
||||
return fmt.Errorf("Expected number of endpoints %d, seen %d", s1.MinioEndpoints.NEndpoints(),
|
||||
s2.MinioEndpoints.NEndpoints())
|
||||
@@ -200,15 +201,18 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
|
||||
srcCfg := getServerSystemCfg()
|
||||
clnts := newBootstrapRESTClients(endpointServerPools)
|
||||
var onlineServers int
|
||||
var offlineEndpoints []string
|
||||
var offlineEndpoints []error
|
||||
var incorrectConfigs []error
|
||||
var retries int
|
||||
for onlineServers < len(clnts)/2 {
|
||||
for _, clnt := range clnts {
|
||||
if err := clnt.Verify(ctx, srcCfg); err != nil {
|
||||
if !isNetworkError(err) {
|
||||
logger.LogIf(ctx, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err))
|
||||
logger.LogOnceIf(ctx, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err), clnt.String())
|
||||
incorrectConfigs = append(incorrectConfigs, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err))
|
||||
} else {
|
||||
offlineEndpoints = append(offlineEndpoints, fmt.Errorf("%s is unreachable: %w", clnt.String(), err))
|
||||
}
|
||||
offlineEndpoints = append(offlineEndpoints, clnt.String())
|
||||
continue
|
||||
}
|
||||
onlineServers++
|
||||
@@ -221,15 +225,19 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
|
||||
// 100% CPU when half the endpoints are offline.
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
retries++
|
||||
// after 5 retries start logging that servers are not reachable yet
|
||||
if retries >= 5 {
|
||||
logger.Info(fmt.Sprintf("Waiting for atleast %d remote servers to be online for bootstrap check", len(clnts)/2))
|
||||
// after 20 retries start logging that servers are not reachable yet
|
||||
if retries >= 20 {
|
||||
logger.Info(fmt.Sprintf("Waiting for atleast %d remote servers with valid configuration to be online", len(clnts)/2))
|
||||
if len(offlineEndpoints) > 0 {
|
||||
logger.Info(fmt.Sprintf("Following servers are currently offline or unreachable %s", offlineEndpoints))
|
||||
}
|
||||
if len(incorrectConfigs) > 0 {
|
||||
logger.Info(fmt.Sprintf("Following servers mismatch in their configuration %s", incorrectConfigs))
|
||||
}
|
||||
retries = 0 // reset to log again after 5 retries.
|
||||
}
|
||||
offlineEndpoints = nil
|
||||
incorrectConfigs = nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/kes"
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
@@ -204,7 +204,7 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
|
||||
}
|
||||
|
||||
// Delete bucket encryption config from object layer
|
||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, nil)
|
||||
updatedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketSSEConfig)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
|
||||
@@ -34,15 +34,6 @@ func NewBucketSSEConfigSys() *BucketSSEConfigSys {
|
||||
|
||||
// Get - gets bucket encryption config for the given bucket.
|
||||
func (sys *BucketSSEConfigSys) Get(bucket string) (*sse.BucketSSEConfig, error) {
|
||||
if globalIsGateway {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return nil, errServerNotInitialized
|
||||
}
|
||||
|
||||
return nil, BucketSSEConfigNotFound{Bucket: bucket}
|
||||
}
|
||||
|
||||
sseCfg, _, err := globalBucketMetadataSys.GetSSEConfig(bucket)
|
||||
return sseCfg, err
|
||||
}
|
||||
|
||||
+21
-34
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
// Copyright (c) 2015-2022 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
@@ -36,7 +36,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/gorilla/mux"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
sse "github.com/minio/minio/internal/bucket/encryption"
|
||||
@@ -300,7 +300,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
|
||||
listBuckets := objectAPI.ListBuckets
|
||||
|
||||
cred, owner, s3Error := checkRequestAuthTypeCredential(ctx, r, policy.ListAllMyBucketsAction, "", "")
|
||||
cred, owner, s3Error := checkRequestAuthTypeCredential(ctx, r, policy.ListAllMyBucketsAction)
|
||||
if s3Error != ErrNone && s3Error != ErrAccessDenied {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
@@ -431,7 +431,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
// Unmarshal list of keys to be deleted.
|
||||
deleteObjectsReq := &DeleteObjectsRequest{}
|
||||
if err := xmlDecoder(r.Body, deleteObjectsReq, maxBodySize); err != nil {
|
||||
logger.LogIf(ctx, err, logger.Application)
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -511,11 +510,10 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
}
|
||||
if object.VersionID != "" && object.VersionID != nullVersionID {
|
||||
if _, err := uuid.Parse(object.VersionID); err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("invalid version-id specified %w", err))
|
||||
apiErr := errorCodes.ToAPIErr(ErrNoSuchVersion)
|
||||
deleteResults[index].errInfo = DeleteError{
|
||||
Code: apiErr.Code,
|
||||
Message: apiErr.Description,
|
||||
Message: fmt.Sprintf("%s (%s)", apiErr.Description, err),
|
||||
Key: object.ObjectName,
|
||||
VersionID: object.VersionID,
|
||||
}
|
||||
@@ -731,7 +729,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
}
|
||||
|
||||
cred, owner, s3Error := checkRequestAuthTypeCredential(ctx, r, policy.CreateBucketAction, bucket, "")
|
||||
cred, owner, s3Error := checkRequestAuthTypeCredential(ctx, r, policy.CreateBucketAction)
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
@@ -769,16 +767,12 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
return
|
||||
}
|
||||
|
||||
// check if client is attempting to create more buckets than allowed maximum.
|
||||
// check if client is attempting to create more buckets, complain about it.
|
||||
if currBuckets := globalBucketMetadataSys.Count(); currBuckets+1 > maxBuckets {
|
||||
apiErr := errorCodes.ToAPIErr(ErrTooManyBuckets)
|
||||
apiErr.Description = fmt.Sprintf("You have attempted to create %d buckets than allowed %d", currBuckets+1, maxBuckets)
|
||||
writeErrorResponse(ctx, w, apiErr, r.URL)
|
||||
return
|
||||
logger.LogIf(ctx, fmt.Errorf("An attempt to create %d buckets beyond recommended %d", currBuckets+1, maxBuckets))
|
||||
}
|
||||
|
||||
opts := MakeBucketOptions{
|
||||
Location: location,
|
||||
LockEnabled: objectLockEnabled,
|
||||
ForceCreate: forceCreate,
|
||||
}
|
||||
@@ -790,15 +784,14 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
// exists elsewhere
|
||||
if err == dns.ErrNoEntriesFound || err == dns.ErrNotImplemented {
|
||||
// Proceed to creating a bucket.
|
||||
if err = objectAPI.MakeBucketWithLocation(ctx, bucket, opts); err != nil {
|
||||
if err = objectAPI.MakeBucket(ctx, bucket, opts); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err = globalDNSConfig.Put(bucket); err != nil {
|
||||
objectAPI.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{
|
||||
Force: false,
|
||||
NoRecreate: true,
|
||||
Force: true,
|
||||
SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
|
||||
})
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
@@ -842,7 +835,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
|
||||
// Proceed to creating a bucket.
|
||||
if err := objectAPI.MakeBucketWithLocation(ctx, bucket, opts); err != nil {
|
||||
if err := objectAPI.MakeBucket(ctx, bucket, opts); err != nil {
|
||||
if _, ok := err.(BucketExists); ok {
|
||||
// Though bucket exists locally, we send the site-replication
|
||||
// hook to ensure all sites have this bucket. If the hook
|
||||
@@ -858,10 +851,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
|
||||
|
||||
// Call site replication hook
|
||||
if err := globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts)
|
||||
|
||||
// Make sure to add Location information here only for bucket
|
||||
if cp := pathClean(r.URL.Path); cp != "" {
|
||||
@@ -930,16 +920,18 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
// be loaded in memory, the remaining being put in temporary files.
|
||||
reader, err := r.MultipartReader()
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
|
||||
apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
|
||||
apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
|
||||
writeErrorResponse(ctx, w, apiErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Read multipart data and save in memory and in the disk if needed
|
||||
form, err := reader.ReadForm(maxFormMemory)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err, logger.Application)
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
|
||||
apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
|
||||
apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
|
||||
writeErrorResponse(ctx, w, apiErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -949,8 +941,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
// Extract all form fields
|
||||
fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(ctx, form)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err, logger.Application)
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
|
||||
apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
|
||||
apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
|
||||
writeErrorResponse(ctx, w, apiErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1065,10 +1058,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
sseConfig, _ := globalBucketSSEConfigSys.Get(bucket)
|
||||
sseConfig.Apply(r.Header, sse.ApplyOptions{
|
||||
AutoEncrypt: globalAutoEncryption,
|
||||
Passthrough: globalIsGateway && globalGatewayName == S3BackendGateway,
|
||||
})
|
||||
|
||||
// get gateway encryption options
|
||||
var opts ObjectOptions
|
||||
opts, err = putOpts(ctx, r, bucket, object, metadata)
|
||||
if err != nil {
|
||||
@@ -1398,10 +1389,6 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketObjectLockConfigurationAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
@@ -1618,7 +1605,7 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, nil)
|
||||
updatedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketTaggingConfig)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
|
||||
@@ -168,7 +168,7 @@ func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter,
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, nil); err != nil {
|
||||
if _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketLifecycleConfig); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
+22
-32
@@ -58,15 +58,6 @@ type LifecycleSys struct{}
|
||||
|
||||
// Get - gets lifecycle config associated to a given bucket name.
|
||||
func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, err error) {
|
||||
if globalIsGateway {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return nil, errServerNotInitialized
|
||||
}
|
||||
|
||||
return nil, BucketLifecycleNotFound{Bucket: bucketName}
|
||||
}
|
||||
|
||||
return globalBucketMetadataSys.GetLifecycleConfig(bucketName)
|
||||
}
|
||||
|
||||
@@ -155,9 +146,14 @@ type newerNoncurrentTask struct {
|
||||
versions []ObjectToDelete
|
||||
}
|
||||
|
||||
type transitionTask struct {
|
||||
tier string
|
||||
objInfo ObjectInfo
|
||||
}
|
||||
|
||||
type transitionState struct {
|
||||
once sync.Once
|
||||
transitionCh chan ObjectInfo
|
||||
transitionCh chan transitionTask
|
||||
|
||||
ctx context.Context
|
||||
objAPI ObjectLayer
|
||||
@@ -171,13 +167,13 @@ type transitionState struct {
|
||||
lastDayStats map[string]*lastDayTierStats
|
||||
}
|
||||
|
||||
func (t *transitionState) queueTransitionTask(oi ObjectInfo) {
|
||||
func (t *transitionState) queueTransitionTask(oi ObjectInfo, sc string) {
|
||||
select {
|
||||
case <-GlobalContext.Done():
|
||||
t.once.Do(func() {
|
||||
close(t.transitionCh)
|
||||
})
|
||||
case t.transitionCh <- oi:
|
||||
case t.transitionCh <- transitionTask{objInfo: oi, tier: sc}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
@@ -186,7 +182,7 @@ var globalTransitionState *transitionState
|
||||
|
||||
func newTransitionState(ctx context.Context, objAPI ObjectLayer) *transitionState {
|
||||
return &transitionState{
|
||||
transitionCh: make(chan ObjectInfo, 10000),
|
||||
transitionCh: make(chan transitionTask, 10000),
|
||||
ctx: ctx,
|
||||
objAPI: objAPI,
|
||||
killCh: make(chan struct{}),
|
||||
@@ -213,27 +209,25 @@ func (t *transitionState) worker(ctx context.Context, objectAPI ObjectLayer) {
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case oi, ok := <-t.transitionCh:
|
||||
case task, ok := <-t.transitionCh:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
atomic.AddInt32(&t.activeTasks, 1)
|
||||
var tier string
|
||||
var err error
|
||||
if tier, err = transitionObject(ctx, objectAPI, oi); err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w", oi.Bucket, oi.Name, oi.VersionID, err))
|
||||
if err := transitionObject(ctx, objectAPI, task.objInfo, task.tier); err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w",
|
||||
task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
|
||||
} else {
|
||||
ts := tierStats{
|
||||
TotalSize: uint64(oi.Size),
|
||||
TotalSize: uint64(task.objInfo.Size),
|
||||
NumVersions: 1,
|
||||
}
|
||||
if oi.IsLatest {
|
||||
if task.objInfo.IsLatest {
|
||||
ts.NumObjects = 1
|
||||
}
|
||||
t.addLastDayStats(tier, ts)
|
||||
t.addLastDayStats(task.tier, ts)
|
||||
}
|
||||
atomic.AddInt32(&t.activeTasks, -1)
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -304,9 +298,10 @@ func validateTransitionTier(lc *lifecycle.Lifecycle) error {
|
||||
// This is to be called after a successful upload of an object (version).
|
||||
func enqueueTransitionImmediate(obj ObjectInfo) {
|
||||
if lc, err := globalLifecycleSys.Get(obj.Bucket); err == nil {
|
||||
switch lc.ComputeAction(obj.ToLifecycleOpts()) {
|
||||
event := lc.Eval(obj.ToLifecycleOpts())
|
||||
switch event.Action {
|
||||
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
|
||||
globalTransitionState.queueTransitionTask(obj)
|
||||
globalTransitionState.queueTransitionTask(obj, event.StorageClass)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -403,12 +398,7 @@ func genTransitionObjName(bucket string) (string, error) {
|
||||
// storage specified by the transition ARN, the metadata is left behind on source cluster and original content
|
||||
// is moved to the transition tier. Note that in the case of encrypted objects, entire encrypted stream is moved
|
||||
// to the transition tier without decrypting or re-encrypting.
|
||||
func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo) (string, error) {
|
||||
lc, err := globalLifecycleSys.Get(oi.Bucket)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
tier := lc.TransitionTier(oi.ToLifecycleOpts())
|
||||
func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo, tier string) error {
|
||||
opts := ObjectOptions{
|
||||
Transition: TransitionOptions{
|
||||
Status: lifecycle.TransitionPending,
|
||||
@@ -420,7 +410,7 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo)
|
||||
VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
|
||||
MTime: oi.ModTime,
|
||||
}
|
||||
return tier, objectAPI.TransitionObject(ctx, oi.Bucket, oi.Name, opts)
|
||||
return objectAPI.TransitionObject(ctx, oi.Bucket, oi.Name, opts)
|
||||
}
|
||||
|
||||
type auditTierOp struct {
|
||||
@@ -550,7 +540,7 @@ type RestoreObjectRequest struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ RestoreRequest" json:"-"`
|
||||
Days int `xml:"Days,omitempty"`
|
||||
Type RestoreRequestType `xml:"Type,omitempty"`
|
||||
Tier string `xml:"Tier,-"`
|
||||
Tier string `xml:"Tier"`
|
||||
Description string `xml:"Description,omitempty"`
|
||||
SelectParameters *SelectParameters `xml:"SelectParameters,omitempty"`
|
||||
OutputLocation OutputLocation `xml:"OutputLocation,omitempty"`
|
||||
|
||||
@@ -35,7 +35,7 @@ import (
|
||||
// - delimiter if set should be equal to '/', otherwise the request is rejected.
|
||||
// - marker if set should have a common prefix with 'prefix' param, otherwise
|
||||
// the request is rejected.
|
||||
func validateListObjectsArgs(marker, delimiter, encodingType string, maxKeys int) APIErrorCode {
|
||||
func validateListObjectsArgs(prefix, marker, delimiter, encodingType string, maxKeys int) APIErrorCode {
|
||||
// Max keys cannot be negative.
|
||||
if maxKeys < 0 {
|
||||
return ErrInvalidMaxKeys
|
||||
@@ -48,6 +48,14 @@ func validateListObjectsArgs(marker, delimiter, encodingType string, maxKeys int
|
||||
}
|
||||
}
|
||||
|
||||
if !IsValidObjectPrefix(prefix) {
|
||||
return ErrInvalidObjectName
|
||||
}
|
||||
|
||||
if marker != "" && !HasPrefix(marker, prefix) {
|
||||
return ErrNotImplemented
|
||||
}
|
||||
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
@@ -83,7 +91,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
|
||||
}
|
||||
|
||||
// Validate the query params before beginning to serve the request.
|
||||
if s3Error := validateListObjectsArgs(marker, delimiter, encodingType, maxkeys); s3Error != ErrNone {
|
||||
if s3Error := validateListObjectsArgs(prefix, marker, delimiter, encodingType, maxkeys); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -148,7 +156,7 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
|
||||
|
||||
// Validate the query params before beginning to serve the request.
|
||||
// fetch-owner is not validated since it is a boolean
|
||||
if s3Error := validateListObjectsArgs(token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
|
||||
if s3Error := validateListObjectsArgs(prefix, token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -218,7 +226,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
|
||||
// Validate the query params before beginning to serve the request.
|
||||
// fetch-owner is not validated since it is a boolean
|
||||
if s3Error := validateListObjectsArgs(token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
|
||||
if s3Error := validateListObjectsArgs(prefix, token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -325,7 +333,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
|
||||
}
|
||||
|
||||
// Validate all the query params before beginning to serve the request.
|
||||
if s3Error := validateListObjectsArgs(marker, delimiter, encodingType, maxKeys); s3Error != ErrNone {
|
||||
if s3Error := validateListObjectsArgs(prefix, marker, delimiter, encodingType, maxKeys); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
+19
-79
@@ -18,14 +18,13 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
bucketsse "github.com/minio/minio/internal/bucket/encryption"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
@@ -55,9 +54,6 @@ func (sys *BucketMetadataSys) Count() int {
|
||||
|
||||
// Remove bucket metadata from memory.
|
||||
func (sys *BucketMetadataSys) Remove(bucket string) {
|
||||
if globalIsGateway {
|
||||
return
|
||||
}
|
||||
sys.Lock()
|
||||
delete(sys.metadataMap, bucket)
|
||||
globalBucketMonitor.DeleteBucket(bucket)
|
||||
@@ -70,44 +66,24 @@ func (sys *BucketMetadataSys) Remove(bucket string) {
|
||||
// so they should be replaced atomically and not appended to, etc.
|
||||
// Data is not persisted to disk.
|
||||
func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
|
||||
if globalIsGateway {
|
||||
return
|
||||
}
|
||||
|
||||
if bucket != minioMetaBucket {
|
||||
if !isMinioMetaBucketName(bucket) {
|
||||
sys.Lock()
|
||||
sys.metadataMap[bucket] = meta
|
||||
sys.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// Update update bucket metadata for the specified config file.
|
||||
// The configData data should not be modified after being sent here.
|
||||
func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) (updatedAt time.Time, err error) {
|
||||
func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string, configFile string, configData []byte, parse bool) (updatedAt time.Time, err error) {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return updatedAt, errServerNotInitialized
|
||||
}
|
||||
|
||||
if globalIsGateway && globalGatewayName != NASBackendGateway {
|
||||
if configFile == bucketPolicyConfig {
|
||||
if configData == nil {
|
||||
return updatedAt, objAPI.DeleteBucketPolicy(ctx, bucket)
|
||||
}
|
||||
config, err := policy.ParseConfig(bytes.NewReader(configData), bucket)
|
||||
if err != nil {
|
||||
return updatedAt, err
|
||||
}
|
||||
return updatedAt, objAPI.SetBucketPolicy(ctx, bucket, config)
|
||||
}
|
||||
return updatedAt, NotImplemented{}
|
||||
}
|
||||
|
||||
if bucket == minioMetaBucket {
|
||||
if isMinioMetaBucketName(bucket) {
|
||||
return updatedAt, errInvalidArgument
|
||||
}
|
||||
|
||||
meta, err := loadBucketMetadata(ctx, objAPI, bucket)
|
||||
meta, err := loadBucketMetadataParse(ctx, objAPI, bucket, parse)
|
||||
if err != nil {
|
||||
if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
|
||||
// Only single drive mode needs this fallback.
|
||||
@@ -165,6 +141,18 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
|
||||
return updatedAt, nil
|
||||
}
|
||||
|
||||
// Delete delete the bucket metadata for the specified bucket.
|
||||
// must be used by all callers instead of using Update() with nil configData.
|
||||
func (sys *BucketMetadataSys) Delete(ctx context.Context, bucket string, configFile string) (updatedAt time.Time, err error) {
|
||||
return sys.updateAndParse(ctx, bucket, configFile, nil, false)
|
||||
}
|
||||
|
||||
// Update update bucket metadata for the specified bucket.
|
||||
// The configData data should not be modified after being sent here.
|
||||
func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) (updatedAt time.Time, err error) {
|
||||
return sys.updateAndParse(ctx, bucket, configFile, configData, true)
|
||||
}
|
||||
|
||||
// Get metadata for a bucket.
|
||||
// If no metadata exists errConfigNotFound is returned and a new metadata is returned.
|
||||
// Only a shallow copy is returned, so referenced data should not be modified,
|
||||
@@ -176,7 +164,7 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
|
||||
// For all other bucket specific metadata, use the relevant
|
||||
// calls implemented specifically for each of those features.
|
||||
func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
|
||||
if globalIsGateway || bucket == minioMetaBucket {
|
||||
if isMinioMetaBucketName(bucket) {
|
||||
return newBucketMetadata(bucket), errConfigNotFound
|
||||
}
|
||||
|
||||
@@ -239,22 +227,6 @@ func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Co
|
||||
// GetLifecycleConfig returns configured lifecycle config
|
||||
// The returned object may not be modified.
|
||||
func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Lifecycle, error) {
|
||||
if globalIsGateway && globalGatewayName == NASBackendGateway {
|
||||
// Only needed in case of NAS gateway.
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return nil, errServerNotInitialized
|
||||
}
|
||||
meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if meta.lifecycleConfig == nil {
|
||||
return nil, BucketLifecycleNotFound{Bucket: bucket}
|
||||
}
|
||||
return meta.lifecycleConfig, nil
|
||||
}
|
||||
|
||||
meta, err := sys.GetConfig(GlobalContext, bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, errConfigNotFound) {
|
||||
@@ -271,19 +243,6 @@ func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Life
|
||||
// GetNotificationConfig returns configured notification config
|
||||
// The returned object may not be modified.
|
||||
func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Config, error) {
|
||||
if globalIsGateway && globalGatewayName == NASBackendGateway {
|
||||
// Only needed in case of NAS gateway.
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return nil, errServerNotInitialized
|
||||
}
|
||||
meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return meta.notificationConfig, nil
|
||||
}
|
||||
|
||||
meta, err := sys.GetConfig(GlobalContext, bucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -319,15 +278,6 @@ func (sys *BucketMetadataSys) CreatedAt(bucket string) (time.Time, error) {
|
||||
// GetPolicyConfig returns configured bucket policy
|
||||
// The returned object may not be modified.
|
||||
func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, time.Time, error) {
|
||||
if globalIsGateway {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return nil, time.Time{}, errServerNotInitialized
|
||||
}
|
||||
p, err := objAPI.GetBucketPolicy(GlobalContext, bucket)
|
||||
return p, UTCNow(), err
|
||||
}
|
||||
|
||||
meta, err := sys.GetConfig(GlobalContext, bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, errConfigNotFound) {
|
||||
@@ -395,11 +345,7 @@ func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (Buc
|
||||
return newBucketMetadata(bucket), errServerNotInitialized
|
||||
}
|
||||
|
||||
if globalIsGateway {
|
||||
return newBucketMetadata(bucket), NotImplemented{}
|
||||
}
|
||||
|
||||
if bucket == minioMetaBucket {
|
||||
if isMinioMetaBucketName(bucket) {
|
||||
return newBucketMetadata(bucket), errInvalidArgument
|
||||
}
|
||||
|
||||
@@ -426,12 +372,6 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
|
||||
return errServerNotInitialized
|
||||
}
|
||||
|
||||
// In gateway mode, we don't need to load bucket metadata except
|
||||
// NAS gateway backend.
|
||||
if globalIsGateway && !objAPI.IsNotificationSupported() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Load bucket metadata sys in background
|
||||
go sys.load(ctx, buckets, objAPI)
|
||||
return nil
|
||||
|
||||
+30
-30
@@ -29,7 +29,7 @@ import (
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
bucketsse "github.com/minio/minio/internal/bucket/encryption"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
@@ -135,7 +135,7 @@ func (b *BucketMetadata) SetCreatedAt(createdAt time.Time) {
|
||||
func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string) error {
|
||||
if name == "" {
|
||||
logger.LogIf(ctx, errors.New("bucket name cannot be empty"))
|
||||
return errors.New("bucket name cannot be empty")
|
||||
return errInvalidArgument
|
||||
}
|
||||
configFile := path.Join(bucketMetaPrefix, name, bucketMetadataFile)
|
||||
data, err := readConfig(ctx, api, configFile)
|
||||
@@ -162,8 +162,7 @@ func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string)
|
||||
return err
|
||||
}
|
||||
|
||||
// loadBucketMetadata loads and migrates to bucket metadata.
|
||||
func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
|
||||
func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket string, parse bool) (BucketMetadata, error) {
|
||||
b := newBucketMetadata(bucket)
|
||||
err := b.Load(ctx, objectAPI, b.Name)
|
||||
if err != nil && !errors.Is(err, errConfigNotFound) {
|
||||
@@ -172,8 +171,22 @@ func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket strin
|
||||
if err == nil {
|
||||
b.defaultTimestamps()
|
||||
}
|
||||
// Old bucket without bucket metadata. Hence we migrate existing settings.
|
||||
if err := b.convertLegacyConfigs(ctx, objectAPI); err != nil {
|
||||
|
||||
configs, err := b.getAllLegacyConfigs(ctx, objectAPI)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
|
||||
if len(configs) == 0 {
|
||||
if parse {
|
||||
// nothing to update, parse and proceed.
|
||||
err = b.parseAllConfigs(ctx, objectAPI)
|
||||
}
|
||||
} else {
|
||||
// Old bucket without bucket metadata. Hence we migrate existing settings.
|
||||
err = b.convertLegacyConfigs(ctx, objectAPI, configs)
|
||||
}
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
|
||||
@@ -185,6 +198,11 @@ func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket strin
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// loadBucketMetadata loads and migrates to bucket metadata.
|
||||
func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
|
||||
return loadBucketMetadataParse(ctx, objectAPI, bucket, true)
|
||||
}
|
||||
|
||||
// parseAllConfigs will parse all configs and populate the private fields.
|
||||
// The first error encountered is returned.
|
||||
func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLayer) (err error) {
|
||||
@@ -277,7 +295,7 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
func (b *BucketMetadata) getAllLegacyConfigs(ctx context.Context, objectAPI ObjectLayer) (map[string][]byte, error) {
|
||||
legacyConfigs := []string{
|
||||
legacyBucketObjectLockEnabledConfigFile,
|
||||
bucketPolicyConfig,
|
||||
@@ -291,7 +309,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
|
||||
objectLockConfig,
|
||||
}
|
||||
|
||||
configs := make(map[string][]byte)
|
||||
configs := make(map[string][]byte, len(legacyConfigs))
|
||||
|
||||
// Handle migration from lockEnabled to newer format.
|
||||
if b.LockEnabled {
|
||||
@@ -316,16 +334,15 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
|
||||
continue
|
||||
}
|
||||
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
configs[legacyFile] = configData
|
||||
}
|
||||
|
||||
if len(configs) == 0 {
|
||||
// nothing to update, return right away.
|
||||
return b.parseAllConfigs(ctx, objectAPI)
|
||||
}
|
||||
return configs, nil
|
||||
}
|
||||
|
||||
func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer, configs map[string][]byte) error {
|
||||
for legacyFile, configData := range configs {
|
||||
switch legacyFile {
|
||||
case legacyBucketObjectLockEnabledConfigFile:
|
||||
@@ -425,23 +442,6 @@ func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
|
||||
return saveConfig(ctx, api, configFile, data)
|
||||
}
|
||||
|
||||
// deleteBucketMetadata deletes bucket metadata
|
||||
// If config does not exist no error is returned.
|
||||
func deleteBucketMetadata(ctx context.Context, obj objectDeleter, bucket string) error {
|
||||
metadataFiles := []string{
|
||||
dataUsageCacheName,
|
||||
bucketMetadataFile,
|
||||
path.Join(replicationDir, resyncFileName),
|
||||
}
|
||||
for _, metaFile := range metadataFiles {
|
||||
configFile := path.Join(bucketMetaPrefix, bucket, metaFile)
|
||||
if err := deleteConfig(ctx, obj, configFile); err != nil && err != errConfigNotFound {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrate config for remote targets by encrypting data if currently unencrypted and kms is configured.
|
||||
func (b *BucketMetadata) migrateTargetConfig(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
var err error
|
||||
|
||||
@@ -36,15 +36,6 @@ type BucketObjectLockSys struct{}
|
||||
|
||||
// Get - Get retention configuration.
|
||||
func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention, err error) {
|
||||
if globalIsGateway {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return r, errServerNotInitialized
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucketName)
|
||||
if err != nil {
|
||||
if errors.Is(err, BucketObjectLockConfigNotFound{Bucket: bucketName}) {
|
||||
|
||||
@@ -25,7 +25,7 @@ import (
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
)
|
||||
@@ -92,7 +92,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
|
||||
// Version in policy must not be empty
|
||||
if bucketPolicy.Version == "" {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL)
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPolicyInvalidVersion), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -149,7 +149,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, nil)
|
||||
updatedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketPolicyConfig)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
|
||||
@@ -107,7 +107,7 @@ func getAnonWriteOnlyObjectPolicy(bucketName, prefix string) *policy.Policy {
|
||||
|
||||
// Wrapper for calling Create Bucket and ensure we get one and only one success.
|
||||
func TestCreateBucket(t *testing.T) {
|
||||
ExecObjectLayerAPITest(t, testCreateBucket, []string{"MakeBucketWithLocation"})
|
||||
ExecObjectLayerAPITest(t, testCreateBucket, []string{"MakeBucket"})
|
||||
}
|
||||
|
||||
// testCreateBucket - Test for calling Create Bucket and ensure we get one and only one success.
|
||||
@@ -127,7 +127,7 @@ func testCreateBucket(obj ObjectLayer, instanceType, bucketName string, apiRoute
|
||||
defer wg.Done()
|
||||
// Sync start.
|
||||
<-start
|
||||
if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil {
|
||||
if err := obj.MakeBucket(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil {
|
||||
if _, ok := err.(BucketExists); !ok {
|
||||
t.Logf("unexpected error: %T: %v", err, err)
|
||||
return
|
||||
@@ -162,7 +162,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
bucketName1 := fmt.Sprintf("%s-1", bucketName)
|
||||
if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil {
|
||||
if err := obj.MakeBucket(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
miniogopolicy "github.com/minio/minio-go/v7/pkg/policy"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
@@ -125,6 +126,20 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
|
||||
|
||||
cloneHeader := r.Header.Clone()
|
||||
|
||||
if userTags := cloneHeader.Get(xhttp.AmzObjectTagging); userTags != "" {
|
||||
tag, _ := tags.ParseObjectTags(userTags)
|
||||
if tag != nil {
|
||||
tagMap := tag.ToMap()
|
||||
keys := make([]string, 0, len(tagMap))
|
||||
for k, v := range tagMap {
|
||||
args[pathJoin("ExistingObjectTag", k)] = []string{v}
|
||||
args[pathJoin("RequestObjectTag", k)] = []string{v}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
args["RequestObjectTagKeys"] = keys
|
||||
}
|
||||
}
|
||||
|
||||
for _, objLock := range []string{
|
||||
xhttp.AmzObjectLockMode,
|
||||
xhttp.AmzObjectLockLegalHold,
|
||||
@@ -137,6 +152,9 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
|
||||
}
|
||||
|
||||
for key, values := range cloneHeader {
|
||||
if strings.EqualFold(key, xhttp.AmzObjectTagging) {
|
||||
continue
|
||||
}
|
||||
if existingValues, found := args[key]; found {
|
||||
args[key] = append(existingValues, values...)
|
||||
} else {
|
||||
|
||||
+1
-8
@@ -24,7 +24,7 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
)
|
||||
|
||||
@@ -35,13 +35,6 @@ type BucketQuotaSys struct {
|
||||
|
||||
// Get - Get quota configuration.
|
||||
func (sys *BucketQuotaSys) Get(ctx context.Context, bucketName string) (*madmin.BucketQuota, error) {
|
||||
if globalIsGateway {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return nil, errServerNotInitialized
|
||||
}
|
||||
return &madmin.BucketQuota{}, nil
|
||||
}
|
||||
qCfg, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucketName)
|
||||
return qCfg, err
|
||||
}
|
||||
|
||||
@@ -46,10 +46,6 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
if globalIsGateway {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.PutReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
@@ -59,7 +55,10 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if globalSiteReplicationSys.isEnabled() && logger.GetReqInfo(ctx).Cred.AccessKey != globalActiveCred.AccessKey {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
|
||||
return
|
||||
}
|
||||
if versioned := globalBucketVersioningSys.Enabled(bucket); !versioned {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNeedsVersioningError), r.URL)
|
||||
return
|
||||
@@ -165,11 +164,26 @@ func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.Respons
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
|
||||
return
|
||||
}
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, nil); err != nil {
|
||||
if _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketReplicationConfig); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
for _, tgt := range targets.Targets {
|
||||
if err := globalBucketTargetSys.RemoveTarget(ctx, bucket, tgt.Arn); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
if _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketTargetsFile); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
// Write success response.
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
@@ -277,7 +291,13 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if !config.HasExistingObjectReplication(arn) {
|
||||
hasARN, hasExistingObjEnabled := config.HasExistingObjectReplication(arn)
|
||||
if !hasARN {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrRemoteTargetNotFoundError), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if !hasExistingObjEnabled {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNoExistingObjects), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -316,7 +336,27 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
|
||||
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
}
|
||||
}
|
||||
if err := startReplicationResync(ctx, bucket, arn, resetID, resetBeforeDate, objectAPI); err != nil {
|
||||
targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
tgtBytes, err := json.Marshal(&targets)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
|
||||
return
|
||||
}
|
||||
if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err := globalReplicationPool.resyncer.start(ctx, objectAPI, resyncOpts{
|
||||
bucket: bucket,
|
||||
arn: arn,
|
||||
resyncID: resetID,
|
||||
resyncBefore: resetBeforeDate,
|
||||
}); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
|
||||
Bucket: bucket,
|
||||
Err: err,
|
||||
@@ -365,10 +405,13 @@ func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.Response
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
globalReplicationPool.resyncState.RLock()
|
||||
brs, ok := globalReplicationPool.resyncState.statusMap[bucket]
|
||||
globalReplicationPool.resyncState.RUnlock()
|
||||
var tgtStats map[string]TargetReplicationResyncStatus
|
||||
globalReplicationPool.resyncer.RLock()
|
||||
brs, ok := globalReplicationPool.resyncer.statusMap[bucket]
|
||||
if ok {
|
||||
tgtStats = brs.cloneTgtStats()
|
||||
}
|
||||
globalReplicationPool.resyncer.RUnlock()
|
||||
if !ok {
|
||||
brs, err = loadBucketResyncMetadata(ctx, bucket, objectAPI)
|
||||
if err != nil {
|
||||
@@ -378,10 +421,11 @@ func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.Response
|
||||
}), r.URL)
|
||||
return
|
||||
}
|
||||
tgtStats = brs.cloneTgtStats()
|
||||
}
|
||||
|
||||
var rinfo ResyncTargetsInfo
|
||||
for tarn, st := range brs.TargetsMap {
|
||||
for tarn, st := range tgtStats {
|
||||
if arn != "" && tarn != arn {
|
||||
continue
|
||||
}
|
||||
@@ -389,7 +433,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.Response
|
||||
Arn: tarn,
|
||||
ResetID: st.ResyncID,
|
||||
StartTime: st.StartTime,
|
||||
EndTime: st.EndTime,
|
||||
EndTime: st.LastUpdate,
|
||||
ResyncStatus: st.ResyncStatus.String(),
|
||||
ReplicatedSize: st.ReplicatedSize,
|
||||
ReplicatedCount: st.ReplicatedCount,
|
||||
|
||||
@@ -19,6 +19,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -37,12 +38,12 @@ func (b *BucketReplicationStats) hasReplicationUsage() bool {
|
||||
|
||||
// ReplicationStats holds the global in-memory replication stats
|
||||
type ReplicationStats struct {
|
||||
Cache map[string]*BucketReplicationStats
|
||||
UsageCache map[string]*BucketReplicationStats
|
||||
mostRecentStats BucketStatsMap
|
||||
sync.RWMutex // mutex for Cache
|
||||
ulock sync.RWMutex // mutex for UsageCache
|
||||
dlock sync.RWMutex // mutex for mostRecentStats
|
||||
Cache map[string]*BucketReplicationStats
|
||||
UsageCache map[string]*BucketReplicationStats
|
||||
mostRecentStats BucketStatsMap
|
||||
sync.RWMutex // mutex for Cache
|
||||
ulock sync.RWMutex // mutex for UsageCache
|
||||
mostRecentStatsMu sync.Mutex // mutex for mostRecentStats
|
||||
}
|
||||
|
||||
// Delete deletes in-memory replication statistics for a bucket.
|
||||
@@ -245,10 +246,23 @@ outer:
|
||||
r.ulock.Unlock()
|
||||
}
|
||||
|
||||
func (r *ReplicationStats) getAllCachedLatest() BucketStatsMap {
|
||||
r.dlock.RLock()
|
||||
defer r.dlock.RUnlock()
|
||||
return r.mostRecentStats
|
||||
// serializeStats will serialize the current stats.
|
||||
// Will return (nil, nil) if no data.
|
||||
func (r *ReplicationStats) serializeStats() ([]byte, error) {
|
||||
if r == nil {
|
||||
return nil, nil
|
||||
}
|
||||
r.mostRecentStatsMu.Lock()
|
||||
defer r.mostRecentStatsMu.Unlock()
|
||||
if len(r.mostRecentStats.Stats) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
data := make([]byte, 4, 4+r.mostRecentStats.Msgsize())
|
||||
// Add the replication stats meta header.
|
||||
binary.LittleEndian.PutUint16(data[0:2], replStatsMetaFormat)
|
||||
binary.LittleEndian.PutUint16(data[2:4], replStatsVersion)
|
||||
// Add data
|
||||
return r.mostRecentStats.MarshalMsg(data)
|
||||
}
|
||||
|
||||
func (r *ReplicationStats) getAllLatest(bucketsUsage map[string]BucketUsageInfo) (bucketsReplicationStats map[string]BucketReplicationStats) {
|
||||
@@ -318,6 +332,7 @@ func (r *ReplicationStats) calculateBucketReplicationStats(bucket string, u Buck
|
||||
s = BucketReplicationStats{
|
||||
Stats: make(map[string]*BucketReplicationStat, len(stats)),
|
||||
}
|
||||
|
||||
var latestTotReplicatedSize int64
|
||||
for _, st := range u.ReplicationInfo {
|
||||
latestTotReplicatedSize += int64(st.ReplicatedSize)
|
||||
@@ -353,13 +368,15 @@ func (r *ReplicationStats) calculateBucketReplicationStats(bucket string, u Buck
|
||||
// normalize overall stats
|
||||
s.ReplicaSize = int64(math.Max(float64(totReplicaSize), float64(u.ReplicaSize)))
|
||||
s.ReplicatedSize = int64(math.Max(float64(s.ReplicatedSize), float64(latestTotReplicatedSize)))
|
||||
r.dlock.Lock()
|
||||
r.mostRecentStatsMu.Lock()
|
||||
if len(r.mostRecentStats.Stats) == 0 {
|
||||
r.mostRecentStats = BucketStatsMap{Stats: make(map[string]BucketStats, 1), Timestamp: UTCNow()}
|
||||
}
|
||||
r.mostRecentStats.Stats[bucket] = BucketStats{ReplicationStats: s}
|
||||
if len(s.Stats) > 0 {
|
||||
r.mostRecentStats.Stats[bucket] = BucketStats{ReplicationStats: s}
|
||||
}
|
||||
r.mostRecentStats.Timestamp = UTCNow()
|
||||
r.dlock.Unlock()
|
||||
r.mostRecentStatsMu.Unlock()
|
||||
return s
|
||||
}
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
@@ -628,26 +628,40 @@ func (v VersionPurgeStatusType) Pending() bool {
|
||||
return v == Pending || v == Failed
|
||||
}
|
||||
|
||||
type replicationResyncState struct {
|
||||
type replicationResyncer struct {
|
||||
// map of bucket to their resync status
|
||||
statusMap map[string]BucketReplicationResyncStatus
|
||||
statusMap map[string]BucketReplicationResyncStatus
|
||||
workerSize int
|
||||
resyncCancelCh chan struct{}
|
||||
workerCh chan struct{}
|
||||
sync.RWMutex
|
||||
}
|
||||
|
||||
const (
|
||||
replicationDir = "replication"
|
||||
replicationDir = ".replication"
|
||||
resyncFileName = "resync.bin"
|
||||
resyncMetaFormat = 1
|
||||
resyncMetaVersionV1 = 1
|
||||
resyncMetaVersion = resyncMetaVersionV1
|
||||
)
|
||||
|
||||
type resyncOpts struct {
|
||||
bucket string
|
||||
arn string
|
||||
resyncID string
|
||||
resyncBefore time.Time
|
||||
}
|
||||
|
||||
// ResyncStatusType status of resync operation
|
||||
type ResyncStatusType int
|
||||
|
||||
const (
|
||||
// NoResync - no resync in progress
|
||||
NoResync ResyncStatusType = iota
|
||||
// ResyncPending - resync pending
|
||||
ResyncPending
|
||||
// ResyncCanceled - resync canceled
|
||||
ResyncCanceled
|
||||
// ResyncStarted - resync in progress
|
||||
ResyncStarted
|
||||
// ResyncCompleted - resync finished
|
||||
@@ -656,6 +670,10 @@ const (
|
||||
ResyncFailed
|
||||
)
|
||||
|
||||
func (rt ResyncStatusType) isValid() bool {
|
||||
return rt != NoResync
|
||||
}
|
||||
|
||||
func (rt ResyncStatusType) String() string {
|
||||
switch rt {
|
||||
case ResyncStarted:
|
||||
@@ -664,6 +682,10 @@ func (rt ResyncStatusType) String() string {
|
||||
return "Completed"
|
||||
case ResyncFailed:
|
||||
return "Failed"
|
||||
case ResyncPending:
|
||||
return "Pending"
|
||||
case ResyncCanceled:
|
||||
return "Canceled"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
@@ -671,8 +693,8 @@ func (rt ResyncStatusType) String() string {
|
||||
|
||||
// TargetReplicationResyncStatus status of resync of bucket for a specific target
|
||||
type TargetReplicationResyncStatus struct {
|
||||
StartTime time.Time `json:"startTime" msg:"st"`
|
||||
EndTime time.Time `json:"endTime" msg:"et"`
|
||||
StartTime time.Time `json:"startTime" msg:"st"`
|
||||
LastUpdate time.Time `json:"lastUpdated" msg:"lst"`
|
||||
// Resync ID assigned to this reset
|
||||
ResyncID string `json:"resyncID" msg:"id"`
|
||||
// ResyncBeforeDate - resync all objects created prior to this date
|
||||
@@ -701,6 +723,14 @@ type BucketReplicationResyncStatus struct {
|
||||
LastUpdate time.Time `json:"lastUpdate" msg:"lu"`
|
||||
}
|
||||
|
||||
func (rs *BucketReplicationResyncStatus) cloneTgtStats() (m map[string]TargetReplicationResyncStatus) {
|
||||
m = make(map[string]TargetReplicationResyncStatus)
|
||||
for arn, st := range rs.TargetsMap {
|
||||
m[arn] = st
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func newBucketResyncStatus(bucket string) BucketReplicationResyncStatus {
|
||||
return BucketReplicationResyncStatus{
|
||||
TargetsMap: make(map[string]TargetReplicationResyncStatus),
|
||||
@@ -767,9 +797,8 @@ func (ri ReplicateObjectInfo) ToMRFEntry() MRFReplicateEntry {
|
||||
}
|
||||
}
|
||||
|
||||
func getReplicationStatsPath(nodeName string) string {
|
||||
nodeStr := strings.ReplaceAll(nodeName, ":", "_")
|
||||
return bucketMetaPrefix + SlashSeparator + replicationDir + SlashSeparator + nodeStr + ".stats"
|
||||
func getReplicationStatsPath() string {
|
||||
return bucketMetaPrefix + SlashSeparator + replicationDir + SlashSeparator + "replication.stats"
|
||||
}
|
||||
|
||||
const (
|
||||
|
||||
@@ -2117,10 +2117,10 @@ func (z *TargetReplicationResyncStatus) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
err = msgp.WrapError(err, "StartTime")
|
||||
return
|
||||
}
|
||||
case "et":
|
||||
z.EndTime, err = dc.ReadTime()
|
||||
case "lst":
|
||||
z.LastUpdate, err = dc.ReadTime()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "EndTime")
|
||||
err = msgp.WrapError(err, "LastUpdate")
|
||||
return
|
||||
}
|
||||
case "id":
|
||||
@@ -2205,14 +2205,14 @@ func (z *TargetReplicationResyncStatus) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
err = msgp.WrapError(err, "StartTime")
|
||||
return
|
||||
}
|
||||
// write "et"
|
||||
err = en.Append(0xa2, 0x65, 0x74)
|
||||
// write "lst"
|
||||
err = en.Append(0xa3, 0x6c, 0x73, 0x74)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteTime(z.EndTime)
|
||||
err = en.WriteTime(z.LastUpdate)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "EndTime")
|
||||
err = msgp.WrapError(err, "LastUpdate")
|
||||
return
|
||||
}
|
||||
// write "id"
|
||||
@@ -2315,9 +2315,9 @@ func (z *TargetReplicationResyncStatus) MarshalMsg(b []byte) (o []byte, err erro
|
||||
// string "st"
|
||||
o = append(o, 0x8b, 0xa2, 0x73, 0x74)
|
||||
o = msgp.AppendTime(o, z.StartTime)
|
||||
// string "et"
|
||||
o = append(o, 0xa2, 0x65, 0x74)
|
||||
o = msgp.AppendTime(o, z.EndTime)
|
||||
// string "lst"
|
||||
o = append(o, 0xa3, 0x6c, 0x73, 0x74)
|
||||
o = msgp.AppendTime(o, z.LastUpdate)
|
||||
// string "id"
|
||||
o = append(o, 0xa2, 0x69, 0x64)
|
||||
o = msgp.AppendString(o, z.ResyncID)
|
||||
@@ -2372,10 +2372,10 @@ func (z *TargetReplicationResyncStatus) UnmarshalMsg(bts []byte) (o []byte, err
|
||||
err = msgp.WrapError(err, "StartTime")
|
||||
return
|
||||
}
|
||||
case "et":
|
||||
z.EndTime, bts, err = msgp.ReadTimeBytes(bts)
|
||||
case "lst":
|
||||
z.LastUpdate, bts, err = msgp.ReadTimeBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "EndTime")
|
||||
err = msgp.WrapError(err, "LastUpdate")
|
||||
return
|
||||
}
|
||||
case "id":
|
||||
@@ -2450,7 +2450,7 @@ func (z *TargetReplicationResyncStatus) UnmarshalMsg(bts []byte) (o []byte, err
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *TargetReplicationResyncStatus) Msgsize() (s int) {
|
||||
s = 1 + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.StringPrefixSize + len(z.ResyncID) + 4 + msgp.TimeSize + 4 + msgp.IntSize + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.StringPrefixSize + len(z.Bucket) + 4 + msgp.StringPrefixSize + len(z.Object)
|
||||
s = 1 + 3 + msgp.TimeSize + 4 + msgp.TimeSize + 3 + msgp.StringPrefixSize + len(z.ResyncID) + 4 + msgp.TimeSize + 4 + msgp.IntSize + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.StringPrefixSize + len(z.Bucket) + 4 + msgp.StringPrefixSize + len(z.Object)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
+549
-284
File diff suppressed because it is too large
Load Diff
@@ -24,7 +24,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
+41
-26
@@ -25,12 +25,10 @@ import (
|
||||
"time"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio-go/v7"
|
||||
miniogo "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
@@ -209,9 +207,6 @@ func (sys *BucketTargetSys) Delete(bucket string) {
|
||||
|
||||
// SetTarget - sets a new minio-go client target for this bucket.
|
||||
func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *madmin.BucketTarget, update bool) error {
|
||||
if globalIsGateway {
|
||||
return nil
|
||||
}
|
||||
if !tgt.Type.IsValid() && !update {
|
||||
return BucketRemoteArnTypeInvalid{Bucket: bucket}
|
||||
}
|
||||
@@ -220,11 +215,18 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
|
||||
return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
|
||||
}
|
||||
// validate if target credentials are ok
|
||||
if _, err = clnt.BucketExists(ctx, tgt.TargetBucket); err != nil {
|
||||
if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
|
||||
exists, err := clnt.BucketExists(ctx, tgt.TargetBucket)
|
||||
if err != nil {
|
||||
switch minio.ToErrorResponse(err).Code {
|
||||
case "NoSuchBucket":
|
||||
return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
|
||||
case "AccessDenied":
|
||||
return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, AccessKey: tgt.Credentials.AccessKey, Err: err}
|
||||
}
|
||||
return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: err}
|
||||
return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, AccessKey: tgt.Credentials.AccessKey, Err: err}
|
||||
}
|
||||
if !exists {
|
||||
return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
|
||||
}
|
||||
if tgt.Type == madmin.ReplicationService {
|
||||
if !globalBucketVersioningSys.Enabled(bucket) {
|
||||
@@ -232,9 +234,9 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
|
||||
}
|
||||
vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
|
||||
if err != nil {
|
||||
return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: err}
|
||||
return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: err, AccessKey: tgt.Credentials.AccessKey}
|
||||
}
|
||||
if vcfg.Status != string(versioning.Enabled) {
|
||||
if !vcfg.Enabled() {
|
||||
return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
|
||||
}
|
||||
}
|
||||
@@ -254,6 +256,10 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
|
||||
found = true
|
||||
continue
|
||||
}
|
||||
// fail if endpoint is already present in list of targets and not a matching ARN
|
||||
if t.Endpoint == tgt.Endpoint {
|
||||
return BucketRemoteAlreadyExists{Bucket: t.TargetBucket}
|
||||
}
|
||||
}
|
||||
newtgts[idx] = t
|
||||
}
|
||||
@@ -268,9 +274,6 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
|
||||
}
|
||||
|
||||
func (sys *BucketTargetSys) updateBandwidthLimit(bucket string, limit int64) {
|
||||
if globalIsGateway {
|
||||
return
|
||||
}
|
||||
if limit == 0 {
|
||||
globalBucketMonitor.DeleteBucket(bucket)
|
||||
return
|
||||
@@ -282,10 +285,6 @@ func (sys *BucketTargetSys) updateBandwidthLimit(bucket string, limit int64) {
|
||||
|
||||
// RemoveTarget - removes a remote bucket target for this source bucket.
|
||||
func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr string) error {
|
||||
if globalIsGateway {
|
||||
return nil
|
||||
}
|
||||
|
||||
if arnStr == "" {
|
||||
return BucketRemoteArnInvalid{Bucket: bucket}
|
||||
}
|
||||
@@ -449,7 +448,7 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
|
||||
config := tcfg.Credentials
|
||||
creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "")
|
||||
|
||||
api, err := minio.New(tcfg.Endpoint, &miniogo.Options{
|
||||
api, err := minio.New(tcfg.Endpoint, &minio.Options{
|
||||
Creds: creds,
|
||||
Secure: tcfg.Secure,
|
||||
Region: tcfg.Region,
|
||||
@@ -478,20 +477,36 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
|
||||
}
|
||||
|
||||
// getRemoteARN gets existing ARN for an endpoint or generates a new one.
|
||||
func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTarget) string {
|
||||
func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTarget) (arn string, exists bool) {
|
||||
if target == nil {
|
||||
return ""
|
||||
return
|
||||
}
|
||||
tgts := sys.targetsMap[bucket]
|
||||
for _, tgt := range tgts {
|
||||
if tgt.Type == target.Type && tgt.TargetBucket == target.TargetBucket && target.URL().String() == tgt.URL().String() {
|
||||
return tgt.Arn
|
||||
if tgt.Type == target.Type && tgt.TargetBucket == target.TargetBucket && target.URL().String() == tgt.URL().String() && tgt.Credentials.AccessKey == target.Credentials.AccessKey {
|
||||
return tgt.Arn, true
|
||||
}
|
||||
}
|
||||
if !target.Type.IsValid() {
|
||||
return ""
|
||||
return
|
||||
}
|
||||
return generateARN(target)
|
||||
return generateARN(target), false
|
||||
}
|
||||
|
||||
// getRemoteARNForPeer returns the remote target for a peer site in site replication
|
||||
func (sys *BucketTargetSys) getRemoteARNForPeer(bucket string, peer madmin.PeerInfo) string {
|
||||
tgts := sys.targetsMap[bucket]
|
||||
for _, target := range tgts {
|
||||
ep, _ := url.Parse(peer.Endpoint)
|
||||
if target.SourceBucket == bucket &&
|
||||
target.TargetBucket == bucket &&
|
||||
target.Endpoint == ep.Host &&
|
||||
target.Secure == (ep.Scheme == "https") &&
|
||||
target.Type == madmin.ReplicationService {
|
||||
return target.Arn
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// generate ARN that is unique to this target type
|
||||
@@ -540,7 +555,7 @@ func parseBucketTargetConfig(bucket string, cdata, cmetadata []byte) (*madmin.Bu
|
||||
|
||||
// TargetClient is the struct for remote target client.
|
||||
type TargetClient struct {
|
||||
*miniogo.Client
|
||||
*minio.Client
|
||||
healthCheckDuration time.Duration
|
||||
Bucket string // remote bucket target
|
||||
replicateSync bool
|
||||
|
||||
@@ -25,7 +25,7 @@ import (
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
|
||||
@@ -69,11 +69,6 @@ func (sys *BucketVersioningSys) PrefixSuspended(bucket, prefix string) bool {
|
||||
|
||||
// Get returns stored bucket policy
|
||||
func (sys *BucketVersioningSys) Get(bucket string) (*versioning.Versioning, error) {
|
||||
if globalIsGateway {
|
||||
// Gateway does not implement versioning.
|
||||
return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, nil
|
||||
}
|
||||
|
||||
if bucket == minioMetaBucket || strings.HasPrefix(bucket, minioMetaBucket) {
|
||||
return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, nil
|
||||
}
|
||||
|
||||
+114
-64
@@ -18,53 +18,28 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
uatomic "go.uber.org/atomic"
|
||||
)
|
||||
|
||||
const (
|
||||
// callhomeSchemaVersion1 is callhome schema version 1
|
||||
callhomeSchemaVersion1 = "1"
|
||||
|
||||
// callhomeSchemaVersion is current callhome schema version.
|
||||
callhomeSchemaVersion = callhomeSchemaVersion1
|
||||
|
||||
// callhomeCycleDefault is the default interval between two callhome cycles (24hrs)
|
||||
callhomeCycleDefault = 24 * time.Hour
|
||||
)
|
||||
|
||||
// CallhomeInfo - Contains callhome information
|
||||
type CallhomeInfo struct {
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
AdminInfo madmin.InfoMessage `json:"admin_info"`
|
||||
}
|
||||
|
||||
var (
|
||||
enableCallhome = uatomic.NewBool(false)
|
||||
callhomeLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
|
||||
callhomeFreq = uatomic.NewDuration(callhomeCycleDefault)
|
||||
)
|
||||
|
||||
func updateCallhomeParams(ctx context.Context, objAPI ObjectLayer) {
|
||||
alreadyEnabled := enableCallhome.Load()
|
||||
enableCallhome.Store(globalCallhomeConfig.Enable)
|
||||
callhomeFreq.Store(globalCallhomeConfig.Frequency)
|
||||
|
||||
// If callhome was disabled earlier and has now been enabled,
|
||||
// initialize the callhome process again.
|
||||
if !alreadyEnabled && enableCallhome.Load() {
|
||||
initCallhome(ctx, objAPI)
|
||||
}
|
||||
}
|
||||
var callhomeLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
|
||||
|
||||
// initCallhome will start the callhome task in the background.
|
||||
func initCallhome(ctx context.Context, objAPI ObjectLayer) {
|
||||
if !globalCallhomeConfig.Enabled() {
|
||||
return
|
||||
}
|
||||
|
||||
go func() {
|
||||
r := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
// Leader node (that successfully acquires the lock inside runCallhome)
|
||||
@@ -72,79 +47,154 @@ func initCallhome(ctx context.Context, objAPI ObjectLayer) {
|
||||
// the lock will be released and another node will acquire it and take over
|
||||
// because of this loop.
|
||||
for {
|
||||
runCallhome(ctx, objAPI)
|
||||
if !enableCallhome.Load() {
|
||||
if !globalCallhomeConfig.Enabled() {
|
||||
return
|
||||
}
|
||||
|
||||
if !runCallhome(ctx, objAPI) {
|
||||
// callhome was disabled or context was canceled
|
||||
return
|
||||
}
|
||||
|
||||
// callhome running on a different node.
|
||||
// sleep for some time and try again.
|
||||
duration := time.Duration(r.Float64() * float64(callhomeFreq.Load()))
|
||||
duration := time.Duration(r.Float64() * float64(globalCallhomeConfig.FrequencyDur()))
|
||||
if duration < time.Second {
|
||||
// Make sure to sleep atleast a second to avoid high CPU ticks.
|
||||
duration = time.Second
|
||||
}
|
||||
time.Sleep(duration)
|
||||
|
||||
if !enableCallhome.Load() {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func runCallhome(ctx context.Context, objAPI ObjectLayer) {
|
||||
func runCallhome(ctx context.Context, objAPI ObjectLayer) bool {
|
||||
// Make sure only 1 callhome is running on the cluster.
|
||||
locker := objAPI.NewNSLock(minioMetaBucket, "callhome/runCallhome.lock")
|
||||
lkctx, err := locker.GetLock(ctx, callhomeLeaderLockTimeout)
|
||||
if err != nil {
|
||||
return
|
||||
// lock timedout means some other node is the leader,
|
||||
// cycle back return 'true'
|
||||
return true
|
||||
}
|
||||
|
||||
ctx = lkctx.Context()
|
||||
defer locker.Unlock(lkctx.Cancel)
|
||||
defer locker.Unlock(lkctx)
|
||||
|
||||
callhomeTimer := time.NewTimer(callhomeFreq.Load())
|
||||
callhomeTimer := time.NewTimer(globalCallhomeConfig.FrequencyDur())
|
||||
defer callhomeTimer.Stop()
|
||||
|
||||
for {
|
||||
if !globalCallhomeConfig.Enabled() {
|
||||
// Stop the processing as callhome got disabled
|
||||
return false
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
// indicates that we do not need to run callhome anymore
|
||||
return false
|
||||
case <-callhomeTimer.C:
|
||||
if !enableCallhome.Load() {
|
||||
if !globalCallhomeConfig.Enabled() {
|
||||
// Stop the processing as callhome got disabled
|
||||
return
|
||||
return false
|
||||
}
|
||||
|
||||
performCallhome(ctx)
|
||||
|
||||
// Reset the timer for next cycle.
|
||||
callhomeTimer.Reset(callhomeFreq.Load())
|
||||
callhomeTimer.Reset(globalCallhomeConfig.FrequencyDur())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func performCallhome(ctx context.Context) {
|
||||
err := sendCallhomeInfo(
|
||||
CallhomeInfo{
|
||||
SchemaVersion: callhomeSchemaVersion,
|
||||
AdminInfo: getServerInfo(ctx, nil),
|
||||
})
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to perform callhome: %w", err))
|
||||
deadline := 10 * time.Second // Default deadline is 10secs for callhome
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil {
|
||||
logger.LogIf(ctx, errors.New("Callhome: object layer not ready"))
|
||||
return
|
||||
}
|
||||
|
||||
healthCtx, healthCancel := context.WithTimeout(ctx, deadline)
|
||||
defer healthCancel()
|
||||
|
||||
healthInfoCh := make(chan madmin.HealthInfo)
|
||||
|
||||
query := url.Values{}
|
||||
for _, k := range madmin.HealthDataTypesList {
|
||||
query.Set(string(k), "true")
|
||||
}
|
||||
|
||||
healthInfo := madmin.HealthInfo{
|
||||
TimeStamp: time.Now().UTC(),
|
||||
Version: madmin.HealthInfoVersion,
|
||||
Minio: madmin.MinioHealthInfo{
|
||||
Info: madmin.MinioInfo{
|
||||
DeploymentID: globalDeploymentID,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
go fetchHealthInfo(healthCtx, objectAPI, &query, healthInfoCh, healthInfo)
|
||||
|
||||
for {
|
||||
select {
|
||||
case hi, hasMore := <-healthInfoCh:
|
||||
if !hasMore {
|
||||
// Received all data. Send to SUBNET and return
|
||||
err := sendHealthInfo(ctx, healthInfo)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to perform callhome: %w", err))
|
||||
}
|
||||
return
|
||||
}
|
||||
healthInfo = hi
|
||||
case <-healthCtx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
callhomeURL = "https://subnet.min.io/api/callhome"
|
||||
callhomeURLDev = "http://localhost:9000/api/callhome"
|
||||
healthURL = "https://subnet.min.io/api/health/upload"
|
||||
healthURLDev = "http://localhost:9000/api/health/upload"
|
||||
)
|
||||
|
||||
func sendCallhomeInfo(ch CallhomeInfo) error {
|
||||
url := callhomeURL
|
||||
func sendHealthInfo(ctx context.Context, healthInfo madmin.HealthInfo) error {
|
||||
url := healthURL
|
||||
if globalIsCICD {
|
||||
url = callhomeURLDev
|
||||
url = healthURLDev
|
||||
}
|
||||
_, err := globalSubnetConfig.Post(url, ch)
|
||||
|
||||
filename := fmt.Sprintf("health_%s.json.gz", UTCNow().Format("20060102150405"))
|
||||
url += "?filename=" + filename
|
||||
|
||||
_, err := globalSubnetConfig.Upload(url, filename, createHealthJSONGzip(ctx, healthInfo))
|
||||
return err
|
||||
}
|
||||
|
||||
func createHealthJSONGzip(ctx context.Context, healthInfo madmin.HealthInfo) []byte {
|
||||
var b bytes.Buffer
|
||||
gzWriter := gzip.NewWriter(&b)
|
||||
|
||||
header := struct {
|
||||
Version string `json:"version"`
|
||||
}{Version: healthInfo.Version}
|
||||
|
||||
enc := json.NewEncoder(gzWriter)
|
||||
if e := enc.Encode(header); e != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Could not encode health info header: %w", e))
|
||||
return nil
|
||||
}
|
||||
|
||||
if e := enc.Encode(healthInfo); e != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Could not encode health info: %w", e))
|
||||
return nil
|
||||
}
|
||||
|
||||
gzWriter.Flush()
|
||||
gzWriter.Close()
|
||||
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
+16
-31
@@ -52,7 +52,7 @@ import (
|
||||
"github.com/minio/console/restapi"
|
||||
"github.com/minio/console/restapi/operations"
|
||||
"github.com/minio/kes"
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
@@ -117,7 +117,8 @@ func init() {
|
||||
go func() {
|
||||
var t *time.Ticker
|
||||
if containers {
|
||||
t = time.NewTicker(1 * time.Minute)
|
||||
// k8s DNS TTL is 30s (Attempt a refresh only after)
|
||||
t = time.NewTicker(30 * time.Second)
|
||||
} else {
|
||||
t = time.NewTicker(10 * time.Minute)
|
||||
}
|
||||
@@ -134,7 +135,7 @@ func init() {
|
||||
|
||||
globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
|
||||
PassHost: true,
|
||||
RoundTripper: newGatewayHTTPTransport(1 * time.Hour),
|
||||
RoundTripper: NewHTTPTransportWithTimeout(1 * time.Hour),
|
||||
Logger: func(err error) {
|
||||
if err != nil && !errors.Is(err, context.Canceled) {
|
||||
logger.LogIf(GlobalContext, err)
|
||||
@@ -151,7 +152,7 @@ func init() {
|
||||
defaultAWSCredProvider = []credentials.Provider{
|
||||
&credentials.IAM{
|
||||
Client: &http.Client{
|
||||
Transport: NewGatewayHTTPTransport(),
|
||||
Transport: NewHTTPTransport(),
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -175,7 +176,9 @@ func minioConfigToConsoleFeatures() {
|
||||
if globalMinioEndpoint != "" {
|
||||
os.Setenv("CONSOLE_MINIO_SERVER", globalMinioEndpoint)
|
||||
} else {
|
||||
os.Setenv("CONSOLE_MINIO_SERVER", getAPIEndpoints()[0])
|
||||
// Explicitly set 127.0.0.1 so Console will automatically bypass TLS verification to the local S3 API.
|
||||
// This will save users from providing a certificate with IP or FQDN SAN that points to the local host.
|
||||
os.Setenv("CONSOLE_MINIO_SERVER", fmt.Sprintf("%s://127.0.0.1:%s", getURLScheme(globalIsTLS), globalMinioPort))
|
||||
}
|
||||
if value := env.Get("MINIO_LOG_QUERY_URL", ""); value != "" {
|
||||
os.Setenv("CONSOLE_LOG_QUERY_URL", value)
|
||||
@@ -202,20 +205,13 @@ func minioConfigToConsoleFeatures() {
|
||||
}
|
||||
}
|
||||
// Enable if LDAP is enabled.
|
||||
if globalLDAPConfig.Enabled {
|
||||
if globalLDAPConfig.Enabled() {
|
||||
os.Setenv("CONSOLE_LDAP_ENABLED", config.EnableOn)
|
||||
}
|
||||
os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
|
||||
os.Setenv("CONSOLE_CERT_PASSWD", env.Get("MINIO_CERT_PASSWD", ""))
|
||||
if globalSubnetConfig.License != "" {
|
||||
os.Setenv("CONSOLE_SUBNET_LICENSE", globalSubnetConfig.License)
|
||||
}
|
||||
if globalSubnetConfig.APIKey != "" {
|
||||
os.Setenv("CONSOLE_SUBNET_API_KEY", globalSubnetConfig.APIKey)
|
||||
}
|
||||
if globalSubnetConfig.ProxyURL != nil {
|
||||
os.Setenv("CONSOLE_SUBNET_PROXY", globalSubnetConfig.ProxyURL.String())
|
||||
}
|
||||
|
||||
globalSubnetConfig.ApplyEnv()
|
||||
}
|
||||
|
||||
func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
|
||||
@@ -236,6 +232,8 @@ func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
|
||||
Userinfo: cfg.ClaimUserinfo,
|
||||
RedirectCallbackDynamic: cfg.RedirectURIDynamic,
|
||||
RedirectCallback: callback,
|
||||
EndSessionEndpoint: cfg.DiscoveryDoc.EndSessionEndpoint,
|
||||
RoleArn: cfg.GetRoleArn(),
|
||||
}
|
||||
}
|
||||
return m
|
||||
@@ -305,22 +303,6 @@ func initConsoleServer() (*restapi.Server, error) {
|
||||
return server, nil
|
||||
}
|
||||
|
||||
func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
|
||||
if strings.HasPrefix(name, "gateway") {
|
||||
if GlobalGatewaySSE.IsSet() && GlobalKMS == nil {
|
||||
uiErr := config.ErrInvalidGWSSEEnvValue(nil).Msg("MINIO_GATEWAY_SSE set but KMS is not configured")
|
||||
logger.Fatal(uiErr, "Unable to start gateway with SSE")
|
||||
}
|
||||
}
|
||||
|
||||
globalCompressConfigMu.Lock()
|
||||
if globalCompressConfig.Enabled && !objAPI.IsCompressionSupported() {
|
||||
logger.Fatal(errInvalidArgument,
|
||||
"Compression support is requested but '%s' does not support compression", name)
|
||||
}
|
||||
globalCompressConfigMu.Unlock()
|
||||
}
|
||||
|
||||
// Check for updates and print a notification message
|
||||
func checkUpdate(mode string) {
|
||||
updateURL := minioReleaseInfoURL
|
||||
@@ -792,6 +774,8 @@ func handleCommonEnvVars() {
|
||||
logger.Info(color.RedBold(msg))
|
||||
}
|
||||
globalActiveCred = cred
|
||||
} else {
|
||||
globalActiveCred = auth.DefaultCredentials
|
||||
}
|
||||
}
|
||||
|
||||
@@ -874,6 +858,7 @@ func handleKMSConfig() {
|
||||
defaultKeyID := env.Get(config.EnvKESKeyName, "")
|
||||
KMS, err := kms.NewWithConfig(kms.Config{
|
||||
Endpoints: endpoints,
|
||||
Enclave: env.Get(config.EnvKESEnclave, ""),
|
||||
DefaultKeyID: defaultKeyID,
|
||||
Certificate: certificate,
|
||||
ReloadCertEvents: reloadCertEvents,
|
||||
|
||||
+14
-6
@@ -29,10 +29,14 @@ import (
|
||||
|
||||
var errConfigNotFound = errors.New("config file not found")
|
||||
|
||||
func readConfigWithMetadata(ctx context.Context, store objectIO, configFile string) ([]byte, ObjectInfo, error) {
|
||||
r, err := store.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, readLock, ObjectOptions{})
|
||||
func readConfigWithMetadata(ctx context.Context, store objectIO, configFile string, opts ObjectOptions) ([]byte, ObjectInfo, error) {
|
||||
lockType := readLock
|
||||
if opts.NoLock {
|
||||
lockType = noLock // erasureObjects.GetObjectNInfo honors lockType argument but not opts.NoLock.
|
||||
}
|
||||
|
||||
r, err := store.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, lockType, opts)
|
||||
if err != nil {
|
||||
// Treat object not found as config not found.
|
||||
if isErrObjectNotFound(err) {
|
||||
return nil, ObjectInfo{}, errConfigNotFound
|
||||
}
|
||||
@@ -52,7 +56,7 @@ func readConfigWithMetadata(ctx context.Context, store objectIO, configFile stri
|
||||
}
|
||||
|
||||
func readConfig(ctx context.Context, store objectIO, configFile string) ([]byte, error) {
|
||||
buf, _, err := readConfigWithMetadata(ctx, store, configFile)
|
||||
buf, _, err := readConfigWithMetadata(ctx, store, configFile, ObjectOptions{})
|
||||
return buf, err
|
||||
}
|
||||
|
||||
@@ -70,16 +74,20 @@ func deleteConfig(ctx context.Context, objAPI objectDeleter, configFile string)
|
||||
return err
|
||||
}
|
||||
|
||||
func saveConfig(ctx context.Context, store objectIO, configFile string, data []byte) error {
|
||||
func saveConfigWithOpts(ctx context.Context, store objectIO, configFile string, data []byte, opts ObjectOptions) error {
|
||||
hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = store.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader), ObjectOptions{MaxParity: true})
|
||||
_, err = store.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader), opts)
|
||||
return err
|
||||
}
|
||||
|
||||
func saveConfig(ctx context.Context, store objectIO, configFile string, data []byte) error {
|
||||
return saveConfigWithOpts(ctx, store, configFile, data, ObjectOptions{MaxParity: true})
|
||||
}
|
||||
|
||||
func checkConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
|
||||
if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
|
||||
// Treat object not found as config not found.
|
||||
|
||||
+109
-153
@@ -24,8 +24,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/config/api"
|
||||
"github.com/minio/minio/internal/config/cache"
|
||||
@@ -65,11 +64,9 @@ func initHelp() {
|
||||
config.SiteSubSys: config.DefaultSiteKVS,
|
||||
config.RegionSubSys: config.DefaultRegionKVS,
|
||||
config.APISubSys: api.DefaultKVS,
|
||||
config.CredentialsSubSys: config.DefaultCredentialKVS,
|
||||
config.LoggerWebhookSubSys: logger.DefaultLoggerWebhookKVS,
|
||||
config.AuditWebhookSubSys: logger.DefaultAuditWebhookKVS,
|
||||
config.AuditKafkaSubSys: logger.DefaultAuditKafkaKVS,
|
||||
config.HealSubSys: heal.DefaultKVS,
|
||||
config.ScannerSubSys: scanner.DefaultKVS,
|
||||
config.SubnetSubSys: subnet.DefaultKVS,
|
||||
config.CallhomeSubSys: callhome.DefaultKVS,
|
||||
@@ -79,27 +76,40 @@ func initHelp() {
|
||||
}
|
||||
if globalIsErasure {
|
||||
kvs[config.StorageClassSubSys] = storageclass.DefaultKVS
|
||||
kvs[config.HealSubSys] = heal.DefaultKVS
|
||||
}
|
||||
config.RegisterDefaultKVS(kvs)
|
||||
|
||||
// Captures help for each sub-system
|
||||
helpSubSys := config.HelpKVS{
|
||||
config.HelpKV{
|
||||
Key: config.SubnetSubSys,
|
||||
Type: "string",
|
||||
Description: "register the cluster to MinIO SUBNET",
|
||||
Optional: true,
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: config.CallhomeSubSys,
|
||||
Type: "string",
|
||||
Description: "enable callhome to MinIO SUBNET",
|
||||
Optional: true,
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: config.SiteSubSys,
|
||||
Description: "label the server and its location",
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: config.CacheSubSys,
|
||||
Description: "add caching storage tier",
|
||||
Key: config.APISubSys,
|
||||
Description: "manage global HTTP API call specific features, such as throttling, authentication types, etc.",
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: config.ScannerSubSys,
|
||||
Description: "manage namespace scanning for usage calculation, lifecycle, healing and more",
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: config.CompressionSubSys,
|
||||
Description: "enable server side compression of objects",
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: config.EtcdSubSys,
|
||||
Description: "federate multiple clusters for IAM and Bucket DNS",
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: config.IdentityOpenIDSubSys,
|
||||
Description: "enable OpenID SSO support",
|
||||
@@ -121,18 +131,6 @@ func initHelp() {
|
||||
Key: config.PolicyPluginSubSys,
|
||||
Description: "enable Access Management Plugin for policy enforcement",
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: config.APISubSys,
|
||||
Description: "manage global HTTP API call specific features, such as throttling, authentication types, etc.",
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: config.HealSubSys,
|
||||
Description: "manage object healing frequency and bitrot verification checks",
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: config.ScannerSubSys,
|
||||
Description: "manage namespace scanning for usage calculation, lifecycle, healing and more",
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: config.LoggerWebhookSubSys,
|
||||
Description: "send server logs to webhook endpoints",
|
||||
@@ -199,26 +197,23 @@ func initHelp() {
|
||||
MultipleTargets: true,
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: config.SubnetSubSys,
|
||||
Type: "string",
|
||||
Description: "set subnet config for the cluster e.g. api key",
|
||||
Optional: true,
|
||||
Key: config.EtcdSubSys,
|
||||
Description: "persist IAM assets externally to etcd",
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: config.CallhomeSubSys,
|
||||
Type: "string",
|
||||
Description: "enable callhome for the cluster",
|
||||
Optional: true,
|
||||
Key: config.CacheSubSys,
|
||||
Description: "[DEPRECATED] add caching storage tier",
|
||||
},
|
||||
}
|
||||
|
||||
if globalIsErasure {
|
||||
helpSubSys = append(helpSubSys, config.HelpKV{})
|
||||
copy(helpSubSys[2:], helpSubSys[1:])
|
||||
helpSubSys[1] = config.HelpKV{
|
||||
helpSubSys = append(helpSubSys, config.HelpKV{
|
||||
Key: config.StorageClassSubSys,
|
||||
Description: "define object level redundancy",
|
||||
}
|
||||
}, config.HelpKV{
|
||||
Key: config.HealSubSys,
|
||||
Description: "manage object healing frequency and bitrot verification checks",
|
||||
})
|
||||
}
|
||||
|
||||
helpMap := map[string]config.HelpKVS{
|
||||
@@ -289,14 +284,12 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
|
||||
return err
|
||||
}
|
||||
case config.StorageClassSubSys:
|
||||
if globalIsErasure {
|
||||
if objAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
for _, setDriveCount := range objAPI.SetDriveCounts() {
|
||||
if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount); err != nil {
|
||||
return err
|
||||
}
|
||||
if objAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
for _, setDriveCount := range objAPI.SetDriveCounts() {
|
||||
if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case config.CacheSubSys:
|
||||
@@ -336,16 +329,16 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
|
||||
}
|
||||
case config.IdentityOpenIDSubSys:
|
||||
if _, err := openid.LookupConfig(s,
|
||||
NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
|
||||
NewHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
|
||||
return err
|
||||
}
|
||||
case config.IdentityLDAPSubSys:
|
||||
cfg, err := xldap.Lookup(s[config.IdentityLDAPSubSys][config.Default], globalRootCAs)
|
||||
cfg, err := xldap.Lookup(s, globalRootCAs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.Enabled {
|
||||
conn, cerr := cfg.Connect()
|
||||
if cfg.Enabled() {
|
||||
conn, cerr := cfg.LDAP.Connect()
|
||||
if cerr != nil {
|
||||
return cerr
|
||||
}
|
||||
@@ -357,7 +350,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
|
||||
}
|
||||
case config.IdentityPluginSubSys:
|
||||
if _, err := idplugin.LookupConfig(s[config.IdentityPluginSubSys][config.Default],
|
||||
NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
|
||||
NewHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
|
||||
return err
|
||||
}
|
||||
case config.SubnetSubSys:
|
||||
@@ -365,22 +358,26 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
|
||||
return err
|
||||
}
|
||||
case config.CallhomeSubSys:
|
||||
if _, err := callhome.LookupConfig(s[config.CallhomeSubSys][config.Default]); err != nil {
|
||||
cfg, err := callhome.LookupConfig(s[config.CallhomeSubSys][config.Default])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// callhome cannot be enabled if license is not registered yet, throw an error.
|
||||
if cfg.Enabled() && !globalSubnetConfig.Registered() {
|
||||
return errors.New("Deployment is not registered with SUBNET. Please register the deployment via 'mc license register ALIAS'")
|
||||
}
|
||||
case config.PolicyOPASubSys:
|
||||
// In case legacy OPA config is being set, we treat it as if the
|
||||
// AuthZPlugin is being set.
|
||||
subSys = config.PolicyPluginSubSys
|
||||
fallthrough
|
||||
case config.PolicyPluginSubSys:
|
||||
if ppargs, err := polplugin.LookupConfig(s[config.PolicyPluginSubSys][config.Default],
|
||||
NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
|
||||
if ppargs, err := polplugin.LookupConfig(s, GetDefaultConnSettings(), xhttp.DrainBody); err != nil {
|
||||
return err
|
||||
} else if ppargs.URL == nil {
|
||||
// Check if legacy opa is configured.
|
||||
if _, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
|
||||
NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
|
||||
NewHTTPTransport(), xhttp.DrainBody); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -393,7 +390,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
|
||||
}
|
||||
|
||||
if config.NotifySubSystems.Contains(subSys) {
|
||||
if err := notify.TestSubSysNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), globalEventNotifier.ConfiguredTargetIDs(), subSys); err != nil {
|
||||
if err := notify.TestSubSysNotificationTargets(GlobalContext, s, subSys, NewHTTPTransport()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -428,53 +425,28 @@ func validateConfig(s config.Config, subSys string) error {
|
||||
func lookupConfigs(s config.Config, objAPI ObjectLayer) {
|
||||
ctx := GlobalContext
|
||||
|
||||
var err error
|
||||
if !globalActiveCred.IsValid() {
|
||||
// Env doesn't seem to be set, we fallback to lookup creds from the config.
|
||||
globalActiveCred, err = config.LookupCreds(s[config.CredentialsSubSys][config.Default])
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Invalid credentials configuration: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
dnsURL, dnsUser, dnsPass, err := env.LookupEnv(config.EnvDNSWebhook)
|
||||
if err != nil {
|
||||
if globalIsGateway {
|
||||
logger.FatalIf(err, "Unable to initialize remote webhook DNS config")
|
||||
} else {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
|
||||
}
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
|
||||
}
|
||||
if err == nil && dnsURL != "" {
|
||||
globalDNSConfig, err = dns.NewOperatorDNS(dnsURL,
|
||||
dns.Authentication(dnsUser, dnsPass),
|
||||
dns.RootCAs(globalRootCAs))
|
||||
if err != nil {
|
||||
if globalIsGateway {
|
||||
logger.FatalIf(err, "Unable to initialize remote webhook DNS config")
|
||||
} else {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
|
||||
}
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
|
||||
if err != nil {
|
||||
if globalIsGateway {
|
||||
logger.FatalIf(err, "Unable to initialize etcd config")
|
||||
} else {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
|
||||
}
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
|
||||
}
|
||||
|
||||
if etcdCfg.Enabled {
|
||||
globalEtcdClient, err = etcd.New(etcdCfg)
|
||||
if err != nil {
|
||||
if globalIsGateway {
|
||||
logger.FatalIf(err, "Unable to initialize etcd config")
|
||||
} else {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
|
||||
}
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
|
||||
}
|
||||
|
||||
if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
|
||||
@@ -490,12 +462,8 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
|
||||
dns.CoreDNSPath(etcdCfg.CoreDNSPath),
|
||||
)
|
||||
if err != nil {
|
||||
if globalIsGateway {
|
||||
logger.FatalIf(err, "Unable to initialize DNS config")
|
||||
} else {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
|
||||
globalDomainNames, err))
|
||||
}
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
|
||||
globalDomainNames, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -515,11 +483,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
|
||||
|
||||
globalCacheConfig, err = cache.LookupConfig(s[config.CacheSubSys][config.Default])
|
||||
if err != nil {
|
||||
if globalIsGateway {
|
||||
logger.FatalIf(err, "Unable to setup cache")
|
||||
} else {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
|
||||
}
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
|
||||
}
|
||||
|
||||
if globalCacheConfig.Enabled {
|
||||
@@ -545,49 +509,39 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
|
||||
logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
|
||||
}
|
||||
|
||||
globalSubnetConfig, err = subnet.LookupConfig(s[config.SubnetSubSys][config.Default], globalProxyTransport)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
|
||||
}
|
||||
transport := NewHTTPTransport()
|
||||
|
||||
transport := NewGatewayHTTPTransport()
|
||||
|
||||
globalConfigTargetList, err = notify.GetNotificationTargets(GlobalContext, s, transport, false)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
|
||||
}
|
||||
|
||||
globalEnvTargetList, err = notify.GetNotificationTargets(GlobalContext, newServerConfig(), transport, true)
|
||||
bootstrapTrace("lookup the event notification targets")
|
||||
globalConfigTargetList, err = notify.FetchEnabledTargets(GlobalContext, s, transport)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
|
||||
}
|
||||
|
||||
bootstrapTrace("applying the dynamic configuration")
|
||||
// Apply dynamic config values
|
||||
if err := applyDynamicConfig(ctx, objAPI, s); err != nil {
|
||||
if globalIsGateway {
|
||||
logger.FatalIf(err, "Unable to initialize dynamic configuration")
|
||||
} else {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}
|
||||
|
||||
func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s config.Config, subSys string) error {
|
||||
if objAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
|
||||
setDriveCounts := objAPI.SetDriveCounts()
|
||||
switch subSys {
|
||||
case config.APISubSys:
|
||||
apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
|
||||
}
|
||||
var setDriveCounts []int
|
||||
if objAPI != nil {
|
||||
setDriveCounts = objAPI.SetDriveCounts()
|
||||
}
|
||||
|
||||
globalAPIConfig.init(apiConfig, setDriveCounts)
|
||||
|
||||
// Initialize remote instance transport once.
|
||||
getRemoteInstanceTransportOnce.Do(func() {
|
||||
getRemoteInstanceTransport = newGatewayHTTPTransport(apiConfig.RemoteTransportDeadline)
|
||||
getRemoteInstanceTransport = NewHTTPTransportWithTimeout(apiConfig.RemoteTransportDeadline)
|
||||
})
|
||||
case config.CompressionSubSys:
|
||||
cmpCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
|
||||
@@ -595,10 +549,8 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
|
||||
return fmt.Errorf("Unable to setup Compression: %w", err)
|
||||
}
|
||||
// Validate if the object layer supports compression.
|
||||
if objAPI != nil {
|
||||
if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
|
||||
return fmt.Errorf("Backend does not support compression")
|
||||
}
|
||||
if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
|
||||
return fmt.Errorf("Backend does not support compression")
|
||||
}
|
||||
globalCompressConfigMu.Lock()
|
||||
globalCompressConfig = cmpCfg
|
||||
@@ -627,12 +579,12 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
|
||||
if l.Enabled {
|
||||
l.LogOnce = logger.LogOnceConsoleIf
|
||||
l.UserAgent = userAgent
|
||||
l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
|
||||
loggerCfg.HTTP[n] = l
|
||||
l.Transport = NewHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
|
||||
}
|
||||
loggerCfg.HTTP[n] = l
|
||||
}
|
||||
if err = logger.UpdateSystemTargets(loggerCfg); err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %w", err))
|
||||
if errs := logger.UpdateSystemTargets(loggerCfg); len(errs) > 0 {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %v", errs))
|
||||
}
|
||||
case config.AuditWebhookSubSys:
|
||||
loggerCfg, err := logger.LookupConfigForSubSys(s, config.AuditWebhookSubSys)
|
||||
@@ -644,13 +596,13 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
|
||||
if l.Enabled {
|
||||
l.LogOnce = logger.LogOnceConsoleIf
|
||||
l.UserAgent = userAgent
|
||||
l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
|
||||
loggerCfg.AuditWebhook[n] = l
|
||||
l.Transport = NewHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
|
||||
}
|
||||
loggerCfg.AuditWebhook[n] = l
|
||||
}
|
||||
|
||||
if err = logger.UpdateAuditWebhookTargets(loggerCfg); err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %w", err))
|
||||
if errs := logger.UpdateAuditWebhookTargets(loggerCfg); len(errs) > 0 {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %v", errs))
|
||||
}
|
||||
case config.AuditKafkaSubSys:
|
||||
loggerCfg, err := logger.LookupConfigForSubSys(s, config.AuditKafkaSubSys)
|
||||
@@ -663,32 +615,40 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
|
||||
loggerCfg.AuditKafka[n] = l
|
||||
}
|
||||
}
|
||||
if err = logger.UpdateAuditKafkaTargets(loggerCfg); err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to update audit kafka targets: %w", err))
|
||||
if errs := logger.UpdateAuditKafkaTargets(loggerCfg); len(errs) > 0 {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to update audit kafka targets: %v", errs))
|
||||
}
|
||||
case config.StorageClassSubSys:
|
||||
if globalIsErasure && objAPI != nil {
|
||||
setDriveCounts := objAPI.SetDriveCounts()
|
||||
for i, setDriveCount := range setDriveCounts {
|
||||
sc, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
|
||||
break
|
||||
}
|
||||
// if we validated all setDriveCounts and it was successful
|
||||
// proceed to store the correct storage class globally.
|
||||
if i == len(setDriveCounts)-1 {
|
||||
globalStorageClass.Update(sc)
|
||||
}
|
||||
for i, setDriveCount := range setDriveCounts {
|
||||
sc, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
|
||||
break
|
||||
}
|
||||
// if we validated all setDriveCounts and it was successful
|
||||
// proceed to store the correct storage class globally.
|
||||
if i == len(setDriveCounts)-1 {
|
||||
globalStorageClass.Update(sc)
|
||||
}
|
||||
}
|
||||
case config.SubnetSubSys:
|
||||
subnetConfig, err := subnet.LookupConfig(s[config.SubnetSubSys][config.Default], globalProxyTransport)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
|
||||
} else {
|
||||
globalSubnetConfig.Update(subnetConfig)
|
||||
globalSubnetConfig.ApplyEnv() // update environment settings for Console UI
|
||||
}
|
||||
case config.CallhomeSubSys:
|
||||
callhomeCfg, err := callhome.LookupConfig(s[config.CallhomeSubSys][config.Default])
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to load callhome config: %w", err))
|
||||
} else {
|
||||
globalCallhomeConfig = callhomeCfg
|
||||
updateCallhomeParams(ctx, objAPI)
|
||||
enable := callhomeCfg.Enable && !globalCallhomeConfig.Enabled()
|
||||
globalCallhomeConfig.Update(callhomeCfg)
|
||||
if enable {
|
||||
initCallhome(ctx, objAPI)
|
||||
}
|
||||
}
|
||||
}
|
||||
globalServerConfigMu.Lock()
|
||||
@@ -796,13 +756,6 @@ func newSrvConfig(objAPI ObjectLayer) error {
|
||||
// Initialize server config.
|
||||
srvCfg := newServerConfig()
|
||||
|
||||
if globalActiveCred.IsValid() && !globalActiveCred.Equal(auth.DefaultCredentials) {
|
||||
kvs := srvCfg[config.CredentialsSubSys][config.Default]
|
||||
kvs.Set(config.AccessKey, globalActiveCred.AccessKey)
|
||||
kvs.Set(config.SecretKey, globalActiveCred.SecretKey)
|
||||
srvCfg[config.CredentialsSubSys][config.Default] = kvs
|
||||
}
|
||||
|
||||
// hold the mutex lock before a new config is assigned.
|
||||
globalServerConfigMu.Lock()
|
||||
globalServerConfig = srvCfg
|
||||
@@ -813,17 +766,20 @@ func newSrvConfig(objAPI ObjectLayer) error {
|
||||
}
|
||||
|
||||
func getValidConfig(objAPI ObjectLayer) (config.Config, error) {
|
||||
return readServerConfig(GlobalContext, objAPI)
|
||||
return readServerConfig(GlobalContext, objAPI, nil)
|
||||
}
|
||||
|
||||
// loadConfig - loads a new config from disk, overrides params
|
||||
// from env if found and valid
|
||||
func loadConfig(objAPI ObjectLayer) error {
|
||||
srvCfg, err := getValidConfig(objAPI)
|
||||
// data is optional. If nil it will be loaded from backend.
|
||||
func loadConfig(objAPI ObjectLayer, data []byte) error {
|
||||
bootstrapTrace("load the configuration")
|
||||
srvCfg, err := readServerConfig(GlobalContext, objAPI, data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
bootstrapTrace("lookup the configuration")
|
||||
// Override any values from ENVs.
|
||||
lookupConfigs(srvCfg, objAPI)
|
||||
|
||||
|
||||
@@ -26,7 +26,10 @@ import (
|
||||
)
|
||||
|
||||
func TestServerConfig(t *testing.T) {
|
||||
objLayer, fsDir, err := prepareFS()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
objLayer, fsDir, err := prepareFS(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -58,7 +61,7 @@ func TestServerConfig(t *testing.T) {
|
||||
}
|
||||
|
||||
// Initialize server config.
|
||||
if err := loadConfig(objLayer); err != nil {
|
||||
if err := loadConfig(objLayer, nil); err != nil {
|
||||
t.Fatalf("Unable to initialize from updated config file %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
+15
-10
@@ -25,7 +25,7 @@ import (
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
@@ -37,7 +37,10 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("Unable to encrypt config %w", err)
|
||||
}
|
||||
if err = migrateConfigPrefixToEncrypted(objAPI, encrypted); err != nil {
|
||||
if !encrypted {
|
||||
return nil
|
||||
}
|
||||
if err = migrateConfigPrefixToEncrypted(objAPI); err != nil {
|
||||
return fmt.Errorf("Unable to migrate all config at .minio.sys/config/: %w", err)
|
||||
}
|
||||
return nil
|
||||
@@ -48,6 +51,7 @@ const backendEncryptedFile = "backend-encrypted"
|
||||
var backendEncryptedMigrationComplete = []byte("encrypted")
|
||||
|
||||
func checkBackendEtcdEncrypted(ctx context.Context, client *etcd.Client) (bool, error) {
|
||||
bootstrapTrace("check if etcd backend is encrypted")
|
||||
data, err := readKeyEtcd(ctx, client, backendEncryptedFile)
|
||||
if err != nil && err != errConfigNotFound {
|
||||
return false, err
|
||||
@@ -56,6 +60,7 @@ func checkBackendEtcdEncrypted(ctx context.Context, client *etcd.Client) (bool,
|
||||
}
|
||||
|
||||
func checkBackendEncrypted(objAPI ObjectLayer) (bool, error) {
|
||||
bootstrapTrace("check if the config backend is encrypted")
|
||||
data, err := readConfig(GlobalContext, objAPI, backendEncryptedFile)
|
||||
if err != nil && err != errConfigNotFound {
|
||||
return false, err
|
||||
@@ -75,7 +80,9 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
|
||||
return nil
|
||||
}
|
||||
|
||||
if encrypted && GlobalKMS != nil {
|
||||
bootstrapTrace("encrypt etcd config")
|
||||
|
||||
if GlobalKMS != nil {
|
||||
stat, err := GlobalKMS.Stat(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -136,18 +143,16 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
|
||||
}
|
||||
}
|
||||
|
||||
if encrypted && GlobalKMS != nil {
|
||||
if GlobalKMS != nil {
|
||||
logger.Info("Migration of encrypted IAM config data completed. All data is now encrypted on etcd.")
|
||||
}
|
||||
|
||||
return deleteKeyEtcd(ctx, client, backendEncryptedFile)
|
||||
}
|
||||
|
||||
func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, encrypted bool) error {
|
||||
if !encrypted {
|
||||
return nil
|
||||
}
|
||||
if encrypted && GlobalKMS != nil {
|
||||
func migrateConfigPrefixToEncrypted(objAPI ObjectLayer) error {
|
||||
bootstrapTrace("migrating config prefix to encrypted")
|
||||
if GlobalKMS != nil {
|
||||
stat, err := GlobalKMS.Stat(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -202,7 +207,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, encrypted bool) error {
|
||||
}
|
||||
}
|
||||
|
||||
if encrypted && GlobalKMS != nil {
|
||||
if GlobalKMS != nil {
|
||||
logger.Info("Migration of encrypted config data completed. All config data is now encrypted.")
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
)
|
||||
|
||||
|
||||
+19
-3
@@ -27,7 +27,7 @@ import (
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/config/cache"
|
||||
@@ -2425,6 +2425,7 @@ func migrateV27ToV28() error {
|
||||
// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
|
||||
// if etcd is configured then migrates /config/config.json to '<export_path>/.minio.sys/config/config.json'
|
||||
func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
|
||||
bootstrapTrace("migrate config to .minio.sys/config/config.json")
|
||||
// Construct path to config.json for the given bucket.
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
|
||||
@@ -2472,6 +2473,7 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
|
||||
|
||||
// Migrates '.minio.sys/config.json' to v33.
|
||||
func migrateMinioSysConfig(objAPI ObjectLayer) error {
|
||||
bootstrapTrace("migrate .minio.sys/config/config.json to latest version")
|
||||
// Construct path to config.json for the given bucket.
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
|
||||
@@ -2533,6 +2535,21 @@ func checkConfigVersion(objAPI ObjectLayer, configFile string, version string) (
|
||||
}
|
||||
}
|
||||
|
||||
if version == "kvs" {
|
||||
// Check api config values are present.
|
||||
var vcfg struct {
|
||||
API struct {
|
||||
E []struct {
|
||||
Key string `json:"key"`
|
||||
Value string `json:"value"`
|
||||
} `json:"_"`
|
||||
} `json:"api"`
|
||||
}
|
||||
if err = json.Unmarshal(data, &vcfg); err != nil {
|
||||
return false, nil, nil
|
||||
}
|
||||
return len(vcfg.API.E) > 0, data, nil
|
||||
}
|
||||
var versionConfig struct {
|
||||
Version string `json:"version"`
|
||||
}
|
||||
@@ -2722,6 +2739,7 @@ func migrateV32ToV33MinioSys(objAPI ObjectLayer) error {
|
||||
}
|
||||
|
||||
func migrateMinioSysConfigToKV(objAPI ObjectLayer) error {
|
||||
bootstrapTrace("migrate config to KV style")
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
|
||||
// Check if the config version is latest, if not migrate.
|
||||
@@ -2740,9 +2758,7 @@ func migrateMinioSysConfigToKV(objAPI ObjectLayer) error {
|
||||
|
||||
newCfg := newServerConfig()
|
||||
|
||||
config.SetCredentials(newCfg, cfg.Credential)
|
||||
config.SetRegion(newCfg, cfg.Region)
|
||||
|
||||
storageclass.SetStorageClass(newCfg, cfg.StorageClass)
|
||||
|
||||
for k, loggerArgs := range cfg.Logger.HTTP {
|
||||
|
||||
+10
-17
@@ -18,16 +18,18 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/config"
|
||||
)
|
||||
|
||||
// Test if config v1 is purged
|
||||
func TestServerConfigMigrateV1(t *testing.T) {
|
||||
objLayer, fsDir, err := prepareFS()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
objLayer, fsDir, err := prepareFS(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -61,7 +63,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
|
||||
}
|
||||
|
||||
// Initialize server config and check again if everything is fine
|
||||
if err := loadConfig(objLayer); err != nil {
|
||||
if err := loadConfig(objLayer, nil); err != nil {
|
||||
t.Fatalf("Unable to initialize from updated config file %s", err)
|
||||
}
|
||||
}
|
||||
@@ -154,10 +156,12 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
|
||||
// Test if a config migration from v2 to v33 is successfully done
|
||||
func TestServerConfigMigrateV2toV33(t *testing.T) {
|
||||
rootPath := t.TempDir()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
globalConfigDir = &ConfigDir{path: rootPath}
|
||||
|
||||
objLayer, fsDir, err := prepareFS()
|
||||
objLayer, fsDir, err := prepareFS(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -201,20 +205,9 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
|
||||
}
|
||||
|
||||
// Initialize server config and check again if everything is fine
|
||||
if err := loadConfig(objLayer); err != nil {
|
||||
if err := loadConfig(objLayer, nil); err != nil {
|
||||
t.Fatalf("Unable to initialize from updated config file %s", err)
|
||||
}
|
||||
|
||||
// Check if accessKey and secretKey are not altered during migration
|
||||
caccessKey := globalServerConfig[config.CredentialsSubSys][config.Default].Get(config.AccessKey)
|
||||
if caccessKey != accessKey {
|
||||
t.Fatalf("Access key lost during migration, expected: %v, found:%v", accessKey, caccessKey)
|
||||
}
|
||||
|
||||
csecretKey := globalServerConfig[config.CredentialsSubSys][config.Default].Get(config.SecretKey)
|
||||
if csecretKey != secretKey {
|
||||
t.Fatalf("Secret key lost during migration, expected: %v, found: %v", secretKey, csecretKey)
|
||||
}
|
||||
}
|
||||
|
||||
// Test if all migrate code returns error with corrupted config files
|
||||
|
||||
@@ -855,5 +855,5 @@ type serverConfigV33 struct {
|
||||
// Add new external policy enforcements here.
|
||||
} `json:"policy"`
|
||||
|
||||
LDAPServerConfig xldap.Config `json:"ldapserverconfig"`
|
||||
LDAPServerConfig xldap.LegacyConfig `json:"ldapserverconfig"`
|
||||
}
|
||||
|
||||
+34
-19
@@ -28,7 +28,7 @@ import (
|
||||
"unicode/utf8"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
)
|
||||
@@ -152,30 +152,34 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, cfg interface{})
|
||||
return saveConfig(ctx, objAPI, configFile, data)
|
||||
}
|
||||
|
||||
func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, error) {
|
||||
// data is optional. If nil it will be loaded from backend.
|
||||
func readServerConfig(ctx context.Context, objAPI ObjectLayer, data []byte) (config.Config, error) {
|
||||
srvCfg := config.New()
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
data, err := readConfig(ctx, objAPI, configFile)
|
||||
if err != nil {
|
||||
if errors.Is(err, errConfigNotFound) {
|
||||
lookupConfigs(srvCfg, objAPI)
|
||||
return srvCfg, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if GlobalKMS != nil && !utf8.Valid(data) {
|
||||
data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
|
||||
minioMetaBucket: path.Join(minioMetaBucket, configFile),
|
||||
})
|
||||
var err error
|
||||
if len(data) == 0 {
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
data, err = readConfig(ctx, objAPI, configFile)
|
||||
if err != nil {
|
||||
lookupConfigs(srvCfg, objAPI)
|
||||
if errors.Is(err, errConfigNotFound) {
|
||||
lookupConfigs(srvCfg, objAPI)
|
||||
return srvCfg, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if GlobalKMS != nil && !utf8.Valid(data) {
|
||||
data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
|
||||
minioMetaBucket: path.Join(minioMetaBucket, configFile),
|
||||
})
|
||||
if err != nil {
|
||||
lookupConfigs(srvCfg, objAPI)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
json := jsoniter.ConfigCompatibleWithStandardLibrary
|
||||
if err = json.Unmarshal(data, &srvCfg); err != nil {
|
||||
if err := json.Unmarshal(data, &srvCfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -202,6 +206,8 @@ func NewConfigSys() *ConfigSys {
|
||||
|
||||
// Initialize and load config from remote etcd or local config directory
|
||||
func initConfig(objAPI ObjectLayer) error {
|
||||
bootstrapTrace("load the configuration")
|
||||
|
||||
if objAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
@@ -212,6 +218,15 @@ func initConfig(objAPI ObjectLayer) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Check if the config version is latest (kvs), if not migrate.
|
||||
ok, data, err := checkConfigVersion(objAPI, path.Join(minioConfigPrefix, minioConfigFile), "kvs")
|
||||
if err != nil && !errors.Is(err, errConfigNotFound) {
|
||||
return err
|
||||
}
|
||||
if ok {
|
||||
return loadConfig(objAPI, data)
|
||||
}
|
||||
|
||||
// Migrates ${HOME}/.minio/config.json or config.json.deprecated
|
||||
// to '<export_path>/.minio.sys/config/config.json'
|
||||
// ignore if the file doesn't exist.
|
||||
@@ -232,5 +247,5 @@ func initConfig(objAPI ObjectLayer) error {
|
||||
return fmt.Errorf("migrateMinioSysConfigToKV: %w", err)
|
||||
}
|
||||
|
||||
return loadConfig(objAPI)
|
||||
return loadConfig(objAPI, nil)
|
||||
}
|
||||
|
||||
+30
-9
@@ -21,8 +21,9 @@ import (
|
||||
"container/ring"
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/logger/message/log"
|
||||
"github.com/minio/minio/internal/logger/target/console"
|
||||
@@ -36,8 +37,11 @@ const defaultLogBufferCount = 10000
|
||||
|
||||
// HTTPConsoleLoggerSys holds global console logger state
|
||||
type HTTPConsoleLoggerSys struct {
|
||||
totalMessages int64
|
||||
failedMessages int64
|
||||
|
||||
sync.RWMutex
|
||||
pubsub *pubsub.PubSub
|
||||
pubsub *pubsub.PubSub[log.Info, madmin.LogMask]
|
||||
console *console.Target
|
||||
nodeName string
|
||||
logBuf *ring.Ring
|
||||
@@ -46,14 +50,18 @@ type HTTPConsoleLoggerSys struct {
|
||||
// NewConsoleLogger - creates new HTTPConsoleLoggerSys with all nodes subscribed to
|
||||
// the console logging pub sub system
|
||||
func NewConsoleLogger(ctx context.Context) *HTTPConsoleLoggerSys {
|
||||
ps := pubsub.New(8)
|
||||
return &HTTPConsoleLoggerSys{
|
||||
pubsub: ps,
|
||||
pubsub: pubsub.New[log.Info, madmin.LogMask](8),
|
||||
console: console.New(),
|
||||
logBuf: ring.New(defaultLogBufferCount),
|
||||
}
|
||||
}
|
||||
|
||||
// IsOnline always true in case of console logger
|
||||
func (sys *HTTPConsoleLoggerSys) IsOnline() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// SetNodeName - sets the node name if any after distributed setup has initialized
|
||||
func (sys *HTTPConsoleLoggerSys) SetNodeName(nodeName string) {
|
||||
if !globalIsDistErasure {
|
||||
@@ -72,11 +80,11 @@ func (sys *HTTPConsoleLoggerSys) SetNodeName(nodeName string) {
|
||||
// HasLogListeners returns true if console log listeners are registered
|
||||
// for this node or peers
|
||||
func (sys *HTTPConsoleLoggerSys) HasLogListeners() bool {
|
||||
return sys != nil && sys.pubsub.NumSubscribers(madmin.LogMaskAll) > 0
|
||||
return sys != nil && sys.pubsub.Subscribers() > 0
|
||||
}
|
||||
|
||||
// Subscribe starts console logging for this node.
|
||||
func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan pubsub.Maskable, doneCh <-chan struct{}, node string, last int, logKind madmin.LogMask, filter func(entry pubsub.Maskable) bool) error {
|
||||
func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan log.Info, doneCh <-chan struct{}, node string, last int, logKind madmin.LogMask, filter func(entry log.Info) bool) error {
|
||||
// Enable console logging for remote client.
|
||||
if !sys.HasLogListeners() {
|
||||
logger.AddSystemTarget(sys)
|
||||
@@ -116,7 +124,7 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan pubsub.Maskable, doneCh <-
|
||||
}
|
||||
}
|
||||
}
|
||||
return sys.pubsub.Subscribe(pubsub.MaskFromMaskable(madmin.LogMaskAll), subCh, doneCh, filter)
|
||||
return sys.pubsub.Subscribe(madmin.LogMaskAll, subCh, doneCh, filter)
|
||||
}
|
||||
|
||||
// Init if HTTPConsoleLoggerSys is valid, always returns nil right now
|
||||
@@ -134,6 +142,15 @@ func (sys *HTTPConsoleLoggerSys) String() string {
|
||||
return logger.ConsoleLoggerTgt
|
||||
}
|
||||
|
||||
// Stats returns the target statistics.
|
||||
func (sys *HTTPConsoleLoggerSys) Stats() types.TargetStats {
|
||||
return types.TargetStats{
|
||||
TotalMessages: atomic.LoadInt64(&sys.totalMessages),
|
||||
FailedMessages: atomic.LoadInt64(&sys.failedMessages),
|
||||
QueueLength: 0,
|
||||
}
|
||||
}
|
||||
|
||||
// Content returns the console stdout log
|
||||
func (sys *HTTPConsoleLoggerSys) Content() (logs []log.Entry) {
|
||||
sys.RLock()
|
||||
@@ -171,6 +188,7 @@ func (sys *HTTPConsoleLoggerSys) Send(entry interface{}) error {
|
||||
case string:
|
||||
lg = log.Info{ConsoleMsg: e, NodeName: sys.nodeName}
|
||||
}
|
||||
atomic.AddInt64(&sys.totalMessages, 1)
|
||||
|
||||
sys.pubsub.Publish(lg)
|
||||
sys.Lock()
|
||||
@@ -178,6 +196,9 @@ func (sys *HTTPConsoleLoggerSys) Send(entry interface{}) error {
|
||||
sys.logBuf.Value = lg
|
||||
sys.logBuf = sys.logBuf.Next()
|
||||
sys.Unlock()
|
||||
|
||||
return sys.console.Send(entry, string(logger.All))
|
||||
err := sys.console.Send(entry, string(logger.All))
|
||||
if err != nil {
|
||||
atomic.AddInt64(&sys.failedMessages, 1)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
+11
-15
@@ -7,7 +7,7 @@ import (
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
)
|
||||
|
||||
@@ -47,6 +47,8 @@ const (
|
||||
scannerMetricILM
|
||||
scannerMetricCheckReplication
|
||||
scannerMetricYield
|
||||
scannerMetricCleanAbandoned
|
||||
scannerMetricApplyNonCurrent
|
||||
|
||||
// START Trace metrics:
|
||||
scannerMetricStartTrace
|
||||
@@ -56,9 +58,10 @@ const (
|
||||
scannerMetricLastRealtime
|
||||
|
||||
// Trace only metrics:
|
||||
scannerMetricScanFolder // Scan a folder on disk, recursively.
|
||||
scannerMetricScanCycle // Full cycle, cluster global
|
||||
scannerMetricScanBucketDisk // Single bucket on one disk
|
||||
scannerMetricScanFolder // Scan a folder on disk, recursively.
|
||||
scannerMetricScanCycle // Full cycle, cluster global
|
||||
scannerMetricScanBucketDrive // Single bucket on one drive
|
||||
scannerMetricCompactFolder // Folder compacted.
|
||||
|
||||
// Must be last:
|
||||
scannerMetricLast
|
||||
@@ -119,13 +122,6 @@ func (p *scannerMetrics) incTime(s scannerMetric, d time.Duration) {
|
||||
}
|
||||
}
|
||||
|
||||
func (p *scannerMetrics) incNoTime(s scannerMetric) {
|
||||
atomic.AddUint64(&p.operations[s], 1)
|
||||
if s < scannerMetricLastRealtime {
|
||||
p.latency[s].add(0)
|
||||
}
|
||||
}
|
||||
|
||||
// timeILM times an ILM action.
|
||||
// lifecycle.NoneAction is ignored.
|
||||
// Use for s < scannerMetricLastRealtime
|
||||
@@ -186,9 +182,9 @@ func (p *scannerMetrics) getCurrentPaths() []string {
|
||||
return res
|
||||
}
|
||||
|
||||
// activeDisks returns the number of currently active disks.
|
||||
// activeDrives returns the number of currently active disks.
|
||||
// (since this is concurrent it may not be 100% reliable)
|
||||
func (p *scannerMetrics) activeDisks() int {
|
||||
func (p *scannerMetrics) activeDrives() int {
|
||||
var i int
|
||||
p.currentPaths.Range(func(k, v interface{}) bool {
|
||||
i++
|
||||
@@ -199,7 +195,7 @@ func (p *scannerMetrics) activeDisks() int {
|
||||
|
||||
// lifetime returns the lifetime count of the specified metric.
|
||||
func (p *scannerMetrics) lifetime(m scannerMetric) uint64 {
|
||||
if m < 0 || m >= scannerMetricLast {
|
||||
if m >= scannerMetricLast {
|
||||
return 0
|
||||
}
|
||||
val := atomic.LoadUint64(&p.operations[m])
|
||||
@@ -209,7 +205,7 @@ func (p *scannerMetrics) lifetime(m scannerMetric) uint64 {
|
||||
// lastMinute returns the last minute statistics of a metric.
|
||||
// m should be < scannerMetricLastRealtime
|
||||
func (p *scannerMetrics) lastMinute(m scannerMetric) AccElem {
|
||||
if m < 0 || m >= scannerMetricLastRealtime {
|
||||
if m >= scannerMetricLastRealtime {
|
||||
return AccElem{}
|
||||
}
|
||||
val := p.latency[m].total()
|
||||
|
||||
+69
-61
@@ -35,7 +35,7 @@ import (
|
||||
|
||||
"github.com/bits-and-blooms/bloom/v3"
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
"github.com/minio/minio/internal/bucket/object/lock"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
@@ -64,11 +64,6 @@ const (
|
||||
var (
|
||||
globalHealConfig heal.Config
|
||||
|
||||
dataScannerLeaderLockTimeout = newDynamicTimeoutWithOpts(dynamicTimeoutOpts{
|
||||
timeout: 30 * time.Second,
|
||||
minimum: 10 * time.Second,
|
||||
retryInterval: time.Second,
|
||||
})
|
||||
// Sleeper values are updated when config is loaded.
|
||||
scannerSleeper = newDynamicSleeper(10, 10*time.Second, true)
|
||||
scannerCycle = uatomic.NewDuration(dataScannerStartDelay)
|
||||
@@ -118,6 +113,10 @@ type backgroundHealInfo struct {
|
||||
}
|
||||
|
||||
func readBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer) backgroundHealInfo {
|
||||
if globalIsErasureSD {
|
||||
return backgroundHealInfo{}
|
||||
}
|
||||
|
||||
// Get last healing information
|
||||
buf, err := readConfig(ctx, objAPI, backgroundHealInfoPath)
|
||||
if err != nil {
|
||||
@@ -127,15 +126,17 @@ func readBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer) backgroundH
|
||||
return backgroundHealInfo{}
|
||||
}
|
||||
var info backgroundHealInfo
|
||||
err = json.Unmarshal(buf, &info)
|
||||
if err != nil {
|
||||
if err = json.Unmarshal(buf, &info); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return backgroundHealInfo{}
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
func saveBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer, info backgroundHealInfo) {
|
||||
if globalIsErasureSD {
|
||||
return
|
||||
}
|
||||
|
||||
b, err := json.Marshal(info)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
@@ -151,19 +152,9 @@ func saveBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer, info backgr
|
||||
// runDataScanner will start a data scanner.
|
||||
// The function will block until the context is canceled.
|
||||
// There should only ever be one scanner running per cluster.
|
||||
func runDataScanner(pctx context.Context, objAPI ObjectLayer) {
|
||||
// Make sure only 1 scanner is running on the cluster.
|
||||
locker := objAPI.NewNSLock(minioMetaBucket, "scanner/runDataScanner.lock")
|
||||
lkctx, err := locker.GetLock(pctx, dataScannerLeaderLockTimeout)
|
||||
if err != nil {
|
||||
if intDataUpdateTracker.debug {
|
||||
logger.LogIf(pctx, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
ctx := lkctx.Context()
|
||||
defer lkctx.Cancel()
|
||||
// No unlock for "leader" lock.
|
||||
func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
|
||||
ctx, cancel := globalLeaderLock.GetLock(ctx)
|
||||
defer cancel()
|
||||
|
||||
// Load current bloom cycle
|
||||
var cycleInfo currentScannerCycle
|
||||
@@ -175,7 +166,7 @@ func runDataScanner(pctx context.Context, objAPI ObjectLayer) {
|
||||
} else if len(buf) > 8 {
|
||||
cycleInfo.next = binary.LittleEndian.Uint64(buf[:8])
|
||||
buf = buf[8:]
|
||||
_, err = cycleInfo.UnmarshalMsg(buf)
|
||||
_, err := cycleInfo.UnmarshalMsg(buf)
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
|
||||
@@ -429,7 +420,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
|
||||
filter := f.withFilter
|
||||
_, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
|
||||
var activeLifeCycle *lifecycle.Lifecycle
|
||||
if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) {
|
||||
if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix) {
|
||||
if f.dataUsageScannerDebug {
|
||||
console.Debugf(scannerLogPrefix+" Prefix %q has active rules\n", prefix)
|
||||
}
|
||||
@@ -770,7 +761,9 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
|
||||
object: entry.name,
|
||||
versionID: "",
|
||||
}, madmin.HealItemObject)
|
||||
logger.LogIf(ctx, err)
|
||||
if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
foundObjs = foundObjs || err == nil
|
||||
return
|
||||
}
|
||||
@@ -785,7 +778,9 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
|
||||
object: fiv.Name,
|
||||
versionID: ver.VersionID,
|
||||
}, madmin.HealItemObject)
|
||||
logger.LogIf(ctx, err)
|
||||
if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
foundObjs = foundObjs || err == nil
|
||||
}
|
||||
},
|
||||
@@ -822,9 +817,8 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
|
||||
flat.Compacted = true
|
||||
var compact bool
|
||||
if flat.Objects < dataScannerCompactLeastObject {
|
||||
if f.dataUsageScannerDebug && flat.Objects > 1 {
|
||||
// Disabled, rather chatty:
|
||||
// console.Debugf(scannerLogPrefix+" Only %d objects, compacting %s -> %+v\n", flat.Objects, folder.name, flat)
|
||||
if flat.Objects > 1 {
|
||||
globalScannerMetrics.log(scannerMetricCompactFolder, folder.name)
|
||||
}
|
||||
compact = true
|
||||
} else {
|
||||
@@ -838,9 +832,8 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
|
||||
}
|
||||
}
|
||||
}
|
||||
if f.dataUsageScannerDebug && compact {
|
||||
// Disabled, rather chatty:
|
||||
// console.Debugf(scannerLogPrefix+" Only objects (%d), compacting %s -> %+v\n", flat.Objects, folder.name, flat)
|
||||
if compact {
|
||||
globalScannerMetrics.log(scannerMetricCompactFolder, folder.name)
|
||||
}
|
||||
}
|
||||
if compact {
|
||||
@@ -938,7 +931,10 @@ func (i *scannerItem) applyHealing(ctx context.Context, o ObjectLayer, oi Object
|
||||
ScanMode: scanMode,
|
||||
}
|
||||
res, err := o.HealObject(ctx, i.bucket, i.objectPath(), oi.VersionID, healOpts)
|
||||
if err != nil && !errors.Is(err, NotImplemented{}) {
|
||||
if err != nil {
|
||||
if errors.Is(err, NotImplemented{}) || isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||
err = nil
|
||||
}
|
||||
logger.LogIf(ctx, err)
|
||||
return 0
|
||||
}
|
||||
@@ -951,30 +947,26 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
if i.lifeCycle == nil {
|
||||
if i.debug {
|
||||
// disabled, very chatty:
|
||||
// console.Debugf(applyActionsLogPrefix+" no lifecycle rules to apply: %q\n", i.objectPath())
|
||||
}
|
||||
return false, size
|
||||
}
|
||||
|
||||
versionID := oi.VersionID
|
||||
rCfg, _ := globalBucketObjectLockSys.Get(i.bucket)
|
||||
action := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, oi, false)
|
||||
lcEvt := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, oi)
|
||||
if i.debug {
|
||||
if versionID != "" {
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, action)
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, lcEvt.Action)
|
||||
} else {
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), action)
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), lcEvt.Action)
|
||||
}
|
||||
}
|
||||
defer globalScannerMetrics.timeILM(action)
|
||||
defer globalScannerMetrics.timeILM(lcEvt.Action)
|
||||
|
||||
switch action {
|
||||
switch lcEvt.Action {
|
||||
case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
|
||||
return applyLifecycleAction(action, oi), 0
|
||||
return applyLifecycleAction(lcEvt.Action, oi, ""), 0
|
||||
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
|
||||
return applyLifecycleAction(action, oi), size
|
||||
return applyLifecycleAction(lcEvt.Action, oi, lcEvt.StorageClass), size
|
||||
default:
|
||||
// No action.
|
||||
return false, size
|
||||
@@ -1005,7 +997,8 @@ func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi O
|
||||
|
||||
// Remove this free version
|
||||
_, err = o.DeleteObject(ctx, oi.Bucket, oi.Name, ObjectOptions{
|
||||
VersionID: oi.VersionID,
|
||||
VersionID: oi.VersionID,
|
||||
InclFreeVersions: true,
|
||||
})
|
||||
if err == nil {
|
||||
auditLogLifecycle(ctx, oi, ILMFreeVersionDelete)
|
||||
@@ -1021,6 +1014,8 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
|
||||
if i.lifeCycle == nil {
|
||||
return fivs, nil
|
||||
}
|
||||
done := globalScannerMetrics.time(scannerMetricApplyNonCurrent)
|
||||
defer done()
|
||||
|
||||
_, days, lim := i.lifeCycle.NoncurrentVersionsExpirationLimit(lifecycle.ObjectOpts{Name: i.objectPath()})
|
||||
if lim == 0 || len(fivs) <= lim+1 { // fewer than lim _noncurrent_ versions
|
||||
@@ -1077,6 +1072,17 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
|
||||
// applyVersionActions will apply lifecycle checks on all versions of a scanned item. Returns versions that remain
|
||||
// after applying lifecycle checks configured.
|
||||
func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fivs []FileInfo) ([]FileInfo, error) {
|
||||
if i.heal.enabled {
|
||||
if healDeleteDangling {
|
||||
done := globalScannerMetrics.time(scannerMetricCleanAbandoned)
|
||||
err := o.CheckAbandonedParts(ctx, i.bucket, i.objectPath(), madmin.HealOpts{Remove: healDeleteDangling})
|
||||
done()
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", i.bucket, i.objectPath(), err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return i.applyNewerNoncurrentVersionLimit(ctx, o, fivs)
|
||||
}
|
||||
|
||||
@@ -1106,42 +1112,42 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
|
||||
return size
|
||||
}
|
||||
|
||||
func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, obj ObjectInfo, debug bool) (action lifecycle.Action) {
|
||||
action = lc.ComputeAction(obj.ToLifecycleOpts())
|
||||
if debug {
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", action)
|
||||
func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, obj ObjectInfo) lifecycle.Event {
|
||||
event := lc.Eval(obj.ToLifecycleOpts())
|
||||
if serverDebugLog {
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", event.Action)
|
||||
}
|
||||
|
||||
if action == lifecycle.NoneAction {
|
||||
return action
|
||||
if event.Action == lifecycle.NoneAction {
|
||||
return event
|
||||
}
|
||||
|
||||
switch action {
|
||||
switch event.Action {
|
||||
case lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredVersionAction:
|
||||
// Defensive code, should never happen
|
||||
if obj.VersionID == "" {
|
||||
return lifecycle.NoneAction
|
||||
return lifecycle.Event{Action: lifecycle.NoneAction}
|
||||
}
|
||||
if lr.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
|
||||
if debug {
|
||||
if serverDebugLog {
|
||||
if obj.VersionID != "" {
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: %s v(%s) is locked, not deleting\n", obj.Name, obj.VersionID)
|
||||
} else {
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", obj.Name)
|
||||
}
|
||||
}
|
||||
return lifecycle.NoneAction
|
||||
return lifecycle.Event{Action: lifecycle.NoneAction}
|
||||
}
|
||||
}
|
||||
|
||||
return action
|
||||
return event
|
||||
}
|
||||
|
||||
func applyTransitionRule(obj ObjectInfo) bool {
|
||||
func applyTransitionRule(obj ObjectInfo, storageClass string) bool {
|
||||
if obj.DeleteMarker {
|
||||
return false
|
||||
}
|
||||
globalTransitionState.queueTransitionTask(obj)
|
||||
globalTransitionState.queueTransitionTask(obj, storageClass)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -1210,14 +1216,14 @@ func applyExpiryRule(obj ObjectInfo, restoredObject, applyOnVersion bool) bool {
|
||||
}
|
||||
|
||||
// Perform actions (removal or transitioning of objects), return true the action is successfully performed
|
||||
func applyLifecycleAction(action lifecycle.Action, obj ObjectInfo) (success bool) {
|
||||
func applyLifecycleAction(action lifecycle.Action, obj ObjectInfo, storageClass string) (success bool) {
|
||||
switch action {
|
||||
case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
|
||||
success = applyExpiryRule(obj, false, action == lifecycle.DeleteVersionAction)
|
||||
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
|
||||
success = applyExpiryRule(obj, true, action == lifecycle.DeleteRestoredVersionAction)
|
||||
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
|
||||
success = applyTransitionRule(obj)
|
||||
success = applyTransitionRule(obj, storageClass)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -1438,9 +1444,11 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string) {
|
||||
case ILMTransition:
|
||||
apiName = "ILMTransition"
|
||||
}
|
||||
auditLogInternal(ctx, oi.Bucket, oi.Name, AuditLogOptions{
|
||||
auditLogInternal(ctx, AuditLogOptions{
|
||||
Event: event,
|
||||
APIName: apiName,
|
||||
Bucket: oi.Bucket,
|
||||
Object: oi.Name,
|
||||
VersionID: oi.VersionID,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -62,6 +62,15 @@ func (t *testingLogger) Type() types.TargetType {
|
||||
return types.TargetHTTP
|
||||
}
|
||||
|
||||
func (t *testingLogger) IsOnline() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Stats returns the target statistics.
|
||||
func (t *testingLogger) Stats() types.TargetStats {
|
||||
return types.TargetStats{}
|
||||
}
|
||||
|
||||
func (t *testingLogger) Send(entry interface{}) error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
|
||||
+24
-52
@@ -18,6 +18,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -31,7 +32,7 @@ import (
|
||||
|
||||
"github.com/cespare/xxhash/v2"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
@@ -392,13 +393,6 @@ func (e *dataUsageEntry) addChild(hash dataUsageHash) {
|
||||
e.Children[hash.Key()] = struct{}{}
|
||||
}
|
||||
|
||||
// removeChild will remove a child based on its hash.
|
||||
func (e *dataUsageEntry) removeChild(hash dataUsageHash) {
|
||||
if len(e.Children) > 0 {
|
||||
delete(e.Children, hash.Key())
|
||||
}
|
||||
}
|
||||
|
||||
// Create a clone of the entry.
|
||||
func (e dataUsageEntry) clone() dataUsageEntry {
|
||||
// We operate on a copy from the receiver.
|
||||
@@ -488,43 +482,6 @@ func (d *dataUsageCache) deleteRecursive(h dataUsageHash) {
|
||||
}
|
||||
}
|
||||
|
||||
// keepBuckets will keep only the buckets specified specified by delete all others.
|
||||
func (d *dataUsageCache) keepBuckets(b []BucketInfo) {
|
||||
lu := make(map[dataUsageHash]struct{})
|
||||
for _, v := range b {
|
||||
lu[hashPath(v.Name)] = struct{}{}
|
||||
}
|
||||
d.keepRootChildren(lu)
|
||||
}
|
||||
|
||||
// keepRootChildren will keep the root children specified by delete all others.
|
||||
func (d *dataUsageCache) keepRootChildren(list map[dataUsageHash]struct{}) {
|
||||
root := d.root()
|
||||
if root == nil {
|
||||
return
|
||||
}
|
||||
rh := d.rootHash()
|
||||
for k := range d.Cache {
|
||||
h := dataUsageHash(k)
|
||||
if h == rh {
|
||||
continue
|
||||
}
|
||||
if _, ok := list[h]; !ok {
|
||||
delete(d.Cache, k)
|
||||
d.deleteRecursive(h)
|
||||
root.removeChild(h)
|
||||
}
|
||||
}
|
||||
// Clean up abandoned children.
|
||||
for k := range root.Children {
|
||||
h := dataUsageHash(k)
|
||||
if _, ok := list[h]; !ok {
|
||||
delete(root.Children, k)
|
||||
}
|
||||
}
|
||||
d.Cache[rh.Key()] = *root
|
||||
}
|
||||
|
||||
// dui converts the flattened version of the path to madmin.DataUsageInfo.
|
||||
// As a side effect d will be flattened, use a clone if this is not ok.
|
||||
func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
|
||||
@@ -923,13 +880,28 @@ func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string)
|
||||
|
||||
// save the content of the cache to minioMetaBackgroundOpsBucket with the provided name.
|
||||
func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string) error {
|
||||
pr, pw := io.Pipe()
|
||||
go func() {
|
||||
pw.CloseWithError(d.serializeTo(pw))
|
||||
}()
|
||||
defer pr.Close()
|
||||
var r io.Reader
|
||||
|
||||
r, err := hash.NewReader(pr, -1, "", "", -1)
|
||||
// If big, do streaming...
|
||||
size := int64(-1)
|
||||
if len(d.Cache) > 10000 {
|
||||
pr, pw := io.Pipe()
|
||||
go func() {
|
||||
pw.CloseWithError(d.serializeTo(pw))
|
||||
}()
|
||||
defer pr.Close()
|
||||
r = pr
|
||||
} else {
|
||||
var buf bytes.Buffer
|
||||
err := d.serializeTo(&buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r = &buf
|
||||
size = int64(buf.Len())
|
||||
}
|
||||
|
||||
hr, err := hash.NewReader(r, size, "", "", size)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -940,7 +912,7 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
|
||||
_, err = store.PutObject(ctx,
|
||||
dataUsageBucket,
|
||||
name,
|
||||
NewPutObjReader(r),
|
||||
NewPutObjReader(hr),
|
||||
ObjectOptions{})
|
||||
if isErrBucketNotFound(err) {
|
||||
return nil
|
||||
|
||||
@@ -21,7 +21,7 @@ import (
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
)
|
||||
|
||||
// BucketTargetUsageInfo - bucket target usage info provides
|
||||
|
||||
+13
-12
@@ -512,7 +512,7 @@ func (c *diskCache) statCachedMeta(ctx context.Context, cacheObjPath string) (me
|
||||
return
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer cLock.RUnlock(lkctx.Cancel)
|
||||
defer cLock.RUnlock(lkctx)
|
||||
return c.statCache(ctx, cacheObjPath)
|
||||
}
|
||||
|
||||
@@ -597,7 +597,7 @@ func (c *diskCache) SaveMetadata(ctx context.Context, bucket, object string, met
|
||||
return err
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer cLock.Unlock(lkctx.Cancel)
|
||||
defer cLock.Unlock(lkctx)
|
||||
if err = c.saveMetadata(ctx, bucket, object, meta, actualSize, rs, rsFileName, incHitsOnly); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -706,7 +706,7 @@ func (c *diskCache) updateMetadata(ctx context.Context, bucket, object, etag str
|
||||
|
||||
if globalCacheKMS != nil {
|
||||
// Calculating object encryption key
|
||||
key, err = decryptObjectInfo(key, bucket, object, m.Meta)
|
||||
key, err = decryptObjectMeta(key, bucket, object, m.Meta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -842,7 +842,7 @@ func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Read
|
||||
return oi, err
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer cLock.Unlock(lkctx.Cancel)
|
||||
defer cLock.Unlock(lkctx)
|
||||
|
||||
return c.put(ctx, bucket, object, data, size, rs, opts, incHitsOnly, writeback)
|
||||
}
|
||||
@@ -953,7 +953,7 @@ func (c *diskCache) putRange(ctx context.Context, bucket, object string, data io
|
||||
objSize, _ = sio.EncryptedSize(uint64(size))
|
||||
|
||||
}
|
||||
cacheFile := MustGetUUID()
|
||||
cacheFile := mustGetUUID()
|
||||
n, _, err := c.bitrotWriteToCache(cachePath, cacheFile, reader, actualSize)
|
||||
if IsErr(err, baseErrs...) {
|
||||
// take the cache drive offline
|
||||
@@ -1003,6 +1003,7 @@ func (c *diskCache) bitrotReadFromCache(ctx context.Context, filePath string, of
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rc.Close()
|
||||
bufp := c.pool.Get().(*[]byte)
|
||||
defer c.pool.Put(bufp)
|
||||
|
||||
@@ -1075,7 +1076,7 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
|
||||
return nil, numHits, err
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer cLock.RUnlock(lkctx.Cancel)
|
||||
defer cLock.RUnlock(lkctx)
|
||||
|
||||
var objInfo ObjectInfo
|
||||
var rngInfo RangeInfo
|
||||
@@ -1212,7 +1213,7 @@ func (c *diskCache) Delete(ctx context.Context, bucket, object string) (err erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cLock.Unlock(lkctx.Cancel)
|
||||
defer cLock.Unlock(lkctx)
|
||||
return removeAll(cacheObjPath)
|
||||
}
|
||||
|
||||
@@ -1323,7 +1324,7 @@ func (c *diskCache) PutObjectPart(ctx context.Context, bucket, object, uploadID
|
||||
}
|
||||
|
||||
ctx = lkctx.Context()
|
||||
defer partIDLock.Unlock(lkctx.Cancel)
|
||||
defer partIDLock.Unlock(lkctx)
|
||||
meta, _, _, err := c.statCache(ctx, uploadIDDir)
|
||||
// Case where object not yet cached
|
||||
if err != nil {
|
||||
@@ -1380,7 +1381,7 @@ func (c *diskCache) SavePartMetadata(ctx context.Context, bucket, object, upload
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer uploadLock.Unlock(ulkctx.Cancel)
|
||||
defer uploadLock.Unlock(ulkctx)
|
||||
|
||||
metaPath := pathJoin(uploadDir, cacheMetaJSONFile)
|
||||
f, err := OpenFile(metaPath, os.O_RDWR|writeMode, 0o666)
|
||||
@@ -1397,7 +1398,7 @@ func (c *diskCache) SavePartMetadata(ctx context.Context, bucket, object, upload
|
||||
var objectEncryptionKey crypto.ObjectKey
|
||||
if globalCacheKMS != nil {
|
||||
// Calculating object encryption key
|
||||
key, err = decryptObjectInfo(key, bucket, object, m.Meta)
|
||||
key, err = decryptObjectMeta(key, bucket, object, m.Meta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1427,7 +1428,7 @@ func newCachePartEncryptReader(ctx context.Context, bucket, object string, partI
|
||||
var objectEncryptionKey, partEncryptionKey crypto.ObjectKey
|
||||
|
||||
// Calculating object encryption key
|
||||
key, err = decryptObjectInfo(key, bucket, object, metadata)
|
||||
key, err = decryptObjectMeta(key, bucket, object, metadata)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1479,7 +1480,7 @@ func (c *diskCache) CompleteMultipartUpload(ctx context.Context, bucket, object,
|
||||
}
|
||||
|
||||
ctx = lkctx.Context()
|
||||
defer cLock.Unlock(lkctx.Cancel)
|
||||
defer cLock.Unlock(lkctx)
|
||||
mpartCachePath := getMultipartCacheSHADir(c.dir, bucket, object)
|
||||
uploadIDDir := path.Join(mpartCachePath, uploadID)
|
||||
|
||||
|
||||
+1
-1
@@ -728,7 +728,7 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
|
||||
if cerr != nil {
|
||||
return putObjectFn(ctx, bucket, object, r, opts)
|
||||
}
|
||||
defer cLock.Unlock(lkctx.Cancel)
|
||||
defer cLock.Unlock(lkctx)
|
||||
// Initialize pipe to stream data to backend
|
||||
pipeReader, pipeWriter := io.Pipe()
|
||||
hashReader, err := hash.NewReader(pipeReader, size, "", "", r.ActualSize())
|
||||
|
||||
@@ -130,7 +130,6 @@ func TestDynamicTimeoutConcurrent(t *testing.T) {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < 100; i++ {
|
||||
timeout.LogFailure()
|
||||
for j := 0; j < 100; j++ {
|
||||
timeout.LogSuccess(time.Duration(float64(time.Second) * rng.Float64()))
|
||||
}
|
||||
|
||||
+43
-17
@@ -473,7 +473,7 @@ func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, m
|
||||
return newEncryptReader(r.Context(), content, kind, keyID, key, bucket, object, metadata, ctx)
|
||||
}
|
||||
|
||||
func decryptObjectInfo(key []byte, bucket, object string, metadata map[string]string) ([]byte, error) {
|
||||
func decryptObjectMeta(key []byte, bucket, object string, metadata map[string]string) ([]byte, error) {
|
||||
switch kind, _ := crypto.IsEncrypted(metadata); kind {
|
||||
case crypto.S3:
|
||||
var KMS kms.KMS = GlobalKMS
|
||||
@@ -544,7 +544,7 @@ func DecryptCopyRequestR(client io.Reader, h http.Header, bucket, object string,
|
||||
}
|
||||
|
||||
func newDecryptReader(client io.Reader, key []byte, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
|
||||
objectEncryptionKey, err := decryptObjectInfo(key, bucket, object, metadata)
|
||||
objectEncryptionKey, err := decryptObjectMeta(key, bucket, object, metadata)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -656,7 +656,7 @@ func (d *DecryptBlocksReader) buildDecrypter(partID int) error {
|
||||
return err
|
||||
}
|
||||
|
||||
objectEncryptionKey, err := decryptObjectInfo(key, d.bucket, d.object, m)
|
||||
objectEncryptionKey, err := decryptObjectMeta(key, d.bucket, d.object, m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -822,7 +822,7 @@ func getDecryptedETag(headers http.Header, objInfo ObjectInfo, copySource bool)
|
||||
return objInfo.ETag[len(objInfo.ETag)-32:]
|
||||
}
|
||||
|
||||
objectEncryptionKey, err := decryptObjectInfo(key[:], objInfo.Bucket, objInfo.Name, objInfo.UserDefined)
|
||||
objectEncryptionKey, err := decryptObjectMeta(key[:], objInfo.Bucket, objInfo.Name, objInfo.UserDefined)
|
||||
if err != nil {
|
||||
return objInfo.ETag
|
||||
}
|
||||
@@ -1043,18 +1043,6 @@ func DecryptObjectInfo(info *ObjectInfo, r *http.Request) (encrypted bool, err e
|
||||
return encrypted, nil
|
||||
}
|
||||
|
||||
// The customer key in the header is used by the gateway for encryption in the case of
|
||||
// s3 gateway double encryption. A new client key is derived from the customer provided
|
||||
// key to be sent to the s3 backend for encryption at the backend.
|
||||
func deriveClientKey(clientKey [32]byte, bucket, object string) [32]byte {
|
||||
var key [32]byte
|
||||
mac := hmac.New(sha256.New, clientKey[:])
|
||||
mac.Write([]byte(crypto.SSEC.String()))
|
||||
mac.Write([]byte(path.Join(bucket, object)))
|
||||
mac.Sum(key[:0])
|
||||
return key
|
||||
}
|
||||
|
||||
type (
|
||||
objectMetaEncryptFn func(baseKey string, data []byte) []byte
|
||||
objectMetaDecryptFn func(baseKey string, data []byte) ([]byte, error)
|
||||
@@ -1085,7 +1073,7 @@ func (o *ObjectInfo) metadataDecrypter() objectMetaDecryptFn {
|
||||
return input, nil
|
||||
}
|
||||
|
||||
key, err := decryptObjectInfo(nil, o.Bucket, o.Name, o.UserDefined)
|
||||
key, err := decryptObjectMeta(nil, o.Bucket, o.Name, o.UserDefined)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1095,6 +1083,44 @@ func (o *ObjectInfo) metadataDecrypter() objectMetaDecryptFn {
|
||||
}
|
||||
}
|
||||
|
||||
// metadataEncryptFn provides an encryption function for metadata.
|
||||
// Will return nil, nil if unencrypted.
|
||||
func (o *ObjectInfo) metadataEncryptFn(headers http.Header) (objectMetaEncryptFn, error) {
|
||||
kind, _ := crypto.IsEncrypted(o.UserDefined)
|
||||
switch kind {
|
||||
case crypto.SSEC:
|
||||
if crypto.SSECopy.IsRequested(headers) {
|
||||
key, err := crypto.SSECopy.ParseHTTP(headers)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
objectEncryptionKey, err := decryptObjectMeta(key[:], o.Bucket, o.Name, o.UserDefined)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(objectEncryptionKey) == 32 {
|
||||
var key crypto.ObjectKey
|
||||
copy(key[:], objectEncryptionKey)
|
||||
return metadataEncrypter(key), nil
|
||||
}
|
||||
return nil, errors.New("metadataEncryptFn: unexpected key size")
|
||||
}
|
||||
case crypto.S3, crypto.S3KMS:
|
||||
objectEncryptionKey, err := decryptObjectMeta(nil, o.Bucket, o.Name, o.UserDefined)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(objectEncryptionKey) == 32 {
|
||||
var key crypto.ObjectKey
|
||||
copy(key[:], objectEncryptionKey)
|
||||
return metadataEncrypter(key), nil
|
||||
}
|
||||
return nil, errors.New("metadataEncryptFn: unexpected key size")
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// decryptChecksums will attempt to decode checksums and return it/them if set.
|
||||
func (o *ObjectInfo) decryptChecksums() map[string]string {
|
||||
data := o.Checksum
|
||||
|
||||
@@ -79,7 +79,7 @@ func TestEncryptRequest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
var decryptObjectInfoTests = []struct {
|
||||
var decryptObjectMetaTests = []struct {
|
||||
info ObjectInfo
|
||||
request *http.Request
|
||||
expErr error
|
||||
@@ -122,7 +122,7 @@ var decryptObjectInfoTests = []struct {
|
||||
}
|
||||
|
||||
func TestDecryptObjectInfo(t *testing.T) {
|
||||
for i, test := range decryptObjectInfoTests {
|
||||
for i, test := range decryptObjectMetaTests {
|
||||
if encrypted, err := DecryptObjectInfo(&test.info, test.request); err != test.expErr {
|
||||
t.Errorf("Test %d: Decryption returned wrong error code: got %d , want %d", i, err, test.expErr)
|
||||
} else if _, enc := crypto.IsEncrypted(test.info.UserDefined); encrypted && enc != encrypted {
|
||||
|
||||
@@ -207,7 +207,7 @@ func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint6
|
||||
}
|
||||
|
||||
// Returns all the expanded endpoints, each argument is expanded separately.
|
||||
func (s endpointSet) getEndpoints() (endpoints []string) {
|
||||
func (s *endpointSet) getEndpoints() (endpoints []string) {
|
||||
if len(s.endpoints) != 0 {
|
||||
return s.endpoints
|
||||
}
|
||||
@@ -343,7 +343,13 @@ func createServerEndpoints(serverAddr string, args ...string) (
|
||||
return nil, -1, errInvalidArgument
|
||||
}
|
||||
|
||||
if !ellipses.HasEllipses(args...) {
|
||||
ok := true
|
||||
for _, arg := range args {
|
||||
ok = ok && !ellipses.HasEllipses(arg)
|
||||
}
|
||||
|
||||
// None of the args have ellipses use the old style.
|
||||
if ok {
|
||||
setArgs, err := GetAllSets(args...)
|
||||
if err != nil {
|
||||
return nil, -1, err
|
||||
@@ -365,6 +371,10 @@ func createServerEndpoints(serverAddr string, args ...string) (
|
||||
|
||||
var foundPrevLocal bool
|
||||
for _, arg := range args {
|
||||
if !ellipses.HasEllipses(arg) && len(args) > 1 {
|
||||
// TODO: support SNSD deployments to be decommissioned in future
|
||||
return nil, -1, fmt.Errorf("all args must have ellipses for pool expansion (%w) args: %s", errInvalidArgument, args)
|
||||
}
|
||||
setArgs, err := GetAllSets(arg)
|
||||
if err != nil {
|
||||
return nil, -1, err
|
||||
|
||||
+45
-4
@@ -417,6 +417,27 @@ func (endpoints Endpoints) atleastOneEndpointLocal() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func hostResolveToLocalhost(endpoint Endpoint) bool {
|
||||
hostIPs, err := getHostIP(endpoint.Hostname())
|
||||
if err != nil {
|
||||
// Log the message to console about the host resolving
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags(
|
||||
"host",
|
||||
endpoint.Hostname(),
|
||||
)
|
||||
ctx := logger.SetReqInfo(GlobalContext, reqInfo)
|
||||
logger.LogOnceIf(ctx, err, endpoint.Hostname(), logger.Application)
|
||||
return false
|
||||
}
|
||||
var loopback int
|
||||
for _, hostIP := range hostIPs.ToSlice() {
|
||||
if net.ParseIP(hostIP).IsLoopback() {
|
||||
loopback++
|
||||
}
|
||||
}
|
||||
return loopback == len(hostIPs)
|
||||
}
|
||||
|
||||
// UpdateIsLocal - resolves the host and discovers the local host.
|
||||
func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
|
||||
orchestrated := IsDocker() || IsKubernetes()
|
||||
@@ -452,6 +473,26 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
|
||||
endpoints[i].Hostname(),
|
||||
)
|
||||
|
||||
if orchestrated && hostResolveToLocalhost(endpoints[i]) {
|
||||
// time elapsed
|
||||
timeElapsed := time.Since(startTime)
|
||||
// log error only if more than a second has elapsed
|
||||
if timeElapsed > time.Second {
|
||||
reqInfo.AppendTags("elapsedTime",
|
||||
humanize.RelTime(startTime,
|
||||
startTime.Add(timeElapsed),
|
||||
"elapsed",
|
||||
"",
|
||||
))
|
||||
ctx := logger.SetReqInfo(GlobalContext,
|
||||
reqInfo)
|
||||
logger.LogOnceIf(ctx, fmt.Errorf("%s resolves to localhost in a containerized deployment, waiting for it to resolve to a valid IP",
|
||||
endpoints[i].Hostname()), endpoints[i].Hostname(), logger.Application)
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// return err if not Docker or Kubernetes
|
||||
// We use IsDocker() to check for Docker environment
|
||||
// We use IsKubernetes() to check for Kubernetes environment
|
||||
@@ -465,7 +506,7 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
|
||||
if err != nil {
|
||||
// time elapsed
|
||||
timeElapsed := time.Since(startTime)
|
||||
// log error only if more than 1s elapsed
|
||||
// log error only if more than a second has elapsed
|
||||
if timeElapsed > time.Second {
|
||||
reqInfo.AppendTags("elapsedTime",
|
||||
humanize.RelTime(startTime,
|
||||
@@ -475,7 +516,7 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
|
||||
))
|
||||
ctx := logger.SetReqInfo(GlobalContext,
|
||||
reqInfo)
|
||||
logger.LogIf(ctx, err, logger.Application)
|
||||
logger.LogOnceIf(ctx, err, endpoints[i].Hostname(), logger.Application)
|
||||
}
|
||||
} else {
|
||||
resolvedList[i] = true
|
||||
@@ -592,14 +633,14 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
|
||||
return endpoints, setupType, err
|
||||
}
|
||||
if endpoint.Type() != PathEndpointType {
|
||||
return endpoints, setupType, config.ErrInvalidFSEndpoint(nil).Msg("use path style endpoint for FS setup")
|
||||
return endpoints, setupType, config.ErrInvalidEndpoint(nil).Msg("use path style endpoint for single node setup")
|
||||
}
|
||||
endpoints = append(endpoints, endpoint)
|
||||
setupType = ErasureSDSetupType
|
||||
|
||||
// Check for cross device mounts if any.
|
||||
if err = checkCrossDeviceMounts(endpoints); err != nil {
|
||||
return endpoints, setupType, config.ErrInvalidFSEndpoint(nil).Msg(err.Error())
|
||||
return endpoints, setupType, config.ErrInvalidEndpoint(nil).Msg(err.Error())
|
||||
}
|
||||
|
||||
return endpoints, setupType, nil
|
||||
|
||||
@@ -21,8 +21,6 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/minio/minio-go/v7/pkg/s3utils"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -32,181 +30,6 @@ var bucketOpIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied, errUnform
|
||||
// list all errors that can be ignored in a bucket metadata operation.
|
||||
var bucketMetadataOpIgnoredErrs = append(bucketOpIgnoredErrs, errVolumeNotFound)
|
||||
|
||||
// Bucket operations
|
||||
|
||||
// MakeBucket - make a bucket.
|
||||
func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts MakeBucketOptions) error {
|
||||
defer NSUpdated(bucket, slashSeparator)
|
||||
|
||||
// Verify if bucket is valid.
|
||||
if !isMinioMetaBucketName(bucket) {
|
||||
if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
|
||||
return BucketNameInvalid{Bucket: bucket}
|
||||
}
|
||||
}
|
||||
|
||||
storageDisks := er.getDisks()
|
||||
|
||||
g := errgroup.WithNErrs(len(storageDisks))
|
||||
|
||||
// Make a volume entry on all underlying storage disks.
|
||||
for index := range storageDisks {
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
if storageDisks[index] != nil {
|
||||
if err := storageDisks[index].MakeVol(ctx, bucket); err != nil {
|
||||
if opts.ForceCreate && errors.Is(err, errVolumeExists) {
|
||||
// No need to return error when force create was
|
||||
// requested.
|
||||
return nil
|
||||
}
|
||||
if !errors.Is(err, errVolumeExists) {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return errDiskNotFound
|
||||
}, index)
|
||||
}
|
||||
|
||||
err := reduceWriteQuorumErrs(ctx, g.Wait(), bucketOpIgnoredErrs, er.defaultWQuorum())
|
||||
return toObjectErr(err, bucket)
|
||||
}
|
||||
|
||||
func undoDeleteBucket(storageDisks []StorageAPI, bucket string) {
|
||||
g := errgroup.WithNErrs(len(storageDisks))
|
||||
// Undo previous make bucket entry on all underlying storage disks.
|
||||
for index := range storageDisks {
|
||||
if storageDisks[index] == nil {
|
||||
continue
|
||||
}
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
_ = storageDisks[index].MakeVol(context.Background(), bucket)
|
||||
return nil
|
||||
}, index)
|
||||
}
|
||||
|
||||
// Wait for all make vol to finish.
|
||||
g.Wait()
|
||||
}
|
||||
|
||||
// getBucketInfo - returns the BucketInfo from one of the load balanced disks.
|
||||
func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string, opts BucketOptions) (bucketInfo BucketInfo, err error) {
|
||||
storageDisks := er.getDisks()
|
||||
|
||||
g := errgroup.WithNErrs(len(storageDisks))
|
||||
bucketsInfo := make([]BucketInfo, len(storageDisks))
|
||||
// Undo previous make bucket entry on all underlying storage disks.
|
||||
for index := range storageDisks {
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
if storageDisks[index] == nil {
|
||||
return errDiskNotFound
|
||||
}
|
||||
volInfo, err := storageDisks[index].StatVol(ctx, bucketName)
|
||||
if err != nil {
|
||||
if opts.Deleted {
|
||||
dvi, derr := storageDisks[index].StatVol(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, bucketName))
|
||||
if derr != nil {
|
||||
return err
|
||||
}
|
||||
bucketsInfo[index] = BucketInfo{Name: bucketName, Deleted: dvi.Created}
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
bucketsInfo[index] = BucketInfo{Name: volInfo.Name, Created: volInfo.Created}
|
||||
return nil
|
||||
}, index)
|
||||
}
|
||||
|
||||
errs := g.Wait()
|
||||
|
||||
for i, err := range errs {
|
||||
if err == nil {
|
||||
return bucketsInfo[i], nil
|
||||
}
|
||||
}
|
||||
|
||||
// If all our errors were ignored, then we try to
|
||||
// reduce to one error based on read quorum.
|
||||
// `nil` is deliberately passed for ignoredErrs
|
||||
// because these errors were already ignored.
|
||||
return BucketInfo{}, reduceReadQuorumErrs(ctx, errs, nil, er.defaultRQuorum())
|
||||
}
|
||||
|
||||
// GetBucketInfo - returns BucketInfo for a bucket.
|
||||
func (er erasureObjects) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bi BucketInfo, e error) {
|
||||
bucketInfo, err := er.getBucketInfo(ctx, bucket, opts)
|
||||
if err != nil {
|
||||
return bi, toObjectErr(err, bucket)
|
||||
}
|
||||
return bucketInfo, nil
|
||||
}
|
||||
|
||||
// DeleteBucket - deletes a bucket.
|
||||
func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
|
||||
// Collect if all disks report volume not found.
|
||||
defer NSUpdated(bucket, slashSeparator)
|
||||
|
||||
storageDisks := er.getDisks()
|
||||
|
||||
g := errgroup.WithNErrs(len(storageDisks))
|
||||
|
||||
for index := range storageDisks {
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
if storageDisks[index] != nil {
|
||||
return storageDisks[index].DeleteVol(ctx, bucket, opts.Force)
|
||||
}
|
||||
return errDiskNotFound
|
||||
}, index)
|
||||
}
|
||||
|
||||
// Wait for all the delete vols to finish.
|
||||
dErrs := g.Wait()
|
||||
|
||||
if opts.Force {
|
||||
for _, err := range dErrs {
|
||||
if err != nil {
|
||||
undoDeleteBucket(storageDisks, bucket)
|
||||
return toObjectErr(err, bucket)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
err := reduceWriteQuorumErrs(ctx, dErrs, bucketOpIgnoredErrs, er.defaultWQuorum())
|
||||
if err == errErasureWriteQuorum && !opts.NoRecreate {
|
||||
undoDeleteBucket(storageDisks, bucket)
|
||||
}
|
||||
|
||||
if err == nil || errors.Is(err, errVolumeNotFound) {
|
||||
var purgedDangling bool
|
||||
// At this point we have `err == nil` but some errors might be `errVolumeNotEmpty`
|
||||
// we should proceed to attempt a force delete of such buckets.
|
||||
for index, err := range dErrs {
|
||||
if err == errVolumeNotEmpty && storageDisks[index] != nil {
|
||||
storageDisks[index].RenameFile(ctx, bucket, "", minioMetaTmpDeletedBucket, mustGetUUID())
|
||||
purgedDangling = true
|
||||
}
|
||||
}
|
||||
// if we purged dangling buckets, ignore errVolumeNotFound error.
|
||||
if purgedDangling {
|
||||
err = nil
|
||||
}
|
||||
if opts.SRDeleteOp == MarkDelete {
|
||||
er.markDelete(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix, bucket))
|
||||
}
|
||||
}
|
||||
|
||||
return toObjectErr(err, bucket)
|
||||
}
|
||||
|
||||
// markDelete creates a vol entry in .minio.sys/buckets/.deleted until site replication
|
||||
// syncs the delete to peers
|
||||
func (er erasureObjects) markDelete(ctx context.Context, bucket, prefix string) error {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user