Compare commits

..

49 Commits

Author SHA1 Message Date
Harshavardhana c7f7e67a10 Do not allow adding root user to IAM subsystem (#16803) 2023-03-13 12:46:17 -07:00
Klaus Post 628042e65e tests: Protect globalLocalDrives against races (#16800) 2023-03-13 06:04:20 -07:00
ferhat elmas cde7eeb660 chore: drop go versions in static analysis (#16790) 2023-03-11 22:10:33 -08:00
Aditya Manthramurthy 6305b206e1 fix: site-repl should heal STS with virtual parent (#16792) 2023-03-10 16:21:51 -08:00
Klaus Post d85da9236e Add Object Version count histogram (#16739) 2023-03-10 08:53:59 -08:00
Minio Trusted 9800760cb3 Update yaml files to latest version RELEASE.2023-03-09T23-16-13Z 2023-03-10 00:59:46 +00:00
Anis Elleuch 5c087bdcad fix: a cosmetic error reporting with a lock timeout (#16788) 2023-03-09 15:16:13 -08:00
Klaus Post a547bf517d Remove locks on usage cache (#16786) 2023-03-09 15:15:46 -08:00
Harshavardhana b984bf8d1a allow expiration of all versions during Listing() (#16757) 2023-03-09 15:15:30 -08:00
Daniel Valdivia 18f9cccfa7 Upgrade Console to v0.25.0 (#16782)
Signed-off-by: Daniel Valdivia <18384552+dvaldivia@users.noreply.github.com>
2023-03-08 12:15:13 -08:00
Poorna fb6ab1cca2 fix: allow replication of 'null' delete markers (#16773) 2023-03-08 07:03:29 -08:00
Krishnan Parthasarathi 56c57e2c53 tier-stats: Avoid repeated logs (#16774) 2023-03-07 16:43:05 -08:00
Poorna a6057c35cc Avoid peer notification when peer is offline, tune retries (#16737) 2023-03-07 08:13:28 -08:00
Harshavardhana 901887e6bf feat: add lambda transformation functions target (#16507) 2023-03-07 08:12:41 -08:00
Poorna ee54643004 Avoid unnecessary replication heal attempts (#16769) 2023-03-07 07:43:38 -08:00
Harshavardhana 0a17acdb34 return error if policy changes on disabled groups (#16766) 2023-03-06 10:46:24 -08:00
Harshavardhana 72e5212842 fix: handle syscall.EROFS also for osIsPermission() (#16765) 2023-03-06 08:56:29 -08:00
ferhat elmas 714283fae2 cleanup ignored static analysis (#16767) 2023-03-06 08:56:10 -08:00
ferhat elmas 3423028713 cleanup Go linter settings (#16736) 2023-03-04 20:57:35 -08:00
jiuker 9d062b37d7 return underlying error with BackendDown{} error (#16738) 2023-03-03 23:56:53 -08:00
Harshavardhana 4636d3a9c3 upgrade all dependencies (#16753) 2023-03-03 18:22:40 -08:00
Aditya Manthramurthy c95ede35c1 Switch windows CI back to go 1.19.x (#16755) 2023-03-03 15:19:28 -08:00
Aditya Manthramurthy 7415e1aa56 Switch to go1.20 in CI (#16743) 2023-03-03 10:15:03 -08:00
jiuker f350953a19 calculate disk cache usage percent accurately (#16740) 2023-03-02 20:32:22 -08:00
Daniel Valdivia 958bba5b42 Attach creds, owner and region to madmin calls (#16658)
Signed-off-by: Daniel Valdivia <18384552+dvaldivia@users.noreply.github.com>
2023-03-02 14:35:24 -08:00
Poorna 0f2b95b497 fix a data race in IAM loading (#16742) 2023-03-02 14:32:54 -08:00
Klaus Post d07089ceac Fix scanner deadlock on lost global lock (#16726) 2023-02-28 21:34:45 -08:00
Aditya Manthramurthy 47dfa62384 Update LDAP doc for new policy attach|detach cmds (#16723) 2023-02-27 21:04:27 -08:00
Minio Trusted 3a3265cf88 Update yaml files to latest version RELEASE.2023-02-27T18-10-45Z 2023-02-28 00:30:32 +00:00
Harshavardhana 0ff931dc76 fix: allow CORS to work by default (#16713) 2023-02-27 10:10:45 -08:00
Praveen raj Mani 4d708cebe9 Support adding service accounts with expiration (#16430)
Co-authored-by: Harshavardhana <harsha@minio.io>
2023-02-27 10:10:22 -08:00
dorman 4d7c8e3bb8 Remove redundant log (#16710)
Co-authored-by: z30001483 <zekaifeng2@huawei.com>
2023-02-27 09:59:47 -08:00
Aditya Manthramurthy 8cde38404d Add metrics for custom auth plugin (#16701) 2023-02-27 09:55:18 -08:00
Krishnan Parthasarathi fe7bf6cbbc Support tier-add if tier backend not empty (#16715) 2023-02-27 09:26:26 -08:00
Shubhendu 8b4eb2304b Set logger webhook proxy on subnet proxy change (#16665)
Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>
2023-02-27 08:35:36 -08:00
Harshavardhana ae029191a3 liveness returns "busy" if queued requests > available capacity (#16719) 2023-02-27 08:34:52 -08:00
Harshavardhana bfedea9bad fix: disk healing should honor the right pool/set index (#16712) 2023-02-27 04:55:32 -08:00
Aditya Manthramurthy 7777d3b43a Remove globalSTSTLSConfig (#16709) 2023-02-26 23:37:00 -08:00
Aditya Manthramurthy 9ed4fc9687 Remove globalOpenIDConfig (#16708) 2023-02-25 21:01:37 -08:00
jiuker b49b39e99d fix: errNoSuchPolicy should use errors.Is (#16656) 2023-02-25 00:01:37 -08:00
jiuker cd3a2de5a3 add isValidLocation to common parseLocation (#16690) 2023-02-25 08:09:20 +05:30
jiuker 6e8960ccdd fix: delete globalProfiler should lock (#16697) 2023-02-25 08:07:44 +05:30
Aditya Manthramurthy e05f3d5d84 Remove globalLDAPConfig (#16706) 2023-02-25 08:07:22 +05:30
Anis Elleuch 94c6cb1323 tests: Add test for S3 API error codes (#16705) 2023-02-25 08:06:29 +05:30
Aditya Manthramurthy 3f81cd1b22 Update OpenID doc with info on redirection params (#16704) 2023-02-24 12:13:00 -08:00
Anis Elleuch 8da0f4c5bb Better error message when TLS certs do not have proper permissions (#16703) 2023-02-24 06:34:55 -08:00
Klaus Post 9acf1024e4 Remove bloom filter (#16682)
Removes the bloom filter since it has so limited usability, often gets saturated anyway and adds a bunch of complexity to the scanner.

Also removes a tiny bit of CPU by each write operation.
2023-02-24 09:03:31 +05:30
Harshavardhana b21d3f9b82 event target registration failures must be returned (#16700) 2023-02-23 21:59:14 +05:30
Minio Trusted 2bbf380262 Update yaml files to latest version RELEASE.2023-02-22T18-23-45Z 2023-02-22 21:50:11 +00:00
152 changed files with 4075 additions and 2504 deletions
+3 -3
View File
@@ -20,11 +20,11 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [1.19.x]
go-version: [1.20.x]
os: [ubuntu-latest]
steps:
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
- uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
- uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3
with:
go-version: ${{ matrix.go-version }}
check-latest: true
+15 -8
View File
@@ -20,21 +20,28 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [1.19.x]
go-version: [1.20.x]
os: [ubuntu-latest]
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Setup dockerfile for build test
run: |
echo "FROM golang:1.19.4" >> Dockerfile.fips.test
echo "COPY . /minio" >> Dockerfile.fips.test
echo "WORKDIR /minio" >> Dockerfile.fips.test
echo "ENV GOEXPERIMENT=boringcrypto" >> Dockerfile.fips.test
echo "RUN make" >> Dockerfile.fips.test
GO_VERSION=$(go version | cut -d ' ' -f 3 | sed 's/go//')
echo Detected go version $GO_VERSION
cat > Dockerfile.fips.test <<EOF
FROM golang:${GO_VERSION}
COPY . /minio
WORKDIR /minio
ENV GOEXPERIMENT=boringcrypto
RUN make
EOF
- name: Build
uses: docker/build-push-action@v3
@@ -49,4 +56,4 @@ jobs:
- name: Test binary
run: |
docker run --rm minio/fips-test:latest ./minio --version
docker run --rm -i minio/fips-test:latest /bin/bash -c 'go tool nm ./minio' | grep -q FIPS
docker run --rm -i minio/fips-test:latest /bin/bash -c 'go tool nm ./minio | grep FIPS | grep -q FIPS'
+2 -2
View File
@@ -20,10 +20,10 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [1.19.x]
go-version: [1.20.x]
os: [ubuntu-latest]
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
+7 -2
View File
@@ -20,10 +20,15 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [1.19.x]
go-version: [1.20.x, 1.19.x]
os: [ubuntu-latest, windows-latest]
exclude:
- os: ubuntu-latest
go-version: 1.19.x
- os: windows-latest
go-version: 1.20.x
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
+2 -2
View File
@@ -20,10 +20,10 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [1.19.x]
go-version: [1.20.x]
os: [ubuntu-latest]
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
+2 -2
View File
@@ -61,7 +61,7 @@ jobs:
# are turned off - i.e. if ldap="", then ldap server is not enabled for
# the tests.
matrix:
go-version: [1.19.x]
go-version: [1.20.x]
ldap: ["", "localhost:389"]
etcd: ["", "http://localhost:2379"]
openid: ["", "http://127.0.0.1:5556/dex"]
@@ -75,7 +75,7 @@ jobs:
openid: "http://127.0.0.1:5556/dex"
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
+1 -1
View File
@@ -20,7 +20,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Lint all docs
run: |
+2 -2
View File
@@ -21,10 +21,10 @@ jobs:
strategy:
matrix:
go-version: [1.19.x]
go-version: [1.20.x]
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
+2 -2
View File
@@ -20,11 +20,11 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [1.19.x]
go-version: [1.20.x]
os: [ubuntu-latest]
steps:
- uses: actions/checkout@v1
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
+1 -1
View File
@@ -20,7 +20,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: 1.19
go-version: 1.20.x
check-latest: true
- name: Get official govulncheck
run: go install golang.org/x/vuln/cmd/govulncheck@latest
+1
View File
@@ -38,3 +38,4 @@ docs/debugging/s3-check-md5/s3-check-md5
docs/debugging/hash-set/hash-set
docs/debugging/healing-bin/healing-bin
docs/debugging/inspect/inspect
.bin/
+18 -23
View File
@@ -1,39 +1,34 @@
linters-settings:
gofumpt:
lang-version: "1.18"
simplify: true
misspell:
locale: US
staticcheck:
checks: ['all', '-ST1005', '-ST1000', '-SA4000', '-SA9004', '-SA1019', '-SA1008', '-U1000', '-ST1016']
linters:
disable-all: true
enable:
- typecheck
- goimports
- misspell
- govet
- revive
- ineffassign
- gomodguard
- durationcheck
- gocritic
- gofmt
- gofumpt
- goimports
- gomodguard
- govet
- ineffassign
- misspell
- revive
- staticcheck
- tenv
- typecheck
- unconvert
- unused
- gocritic
- gofumpt
- tenv
- durationcheck
issues:
exclude-use-default: false
exclude:
- should have a package comment
- error strings should not be capitalized or end with punctuation or a newline
# todo fix these when we get enough time.
- "singleCaseSwitch: should rewrite switch statement to if statement"
- "unlambda: replace"
- "captLocal:"
- "ifElseChain:"
- "elseif:"
service:
golangci-lint-version: 1.43.0 # use the fixed version to not introduce new linters unexpectedly
- should have a package comment
- error strings should not be capitalized or end with punctuation or a newline
+24 -143
View File
@@ -3175,69 +3175,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
================================================================
github.com/bits-and-blooms/bitset
https://github.com/bits-and-blooms/bitset
----------------------------------------------------------------
Copyright (c) 2014 Will Fitzgerald. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
================================================================
github.com/bits-and-blooms/bloom/v3
https://github.com/bits-and-blooms/bloom/v3
----------------------------------------------------------------
Copyright (c) 2014 Will Fitzgerald. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
================================================================
github.com/buger/jsonparser
https://github.com/buger/jsonparser
----------------------------------------------------------------
@@ -15500,37 +15437,36 @@ https://github.com/matttproud/golang_protobuf_extensions
github.com/miekg/dns
https://github.com/miekg/dns
----------------------------------------------------------------
Copyright (c) 2009 The Go Authors. All rights reserved.
BSD 3-Clause License
Copyright (c) 2009, The Go Authors. Extensions copyright (c) 2011, Miek Gieben.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
As this is fork of the official Go code the same license applies.
Extensions of the original work are copyright (c) 2011 Miek Gieben
================================================================
github.com/minio/cli
@@ -25887,61 +25823,6 @@ https://github.com/tklauser/numcpus
================================================================
github.com/twmb/murmur3
https://github.com/twmb/murmur3
----------------------------------------------------------------
Copyright 2013, Sébastien Paolacci.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of the library nor the
names of its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Copyright 2018, Travis Bischel.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of the library nor the
names of its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
================================================================
github.com/unrolled/secure
https://github.com/unrolled/secure
----------------------------------------------------------------
+8 -6
View File
@@ -8,6 +8,10 @@ GOOS := $(shell go env GOOS)
VERSION ?= $(shell git describe --tags)
TAG ?= "minio/minio:$(VERSION)"
GOLANGCI_VERSION = v1.51.2
GOLANGCI_DIR = .bin/golangci/$(GOLANGCI_VERSION)
GOLANGCI = $(GOLANGCI_DIR)/golangci-lint
all: build
checks: ## check dependencies
@@ -19,24 +23,22 @@ help: ## print this help
getdeps: ## fetch necessary dependencies
@mkdir -p ${GOPATH}/bin
@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin
@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR) $(GOLANGCI_VERSION)
@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7
@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
@echo "Installing staticcheck" && go install honnef.co/go/tools/cmd/staticcheck@latest
crosscompile: ## cross compile minio
@(env bash $(PWD)/buildscripts/cross-compile.sh)
verifiers: getdeps lint check-gen
verifiers: lint check-gen
check-gen: ## check for updated autogenerated files
@go generate ./... >/dev/null
@(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false)
lint: ## runs golangci-lint suite of linters
lint: getdeps ## runs golangci-lint suite of linters
@echo "Running $@ check"
@${GOPATH}/bin/golangci-lint run --build-tags kqueue --timeout=10m --config ./.golangci.yml
@${GOPATH}/bin/staticcheck --tests=false ./...
@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml --fix
check: test
test: verifiers build ## builds minio, runs linters, tests
+1 -2
View File
@@ -5,7 +5,6 @@ set -e
export GORACE="history_size=7"
export MINIO_API_REQUESTS_MAX=10000
## TODO remove `dsync` from race detector once this is merged and released https://go-review.googlesource.com/c/go/+/333529/
for d in $(go list ./... | grep -v dsync); do
for d in $(go list ./...); do
CGO_ENABLED=1 go test -v -race --timeout 100m "$d"
done
+2 -4
View File
@@ -686,8 +686,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
continue
}
bucket, fileName := slc[0], slc[1]
switch fileName {
case objectLockConfig:
if fileName == objectLockConfig {
reader, err := file.Open()
if err != nil {
rpt.SetStatus(bucket, fileName, err)
@@ -756,8 +755,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
continue
}
bucket, fileName := slc[0], slc[1]
switch fileName {
case bucketVersioningConfig:
if fileName == bucketVersioningConfig {
reader, err := file.Open()
if err != nil {
rpt.SetStatus(bucket, fileName, err)
+38 -4
View File
@@ -35,6 +35,7 @@ import (
idplugin "github.com/minio/minio/internal/config/identity/plugin"
polplugin "github.com/minio/minio/internal/config/policy/plugin"
"github.com/minio/minio/internal/config/storageclass"
"github.com/minio/minio/internal/config/subnet"
"github.com/minio/minio/internal/logger"
"github.com/minio/mux"
iampolicy "github.com/minio/pkg/iam/policy"
@@ -87,6 +88,10 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
return
}
// Check if subnet proxy being deleted and if so the value of proxy of subnet
// target of logger webhook configuration also should be deleted
loggerWebhookProxyDeleted := setLoggerWebhookSubnetProxy(subSys, cfg)
if err = saveServerConfig(ctx, objectAPI, cfg); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
@@ -101,6 +106,10 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
dynamic := config.SubSystemsDynamic.Contains(subSys)
if dynamic {
applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
if subSys == config.SubnetSubSys && loggerWebhookProxyDeleted {
// Logger webhook proxy deleted, apply the dynamic changes
applyDynamic(ctx, objectAPI, cfg, config.LoggerWebhookSubSys, r, w)
}
}
}
@@ -132,9 +141,10 @@ func (bce badConfigErr) Unwrap() error {
}
type setConfigResult struct {
Cfg config.Config
SubSys string
Dynamic bool
Cfg config.Config
SubSys string
Dynamic bool
LoggerWebhookCfgUpdated bool
}
// SetConfigKVHandler - PUT /minio/admin/v3/set-config-kv
@@ -175,6 +185,11 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
if result.Dynamic {
applyDynamic(ctx, objectAPI, result.Cfg, result.SubSys, r, w)
// If logger webhook config updated (proxy due to callhome), explicitly dynamically
// apply the config
if result.LoggerWebhookCfgUpdated {
applyDynamic(ctx, objectAPI, result.Cfg, config.LoggerWebhookSubSys, r, w)
}
}
writeSuccessResponseHeadersOnly(w)
@@ -201,6 +216,10 @@ func setConfigKV(ctx context.Context, objectAPI ObjectLayer, kvBytes []byte) (re
return
}
// Check if subnet proxy being set and if so set the same value to proxy of subnet
// target of logger webhook configuration
result.LoggerWebhookCfgUpdated = setLoggerWebhookSubnetProxy(result.SubSys, result.Cfg)
// Update the actual server config on disk.
if err = saveServerConfig(ctx, objectAPI, result.Cfg); err != nil {
return
@@ -501,7 +520,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
case config.IdentityLDAPSubSys:
off = !xldap.Enabled(item.Config)
case config.IdentityTLSSubSys:
off = !globalSTSTLSConfig.Enabled
off = !globalIAMSys.STSTLSConfig.Enabled
case config.IdentityPluginSubSys:
off = !idplugin.Enabled(item.Config)
}
@@ -518,3 +537,18 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
writeSuccessResponseJSON(w, econfigData)
}
// setLoggerWebhookSubnetProxy - Sets the logger webhook's subnet proxy value to
// one being set for subnet proxy
func setLoggerWebhookSubnetProxy(subSys string, cfg config.Config) bool {
if subSys == config.SubnetSubSys {
subnetWebhookCfg := cfg[config.LoggerWebhookSubSys][subnet.LoggerWebhookName]
loggerWebhookSubnetProxy := subnetWebhookCfg.Get(logger.Proxy)
subnetProxy := cfg[config.SubnetSubSys][config.Default].Get(logger.Proxy)
if loggerWebhookSubnetProxy != subnetProxy {
subnetWebhookCfg.Set(logger.Proxy, subnetProxy)
return true
}
}
return false
}
+8 -8
View File
@@ -178,9 +178,9 @@ func handleCreateUpdateValidation(s config.Config, subSys, cfgTarget string, isU
var cfgInfos []madmin.IDPCfgInfo
switch subSys {
case madmin.IdentityOpenIDSubSys:
cfgInfos, _ = globalOpenIDConfig.GetConfigInfo(s, cfgTarget)
cfgInfos, _ = globalIAMSys.OpenIDConfig.GetConfigInfo(s, cfgTarget)
case madmin.IdentityLDAPSubSys:
cfgInfos, _ = globalLDAPConfig.GetConfigInfo(s, cfgTarget)
cfgInfos, _ = globalIAMSys.LDAPConfig.GetConfigInfo(s, cfgTarget)
}
if len(cfgInfos) > 0 && !isUpdate {
@@ -240,10 +240,10 @@ func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http
switch idpCfgType {
case madmin.OpenidIDPCfg:
cfg := globalServerConfig.Clone()
cfgList, err = globalOpenIDConfig.GetConfigList(cfg)
cfgList, err = globalIAMSys.OpenIDConfig.GetConfigList(cfg)
case madmin.LDAPIDPCfg:
cfg := globalServerConfig.Clone()
cfgList, err = globalLDAPConfig.GetConfigList(cfg)
cfgList, err = globalIAMSys.LDAPConfig.GetConfigList(cfg)
default:
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
@@ -296,9 +296,9 @@ func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.
var err error
switch idpCfgType {
case madmin.OpenidIDPCfg:
cfgInfos, err = globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
cfgInfos, err = globalIAMSys.OpenIDConfig.GetConfigInfo(cfg, cfgName)
case madmin.LDAPIDPCfg:
cfgInfos, err = globalLDAPConfig.GetConfigInfo(cfg, cfgName)
cfgInfos, err = globalIAMSys.LDAPConfig.GetConfigInfo(cfg, cfgName)
}
if err != nil {
if errors.Is(err, openid.ErrProviderConfigNotFound) || errors.Is(err, cfgldap.ErrProviderConfigNotFound) {
@@ -355,7 +355,7 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
switch idpCfgType {
case madmin.OpenidIDPCfg:
subSys = config.IdentityOpenIDSubSys
cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfgCopy, cfgName)
cfgInfos, err := globalIAMSys.OpenIDConfig.GetConfigInfo(cfgCopy, cfgName)
if err != nil {
if errors.Is(err, openid.ErrProviderConfigNotFound) {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
@@ -380,7 +380,7 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
}
case madmin.LDAPIDPCfg:
subSys = config.IdentityLDAPSubSys
cfgInfos, err := globalLDAPConfig.GetConfigInfo(cfgCopy, cfgName)
cfgInfos, err := globalIAMSys.LDAPConfig.GetConfigInfo(cfgCopy, cfgName)
if err != nil {
if errors.Is(err, openid.ErrProviderConfigNotFound) {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
+15 -12
View File
@@ -27,12 +27,12 @@ import (
"context"
"fmt"
"runtime"
"sync"
"testing"
"time"
"github.com/minio/madmin-go/v2"
minio "github.com/minio/minio-go/v7"
"github.com/minio/minio/internal/sync/errgroup"
)
func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {
@@ -129,18 +129,21 @@ func (s *TestSuiteIAM) TestDeleteUserRace(c *check) {
secretKeys[i] = secretKey
}
wg := sync.WaitGroup{}
g := errgroup.Group{}
for i := 0; i < userCount; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
uClient := s.getUserClient(c, accessKeys[i], secretKeys[i], "")
err := s.adm.RemoveUser(ctx, accessKeys[i])
if err != nil {
c.Fatalf("unable to remove user: %v", err)
g.Go(func(i int) func() error {
return func() error {
uClient := s.getUserClient(c, accessKeys[i], secretKeys[i], "")
err := s.adm.RemoveUser(ctx, accessKeys[i])
if err != nil {
return err
}
c.mustNotListObjects(ctx, uClient, bucket)
return nil
}
c.mustNotListObjects(ctx, uClient, bucket)
}(i)
}(i), i)
}
if errs := g.Wait(); len(errs) > 0 {
c.Fatalf("unable to remove users: %v", errs)
}
wg.Wait()
}
+34 -19
View File
@@ -431,7 +431,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
}
// Not allowed to add a user with same access key as root credential
if owner && accessKey == cred.AccessKey {
if accessKey == globalActiveCred.AccessKey {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
return
}
@@ -650,10 +650,11 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
}
opts := newServiceAccountOpts{
accessKey: createReq.AccessKey,
secretKey: createReq.SecretKey,
comment: createReq.Comment,
claims: make(map[string]interface{}),
accessKey: createReq.AccessKey,
secretKey: createReq.SecretKey,
comment: createReq.Comment,
expiration: createReq.Expiration,
claims: make(map[string]interface{}),
}
// Find the user for the request sender (as it may be sent via a service
@@ -741,9 +742,9 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
// In case of LDAP we need to resolve the targetUser to a DN and
// query their groups:
if globalLDAPConfig.Enabled() {
if globalIAMSys.LDAPConfig.Enabled() {
opts.claims[ldapUserN] = targetUser // simple username
targetUser, targetGroups, err = globalLDAPConfig.LookupUserDN(targetUser)
targetUser, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(targetUser)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
@@ -775,8 +776,9 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
createResp := madmin.AddServiceAccountResp{
Credentials: madmin.Credentials{
AccessKey: newCred.AccessKey,
SecretKey: newCred.SecretKey,
AccessKey: newCred.AccessKey,
SecretKey: newCred.SecretKey,
Expiration: newCred.Expiration,
},
}
@@ -809,6 +811,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
Claims: opts.claims,
SessionPolicy: createReq.Policy,
Status: auth.AccountOn,
Expiration: createReq.Expiration,
},
},
UpdatedAt: updatedAt,
@@ -891,6 +894,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
secretKey: updateReq.NewSecretKey,
status: updateReq.NewStatus,
comment: updateReq.NewComment,
expiration: updateReq.NewExpiration,
sessionPolicy: sp,
}
updatedAt, err := globalIAMSys.UpdateServiceAccount(ctx, accessKey, opts)
@@ -910,6 +914,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
Status: opts.status,
Comment: opts.comment,
SessionPolicy: updateReq.NewPolicy,
Expiration: updateReq.NewExpiration,
},
},
UpdatedAt: updatedAt,
@@ -988,12 +993,18 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
return
}
var expiration *time.Time
if !svcAccount.Expiration.IsZero() && !svcAccount.Expiration.Equal(timeSentinel) {
expiration = &svcAccount.Expiration
}
infoResp := madmin.InfoServiceAccountResp{
ParentUser: svcAccount.ParentUser,
Comment: svcAccount.Comment,
AccountStatus: svcAccount.Status,
ImpliedPolicy: policy == nil,
Policy: string(policyJSON),
Expiration: expiration,
}
data, err := json.Marshal(infoResp)
@@ -1321,11 +1332,12 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
// Fetch the data usage of the current bucket
var size uint64
var objectsCount uint64
var objectsHist map[string]uint64
var objectsHist, versionsHist map[string]uint64
if !dataUsageInfo.LastUpdate.IsZero() {
size = dataUsageInfo.BucketsUsage[bucket.Name].Size
objectsCount = dataUsageInfo.BucketsUsage[bucket.Name].ObjectsCount
objectsHist = dataUsageInfo.BucketsUsage[bucket.Name].ObjectSizesHistogram
versionsHist = dataUsageInfo.BucketsUsage[bucket.Name].ObjectVersionsHistogram
}
// Fetch the prefix usage of the current bucket
var prefixUsage map[string]uint64
@@ -1339,12 +1351,13 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
tcfg, _, _ := globalBucketMetadataSys.GetTaggingConfig(bucket.Name)
acctInfo.Buckets = append(acctInfo.Buckets, madmin.BucketAccessInfo{
Name: bucket.Name,
Created: bucket.Created,
Size: size,
Objects: objectsCount,
ObjectSizesHistogram: objectsHist,
PrefixUsage: prefixUsage,
Name: bucket.Name,
Created: bucket.Created,
Size: size,
Objects: objectsCount,
ObjectSizesHistogram: objectsHist,
ObjectVersionsHistogram: versionsHist,
PrefixUsage: prefixUsage,
Details: &madmin.BucketDetails{
Versioning: globalBucketVersioningSys.Enabled(bucket.Name),
VersioningSuspended: globalBucketVersioningSys.Suspended(bucket.Name),
@@ -2284,7 +2297,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
}
for accessKey, ureq := range userAccts {
// Not allowed to add a user with same access key as root credential
if owner && accessKey == cred.AccessKey {
if accessKey == globalActiveCred.AccessKey {
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAddUserInvalidArgument, err, allUsersFile, accessKey), r.URL)
return
}
@@ -2436,6 +2449,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
secretKey: svcAcctReq.SecretKey,
status: svcAcctReq.Status,
comment: svcAcctReq.Comment,
expiration: svcAcctReq.Expiration,
sessionPolicy: sp,
}
_, err = globalIAMSys.UpdateServiceAccount(ctx, svcAcctReq.AccessKey, opts)
@@ -2451,13 +2465,14 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
sessionPolicy: sp,
claims: svcAcctReq.Claims,
comment: svcAcctReq.Comment,
expiration: svcAcctReq.Expiration,
}
// In case of LDAP we need to resolve the targetUser to a DN and
// query their groups:
if globalLDAPConfig.Enabled() {
if globalIAMSys.LDAPConfig.Enabled() {
opts.claims[ldapUserN] = svcAcctReq.AccessKey // simple username
targetUser, _, err := globalLDAPConfig.LookupUserDN(svcAcctReq.AccessKey)
targetUser, _, err := globalIAMSys.LDAPConfig.LookupUserDN(svcAcctReq.AccessKey)
if err != nil {
writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
return
+1 -1
View File
@@ -825,7 +825,7 @@ func (s *TestSuiteIAM) TestGroupAddRemove(c *check) {
if set.CreateStringSet(groups...).Contains(group) {
c.Fatalf("created group still present!")
}
groupInfo, err = s.adm.GetGroupDescription(ctx, group)
_, err = s.adm.GetGroupDescription(ctx, group)
if err == nil {
c.Fatalf("group appears to exist")
}
+5 -3
View File
@@ -803,6 +803,8 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
for {
select {
case <-ctx.Done():
globalProfilerMu.Lock()
defer globalProfilerMu.Unlock()
for k, v := range globalProfiler {
v.Stop()
delete(globalProfiler, k)
@@ -1819,8 +1821,8 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
kmsStat := fetchKMSStatus()
ldap := madmin.LDAP{}
if globalLDAPConfig.Enabled() {
ldapConn, err := globalLDAPConfig.LDAP.Connect()
if globalIAMSys.LDAPConfig.Enabled() {
ldapConn, err := globalIAMSys.LDAPConfig.LDAP.Connect()
//nolint:gocritic
if err != nil {
ldap.Status = string(madmin.ItemOffline)
@@ -2427,7 +2429,7 @@ func assignPoolNumbers(servers []madmin.ServerProperties) {
func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
lambdaMap := make(map[string][]madmin.TargetIDStatus)
for _, tgt := range globalConfigTargetList.Targets() {
for _, tgt := range globalNotifyTargetList.Targets() {
targetIDStatus := make(map[string]madmin.Status)
active, _ := tgt.IsActive()
targetID := tgt.ID()
+54 -62
View File
@@ -42,6 +42,7 @@ import (
objectlock "github.com/minio/minio/internal/bucket/object/lock"
"github.com/minio/minio/internal/bucket/versioning"
levent "github.com/minio/minio/internal/config/lambda/event"
"github.com/minio/minio/internal/event"
"github.com/minio/minio/internal/hash"
"github.com/minio/pkg/bucket/policy"
@@ -165,7 +166,6 @@ const (
ErrMalformedDate
ErrMalformedPresignedDate
ErrMalformedCredentialDate
ErrMalformedCredentialRegion
ErrMalformedExpires
ErrNegativeExpires
ErrAuthHeaderEmpty
@@ -244,8 +244,6 @@ const (
// Add new extended error codes here.
// MinIO extended errors.
ErrReadQuorum
ErrWriteQuorum
ErrStorageFull
ErrRequestBodyParse
ErrObjectExistsAsDirectory
@@ -269,6 +267,7 @@ const (
ErrAdminNoSuchUser
ErrAdminNoSuchGroup
ErrAdminGroupNotEmpty
ErrAdminGroupDisabled
ErrAdminNoSuchJob
ErrAdminNoSuchPolicy
ErrAdminPolicyChangeAlreadyApplied
@@ -412,6 +411,12 @@ const (
ErrPostPolicyConditionInvalidFormat
ErrInvalidChecksum
// Lambda functions
ErrLambdaARNInvalid
ErrLambdaARNNotFound
apiErrCodeEnd // This is used only for the testing code
)
type errorCodeMap map[APIErrorCode]APIError
@@ -425,8 +430,7 @@ func (e errorCodeMap) ToAPIErrWithErr(errCode APIErrorCode, err error) APIError
apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
}
if globalSite.Region != "" {
switch errCode {
case ErrAuthorizationHeaderMalformed:
if errCode == ErrAuthorizationHeaderMalformed {
apiErr.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalSite.Region)
return apiErr
}
@@ -1262,6 +1266,11 @@ var errorCodes = errorCodeMap{
Description: "The specified group is not empty - cannot remove it.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrAdminGroupDisabled: {
Code: "XMinioAdminGroupDisabled",
Description: "The specified group is disabled.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrAdminNoSuchPolicy: {
Code: "XMinioAdminNoSuchPolicy",
Description: "The canned policy does not exist.",
@@ -1521,7 +1530,7 @@ var errorCodes = errorCodeMap{
HTTPStatusCode: http.StatusBadRequest,
},
ErrBusy: {
Code: "Busy",
Code: "ServerBusy",
Description: "The service is unavailable. Please retry.",
HTTPStatusCode: http.StatusServiceUnavailable,
},
@@ -1960,6 +1969,16 @@ var errorCodes = errorCodeMap{
Description: "Invalid checksum provided.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrLambdaARNInvalid: {
Code: "LambdaARNInvalid",
Description: "The specified lambda ARN is invalid",
HTTPStatusCode: http.StatusBadRequest,
},
ErrLambdaARNNotFound: {
Code: "LambdaARNNotFound",
Description: "The specified lambda ARN does not exist",
HTTPStatusCode: http.StatusNotFound,
},
ErrPolicyAlreadyAttached: {
Code: "XMinioPolicyAlreadyAttached",
Description: "The specified policy is already attached.",
@@ -1983,15 +2002,15 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
// Only return ErrClientDisconnected if the provided context is actually canceled.
// This way downstream context.Canceled will still report ErrOperationTimedOut
if contextCanceled(ctx) {
if ctx.Err() == context.Canceled {
return ErrClientDisconnected
}
if contextCanceled(ctx) && errors.Is(ctx.Err(), context.Canceled) {
return ErrClientDisconnected
}
switch err {
case errInvalidArgument:
apiErr = ErrAdminInvalidArgument
case errNoSuchPolicy:
apiErr = ErrAdminNoSuchPolicy
case errNoSuchUser:
apiErr = ErrAdminNoSuchUser
case errNoSuchServiceAccount:
@@ -2002,8 +2021,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
apiErr = ErrAdminGroupNotEmpty
case errNoSuchJob:
apiErr = ErrAdminNoSuchJob
case errNoSuchPolicy:
apiErr = ErrAdminNoSuchPolicy
case errNoPolicyToAttachOrDetach:
apiErr = ErrAdminPolicyChangeAlreadyApplied
case errSignatureMismatch:
@@ -2022,6 +2039,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
apiErr = ErrAdminInvalidSecretKey
case errInvalidStorageClass:
apiErr = ErrInvalidStorageClass
case errErasureReadQuorum:
apiErr = ErrSlowDown
case errErasureWriteQuorum:
apiErr = ErrSlowDown
// SSE errors
case errInvalidEncryptionParameters:
apiErr = ErrInvalidEncryptionParameters
@@ -2055,7 +2076,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
apiErr = ErrKMSKeyNotFoundException
case errKMSDefaultKeyAlreadyConfigured:
apiErr = ErrKMSDefaultKeyAlreadyConfigured
case context.Canceled, context.DeadlineExceeded:
apiErr = ErrOperationTimedOut
case errDiskNotFound:
@@ -2073,8 +2093,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
}
// Compression errors
switch err {
case errInvalidDecompressedSize:
if err == errInvalidDecompressedSize {
apiErr = ErrInvalidDecompressedSize
}
@@ -2085,7 +2104,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
// etcd specific errors, a key is always a bucket for us return
// ErrNoSuchBucket in such a case.
if err == dns.ErrNoEntriesFound {
if errors.Is(err, dns.ErrNoEntriesFound) {
return ErrNoSuchBucket
}
@@ -2210,6 +2229,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
apiErr = ErrARNNotification
case *event.ErrARNNotFound:
apiErr = ErrARNNotification
case *levent.ErrInvalidARN:
apiErr = ErrLambdaARNInvalid
case *levent.ErrARNNotFound:
apiErr = ErrLambdaARNNotFound
case *event.ErrUnknownRegion:
apiErr = ErrRegionNotification
case *event.ErrInvalidFilterName:
@@ -2273,34 +2296,17 @@ func toAPIError(ctx context.Context, err error) APIError {
}
apiErr := errorCodes.ToAPIErr(toAPIErrorCode(ctx, err))
e, ok := err.(dns.ErrInvalidBucketName)
if ok {
code := toAPIErrorCode(ctx, e)
apiErr = errorCodes.ToAPIErrWithErr(code, e)
}
if apiErr.Code == "NotImplemented" {
switch e := err.(type) {
case NotImplemented:
desc := e.Error()
if desc == "" {
desc = apiErr.Description
}
apiErr = APIError{
Code: apiErr.Code,
Description: desc,
HTTPStatusCode: apiErr.HTTPStatusCode,
}
return apiErr
switch apiErr.Code {
case "NotImplemented":
desc := fmt.Sprintf("%s (%v)", apiErr.Description, err)
apiErr = APIError{
Code: apiErr.Code,
Description: desc,
HTTPStatusCode: apiErr.HTTPStatusCode,
}
}
if apiErr.Code == "XMinioBackendDown" {
case "XMinioBackendDown":
apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err)
return apiErr
}
if apiErr.Code == "InternalError" {
case "InternalError":
// If we see an internal error try to interpret
// any underlying errors if possible depending on
// their internal error types.
@@ -2315,22 +2321,20 @@ func toAPIError(ctx context.Context, err error) APIError {
}
case *xml.SyntaxError:
apiErr = APIError{
Code: "MalformedXML",
Description: fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description,
e.Error()),
Code: "MalformedXML",
Description: fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description, e),
HTTPStatusCode: errorCodes[ErrMalformedXML].HTTPStatusCode,
}
case url.EscapeError:
apiErr = APIError{
Code: "XMinioInvalidObjectName",
Description: fmt.Sprintf("%s (%s)", errorCodes[ErrInvalidObjectName].Description,
e.Error()),
Code: "XMinioInvalidObjectName",
Description: fmt.Sprintf("%s (%s)", errorCodes[ErrInvalidObjectName].Description, e),
HTTPStatusCode: http.StatusBadRequest,
}
case versioning.Error:
apiErr = APIError{
Code: "IllegalVersioningConfigurationException",
Description: fmt.Sprintf("Versioning configuration specified in the request is invalid. (%s)", e.Error()),
Description: fmt.Sprintf("Versioning configuration specified in the request is invalid. (%s)", e),
HTTPStatusCode: http.StatusBadRequest,
}
case lifecycle.Error:
@@ -2396,19 +2400,7 @@ func toAPIError(ctx context.Context, err error) APIError {
// Add more other SDK related errors here if any in future.
default:
//nolint:gocritic
if errors.Is(err, errMalformedEncoding) {
apiErr = APIError{
Code: "BadRequest",
Description: err.Error(),
HTTPStatusCode: http.StatusBadRequest,
}
} else if errors.Is(err, errChunkTooBig) {
apiErr = APIError{
Code: "BadRequest",
Description: err.Error(),
HTTPStatusCode: http.StatusBadRequest,
}
} else if errors.Is(err, strconv.ErrRange) {
if errors.Is(err, errMalformedEncoding) || errors.Is(err, errChunkTooBig) || errors.Is(err, strconv.ErrRange) {
apiErr = APIError{
Code: "BadRequest",
Description: err.Error(),
+16
View File
@@ -80,3 +80,19 @@ func TestAPIErrCode(t *testing.T) {
}
}
}
// Check if an API error is properly defined
func TestAPIErrCodeDefinition(t *testing.T) {
for errAPI := ErrNone + 1; errAPI < apiErrCodeEnd; errAPI++ {
errCode, ok := errorCodes[errAPI]
if !ok {
t.Fatal(errAPI, "error code is not defined in the API error code table")
}
if errCode.Code == "" {
t.Fatal(errAPI, "error code has an empty XML code")
}
if errCode.HTTPStatusCode == 0 {
t.Fatal(errAPI, "error code has a zero HTTP status code")
}
}
}
+9 -2
View File
@@ -285,7 +285,10 @@ func registerAPIRouter(router *mux.Router) {
// GetObjectLegalHold
router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
collectAPIStats("getobjectlegalhold", maxClients(gz(httpTraceAll(api.GetObjectLegalHoldHandler))))).Queries("legal-hold", "")
// GetObject - note gzip compression is *not* added due to Range requests.
// GetObject with lambda ARNs
router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
collectAPIStats("getobject", maxClients(gz(httpTraceHdrs(api.GetObjectLambdaHandler))))).Queries("lambdaArn", "{lambdaArn:.*}")
// GetObject
router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
collectAPIStats("getobject", maxClients(gz(httpTraceHdrs(api.GetObjectHandler)))))
// CopyObject
@@ -516,7 +519,11 @@ func corsHandler(handler http.Handler) http.Handler {
return cors.New(cors.Options{
AllowOriginFunc: func(origin string) bool {
for _, allowedOrigin := range globalAPIConfig.getCorsAllowOrigins() {
allowedOrigins := globalAPIConfig.getCorsAllowOrigins()
if len(allowedOrigins) == 0 {
allowedOrigins = []string{"*"}
}
for _, allowedOrigin := range allowedOrigins {
if wildcard.MatchSimple(allowedOrigin, origin) {
return true
}
+2 -8
View File
@@ -94,14 +94,8 @@ func s3URLEncode(s string) string {
}
// s3EncodeName encodes string in response when encodingType is specified in AWS S3 requests.
func s3EncodeName(name string, encodingType string) (result string) {
// Quick path to exit
if encodingType == "" {
return name
}
encodingType = strings.ToLower(encodingType)
switch encodingType {
case "url":
func s3EncodeName(name, encodingType string) string {
if strings.ToLower(encodingType) == "url" {
return s3URLEncode(name)
}
return name
+219 -218
View File
File diff suppressed because one or more lines are too long
+7 -2
View File
@@ -154,6 +154,10 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
return cred, owner, s3Err
}
logger.GetReqInfo(ctx).Cred = cred
logger.GetReqInfo(ctx).Owner = owner
logger.GetReqInfo(ctx).Region = globalSite.Region
return cred, owner, ErrNone
}
@@ -253,7 +257,7 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
return nil, ErrNoAccessKey
}
if token == "" && cred.IsTemp() {
if token == "" && cred.IsTemp() && !cred.IsServiceAccount() {
// Temporary credentials should always have x-amz-security-token
return nil, ErrInvalidToken
}
@@ -263,7 +267,7 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
return nil, ErrInvalidToken
}
if cred.IsTemp() && subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
if !cred.IsServiceAccount() && cred.IsTemp() && subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
// validate token for temporary credentials only.
return nil, ErrInvalidToken
}
@@ -344,6 +348,7 @@ func authenticateRequest(ctx context.Context, r *http.Request, action policy.Act
logger.GetReqInfo(ctx).Cred = cred
logger.GetReqInfo(ctx).Owner = owner
logger.GetReqInfo(ctx).Region = globalSite.Region
// region is valid only for CreateBucketAction.
var region string
+23 -17
View File
@@ -271,6 +271,9 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
}
func getLocalDisksToHeal() (disksToHeal Endpoints) {
globalLocalDrivesMu.RLock()
globalLocalDrives := globalLocalDrives
globalLocalDrivesMu.RUnlock()
for _, disk := range globalLocalDrives {
_, err := disk.GetDiskID()
if errors.Is(err, errUnformattedDisk) {
@@ -299,7 +302,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
defer disk.Close()
poolIdx := globalEndpoints.GetLocalPoolIdx(disk.Endpoint())
if poolIdx < 0 {
return fmt.Errorf("unexpected pool index (%d) found in %s", poolIdx, disk.Endpoint())
return fmt.Errorf("unexpected pool index (%d) found for %s", poolIdx, disk.Endpoint())
}
// Calculate the set index where the current endpoint belongs
@@ -310,14 +313,15 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
return err
}
if setIdx < 0 {
return fmt.Errorf("unexpected set index (%d) found in %s", setIdx, disk.Endpoint())
return fmt.Errorf("unexpected set index (%d) found for %s", setIdx, disk.Endpoint())
}
// Prevent parallel erasure set healing
locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-drive-healing/%d/%d", poolIdx, setIdx))
lkctx, err := locker.GetLock(ctx, newDiskHealingTimeout)
if err != nil {
return err
return fmt.Errorf("Healing of drive '%v' on %s pool, belonging to %s erasure set already in progress: %w",
disk, humanize.Ordinal(poolIdx+1), humanize.Ordinal(setIdx+1), err)
}
ctx = lkctx.Context()
defer locker.Unlock(lkctx)
@@ -325,19 +329,20 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
// Load healing tracker in this disk
tracker, err := loadHealingTracker(ctx, disk)
if err != nil {
// A healing track can be not found when another disk in the same
// erasure set and same healing-id successfully finished healing.
if err == errFileNotFound {
// A healing tracker may be deleted if another disk in the
// same erasure set with same healing-id successfully finished
// healing.
if errors.Is(err, errFileNotFound) {
return nil
}
logger.LogIf(ctx, fmt.Errorf("Unable to load a healing tracker on '%s': %w", disk, err))
logger.LogIf(ctx, fmt.Errorf("Unable to load healing tracker on '%s': %w, re-initializing..", disk, err))
tracker = newHealingTracker(disk, mustGetUUID())
}
logger.Info(fmt.Sprintf("Proceeding to heal '%s' - 'mc admin heal alias/ --verbose' to check the status.", endpoint))
logger.Info(fmt.Sprintf("Healing drive '%s' - 'mc admin heal alias/ --verbose' to check the current status.", endpoint))
buckets, _ := z.ListBuckets(ctx, BucketOptions{})
// Buckets data are dispersed in multiple zones/sets, make
// Buckets data are dispersed in multiple pools/sets, make
// sure to heal all bucket metadata configuration.
buckets = append(buckets, BucketInfo{
Name: pathJoin(minioMetaBucket, minioConfigPrefix),
@@ -355,7 +360,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
})
if serverDebugLog {
logger.Info("Healing drive '%v' on %s pool", disk, humanize.Ordinal(poolIdx+1))
logger.Info("Healing drive '%v' on %s pool, belonging to %s erasure set", disk, humanize.Ordinal(poolIdx+1), humanize.Ordinal(setIdx+1))
}
// Load bucket totals
@@ -378,9 +383,9 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
}
if tracker.ItemsFailed > 0 {
logger.Info("Healing drive '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
logger.Info("Healing of drive '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
} else {
logger.Info("Healing drive '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
logger.Info("Healing of drive '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
}
if len(tracker.QueuedBuckets) > 0 {
@@ -392,7 +397,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
logger.Info("\n")
}
if tracker.HealID == "" { // HealID is empty only before Feb 2023
if tracker.HealID == "" { // HealID was empty only before Feb 2023
logger.LogIf(ctx, tracker.delete(ctx))
return nil
}
@@ -401,7 +406,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
for _, disk := range z.serverPools[poolIdx].sets[setIdx].getDisks() {
t, err := loadHealingTracker(ctx, disk)
if err != nil {
if err != errFileNotFound {
if !errors.Is(err, errFileNotFound) {
logger.LogIf(ctx, err)
}
continue
@@ -446,10 +451,11 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) {
for _, disk := range healDisks {
go func(disk Endpoint) {
globalBackgroundHealState.setDiskHealingStatus(disk, true)
err := healFreshDisk(ctx, z, disk)
if err != nil {
if err := healFreshDisk(ctx, z, disk); err != nil {
globalBackgroundHealState.setDiskHealingStatus(disk, false)
printEndpointError(disk, err, false)
if !errors.Is(err, context.Canceled) {
printEndpointError(disk, err, false)
}
return
}
// Only upon success pop the healed disk.
+1 -8
View File
@@ -755,19 +755,12 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
}
// Parse incoming location constraint.
location, s3Error := parseLocationConstraint(r)
_, s3Error = parseLocationConstraint(r)
if s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
// Validate if location sent by the client is valid, reject
// requests which do not follow valid region requirements.
if !isValidLocation(location) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRegion), r.URL)
return
}
// check if client is attempting to create more buckets, complain about it.
if currBuckets := globalBucketMetadataSys.Count(); currBuckets+1 > maxBuckets {
logger.LogIf(ctx, fmt.Errorf("An attempt to create %d buckets beyond recommended %d", currBuckets+1, maxBuckets))
+1 -2
View File
@@ -323,8 +323,7 @@ func (b *BucketMetadata) getAllLegacyConfigs(ctx context.Context, objectAPI Obje
configData, err := readConfig(ctx, objectAPI, configFile)
if err != nil {
switch err.(type) {
case ObjectExistsAsDirectory:
if _, ok := err.(ObjectExistsAsDirectory); ok {
// in FS mode it possible that we have actual
// files in this folder with `.minio.sys/buckets/bucket/configFile`
continue
+6 -10
View File
@@ -90,8 +90,8 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
opts.VersionID = object.VersionID
if gerr != nil { // error from GetObjectInfo
switch gerr.(type) {
case MethodNotAllowed: // This happens usually for a delete marker
if _, ok := gerr.(MethodNotAllowed); ok {
// This happens usually for a delete marker
if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
// Delete marker should be present and valid.
return ErrNone
@@ -192,8 +192,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
days, objRetention.RetainUntilDate.Time,
objRetention.Mode, byPassSet, r, cred,
owner)
switch apiErr {
case ErrAccessDenied:
if apiErr == ErrAccessDenied {
return errAuthentication
}
return nil
@@ -210,8 +209,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
}
}
switch govPerm {
case ErrAccessDenied:
if govPerm == ErrAccessDenied {
return errAuthentication
}
return nil
@@ -224,8 +222,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
days, objRetention.RetainUntilDate.Time, objRetention.Mode,
false, r, cred, owner)
switch apiErr {
case ErrAccessDenied:
if apiErr == ErrAccessDenied {
return errAuthentication
}
return nil
@@ -236,8 +233,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
days, objRetention.RetainUntilDate.Time,
objRetention.Mode, byPassSet, r, cred, owner)
switch apiErr {
case ErrAccessDenied:
if apiErr == ErrAccessDenied {
return errAuthentication
}
return nil
+1 -2
View File
@@ -496,8 +496,7 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
oi.VersionPurgeStatusInternal = fmt.Sprintf("%s=%s;", rcfg.Config.RoleArn, oi.VersionPurgeStatus)
}
for k, v := range oi.UserDefined {
switch {
case strings.EqualFold(k, ReservedMetadataPrefixLower+ReplicationReset):
if strings.EqualFold(k, ReservedMetadataPrefixLower+ReplicationReset) {
delete(oi.UserDefined, k)
oi.UserDefined[targetResetHeader(rcfg.Config.RoleArn)] = v
}
+22 -17
View File
@@ -586,11 +586,13 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
return
}
}
// mark delete marker replication as failed if target cluster not ready to receive
// this request yet (object version not replicated yet)
if err != nil && !toi.ReplicationReady {
rinfo.ReplicationStatus = replication.Failed
return
if !isErrObjectNotFound(ErrorRespToObjectError(err, dobj.Bucket, dobj.ObjectName)) {
// mark delete marker replication as failed if target cluster not ready to receive
// this request yet (object version not replicated yet)
if err != nil && !toi.ReplicationReady {
rinfo.ReplicationStatus = replication.Failed
return
}
}
}
@@ -802,10 +804,12 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
oi1.ModTime.Unix() > oi2.LastModified.Unix() && oi1.VersionID == nullVersionID {
return replicateNone
}
sz, _ := oi1.GetActualSize()
// needs full replication
if oi1.ETag != oi2.ETag ||
oi1.VersionID != oi2.VersionID ||
oi1.Size != oi2.Size ||
sz != oi2.Size ||
oi1.DeleteMarker != oi2.IsDeleteMarker ||
oi1.ModTime.Unix() != oi2.LastModified.Unix() {
return replicateAll
@@ -1202,12 +1206,6 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
ReplicationAction: rAction,
}
if ri.ObjectInfo.TargetReplicationStatus(tgt.ARN) == replication.Completed && !ri.ExistingObjResync.Empty() && !ri.ExistingObjResync.mustResyncTarget(tgt.ARN) {
rinfo.ReplicationStatus = replication.Completed
rinfo.ReplicationResynced = true
return
}
if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
logger.LogIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN))
sendEvent(eventArgs{
@@ -1243,6 +1241,13 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
objInfo = gr.ObjInfo
// use latest ObjectInfo to check if previous replication attempt succeeded
if objInfo.TargetReplicationStatus(tgt.ARN) == replication.Completed && !ri.ExistingObjResync.Empty() && !ri.ExistingObjResync.mustResyncTarget(tgt.ARN) {
rinfo.ReplicationStatus = replication.Completed
rinfo.ReplicationResynced = true
return
}
size, err := objInfo.GetActualSize()
if err != nil {
logger.LogIf(ctx, err)
@@ -1256,7 +1261,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
}
if tgt.Bucket == "" {
logger.LogIf(ctx, fmt.Errorf("Unable to replicate object %s(%s), bucket is empty", objInfo.Name, objInfo.VersionID))
logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s), bucket is empty", objInfo.Name, objInfo.VersionID))
sendEvent(eventArgs{
EventName: event.ObjectReplicationNotTracked,
BucketName: bucket,
@@ -1286,7 +1291,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
if rAction == replicateNone {
if ri.OpType == replication.ExistingObjectReplicationType &&
objInfo.ModTime.Unix() > oi.LastModified.Unix() && objInfo.VersionID == nullVersionID {
logger.LogIf(ctx, fmt.Errorf("Unable to replicate %s/%s (null). Newer version exists on target", bucket, object))
logger.LogIf(ctx, fmt.Errorf("unable to replicate %s/%s (null). Newer version exists on target", bucket, object))
sendEvent(eventArgs{
EventName: event.ObjectReplicationNotTracked,
BucketName: bucket,
@@ -1328,7 +1333,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
}
if _, err = c.CopyObject(ctx, tgt.Bucket, object, tgt.Bucket, object, getCopyObjMetadata(objInfo, tgt.StorageClass), srcOpts, dstOpts); err != nil {
rinfo.ReplicationStatus = replication.Failed
logger.LogIf(ctx, fmt.Errorf("Unable to replicate metadata for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
logger.LogIf(ctx, fmt.Errorf("unable to replicate metadata for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
}
} else {
var putOpts minio.PutObjectOptions
@@ -1365,7 +1370,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
r, objInfo, putOpts); err != nil {
if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
rinfo.ReplicationStatus = replication.Failed
logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
} else {
rinfo.ReplicationStatus = replication.Completed
}
@@ -1374,7 +1379,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
if _, err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts); err != nil {
if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
rinfo.ReplicationStatus = replication.Failed
logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
} else {
rinfo.ReplicationStatus = replication.Completed
}
+6 -5
View File
@@ -205,7 +205,7 @@ func minioConfigToConsoleFeatures() {
}
}
// Enable if LDAP is enabled.
if globalLDAPConfig.Enabled() {
if globalIAMSys.LDAPConfig.Enabled() {
os.Setenv("CONSOLE_LDAP_ENABLED", config.EnableOn)
}
os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
@@ -215,8 +215,9 @@ func minioConfigToConsoleFeatures() {
}
func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
m := make(map[string]consoleoauth2.ProviderConfig, len(globalOpenIDConfig.ProviderCfgs))
for name, cfg := range globalOpenIDConfig.ProviderCfgs {
pcfgs := globalIAMSys.OpenIDConfig.ProviderCfgs
m := make(map[string]consoleoauth2.ProviderConfig, len(pcfgs))
for name, cfg := range pcfgs {
callback := getConsoleEndpoints()[0] + "/oauth_callback"
if cfg.RedirectURI != "" {
callback = cfg.RedirectURI
@@ -660,6 +661,7 @@ func handleCommonEnvVars() {
}
u.Path = "" // remove any path component such as `/`
globalMinioEndpoint = u.String()
globalMinioEndpointURL = u
}
globalFSOSync, err = config.ParseBool(env.Get(config.EnvFSOSync, config.EnableOff))
@@ -782,8 +784,7 @@ func handleCommonEnvVars() {
// Initialize KMS global variable after valiadating and loading the configuration.
// It depends on KMS env variables and global cli flags.
func handleKMSConfig() {
switch {
case env.IsSet(kms.EnvKMSSecretKey) && env.IsSet(kms.EnvKESEndpoint):
if env.IsSet(kms.EnvKMSSecretKey) && env.IsSet(kms.EnvKESEndpoint) {
logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", kms.EnvKMSSecretKey, kms.EnvKESEndpoint))
}
+27 -11
View File
@@ -37,6 +37,7 @@ import (
"github.com/minio/minio/internal/config/identity/openid"
idplugin "github.com/minio/minio/internal/config/identity/plugin"
xtls "github.com/minio/minio/internal/config/identity/tls"
"github.com/minio/minio/internal/config/lambda"
"github.com/minio/minio/internal/config/notify"
"github.com/minio/minio/internal/config/policy/opa"
polplugin "github.com/minio/minio/internal/config/policy/plugin"
@@ -74,6 +75,9 @@ func initHelp() {
for k, v := range notify.DefaultNotificationKVS {
kvs[k] = v
}
for k, v := range lambda.DefaultLambdaKVS {
kvs[k] = v
}
if globalIsErasure {
kvs[config.StorageClassSubSys] = storageclass.DefaultKVS
kvs[config.HealSubSys] = heal.DefaultKVS
@@ -196,6 +200,11 @@ func initHelp() {
Description: "publish bucket notifications to Redis datastores",
MultipleTargets: true,
},
config.HelpKV{
Key: config.LambdaWebhookSubSys,
Description: "manage remote lambda functions",
MultipleTargets: true,
},
config.HelpKV{
Key: config.EtcdSubSys,
Description: "persist IAM assets externally to etcd",
@@ -246,6 +255,7 @@ func initHelp() {
config.NotifyRedisSubSys: notify.HelpRedis,
config.NotifyWebhookSubSys: notify.HelpWebhook,
config.NotifyESSubSys: notify.HelpES,
config.LambdaWebhookSubSys: lambda.HelpWebhook,
config.SubnetSubSys: subnet.HelpSubnet,
config.CallhomeSubSys: callhome.HelpCallhome,
}
@@ -387,6 +397,13 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
return err
}
}
if config.LambdaSubSystems.Contains(subSys) {
if err := lambda.TestSubSysLambdaTargets(GlobalContext, s, subSys, NewHTTPTransport()); err != nil {
return err
}
}
return nil
}
@@ -423,6 +440,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
}
if err == nil && dnsURL != "" {
bootstrapTrace("initialize remote bucket DNS store")
globalDNSConfig, err = dns.NewOperatorDNS(dnsURL,
dns.Authentication(dnsUser, dnsPass),
dns.RootCAs(globalRootCAs))
@@ -437,6 +455,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
}
if etcdCfg.Enabled {
bootstrapTrace("initialize etcd store")
globalEtcdClient, err = etcd.New(etcdCfg)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
@@ -493,23 +512,20 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
logger.Fatal(errors.New("no KMS configured"), "MINIO_KMS_AUTO_ENCRYPTION requires a valid KMS configuration")
}
globalSTSTLSConfig, err = xtls.Lookup(s[config.IdentityTLSSubSys][config.Default])
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize X.509/TLS STS API: %w", err))
}
if globalSTSTLSConfig.InsecureSkipVerify {
logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
}
transport := NewHTTPTransport()
bootstrapTrace("lookup the event notification targets")
globalConfigTargetList, err = notify.FetchEnabledTargets(GlobalContext, s, transport)
bootstrapTrace("initialize the event notification targets")
globalNotifyTargetList, err = notify.FetchEnabledTargets(GlobalContext, s, transport)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
}
bootstrapTrace("initialize the lambda targets")
globalLambdaTargetList, err = lambda.FetchEnabledTargets(GlobalContext, s, transport)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize lambda target(s): %w", err))
}
bootstrapTrace("applying the dynamic configuration")
// Apply dynamic config values
if err := applyDynamicConfig(ctx, objAPI, s); err != nil {
+1 -2
View File
@@ -33,8 +33,7 @@ const crossDomainXMLEntity = "/crossdomain.xml"
func setCrossDomainPolicy(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Look for 'crossdomain.xml' in the incoming request.
switch r.URL.Path {
case crossDomainXMLEntity:
if r.URL.Path == crossDomainXMLEntity {
// Write the standard cross domain policy xml.
w.Write([]byte(crossDomainXML))
// Request completed, no need to serve to other handlers.
+7 -53
View File
@@ -18,7 +18,6 @@
package cmd
import (
"bytes"
"context"
"encoding/binary"
"encoding/json"
@@ -34,7 +33,6 @@ import (
"sync"
"time"
"github.com/bits-and-blooms/bloom/v3"
"github.com/dustin/go-humanize"
"github.com/minio/madmin-go/v2"
"github.com/minio/minio/internal/bucket/lifecycle"
@@ -162,7 +160,6 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
// Load current bloom cycle
var cycleInfo currentScannerCycle
cycleInfo.next = intDataUpdateTracker.current() + 1
buf, _ := readConfig(ctx, objAPI, dataUsageBloomNamePath)
if len(buf) == 8 {
@@ -207,9 +204,7 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
// Wait before starting next cycle and wait on startup.
results := make(chan DataUsageInfo, 1)
go storeDataUsageInBackend(ctx, objAPI, results)
bf, err := globalNotificationSys.updateBloomFilter(ctx, cycleInfo.current)
logger.LogIf(ctx, err)
err = objAPI.NSScanner(ctx, bf, results, uint32(cycleInfo.current), scanMode)
err := objAPI.NSScanner(ctx, results, uint32(cycleInfo.current), scanMode)
logger.LogIf(ctx, err)
res := map[string]string{"cycle": fmt.Sprint(cycleInfo.current)}
if err != nil {
@@ -248,10 +243,8 @@ type folderScanner struct {
oldCache dataUsageCache
newCache dataUsageCache
updateCache dataUsageCache
withFilter *bloomFilter
dataUsageScannerDebug bool
healFolderInclude uint32 // Include a clean folder one in n cycles.
healObjectSelect uint32 // Do a heal check on an object once every n cycles. Must divide into healFolderInclude
scanMode madmin.HealScanMode
@@ -310,8 +303,6 @@ type folderScanner struct {
// Before each operation sleepDuration is called which can be used to temporarily halt the scanner.
// If the supplied context is canceled the function will return at the first chance.
func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, cache dataUsageCache, getSize getSizeFn, scanMode madmin.HealScanMode) (dataUsageCache, error) {
logPrefix := color.Green("data-usage: ")
switch cache.Info.Name {
case "", dataUsageRoot:
return cache, errors.New("internal error: root scan attempted")
@@ -325,8 +316,7 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
oldCache: cache,
newCache: dataUsageCache{Info: cache.Info},
updateCache: dataUsageCache{Info: cache.Info},
dataUsageScannerDebug: intDataUpdateTracker.debug,
healFolderInclude: 0,
dataUsageScannerDebug: false,
healObjectSelect: 0,
scanMode: scanMode,
updates: cache.Info.updates,
@@ -349,19 +339,9 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
// Enable healing in XL mode.
if globalIsErasure && !cache.Info.SkipHealing {
// Include a clean folder one in n cycles.
s.healFolderInclude = healFolderIncludeProb
// Do a heal check on an object once every n cycles. Must divide into healFolderInclude
s.healObjectSelect = healObjectSelectProb
}
if len(cache.Info.BloomFilter) > 0 {
s.withFilter = &bloomFilter{BloomFilter: &bloom.BloomFilter{}}
_, err := s.withFilter.ReadFrom(bytes.NewReader(cache.Info.BloomFilter))
if err != nil {
logger.LogIf(ctx, err, logPrefix+"Error reading bloom filter")
s.withFilter = nil
}
}
done := ctx.Done()
@@ -418,14 +398,12 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
return ctx.Err()
default:
}
existing, ok := f.oldCache.Cache[thisHash.Key()]
var abandonedChildren dataUsageHashMap
if !into.Compacted {
abandonedChildren = f.oldCache.findChildrenCopy(thisHash)
}
// If there are lifecycle rules for the prefix, remove the filter.
filter := f.withFilter
_, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
var activeLifeCycle *lifecycle.Lifecycle
if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix) {
@@ -433,33 +411,13 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
console.Debugf(scannerLogPrefix+" Prefix %q has active rules\n", prefix)
}
activeLifeCycle = f.oldCache.Info.lifeCycle
filter = nil
}
// If there are replication rules for the prefix, remove the filter.
var replicationCfg replicationConfig
if !f.oldCache.Info.replication.Empty() && f.oldCache.Info.replication.Config.HasActiveRules(prefix, true) {
replicationCfg = f.oldCache.Info.replication
filter = nil
}
// Check if we can skip it due to bloom filter...
if filter != nil && ok && existing.Compacted {
// If folder isn't in filter and we have data, skip it completely.
if folder.name != dataUsageRoot && !filter.containsDir(folder.name) {
if f.healObjectSelect == 0 || !thisHash.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healFolderInclude/folder.objectHealProbDiv) {
f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
f.updateCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
if f.dataUsageScannerDebug {
console.Debugf(scannerLogPrefix+" Skipping non-updated folder: %v\n", folder.name)
}
return nil
}
if f.dataUsageScannerDebug {
console.Debugf(scannerLogPrefix+" Adding non-updated folder to heal check: %v\n", folder.name)
}
// If probability was already scannerHealFolderInclude, keep it.
folder.objectHealProbDiv = f.healFolderInclude
}
}
scannerSleeper.Sleep(ctx, dataScannerSleepPerFolder)
var existingFolders, newFolders []cachedFolder
@@ -673,13 +631,10 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
// and the entry itself is compacted.
if !into.Compacted && f.oldCache.isCompacted(h) {
if !h.mod(f.oldCache.Info.NextCycle, dataUsageUpdateDirCycles) {
if f.healObjectSelect == 0 || !h.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healFolderInclude/folder.objectHealProbDiv) {
// Transfer and add as child...
f.newCache.copyWithChildren(&f.oldCache, h, folder.parent)
into.addChild(h)
continue
}
folder.objectHealProbDiv = f.healFolderInclude
// Transfer and add as child...
f.newCache.copyWithChildren(&f.oldCache, h, folder.parent)
into.addChild(h)
continue
}
}
f.updateCurrentPath(folder.name)
@@ -1318,8 +1273,7 @@ func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi Obj
sizeS.replTargetStats[arn] = tgtSizeS
}
switch oi.ReplicationStatus {
case replication.Replica:
if oi.ReplicationStatus == replication.Replica {
sizeS.replicaSize += oi.Size
}
}
-682
View File
@@ -1,682 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bufio"
"bytes"
"context"
"encoding/binary"
"errors"
"io"
"os"
"path"
"sort"
"strings"
"sync"
"time"
"github.com/bits-and-blooms/bloom/v3"
"github.com/minio/minio/internal/color"
"github.com/minio/minio/internal/logger"
"github.com/minio/pkg/console"
)
const (
// Estimate bloom filter size. With this many items
dataUpdateTrackerEstItems = 200000
// ... we want this false positive rate:
dataUpdateTrackerFP = 0.1
dataUpdateTrackerQueueSize = 0
dataUpdateTrackerFilename = dataUsageBucket + SlashSeparator + ".tracker.bin"
dataUpdateTrackerVersion = 7
dataUpdateTrackerSaveInterval = 5 * time.Minute
)
var intDataUpdateTracker *dataUpdateTracker
func init() {
intDataUpdateTracker = newDataUpdateTracker()
}
type dataUpdateTracker struct {
mu sync.Mutex
input chan string
save chan struct{}
debug bool
saveExited chan struct{}
dirty bool
Current dataUpdateFilter
History dataUpdateTrackerHistory
Saved time.Time
}
// newDataUpdateTracker returns a dataUpdateTracker with default settings.
func newDataUpdateTracker() *dataUpdateTracker {
d := &dataUpdateTracker{
Current: dataUpdateFilter{
idx: 1,
},
debug: serverDebugLog,
input: make(chan string, dataUpdateTrackerQueueSize),
save: make(chan struct{}, 1),
saveExited: make(chan struct{}),
}
d.Current.bf = d.newBloomFilter()
d.dirty = true
return d
}
type dataUpdateTrackerHistory []dataUpdateFilter
type dataUpdateFilter struct {
idx uint64
bf bloomFilter
}
type bloomFilter struct {
*bloom.BloomFilter
}
// emptyBloomFilter returns an empty bloom filter.
func emptyBloomFilter() bloomFilter {
return bloomFilter{BloomFilter: &bloom.BloomFilter{}}
}
// containsDir returns whether the bloom filter contains a directory.
// Note that objects in XL mode are also considered directories.
func (b bloomFilter) containsDir(in string) bool {
split := splitPathDeterministic(path.Clean(in))
if len(split) == 0 {
return false
}
return b.TestString(hashPath(path.Join(split...)).String())
}
// bytes returns the bloom filter serialized as a byte slice.
func (b *bloomFilter) bytes() []byte {
if b == nil || b.BloomFilter == nil {
return nil
}
var buf bytes.Buffer
_, err := b.WriteTo(&buf)
if err != nil {
logger.LogIf(GlobalContext, err)
return nil
}
return buf.Bytes()
}
// sort the dataUpdateTrackerHistory, newest first.
// Returns whether the history is complete.
func (d dataUpdateTrackerHistory) sort() bool {
if len(d) == 0 {
return true
}
sort.Slice(d, func(i, j int) bool {
return d[i].idx > d[j].idx
})
return d[0].idx-d[len(d)-1].idx == uint64(len(d))
}
// removeOlderThan will remove entries older than index 'n'.
func (d *dataUpdateTrackerHistory) removeOlderThan(n uint64) {
d.sort()
dd := *d
end := len(dd)
for i := end - 1; i >= 0; i-- {
if dd[i].idx < n {
end = i
}
}
dd = dd[:end]
*d = dd
}
// newBloomFilter returns a new bloom filter with default settings.
func (d *dataUpdateTracker) newBloomFilter() bloomFilter {
return bloomFilter{bloom.NewWithEstimates(dataUpdateTrackerEstItems, dataUpdateTrackerFP)}
}
// current returns the current index.
func (d *dataUpdateTracker) current() uint64 {
d.mu.Lock()
defer d.mu.Unlock()
return d.Current.idx
}
// latestWithDir returns the highest index that contains the directory.
// This means that any cycle higher than this does NOT contain the entry.
func (d *dataUpdateTracker) latestWithDir(dir string) uint64 {
dateUpdateTrackerLogPrefix := color.Green("dataUpdateTracker:")
bucket, _ := path2BucketObjectWithBasePath("", dir)
if bucket == "" {
if d.debug && len(dir) > 0 {
console.Debugf(dateUpdateTrackerLogPrefix+" no bucket (%s)\n", dir)
}
return d.current()
}
if isReservedOrInvalidBucket(bucket, false) {
return d.current()
}
d.mu.Lock()
defer d.mu.Unlock()
if d.Current.bf.containsDir(dir) || d.Current.idx == 0 {
return d.Current.idx
}
if d.debug {
console.Debugf(dateUpdateTrackerLogPrefix+" current bloom does NOT contains dir %s\n", dir)
}
idx := d.Current.idx - 1
for {
f := d.History.find(idx)
if f == nil || f.bf.containsDir(dir) || idx == 0 {
break
}
idx--
}
return idx
}
// start will load the current data from the drives start collecting information and
// start a saver goroutine.
// All of these will exit when the context is canceled.
func (d *dataUpdateTracker) start(ctx context.Context, drives ...string) {
if len(drives) == 0 {
logger.LogIf(ctx, errors.New("dataUpdateTracker.start: No local drives specified"))
return
}
d.load(ctx, drives...)
go d.startCollector(ctx)
// startSaver will unlock.
d.mu.Lock()
go d.startSaver(ctx, dataUpdateTrackerSaveInterval, drives...)
}
// load will attempt to load data tracking information from the supplied drives.
// The data will only be loaded if d.Saved is older than the one found on disk.
// The newest working cache will be kept in d.
// If no valid data usage tracker can be found d will remain unchanged.
// If object is shared the caller should lock it.
func (d *dataUpdateTracker) load(ctx context.Context, drives ...string) {
if len(drives) == 0 {
logger.LogIf(ctx, errors.New("dataUpdateTracker.load: No local drives specified"))
return
}
for _, drive := range drives {
cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
f, err := OpenFile(cacheFormatPath, readMode, 0o666)
if err != nil {
if osIsNotExist(err) {
continue
}
logger.LogIf(ctx, err)
continue
}
err = d.deserialize(f, d.Saved)
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
logger.LogIf(ctx, err)
}
f.Close()
}
}
// startSaver will start a saver that will write d to all supplied drives at specific intervals.
// 'd' must be write locked when started and will be unlocked.
// The saver will save and exit when supplied context is closed.
func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Duration, drives ...string) {
if len(drives) == 0 {
return
}
saveNow := d.save
exited := make(chan struct{})
d.saveExited = exited
d.mu.Unlock()
t := time.NewTicker(interval)
defer t.Stop()
defer close(exited)
var buf bytes.Buffer
for {
var exit bool
select {
case <-ctx.Done():
exit = true
case <-t.C:
case <-saveNow:
}
buf.Reset()
d.mu.Lock()
if !d.dirty {
d.mu.Unlock()
if exit {
return
}
continue
}
d.Saved = UTCNow()
err := d.serialize(&buf)
if d.debug {
console.Debugf(color.Green("dataUpdateTracker:")+" Saving: %v bytes, Current idx: %v\n", buf.Len(), d.Current.idx)
}
d.dirty = false
d.mu.Unlock()
if err != nil {
logger.LogIf(ctx, err, "Error serializing usage tracker data")
if exit {
return
}
continue
}
if buf.Len() == 0 {
logger.LogIf(ctx, errors.New("zero sized output, skipping save"))
continue
}
for _, drive := range drives {
cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
err := os.WriteFile(cacheFormatPath, buf.Bytes(), os.ModePerm)
if err != nil {
if osIsNotExist(err) {
continue
}
logger.LogIf(ctx, err)
continue
}
}
if exit {
return
}
}
}
// serialize all data in d to dst.
// Caller should hold lock if d is expected to be shared.
// If an error is returned, there will likely be partial data written to dst.
func (d *dataUpdateTracker) serialize(dst io.Writer) (err error) {
ctx := GlobalContext
var tmp [8]byte
o := bufio.NewWriter(dst)
defer func() {
if err == nil {
err = o.Flush()
}
}()
// Version
if err := o.WriteByte(dataUpdateTrackerVersion); err != nil {
if d.debug {
logger.LogIf(ctx, err)
}
return err
}
// Timestamp.
binary.LittleEndian.PutUint64(tmp[:], uint64(d.Saved.Unix()))
if _, err := o.Write(tmp[:]); err != nil {
if d.debug {
logger.LogIf(ctx, err)
}
return err
}
// Current
binary.LittleEndian.PutUint64(tmp[:], d.Current.idx)
if _, err := o.Write(tmp[:]); err != nil {
if d.debug {
logger.LogIf(ctx, err)
}
return err
}
if _, err := d.Current.bf.WriteTo(o); err != nil {
if d.debug {
logger.LogIf(ctx, err)
}
return err
}
// History
binary.LittleEndian.PutUint64(tmp[:], uint64(len(d.History)))
if _, err := o.Write(tmp[:]); err != nil {
if d.debug {
logger.LogIf(ctx, err)
}
return err
}
for _, bf := range d.History {
// Current
binary.LittleEndian.PutUint64(tmp[:], bf.idx)
if _, err := o.Write(tmp[:]); err != nil {
if d.debug {
logger.LogIf(ctx, err)
}
return err
}
if _, err := bf.bf.WriteTo(o); err != nil {
if d.debug {
logger.LogIf(ctx, err)
}
return err
}
}
return nil
}
// deserialize will deserialize the supplied input if the input is newer than the supplied time.
func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) error {
ctx := GlobalContext
var dst dataUpdateTracker
var tmp [8]byte
// Version
if _, err := io.ReadFull(src, tmp[:1]); err != nil {
if d.debug {
if err != io.EOF {
logger.LogIf(ctx, err)
}
}
return err
}
switch tmp[0] {
case 1, 2, 3, 4, 5, 6:
if intDataUpdateTracker.debug {
console.Debugln(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
}
return nil
case dataUpdateTrackerVersion:
default:
return errors.New("dataUpdateTracker: Unknown data version")
}
// Timestamp.
if _, err := io.ReadFull(src, tmp[:8]); err != nil {
if d.debug {
logger.LogIf(ctx, err)
}
return err
}
t := time.Unix(int64(binary.LittleEndian.Uint64(tmp[:])), 0)
if !t.After(newerThan) {
return nil
}
// Current
if _, err := io.ReadFull(src, tmp[:8]); err != nil {
if d.debug {
logger.LogIf(ctx, err)
}
return err
}
dst.Current.idx = binary.LittleEndian.Uint64(tmp[:])
dst.Current.bf = emptyBloomFilter()
if _, err := dst.Current.bf.ReadFrom(src); err != nil {
if d.debug {
logger.LogIf(ctx, err)
}
return err
}
// History
if _, err := io.ReadFull(src, tmp[:8]); err != nil {
if d.debug {
logger.LogIf(ctx, err)
}
return err
}
n := binary.LittleEndian.Uint64(tmp[:])
dst.History = make(dataUpdateTrackerHistory, int(n))
for i, e := range dst.History {
if _, err := io.ReadFull(src, tmp[:8]); err != nil {
if d.debug {
logger.LogIf(ctx, err)
}
return err
}
e.idx = binary.LittleEndian.Uint64(tmp[:])
e.bf = emptyBloomFilter()
if _, err := e.bf.ReadFrom(src); err != nil {
if d.debug {
logger.LogIf(ctx, err)
}
return err
}
dst.History[i] = e
}
// Ignore what remains on the stream.
// Update d:
d.mu.Lock()
defer d.mu.Unlock()
d.Current = dst.Current
d.History = dst.History
d.Saved = dst.Saved
return nil
}
// start a collector that picks up entries from objectUpdatedCh
// and adds them to the current bloom filter.
func (d *dataUpdateTracker) startCollector(ctx context.Context) {
for {
select {
case <-ctx.Done():
return
case in := <-d.input:
bucket, _ := path2BucketObjectWithBasePath("", in)
if bucket == "" {
if d.debug && len(in) > 0 {
console.Debugf(color.Green("dataUpdateTracker:")+" no bucket (%s)\n", in)
}
continue
}
if isReservedOrInvalidBucket(bucket, false) {
continue
}
split := splitPathDeterministic(in)
// Add all paths until done.
d.mu.Lock()
for i := range split {
d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
}
d.dirty = d.dirty || len(split) > 0
d.mu.Unlock()
}
}
}
// markDirty adds the supplied path to the current bloom filter.
func (d *dataUpdateTracker) markDirty(bucket, prefix string) {
dateUpdateTrackerLogPrefix := color.Green("dataUpdateTracker:")
if bucket == "" && d.debug {
console.Debugf(dateUpdateTrackerLogPrefix + " no bucket specified\n")
return
}
if isReservedOrInvalidBucket(bucket, false) && d.debug {
return
}
split := splitPathDeterministic(pathJoin(bucket, prefix))
// Add all paths until done.
d.mu.Lock()
for i := range split {
d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
}
d.dirty = d.dirty || len(split) > 0
d.mu.Unlock()
}
// find entry with specified index.
// Returns nil if not found.
func (d dataUpdateTrackerHistory) find(idx uint64) *dataUpdateFilter {
for _, f := range d {
if f.idx == idx {
return &f
}
}
return nil
}
// filterFrom will return a combined bloom filter.
func (d *dataUpdateTracker) filterFrom(ctx context.Context, oldest, newest uint64) *bloomFilterResponse {
bf := d.newBloomFilter()
bfr := bloomFilterResponse{
OldestIdx: oldest,
CurrentIdx: d.Current.idx,
Complete: true,
}
// Loop through each index requested.
for idx := oldest; idx <= newest; idx++ {
v := d.History.find(idx)
if v == nil {
if d.Current.idx == idx {
// Merge current.
err := bf.Merge(d.Current.bf.BloomFilter)
logger.LogIf(ctx, err)
if err != nil {
bfr.Complete = false
}
continue
}
bfr.Complete = false
bfr.OldestIdx = idx + 1
continue
}
err := bf.Merge(v.bf.BloomFilter)
if err != nil {
bfr.Complete = false
logger.LogIf(ctx, err)
continue
}
bfr.NewestIdx = idx
}
var dst bytes.Buffer
_, err := bf.WriteTo(&dst)
if err != nil {
logger.LogIf(ctx, err)
return nil
}
bfr.Filter = dst.Bytes()
return &bfr
}
// cycleFilter will cycle the bloom filter to start recording to index y if not already.
// The response will contain a bloom filter starting at index x up to, but not including index y.
// If y is 0, the response will not update y, but return the currently recorded information
// from the oldest (unless 0, then it will be all) until and including current y.
func (d *dataUpdateTracker) cycleFilter(ctx context.Context, req bloomFilterRequest) (*bloomFilterResponse, error) {
if req.OldestClean != "" {
return &bloomFilterResponse{OldestIdx: d.latestWithDir(req.OldestClean)}, nil
}
current := req.Current
oldest := req.Oldest
d.mu.Lock()
defer d.mu.Unlock()
if current == 0 {
if len(d.History) == 0 {
return d.filterFrom(ctx, d.Current.idx, d.Current.idx), nil
}
d.History.sort()
if oldest == 0 {
oldest = d.History[len(d.History)-1].idx
}
return d.filterFrom(ctx, oldest, d.Current.idx), nil
}
// Move current to history if new one requested
if d.Current.idx != current {
d.dirty = true
if d.debug {
console.Debugf(color.Green("dataUpdateTracker:")+" cycle bloom filter: %v -> %v\n", d.Current.idx, current)
}
d.History = append(d.History, d.Current)
d.Current.idx = current
d.Current.bf = d.newBloomFilter()
select {
case d.save <- struct{}{}:
default:
}
}
d.History.removeOlderThan(oldest)
return d.filterFrom(ctx, oldest, current), nil
}
// splitPathDeterministic will split the provided relative path
// deterministically and return up to the first 3 elements of the path.
// slash and dot prefixes are removed.
// Trailing slashes are removed.
// Returns 0 length if no parts are found after trimming.
func splitPathDeterministic(in string) []string {
split := strings.Split(decodeDirObject(in), SlashSeparator)
// Trim empty start/end
for len(split) > 0 {
if len(split[0]) > 0 && split[0] != "." {
break
}
split = split[1:]
}
for len(split) > 0 {
if len(split[len(split)-1]) > 0 {
break
}
split = split[:len(split)-1]
}
return split
}
// bloomFilterRequest request bloom filters.
// Current index will be updated to current and entries back to Oldest is returned.
type bloomFilterRequest struct {
Oldest uint64
Current uint64
// If set the oldest clean version will be returned in OldestIdx
// and the rest of the request will be ignored.
OldestClean string
}
type bloomFilterResponse struct {
// Current index being written to.
CurrentIdx uint64
// Oldest index in the returned bloom filter.
OldestIdx uint64
// Newest Index in the returned bloom filter.
NewestIdx uint64
// Are all indexes between oldest and newest filled?
Complete bool
// Binary data of the bloom filter.
Filter []byte
}
// NSUpdated indicates namespace has been updated.
// The function will block until the entry has been picked up.
func NSUpdated(bucket, prefix string) {
if intDataUpdateTracker != nil {
intDataUpdateTracker.markDirty(bucket, prefix)
}
}
-299
View File
@@ -1,299 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"context"
"fmt"
"math/rand"
"os"
"path"
"path/filepath"
"sync"
"testing"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/logger/target/types"
"github.com/minio/pkg/logger/message/log"
)
type testLoggerI interface {
Helper()
Log(args ...interface{})
}
type testingLogger struct {
mu sync.Mutex
t testLoggerI
}
func (t *testingLogger) Endpoint() string {
return ""
}
func (t *testingLogger) String() string {
return ""
}
func (t *testingLogger) Init() error {
return nil
}
func (t *testingLogger) Cancel() {
}
func (t *testingLogger) Type() types.TargetType {
return types.TargetHTTP
}
func (t *testingLogger) IsOnline() bool {
return true
}
// Stats returns the target statistics.
func (t *testingLogger) Stats() types.TargetStats {
return types.TargetStats{}
}
func (t *testingLogger) Send(entry interface{}) error {
t.mu.Lock()
defer t.mu.Unlock()
if t.t == nil {
return nil
}
e, ok := entry.(log.Entry)
if !ok {
return fmt.Errorf("unexpected log entry structure %#v", entry)
}
t.t.Helper()
t.t.Log(e.Level, ":", e.Message)
return nil
}
func addTestingLogging(t testLoggerI) func() {
tl := &testingLogger{t: t}
logger.AddSystemTarget(tl)
return func() {
tl.mu.Lock()
defer tl.mu.Unlock()
tl.t = nil
}
}
func TestDataUpdateTracker(t *testing.T) {
dut := newDataUpdateTracker()
// Change some defaults.
dut.debug = testing.Verbose()
dut.input = make(chan string)
dut.save = make(chan struct{})
defer addTestingLogging(t)()
dut.Current.bf = dut.newBloomFilter()
tmpDir := t.TempDir()
err := os.MkdirAll(filepath.Dir(filepath.Join(tmpDir, dataUpdateTrackerFilename)), os.ModePerm)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
dut.start(ctx, tmpDir)
tests := []struct {
in string
check []string // if not empty, check against these instead.
exist bool
}{
{
in: "bucket/directory/file.txt",
check: []string{"bucket", "bucket/", "/bucket", "bucket/directory", "bucket/directory/", "bucket/directory/file.txt", "/bucket/directory/file.txt"},
exist: true,
},
{
// System bucket
in: ".minio.sys/ignoreme/pls",
exist: false,
},
{
// Not a valid bucket
in: "./bucket/okfile.txt",
check: []string{"./bucket/okfile.txt", "/bucket/okfile.txt", "bucket/okfile.txt"},
exist: false,
},
{
// Not a valid bucket
in: "æ/okfile.txt",
check: []string{"æ/okfile.txt", "æ/okfile.txt", "æ"},
exist: false,
},
{
in: "/bucket2/okfile2.txt",
check: []string{"./bucket2/okfile2.txt", "/bucket2/okfile2.txt", "bucket2/okfile2.txt", "bucket2"},
exist: true,
},
{
in: "/bucket3/prefix/okfile2.txt",
check: []string{"./bucket3/prefix/okfile2.txt", "/bucket3/prefix/okfile2.txt", "bucket3/prefix/okfile2.txt", "bucket3/prefix", "bucket3"},
exist: true,
},
}
for _, tt := range tests {
t.Run(tt.in, func(t *testing.T) {
dut.input <- tt.in
dut.input <- "" // Sending empty string ensures the previous is added to filter.
dut.mu.Lock()
defer dut.mu.Unlock()
if len(tt.check) == 0 {
got := dut.Current.bf.containsDir(tt.in)
if got != tt.exist {
// For unlimited tests this could lead to false positives,
// but it should be deterministic.
t.Errorf("entry %q, got: %v, want %v", tt.in, got, tt.exist)
}
return
}
for _, check := range tt.check {
got := dut.Current.bf.containsDir(check)
if got != tt.exist {
// For unlimited tests this could lead to false positives,
// but it should be deterministic.
t.Errorf("entry %q, check: %q, got: %v, want %v", tt.in, check, got, tt.exist)
}
continue
}
})
}
// Cycle to history
req := bloomFilterRequest{
Oldest: 1,
Current: 2,
}
_, err = dut.cycleFilter(ctx, req)
if err != nil {
t.Fatal(err)
}
dut.input <- "cycle2/file.txt"
dut.input <- "" // Sending empty string ensures the previous is added to filter.
tests = append(tests, struct {
in string
check []string
exist bool
}{in: "cycle2/file.txt", exist: true})
// Shut down
cancel()
<-dut.saveExited
if dut.current() != 2 {
t.Fatal("wrong current idx after save. want 2, got:", dut.current())
}
ctx, cancel = context.WithCancel(context.Background())
// Reload...
dut = newDataUpdateTracker()
dut.start(ctx, tmpDir)
defer func() {
cancel()
<-dut.saveExited
}()
if dut.current() != 2 {
t.Fatal("current idx after load not preserved. want 2, got:", dut.current())
}
req = bloomFilterRequest{
Oldest: 1,
Current: 3,
}
bfr2, err := dut.cycleFilter(ctx, req)
if err != nil {
t.Fatal(err)
}
if !bfr2.Complete {
t.Fatal("Wanted complete, didn't get it")
}
if bfr2.CurrentIdx != 3 {
t.Fatal("wanted index 3, got", bfr2.CurrentIdx)
}
if bfr2.OldestIdx != 1 {
t.Fatal("wanted oldest index 3, got", bfr2.OldestIdx)
}
t.Logf("Size of filter %d bytes, M: %d, K:%d", len(bfr2.Filter), dut.Current.bf.Cap(), dut.Current.bf.K())
// Rerun test with returned bfr2
bf := dut.newBloomFilter()
_, err = bf.ReadFrom(bytes.NewReader(bfr2.Filter))
if err != nil {
t.Fatal(err)
}
for _, tt := range tests {
t.Run(tt.in+"-reloaded", func(t *testing.T) {
if len(tt.check) == 0 {
got := bf.containsDir(tt.in)
if got != tt.exist {
// For unlimited tests this could lead to false positives,
// but it should be deterministic.
t.Errorf("entry %q, got: %v, want %v", tt.in, got, tt.exist)
}
return
}
for _, check := range tt.check {
got := bf.containsDir(check)
if got != tt.exist {
// For unlimited tests this could lead to false positives,
// but it should be deterministic.
t.Errorf("entry %q, check: %q, got: %v, want %v", tt.in, check, got, tt.exist)
}
continue
}
})
}
}
func BenchmarkDataUpdateTracker(b *testing.B) {
dut := newDataUpdateTracker()
// Change some defaults.
dut.debug = false
dut.input = make(chan string)
dut.save = make(chan struct{})
defer addTestingLogging(b)()
dut.Current.bf = dut.newBloomFilter()
// We do this unbuffered. This will very significantly reduce throughput, so this is a worst case.
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go dut.startCollector(ctx)
input := make([]string, 1000)
rng := rand.New(rand.NewSource(0xabad1dea))
tmp := []string{"bucket", "aprefix", "nextprefixlevel", "maybeobjname", "evendeeper", "ok-one-morelevel", "final.object"}
for i := range input {
tmp := tmp[:1+rng.Intn(cap(tmp)-1)]
input[i] = path.Join(tmp...)
}
b.SetBytes(1)
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
dut.input <- input[rng.Intn(len(input))]
}
}
+71 -26
View File
@@ -23,6 +23,7 @@ import (
"errors"
"fmt"
"io"
"math/rand"
"net/http"
"path"
"path/filepath"
@@ -47,6 +48,9 @@ type dataUsageHash string
// sizeHistogram is a size histogram.
type sizeHistogram [dataUsageBucketLen]uint64
// versionsHistogram is a histogram of number of versions in an object.
type versionsHistogram [dataUsageVersionLen]uint64
type dataUsageEntry struct {
Children dataUsageHashMap `msg:"ch"`
// These fields do no include any children.
@@ -54,6 +58,7 @@ type dataUsageEntry struct {
Objects uint64 `msg:"os"`
Versions uint64 `msg:"vs"` // Versions that are not delete markers.
ObjSizes sizeHistogram `msg:"szs"`
ObjVersions versionsHistogram `msg:"vh"`
ReplicationStats *replicationAllStats `msg:"rs,omitempty"`
AllTierStats *allTierStats `msg:"ats,omitempty"`
Compacted bool `msg:"c"`
@@ -276,7 +281,6 @@ type dataUsageCacheInfo struct {
// indicates if the disk is being healed and scanner
// should skip healing the disk
SkipHealing bool
BloomFilter []byte `msg:"BloomFilter,omitempty"`
// Active lifecycle, if any on the bucket
lifeCycle *lifecycle.Lifecycle `msg:"-"`
@@ -292,6 +296,7 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) {
e.Size += summary.totalSize
e.Versions += summary.versions
e.ObjSizes.add(summary.totalSize)
e.ObjVersions.add(summary.versions)
if e.ReplicationStats == nil {
e.ReplicationStats = &replicationAllStats{
@@ -352,6 +357,10 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
e.ObjSizes[i] += v
}
for i, v := range other.ObjVersions[:] {
e.ObjVersions[i] += v
}
if other.AllTierStats != nil {
if e.AllTierStats == nil {
e.AllTierStats = newAllTierStats()
@@ -649,10 +658,7 @@ func (d *dataUsageCache) reduceChildrenOf(path dataUsageHash, limit int, compact
// StringAll returns a detailed string representation of all entries in the cache.
func (d *dataUsageCache) StringAll() string {
// Remove bloom filter from print.
bf := d.Info.BloomFilter
d.Info.BloomFilter = nil
s := fmt.Sprintf("info:%+v\n", d.Info)
d.Info.BloomFilter = bf
for k, v := range d.Cache {
s += fmt.Sprintf("\t%v: %+v\n", k, v)
}
@@ -698,7 +704,7 @@ func (d *dataUsageCache) flatten(root dataUsageEntry) dataUsageEntry {
func (h *sizeHistogram) add(size int64) {
// Fetch the histogram interval corresponding
// to the passed object size.
for i, interval := range ObjectsHistogramIntervals {
for i, interval := range ObjectsHistogramIntervals[:] {
if size >= interval.start && size <= interval.end {
h[i]++
break
@@ -715,6 +721,27 @@ func (h *sizeHistogram) toMap() map[string]uint64 {
return res
}
// add a version count to the histogram.
func (h *versionsHistogram) add(versions uint64) {
// Fetch the histogram interval corresponding
// to the passed object size.
for i, interval := range ObjectsVersionCountIntervals[:] {
if versions >= uint64(interval.start) && versions <= uint64(interval.end) {
h[i]++
break
}
}
}
// toMap returns the map to a map[string]uint64.
func (h *versionsHistogram) toMap() map[string]uint64 {
res := make(map[string]uint64, dataUsageVersionLen)
for i, count := range h {
res[ObjectsVersionCountIntervals[i].name] = count
}
return res
}
func (d *dataUsageCache) tiersUsageInfo(buckets []BucketInfo) *allTierStats {
dst := newAllTierStats()
for _, bucket := range buckets {
@@ -745,10 +772,11 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
}
flat := d.flatten(*e)
bui := BucketUsageInfo{
Size: uint64(flat.Size),
VersionsCount: flat.Versions,
ObjectsCount: flat.Objects,
ObjectSizesHistogram: flat.ObjSizes.toMap(),
Size: uint64(flat.Size),
VersionsCount: flat.Versions,
ObjectsCount: flat.Objects,
ObjectSizesHistogram: flat.ObjSizes.toMap(),
ObjectVersionsHistogram: flat.ObjVersions.toMap(),
}
if flat.ReplicationStats != nil {
bui.ReplicaSize = flat.ReplicationStats.ReplicaSize
@@ -851,37 +879,54 @@ type objectIO interface {
// load the cache content with name from minioMetaBackgroundOpsBucket.
// Only backend errors are returned as errors.
// The loader is optimistic and has no locking, but tries 5 times before giving up.
// If the object is not found or unable to deserialize d is cleared and nil error is returned.
func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string) error {
// Abandon if more than 5 minutes, so we don't hold up scanner.
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
r, err := store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, readLock, ObjectOptions{})
if err != nil {
switch err.(type) {
case ObjectNotFound:
case BucketNotFound:
case InsufficientReadQuorum:
case StorageErr:
default:
return toObjectErr(err, dataUsageBucket, name)
// Caches are read+written without locks,
retries := 0
for retries < 5 {
r, err := store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, noLock, ObjectOptions{NoLock: true})
if err != nil {
switch err.(type) {
case ObjectNotFound, BucketNotFound:
case InsufficientReadQuorum, StorageErr:
retries++
time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
continue
default:
return toObjectErr(err, dataUsageBucket, name)
}
*d = dataUsageCache{}
return nil
}
*d = dataUsageCache{}
if err := d.deserialize(r); err != nil {
r.Close()
retries++
time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
continue
}
r.Close()
return nil
}
defer r.Close()
if err := d.deserialize(r); err != nil {
*d = dataUsageCache{}
logger.LogOnceIf(ctx, err, err.Error())
}
*d = dataUsageCache{}
return nil
}
// Maximum running concurrent saves on server.
var maxConcurrentScannerSaves = make(chan struct{}, 4)
// save the content of the cache to minioMetaBackgroundOpsBucket with the provided name.
// Note that no locking is done when saving.
func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string) error {
var r io.Reader
maxConcurrentScannerSaves <- struct{}{}
defer func() {
<-maxConcurrentScannerSaves
}()
// If big, do streaming...
size := int64(-1)
if len(d.Cache) > 10000 {
@@ -913,7 +958,7 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
dataUsageBucket,
name,
NewPutObjReader(hr),
ObjectOptions{})
ObjectOptions{NoLock: true})
if isErrBucketNotFound(err) {
return nil
}
+155 -74
View File
@@ -629,12 +629,6 @@ func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
err = msgp.WrapError(err, "SkipHealing")
return
}
case "BloomFilter":
z.BloomFilter, err = dc.ReadBytes(z.BloomFilter)
if err != nil {
err = msgp.WrapError(err, "BloomFilter")
return
}
default:
err = dc.Skip()
if err != nil {
@@ -648,24 +642,9 @@ func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
// EncodeMsg implements msgp.Encodable
func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
// omitempty: check for empty values
zb0001Len := uint32(5)
var zb0001Mask uint8 /* 5 bits */
_ = zb0001Mask
if z.BloomFilter == nil {
zb0001Len--
zb0001Mask |= 0x10
}
// variable map header, size zb0001Len
err = en.Append(0x80 | uint8(zb0001Len))
if err != nil {
return
}
if zb0001Len == 0 {
return
}
// map header, size 4
// write "Name"
err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65)
err = en.Append(0x84, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
if err != nil {
return
}
@@ -704,39 +683,15 @@ func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "SkipHealing")
return
}
if (zb0001Mask & 0x10) == 0 { // if not empty
// write "BloomFilter"
err = en.Append(0xab, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
if err != nil {
return
}
err = en.WriteBytes(z.BloomFilter)
if err != nil {
err = msgp.WrapError(err, "BloomFilter")
return
}
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z *dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// omitempty: check for empty values
zb0001Len := uint32(5)
var zb0001Mask uint8 /* 5 bits */
_ = zb0001Mask
if z.BloomFilter == nil {
zb0001Len--
zb0001Mask |= 0x10
}
// variable map header, size zb0001Len
o = append(o, 0x80|uint8(zb0001Len))
if zb0001Len == 0 {
return
}
// map header, size 4
// string "Name"
o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
o = append(o, 0x84, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
o = msgp.AppendString(o, z.Name)
// string "NextCycle"
o = append(o, 0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65)
@@ -747,11 +702,6 @@ func (z *dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
// string "SkipHealing"
o = append(o, 0xab, 0x53, 0x6b, 0x69, 0x70, 0x48, 0x65, 0x61, 0x6c, 0x69, 0x6e, 0x67)
o = msgp.AppendBool(o, z.SkipHealing)
if (zb0001Mask & 0x10) == 0 { // if not empty
// string "BloomFilter"
o = append(o, 0xab, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
o = msgp.AppendBytes(o, z.BloomFilter)
}
return
}
@@ -797,12 +747,6 @@ func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
err = msgp.WrapError(err, "SkipHealing")
return
}
case "BloomFilter":
z.BloomFilter, bts, err = msgp.ReadBytesBytes(bts, z.BloomFilter)
if err != nil {
err = msgp.WrapError(err, "BloomFilter")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
@@ -817,7 +761,7 @@ func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *dataUsageCacheInfo) Msgsize() (s int) {
s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 10 + msgp.Uint32Size + 11 + msgp.TimeSize + 12 + msgp.BoolSize + 12 + msgp.BytesPrefixSize + len(z.BloomFilter)
s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 10 + msgp.Uint32Size + 11 + msgp.TimeSize + 12 + msgp.BoolSize
return
}
@@ -1596,6 +1540,24 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
}
case "vh":
var zb0003 uint32
zb0003, err = dc.ReadArrayHeader()
if err != nil {
err = msgp.WrapError(err, "ObjVersions")
return
}
if zb0003 != uint32(dataUsageVersionLen) {
err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
return
}
for za0002 := range z.ObjVersions {
z.ObjVersions[za0002], err = dc.ReadUint64()
if err != nil {
err = msgp.WrapError(err, "ObjVersions", za0002)
return
}
}
case "rs":
if dc.IsNil() {
err = dc.ReadNil()
@@ -1652,16 +1614,16 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
// EncodeMsg implements msgp.Encodable
func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
// omitempty: check for empty values
zb0001Len := uint32(8)
var zb0001Mask uint8 /* 8 bits */
zb0001Len := uint32(9)
var zb0001Mask uint16 /* 9 bits */
_ = zb0001Mask
if z.ReplicationStats == nil {
zb0001Len--
zb0001Mask |= 0x20
zb0001Mask |= 0x40
}
if z.AllTierStats == nil {
zb0001Len--
zb0001Mask |= 0x40
zb0001Mask |= 0x80
}
// variable map header, size zb0001Len
err = en.Append(0x80 | uint8(zb0001Len))
@@ -1728,7 +1690,24 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
return
}
}
if (zb0001Mask & 0x20) == 0 { // if not empty
// write "vh"
err = en.Append(0xa2, 0x76, 0x68)
if err != nil {
return
}
err = en.WriteArrayHeader(uint32(dataUsageVersionLen))
if err != nil {
err = msgp.WrapError(err, "ObjVersions")
return
}
for za0002 := range z.ObjVersions {
err = en.WriteUint64(z.ObjVersions[za0002])
if err != nil {
err = msgp.WrapError(err, "ObjVersions", za0002)
return
}
}
if (zb0001Mask & 0x40) == 0 { // if not empty
// write "rs"
err = en.Append(0xa2, 0x72, 0x73)
if err != nil {
@@ -1747,7 +1726,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
}
}
}
if (zb0001Mask & 0x40) == 0 { // if not empty
if (zb0001Mask & 0x80) == 0 { // if not empty
// write "ats"
err = en.Append(0xa3, 0x61, 0x74, 0x73)
if err != nil {
@@ -1783,16 +1762,16 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// omitempty: check for empty values
zb0001Len := uint32(8)
var zb0001Mask uint8 /* 8 bits */
zb0001Len := uint32(9)
var zb0001Mask uint16 /* 9 bits */
_ = zb0001Mask
if z.ReplicationStats == nil {
zb0001Len--
zb0001Mask |= 0x20
zb0001Mask |= 0x40
}
if z.AllTierStats == nil {
zb0001Len--
zb0001Mask |= 0x40
zb0001Mask |= 0x80
}
// variable map header, size zb0001Len
o = append(o, 0x80|uint8(zb0001Len))
@@ -1821,7 +1800,13 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
for za0001 := range z.ObjSizes {
o = msgp.AppendUint64(o, z.ObjSizes[za0001])
}
if (zb0001Mask & 0x20) == 0 { // if not empty
// string "vh"
o = append(o, 0xa2, 0x76, 0x68)
o = msgp.AppendArrayHeader(o, uint32(dataUsageVersionLen))
for za0002 := range z.ObjVersions {
o = msgp.AppendUint64(o, z.ObjVersions[za0002])
}
if (zb0001Mask & 0x40) == 0 { // if not empty
// string "rs"
o = append(o, 0xa2, 0x72, 0x73)
if z.ReplicationStats == nil {
@@ -1834,7 +1819,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
}
}
}
if (zb0001Mask & 0x40) == 0 { // if not empty
if (zb0001Mask & 0x80) == 0 { // if not empty
// string "ats"
o = append(o, 0xa3, 0x61, 0x74, 0x73)
if z.AllTierStats == nil {
@@ -1913,6 +1898,24 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
}
case "vh":
var zb0003 uint32
zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, "ObjVersions")
return
}
if zb0003 != uint32(dataUsageVersionLen) {
err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
return
}
for za0002 := range z.ObjVersions {
z.ObjVersions[za0002], bts, err = msgp.ReadUint64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "ObjVersions", za0002)
return
}
}
case "rs":
if msgp.IsNil(bts) {
bts, err = msgp.ReadNilBytes(bts)
@@ -1967,7 +1970,7 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *dataUsageEntry) Msgsize() (s int) {
s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3
s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 3
if z.ReplicationStats == nil {
s += msgp.NilSize
} else {
@@ -3760,3 +3763,81 @@ func (z tierStats) Msgsize() (s int) {
s = 1 + 3 + msgp.Uint64Size + 3 + msgp.IntSize + 3 + msgp.IntSize
return
}
// DecodeMsg implements msgp.Decodable
func (z *versionsHistogram) DecodeMsg(dc *msgp.Reader) (err error) {
var zb0001 uint32
zb0001, err = dc.ReadArrayHeader()
if err != nil {
err = msgp.WrapError(err)
return
}
if zb0001 != uint32(dataUsageVersionLen) {
err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0001}
return
}
for za0001 := range z {
z[za0001], err = dc.ReadUint64()
if err != nil {
err = msgp.WrapError(err, za0001)
return
}
}
return
}
// EncodeMsg implements msgp.Encodable
func (z *versionsHistogram) EncodeMsg(en *msgp.Writer) (err error) {
err = en.WriteArrayHeader(uint32(dataUsageVersionLen))
if err != nil {
err = msgp.WrapError(err)
return
}
for za0001 := range z {
err = en.WriteUint64(z[za0001])
if err != nil {
err = msgp.WrapError(err, za0001)
return
}
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z *versionsHistogram) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
o = msgp.AppendArrayHeader(o, uint32(dataUsageVersionLen))
for za0001 := range z {
o = msgp.AppendUint64(o, z[za0001])
}
return
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *versionsHistogram) UnmarshalMsg(bts []byte) (o []byte, err error) {
var zb0001 uint32
zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
if zb0001 != uint32(dataUsageVersionLen) {
err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0001}
return
}
for za0001 := range z {
z[za0001], bts, err = msgp.ReadUint64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, za0001)
return
}
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *versionsHistogram) Msgsize() (s int) {
s = msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size))
return
}
+113
View File
@@ -1196,3 +1196,116 @@ func BenchmarkDecodetierStats(b *testing.B) {
}
}
}
func TestMarshalUnmarshalversionsHistogram(t *testing.T) {
v := versionsHistogram{}
bts, err := v.MarshalMsg(nil)
if err != nil {
t.Fatal(err)
}
left, err := v.UnmarshalMsg(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
}
left, err = msgp.Skip(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
}
}
func BenchmarkMarshalMsgversionsHistogram(b *testing.B) {
v := versionsHistogram{}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.MarshalMsg(nil)
}
}
func BenchmarkAppendMsgversionsHistogram(b *testing.B) {
v := versionsHistogram{}
bts := make([]byte, 0, v.Msgsize())
bts, _ = v.MarshalMsg(bts[0:0])
b.SetBytes(int64(len(bts)))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
bts, _ = v.MarshalMsg(bts[0:0])
}
}
func BenchmarkUnmarshalversionsHistogram(b *testing.B) {
v := versionsHistogram{}
bts, _ := v.MarshalMsg(nil)
b.ReportAllocs()
b.SetBytes(int64(len(bts)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := v.UnmarshalMsg(bts)
if err != nil {
b.Fatal(err)
}
}
}
func TestEncodeDecodeversionsHistogram(t *testing.T) {
v := versionsHistogram{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
m := v.Msgsize()
if buf.Len() > m {
t.Log("WARNING: TestEncodeDecodeversionsHistogram Msgsize() is inaccurate")
}
vn := versionsHistogram{}
err := msgp.Decode(&buf, &vn)
if err != nil {
t.Error(err)
}
buf.Reset()
msgp.Encode(&buf, &v)
err = msgp.NewReader(&buf).Skip()
if err != nil {
t.Error(err)
}
}
func BenchmarkEncodeversionsHistogram(b *testing.B) {
v := versionsHistogram{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
en := msgp.NewWriter(msgp.Nowhere)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.EncodeMsg(en)
}
en.Flush()
}
func BenchmarkDecodeversionsHistogram(b *testing.B) {
v := versionsHistogram{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
rd := msgp.NewEndlessReader(buf.Bytes(), b)
dc := msgp.NewReader(rd)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
err := v.DecodeMsg(dc)
if err != nil {
b.Fatal(err)
}
}
}
+6 -5
View File
@@ -58,11 +58,12 @@ type BucketUsageInfo struct {
// Total number of objects that failed replication
ReplicationFailedCountV1 uint64 `json:"objectsFailedReplicationCount"`
ObjectsCount uint64 `json:"objectsCount"`
ObjectSizesHistogram map[string]uint64 `json:"objectsSizesHistogram"`
VersionsCount uint64 `json:"versionsCount"`
ReplicaSize uint64 `json:"objectReplicaTotalSize"`
ReplicationInfo map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
ObjectsCount uint64 `json:"objectsCount"`
ObjectSizesHistogram map[string]uint64 `json:"objectsSizesHistogram"`
ObjectVersionsHistogram map[string]uint64 `json:"objectsVersionsHistogram"`
VersionsCount uint64 `json:"versionsCount"`
ReplicaSize uint64 `json:"objectReplicaTotalSize"`
ReplicationInfo map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
}
// DataUsageInfo represents data usage stats of the underlying Object API
+15 -4
View File
@@ -25,7 +25,6 @@ import (
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
objectlock "github.com/minio/minio/internal/bucket/object/lock"
@@ -35,6 +34,7 @@ import (
"github.com/minio/minio/internal/hash"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/sync/errgroup"
xnet "github.com/minio/pkg/net"
"github.com/minio/pkg/wildcard"
)
@@ -448,7 +448,10 @@ func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string,
return cachedObjInfo, nil
}
c.cacheStats.incMiss()
return ObjectInfo{}, BackendDown{}
if xnet.IsNetworkOrHostDown(err, false) {
return ObjectInfo{}, BackendDown{Err: err.Error()}
}
return ObjectInfo{}, err
}
// Reaching here implies cache miss
c.cacheStats.incMiss()
@@ -893,8 +896,16 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
cacheDiskStats[i].UsageSize = info.Used
cacheDiskStats[i].TotalCapacity = info.Total
cacheDiskStats[i].Dir = dcache.stats.Dir
atomic.StoreInt32(&cacheDiskStats[i].UsageState, atomic.LoadInt32(&dcache.stats.UsageState))
atomic.StoreUint64(&cacheDiskStats[i].UsagePercent, atomic.LoadUint64(&dcache.stats.UsagePercent))
if info.Total != 0 {
// UsageState
gcTriggerPct := dcache.quotaPct * dcache.highWatermark / 100
usedPercent := float64(info.Used) * 100 / float64(info.Total)
if usedPercent >= float64(gcTriggerPct) {
cacheDiskStats[i].UsageState = 1
}
// UsagePercent
cacheDiskStats[i].UsagePercent = uint64(usedPercent)
}
}
}
return cacheDiskStats
+2 -2
View File
@@ -110,7 +110,7 @@ func kmsKeyIDFromMetadata(metadata map[string]string) string {
//
// DecryptETags uses a KMS bulk decryption API, if available, which
// is more efficient than decrypting ETags sequentually.
func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo) error {
func DecryptETags(ctx context.Context, k kms.KMS, objects []ObjectInfo) error {
const BatchSize = 250 // We process the objects in batches - 250 is a reasonable default.
var (
metadata = make([]map[string]string, 0, BatchSize)
@@ -170,7 +170,7 @@ func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo) error
// For all SSE-S3 single-part objects we have to
// fetch their decryption keys. We do this using
// a Bulk-Decryption API call, if available.
keys, err := crypto.S3.UnsealObjectKeys(ctx, KMS, metadata, buckets, names)
keys, err := crypto.S3.UnsealObjectKeys(ctx, k, metadata, buckets, names)
if err != nil {
return err
}
+8 -6
View File
@@ -62,13 +62,14 @@ func TestNewEndpoint(t *testing.T) {
err = endpoint.UpdateIsLocal()
}
if test.expectedErr == nil {
switch {
case test.expectedErr == nil:
if err != nil {
t.Errorf("error: expected = <nil>, got = %v", err)
}
} else if err == nil {
case err == nil:
t.Errorf("error: expected = %v, got = <nil>", test.expectedErr)
} else if test.expectedErr.Error() != err.Error() {
case test.expectedErr.Error() != err.Error():
t.Errorf("error: expected = %v, got = %v", test.expectedErr, err)
}
@@ -115,13 +116,14 @@ func TestNewEndpoints(t *testing.T) {
for _, testCase := range testCases {
_, err := NewEndpoints(testCase.args...)
if testCase.expectedErr == nil {
switch {
case testCase.expectedErr == nil:
if err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
}
} else if err == nil {
case err == nil:
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
} else if testCase.expectedErr.Error() != err.Error() {
case testCase.expectedErr.Error() != err.Error():
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
}
}
+1 -5
View File
@@ -24,7 +24,7 @@ import (
"io"
"testing"
humanize "github.com/dustin/go-humanize"
"github.com/dustin/go-humanize"
)
type badDisk struct{ StorageAPI }
@@ -41,10 +41,6 @@ func (a badDisk) ReadFileStream(ctx context.Context, volume, path string, offset
return nil, errFaultyDisk
}
func (a badDisk) UpdateBloomFilter(ctx context.Context, oldest, current uint64) (*bloomFilterResponse, error) {
return nil, errFaultyDisk
}
func (a badDisk) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
return errFaultyDisk
}
+9 -12
View File
@@ -72,10 +72,6 @@ func (fi FileInfo) DataShardFixed() bool {
func (er erasureObjects) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (
result madmin.HealResultItem, err error,
) {
if !opts.DryRun {
defer NSUpdated(bucket, slashSeparator)
}
storageDisks := er.getDisks()
storageEndpoints := er.getEndpoints()
@@ -754,7 +750,6 @@ func (er *erasureObjects) healObjectDir(ctx context.Context, bucket, object stri
}(index, disk)
}
wg.Wait()
NSUpdated(bucket, object)
}
}
@@ -915,13 +910,14 @@ func isObjectDirDangling(errs []error) (ok bool) {
var foundNotEmpty int
var otherFound int
for _, readErr := range errs {
if readErr == nil {
switch {
case readErr == nil:
found++
} else if readErr == errFileNotFound || readErr == errVolumeNotFound {
case readErr == errFileNotFound || readErr == errVolumeNotFound:
notFound++
} else if readErr == errVolumeNotEmpty {
case readErr == errVolumeNotEmpty:
foundNotEmpty++
} else {
default:
otherFound++
}
}
@@ -943,11 +939,12 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
diskNotFoundCount int
)
for _, readErr := range cerrs {
if errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound) {
switch {
case errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound):
notFoundCount++
} else if errors.Is(readErr, errFileCorrupt) {
case errors.Is(readErr, errFileCorrupt):
corruptedCount++
} else if errors.Is(readErr, errDiskNotFound) {
case errors.Is(readErr, errDiskNotFound):
diskNotFoundCount++
}
}
+2 -2
View File
@@ -1621,10 +1621,10 @@ func TestHealLastDataShard(t *testing.T) {
}
firstGr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, noLock, ObjectOptions{})
defer firstGr.Close()
if err != nil {
t.Fatal(err)
}
defer firstGr.Close()
firstHealedH := sha256.New()
_, err = io.Copy(firstHealedH, firstGr)
@@ -1651,10 +1651,10 @@ func TestHealLastDataShard(t *testing.T) {
}
secondGr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, noLock, ObjectOptions{})
defer secondGr.Close()
if err != nil {
t.Fatal(err)
}
defer secondGr.Close()
secondHealedH := sha256.New()
_, err = io.Copy(secondHealedH, secondGr)
-1
View File
@@ -1216,7 +1216,6 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
if err != nil {
return oi, toObjectErr(err, bucket, object)
}
defer NSUpdated(bucket, object)
if !opts.Speedtest && versionsDisparity {
listAndHeal(ctx, bucket, object, &er, healObjectVersionsDisparity)
+1 -13
View File
@@ -72,8 +72,6 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
return oi, NotImplemented{}
}
defer NSUpdated(dstBucket, dstObject)
if !dstOpts.NoLock {
lk := er.NewNSLock(dstBucket, dstObject)
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
@@ -483,7 +481,6 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
if opts.VersionID != "" {
err = errFileVersionNotFound
}
defer NSUpdated(bucket, object)
fi := FileInfo{
VersionID: m.VersionID,
@@ -1271,8 +1268,6 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
return ObjectInfo{}, toObjectErr(err, bucket, object)
}
defer NSUpdated(bucket, object)
for i := 0; i < len(onlineDisks); i++ {
if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
// Object info is the same in all disks, so we can pick
@@ -1475,8 +1470,6 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
} else {
errs[objIndex] = toObjectErr(err, bucket, objects[objIndex].ObjectName)
}
defer NSUpdated(bucket, objects[objIndex].ObjectName)
}
// Check failed deletes across multiple objects
@@ -1568,16 +1561,12 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
}
}
objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
defer NSUpdated(bucket, object)
storageDisks := er.getDisks()
versionFound := true
objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
goi, _, gerr := er.getObjectInfoAndQuorum(ctx, bucket, object, opts)
if gerr != nil && goi.Name == "" {
switch gerr.(type) {
case InsufficientReadQuorum:
if _, ok := gerr.(InsufficientReadQuorum); ok {
return objInfo, InsufficientWriteQuorum{}
}
// For delete marker replication, versionID being replicated will not exist on disk
@@ -1940,7 +1929,6 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
if fi.TransitionStatus == lifecycle.TransitionComplete {
return nil
}
defer NSUpdated(bucket, object)
if fi.XLV1 {
if _, err = er.HealObject(ctx, bucket, object, "", madmin.HealOpts{NoLock: true}); err != nil {
+7 -10
View File
@@ -32,7 +32,6 @@ import (
"github.com/dustin/go-humanize"
"github.com/minio/madmin-go/v2"
"github.com/minio/minio/internal/bucket/lifecycle"
"github.com/minio/minio/internal/hash"
"github.com/minio/minio/internal/logger"
"github.com/minio/pkg/console"
@@ -735,17 +734,15 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
}
versioned := vc != nil && vc.Versioned(object)
objInfo := fi.ToObjectInfo(bucket, object, versioned)
evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
switch evt.Action {
case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
globalExpiryState.enqueueByDays(objInfo, false, evt.Action == lifecycle.DeleteVersionAction)
// Skip this entry.
return true
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
globalExpiryState.enqueueByDays(objInfo, true, evt.Action == lifecycle.DeleteRestoredVersionAction)
// Skip this entry.
return true
if evt.Action.Delete() {
globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
if !evt.Action.DeleteRestored() {
return true
}
}
return false
}
+5 -10
View File
@@ -32,7 +32,6 @@ import (
"github.com/lithammer/shortuuid/v4"
"github.com/minio/madmin-go/v2"
"github.com/minio/minio/internal/bucket/lifecycle"
"github.com/minio/minio/internal/hash"
"github.com/minio/minio/internal/logger"
"github.com/minio/pkg/env"
@@ -463,17 +462,13 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
}
versioned := vc != nil && vc.Versioned(object)
objInfo := fi.ToObjectInfo(bucket, object, versioned)
event := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
switch action := event.Action; action {
case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
globalExpiryState.enqueueByDays(objInfo, false, action == lifecycle.DeleteVersionAction)
// Skip this entry.
return true
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
globalExpiryState.enqueueByDays(objInfo, true, action == lifecycle.DeleteRestoredVersionAction)
// Skip this entry.
evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
if evt.Action.Delete() {
globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
return true
}
return false
}
+28 -42
View File
@@ -37,7 +37,6 @@ import (
"github.com/minio/minio-go/v7/pkg/s3utils"
"github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio-go/v7/pkg/tags"
"github.com/minio/minio/internal/bucket/lifecycle"
"github.com/minio/minio/internal/config/storageclass"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/sync/errgroup"
@@ -56,9 +55,6 @@ type erasureServerPools struct {
serverPools []*erasureSets
// Shut down async operations
shutdown context.CancelFunc
// Active decommission canceler
decommissionCancelers []context.CancelFunc
@@ -160,14 +156,9 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
break
}
drives := make([]string, 0, len(localDrives))
for _, localDrive := range localDrives {
drives = append(drives, localDrive.Endpoint().Path)
}
globalLocalDrivesMu.Lock()
globalLocalDrives = localDrives
ctx, z.shutdown = context.WithCancel(ctx)
go intDataUpdateTracker.start(ctx, drives...)
defer globalLocalDrivesMu.Unlock()
return z, nil
}
@@ -431,6 +422,7 @@ func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bu
return mtime1.After(mtime2)
})
defPool := PoolObjInfo{Index: -1}
for _, pinfo := range poolObjInfos {
// skip all objects from suspended pools if asked by the
// caller.
@@ -445,14 +437,13 @@ func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bu
if pinfo.Err != nil && !isErrObjectNotFound(pinfo.Err) {
return pinfo, pinfo.Err
}
defPool = pinfo
if isErrObjectNotFound(pinfo.Err) {
// No object exists or its a delete marker,
// check objInfo to confirm.
if pinfo.ObjInfo.DeleteMarker && pinfo.ObjInfo.Name != "" {
return pinfo, nil
}
// objInfo is not valid, truly the object doesn't
// exist proceed to next pool.
continue
@@ -460,7 +451,12 @@ func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bu
return pinfo, nil
}
if opts.ReplicationRequest && opts.DeleteMarker && defPool.Index >= 0 {
// If the request is a delete marker replication request, return a default pool
// in cases where the object does not exist.
// This is to ensure that the delete marker is replicated to the destination.
return defPool, nil
}
return PoolObjInfo{}, toObjectErr(errFileNotFound, bucket, object)
}
@@ -524,8 +520,6 @@ func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object stri
}
func (z *erasureServerPools) Shutdown(ctx context.Context) error {
defer z.shutdown()
g := errgroup.WithNErrs(len(z.serverPools))
for index := range z.serverPools {
@@ -594,7 +588,7 @@ func (z *erasureServerPools) StorageInfo(ctx context.Context) StorageInfo {
return globalNotificationSys.StorageInfo(z)
}
func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32, healScanMode madmin.HealScanMode) error {
func (z *erasureServerPools) NSScanner(ctx context.Context, updates chan<- DataUsageInfo, wantCycle uint32, healScanMode madmin.HealScanMode) error {
// Updates must be closed before we return.
defer close(updates)
@@ -639,7 +633,7 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
}
}()
// Start scanner. Blocks until done.
err := erObj.nsScanner(ctx, allBuckets, bf, wantCycle, updates, healScanMode)
err := erObj.nsScanner(ctx, allBuckets, wantCycle, updates, healScanMode)
if err != nil {
logger.LogIf(ctx, err)
mu.Lock()
@@ -701,9 +695,11 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
case updateCloser <- ch:
<-ch
case <-ctx.Done():
mu.Lock()
if firstErr == nil {
firstErr = ctx.Err()
}
defer mu.Unlock()
}
return firstErr
}
@@ -712,8 +708,6 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
// even if one of the sets fail to create buckets, we proceed all the successful
// operations.
func (z *erasureServerPools) MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error {
defer NSUpdated(bucket, slashSeparator)
// Verify if bucket is valid.
if !isMinioMetaBucketName(bucket) {
if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
@@ -999,8 +993,7 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
gopts.NoLock = true
pinfo, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, gopts)
if err != nil {
switch err.(type) {
case InsufficientReadQuorum:
if _, ok := err.(InsufficientReadQuorum); ok {
return objInfo, InsufficientWriteQuorum{}
}
return objInfo, err
@@ -1313,11 +1306,12 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
if err == nil {
if opts.Lifecycle != nil {
evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, objInfo)
switch evt.Action {
case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
fallthrough
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
return loi, nil
if evt.Action.Delete() {
globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
if !evt.Action.DeleteRestored() {
// Skip entry if ILM action was DeleteVersionAction or DeleteAction
return loi, nil
}
}
}
loi.Objects = append(loi.Objects, objInfo)
@@ -1466,8 +1460,7 @@ func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object,
if err == nil {
return pi, nil
}
switch err.(type) {
case InvalidUploadID:
if _, ok := err.(InvalidUploadID); ok {
// Look for information on the next pool
continue
}
@@ -1498,8 +1491,7 @@ func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, objec
if err == nil {
return mi, nil
}
switch err.(type) {
case InvalidUploadID:
if _, ok := err.(InvalidUploadID); ok {
// upload id not found, continue to the next pool.
continue
}
@@ -1530,8 +1522,7 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
if err == nil {
return result, nil
}
switch err.(type) {
case InvalidUploadID:
if _, ok := err.(InvalidUploadID); ok {
continue
}
return ListPartsInfo{}, err
@@ -1561,8 +1552,7 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
if err == nil {
return nil
}
switch err.(type) {
case InvalidUploadID:
if _, ok := err.(InvalidUploadID); ok {
// upload id not found move to next pool
continue
}
@@ -1593,8 +1583,7 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
if err == nil {
return objInfo, nil
}
switch err.(type) {
case InvalidUploadID:
if _, ok := err.(InvalidUploadID); ok {
// upload id not found move to next pool
continue
}
@@ -1637,7 +1626,6 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
return BucketNameInvalid{Bucket: bucket}
}
defer NSUpdated(bucket, slashSeparator)
if !opts.NoLock {
// Lock the bucket name before creating.
lk := z.NewNSLock(minioMetaTmpBucket, bucket+".lck")
@@ -1652,8 +1640,7 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
err := z.s3Peer.DeleteBucket(ctx, bucket, opts)
if err == nil || errors.Is(err, errVolumeNotFound) {
// If site replication is configured, hold on to deleted bucket state until sites sync
switch opts.SRDeleteOp {
case MarkDelete:
if opts.SRDeleteOp == MarkDelete {
z.s3Peer.MakeBucket(context.Background(), pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, bucket), MakeBucketOptions{})
}
}
@@ -1757,8 +1744,7 @@ func (z *erasureServerPools) HealBucket(ctx context.Context, bucket string, opts
for _, pool := range z.serverPools {
result, err := pool.HealBucket(ctx, bucket, opts)
if err != nil {
switch err.(type) {
case BucketNotFound:
if _, ok := err.(BucketNotFound); ok {
continue
}
return result, err
+12 -17
View File
@@ -33,7 +33,6 @@ import (
"github.com/minio/minio/internal/dsync"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/sync/errgroup"
"github.com/minio/pkg/console"
)
// list all errors that can be ignore in a bucket operation.
@@ -351,7 +350,7 @@ func (er erasureObjects) cleanupDeletedObjects(ctx context.Context) {
// nsScanner will start scanning buckets and send updated totals as they are traversed.
// Updates are sent on a regular basis and the caller *must* consume them.
func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, wantCycle uint32, updates chan<- dataUsageCache, healScanMode madmin.HealScanMode) error {
func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wantCycle uint32, updates chan<- dataUsageCache, healScanMode madmin.HealScanMode) error {
if len(buckets) == 0 {
return nil
}
@@ -377,7 +376,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
},
Cache: make(map[string]dataUsageEntry, len(oldCache.Cache)),
}
bloom := bf.bytes()
// Put all buckets into channel.
bucketCh := make(chan BucketInfo, len(buckets))
@@ -414,9 +412,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
for {
select {
case <-ctx.Done():
// Return without saving.
return
case <-t.C:
if cache.Info.LastUpdate.Equal(lastSave) {
continue
@@ -454,6 +449,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
// Start one scanner per disk
var wg sync.WaitGroup
wg.Add(len(disks))
for i := range disks {
go func(i int) {
defer wg.Done()
@@ -473,7 +469,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
if cache.Info.Name == "" {
cache.Info.Name = bucket.Name
}
cache.Info.BloomFilter = bloom
cache.Info.SkipHealing = healing
cache.Info.NextCycle = wantCycle
if cache.Info.Name != bucket.Name {
@@ -491,13 +486,13 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
go func(name string) {
defer wg.Done()
for update := range updates {
bucketResults <- dataUsageEntryInfo{
select {
case <-ctx.Done():
case bucketResults <- dataUsageEntryInfo{
Name: name,
Parent: dataUsageRoot,
Entry: update,
}
if intDataUpdateTracker.debug {
console.Debugln("z:", er.poolIndex, "s:", er.setIndex, "bucket", name, "got update", update)
}:
}
}
}(cache.Info.Name)
@@ -505,7 +500,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
before := cache.Info.LastUpdate
var err error
cache, err = disk.NSScanner(ctx, cache, updates, healScanMode)
cache.Info.BloomFilter = nil
if err != nil {
if !cache.Info.LastUpdate.IsZero() && cache.Info.LastUpdate.After(before) {
logger.LogIf(ctx, cache.save(ctx, er, cacheName))
@@ -525,15 +519,16 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
if r := cache.root(); r != nil {
root = cache.flatten(*r)
}
t := time.Now()
bucketResults <- dataUsageEntryInfo{
select {
case <-ctx.Done():
return
case bucketResults <- dataUsageEntryInfo{
Name: cache.Info.Name,
Parent: dataUsageRoot,
Entry: root,
}:
}
// We want to avoid synchronizing up all writes in case
// the results are piled up.
time.Sleep(time.Duration(float64(time.Since(t)) * rand.Float64()))
// Save cache
logger.LogIf(ctx, cache.save(ctx, er, cacheName))
}
+1 -1
View File
@@ -101,7 +101,7 @@ func (evnot *EventNotifier) InitBucketTargets(ctx context.Context, objAPI Object
return errServerNotInitialized
}
if err := evnot.targetList.Add(globalConfigTargetList.Targets()...); err != nil {
if err := evnot.targetList.Add(globalNotifyTargetList.Targets()...); err != nil {
return err
}
+25 -5
View File
@@ -269,7 +269,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
}()
// Note: updates from healEntry to tracker must be sent on results channel.
healEntry := func(entry metaCacheEntry) {
healEntry := func(bucket string, entry metaCacheEntry) {
if entry.name == "" && len(entry.metadata) == 0 {
// ignore entries that don't have metadata.
return
@@ -278,6 +278,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
// ignore healing entry.name's with `/` suffix.
return
}
// We might land at .metacache, .trash, .multipart
// no need to heal them skip, only when bucket
// is '.minio.sys'
@@ -302,6 +303,11 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
versionID: "",
}, madmin.HealItemObject)
if err != nil {
if isErrObjectNotFound(err) {
// queueing happens across namespace, ignore
// objects that are not found.
return
}
result = healEntryFailure(0)
logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, entry.name, err))
} else {
@@ -317,12 +323,19 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
return
}
var versionNotFound int
for _, version := range fivs.Versions {
if err := bgSeq.queueHealTask(healSource{
bucket: bucket,
object: version.Name,
versionID: version.VersionID,
}, madmin.HealItemObject); err != nil {
if isErrObjectNotFound(err) {
// queueing happens across namespace, ignore
// objects that are not found.
versionNotFound++
continue
}
// If not deleted, assume they failed.
result = healEntryFailure(uint64(version.Size))
if version.VersionID != "" {
@@ -341,6 +354,10 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
case results <- result:
}
}
// All versions resulted in 'ObjectNotFound'
if versionNotFound == len(fivs.Versions) {
return
}
select {
case <-ctx.Done():
return
@@ -351,22 +368,25 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
waitForLowHTTPReq()
}
actualBucket, prefix := path2BucketObject(bucket)
// How to resolve partial results.
resolver := metadataResolutionParams{
dirQuorum: 1,
objQuorum: 1,
bucket: bucket,
bucket: actualBucket,
}
err := listPathRaw(ctx, listPathRawOptions{
disks: disks,
bucket: bucket,
bucket: actualBucket,
path: prefix,
recursive: true,
forwardTo: forwardTo,
minDisks: 1,
reportNotFound: false,
agreed: func(entry metaCacheEntry) {
healEntry(entry)
healEntry(actualBucket, entry)
},
partial: func(entries metaCacheEntries, _ []error) {
entry, ok := entries.resolve(&resolver)
@@ -375,7 +395,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
// proceed to heal nonetheless.
entry, _ = entries.firstFound()
}
healEntry(*entry)
healEntry(actualBucket, *entry)
},
finished: nil,
})
+9 -11
View File
@@ -42,16 +42,14 @@ import (
"github.com/minio/minio/internal/config/callhome"
"github.com/minio/minio/internal/config/compress"
"github.com/minio/minio/internal/config/dns"
xldap "github.com/minio/minio/internal/config/identity/ldap"
"github.com/minio/minio/internal/config/identity/openid"
idplugin "github.com/minio/minio/internal/config/identity/plugin"
xtls "github.com/minio/minio/internal/config/identity/tls"
polplugin "github.com/minio/minio/internal/config/policy/plugin"
"github.com/minio/minio/internal/config/storageclass"
"github.com/minio/minio/internal/config/subnet"
xhttp "github.com/minio/minio/internal/http"
etcd "go.etcd.io/etcd/client/v3"
levent "github.com/minio/minio/internal/config/lambda/event"
"github.com/minio/minio/internal/event"
"github.com/minio/minio/internal/pubsub"
"github.com/minio/pkg/certs"
@@ -177,7 +175,8 @@ var (
globalMinioConsoleHost = ""
// Holds the possible host endpoint.
globalMinioEndpoint = ""
globalMinioEndpoint = ""
globalMinioEndpointURL *xnet.URL
// globalConfigSys server config system.
globalConfigSys *ConfigSys
@@ -185,7 +184,8 @@ var (
globalNotificationSys *NotificationSys
globalEventNotifier *EventNotifier
globalConfigTargetList *event.TargetList
globalNotifyTargetList *event.TargetList
globalLambdaTargetList *levent.TargetList
globalBucketMetadataSys *BucketMetadataSys
globalBucketMonitor *bandwidth.Monitor
@@ -201,10 +201,6 @@ var (
globalStorageClass storageclass.Config
globalLDAPConfig xldap.Config
globalOpenIDConfig openid.Config
globalSTSTLSConfig xtls.Config
globalAuthNPlugin *idplugin.AuthNPlugin
// CA root certificates, a nil value means system certs pool will be used
@@ -357,8 +353,10 @@ var (
globalServiceFreezeCnt int32
globalServiceFreezeMu sync.Mutex // Updates.
// List of local drives to this node, this is only set during server startup.
globalLocalDrives []StorageAPI
// List of local drives to this node, this is only set during server startup,
// and should never be mutated. Hold globalLocalDrivesMu to access.
globalLocalDrives []StorageAPI
globalLocalDrivesMu sync.RWMutex
// Is MINIO_CI_CD set?
globalIsCICD bool
+7
View File
@@ -227,6 +227,13 @@ func (t *apiConfig) getClusterDeadline() time.Duration {
return t.clusterDeadline
}
func (t *apiConfig) getRequestsPoolCapacity() int {
t.mu.RLock()
defer t.mu.RUnlock()
return cap(t.requestsPool)
}
func (t *apiConfig) getRequestsPool() (chan struct{}, time.Duration) {
t.mu.RLock()
defer t.mu.RUnlock()
+4
View File
@@ -58,6 +58,10 @@ func parseLocationConstraint(r *http.Request) (location string, s3Error APIError
if location == "" {
location = globalSite.Region
}
if !isValidLocation(location) {
return location, ErrInvalidRegion
}
return location, ErrNone
}
+11
View File
@@ -106,6 +106,17 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set(xhttp.MinIOServerStatus, unavailable)
}
if int(globalHTTPStats.loadRequestsInQueue()) > globalAPIConfig.getRequestsPoolCapacity() {
apiErr := getAPIError(ErrBusy)
switch r.Method {
case http.MethodHead:
writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
case http.MethodGet:
writeErrorResponse(r.Context(), w, apiErr, r.URL)
}
return
}
// Verify if KMS is reachable if its configured
if GlobalKMS != nil {
ctx, cancel := context.WithTimeout(r.Context(), time.Minute)
+4
View File
@@ -255,6 +255,10 @@ type HTTPStats struct {
totalS3Canceled HTTPAPIStats
}
func (st *HTTPStats) loadRequestsInQueue() int32 {
return atomic.LoadInt32(&st.s3RequestsInQueue)
}
func (st *HTTPStats) addRequestsInQueue(i int32) {
atomic.AddInt32(&st.s3RequestsInQueue, i)
}
+7
View File
@@ -245,6 +245,13 @@ func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMU
if u.Credentials.AccessKey == "" {
u.Credentials.AccessKey = user
}
if u.Credentials.SessionToken != "" {
jwtClaims, err := extractJWTClaims(u)
if err != nil {
return err
}
u.Credentials.Claims = jwtClaims.Map()
}
m[user] = u
return nil
}
+11 -1
View File
@@ -184,6 +184,14 @@ func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType
u.Credentials.AccessKey = user
}
if u.Credentials.SessionToken != "" {
jwtClaims, err := extractJWTClaims(u)
if err != nil {
return err
}
u.Credentials.Claims = jwtClaims.Map()
}
m[user] = u
return nil
}
@@ -341,7 +349,9 @@ func (iamOS *IAMObjectStore) listAllIAMConfigItems(ctx context.Context) (map[str
// Assumes cache is locked by caller.
func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iamCache) error {
bootstrapTrace("loading all IAM items")
if iamOS.objAPI == nil {
return errServerNotInitialized
}
listedConfigItems, err := iamOS.listAllIAMConfigItems(ctx)
if err != nil {
return err
+72 -22
View File
@@ -33,6 +33,7 @@ import (
"github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/config/identity/openid"
"github.com/minio/minio/internal/jwt"
"github.com/minio/minio/internal/logger"
iampolicy "github.com/minio/pkg/iam/policy"
)
@@ -76,8 +77,13 @@ const (
iamFormatFile = "format.json"
iamFormatVersion1 = 1
minServiceAccountExpiry time.Duration = 15 * time.Minute
maxServiceAccountExpiry time.Duration = 365 * 24 * time.Hour
)
var errInvalidSvcAcctExpiration = errors.New("invalid service account expiration")
type iamFormat struct {
Version int `json:"version"`
}
@@ -394,6 +400,19 @@ func (c *iamCache) policyDBGet(mode UsersSysType, name string, isGroup bool) ([]
return policies, mp.UpdatedAt, nil
}
func (c *iamCache) updateUserWithClaims(key string, u UserIdentity) error {
if u.Credentials.SessionToken != "" {
jwtClaims, err := extractJWTClaims(u)
if err != nil {
return err
}
u.Credentials.Claims = jwtClaims.Map()
}
c.iamUsersMap[key] = u
c.updatedAt = time.Now()
return nil
}
// IAMStorageAPI defines an interface for the IAM persistence layer
type IAMStorageAPI interface {
// The role of the read-write lock is to prevent go routines from
@@ -918,8 +937,7 @@ func (store *IAMStoreSys) PolicyDBUpdate(ctx context.Context, name string, isGro
}
if g.Status == statusDisabled {
// TODO: return an error?
return updatedAt, nil, nil
return updatedAt, nil, errGroupDisabled
}
}
mp = cache.iamGroupPolicyMap[name]
@@ -1749,14 +1767,15 @@ func (store *IAMStoreSys) DeleteUser(ctx context.Context, accessKey string, user
if userType == regUser {
for _, ui := range cache.iamUsersMap {
u := ui.Credentials
if u.IsServiceAccount() && u.ParentUser == accessKey {
_ = store.deleteUserIdentity(ctx, u.AccessKey, svcUser)
delete(cache.iamUsersMap, u.AccessKey)
}
// Delete any associated STS users.
if u.IsTemp() && u.ParentUser == accessKey {
_ = store.deleteUserIdentity(ctx, u.AccessKey, stsUser)
delete(cache.iamUsersMap, u.AccessKey)
if u.ParentUser == accessKey {
switch {
case u.IsServiceAccount():
_ = store.deleteUserIdentity(ctx, u.AccessKey, svcUser)
delete(cache.iamUsersMap, u.AccessKey)
case u.IsTemp():
_ = store.deleteUserIdentity(ctx, u.AccessKey, stsUser)
delete(cache.iamUsersMap, u.AccessKey)
}
}
}
}
@@ -2106,8 +2125,9 @@ func (store *IAMStoreSys) SetUserStatus(ctx context.Context, accessKey string, s
return updatedAt, err
}
cache.iamUsersMap[accessKey] = uinfo
cache.updatedAt = time.Now()
if err := cache.updateUserWithClaims(accessKey, uinfo); err != nil {
return updatedAt, err
}
return uinfo.UpdatedAt, nil
}
@@ -2142,8 +2162,7 @@ func (store *IAMStoreSys) AddServiceAccount(ctx context.Context, cred auth.Crede
return updatedAt, err
}
cache.iamUsersMap[u.Credentials.AccessKey] = u
cache.updatedAt = time.Now()
cache.updateUserWithClaims(u.Credentials.AccessKey, u)
return u.UpdatedAt, nil
}
@@ -2170,6 +2189,14 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
cr.Comment = opts.comment
}
if opts.expiration != nil {
expirationInUTC := opts.expiration.UTC()
if err := validateSvcExpirationInUTC(expirationInUTC); err != nil {
return updatedAt, err
}
cr.Expiration = expirationInUTC
}
switch opts.status {
// The caller did not ask to update status account, do nothing
case "":
@@ -2229,8 +2256,9 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
return updatedAt, err
}
cache.iamUsersMap[u.Credentials.AccessKey] = u
cache.updatedAt = time.Now()
if err := cache.updateUserWithClaims(u.Credentials.AccessKey, u); err != nil {
return updatedAt, err
}
return u.UpdatedAt, nil
}
@@ -2331,8 +2359,9 @@ func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, ureq ma
if err := store.saveUserIdentity(ctx, accessKey, regUser, u); err != nil {
return updatedAt, err
}
cache.iamUsersMap[accessKey] = u
if err := cache.updateUserWithClaims(accessKey, u); err != nil {
return updatedAt, err
}
return u.UpdatedAt, nil
}
@@ -2355,8 +2384,7 @@ func (store *IAMStoreSys) UpdateUserSecretKey(ctx context.Context, accessKey, se
return err
}
cache.iamUsersMap[accessKey] = u
return nil
return cache.updateUserWithClaims(accessKey, u)
}
// GetSTSAndServiceAccounts - returns all STS and Service account credentials.
@@ -2393,8 +2421,8 @@ func (store *IAMStoreSys) UpdateUserIdentity(ctx context.Context, cred auth.Cred
if err := store.saveUserIdentity(ctx, cred.AccessKey, userType, ui); err != nil {
return err
}
cache.iamUsersMap[cred.AccessKey] = ui
return nil
return cache.updateUserWithClaims(cred.AccessKey, ui)
}
// LoadUser - attempts to load user info from storage and updates cache.
@@ -2437,3 +2465,25 @@ func (store *IAMStoreSys) LoadUser(ctx context.Context, accessKey string) {
}
}
}
func extractJWTClaims(u UserIdentity) (*jwt.MapClaims, error) {
jwtClaims, err := auth.ExtractClaims(u.Credentials.SessionToken, u.Credentials.SecretKey)
if err != nil {
// Session tokens for STS creds will be generated with root secret
jwtClaims, err = auth.ExtractClaims(u.Credentials.SessionToken, globalActiveCred.SecretKey)
if err != nil {
return nil, err
}
}
return jwtClaims, nil
}
func validateSvcExpirationInUTC(expirationInUTC time.Time) error {
currentTime := time.Now().UTC()
minExpiration := currentTime.Add(minServiceAccountExpiry)
maxExpiration := currentTime.Add(maxServiceAccountExpiry)
if expirationInUTC.Before(minExpiration) || expirationInUTC.After(maxExpiration) {
return errInvalidSvcAcctExpiration
}
return nil
}
+48 -30
View File
@@ -42,6 +42,7 @@ import (
xldap "github.com/minio/minio/internal/config/identity/ldap"
"github.com/minio/minio/internal/config/identity/openid"
idplugin "github.com/minio/minio/internal/config/identity/plugin"
xtls "github.com/minio/minio/internal/config/identity/tls"
"github.com/minio/minio/internal/config/policy/opa"
polplugin "github.com/minio/minio/internal/config/policy/plugin"
xhttp "github.com/minio/minio/internal/http"
@@ -87,8 +88,10 @@ type IAMSys struct {
sync.Mutex
iamRefreshInterval time.Duration
ldapConfig xldap.Config // only valid if usersSysType is LDAPUsers
openIDConfig openid.Config // only valid if OpenID is configured
LDAPConfig xldap.Config // only valid if usersSysType is LDAPUsers
OpenIDConfig openid.Config // only valid if OpenID is configured
STSTLSConfig xtls.Config // only valid if STS TLS is configured
usersSysType UsersSysType
@@ -163,7 +166,7 @@ func (sys *IAMSys) LoadServiceAccount(ctx context.Context, accessKey string) err
// initStore initializes IAM stores
func (sys *IAMSys) initStore(objAPI ObjectLayer, etcdClient *etcd.Client) {
if sys.ldapConfig.Enabled() {
if sys.LDAPConfig.Enabled() {
sys.SetUsersSysType(LDAPUsersSysType)
}
@@ -213,26 +216,34 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
s := globalServerConfig
globalServerConfigMu.RUnlock()
var err error
globalOpenIDConfig, err = openid.LookupConfig(s,
openidConfig, err := openid.LookupConfig(s,
NewHTTPTransport(), xhttp.DrainBody, globalSite.Region)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err))
}
// Initialize if LDAP is enabled
globalLDAPConfig, err = xldap.Lookup(s, globalRootCAs)
ldapConfig, err := xldap.Lookup(s, globalRootCAs)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to parse LDAP configuration: %w", err))
}
stsTLSConfig, err := xtls.Lookup(s[config.IdentityTLSSubSys][config.Default])
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize X.509/TLS STS API: %w", err))
}
if stsTLSConfig.InsecureSkipVerify {
logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
}
authNPluginCfg, err := idplugin.LookupConfig(s[config.IdentityPluginSubSys][config.Default],
NewHTTPTransport(), xhttp.DrainBody, globalSite.Region)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthNPlugin: %w", err))
}
setGlobalAuthNPlugin(idplugin.New(authNPluginCfg))
setGlobalAuthNPlugin(idplugin.New(GlobalContext, authNPluginCfg))
authZPluginCfg, err := polplugin.LookupConfig(s, GetDefaultConnSettings(), xhttp.DrainBody)
if err != nil {
@@ -257,8 +268,10 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
sys.Lock()
defer sys.Unlock()
sys.ldapConfig = globalLDAPConfig.Clone()
sys.openIDConfig = globalOpenIDConfig.Clone()
sys.LDAPConfig = ldapConfig
sys.OpenIDConfig = openidConfig
sys.STSTLSConfig = stsTLSConfig
sys.iamRefreshInterval = iamRefreshInterval
// Initialize IAM store
@@ -322,7 +335,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
// Set up polling for expired accounts and credentials purging.
switch {
case sys.openIDConfig.ProviderEnabled():
case sys.OpenIDConfig.ProviderEnabled():
go func() {
timer := time.NewTimer(refreshInterval)
defer timer.Stop()
@@ -337,7 +350,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
}
}
}()
case sys.ldapConfig.Enabled():
case sys.LDAPConfig.Enabled():
go func() {
timer := time.NewTimer(refreshInterval)
defer timer.Stop()
@@ -363,7 +376,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
sys.rolesMap = make(map[arn.ARN]string)
// From OpenID
if riMap := globalOpenIDConfig.GetRoleInfo(); riMap != nil {
if riMap := sys.OpenIDConfig.GetRoleInfo(); riMap != nil {
sys.validateAndAddRolePolicyMappings(ctx, riMap)
}
@@ -800,7 +813,7 @@ func (sys *IAMSys) QueryLDAPPolicyEntities(ctx context.Context, q madmin.PolicyE
select {
case <-sys.configLoaded:
pe := sys.store.ListLDAPPolicyMappings(q, sys.ldapConfig.IsLDAPUserDN, sys.ldapConfig.IsLDAPGroupDN)
pe := sys.store.ListLDAPPolicyMappings(q, sys.LDAPConfig.IsLDAPUserDN, sys.LDAPConfig.IsLDAPGroupDN)
pe.Timestamp = UTCNow()
return &pe, nil
case <-ctx.Done():
@@ -920,6 +933,7 @@ type newServiceAccountOpts struct {
accessKey string
secretKey string
comment string
expiration *time.Time
claims map[string]interface{}
}
@@ -992,6 +1006,14 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
cred.Status = string(auth.AccountOn)
cred.Comment = opts.comment
if opts.expiration != nil {
expirationInUTC := opts.expiration.UTC()
if err := validateSvcExpirationInUTC(expirationInUTC); err != nil {
return auth.Credentials{}, time.Time{}, err
}
cred.Expiration = expirationInUTC
}
updatedAt, err := sys.store.AddServiceAccount(ctx, cred)
if err != nil {
return auth.Credentials{}, time.Time{}, err
@@ -1006,6 +1028,7 @@ type updateServiceAccountOpts struct {
secretKey string
status string
comment string
expiration *time.Time
}
// UpdateServiceAccount - edit a service account
@@ -1145,12 +1168,9 @@ func (sys *IAMSys) getAccountWithClaims(ctx context.Context, accessKey string) (
return UserIdentity{}, nil, errNoSuchAccount
}
jwtClaims, err := auth.ExtractClaims(acc.Credentials.SessionToken, acc.Credentials.SecretKey)
jwtClaims, err := extractJWTClaims(acc)
if err != nil {
jwtClaims, err = auth.ExtractClaims(acc.Credentials.SessionToken, globalActiveCred.SecretKey)
if err != nil {
return UserIdentity{}, nil, err
}
return UserIdentity{}, nil, err
}
return acc, jwtClaims, nil
@@ -1171,13 +1191,11 @@ func (sys *IAMSys) GetClaimsForSvcAcc(ctx context.Context, accessKey string) (ma
return nil, errNoSuchServiceAccount
}
jwtClaims, err := auth.ExtractClaims(sa.Credentials.SessionToken, sa.Credentials.SecretKey)
jwtClaims, err := extractJWTClaims(sa)
if err != nil {
jwtClaims, err = auth.ExtractClaims(sa.Credentials.SessionToken, globalActiveCred.SecretKey)
if err != nil {
return nil, err
}
return nil, err
}
return jwtClaims.Map(), nil
}
@@ -1280,7 +1298,7 @@ func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) {
continue
}
roleArn = roleArns[0]
u, err := sys.openIDConfig.LookupUser(roleArn, puInfo.subClaimValue)
u, err := sys.OpenIDConfig.LookupUser(roleArn, puInfo.subClaimValue)
if err != nil {
logger.LogIf(GlobalContext, err)
continue
@@ -1302,14 +1320,14 @@ func (sys *IAMSys) purgeExpiredCredentialsForLDAP(ctx context.Context) {
parentUsers := sys.store.GetAllParentUsers()
var allDistNames []string
for parentUser := range parentUsers {
if !sys.ldapConfig.IsLDAPUserDN(parentUser) {
if !sys.LDAPConfig.IsLDAPUserDN(parentUser) {
continue
}
allDistNames = append(allDistNames, parentUser)
}
expiredUsers, err := sys.ldapConfig.GetNonEligibleUserDistNames(allDistNames)
expiredUsers, err := sys.LDAPConfig.GetNonEligibleUserDistNames(allDistNames)
if err != nil {
// Log and return on error - perhaps it'll work the next time.
logger.LogIf(GlobalContext, err)
@@ -1331,7 +1349,7 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
// DN to ldap username mapping for each LDAP user
parentUserToLDAPUsernameMap := make(map[string]string)
for _, cred := range allCreds {
if !sys.ldapConfig.IsLDAPUserDN(cred.ParentUser) {
if !sys.LDAPConfig.IsLDAPUserDN(cred.ParentUser) {
continue
}
// Check if this is the first time we are
@@ -1379,7 +1397,7 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
}
// 2. Query LDAP server for groups of the LDAP users collected.
updatedGroups, err := sys.ldapConfig.LookupGroupMemberships(parentUsers, parentUserToLDAPUsernameMap)
updatedGroups, err := sys.LDAPConfig.LookupGroupMemberships(parentUsers, parentUserToLDAPUsernameMap)
if err != nil {
// Log and return on error - perhaps it'll work the next time.
logger.LogIf(GlobalContext, err)
@@ -1566,7 +1584,7 @@ func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool,
var dn string
var isGroup bool
if r.User != "" {
dn, err = globalLDAPConfig.DoesUsernameExist(r.User)
dn, err = sys.LDAPConfig.DoesUsernameExist(r.User)
if err != nil {
logger.LogIf(ctx, err)
return updatedAt, nil, err
@@ -1576,7 +1594,7 @@ func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool,
}
isGroup = false
} else {
if exists, err := globalLDAPConfig.DoesGroupDNExist(r.Group); err != nil {
if exists, err := sys.LDAPConfig.DoesGroupDNExist(r.Group); err != nil {
logger.LogIf(ctx, err)
return updatedAt, nil, err
} else if !exists {
+1 -1
View File
@@ -419,7 +419,7 @@ func Test_localLocker_RUnlock(t *testing.T) {
}
start = time.Now()
for _, lock := range toUnLock {
ok, err := l.RUnlock(nil, lock)
ok, err := l.RUnlock(context.TODO(), lock)
if err != nil || !ok {
t.Fatal(err)
}
+4 -3
View File
@@ -816,15 +816,16 @@ func (m *metaCacheEntriesSorted) merge(other metaCacheEntriesSorted, limit int)
a := m.entries()
b := other.entries()
for len(a) > 0 && len(b) > 0 {
if a[0].name == b[0].name && bytes.Equal(a[0].metadata, b[0].metadata) {
switch {
case a[0].name == b[0].name && bytes.Equal(a[0].metadata, b[0].metadata):
// Same, discard one.
merged = append(merged, a[0])
a = a[1:]
b = b[1:]
} else if a[0].name < b[0].name {
case a[0].name < b[0].name:
merged = append(merged, a[0])
a = a[1:]
} else {
default:
merged = append(merged, b[0])
b = b[1:]
}
+48 -26
View File
@@ -28,7 +28,6 @@ import (
"sync"
"time"
"github.com/minio/minio/internal/bucket/lifecycle"
"github.com/minio/minio/internal/logger"
)
@@ -341,7 +340,6 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
func applyBucketActions(ctx context.Context, o listPathOptions, in <-chan metaCacheEntry, out chan<- metaCacheEntry) {
defer close(out)
vcfg, _ := globalBucketVersioningSys.Get(o.Bucket)
for {
var obj metaCacheEntry
var ok bool
@@ -354,37 +352,61 @@ func applyBucketActions(ctx context.Context, o listPathOptions, in <-chan metaCa
}
}
fi, err := obj.fileInfo(o.Bucket)
var skip bool
versioned := o.Versioning != nil && o.Versioning.Versioned(obj.name)
// skip latest object from listing only for regular
// listObjects calls, versioned based listing cannot
// filter out between versions 'obj' cannot be truncated
// in such a manner, so look for skipping an object only
// for regular ListObjects() call only.
if !o.Versioned {
fi, err := obj.fileInfo(o.Bucket)
if err != nil {
continue
}
objInfo := fi.ToObjectInfo(o.Bucket, obj.name, versioned)
if o.Lifecycle != nil {
act := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, objInfo).Action
skip = act.Delete()
if act.DeleteRestored() {
// do not skip DeleteRestored* actions
skip = false
}
}
}
// Skip entry only if needed via ILM, skipping is never true for versioned listing.
if !skip {
select {
case <-ctx.Done():
return
case out <- obj:
}
}
fiv, err := obj.fileInfoVersions(o.Bucket)
if err != nil {
continue
}
versioned := vcfg != nil && vcfg.Versioned(obj.name)
// Expire all versions if needed, if not attempt to queue for replication.
for _, version := range fiv.Versions {
objInfo := version.ToObjectInfo(o.Bucket, obj.name, versioned)
objInfo := fi.ToObjectInfo(o.Bucket, obj.name, versioned)
if o.Lifecycle != nil {
evt := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, objInfo)
switch evt.Action {
case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
globalExpiryState.enqueueByDays(objInfo, false, evt.Action == lifecycle.DeleteVersionAction)
// Skip this entry.
continue
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
globalExpiryState.enqueueByDays(objInfo, true, evt.Action == lifecycle.DeleteRestoredVersionAction)
// Skip this entry.
continue
}
}
select {
case <-ctx.Done():
return
case out <- obj:
if fiv, err := obj.fileInfoVersions(o.Bucket); err == nil {
for _, version := range fiv.Versions {
objInfo := version.ToObjectInfo(o.Bucket, obj.name, versioned)
queueReplicationHeal(ctx, o.Bucket, objInfo, o.Replication)
if o.Lifecycle != nil {
evt := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, objInfo)
if evt.Action.Delete() {
globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
if !evt.Action.DeleteRestored() {
continue
} // queue version for replication upon expired restored copies if needed.
}
}
queueReplicationHeal(ctx, o.Bucket, objInfo, o.Replication)
}
}
}
+8 -2
View File
@@ -34,6 +34,7 @@ import (
jsoniter "github.com/json-iterator/go"
"github.com/minio/minio/internal/bucket/lifecycle"
"github.com/minio/minio/internal/bucket/object/lock"
"github.com/minio/minio/internal/bucket/versioning"
"github.com/minio/minio/internal/color"
"github.com/minio/minio/internal/hash"
"github.com/minio/minio/internal/logger"
@@ -95,6 +96,10 @@ type listPathOptions struct {
// Versioned is this a ListObjectVersions call.
Versioned bool
// Versioning config is used for if the path
// has versioning enabled.
Versioning *versioning.Versioning
// Lifecycle performs filtering based on lifecycle.
// This will filter out objects if the most recent version should be deleted by lifecycle.
// Is not transferred across request calls.
@@ -119,12 +124,14 @@ func init() {
func (o *listPathOptions) setBucketMeta(ctx context.Context) {
lc, _ := globalLifecycleSys.Get(o.Bucket)
vc, _ := globalBucketVersioningSys.Get(o.Bucket)
// Check if bucket is object locked.
rcfg, _ := globalBucketObjectLockSys.Get(o.Bucket)
replCfg, _, _ := globalBucketMetadataSys.GetReplicationConfig(ctx, o.Bucket)
tgts, _ := globalBucketTargetSys.ListBucketTargets(ctx, o.Bucket)
o.Lifecycle = lc
o.Versioning = vc
o.Replication = replicationConfig{
Config: replCfg,
remotes: tgts,
@@ -870,8 +877,7 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
var fdMu sync.Mutex
fds := opts.fallbackDisks
fallback := func(err error) StorageAPI {
switch err.(type) {
case StorageErr:
if _, ok := err.(StorageErr); ok {
// Attempt to grab a fallback disk
fdMu.Lock()
defer fdMu.Unlock()
+1 -2
View File
@@ -562,8 +562,7 @@ func (r *metacacheReader) readAll(ctx context.Context, dst chan<- metaCacheEntry
}
for {
if more, err := r.mr.ReadBool(); !more {
switch err {
case io.EOF:
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
r.err = err
+125 -10
View File
@@ -131,6 +131,7 @@ const (
iamSubsystem MetricSubsystem = "iam"
kmsSubsystem MetricSubsystem = "kms"
notifySubsystem MetricSubsystem = "notify"
lambdaSubsystem MetricSubsystem = "lambda"
auditSubsystem MetricSubsystem = "audit"
)
@@ -182,8 +183,9 @@ const (
usageInfo MetricName = "usage_info"
versionInfo MetricName = "version_info"
sizeDistribution = "size_distribution"
ttfbDistribution = "ttfb_seconds_distribution"
sizeDistribution = "size_distribution"
versionDistribution = "version_distribution"
ttfbDistribution = "ttfb_seconds_distribution"
lastActivityTime = "last_activity_nano_seconds"
startTime = "starttime_seconds"
@@ -564,6 +566,16 @@ func getBucketObjectDistributionMD() MetricDescription {
}
}
func getBucketObjectVersionsMD() MetricDescription {
return MetricDescription{
Namespace: bucketMetricNamespace,
Subsystem: objectsSubsystem,
Name: versionDistribution,
Help: "Distribution of object sizes in the bucket, includes label for the bucket name",
Type: histogramMetric,
}
}
func getInternodeFailedRequests() MetricDescription {
return MetricDescription{
Namespace: interNodeMetricNamespace,
@@ -1385,6 +1397,7 @@ func getIAMNodeMetrics() *MetricsGroup {
sinceLastSyncMillis = (uint64(time.Now().UnixNano()) - lastSyncTime) / uint64(time.Millisecond)
}
pluginAuthNMetrics := globalAuthNPlugin.Metrics()
metrics = []Metric{
{
Description: MetricDescription{
@@ -1426,7 +1439,68 @@ func getIAMNodeMetrics() *MetricsGroup {
},
Value: float64(atomic.LoadUint64(&globalIAMSys.TotalRefreshFailures)),
},
{
Description: MetricDescription{
Namespace: nodeMetricNamespace,
Subsystem: iamSubsystem,
Name: "plugin_authn_service_last_succ_seconds",
Help: "When plugin authentication is configured, returns time (in seconds) since the last successful request to the service",
Type: gaugeMetric,
},
Value: pluginAuthNMetrics.LastReachableSecs,
},
{
Description: MetricDescription{
Namespace: nodeMetricNamespace,
Subsystem: iamSubsystem,
Name: "plugin_authn_service_last_fail_seconds",
Help: "When plugin authentication is configured, returns time (in seconds) since the last failed request to the service",
Type: gaugeMetric,
},
Value: pluginAuthNMetrics.LastUnreachableSecs,
},
{
Description: MetricDescription{
Namespace: nodeMetricNamespace,
Subsystem: iamSubsystem,
Name: "plugin_authn_service_total_requests_minute",
Help: "When plugin authentication is configured, returns total requests count in the last full minute",
Type: gaugeMetric,
},
Value: float64(pluginAuthNMetrics.TotalRequests),
},
{
Description: MetricDescription{
Namespace: nodeMetricNamespace,
Subsystem: iamSubsystem,
Name: "plugin_authn_service_failed_requests_minute",
Help: "When plugin authentication is configured, returns failed requests count in the last full minute",
Type: gaugeMetric,
},
Value: float64(pluginAuthNMetrics.FailedRequests),
},
{
Description: MetricDescription{
Namespace: nodeMetricNamespace,
Subsystem: iamSubsystem,
Name: "plugin_authn_service_succ_avg_rtt_ms_minute",
Help: "When plugin authentication is configured, returns average round-trip-time of successful requests in the last full minute",
Type: gaugeMetric,
},
Value: pluginAuthNMetrics.AvgSuccRTTMs,
},
{
Description: MetricDescription{
Namespace: nodeMetricNamespace,
Subsystem: iamSubsystem,
Name: "plugin_authn_service_succ_max_rtt_ms_minute",
Help: "When plugin authentication is configured, returns maximum round-trip-time of successful requests in the last full minute",
Type: gaugeMetric,
},
Value: pluginAuthNMetrics.MaxSuccRTTMs,
},
}
return metrics
})
return mg
@@ -1594,8 +1668,8 @@ func getNotificationMetrics() *MetricsGroup {
cacheInterval: 10 * time.Second,
}
mg.RegisterRead(func(ctx context.Context) []Metric {
stats := globalConfigTargetList.Stats()
metrics := make([]Metric, 0, 1+len(stats.TargetStats))
nstats := globalNotifyTargetList.Stats()
metrics := make([]Metric, 0, 1+len(nstats.TargetStats))
metrics = append(metrics, Metric{
Description: MetricDescription{
Namespace: minioNamespace,
@@ -1604,9 +1678,9 @@ func getNotificationMetrics() *MetricsGroup {
Help: "Number of concurrent async Send calls active to all targets",
Type: gaugeMetric,
},
Value: float64(stats.CurrentSendCalls),
Value: float64(nstats.CurrentSendCalls),
})
for _, st := range stats.TargetStats {
for _, st := range nstats.TargetStats {
metrics = append(metrics, Metric{
Description: MetricDescription{
Namespace: minioNamespace,
@@ -1619,6 +1693,43 @@ func getNotificationMetrics() *MetricsGroup {
Value: float64(st.CurrentQueue),
})
}
lstats := globalLambdaTargetList.Stats()
for _, st := range lstats.TargetStats {
metrics = append(metrics, Metric{
Description: MetricDescription{
Namespace: minioNamespace,
Subsystem: lambdaSubsystem,
Name: "active_requests",
Help: "Number of in progress requests",
},
VariableLabels: map[string]string{"target_id": st.ID.ID, "target_name": st.ID.Name},
Value: float64(st.ActiveRequests),
})
metrics = append(metrics, Metric{
Description: MetricDescription{
Namespace: minioNamespace,
Subsystem: lambdaSubsystem,
Name: "total_requests",
Help: "Total number of requests sent since start",
Type: counterMetric,
},
VariableLabels: map[string]string{"target_id": st.ID.ID, "target_name": st.ID.Name},
Value: float64(st.TotalRequests),
})
metrics = append(metrics, Metric{
Description: MetricDescription{
Namespace: minioNamespace,
Subsystem: lambdaSubsystem,
Name: "failed_requests",
Help: "Total number of requests that failed to send since start",
Type: counterMetric,
},
VariableLabels: map[string]string{"target_id": st.ID.ID, "target_name": st.ID.Name},
Value: float64(st.FailedRequests),
})
}
// Audit and system:
audit := logger.CurrentStats()
for id, st := range audit {
@@ -1899,6 +2010,12 @@ func getBucketUsageMetrics() *MetricsGroup {
HistogramBucketLabel: "range",
VariableLabels: map[string]string{"bucket": bucket},
})
metrics = append(metrics, Metric{
Description: getBucketObjectVersionsMD(),
Histogram: usage.ObjectVersionsHistogram,
HistogramBucketLabel: "range",
VariableLabels: map[string]string{"bucket": bucket},
})
}
return
})
@@ -2260,8 +2377,7 @@ func (c *minioClusterCollector) Collect(out chan<- prometheus.Metric) {
continue
}
metricType := prometheus.GaugeValue
switch metric.Description.Type {
case counterMetric:
if metric.Description.Type == counterMetric {
metricType = prometheus.CounterValue
}
toPost := prometheus.MustNewConstMetric(
@@ -2369,8 +2485,7 @@ func (c *minioNodeCollector) Collect(ch chan<- prometheus.Metric) {
}
metricType := prometheus.GaugeValue
switch metric.Description.Type {
case counterMetric:
if metric.Description.Type == counterMetric {
metricType = prometheus.CounterValue
}
ch <- prometheus.MustNewConstMetric(
+12
View File
@@ -441,6 +441,18 @@ func bucketUsageMetricsPrometheus(ch chan<- prometheus.Metric) {
k,
)
}
for k, v := range usageInfo.ObjectVersionsHistogram {
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName(bucketNamespace, "objects", "histogram"),
"Total number of versions of objects in a bucket",
[]string{"bucket", "object_versions"}, nil),
prometheus.GaugeValue,
float64(v),
bucket,
k,
)
}
}
}
+6 -10
View File
@@ -170,9 +170,8 @@ func (di *distLockInstance) GetLock(ctx context.Context, timeout *dynamicTimeout
RetryInterval: timeout.RetryInterval(),
}) {
timeout.LogFailure()
cancel()
switch err := newCtx.Err(); err {
case context.Canceled:
defer cancel()
if err := newCtx.Err(); err == context.Canceled {
return LockContext{ctx: ctx, cancel: func() {}}, err
}
return LockContext{ctx: ctx, cancel: func() {}}, OperationTimedOut{}
@@ -200,9 +199,8 @@ func (di *distLockInstance) GetRLock(ctx context.Context, timeout *dynamicTimeou
RetryInterval: timeout.RetryInterval(),
}) {
timeout.LogFailure()
cancel()
switch err := newCtx.Err(); err {
case context.Canceled:
defer cancel()
if err := newCtx.Err(); err == context.Canceled {
return LockContext{ctx: ctx, cancel: func() {}}, err
}
return LockContext{ctx: ctx, cancel: func() {}}, OperationTimedOut{}
@@ -257,8 +255,7 @@ func (li *localLockInstance) GetLock(ctx context.Context, timeout *dynamicTimeou
li.ns.unlock(li.volume, li.paths[si], readLock)
}
}
switch err := ctx.Err(); err {
case context.Canceled:
if err := ctx.Err(); err == context.Canceled {
return LockContext{}, err
}
return LockContext{}, OperationTimedOut{}
@@ -294,8 +291,7 @@ func (li *localLockInstance) GetRLock(ctx context.Context, timeout *dynamicTimeo
li.ns.unlock(li.volume, li.paths[si], readLock)
}
}
switch err := ctx.Err(); err {
case context.Canceled:
if err := ctx.Err(); err == context.Canceled {
return LockContext{}, err
}
return LockContext{}, OperationTimedOut{}
+12 -9
View File
@@ -138,13 +138,14 @@ func TestGetHostIP(t *testing.T) {
for _, testCase := range testCases {
ipList, err := getHostIP(testCase.host)
if testCase.expectedErr == nil {
switch {
case testCase.expectedErr == nil:
if err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
}
} else if err == nil {
case err == nil:
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
} else if testCase.expectedErr.Error() != err.Error() {
case testCase.expectedErr.Error() != err.Error():
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
}
@@ -221,13 +222,14 @@ func TestCheckPortAvailability(t *testing.T) {
}
err := checkPortAvailability(testCase.host, testCase.port)
if testCase.expectedErr == nil {
switch {
case testCase.expectedErr == nil:
if err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
}
} else if err == nil {
case err == nil:
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
} else if testCase.expectedErr.Error() != err.Error() {
case testCase.expectedErr.Error() != err.Error():
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
}
}
@@ -252,13 +254,14 @@ func TestCheckLocalServerAddr(t *testing.T) {
testCase := testCase
t.Run("", func(t *testing.T) {
err := CheckLocalServerAddr(testCase.serverAddr)
if testCase.expectedErr == nil {
switch {
case testCase.expectedErr == nil:
if err != nil {
t.Errorf("error: expected = <nil>, got = %v", err)
}
} else if err == nil {
case err == nil:
t.Errorf("error: expected = %v, got = <nil>", testCase.expectedErr)
} else if testCase.expectedErr.Error() != err.Error() {
case testCase.expectedErr.Error() != err.Error():
t.Errorf("error: expected = %v, got = %v", testCase.expectedErr, err)
}
})
+26 -89
View File
@@ -18,9 +18,7 @@
package cmd
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
@@ -29,7 +27,6 @@ import (
"sync"
"time"
"github.com/bits-and-blooms/bloom/v3"
"github.com/cespare/xxhash/v2"
"github.com/klauspost/compress/zip"
"github.com/minio/madmin-go/v2"
@@ -59,14 +56,23 @@ type NotificationPeerErr struct {
//
// A zero NotificationGroup is valid and does not cancel on error.
type NotificationGroup struct {
wg sync.WaitGroup
errs []NotificationPeerErr
wg sync.WaitGroup
errs []NotificationPeerErr
retryCount int
}
// WithNPeers returns a new NotificationGroup with length of errs slice upto nerrs,
// upon Wait() errors are returned collected from all tasks.
func WithNPeers(nerrs int) *NotificationGroup {
return &NotificationGroup{errs: make([]NotificationPeerErr, nerrs)}
return &NotificationGroup{errs: make([]NotificationPeerErr, nerrs), retryCount: 3}
}
// WithRetries sets the retry count for all function calls from the Go method.
func (g *NotificationGroup) WithRetries(retryCount int) *NotificationGroup {
if g != nil {
g.retryCount = retryCount
}
return g
}
// Wait blocks until all function calls from the Go method have returned, then
@@ -88,17 +94,17 @@ func (g *NotificationGroup) Go(ctx context.Context, f func() error, index int, a
g.errs[index] = NotificationPeerErr{
Host: addr,
}
for i := 0; i < 3; i++ {
for i := 0; i < g.retryCount; i++ {
if err := f(); err != nil {
g.errs[index].Err = err
// Last iteration log the error.
if i == 2 {
if i == g.retryCount-1 {
reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
ctx := logger.SetReqInfo(ctx, reqInfo)
logger.LogIf(ctx, err)
}
// Wait for one second and no need wait after last attempt.
if i < 2 {
if i < g.retryCount-1 {
time.Sleep(1 * time.Second)
}
continue
@@ -110,7 +116,7 @@ func (g *NotificationGroup) Go(ctx context.Context, f func() error, index int, a
// DeletePolicy - deletes policy across all peers.
func (sys *NotificationSys) DeletePolicy(policyName string) []NotificationPeerErr {
ng := WithNPeers(len(sys.peerClients))
ng := WithNPeers(len(sys.peerClients)).WithRetries(1)
for idx, client := range sys.peerClients {
if client == nil {
continue
@@ -125,7 +131,7 @@ func (sys *NotificationSys) DeletePolicy(policyName string) []NotificationPeerEr
// LoadPolicy - reloads a specific modified policy across all peers
func (sys *NotificationSys) LoadPolicy(policyName string) []NotificationPeerErr {
ng := WithNPeers(len(sys.peerClients))
ng := WithNPeers(len(sys.peerClients)).WithRetries(1)
for idx, client := range sys.peerClients {
if client == nil {
continue
@@ -140,7 +146,7 @@ func (sys *NotificationSys) LoadPolicy(policyName string) []NotificationPeerErr
// LoadPolicyMapping - reloads a policy mapping across all peers
func (sys *NotificationSys) LoadPolicyMapping(userOrGroup string, userType IAMUserType, isGroup bool) []NotificationPeerErr {
ng := WithNPeers(len(sys.peerClients))
ng := WithNPeers(len(sys.peerClients)).WithRetries(1)
for idx, client := range sys.peerClients {
if client == nil {
continue
@@ -155,7 +161,7 @@ func (sys *NotificationSys) LoadPolicyMapping(userOrGroup string, userType IAMUs
// DeleteUser - deletes a specific user across all peers
func (sys *NotificationSys) DeleteUser(accessKey string) []NotificationPeerErr {
ng := WithNPeers(len(sys.peerClients))
ng := WithNPeers(len(sys.peerClients)).WithRetries(1)
for idx, client := range sys.peerClients {
if client == nil {
continue
@@ -170,7 +176,7 @@ func (sys *NotificationSys) DeleteUser(accessKey string) []NotificationPeerErr {
// LoadUser - reloads a specific user across all peers
func (sys *NotificationSys) LoadUser(accessKey string, temp bool) []NotificationPeerErr {
ng := WithNPeers(len(sys.peerClients))
ng := WithNPeers(len(sys.peerClients)).WithRetries(1)
for idx, client := range sys.peerClients {
if client == nil {
continue
@@ -185,7 +191,7 @@ func (sys *NotificationSys) LoadUser(accessKey string, temp bool) []Notification
// LoadGroup - loads a specific group on all peers.
func (sys *NotificationSys) LoadGroup(group string) []NotificationPeerErr {
ng := WithNPeers(len(sys.peerClients))
ng := WithNPeers(len(sys.peerClients)).WithRetries(1)
for idx, client := range sys.peerClients {
if client == nil {
continue
@@ -198,7 +204,7 @@ func (sys *NotificationSys) LoadGroup(group string) []NotificationPeerErr {
// DeleteServiceAccount - deletes a specific service account across all peers
func (sys *NotificationSys) DeleteServiceAccount(accessKey string) []NotificationPeerErr {
ng := WithNPeers(len(sys.peerClients))
ng := WithNPeers(len(sys.peerClients)).WithRetries(1)
for idx, client := range sys.peerClients {
if client == nil {
continue
@@ -213,7 +219,7 @@ func (sys *NotificationSys) DeleteServiceAccount(accessKey string) []Notificatio
// LoadServiceAccount - reloads a specific service account across all peers
func (sys *NotificationSys) LoadServiceAccount(accessKey string) []NotificationPeerErr {
ng := WithNPeers(len(sys.peerClients))
ng := WithNPeers(len(sys.peerClients)).WithRetries(1)
for idx, client := range sys.peerClients {
if client == nil {
continue
@@ -384,75 +390,6 @@ func (sys *NotificationSys) SignalService(sig serviceSignal) []NotificationPeerE
return ng.Wait()
}
// updateBloomFilter will cycle all servers to the current index and
// return a merged bloom filter if a complete one can be retrieved.
func (sys *NotificationSys) updateBloomFilter(ctx context.Context, current uint64) (*bloomFilter, error) {
req := bloomFilterRequest{
Current: current,
Oldest: current - dataUsageUpdateDirCycles,
}
if current < dataUsageUpdateDirCycles {
req.Oldest = 0
}
// Load initial state from local...
var bf *bloomFilter
bfr, err := intDataUpdateTracker.cycleFilter(ctx, req)
logger.LogIf(ctx, err)
if err == nil && bfr.Complete {
nbf := intDataUpdateTracker.newBloomFilter()
bf = &nbf
_, err = bf.ReadFrom(bytes.NewReader(bfr.Filter))
logger.LogIf(ctx, err)
}
var mu sync.Mutex
g := errgroup.WithNErrs(len(sys.peerClients))
for idx, client := range sys.peerClients {
if client == nil {
continue
}
client := client
g.Go(func() error {
serverBF, err := client.cycleServerBloomFilter(ctx, req)
if false && intDataUpdateTracker.debug {
b, _ := json.MarshalIndent(serverBF, "", " ")
logger.Info("Drive %v, Bloom filter: %v", client.host.Name, string(b))
}
// Keep lock while checking result.
mu.Lock()
defer mu.Unlock()
if err != nil || !serverBF.Complete || bf == nil {
logger.LogOnceIf(ctx, err, client.host.String(), client.cycleServerBloomFilter)
bf = nil
return nil
}
var tmp bloom.BloomFilter
_, err = tmp.ReadFrom(bytes.NewReader(serverBF.Filter))
if err != nil {
logger.LogIf(ctx, err)
bf = nil
return nil
}
if bf.BloomFilter == nil {
bf.BloomFilter = &tmp
} else {
err = bf.Merge(&tmp)
if err != nil {
logger.LogIf(ctx, err)
bf = nil
return nil
}
}
return nil
}, idx)
}
g.Wait()
return bf, nil
}
var errPeerNotReachable = errors.New("peer is not reachable")
// GetLocks - makes GetLocks RPC call on all peers.
@@ -540,7 +477,7 @@ func (sys *NotificationSys) DeleteBucketMetadata(ctx context.Context, bucketName
// GetClusterAllBucketStats - returns bucket stats for all buckets from all remote peers.
func (sys *NotificationSys) GetClusterAllBucketStats(ctx context.Context) []BucketStatsMap {
ng := WithNPeers(len(sys.peerClients))
ng := WithNPeers(len(sys.peerClients)).WithRetries(1)
replicationStats := make([]BucketStatsMap, len(sys.peerClients))
for index, client := range sys.peerClients {
index := index
@@ -581,7 +518,7 @@ func (sys *NotificationSys) GetClusterAllBucketStats(ctx context.Context) []Buck
// GetClusterBucketStats - calls GetClusterBucketStats call on all peers for a cluster statistics view.
func (sys *NotificationSys) GetClusterBucketStats(ctx context.Context, bucketName string) []BucketStats {
ng := WithNPeers(len(sys.peerClients))
ng := WithNPeers(len(sys.peerClients)).WithRetries(1)
bucketStats := make([]BucketStats, len(sys.peerClients))
for index, client := range sys.peerClients {
index := index
@@ -1440,7 +1377,7 @@ func (sys *NotificationSys) GetLastDayTierStats(ctx context.Context) DailyAllTie
merged := globalTransitionState.getDailyAllTierStats()
for i, stat := range lastDayStats {
if errs[i] != nil {
logger.LogIf(ctx, fmt.Errorf("failed to fetch last day tier stats: %w", errs[i]))
logger.LogOnceIf(ctx, fmt.Errorf("failed to fetch last day tier stats: %w", errs[i]), sys.peerClients[i].host.String())
continue
}
merged.merge(stat)
+15 -2
View File
@@ -54,12 +54,13 @@ type objectHistogramInterval struct {
const (
// dataUsageBucketLen must be length of ObjectsHistogramIntervals
dataUsageBucketLen = 7
dataUsageBucketLen = 7
dataUsageVersionLen = 7
)
// ObjectsHistogramIntervals is the list of all intervals
// of object sizes to be included in objects histogram.
var ObjectsHistogramIntervals = []objectHistogramInterval{
var ObjectsHistogramIntervals = [dataUsageBucketLen]objectHistogramInterval{
{"LESS_THAN_1024_B", 0, humanize.KiByte - 1},
{"BETWEEN_1024_B_AND_1_MB", humanize.KiByte, humanize.MiByte - 1},
{"BETWEEN_1_MB_AND_10_MB", humanize.MiByte, humanize.MiByte*10 - 1},
@@ -69,6 +70,18 @@ var ObjectsHistogramIntervals = []objectHistogramInterval{
{"GREATER_THAN_512_MB", humanize.MiByte * 512, math.MaxInt64},
}
// ObjectsVersionCountIntervals is the list of all intervals
// of object version count to be included in objects histogram.
var ObjectsVersionCountIntervals = [dataUsageVersionLen]objectHistogramInterval{
{"UNVERSIONED", 0, 0},
{"SINGLE_VERSION", 1, 1},
{"BETWEEN_2_AND_10", 2, 9},
{"BETWEEN_10_AND_100", 10, 99},
{"BETWEEN_100_AND_1000", 100, 999},
{"BETWEEN_1000_AND_10000", 1000, 9999},
{"GREATER_THAN_10000", 10000, math.MaxInt64},
}
// BucketInfo - represents bucket metadata.
type BucketInfo struct {
// Name of the bucket.
+1 -1
View File
@@ -195,7 +195,7 @@ type ObjectLayer interface {
// Storage operations.
Shutdown(context.Context) error
NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32, scanMode madmin.HealScanMode) error
NSScanner(ctx context.Context, updates chan<- DataUsageInfo, wantCycle uint32, scanMode madmin.HealScanMode) error
BackendInfo() madmin.BackendInfo
StorageInfo(ctx context.Context) StorageInfo
LocalStorageInfo(ctx context.Context) StorageInfo
+21 -30
View File
@@ -472,22 +472,18 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
}
if !proxy.Proxy { // apply lifecycle rules only for local requests
// Automatically remove the object/version is an expiry lifecycle rule can be applied
// Automatically remove the object/version if an expiry lifecycle rule can be applied
if lc, err := globalLifecycleSys.Get(bucket); err == nil {
rcfg, _ := globalBucketObjectLockSys.Get(bucket)
evt := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo)
var success bool
switch evt.Action {
case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
success = applyExpiryRule(objInfo, false, evt.Action == lifecycle.DeleteVersionAction)
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
// Restored object delete would be still allowed to proceed as success
// since transition behavior is slightly different.
applyExpiryRule(objInfo, true, evt.Action == lifecycle.DeleteRestoredVersionAction)
}
if success {
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey))
return
act := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo).Action
if act.Delete() {
// apply whatever the expiry rule is.
applyExpiryRule(objInfo, act.DeleteRestored(), act.DeleteVersioned())
if !act.DeleteRestored() {
// If the ILM action is not on restored object return error.
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey))
return
}
}
}
@@ -729,22 +725,18 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
}
if !proxy.Proxy { // apply lifecycle rules only locally not for proxied requests
// Automatically remove the object/version is an expiry lifecycle rule can be applied
// Automatically remove the object/version if an expiry lifecycle rule can be applied
if lc, err := globalLifecycleSys.Get(bucket); err == nil {
rcfg, _ := globalBucketObjectLockSys.Get(bucket)
evt := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo)
var success bool
switch evt.Action {
case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
success = applyExpiryRule(objInfo, false, evt.Action == lifecycle.DeleteVersionAction)
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
// Restored object delete would be still allowed to proceed as success
// since transition behavior is slightly different.
applyExpiryRule(objInfo, true, evt.Action == lifecycle.DeleteRestoredVersionAction)
}
if success {
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey))
return
act := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo).Action
if act.Delete() {
// apply whatever the expiry rule is.
applyExpiryRule(objInfo, act.DeleteRestored(), act.DeleteVersioned())
if !act.DeleteRestored() {
// If the ILM action is not on restored object return error.
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey))
return
}
}
}
QueueReplicationHeal(ctx, bucket, objInfo)
@@ -2368,8 +2360,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html
objInfo, err := deleteObject(ctx, bucket, object, opts)
if err != nil {
switch err.(type) {
case BucketNotFound:
if _, ok := err.(BucketNotFound); ok {
// When bucket doesn't exist specially handle it.
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
+13 -15
View File
@@ -1197,17 +1197,18 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam
rec := httptest.NewRecorder()
// construct HTTP request for Put Object end point.
var req *http.Request
if testCase.fault == chunkDateMismatch {
switch {
case testCase.fault == chunkDateMismatch:
req, err = newTestStreamingSignedBadChunkDateRequest(http.MethodPut,
getPutObjectURL("", testCase.bucketName, testCase.objectName),
int64(testCase.dataLen), testCase.chunkSize, bytes.NewReader(testCase.data),
testCase.accessKey, testCase.secretKey)
} else if testCase.contentEncoding == "" {
case testCase.contentEncoding == "":
req, err = newTestStreamingSignedRequest(http.MethodPut,
getPutObjectURL("", testCase.bucketName, testCase.objectName),
int64(testCase.dataLen), testCase.chunkSize, bytes.NewReader(testCase.data),
testCase.accessKey, testCase.secretKey)
} else if testCase.contentEncoding != "" {
case testCase.contentEncoding != "":
req, err = newTestStreamingSignedCustomEncodingRequest(http.MethodPut,
getPutObjectURL("", testCase.bucketName, testCase.objectName),
int64(testCase.dataLen), testCase.chunkSize, bytes.NewReader(testCase.data),
@@ -2045,12 +2046,13 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
var req *http.Request
// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
rec := httptest.NewRecorder()
if !testCase.invalidPartNumber || !testCase.maximumPartNumber {
switch {
case !testCase.invalidPartNumber || !testCase.maximumPartNumber:
// construct HTTP request for copy object.
req, err = newTestSignedRequestV4(http.MethodPut, getCopyObjectPartURL("", testCase.bucketName, testObject, testCase.uploadID, "1"), 0, nil, testCase.accessKey, testCase.secretKey, nil)
} else if testCase.invalidPartNumber {
case testCase.invalidPartNumber:
req, err = newTestSignedRequestV4(http.MethodPut, getCopyObjectPartURL("", testCase.bucketName, testObject, testCase.uploadID, "abc"), 0, nil, testCase.accessKey, testCase.secretKey, nil)
} else if testCase.maximumPartNumber {
case testCase.maximumPartNumber:
req, err = newTestSignedRequestV4(http.MethodPut, getCopyObjectPartURL("", testCase.bucketName, testObject, testCase.uploadID, "99999"), 0, nil, testCase.accessKey, testCase.secretKey, nil)
}
if err != nil {
@@ -3545,11 +3547,9 @@ func testAPIPutObjectPartHandlerStreaming(obj ObjectLayer, instanceType, bucketN
t.Errorf("Test %d %s expected to fail with error %s, but received %s", i+1, instanceType,
test.expectedErr.Code, errXML.Code)
}
} else {
if rec.Code != http.StatusOK {
t.Errorf("Test %d %s expected to succeed, but failed with HTTP status code %d",
i+1, instanceType, rec.Code)
}
} else if rec.Code != http.StatusOK {
t.Errorf("Test %d %s expected to succeed, but failed with HTTP status code %d",
i+1, instanceType, rec.Code)
}
}
}
@@ -4119,10 +4119,8 @@ func testAPIListObjectPartsHandler(obj ObjectLayer, instanceType, bucketName str
reqType, test.expectedErr.Code, errXML.Code)
}
// in case error is not expected response status should be 200OK.
} else {
if rec.Code != http.StatusOK {
t.Errorf("%s, Expected to succeed with response HTTP status 200OK, but failed with HTTP status code %d.", reqType, rec.Code)
}
} else if rec.Code != http.StatusOK {
t.Errorf("%s, Expected to succeed with response HTTP status 200OK, but failed with HTTP status code %d.", reqType, rec.Code)
}
}
})
+291
View File
@@ -0,0 +1,291 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"crypto/subtle"
"io"
"net/http"
"net/url"
"strings"
"time"
"github.com/klauspost/compress/gzhttp"
"github.com/lithammer/shortuuid/v4"
miniogo "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/mux"
"github.com/minio/pkg/bucket/policy"
"github.com/minio/minio/internal/auth"
levent "github.com/minio/minio/internal/config/lambda/event"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/logger"
)
func getLambdaEventData(bucket, object string, cred auth.Credentials, r *http.Request) (levent.Event, error) {
host := globalLocalNodeName
secure := globalIsTLS
if globalMinioEndpointURL != nil {
host = globalMinioEndpointURL.Host
secure = globalMinioEndpointURL.Scheme == "https"
}
duration := time.Since(cred.Expiration)
if cred.Expiration.IsZero() {
duration = time.Hour
}
clnt, err := miniogo.New(host, &miniogo.Options{
Creds: credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
Secure: secure,
Transport: globalRemoteTargetTransport,
Region: globalSite.Region,
})
if err != nil {
return levent.Event{}, err
}
reqParams := url.Values{}
if partNumberStr := r.Form.Get("partNumber"); partNumberStr != "" {
reqParams.Set("partNumber", partNumberStr)
}
for k := range supportedHeadGetReqParams {
if v := r.Form.Get(k); v != "" {
reqParams.Set(k, v)
}
}
extraHeaders := http.Header{}
if rng := r.Header.Get(xhttp.Range); rng != "" {
extraHeaders.Set(xhttp.Range, r.Header.Get(xhttp.Range))
}
u, err := clnt.PresignHeader(r.Context(), http.MethodGet, bucket, object, duration, reqParams, extraHeaders)
if err != nil {
return levent.Event{}, err
}
token, err := authenticateNode(cred.AccessKey, cred.SecretKey, u.RawQuery)
if err != nil {
return levent.Event{}, err
}
eventData := levent.Event{
GetObjectContext: &levent.GetObjectContext{
InputS3URL: u.String(),
OutputRoute: shortuuid.New(),
OutputToken: token,
},
UserRequest: levent.UserRequest{
URL: r.URL.String(),
Headers: r.Header.Clone(),
},
UserIdentity: levent.Identity{
Type: "IAMUser",
PrincipalID: cred.AccessKey,
AccessKeyID: cred.SecretKey,
},
}
return eventData, nil
}
var statusTextToCode = map[string]int{
"Continue": http.StatusContinue,
"Switching Protocols": http.StatusSwitchingProtocols,
"Processing": http.StatusProcessing,
"Early Hints": http.StatusEarlyHints,
"OK": http.StatusOK,
"Created": http.StatusCreated,
"Accepted": http.StatusAccepted,
"Non-Authoritative Information": http.StatusNonAuthoritativeInfo,
"No Content": http.StatusNoContent,
"Reset Content": http.StatusResetContent,
"Partial Content": http.StatusPartialContent,
"Multi-Status": http.StatusMultiStatus,
"Already Reported": http.StatusAlreadyReported,
"IM Used": http.StatusIMUsed,
"Multiple Choices": http.StatusMultipleChoices,
"Moved Permanently": http.StatusMovedPermanently,
"Found": http.StatusFound,
"See Other": http.StatusSeeOther,
"Not Modified": http.StatusNotModified,
"Use Proxy": http.StatusUseProxy,
"Temporary Redirect": http.StatusTemporaryRedirect,
"Permanent Redirect": http.StatusPermanentRedirect,
"Bad Request": http.StatusBadRequest,
"Unauthorized": http.StatusUnauthorized,
"Payment Required": http.StatusPaymentRequired,
"Forbidden": http.StatusForbidden,
"Not Found": http.StatusNotFound,
"Method Not Allowed": http.StatusMethodNotAllowed,
"Not Acceptable": http.StatusNotAcceptable,
"Proxy Authentication Required": http.StatusProxyAuthRequired,
"Request Timeout": http.StatusRequestTimeout,
"Conflict": http.StatusConflict,
"Gone": http.StatusGone,
"Length Required": http.StatusLengthRequired,
"Precondition Failed": http.StatusPreconditionFailed,
"Request Entity Too Large": http.StatusRequestEntityTooLarge,
"Request URI Too Long": http.StatusRequestURITooLong,
"Unsupported Media Type": http.StatusUnsupportedMediaType,
"Requested Range Not Satisfiable": http.StatusRequestedRangeNotSatisfiable,
"Expectation Failed": http.StatusExpectationFailed,
"I'm a teapot": http.StatusTeapot,
"Misdirected Request": http.StatusMisdirectedRequest,
"Unprocessable Entity": http.StatusUnprocessableEntity,
"Locked": http.StatusLocked,
"Failed Dependency": http.StatusFailedDependency,
"Too Early": http.StatusTooEarly,
"Upgrade Required": http.StatusUpgradeRequired,
"Precondition Required": http.StatusPreconditionRequired,
"Too Many Requests": http.StatusTooManyRequests,
"Request Header Fields Too Large": http.StatusRequestHeaderFieldsTooLarge,
"Unavailable For Legal Reasons": http.StatusUnavailableForLegalReasons,
"Internal Server Error": http.StatusInternalServerError,
"Not Implemented": http.StatusNotImplemented,
"Bad Gateway": http.StatusBadGateway,
"Service Unavailable": http.StatusServiceUnavailable,
"Gateway Timeout": http.StatusGatewayTimeout,
"HTTP Version Not Supported": http.StatusHTTPVersionNotSupported,
"Variant Also Negotiates": http.StatusVariantAlsoNegotiates,
"Insufficient Storage": http.StatusInsufficientStorage,
"Loop Detected": http.StatusLoopDetected,
"Not Extended": http.StatusNotExtended,
"Network Authentication Required": http.StatusNetworkAuthenticationRequired,
}
// StatusCode returns a HTTP Status code for the HTTP text. It returns -1
// if the text is unknown.
func StatusCode(text string) int {
if code, ok := statusTextToCode[text]; ok {
return code
}
return -1
}
func fwdHeadersToS3(h http.Header, w http.ResponseWriter) {
const trim = "x-amz-fwd-header-"
for k, v := range h {
if strings.HasPrefix(strings.ToLower(k), trim) {
w.Header()[k[len(trim):]] = v
}
}
}
func fwdStatusToAPIError(resp *http.Response) *APIError {
if status := resp.Header.Get(xhttp.AmzFwdStatus); status != "" && StatusCode(status) > -1 {
apiErr := &APIError{
HTTPStatusCode: StatusCode(status),
Description: resp.Header.Get(xhttp.AmzFwdErrorMessage),
Code: resp.Header.Get(xhttp.AmzFwdErrorCode),
}
if apiErr.HTTPStatusCode == http.StatusOK {
return nil
}
return apiErr
}
return nil
}
// GetObjectLamdbaHandler - GET Object with transformed data via lambda functions
// ----------
// This implementation of the GET operation applies lambda functions and returns the
// response generated via the lambda functions. To use this API, you must have READ access
// to the object.
func (api objectAPIHandlers) GetObjectLambdaHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "GetObjectLambda")
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
objectAPI := api.ObjectAPI()
if objectAPI == nil {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
return
}
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
// Check for auth type to return S3 compatible error.
cred, _, s3Error := checkRequestAuthTypeCredential(ctx, r, policy.GetObjectAction)
if s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
target, err := globalLambdaTargetList.Lookup(r.Form.Get("lambdaArn"))
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
eventData, err := getLambdaEventData(bucket, object, cred, r)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
resp, err := target.Send(eventData)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
defer resp.Body.Close()
if eventData.GetObjectContext.OutputRoute != resp.Header.Get(xhttp.AmzRequestRoute) {
tokenErr := errorCodes.ToAPIErr(ErrInvalidRequest)
tokenErr.Description = "The request route included in the request is invalid"
writeErrorResponse(ctx, w, tokenErr, r.URL)
return
}
if subtle.ConstantTimeCompare([]byte(resp.Header.Get(xhttp.AmzRequestToken)), []byte(eventData.GetObjectContext.OutputToken)) != 1 {
tokenErr := errorCodes.ToAPIErr(ErrInvalidToken)
tokenErr.Description = "The request token included in the request is invalid"
writeErrorResponse(ctx, w, tokenErr, r.URL)
return
}
// Set all the relevant lambda forward headers if found.
fwdHeadersToS3(resp.Header, w)
if apiErr := fwdStatusToAPIError(resp); apiErr != nil {
writeErrorResponse(ctx, w, *apiErr, r.URL)
return
}
if resp.StatusCode != http.StatusOK {
writeErrorResponse(ctx, w, APIError{
Code: "LambdaFunctionError",
HTTPStatusCode: resp.StatusCode,
Description: "unexpected failure reported from lambda function",
}, r.URL)
return
}
if !globalAPIConfig.shouldGzipObjects() {
w.Header().Set(gzhttp.HeaderNoCompression, "true")
}
io.Copy(w, resp.Body)
}
+4 -19
View File
@@ -57,6 +57,10 @@ func (client *peerRESTClient) call(method string, values url.Values, body io.Rea
// permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
// after verifying format.json
func (client *peerRESTClient) callWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
if client == nil || !client.IsOnline() {
return nil, errPeerNotReachable
}
if values == nil {
values = make(url.Values)
}
@@ -309,25 +313,6 @@ func (client *peerRESTClient) DeleteBucketMetadata(bucket string) error {
return nil
}
// cycleServerBloomFilter will cycle the bloom filter to start recording to index y if not already.
// The response will contain a bloom filter starting at index x up to, but not including index y.
// If y is 0, the response will not update y, but return the currently recorded information
// from the current x to y-1.
func (client *peerRESTClient) cycleServerBloomFilter(ctx context.Context, req bloomFilterRequest) (*bloomFilterResponse, error) {
var reader bytes.Buffer
err := gob.NewEncoder(&reader).Encode(req)
if err != nil {
return nil, err
}
respBody, err := client.callWithContext(ctx, peerRESTMethodCycleBloom, nil, &reader, -1)
if err != nil {
return nil, err
}
var resp bloomFilterResponse
defer xhttp.DrainBody(respBody)
return &resp, gob.NewDecoder(respBody).Decode(&resp)
}
// DeletePolicy - delete a specific canned policy.
func (client *peerRESTClient) DeletePolicy(policyName string) (err error) {
values := make(url.Values)
+1 -1
View File
@@ -18,7 +18,7 @@
package cmd
const (
peerRESTVersion = "v29" // Added LocalStorageInfo peer API
peerRESTVersion = "v30" // Removed bloom filter
peerRESTVersionPrefix = SlashSeparator + peerRESTVersion
peerRESTPrefix = minioReservedBucketPath + "/peer"
-25
View File
@@ -631,30 +631,6 @@ func (s *peerRESTServer) LoadBucketMetadataHandler(w http.ResponseWriter, r *htt
}
}
// CycleServerBloomFilterHandler cycles bloom filter on server.
func (s *peerRESTServer) CycleServerBloomFilterHandler(w http.ResponseWriter, r *http.Request) {
if !s.IsValid(w, r) {
s.writeErrorResponse(w, errors.New("Invalid request"))
return
}
ctx := newContext(r, w, "CycleServerBloomFilter")
var req bloomFilterRequest
err := gob.NewDecoder(r.Body).Decode(&req)
if err != nil {
s.writeErrorResponse(w, err)
return
}
bf, err := intDataUpdateTracker.cycleFilter(ctx, req)
if err != nil {
s.writeErrorResponse(w, err)
return
}
logger.LogIf(ctx, gob.NewEncoder(w).Encode(bf))
}
func (s *peerRESTServer) GetMetacacheListingHandler(w http.ResponseWriter, r *http.Request) {
if !s.IsValid(w, r) {
s.writeErrorResponse(w, errors.New("Invalid request"))
@@ -1390,7 +1366,6 @@ func registerPeerRESTHandlers(router *mux.Router) {
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodOsInfo).HandlerFunc(httpTraceHdrs(server.GetOSInfoHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDiskHwInfo).HandlerFunc(httpTraceHdrs(server.GetPartitionsHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCPUInfo).HandlerFunc(httpTraceHdrs(server.GetCPUsHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCycleBloom).HandlerFunc(httpTraceHdrs(server.CycleServerBloomFilterHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetAllBucketStats).HandlerFunc(httpTraceHdrs(server.GetAllBucketStatsHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeleteBucketMetadata).HandlerFunc(httpTraceHdrs(server.DeleteBucketMetadataHandler)).Queries(restQueries(peerRESTBucket)...)
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadBucketMetadata).HandlerFunc(httpTraceHdrs(server.LoadBucketMetadataHandler)).Queries(restQueries(peerRESTBucket)...)
+16
View File
@@ -80,6 +80,10 @@ func (s *peerS3Server) HealthHandler(w http.ResponseWriter, r *http.Request) {
}
func listBucketsLocal(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) {
globalLocalDrivesMu.RLock()
globalLocalDrives := globalLocalDrives
globalLocalDrivesMu.RUnlock()
quorum := (len(globalLocalDrives) / 2)
buckets = make([]BucketInfo, 0, 32)
@@ -128,6 +132,10 @@ func listBucketsLocal(ctx context.Context, opts BucketOptions) (buckets []Bucket
}
func getBucketInfoLocal(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) {
globalLocalDrivesMu.RLock()
globalLocalDrives := globalLocalDrives
globalLocalDrivesMu.RUnlock()
g := errgroup.WithNErrs(len(globalLocalDrives)).WithConcurrency(32)
bucketsInfo := make([]BucketInfo, len(globalLocalDrives))
@@ -173,6 +181,10 @@ func getBucketInfoLocal(ctx context.Context, bucket string, opts BucketOptions)
}
func deleteBucketLocal(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
globalLocalDrivesMu.RLock()
globalLocalDrives := globalLocalDrives
globalLocalDrivesMu.RUnlock()
g := errgroup.WithNErrs(len(globalLocalDrives)).WithConcurrency(32)
// Make a volume entry on all underlying storage disks.
@@ -208,6 +220,10 @@ func deleteBucketLocal(ctx context.Context, bucket string, opts DeleteBucketOpti
}
func makeBucketLocal(ctx context.Context, bucket string, opts MakeBucketOptions) error {
globalLocalDrivesMu.RLock()
globalLocalDrives := globalLocalDrives
globalLocalDrivesMu.RUnlock()
g := errgroup.WithNErrs(len(globalLocalDrives)).WithConcurrency(32)
// Make a volume entry on all underlying storage disks.
+14 -1
View File
@@ -136,6 +136,7 @@ func printServerCommonMsg(apiEndpoints []string) {
}
}
printEventNotifiers()
printLambdaTargets()
if globalBrowserEnabled {
consoleEndpointStr := strings.Join(stripStandardPorts(getConsoleEndpoints(), globalMinioConsoleHost), " ")
@@ -152,6 +153,18 @@ func printObjectAPIMsg() {
logger.Info(color.Blue("\nDocumentation: ") + "https://min.io/docs/minio/linux/index.html")
}
func printLambdaTargets() {
if globalLambdaTargetList == nil || globalLambdaTargetList.Empty() {
return
}
arnMsg := color.Blue("Object Lambda ARNs: ")
for _, arn := range globalLambdaTargetList.List(globalSite.Region) {
arnMsg += color.Bold(fmt.Sprintf("%s ", arn))
}
logger.Info(arnMsg + "\n")
}
// Prints bucket notification configurations.
func printEventNotifiers() {
if globalNotificationSys == nil {
@@ -168,7 +181,7 @@ func printEventNotifiers() {
arnMsg += color.Bold(fmt.Sprintf("%s ", arn))
}
logger.Info(arnMsg)
logger.Info(arnMsg + "\n")
}
// Prints startup message for command line access. Prints link to our documentation
+1 -2
View File
@@ -109,8 +109,7 @@ func parseCredentialHeader(credElement string, region string, stype serviceType)
return ch, ErrAuthorizationHeaderMalformed
}
if credElements[2] != string(stype) {
switch stype {
case serviceSTS:
if stype == serviceSTS {
return ch, ErrInvalidServiceSTS
}
return ch, ErrInvalidServiceS3
+2 -3
View File
@@ -630,9 +630,8 @@ func TestDoesV4PresignParamsExist(t *testing.T) {
// TestParsePreSignV4 - Validates the parsing logic of Presignied v4 request from its url query values.
func TestParsePreSignV4(t *testing.T) {
// converts the duration in seconds into string format.
getDurationStr := func(expires int) string {
return strconv.Itoa(expires)
}
getDurationStr := strconv.Itoa
// used in expected preSignValues, preSignValues.Date is of type time.Time .
queryTime := UTCNow()
+2 -4
View File
@@ -150,10 +150,8 @@ func TestSkipContentSha256Cksum(t *testing.T) {
q.Add(testCase.inputHeaderKey, testCase.inputHeaderValue)
}
inputReq.URL.RawQuery = q.Encode()
} else {
if testCase.inputHeaderKey != "" {
inputReq.Header.Set(testCase.inputHeaderKey, testCase.inputHeaderValue)
}
} else if testCase.inputHeaderKey != "" {
inputReq.Header.Set(testCase.inputHeaderKey, testCase.inputHeaderValue)
}
inputReq.ParseForm()
+24 -12
View File
@@ -599,13 +599,13 @@ func (c *SiteReplicationSys) PeerJoinReq(ctx context.Context, arg madmin.SRPeerJ
func (c *SiteReplicationSys) GetIDPSettings(ctx context.Context) madmin.IDPSettings {
s := madmin.IDPSettings{}
s.LDAP = madmin.LDAPSettings{
IsLDAPEnabled: globalLDAPConfig.Enabled(),
LDAPUserDNSearchBase: globalLDAPConfig.LDAP.UserDNSearchBaseDistName,
LDAPUserDNSearchFilter: globalLDAPConfig.LDAP.UserDNSearchFilter,
LDAPGroupSearchBase: globalLDAPConfig.LDAP.GroupSearchBaseDistName,
LDAPGroupSearchFilter: globalLDAPConfig.LDAP.GroupSearchFilter,
IsLDAPEnabled: globalIAMSys.LDAPConfig.Enabled(),
LDAPUserDNSearchBase: globalIAMSys.LDAPConfig.LDAP.UserDNSearchBaseDistName,
LDAPUserDNSearchFilter: globalIAMSys.LDAPConfig.LDAP.UserDNSearchFilter,
LDAPGroupSearchBase: globalIAMSys.LDAPConfig.LDAP.GroupSearchBaseDistName,
LDAPGroupSearchFilter: globalIAMSys.LDAPConfig.LDAP.GroupSearchFilter,
}
s.OpenID = globalOpenIDConfig.GetSettings()
s.OpenID = globalIAMSys.OpenIDConfig.GetSettings()
if s.OpenID.Enabled {
s.OpenID.Region = globalSite.Region
}
@@ -1219,6 +1219,7 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
sessionPolicy: sp,
claims: change.Create.Claims,
comment: change.Create.Comment,
expiration: change.Create.Expiration,
}
_, _, err = globalIAMSys.NewServiceAccount(ctx, change.Create.Parent, change.Create.Groups, opts)
if err != nil {
@@ -1245,6 +1246,7 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
status: change.Update.Status,
comment: change.Update.Comment,
sessionPolicy: sp,
expiration: change.Update.Expiration,
}
_, err = globalIAMSys.UpdateServiceAccount(ctx, change.Update.AccessKey, opts)
@@ -1325,10 +1327,9 @@ func (c *SiteReplicationSys) PeerSTSAccHandler(ctx context.Context, stsCred *mad
// Extract the username and lookup DN and groups in LDAP.
ldapUser, isLDAPSTS := claims.Lookup(ldapUserN)
switch {
case isLDAPSTS:
if isLDAPSTS {
// Need to lookup the groups from LDAP.
_, ldapGroups, err := globalLDAPConfig.LookupUserDN(ldapUser)
_, ldapGroups, err := globalIAMSys.LDAPConfig.LookupUserDN(ldapUser)
if err != nil {
return fmt.Errorf("unable to query LDAP server for %s: %w", ldapUser, err)
}
@@ -1848,6 +1849,7 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
SessionPolicy: json.RawMessage(policyJSON),
Status: acc.Credentials.Status,
Comment: acc.Credentials.Comment,
Expiration: &acc.Credentials.Expiration,
},
},
UpdatedAt: acc.UpdatedAt,
@@ -4716,6 +4718,7 @@ func (c *SiteReplicationSys) healUsers(ctx context.Context, objAPI ObjectLayer,
SessionPolicy: json.RawMessage(policyJSON),
Status: creds.Status,
Comment: creds.Comment,
Expiration: &creds.Expiration,
},
},
UpdatedAt: lastUpdate,
@@ -4725,10 +4728,19 @@ func (c *SiteReplicationSys) healUsers(ctx context.Context, objAPI ObjectLayer,
continue
}
if creds.IsTemp() && !creds.IsExpired() {
var parentPolicy string
u, err := globalIAMSys.GetUserInfo(ctx, creds.ParentUser)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to heal temporary credentials %s from peer site %s -> %s : %w", user, latestPeerName, peerName, err))
continue
// Parent may be "virtual" (for ldap, oidc, client tls auth,
// custom auth plugin), so in such cases we apply no parent
// policy. The session token will contain info about policy to
// be applied.
if !errors.Is(err, errNoSuchUser) {
logger.LogIf(ctx, fmt.Errorf("Unable to heal temporary credentials %s from peer site %s -> %s : %w", user, latestPeerName, peerName, err))
continue
}
} else {
parentPolicy = u.PolicyName
}
// Call hook for site replication.
if err := c.IAMChangeHook(ctx, madmin.SRIAMItem{
@@ -4738,7 +4750,7 @@ func (c *SiteReplicationSys) healUsers(ctx context.Context, objAPI ObjectLayer,
SecretKey: creds.SecretKey,
SessionToken: creds.SessionToken,
ParentUser: creds.ParentUser,
ParentPolicyMapping: u.PolicyName,
ParentPolicyMapping: parentPolicy,
},
UpdatedAt: lastUpdate,
}); err != nil {
+6 -6
View File
@@ -361,7 +361,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
}
// Validate JWT; check clientID in claims matches the one associated with the roleArn
if err := globalOpenIDConfig.Validate(r.Context(), roleArn, token, accessToken, r.Form.Get(stsDurationSeconds), claims); err != nil {
if err := globalIAMSys.OpenIDConfig.Validate(r.Context(), roleArn, token, accessToken, r.Form.Get(stsDurationSeconds), claims); err != nil {
switch err {
case openid.ErrTokenExpired:
switch action {
@@ -598,7 +598,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
}
}
ldapUserDN, groupDistNames, err := globalLDAPConfig.Bind(ldapUsername, ldapPassword)
ldapUserDN, groupDistNames, err := globalIAMSys.LDAPConfig.Bind(ldapUsername, ldapPassword)
if err != nil {
err = fmt.Errorf("LDAP server error: %w", err)
writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
@@ -614,7 +614,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
return
}
expiryDur, err := globalLDAPConfig.GetExpiryDuration(r.Form.Get(stsDurationSeconds))
expiryDur, err := globalIAMSys.LDAPConfig.GetExpiryDuration(r.Form.Get(stsDurationSeconds))
if err != nil {
writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
return
@@ -686,7 +686,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
claims := make(map[string]interface{})
defer logger.AuditLog(ctx, w, r, claims)
if !globalSTSTLSConfig.Enabled {
if !globalIAMSys.STSTLSConfig.Enabled {
writeSTSErrorResponse(ctx, w, true, ErrSTSNotInitialized, errors.New("STS API 'AssumeRoleWithCertificate' is disabled"))
return
}
@@ -727,7 +727,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
}
certificate := r.TLS.PeerCertificates[0]
if !globalSTSTLSConfig.InsecureSkipVerify { // Verify whether the client certificate has been issued by a trusted CA.
if !globalIAMSys.STSTLSConfig.InsecureSkipVerify { // Verify whether the client certificate has been issued by a trusted CA.
_, err := certificate.Verify(x509.VerifyOptions{
KeyUsages: []x509.ExtKeyUsage{
x509.ExtKeyUsageClientAuth,
@@ -776,7 +776,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
return
}
expiry, err := globalSTSTLSConfig.GetExpiryDuration(r.Form.Get(stsDurationSeconds))
expiry, err := globalIAMSys.STSTLSConfig.GetExpiryDuration(r.Form.Get(stsDurationSeconds))
if err != nil {
writeSTSErrorResponse(ctx, w, true, ErrSTSMissingParameter, err)
return
+5
View File
@@ -627,6 +627,11 @@ func (s *TestSuiteIAM) TestSTSForRoot(c *check) {
if !gotBuckets.Equals(shouldHaveBuckets) {
c.Fatalf("root user should have access to all buckets")
}
// This must fail.
if err := userAdmClient.AddUser(ctx, globalActiveCred.AccessKey, globalActiveCred.SecretKey); err == nil {
c.Fatal("AddUser() for root credential must fail via root STS creds")
}
}
// SetUpLDAP - expects to setup an LDAP test server using the test LDAP
+2 -2
View File
@@ -1477,9 +1477,9 @@ func getListenNotificationURL(endPoint, bucketName string, prefixes, suffixes, e
}
// getRandomDisks - Creates a slice of N random disks, each of the form - minio-XXX
func getRandomDisks(N int) ([]string, error) {
func getRandomDisks(n int) ([]string, error) {
var erasureDisks []string
for i := 0; i < N; i++ {
for i := 0; i < n; i++ {
path, err := os.MkdirTemp(globalTestTmpDir, "minio-")
if err != nil {
// Remove directories created so far.
+7 -1
View File
@@ -21,6 +21,7 @@ import (
"encoding/json"
"io"
"net/http"
"strconv"
jsoniter "github.com/json-iterator/go"
"github.com/minio/madmin-go/v2"
@@ -94,6 +95,11 @@ func (api adminAPIHandlers) AddTierHandler(w http.ResponseWriter, r *http.Reques
return
}
var ignoreInUse bool
if forceStr := r.Form.Get("force"); forceStr != "" {
ignoreInUse, _ = strconv.ParseBool(forceStr)
}
// Disallow remote tiers with internal storage class names
switch cfg.Name {
case storageclass.STANDARD, storageclass.RRS:
@@ -107,7 +113,7 @@ func (api adminAPIHandlers) AddTierHandler(w http.ResponseWriter, r *http.Reques
return
}
err = globalTierConfigMgr.Add(ctx, cfg)
err = globalTierConfigMgr.Add(ctx, cfg, ignoreInUse)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
+11 -8
View File
@@ -103,7 +103,7 @@ func (config *TierConfigMgr) isTierNameInUse(tierName string) (madmin.TierType,
}
// Add adds tier to config if it passes all validations.
func (config *TierConfigMgr) Add(ctx context.Context, tier madmin.TierConfig) error {
func (config *TierConfigMgr) Add(ctx context.Context, tier madmin.TierConfig, ignoreInUse bool) error {
config.Lock()
defer config.Unlock()
@@ -122,13 +122,16 @@ func (config *TierConfigMgr) Add(ctx context.Context, tier madmin.TierConfig) er
if err != nil {
return err
}
// Check if warmbackend is in use by other MinIO tenants
inUse, err := d.InUse(ctx)
if err != nil {
return err
}
if inUse {
return errTierBackendInUse
if !ignoreInUse {
// Check if warmbackend is in use by other MinIO tenants
inUse, err := d.InUse(ctx)
if err != nil {
return err
}
if inUse {
return errTierBackendInUse
}
}
config.Tiers[tierName] = tier
+3
View File
@@ -88,6 +88,9 @@ var errNoPolicyToAttachOrDetach = errors.New("Specified policy update has no net
// deleted.
var errGroupNotEmpty = errors.New("Specified group is not empty - cannot remove it")
// error returned in IAM subsystem when a group is disabled
var errGroupDisabled = errors.New("Specified group is disabled")
// error returned in IAM subsystem when policy doesn't exist.
var errNoSuchPolicy = errors.New("Specified canned policy does not exist")
+1 -1
View File
@@ -243,7 +243,7 @@ func untar(ctx context.Context, r io.Reader, putObject func(reader io.Reader, in
rc.Close()
<-asyncWriters
wg.Done()
//lint:ignore SA6002 we are fine with the tiny alloc
//nolint:staticcheck // SA6002 we are fine with the tiny alloc
poolBuf128k.Put(b)
}()
if err := putObject(&rc, fi, name); err != nil {
+4 -3
View File
@@ -300,13 +300,14 @@ func TestDownloadReleaseData(t *testing.T) {
}
result, err := downloadReleaseURL(u, 1*time.Second, "")
if testCase.expectedErr == nil {
switch {
case testCase.expectedErr == nil:
if err != nil {
t.Fatalf("error: expected: %v, got: %v", testCase.expectedErr, err)
}
} else if err == nil {
case err == nil:
t.Fatalf("error: expected: %v, got: %v", testCase.expectedErr, err)
} else if testCase.expectedErr.Error() != err.Error() {
case testCase.expectedErr.Error() != err.Error():
t.Fatalf("error: expected: %v, got: %v", testCase.expectedErr, err)
}
+11 -11
View File
@@ -159,8 +159,7 @@ func ErrorRespToObjectError(err error, params ...string) error {
err = PartTooSmall{}
}
switch minioErr.StatusCode {
case http.StatusMethodNotAllowed:
if minioErr.StatusCode == http.StatusMethodNotAllowed {
err = toObjectErr(errMethodNotAllowed, bucket, object)
}
return err
@@ -908,19 +907,20 @@ func lcp(strs []string, pre bool) string {
// Returns the mode in which MinIO is running
func getMinioMode() string {
mode := globalMinioModeFS
if globalIsDistErasure {
mode = globalMinioModeDistErasure
} else if globalIsErasure {
mode = globalMinioModeErasure
} else if globalIsErasureSD {
mode = globalMinioModeErasureSD
switch {
case globalIsDistErasure:
return globalMinioModeDistErasure
case globalIsErasure:
return globalMinioModeErasure
case globalIsErasureSD:
return globalMinioModeErasureSD
default:
return globalMinioModeFS
}
return mode
}
func iamPolicyClaimNameOpenID() string {
return globalOpenIDConfig.GetIAMPolicyClaimName()
return globalIAMSys.OpenIDConfig.GetIAMPolicyClaimName()
}
func iamPolicyClaimNameSA() string {

Some files were not shown because too many files have changed in this diff Show More