mirror of
https://github.com/pgsty/minio.git
synced 2026-08-14 02:33:16 +03:00
Compare commits
49 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c7f7e67a10 | |||
| 628042e65e | |||
| cde7eeb660 | |||
| 6305b206e1 | |||
| d85da9236e | |||
| 9800760cb3 | |||
| 5c087bdcad | |||
| a547bf517d | |||
| b984bf8d1a | |||
| 18f9cccfa7 | |||
| fb6ab1cca2 | |||
| 56c57e2c53 | |||
| a6057c35cc | |||
| 901887e6bf | |||
| ee54643004 | |||
| 0a17acdb34 | |||
| 72e5212842 | |||
| 714283fae2 | |||
| 3423028713 | |||
| 9d062b37d7 | |||
| 4636d3a9c3 | |||
| c95ede35c1 | |||
| 7415e1aa56 | |||
| f350953a19 | |||
| 958bba5b42 | |||
| 0f2b95b497 | |||
| d07089ceac | |||
| 47dfa62384 | |||
| 3a3265cf88 | |||
| 0ff931dc76 | |||
| 4d708cebe9 | |||
| 4d7c8e3bb8 | |||
| 8cde38404d | |||
| fe7bf6cbbc | |||
| 8b4eb2304b | |||
| ae029191a3 | |||
| bfedea9bad | |||
| 7777d3b43a | |||
| 9ed4fc9687 | |||
| b49b39e99d | |||
| cd3a2de5a3 | |||
| 6e8960ccdd | |||
| e05f3d5d84 | |||
| 94c6cb1323 | |||
| 3f81cd1b22 | |||
| 8da0f4c5bb | |||
| 9acf1024e4 | |||
| b21d3f9b82 | |||
| 2bbf380262 |
@@ -20,11 +20,11 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.19.x]
|
||||
go-version: [1.20.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
- uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
|
||||
- uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
|
||||
@@ -20,21 +20,28 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.19.x]
|
||||
go-version: [1.20.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
|
||||
- name: Setup dockerfile for build test
|
||||
run: |
|
||||
echo "FROM golang:1.19.4" >> Dockerfile.fips.test
|
||||
echo "COPY . /minio" >> Dockerfile.fips.test
|
||||
echo "WORKDIR /minio" >> Dockerfile.fips.test
|
||||
echo "ENV GOEXPERIMENT=boringcrypto" >> Dockerfile.fips.test
|
||||
echo "RUN make" >> Dockerfile.fips.test
|
||||
GO_VERSION=$(go version | cut -d ' ' -f 3 | sed 's/go//')
|
||||
echo Detected go version $GO_VERSION
|
||||
cat > Dockerfile.fips.test <<EOF
|
||||
FROM golang:${GO_VERSION}
|
||||
COPY . /minio
|
||||
WORKDIR /minio
|
||||
ENV GOEXPERIMENT=boringcrypto
|
||||
RUN make
|
||||
EOF
|
||||
|
||||
- name: Build
|
||||
uses: docker/build-push-action@v3
|
||||
@@ -49,4 +56,4 @@ jobs:
|
||||
- name: Test binary
|
||||
run: |
|
||||
docker run --rm minio/fips-test:latest ./minio --version
|
||||
docker run --rm -i minio/fips-test:latest /bin/bash -c 'go tool nm ./minio' | grep -q FIPS
|
||||
docker run --rm -i minio/fips-test:latest /bin/bash -c 'go tool nm ./minio | grep FIPS | grep -q FIPS'
|
||||
|
||||
@@ -20,10 +20,10 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.19.x]
|
||||
go-version: [1.20.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
|
||||
@@ -20,10 +20,15 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.19.x]
|
||||
go-version: [1.20.x, 1.19.x]
|
||||
os: [ubuntu-latest, windows-latest]
|
||||
exclude:
|
||||
- os: ubuntu-latest
|
||||
go-version: 1.19.x
|
||||
- os: windows-latest
|
||||
go-version: 1.20.x
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
|
||||
@@ -20,10 +20,10 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.19.x]
|
||||
go-version: [1.20.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
# are turned off - i.e. if ldap="", then ldap server is not enabled for
|
||||
# the tests.
|
||||
matrix:
|
||||
go-version: [1.19.x]
|
||||
go-version: [1.20.x]
|
||||
ldap: ["", "localhost:389"]
|
||||
etcd: ["", "http://localhost:2379"]
|
||||
openid: ["", "http://127.0.0.1:5556/dex"]
|
||||
@@ -75,7 +75,7 @@ jobs:
|
||||
openid: "http://127.0.0.1:5556/dex"
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Lint all docs
|
||||
run: |
|
||||
|
||||
@@ -21,10 +21,10 @@ jobs:
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.19.x]
|
||||
go-version: [1.20.x]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
|
||||
@@ -20,11 +20,11 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.19.x]
|
||||
go-version: [1.20.x]
|
||||
os: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: 1.19
|
||||
go-version: 1.20.x
|
||||
check-latest: true
|
||||
- name: Get official govulncheck
|
||||
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
|
||||
@@ -38,3 +38,4 @@ docs/debugging/s3-check-md5/s3-check-md5
|
||||
docs/debugging/hash-set/hash-set
|
||||
docs/debugging/healing-bin/healing-bin
|
||||
docs/debugging/inspect/inspect
|
||||
.bin/
|
||||
|
||||
+18
-23
@@ -1,39 +1,34 @@
|
||||
linters-settings:
|
||||
gofumpt:
|
||||
lang-version: "1.18"
|
||||
simplify: true
|
||||
|
||||
misspell:
|
||||
locale: US
|
||||
|
||||
staticcheck:
|
||||
checks: ['all', '-ST1005', '-ST1000', '-SA4000', '-SA9004', '-SA1019', '-SA1008', '-U1000', '-ST1016']
|
||||
|
||||
linters:
|
||||
disable-all: true
|
||||
enable:
|
||||
- typecheck
|
||||
- goimports
|
||||
- misspell
|
||||
- govet
|
||||
- revive
|
||||
- ineffassign
|
||||
- gomodguard
|
||||
- durationcheck
|
||||
- gocritic
|
||||
- gofmt
|
||||
- gofumpt
|
||||
- goimports
|
||||
- gomodguard
|
||||
- govet
|
||||
- ineffassign
|
||||
- misspell
|
||||
- revive
|
||||
- staticcheck
|
||||
- tenv
|
||||
- typecheck
|
||||
- unconvert
|
||||
- unused
|
||||
- gocritic
|
||||
- gofumpt
|
||||
- tenv
|
||||
- durationcheck
|
||||
|
||||
issues:
|
||||
exclude-use-default: false
|
||||
exclude:
|
||||
- should have a package comment
|
||||
- error strings should not be capitalized or end with punctuation or a newline
|
||||
# todo fix these when we get enough time.
|
||||
- "singleCaseSwitch: should rewrite switch statement to if statement"
|
||||
- "unlambda: replace"
|
||||
- "captLocal:"
|
||||
- "ifElseChain:"
|
||||
- "elseif:"
|
||||
|
||||
service:
|
||||
golangci-lint-version: 1.43.0 # use the fixed version to not introduce new linters unexpectedly
|
||||
- should have a package comment
|
||||
- error strings should not be capitalized or end with punctuation or a newline
|
||||
|
||||
@@ -3175,69 +3175,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/bits-and-blooms/bitset
|
||||
https://github.com/bits-and-blooms/bitset
|
||||
----------------------------------------------------------------
|
||||
Copyright (c) 2014 Will Fitzgerald. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/bits-and-blooms/bloom/v3
|
||||
https://github.com/bits-and-blooms/bloom/v3
|
||||
----------------------------------------------------------------
|
||||
Copyright (c) 2014 Will Fitzgerald. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/buger/jsonparser
|
||||
https://github.com/buger/jsonparser
|
||||
----------------------------------------------------------------
|
||||
@@ -15500,37 +15437,36 @@ https://github.com/matttproud/golang_protobuf_extensions
|
||||
github.com/miekg/dns
|
||||
https://github.com/miekg/dns
|
||||
----------------------------------------------------------------
|
||||
Copyright (c) 2009 The Go Authors. All rights reserved.
|
||||
BSD 3-Clause License
|
||||
|
||||
Copyright (c) 2009, The Go Authors. Extensions copyright (c) 2011, Miek Gieben.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of the copyright holder nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
As this is fork of the official Go code the same license applies.
|
||||
Extensions of the original work are copyright (c) 2011 Miek Gieben
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/minio/cli
|
||||
@@ -25887,61 +25823,6 @@ https://github.com/tklauser/numcpus
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/twmb/murmur3
|
||||
https://github.com/twmb/murmur3
|
||||
----------------------------------------------------------------
|
||||
Copyright 2013, Sébastien Paolacci.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
* Neither the name of the library nor the
|
||||
names of its contributors may be used to endorse or promote products
|
||||
derived from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
|
||||
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
Copyright 2018, Travis Bischel.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
* Neither the name of the library nor the
|
||||
names of its contributors may be used to endorse or promote products
|
||||
derived from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
|
||||
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/unrolled/secure
|
||||
https://github.com/unrolled/secure
|
||||
----------------------------------------------------------------
|
||||
|
||||
@@ -8,6 +8,10 @@ GOOS := $(shell go env GOOS)
|
||||
VERSION ?= $(shell git describe --tags)
|
||||
TAG ?= "minio/minio:$(VERSION)"
|
||||
|
||||
GOLANGCI_VERSION = v1.51.2
|
||||
GOLANGCI_DIR = .bin/golangci/$(GOLANGCI_VERSION)
|
||||
GOLANGCI = $(GOLANGCI_DIR)/golangci-lint
|
||||
|
||||
all: build
|
||||
|
||||
checks: ## check dependencies
|
||||
@@ -19,24 +23,22 @@ help: ## print this help
|
||||
|
||||
getdeps: ## fetch necessary dependencies
|
||||
@mkdir -p ${GOPATH}/bin
|
||||
@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin
|
||||
@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR) $(GOLANGCI_VERSION)
|
||||
@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7
|
||||
@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
|
||||
@echo "Installing staticcheck" && go install honnef.co/go/tools/cmd/staticcheck@latest
|
||||
|
||||
crosscompile: ## cross compile minio
|
||||
@(env bash $(PWD)/buildscripts/cross-compile.sh)
|
||||
|
||||
verifiers: getdeps lint check-gen
|
||||
verifiers: lint check-gen
|
||||
|
||||
check-gen: ## check for updated autogenerated files
|
||||
@go generate ./... >/dev/null
|
||||
@(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false)
|
||||
|
||||
lint: ## runs golangci-lint suite of linters
|
||||
lint: getdeps ## runs golangci-lint suite of linters
|
||||
@echo "Running $@ check"
|
||||
@${GOPATH}/bin/golangci-lint run --build-tags kqueue --timeout=10m --config ./.golangci.yml
|
||||
@${GOPATH}/bin/staticcheck --tests=false ./...
|
||||
@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml --fix
|
||||
|
||||
check: test
|
||||
test: verifiers build ## builds minio, runs linters, tests
|
||||
|
||||
@@ -5,7 +5,6 @@ set -e
|
||||
export GORACE="history_size=7"
|
||||
export MINIO_API_REQUESTS_MAX=10000
|
||||
|
||||
## TODO remove `dsync` from race detector once this is merged and released https://go-review.googlesource.com/c/go/+/333529/
|
||||
for d in $(go list ./... | grep -v dsync); do
|
||||
for d in $(go list ./...); do
|
||||
CGO_ENABLED=1 go test -v -race --timeout 100m "$d"
|
||||
done
|
||||
|
||||
@@ -686,8 +686,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
bucket, fileName := slc[0], slc[1]
|
||||
switch fileName {
|
||||
case objectLockConfig:
|
||||
if fileName == objectLockConfig {
|
||||
reader, err := file.Open()
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
@@ -756,8 +755,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
bucket, fileName := slc[0], slc[1]
|
||||
switch fileName {
|
||||
case bucketVersioningConfig:
|
||||
if fileName == bucketVersioningConfig {
|
||||
reader, err := file.Open()
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
|
||||
@@ -35,6 +35,7 @@ import (
|
||||
idplugin "github.com/minio/minio/internal/config/identity/plugin"
|
||||
polplugin "github.com/minio/minio/internal/config/policy/plugin"
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
"github.com/minio/minio/internal/config/subnet"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
@@ -87,6 +88,10 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
// Check if subnet proxy being deleted and if so the value of proxy of subnet
|
||||
// target of logger webhook configuration also should be deleted
|
||||
loggerWebhookProxyDeleted := setLoggerWebhookSubnetProxy(subSys, cfg)
|
||||
|
||||
if err = saveServerConfig(ctx, objectAPI, cfg); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -101,6 +106,10 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
||||
dynamic := config.SubSystemsDynamic.Contains(subSys)
|
||||
if dynamic {
|
||||
applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
|
||||
if subSys == config.SubnetSubSys && loggerWebhookProxyDeleted {
|
||||
// Logger webhook proxy deleted, apply the dynamic changes
|
||||
applyDynamic(ctx, objectAPI, cfg, config.LoggerWebhookSubSys, r, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -132,9 +141,10 @@ func (bce badConfigErr) Unwrap() error {
|
||||
}
|
||||
|
||||
type setConfigResult struct {
|
||||
Cfg config.Config
|
||||
SubSys string
|
||||
Dynamic bool
|
||||
Cfg config.Config
|
||||
SubSys string
|
||||
Dynamic bool
|
||||
LoggerWebhookCfgUpdated bool
|
||||
}
|
||||
|
||||
// SetConfigKVHandler - PUT /minio/admin/v3/set-config-kv
|
||||
@@ -175,6 +185,11 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
||||
|
||||
if result.Dynamic {
|
||||
applyDynamic(ctx, objectAPI, result.Cfg, result.SubSys, r, w)
|
||||
// If logger webhook config updated (proxy due to callhome), explicitly dynamically
|
||||
// apply the config
|
||||
if result.LoggerWebhookCfgUpdated {
|
||||
applyDynamic(ctx, objectAPI, result.Cfg, config.LoggerWebhookSubSys, r, w)
|
||||
}
|
||||
}
|
||||
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
@@ -201,6 +216,10 @@ func setConfigKV(ctx context.Context, objectAPI ObjectLayer, kvBytes []byte) (re
|
||||
return
|
||||
}
|
||||
|
||||
// Check if subnet proxy being set and if so set the same value to proxy of subnet
|
||||
// target of logger webhook configuration
|
||||
result.LoggerWebhookCfgUpdated = setLoggerWebhookSubnetProxy(result.SubSys, result.Cfg)
|
||||
|
||||
// Update the actual server config on disk.
|
||||
if err = saveServerConfig(ctx, objectAPI, result.Cfg); err != nil {
|
||||
return
|
||||
@@ -501,7 +520,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
|
||||
case config.IdentityLDAPSubSys:
|
||||
off = !xldap.Enabled(item.Config)
|
||||
case config.IdentityTLSSubSys:
|
||||
off = !globalSTSTLSConfig.Enabled
|
||||
off = !globalIAMSys.STSTLSConfig.Enabled
|
||||
case config.IdentityPluginSubSys:
|
||||
off = !idplugin.Enabled(item.Config)
|
||||
}
|
||||
@@ -518,3 +537,18 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
|
||||
|
||||
writeSuccessResponseJSON(w, econfigData)
|
||||
}
|
||||
|
||||
// setLoggerWebhookSubnetProxy - Sets the logger webhook's subnet proxy value to
|
||||
// one being set for subnet proxy
|
||||
func setLoggerWebhookSubnetProxy(subSys string, cfg config.Config) bool {
|
||||
if subSys == config.SubnetSubSys {
|
||||
subnetWebhookCfg := cfg[config.LoggerWebhookSubSys][subnet.LoggerWebhookName]
|
||||
loggerWebhookSubnetProxy := subnetWebhookCfg.Get(logger.Proxy)
|
||||
subnetProxy := cfg[config.SubnetSubSys][config.Default].Get(logger.Proxy)
|
||||
if loggerWebhookSubnetProxy != subnetProxy {
|
||||
subnetWebhookCfg.Set(logger.Proxy, subnetProxy)
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -178,9 +178,9 @@ func handleCreateUpdateValidation(s config.Config, subSys, cfgTarget string, isU
|
||||
var cfgInfos []madmin.IDPCfgInfo
|
||||
switch subSys {
|
||||
case madmin.IdentityOpenIDSubSys:
|
||||
cfgInfos, _ = globalOpenIDConfig.GetConfigInfo(s, cfgTarget)
|
||||
cfgInfos, _ = globalIAMSys.OpenIDConfig.GetConfigInfo(s, cfgTarget)
|
||||
case madmin.IdentityLDAPSubSys:
|
||||
cfgInfos, _ = globalLDAPConfig.GetConfigInfo(s, cfgTarget)
|
||||
cfgInfos, _ = globalIAMSys.LDAPConfig.GetConfigInfo(s, cfgTarget)
|
||||
}
|
||||
|
||||
if len(cfgInfos) > 0 && !isUpdate {
|
||||
@@ -240,10 +240,10 @@ func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http
|
||||
switch idpCfgType {
|
||||
case madmin.OpenidIDPCfg:
|
||||
cfg := globalServerConfig.Clone()
|
||||
cfgList, err = globalOpenIDConfig.GetConfigList(cfg)
|
||||
cfgList, err = globalIAMSys.OpenIDConfig.GetConfigList(cfg)
|
||||
case madmin.LDAPIDPCfg:
|
||||
cfg := globalServerConfig.Clone()
|
||||
cfgList, err = globalLDAPConfig.GetConfigList(cfg)
|
||||
cfgList, err = globalIAMSys.LDAPConfig.GetConfigList(cfg)
|
||||
|
||||
default:
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
@@ -296,9 +296,9 @@ func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.
|
||||
var err error
|
||||
switch idpCfgType {
|
||||
case madmin.OpenidIDPCfg:
|
||||
cfgInfos, err = globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
|
||||
cfgInfos, err = globalIAMSys.OpenIDConfig.GetConfigInfo(cfg, cfgName)
|
||||
case madmin.LDAPIDPCfg:
|
||||
cfgInfos, err = globalLDAPConfig.GetConfigInfo(cfg, cfgName)
|
||||
cfgInfos, err = globalIAMSys.LDAPConfig.GetConfigInfo(cfg, cfgName)
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, openid.ErrProviderConfigNotFound) || errors.Is(err, cfgldap.ErrProviderConfigNotFound) {
|
||||
@@ -355,7 +355,7 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
|
||||
switch idpCfgType {
|
||||
case madmin.OpenidIDPCfg:
|
||||
subSys = config.IdentityOpenIDSubSys
|
||||
cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfgCopy, cfgName)
|
||||
cfgInfos, err := globalIAMSys.OpenIDConfig.GetConfigInfo(cfgCopy, cfgName)
|
||||
if err != nil {
|
||||
if errors.Is(err, openid.ErrProviderConfigNotFound) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
|
||||
@@ -380,7 +380,7 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
|
||||
}
|
||||
case madmin.LDAPIDPCfg:
|
||||
subSys = config.IdentityLDAPSubSys
|
||||
cfgInfos, err := globalLDAPConfig.GetConfigInfo(cfgCopy, cfgName)
|
||||
cfgInfos, err := globalIAMSys.LDAPConfig.GetConfigInfo(cfgCopy, cfgName)
|
||||
if err != nil {
|
||||
if errors.Is(err, openid.ErrProviderConfigNotFound) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
|
||||
|
||||
@@ -27,12 +27,12 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v2"
|
||||
minio "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio/internal/sync/errgroup"
|
||||
)
|
||||
|
||||
func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {
|
||||
@@ -129,18 +129,21 @@ func (s *TestSuiteIAM) TestDeleteUserRace(c *check) {
|
||||
secretKeys[i] = secretKey
|
||||
}
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
g := errgroup.Group{}
|
||||
for i := 0; i < userCount; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
uClient := s.getUserClient(c, accessKeys[i], secretKeys[i], "")
|
||||
err := s.adm.RemoveUser(ctx, accessKeys[i])
|
||||
if err != nil {
|
||||
c.Fatalf("unable to remove user: %v", err)
|
||||
g.Go(func(i int) func() error {
|
||||
return func() error {
|
||||
uClient := s.getUserClient(c, accessKeys[i], secretKeys[i], "")
|
||||
err := s.adm.RemoveUser(ctx, accessKeys[i])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.mustNotListObjects(ctx, uClient, bucket)
|
||||
return nil
|
||||
}
|
||||
c.mustNotListObjects(ctx, uClient, bucket)
|
||||
}(i)
|
||||
}(i), i)
|
||||
}
|
||||
if errs := g.Wait(); len(errs) > 0 {
|
||||
c.Fatalf("unable to remove users: %v", errs)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
+34
-19
@@ -431,7 +431,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// Not allowed to add a user with same access key as root credential
|
||||
if owner && accessKey == cred.AccessKey {
|
||||
if accessKey == globalActiveCred.AccessKey {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -650,10 +650,11 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
|
||||
opts := newServiceAccountOpts{
|
||||
accessKey: createReq.AccessKey,
|
||||
secretKey: createReq.SecretKey,
|
||||
comment: createReq.Comment,
|
||||
claims: make(map[string]interface{}),
|
||||
accessKey: createReq.AccessKey,
|
||||
secretKey: createReq.SecretKey,
|
||||
comment: createReq.Comment,
|
||||
expiration: createReq.Expiration,
|
||||
claims: make(map[string]interface{}),
|
||||
}
|
||||
|
||||
// Find the user for the request sender (as it may be sent via a service
|
||||
@@ -741,9 +742,9 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
// In case of LDAP we need to resolve the targetUser to a DN and
|
||||
// query their groups:
|
||||
if globalLDAPConfig.Enabled() {
|
||||
if globalIAMSys.LDAPConfig.Enabled() {
|
||||
opts.claims[ldapUserN] = targetUser // simple username
|
||||
targetUser, targetGroups, err = globalLDAPConfig.LookupUserDN(targetUser)
|
||||
targetUser, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(targetUser)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -775,8 +776,9 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
createResp := madmin.AddServiceAccountResp{
|
||||
Credentials: madmin.Credentials{
|
||||
AccessKey: newCred.AccessKey,
|
||||
SecretKey: newCred.SecretKey,
|
||||
AccessKey: newCred.AccessKey,
|
||||
SecretKey: newCred.SecretKey,
|
||||
Expiration: newCred.Expiration,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -809,6 +811,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
|
||||
Claims: opts.claims,
|
||||
SessionPolicy: createReq.Policy,
|
||||
Status: auth.AccountOn,
|
||||
Expiration: createReq.Expiration,
|
||||
},
|
||||
},
|
||||
UpdatedAt: updatedAt,
|
||||
@@ -891,6 +894,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
secretKey: updateReq.NewSecretKey,
|
||||
status: updateReq.NewStatus,
|
||||
comment: updateReq.NewComment,
|
||||
expiration: updateReq.NewExpiration,
|
||||
sessionPolicy: sp,
|
||||
}
|
||||
updatedAt, err := globalIAMSys.UpdateServiceAccount(ctx, accessKey, opts)
|
||||
@@ -910,6 +914,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
Status: opts.status,
|
||||
Comment: opts.comment,
|
||||
SessionPolicy: updateReq.NewPolicy,
|
||||
Expiration: updateReq.NewExpiration,
|
||||
},
|
||||
},
|
||||
UpdatedAt: updatedAt,
|
||||
@@ -988,12 +993,18 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
var expiration *time.Time
|
||||
if !svcAccount.Expiration.IsZero() && !svcAccount.Expiration.Equal(timeSentinel) {
|
||||
expiration = &svcAccount.Expiration
|
||||
}
|
||||
|
||||
infoResp := madmin.InfoServiceAccountResp{
|
||||
ParentUser: svcAccount.ParentUser,
|
||||
Comment: svcAccount.Comment,
|
||||
AccountStatus: svcAccount.Status,
|
||||
ImpliedPolicy: policy == nil,
|
||||
Policy: string(policyJSON),
|
||||
Expiration: expiration,
|
||||
}
|
||||
|
||||
data, err := json.Marshal(infoResp)
|
||||
@@ -1321,11 +1332,12 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
// Fetch the data usage of the current bucket
|
||||
var size uint64
|
||||
var objectsCount uint64
|
||||
var objectsHist map[string]uint64
|
||||
var objectsHist, versionsHist map[string]uint64
|
||||
if !dataUsageInfo.LastUpdate.IsZero() {
|
||||
size = dataUsageInfo.BucketsUsage[bucket.Name].Size
|
||||
objectsCount = dataUsageInfo.BucketsUsage[bucket.Name].ObjectsCount
|
||||
objectsHist = dataUsageInfo.BucketsUsage[bucket.Name].ObjectSizesHistogram
|
||||
versionsHist = dataUsageInfo.BucketsUsage[bucket.Name].ObjectVersionsHistogram
|
||||
}
|
||||
// Fetch the prefix usage of the current bucket
|
||||
var prefixUsage map[string]uint64
|
||||
@@ -1339,12 +1351,13 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
tcfg, _, _ := globalBucketMetadataSys.GetTaggingConfig(bucket.Name)
|
||||
|
||||
acctInfo.Buckets = append(acctInfo.Buckets, madmin.BucketAccessInfo{
|
||||
Name: bucket.Name,
|
||||
Created: bucket.Created,
|
||||
Size: size,
|
||||
Objects: objectsCount,
|
||||
ObjectSizesHistogram: objectsHist,
|
||||
PrefixUsage: prefixUsage,
|
||||
Name: bucket.Name,
|
||||
Created: bucket.Created,
|
||||
Size: size,
|
||||
Objects: objectsCount,
|
||||
ObjectSizesHistogram: objectsHist,
|
||||
ObjectVersionsHistogram: versionsHist,
|
||||
PrefixUsage: prefixUsage,
|
||||
Details: &madmin.BucketDetails{
|
||||
Versioning: globalBucketVersioningSys.Enabled(bucket.Name),
|
||||
VersioningSuspended: globalBucketVersioningSys.Suspended(bucket.Name),
|
||||
@@ -2284,7 +2297,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
for accessKey, ureq := range userAccts {
|
||||
// Not allowed to add a user with same access key as root credential
|
||||
if owner && accessKey == cred.AccessKey {
|
||||
if accessKey == globalActiveCred.AccessKey {
|
||||
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAddUserInvalidArgument, err, allUsersFile, accessKey), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -2436,6 +2449,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
secretKey: svcAcctReq.SecretKey,
|
||||
status: svcAcctReq.Status,
|
||||
comment: svcAcctReq.Comment,
|
||||
expiration: svcAcctReq.Expiration,
|
||||
sessionPolicy: sp,
|
||||
}
|
||||
_, err = globalIAMSys.UpdateServiceAccount(ctx, svcAcctReq.AccessKey, opts)
|
||||
@@ -2451,13 +2465,14 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
sessionPolicy: sp,
|
||||
claims: svcAcctReq.Claims,
|
||||
comment: svcAcctReq.Comment,
|
||||
expiration: svcAcctReq.Expiration,
|
||||
}
|
||||
|
||||
// In case of LDAP we need to resolve the targetUser to a DN and
|
||||
// query their groups:
|
||||
if globalLDAPConfig.Enabled() {
|
||||
if globalIAMSys.LDAPConfig.Enabled() {
|
||||
opts.claims[ldapUserN] = svcAcctReq.AccessKey // simple username
|
||||
targetUser, _, err := globalLDAPConfig.LookupUserDN(svcAcctReq.AccessKey)
|
||||
targetUser, _, err := globalIAMSys.LDAPConfig.LookupUserDN(svcAcctReq.AccessKey)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
|
||||
return
|
||||
|
||||
@@ -825,7 +825,7 @@ func (s *TestSuiteIAM) TestGroupAddRemove(c *check) {
|
||||
if set.CreateStringSet(groups...).Contains(group) {
|
||||
c.Fatalf("created group still present!")
|
||||
}
|
||||
groupInfo, err = s.adm.GetGroupDescription(ctx, group)
|
||||
_, err = s.adm.GetGroupDescription(ctx, group)
|
||||
if err == nil {
|
||||
c.Fatalf("group appears to exist")
|
||||
}
|
||||
|
||||
@@ -803,6 +803,8 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
globalProfilerMu.Lock()
|
||||
defer globalProfilerMu.Unlock()
|
||||
for k, v := range globalProfiler {
|
||||
v.Stop()
|
||||
delete(globalProfiler, k)
|
||||
@@ -1819,8 +1821,8 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
|
||||
kmsStat := fetchKMSStatus()
|
||||
|
||||
ldap := madmin.LDAP{}
|
||||
if globalLDAPConfig.Enabled() {
|
||||
ldapConn, err := globalLDAPConfig.LDAP.Connect()
|
||||
if globalIAMSys.LDAPConfig.Enabled() {
|
||||
ldapConn, err := globalIAMSys.LDAPConfig.LDAP.Connect()
|
||||
//nolint:gocritic
|
||||
if err != nil {
|
||||
ldap.Status = string(madmin.ItemOffline)
|
||||
@@ -2427,7 +2429,7 @@ func assignPoolNumbers(servers []madmin.ServerProperties) {
|
||||
func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
|
||||
lambdaMap := make(map[string][]madmin.TargetIDStatus)
|
||||
|
||||
for _, tgt := range globalConfigTargetList.Targets() {
|
||||
for _, tgt := range globalNotifyTargetList.Targets() {
|
||||
targetIDStatus := make(map[string]madmin.Status)
|
||||
active, _ := tgt.IsActive()
|
||||
targetID := tgt.ID()
|
||||
|
||||
+54
-62
@@ -42,6 +42,7 @@ import (
|
||||
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
levent "github.com/minio/minio/internal/config/lambda/event"
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
@@ -165,7 +166,6 @@ const (
|
||||
ErrMalformedDate
|
||||
ErrMalformedPresignedDate
|
||||
ErrMalformedCredentialDate
|
||||
ErrMalformedCredentialRegion
|
||||
ErrMalformedExpires
|
||||
ErrNegativeExpires
|
||||
ErrAuthHeaderEmpty
|
||||
@@ -244,8 +244,6 @@ const (
|
||||
// Add new extended error codes here.
|
||||
|
||||
// MinIO extended errors.
|
||||
ErrReadQuorum
|
||||
ErrWriteQuorum
|
||||
ErrStorageFull
|
||||
ErrRequestBodyParse
|
||||
ErrObjectExistsAsDirectory
|
||||
@@ -269,6 +267,7 @@ const (
|
||||
ErrAdminNoSuchUser
|
||||
ErrAdminNoSuchGroup
|
||||
ErrAdminGroupNotEmpty
|
||||
ErrAdminGroupDisabled
|
||||
ErrAdminNoSuchJob
|
||||
ErrAdminNoSuchPolicy
|
||||
ErrAdminPolicyChangeAlreadyApplied
|
||||
@@ -412,6 +411,12 @@ const (
|
||||
ErrPostPolicyConditionInvalidFormat
|
||||
|
||||
ErrInvalidChecksum
|
||||
|
||||
// Lambda functions
|
||||
ErrLambdaARNInvalid
|
||||
ErrLambdaARNNotFound
|
||||
|
||||
apiErrCodeEnd // This is used only for the testing code
|
||||
)
|
||||
|
||||
type errorCodeMap map[APIErrorCode]APIError
|
||||
@@ -425,8 +430,7 @@ func (e errorCodeMap) ToAPIErrWithErr(errCode APIErrorCode, err error) APIError
|
||||
apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
|
||||
}
|
||||
if globalSite.Region != "" {
|
||||
switch errCode {
|
||||
case ErrAuthorizationHeaderMalformed:
|
||||
if errCode == ErrAuthorizationHeaderMalformed {
|
||||
apiErr.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalSite.Region)
|
||||
return apiErr
|
||||
}
|
||||
@@ -1262,6 +1266,11 @@ var errorCodes = errorCodeMap{
|
||||
Description: "The specified group is not empty - cannot remove it.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminGroupDisabled: {
|
||||
Code: "XMinioAdminGroupDisabled",
|
||||
Description: "The specified group is disabled.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminNoSuchPolicy: {
|
||||
Code: "XMinioAdminNoSuchPolicy",
|
||||
Description: "The canned policy does not exist.",
|
||||
@@ -1521,7 +1530,7 @@ var errorCodes = errorCodeMap{
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrBusy: {
|
||||
Code: "Busy",
|
||||
Code: "ServerBusy",
|
||||
Description: "The service is unavailable. Please retry.",
|
||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||
},
|
||||
@@ -1960,6 +1969,16 @@ var errorCodes = errorCodeMap{
|
||||
Description: "Invalid checksum provided.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrLambdaARNInvalid: {
|
||||
Code: "LambdaARNInvalid",
|
||||
Description: "The specified lambda ARN is invalid",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrLambdaARNNotFound: {
|
||||
Code: "LambdaARNNotFound",
|
||||
Description: "The specified lambda ARN does not exist",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrPolicyAlreadyAttached: {
|
||||
Code: "XMinioPolicyAlreadyAttached",
|
||||
Description: "The specified policy is already attached.",
|
||||
@@ -1983,15 +2002,15 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
|
||||
// Only return ErrClientDisconnected if the provided context is actually canceled.
|
||||
// This way downstream context.Canceled will still report ErrOperationTimedOut
|
||||
if contextCanceled(ctx) {
|
||||
if ctx.Err() == context.Canceled {
|
||||
return ErrClientDisconnected
|
||||
}
|
||||
if contextCanceled(ctx) && errors.Is(ctx.Err(), context.Canceled) {
|
||||
return ErrClientDisconnected
|
||||
}
|
||||
|
||||
switch err {
|
||||
case errInvalidArgument:
|
||||
apiErr = ErrAdminInvalidArgument
|
||||
case errNoSuchPolicy:
|
||||
apiErr = ErrAdminNoSuchPolicy
|
||||
case errNoSuchUser:
|
||||
apiErr = ErrAdminNoSuchUser
|
||||
case errNoSuchServiceAccount:
|
||||
@@ -2002,8 +2021,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrAdminGroupNotEmpty
|
||||
case errNoSuchJob:
|
||||
apiErr = ErrAdminNoSuchJob
|
||||
case errNoSuchPolicy:
|
||||
apiErr = ErrAdminNoSuchPolicy
|
||||
case errNoPolicyToAttachOrDetach:
|
||||
apiErr = ErrAdminPolicyChangeAlreadyApplied
|
||||
case errSignatureMismatch:
|
||||
@@ -2022,6 +2039,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrAdminInvalidSecretKey
|
||||
case errInvalidStorageClass:
|
||||
apiErr = ErrInvalidStorageClass
|
||||
case errErasureReadQuorum:
|
||||
apiErr = ErrSlowDown
|
||||
case errErasureWriteQuorum:
|
||||
apiErr = ErrSlowDown
|
||||
// SSE errors
|
||||
case errInvalidEncryptionParameters:
|
||||
apiErr = ErrInvalidEncryptionParameters
|
||||
@@ -2055,7 +2076,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrKMSKeyNotFoundException
|
||||
case errKMSDefaultKeyAlreadyConfigured:
|
||||
apiErr = ErrKMSDefaultKeyAlreadyConfigured
|
||||
|
||||
case context.Canceled, context.DeadlineExceeded:
|
||||
apiErr = ErrOperationTimedOut
|
||||
case errDiskNotFound:
|
||||
@@ -2073,8 +2093,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
}
|
||||
|
||||
// Compression errors
|
||||
switch err {
|
||||
case errInvalidDecompressedSize:
|
||||
if err == errInvalidDecompressedSize {
|
||||
apiErr = ErrInvalidDecompressedSize
|
||||
}
|
||||
|
||||
@@ -2085,7 +2104,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
|
||||
// etcd specific errors, a key is always a bucket for us return
|
||||
// ErrNoSuchBucket in such a case.
|
||||
if err == dns.ErrNoEntriesFound {
|
||||
if errors.Is(err, dns.ErrNoEntriesFound) {
|
||||
return ErrNoSuchBucket
|
||||
}
|
||||
|
||||
@@ -2210,6 +2229,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrARNNotification
|
||||
case *event.ErrARNNotFound:
|
||||
apiErr = ErrARNNotification
|
||||
case *levent.ErrInvalidARN:
|
||||
apiErr = ErrLambdaARNInvalid
|
||||
case *levent.ErrARNNotFound:
|
||||
apiErr = ErrLambdaARNNotFound
|
||||
case *event.ErrUnknownRegion:
|
||||
apiErr = ErrRegionNotification
|
||||
case *event.ErrInvalidFilterName:
|
||||
@@ -2273,34 +2296,17 @@ func toAPIError(ctx context.Context, err error) APIError {
|
||||
}
|
||||
|
||||
apiErr := errorCodes.ToAPIErr(toAPIErrorCode(ctx, err))
|
||||
e, ok := err.(dns.ErrInvalidBucketName)
|
||||
if ok {
|
||||
code := toAPIErrorCode(ctx, e)
|
||||
apiErr = errorCodes.ToAPIErrWithErr(code, e)
|
||||
}
|
||||
|
||||
if apiErr.Code == "NotImplemented" {
|
||||
switch e := err.(type) {
|
||||
case NotImplemented:
|
||||
desc := e.Error()
|
||||
if desc == "" {
|
||||
desc = apiErr.Description
|
||||
}
|
||||
apiErr = APIError{
|
||||
Code: apiErr.Code,
|
||||
Description: desc,
|
||||
HTTPStatusCode: apiErr.HTTPStatusCode,
|
||||
}
|
||||
return apiErr
|
||||
switch apiErr.Code {
|
||||
case "NotImplemented":
|
||||
desc := fmt.Sprintf("%s (%v)", apiErr.Description, err)
|
||||
apiErr = APIError{
|
||||
Code: apiErr.Code,
|
||||
Description: desc,
|
||||
HTTPStatusCode: apiErr.HTTPStatusCode,
|
||||
}
|
||||
}
|
||||
|
||||
if apiErr.Code == "XMinioBackendDown" {
|
||||
case "XMinioBackendDown":
|
||||
apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err)
|
||||
return apiErr
|
||||
}
|
||||
|
||||
if apiErr.Code == "InternalError" {
|
||||
case "InternalError":
|
||||
// If we see an internal error try to interpret
|
||||
// any underlying errors if possible depending on
|
||||
// their internal error types.
|
||||
@@ -2315,22 +2321,20 @@ func toAPIError(ctx context.Context, err error) APIError {
|
||||
}
|
||||
case *xml.SyntaxError:
|
||||
apiErr = APIError{
|
||||
Code: "MalformedXML",
|
||||
Description: fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description,
|
||||
e.Error()),
|
||||
Code: "MalformedXML",
|
||||
Description: fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description, e),
|
||||
HTTPStatusCode: errorCodes[ErrMalformedXML].HTTPStatusCode,
|
||||
}
|
||||
case url.EscapeError:
|
||||
apiErr = APIError{
|
||||
Code: "XMinioInvalidObjectName",
|
||||
Description: fmt.Sprintf("%s (%s)", errorCodes[ErrInvalidObjectName].Description,
|
||||
e.Error()),
|
||||
Code: "XMinioInvalidObjectName",
|
||||
Description: fmt.Sprintf("%s (%s)", errorCodes[ErrInvalidObjectName].Description, e),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
case versioning.Error:
|
||||
apiErr = APIError{
|
||||
Code: "IllegalVersioningConfigurationException",
|
||||
Description: fmt.Sprintf("Versioning configuration specified in the request is invalid. (%s)", e.Error()),
|
||||
Description: fmt.Sprintf("Versioning configuration specified in the request is invalid. (%s)", e),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
case lifecycle.Error:
|
||||
@@ -2396,19 +2400,7 @@ func toAPIError(ctx context.Context, err error) APIError {
|
||||
// Add more other SDK related errors here if any in future.
|
||||
default:
|
||||
//nolint:gocritic
|
||||
if errors.Is(err, errMalformedEncoding) {
|
||||
apiErr = APIError{
|
||||
Code: "BadRequest",
|
||||
Description: err.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
} else if errors.Is(err, errChunkTooBig) {
|
||||
apiErr = APIError{
|
||||
Code: "BadRequest",
|
||||
Description: err.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
} else if errors.Is(err, strconv.ErrRange) {
|
||||
if errors.Is(err, errMalformedEncoding) || errors.Is(err, errChunkTooBig) || errors.Is(err, strconv.ErrRange) {
|
||||
apiErr = APIError{
|
||||
Code: "BadRequest",
|
||||
Description: err.Error(),
|
||||
|
||||
@@ -80,3 +80,19 @@ func TestAPIErrCode(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if an API error is properly defined
|
||||
func TestAPIErrCodeDefinition(t *testing.T) {
|
||||
for errAPI := ErrNone + 1; errAPI < apiErrCodeEnd; errAPI++ {
|
||||
errCode, ok := errorCodes[errAPI]
|
||||
if !ok {
|
||||
t.Fatal(errAPI, "error code is not defined in the API error code table")
|
||||
}
|
||||
if errCode.Code == "" {
|
||||
t.Fatal(errAPI, "error code has an empty XML code")
|
||||
}
|
||||
if errCode.HTTPStatusCode == 0 {
|
||||
t.Fatal(errAPI, "error code has a zero HTTP status code")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+9
-2
@@ -285,7 +285,10 @@ func registerAPIRouter(router *mux.Router) {
|
||||
// GetObjectLegalHold
|
||||
router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||
collectAPIStats("getobjectlegalhold", maxClients(gz(httpTraceAll(api.GetObjectLegalHoldHandler))))).Queries("legal-hold", "")
|
||||
// GetObject - note gzip compression is *not* added due to Range requests.
|
||||
// GetObject with lambda ARNs
|
||||
router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||
collectAPIStats("getobject", maxClients(gz(httpTraceHdrs(api.GetObjectLambdaHandler))))).Queries("lambdaArn", "{lambdaArn:.*}")
|
||||
// GetObject
|
||||
router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||
collectAPIStats("getobject", maxClients(gz(httpTraceHdrs(api.GetObjectHandler)))))
|
||||
// CopyObject
|
||||
@@ -516,7 +519,11 @@ func corsHandler(handler http.Handler) http.Handler {
|
||||
|
||||
return cors.New(cors.Options{
|
||||
AllowOriginFunc: func(origin string) bool {
|
||||
for _, allowedOrigin := range globalAPIConfig.getCorsAllowOrigins() {
|
||||
allowedOrigins := globalAPIConfig.getCorsAllowOrigins()
|
||||
if len(allowedOrigins) == 0 {
|
||||
allowedOrigins = []string{"*"}
|
||||
}
|
||||
for _, allowedOrigin := range allowedOrigins {
|
||||
if wildcard.MatchSimple(allowedOrigin, origin) {
|
||||
return true
|
||||
}
|
||||
|
||||
+2
-8
@@ -94,14 +94,8 @@ func s3URLEncode(s string) string {
|
||||
}
|
||||
|
||||
// s3EncodeName encodes string in response when encodingType is specified in AWS S3 requests.
|
||||
func s3EncodeName(name string, encodingType string) (result string) {
|
||||
// Quick path to exit
|
||||
if encodingType == "" {
|
||||
return name
|
||||
}
|
||||
encodingType = strings.ToLower(encodingType)
|
||||
switch encodingType {
|
||||
case "url":
|
||||
func s3EncodeName(name, encodingType string) string {
|
||||
if strings.ToLower(encodingType) == "url" {
|
||||
return s3URLEncode(name)
|
||||
}
|
||||
return name
|
||||
|
||||
+219
-218
File diff suppressed because one or more lines are too long
+7
-2
@@ -154,6 +154,10 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
|
||||
return cred, owner, s3Err
|
||||
}
|
||||
|
||||
logger.GetReqInfo(ctx).Cred = cred
|
||||
logger.GetReqInfo(ctx).Owner = owner
|
||||
logger.GetReqInfo(ctx).Region = globalSite.Region
|
||||
|
||||
return cred, owner, ErrNone
|
||||
}
|
||||
|
||||
@@ -253,7 +257,7 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
|
||||
return nil, ErrNoAccessKey
|
||||
}
|
||||
|
||||
if token == "" && cred.IsTemp() {
|
||||
if token == "" && cred.IsTemp() && !cred.IsServiceAccount() {
|
||||
// Temporary credentials should always have x-amz-security-token
|
||||
return nil, ErrInvalidToken
|
||||
}
|
||||
@@ -263,7 +267,7 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
|
||||
return nil, ErrInvalidToken
|
||||
}
|
||||
|
||||
if cred.IsTemp() && subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
|
||||
if !cred.IsServiceAccount() && cred.IsTemp() && subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
|
||||
// validate token for temporary credentials only.
|
||||
return nil, ErrInvalidToken
|
||||
}
|
||||
@@ -344,6 +348,7 @@ func authenticateRequest(ctx context.Context, r *http.Request, action policy.Act
|
||||
|
||||
logger.GetReqInfo(ctx).Cred = cred
|
||||
logger.GetReqInfo(ctx).Owner = owner
|
||||
logger.GetReqInfo(ctx).Region = globalSite.Region
|
||||
|
||||
// region is valid only for CreateBucketAction.
|
||||
var region string
|
||||
|
||||
@@ -271,6 +271,9 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
|
||||
}
|
||||
|
||||
func getLocalDisksToHeal() (disksToHeal Endpoints) {
|
||||
globalLocalDrivesMu.RLock()
|
||||
globalLocalDrives := globalLocalDrives
|
||||
globalLocalDrivesMu.RUnlock()
|
||||
for _, disk := range globalLocalDrives {
|
||||
_, err := disk.GetDiskID()
|
||||
if errors.Is(err, errUnformattedDisk) {
|
||||
@@ -299,7 +302,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
|
||||
defer disk.Close()
|
||||
poolIdx := globalEndpoints.GetLocalPoolIdx(disk.Endpoint())
|
||||
if poolIdx < 0 {
|
||||
return fmt.Errorf("unexpected pool index (%d) found in %s", poolIdx, disk.Endpoint())
|
||||
return fmt.Errorf("unexpected pool index (%d) found for %s", poolIdx, disk.Endpoint())
|
||||
}
|
||||
|
||||
// Calculate the set index where the current endpoint belongs
|
||||
@@ -310,14 +313,15 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
|
||||
return err
|
||||
}
|
||||
if setIdx < 0 {
|
||||
return fmt.Errorf("unexpected set index (%d) found in %s", setIdx, disk.Endpoint())
|
||||
return fmt.Errorf("unexpected set index (%d) found for %s", setIdx, disk.Endpoint())
|
||||
}
|
||||
|
||||
// Prevent parallel erasure set healing
|
||||
locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-drive-healing/%d/%d", poolIdx, setIdx))
|
||||
lkctx, err := locker.GetLock(ctx, newDiskHealingTimeout)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("Healing of drive '%v' on %s pool, belonging to %s erasure set already in progress: %w",
|
||||
disk, humanize.Ordinal(poolIdx+1), humanize.Ordinal(setIdx+1), err)
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer locker.Unlock(lkctx)
|
||||
@@ -325,19 +329,20 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
|
||||
// Load healing tracker in this disk
|
||||
tracker, err := loadHealingTracker(ctx, disk)
|
||||
if err != nil {
|
||||
// A healing track can be not found when another disk in the same
|
||||
// erasure set and same healing-id successfully finished healing.
|
||||
if err == errFileNotFound {
|
||||
// A healing tracker may be deleted if another disk in the
|
||||
// same erasure set with same healing-id successfully finished
|
||||
// healing.
|
||||
if errors.Is(err, errFileNotFound) {
|
||||
return nil
|
||||
}
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to load a healing tracker on '%s': %w", disk, err))
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to load healing tracker on '%s': %w, re-initializing..", disk, err))
|
||||
tracker = newHealingTracker(disk, mustGetUUID())
|
||||
}
|
||||
|
||||
logger.Info(fmt.Sprintf("Proceeding to heal '%s' - 'mc admin heal alias/ --verbose' to check the status.", endpoint))
|
||||
logger.Info(fmt.Sprintf("Healing drive '%s' - 'mc admin heal alias/ --verbose' to check the current status.", endpoint))
|
||||
|
||||
buckets, _ := z.ListBuckets(ctx, BucketOptions{})
|
||||
// Buckets data are dispersed in multiple zones/sets, make
|
||||
// Buckets data are dispersed in multiple pools/sets, make
|
||||
// sure to heal all bucket metadata configuration.
|
||||
buckets = append(buckets, BucketInfo{
|
||||
Name: pathJoin(minioMetaBucket, minioConfigPrefix),
|
||||
@@ -355,7 +360,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
|
||||
})
|
||||
|
||||
if serverDebugLog {
|
||||
logger.Info("Healing drive '%v' on %s pool", disk, humanize.Ordinal(poolIdx+1))
|
||||
logger.Info("Healing drive '%v' on %s pool, belonging to %s erasure set", disk, humanize.Ordinal(poolIdx+1), humanize.Ordinal(setIdx+1))
|
||||
}
|
||||
|
||||
// Load bucket totals
|
||||
@@ -378,9 +383,9 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
|
||||
}
|
||||
|
||||
if tracker.ItemsFailed > 0 {
|
||||
logger.Info("Healing drive '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
|
||||
logger.Info("Healing of drive '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
|
||||
} else {
|
||||
logger.Info("Healing drive '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
|
||||
logger.Info("Healing of drive '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
|
||||
}
|
||||
|
||||
if len(tracker.QueuedBuckets) > 0 {
|
||||
@@ -392,7 +397,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
|
||||
logger.Info("\n")
|
||||
}
|
||||
|
||||
if tracker.HealID == "" { // HealID is empty only before Feb 2023
|
||||
if tracker.HealID == "" { // HealID was empty only before Feb 2023
|
||||
logger.LogIf(ctx, tracker.delete(ctx))
|
||||
return nil
|
||||
}
|
||||
@@ -401,7 +406,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
|
||||
for _, disk := range z.serverPools[poolIdx].sets[setIdx].getDisks() {
|
||||
t, err := loadHealingTracker(ctx, disk)
|
||||
if err != nil {
|
||||
if err != errFileNotFound {
|
||||
if !errors.Is(err, errFileNotFound) {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
continue
|
||||
@@ -446,10 +451,11 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) {
|
||||
for _, disk := range healDisks {
|
||||
go func(disk Endpoint) {
|
||||
globalBackgroundHealState.setDiskHealingStatus(disk, true)
|
||||
err := healFreshDisk(ctx, z, disk)
|
||||
if err != nil {
|
||||
if err := healFreshDisk(ctx, z, disk); err != nil {
|
||||
globalBackgroundHealState.setDiskHealingStatus(disk, false)
|
||||
printEndpointError(disk, err, false)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
printEndpointError(disk, err, false)
|
||||
}
|
||||
return
|
||||
}
|
||||
// Only upon success pop the healed disk.
|
||||
|
||||
@@ -755,19 +755,12 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
|
||||
// Parse incoming location constraint.
|
||||
location, s3Error := parseLocationConstraint(r)
|
||||
_, s3Error = parseLocationConstraint(r)
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate if location sent by the client is valid, reject
|
||||
// requests which do not follow valid region requirements.
|
||||
if !isValidLocation(location) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRegion), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// check if client is attempting to create more buckets, complain about it.
|
||||
if currBuckets := globalBucketMetadataSys.Count(); currBuckets+1 > maxBuckets {
|
||||
logger.LogIf(ctx, fmt.Errorf("An attempt to create %d buckets beyond recommended %d", currBuckets+1, maxBuckets))
|
||||
|
||||
@@ -323,8 +323,7 @@ func (b *BucketMetadata) getAllLegacyConfigs(ctx context.Context, objectAPI Obje
|
||||
|
||||
configData, err := readConfig(ctx, objectAPI, configFile)
|
||||
if err != nil {
|
||||
switch err.(type) {
|
||||
case ObjectExistsAsDirectory:
|
||||
if _, ok := err.(ObjectExistsAsDirectory); ok {
|
||||
// in FS mode it possible that we have actual
|
||||
// files in this folder with `.minio.sys/buckets/bucket/configFile`
|
||||
continue
|
||||
|
||||
@@ -90,8 +90,8 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
|
||||
|
||||
opts.VersionID = object.VersionID
|
||||
if gerr != nil { // error from GetObjectInfo
|
||||
switch gerr.(type) {
|
||||
case MethodNotAllowed: // This happens usually for a delete marker
|
||||
if _, ok := gerr.(MethodNotAllowed); ok {
|
||||
// This happens usually for a delete marker
|
||||
if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
|
||||
// Delete marker should be present and valid.
|
||||
return ErrNone
|
||||
@@ -192,8 +192,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
|
||||
days, objRetention.RetainUntilDate.Time,
|
||||
objRetention.Mode, byPassSet, r, cred,
|
||||
owner)
|
||||
switch apiErr {
|
||||
case ErrAccessDenied:
|
||||
if apiErr == ErrAccessDenied {
|
||||
return errAuthentication
|
||||
}
|
||||
return nil
|
||||
@@ -210,8 +209,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
|
||||
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
||||
}
|
||||
}
|
||||
switch govPerm {
|
||||
case ErrAccessDenied:
|
||||
if govPerm == ErrAccessDenied {
|
||||
return errAuthentication
|
||||
}
|
||||
return nil
|
||||
@@ -224,8 +222,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
|
||||
apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
|
||||
days, objRetention.RetainUntilDate.Time, objRetention.Mode,
|
||||
false, r, cred, owner)
|
||||
switch apiErr {
|
||||
case ErrAccessDenied:
|
||||
if apiErr == ErrAccessDenied {
|
||||
return errAuthentication
|
||||
}
|
||||
return nil
|
||||
@@ -236,8 +233,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
|
||||
apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
|
||||
days, objRetention.RetainUntilDate.Time,
|
||||
objRetention.Mode, byPassSet, r, cred, owner)
|
||||
switch apiErr {
|
||||
case ErrAccessDenied:
|
||||
if apiErr == ErrAccessDenied {
|
||||
return errAuthentication
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -496,8 +496,7 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
|
||||
oi.VersionPurgeStatusInternal = fmt.Sprintf("%s=%s;", rcfg.Config.RoleArn, oi.VersionPurgeStatus)
|
||||
}
|
||||
for k, v := range oi.UserDefined {
|
||||
switch {
|
||||
case strings.EqualFold(k, ReservedMetadataPrefixLower+ReplicationReset):
|
||||
if strings.EqualFold(k, ReservedMetadataPrefixLower+ReplicationReset) {
|
||||
delete(oi.UserDefined, k)
|
||||
oi.UserDefined[targetResetHeader(rcfg.Config.RoleArn)] = v
|
||||
}
|
||||
|
||||
+22
-17
@@ -586,11 +586,13 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
|
||||
return
|
||||
}
|
||||
}
|
||||
// mark delete marker replication as failed if target cluster not ready to receive
|
||||
// this request yet (object version not replicated yet)
|
||||
if err != nil && !toi.ReplicationReady {
|
||||
rinfo.ReplicationStatus = replication.Failed
|
||||
return
|
||||
if !isErrObjectNotFound(ErrorRespToObjectError(err, dobj.Bucket, dobj.ObjectName)) {
|
||||
// mark delete marker replication as failed if target cluster not ready to receive
|
||||
// this request yet (object version not replicated yet)
|
||||
if err != nil && !toi.ReplicationReady {
|
||||
rinfo.ReplicationStatus = replication.Failed
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -802,10 +804,12 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
|
||||
oi1.ModTime.Unix() > oi2.LastModified.Unix() && oi1.VersionID == nullVersionID {
|
||||
return replicateNone
|
||||
}
|
||||
sz, _ := oi1.GetActualSize()
|
||||
|
||||
// needs full replication
|
||||
if oi1.ETag != oi2.ETag ||
|
||||
oi1.VersionID != oi2.VersionID ||
|
||||
oi1.Size != oi2.Size ||
|
||||
sz != oi2.Size ||
|
||||
oi1.DeleteMarker != oi2.IsDeleteMarker ||
|
||||
oi1.ModTime.Unix() != oi2.LastModified.Unix() {
|
||||
return replicateAll
|
||||
@@ -1202,12 +1206,6 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
|
||||
ReplicationAction: rAction,
|
||||
}
|
||||
|
||||
if ri.ObjectInfo.TargetReplicationStatus(tgt.ARN) == replication.Completed && !ri.ExistingObjResync.Empty() && !ri.ExistingObjResync.mustResyncTarget(tgt.ARN) {
|
||||
rinfo.ReplicationStatus = replication.Completed
|
||||
rinfo.ReplicationResynced = true
|
||||
return
|
||||
}
|
||||
|
||||
if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
|
||||
logger.LogIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN))
|
||||
sendEvent(eventArgs{
|
||||
@@ -1243,6 +1241,13 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
|
||||
|
||||
objInfo = gr.ObjInfo
|
||||
|
||||
// use latest ObjectInfo to check if previous replication attempt succeeded
|
||||
if objInfo.TargetReplicationStatus(tgt.ARN) == replication.Completed && !ri.ExistingObjResync.Empty() && !ri.ExistingObjResync.mustResyncTarget(tgt.ARN) {
|
||||
rinfo.ReplicationStatus = replication.Completed
|
||||
rinfo.ReplicationResynced = true
|
||||
return
|
||||
}
|
||||
|
||||
size, err := objInfo.GetActualSize()
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
@@ -1256,7 +1261,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
|
||||
}
|
||||
|
||||
if tgt.Bucket == "" {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to replicate object %s(%s), bucket is empty", objInfo.Name, objInfo.VersionID))
|
||||
logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s), bucket is empty", objInfo.Name, objInfo.VersionID))
|
||||
sendEvent(eventArgs{
|
||||
EventName: event.ObjectReplicationNotTracked,
|
||||
BucketName: bucket,
|
||||
@@ -1286,7 +1291,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
|
||||
if rAction == replicateNone {
|
||||
if ri.OpType == replication.ExistingObjectReplicationType &&
|
||||
objInfo.ModTime.Unix() > oi.LastModified.Unix() && objInfo.VersionID == nullVersionID {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to replicate %s/%s (null). Newer version exists on target", bucket, object))
|
||||
logger.LogIf(ctx, fmt.Errorf("unable to replicate %s/%s (null). Newer version exists on target", bucket, object))
|
||||
sendEvent(eventArgs{
|
||||
EventName: event.ObjectReplicationNotTracked,
|
||||
BucketName: bucket,
|
||||
@@ -1328,7 +1333,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
|
||||
}
|
||||
if _, err = c.CopyObject(ctx, tgt.Bucket, object, tgt.Bucket, object, getCopyObjMetadata(objInfo, tgt.StorageClass), srcOpts, dstOpts); err != nil {
|
||||
rinfo.ReplicationStatus = replication.Failed
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to replicate metadata for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
|
||||
logger.LogIf(ctx, fmt.Errorf("unable to replicate metadata for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
|
||||
}
|
||||
} else {
|
||||
var putOpts minio.PutObjectOptions
|
||||
@@ -1365,7 +1370,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
|
||||
r, objInfo, putOpts); err != nil {
|
||||
if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
|
||||
rinfo.ReplicationStatus = replication.Failed
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
|
||||
logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
|
||||
} else {
|
||||
rinfo.ReplicationStatus = replication.Completed
|
||||
}
|
||||
@@ -1374,7 +1379,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
|
||||
if _, err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts); err != nil {
|
||||
if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
|
||||
rinfo.ReplicationStatus = replication.Failed
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
|
||||
logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
|
||||
} else {
|
||||
rinfo.ReplicationStatus = replication.Completed
|
||||
}
|
||||
|
||||
+6
-5
@@ -205,7 +205,7 @@ func minioConfigToConsoleFeatures() {
|
||||
}
|
||||
}
|
||||
// Enable if LDAP is enabled.
|
||||
if globalLDAPConfig.Enabled() {
|
||||
if globalIAMSys.LDAPConfig.Enabled() {
|
||||
os.Setenv("CONSOLE_LDAP_ENABLED", config.EnableOn)
|
||||
}
|
||||
os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
|
||||
@@ -215,8 +215,9 @@ func minioConfigToConsoleFeatures() {
|
||||
}
|
||||
|
||||
func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
|
||||
m := make(map[string]consoleoauth2.ProviderConfig, len(globalOpenIDConfig.ProviderCfgs))
|
||||
for name, cfg := range globalOpenIDConfig.ProviderCfgs {
|
||||
pcfgs := globalIAMSys.OpenIDConfig.ProviderCfgs
|
||||
m := make(map[string]consoleoauth2.ProviderConfig, len(pcfgs))
|
||||
for name, cfg := range pcfgs {
|
||||
callback := getConsoleEndpoints()[0] + "/oauth_callback"
|
||||
if cfg.RedirectURI != "" {
|
||||
callback = cfg.RedirectURI
|
||||
@@ -660,6 +661,7 @@ func handleCommonEnvVars() {
|
||||
}
|
||||
u.Path = "" // remove any path component such as `/`
|
||||
globalMinioEndpoint = u.String()
|
||||
globalMinioEndpointURL = u
|
||||
}
|
||||
|
||||
globalFSOSync, err = config.ParseBool(env.Get(config.EnvFSOSync, config.EnableOff))
|
||||
@@ -782,8 +784,7 @@ func handleCommonEnvVars() {
|
||||
// Initialize KMS global variable after valiadating and loading the configuration.
|
||||
// It depends on KMS env variables and global cli flags.
|
||||
func handleKMSConfig() {
|
||||
switch {
|
||||
case env.IsSet(kms.EnvKMSSecretKey) && env.IsSet(kms.EnvKESEndpoint):
|
||||
if env.IsSet(kms.EnvKMSSecretKey) && env.IsSet(kms.EnvKESEndpoint) {
|
||||
logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", kms.EnvKMSSecretKey, kms.EnvKESEndpoint))
|
||||
}
|
||||
|
||||
|
||||
+27
-11
@@ -37,6 +37,7 @@ import (
|
||||
"github.com/minio/minio/internal/config/identity/openid"
|
||||
idplugin "github.com/minio/minio/internal/config/identity/plugin"
|
||||
xtls "github.com/minio/minio/internal/config/identity/tls"
|
||||
"github.com/minio/minio/internal/config/lambda"
|
||||
"github.com/minio/minio/internal/config/notify"
|
||||
"github.com/minio/minio/internal/config/policy/opa"
|
||||
polplugin "github.com/minio/minio/internal/config/policy/plugin"
|
||||
@@ -74,6 +75,9 @@ func initHelp() {
|
||||
for k, v := range notify.DefaultNotificationKVS {
|
||||
kvs[k] = v
|
||||
}
|
||||
for k, v := range lambda.DefaultLambdaKVS {
|
||||
kvs[k] = v
|
||||
}
|
||||
if globalIsErasure {
|
||||
kvs[config.StorageClassSubSys] = storageclass.DefaultKVS
|
||||
kvs[config.HealSubSys] = heal.DefaultKVS
|
||||
@@ -196,6 +200,11 @@ func initHelp() {
|
||||
Description: "publish bucket notifications to Redis datastores",
|
||||
MultipleTargets: true,
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: config.LambdaWebhookSubSys,
|
||||
Description: "manage remote lambda functions",
|
||||
MultipleTargets: true,
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: config.EtcdSubSys,
|
||||
Description: "persist IAM assets externally to etcd",
|
||||
@@ -246,6 +255,7 @@ func initHelp() {
|
||||
config.NotifyRedisSubSys: notify.HelpRedis,
|
||||
config.NotifyWebhookSubSys: notify.HelpWebhook,
|
||||
config.NotifyESSubSys: notify.HelpES,
|
||||
config.LambdaWebhookSubSys: lambda.HelpWebhook,
|
||||
config.SubnetSubSys: subnet.HelpSubnet,
|
||||
config.CallhomeSubSys: callhome.HelpCallhome,
|
||||
}
|
||||
@@ -387,6 +397,13 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if config.LambdaSubSystems.Contains(subSys) {
|
||||
if err := lambda.TestSubSysLambdaTargets(GlobalContext, s, subSys, NewHTTPTransport()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -423,6 +440,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
|
||||
}
|
||||
if err == nil && dnsURL != "" {
|
||||
bootstrapTrace("initialize remote bucket DNS store")
|
||||
globalDNSConfig, err = dns.NewOperatorDNS(dnsURL,
|
||||
dns.Authentication(dnsUser, dnsPass),
|
||||
dns.RootCAs(globalRootCAs))
|
||||
@@ -437,6 +455,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
|
||||
}
|
||||
|
||||
if etcdCfg.Enabled {
|
||||
bootstrapTrace("initialize etcd store")
|
||||
globalEtcdClient, err = etcd.New(etcdCfg)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
|
||||
@@ -493,23 +512,20 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
|
||||
logger.Fatal(errors.New("no KMS configured"), "MINIO_KMS_AUTO_ENCRYPTION requires a valid KMS configuration")
|
||||
}
|
||||
|
||||
globalSTSTLSConfig, err = xtls.Lookup(s[config.IdentityTLSSubSys][config.Default])
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize X.509/TLS STS API: %w", err))
|
||||
}
|
||||
|
||||
if globalSTSTLSConfig.InsecureSkipVerify {
|
||||
logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
|
||||
}
|
||||
|
||||
transport := NewHTTPTransport()
|
||||
|
||||
bootstrapTrace("lookup the event notification targets")
|
||||
globalConfigTargetList, err = notify.FetchEnabledTargets(GlobalContext, s, transport)
|
||||
bootstrapTrace("initialize the event notification targets")
|
||||
globalNotifyTargetList, err = notify.FetchEnabledTargets(GlobalContext, s, transport)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
|
||||
}
|
||||
|
||||
bootstrapTrace("initialize the lambda targets")
|
||||
globalLambdaTargetList, err = lambda.FetchEnabledTargets(GlobalContext, s, transport)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize lambda target(s): %w", err))
|
||||
}
|
||||
|
||||
bootstrapTrace("applying the dynamic configuration")
|
||||
// Apply dynamic config values
|
||||
if err := applyDynamicConfig(ctx, objAPI, s); err != nil {
|
||||
|
||||
@@ -33,8 +33,7 @@ const crossDomainXMLEntity = "/crossdomain.xml"
|
||||
func setCrossDomainPolicy(h http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Look for 'crossdomain.xml' in the incoming request.
|
||||
switch r.URL.Path {
|
||||
case crossDomainXMLEntity:
|
||||
if r.URL.Path == crossDomainXMLEntity {
|
||||
// Write the standard cross domain policy xml.
|
||||
w.Write([]byte(crossDomainXML))
|
||||
// Request completed, no need to serve to other handlers.
|
||||
|
||||
+7
-53
@@ -18,7 +18,6 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
@@ -34,7 +33,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/bits-and-blooms/bloom/v3"
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
@@ -162,7 +160,6 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
|
||||
|
||||
// Load current bloom cycle
|
||||
var cycleInfo currentScannerCycle
|
||||
cycleInfo.next = intDataUpdateTracker.current() + 1
|
||||
|
||||
buf, _ := readConfig(ctx, objAPI, dataUsageBloomNamePath)
|
||||
if len(buf) == 8 {
|
||||
@@ -207,9 +204,7 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
|
||||
// Wait before starting next cycle and wait on startup.
|
||||
results := make(chan DataUsageInfo, 1)
|
||||
go storeDataUsageInBackend(ctx, objAPI, results)
|
||||
bf, err := globalNotificationSys.updateBloomFilter(ctx, cycleInfo.current)
|
||||
logger.LogIf(ctx, err)
|
||||
err = objAPI.NSScanner(ctx, bf, results, uint32(cycleInfo.current), scanMode)
|
||||
err := objAPI.NSScanner(ctx, results, uint32(cycleInfo.current), scanMode)
|
||||
logger.LogIf(ctx, err)
|
||||
res := map[string]string{"cycle": fmt.Sprint(cycleInfo.current)}
|
||||
if err != nil {
|
||||
@@ -248,10 +243,8 @@ type folderScanner struct {
|
||||
oldCache dataUsageCache
|
||||
newCache dataUsageCache
|
||||
updateCache dataUsageCache
|
||||
withFilter *bloomFilter
|
||||
|
||||
dataUsageScannerDebug bool
|
||||
healFolderInclude uint32 // Include a clean folder one in n cycles.
|
||||
healObjectSelect uint32 // Do a heal check on an object once every n cycles. Must divide into healFolderInclude
|
||||
scanMode madmin.HealScanMode
|
||||
|
||||
@@ -310,8 +303,6 @@ type folderScanner struct {
|
||||
// Before each operation sleepDuration is called which can be used to temporarily halt the scanner.
|
||||
// If the supplied context is canceled the function will return at the first chance.
|
||||
func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, cache dataUsageCache, getSize getSizeFn, scanMode madmin.HealScanMode) (dataUsageCache, error) {
|
||||
logPrefix := color.Green("data-usage: ")
|
||||
|
||||
switch cache.Info.Name {
|
||||
case "", dataUsageRoot:
|
||||
return cache, errors.New("internal error: root scan attempted")
|
||||
@@ -325,8 +316,7 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
|
||||
oldCache: cache,
|
||||
newCache: dataUsageCache{Info: cache.Info},
|
||||
updateCache: dataUsageCache{Info: cache.Info},
|
||||
dataUsageScannerDebug: intDataUpdateTracker.debug,
|
||||
healFolderInclude: 0,
|
||||
dataUsageScannerDebug: false,
|
||||
healObjectSelect: 0,
|
||||
scanMode: scanMode,
|
||||
updates: cache.Info.updates,
|
||||
@@ -349,19 +339,9 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
|
||||
|
||||
// Enable healing in XL mode.
|
||||
if globalIsErasure && !cache.Info.SkipHealing {
|
||||
// Include a clean folder one in n cycles.
|
||||
s.healFolderInclude = healFolderIncludeProb
|
||||
// Do a heal check on an object once every n cycles. Must divide into healFolderInclude
|
||||
s.healObjectSelect = healObjectSelectProb
|
||||
}
|
||||
if len(cache.Info.BloomFilter) > 0 {
|
||||
s.withFilter = &bloomFilter{BloomFilter: &bloom.BloomFilter{}}
|
||||
_, err := s.withFilter.ReadFrom(bytes.NewReader(cache.Info.BloomFilter))
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err, logPrefix+"Error reading bloom filter")
|
||||
s.withFilter = nil
|
||||
}
|
||||
}
|
||||
|
||||
done := ctx.Done()
|
||||
|
||||
@@ -418,14 +398,12 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
existing, ok := f.oldCache.Cache[thisHash.Key()]
|
||||
var abandonedChildren dataUsageHashMap
|
||||
if !into.Compacted {
|
||||
abandonedChildren = f.oldCache.findChildrenCopy(thisHash)
|
||||
}
|
||||
|
||||
// If there are lifecycle rules for the prefix, remove the filter.
|
||||
filter := f.withFilter
|
||||
_, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
|
||||
var activeLifeCycle *lifecycle.Lifecycle
|
||||
if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix) {
|
||||
@@ -433,33 +411,13 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
|
||||
console.Debugf(scannerLogPrefix+" Prefix %q has active rules\n", prefix)
|
||||
}
|
||||
activeLifeCycle = f.oldCache.Info.lifeCycle
|
||||
filter = nil
|
||||
}
|
||||
// If there are replication rules for the prefix, remove the filter.
|
||||
var replicationCfg replicationConfig
|
||||
if !f.oldCache.Info.replication.Empty() && f.oldCache.Info.replication.Config.HasActiveRules(prefix, true) {
|
||||
replicationCfg = f.oldCache.Info.replication
|
||||
filter = nil
|
||||
}
|
||||
// Check if we can skip it due to bloom filter...
|
||||
if filter != nil && ok && existing.Compacted {
|
||||
// If folder isn't in filter and we have data, skip it completely.
|
||||
if folder.name != dataUsageRoot && !filter.containsDir(folder.name) {
|
||||
if f.healObjectSelect == 0 || !thisHash.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healFolderInclude/folder.objectHealProbDiv) {
|
||||
f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
|
||||
f.updateCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
|
||||
if f.dataUsageScannerDebug {
|
||||
console.Debugf(scannerLogPrefix+" Skipping non-updated folder: %v\n", folder.name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if f.dataUsageScannerDebug {
|
||||
console.Debugf(scannerLogPrefix+" Adding non-updated folder to heal check: %v\n", folder.name)
|
||||
}
|
||||
// If probability was already scannerHealFolderInclude, keep it.
|
||||
folder.objectHealProbDiv = f.healFolderInclude
|
||||
}
|
||||
}
|
||||
scannerSleeper.Sleep(ctx, dataScannerSleepPerFolder)
|
||||
|
||||
var existingFolders, newFolders []cachedFolder
|
||||
@@ -673,13 +631,10 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
|
||||
// and the entry itself is compacted.
|
||||
if !into.Compacted && f.oldCache.isCompacted(h) {
|
||||
if !h.mod(f.oldCache.Info.NextCycle, dataUsageUpdateDirCycles) {
|
||||
if f.healObjectSelect == 0 || !h.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healFolderInclude/folder.objectHealProbDiv) {
|
||||
// Transfer and add as child...
|
||||
f.newCache.copyWithChildren(&f.oldCache, h, folder.parent)
|
||||
into.addChild(h)
|
||||
continue
|
||||
}
|
||||
folder.objectHealProbDiv = f.healFolderInclude
|
||||
// Transfer and add as child...
|
||||
f.newCache.copyWithChildren(&f.oldCache, h, folder.parent)
|
||||
into.addChild(h)
|
||||
continue
|
||||
}
|
||||
}
|
||||
f.updateCurrentPath(folder.name)
|
||||
@@ -1318,8 +1273,7 @@ func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi Obj
|
||||
sizeS.replTargetStats[arn] = tgtSizeS
|
||||
}
|
||||
|
||||
switch oi.ReplicationStatus {
|
||||
case replication.Replica:
|
||||
if oi.ReplicationStatus == replication.Replica {
|
||||
sizeS.replicaSize += oi.Size
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,682 +0,0 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/bits-and-blooms/bloom/v3"
|
||||
"github.com/minio/minio/internal/color"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/console"
|
||||
)
|
||||
|
||||
const (
|
||||
// Estimate bloom filter size. With this many items
|
||||
dataUpdateTrackerEstItems = 200000
|
||||
// ... we want this false positive rate:
|
||||
dataUpdateTrackerFP = 0.1
|
||||
dataUpdateTrackerQueueSize = 0
|
||||
|
||||
dataUpdateTrackerFilename = dataUsageBucket + SlashSeparator + ".tracker.bin"
|
||||
dataUpdateTrackerVersion = 7
|
||||
dataUpdateTrackerSaveInterval = 5 * time.Minute
|
||||
)
|
||||
|
||||
var intDataUpdateTracker *dataUpdateTracker
|
||||
|
||||
func init() {
|
||||
intDataUpdateTracker = newDataUpdateTracker()
|
||||
}
|
||||
|
||||
type dataUpdateTracker struct {
|
||||
mu sync.Mutex
|
||||
input chan string
|
||||
save chan struct{}
|
||||
debug bool
|
||||
saveExited chan struct{}
|
||||
dirty bool
|
||||
|
||||
Current dataUpdateFilter
|
||||
History dataUpdateTrackerHistory
|
||||
Saved time.Time
|
||||
}
|
||||
|
||||
// newDataUpdateTracker returns a dataUpdateTracker with default settings.
|
||||
func newDataUpdateTracker() *dataUpdateTracker {
|
||||
d := &dataUpdateTracker{
|
||||
Current: dataUpdateFilter{
|
||||
idx: 1,
|
||||
},
|
||||
debug: serverDebugLog,
|
||||
input: make(chan string, dataUpdateTrackerQueueSize),
|
||||
save: make(chan struct{}, 1),
|
||||
saveExited: make(chan struct{}),
|
||||
}
|
||||
d.Current.bf = d.newBloomFilter()
|
||||
d.dirty = true
|
||||
return d
|
||||
}
|
||||
|
||||
type dataUpdateTrackerHistory []dataUpdateFilter
|
||||
|
||||
type dataUpdateFilter struct {
|
||||
idx uint64
|
||||
bf bloomFilter
|
||||
}
|
||||
|
||||
type bloomFilter struct {
|
||||
*bloom.BloomFilter
|
||||
}
|
||||
|
||||
// emptyBloomFilter returns an empty bloom filter.
|
||||
func emptyBloomFilter() bloomFilter {
|
||||
return bloomFilter{BloomFilter: &bloom.BloomFilter{}}
|
||||
}
|
||||
|
||||
// containsDir returns whether the bloom filter contains a directory.
|
||||
// Note that objects in XL mode are also considered directories.
|
||||
func (b bloomFilter) containsDir(in string) bool {
|
||||
split := splitPathDeterministic(path.Clean(in))
|
||||
|
||||
if len(split) == 0 {
|
||||
return false
|
||||
}
|
||||
return b.TestString(hashPath(path.Join(split...)).String())
|
||||
}
|
||||
|
||||
// bytes returns the bloom filter serialized as a byte slice.
|
||||
func (b *bloomFilter) bytes() []byte {
|
||||
if b == nil || b.BloomFilter == nil {
|
||||
return nil
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
_, err := b.WriteTo(&buf)
|
||||
if err != nil {
|
||||
logger.LogIf(GlobalContext, err)
|
||||
return nil
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// sort the dataUpdateTrackerHistory, newest first.
|
||||
// Returns whether the history is complete.
|
||||
func (d dataUpdateTrackerHistory) sort() bool {
|
||||
if len(d) == 0 {
|
||||
return true
|
||||
}
|
||||
sort.Slice(d, func(i, j int) bool {
|
||||
return d[i].idx > d[j].idx
|
||||
})
|
||||
return d[0].idx-d[len(d)-1].idx == uint64(len(d))
|
||||
}
|
||||
|
||||
// removeOlderThan will remove entries older than index 'n'.
|
||||
func (d *dataUpdateTrackerHistory) removeOlderThan(n uint64) {
|
||||
d.sort()
|
||||
dd := *d
|
||||
end := len(dd)
|
||||
for i := end - 1; i >= 0; i-- {
|
||||
if dd[i].idx < n {
|
||||
end = i
|
||||
}
|
||||
}
|
||||
dd = dd[:end]
|
||||
*d = dd
|
||||
}
|
||||
|
||||
// newBloomFilter returns a new bloom filter with default settings.
|
||||
func (d *dataUpdateTracker) newBloomFilter() bloomFilter {
|
||||
return bloomFilter{bloom.NewWithEstimates(dataUpdateTrackerEstItems, dataUpdateTrackerFP)}
|
||||
}
|
||||
|
||||
// current returns the current index.
|
||||
func (d *dataUpdateTracker) current() uint64 {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return d.Current.idx
|
||||
}
|
||||
|
||||
// latestWithDir returns the highest index that contains the directory.
|
||||
// This means that any cycle higher than this does NOT contain the entry.
|
||||
func (d *dataUpdateTracker) latestWithDir(dir string) uint64 {
|
||||
dateUpdateTrackerLogPrefix := color.Green("dataUpdateTracker:")
|
||||
bucket, _ := path2BucketObjectWithBasePath("", dir)
|
||||
if bucket == "" {
|
||||
if d.debug && len(dir) > 0 {
|
||||
console.Debugf(dateUpdateTrackerLogPrefix+" no bucket (%s)\n", dir)
|
||||
}
|
||||
return d.current()
|
||||
}
|
||||
if isReservedOrInvalidBucket(bucket, false) {
|
||||
return d.current()
|
||||
}
|
||||
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if d.Current.bf.containsDir(dir) || d.Current.idx == 0 {
|
||||
return d.Current.idx
|
||||
}
|
||||
if d.debug {
|
||||
console.Debugf(dateUpdateTrackerLogPrefix+" current bloom does NOT contains dir %s\n", dir)
|
||||
}
|
||||
|
||||
idx := d.Current.idx - 1
|
||||
for {
|
||||
f := d.History.find(idx)
|
||||
if f == nil || f.bf.containsDir(dir) || idx == 0 {
|
||||
break
|
||||
}
|
||||
idx--
|
||||
}
|
||||
return idx
|
||||
}
|
||||
|
||||
// start will load the current data from the drives start collecting information and
|
||||
// start a saver goroutine.
|
||||
// All of these will exit when the context is canceled.
|
||||
func (d *dataUpdateTracker) start(ctx context.Context, drives ...string) {
|
||||
if len(drives) == 0 {
|
||||
logger.LogIf(ctx, errors.New("dataUpdateTracker.start: No local drives specified"))
|
||||
return
|
||||
}
|
||||
d.load(ctx, drives...)
|
||||
go d.startCollector(ctx)
|
||||
// startSaver will unlock.
|
||||
d.mu.Lock()
|
||||
go d.startSaver(ctx, dataUpdateTrackerSaveInterval, drives...)
|
||||
}
|
||||
|
||||
// load will attempt to load data tracking information from the supplied drives.
|
||||
// The data will only be loaded if d.Saved is older than the one found on disk.
|
||||
// The newest working cache will be kept in d.
|
||||
// If no valid data usage tracker can be found d will remain unchanged.
|
||||
// If object is shared the caller should lock it.
|
||||
func (d *dataUpdateTracker) load(ctx context.Context, drives ...string) {
|
||||
if len(drives) == 0 {
|
||||
logger.LogIf(ctx, errors.New("dataUpdateTracker.load: No local drives specified"))
|
||||
return
|
||||
}
|
||||
for _, drive := range drives {
|
||||
|
||||
cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
|
||||
f, err := OpenFile(cacheFormatPath, readMode, 0o666)
|
||||
if err != nil {
|
||||
if osIsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
logger.LogIf(ctx, err)
|
||||
continue
|
||||
}
|
||||
err = d.deserialize(f, d.Saved)
|
||||
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
f.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// startSaver will start a saver that will write d to all supplied drives at specific intervals.
|
||||
// 'd' must be write locked when started and will be unlocked.
|
||||
// The saver will save and exit when supplied context is closed.
|
||||
func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Duration, drives ...string) {
|
||||
if len(drives) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
saveNow := d.save
|
||||
exited := make(chan struct{})
|
||||
d.saveExited = exited
|
||||
d.mu.Unlock()
|
||||
t := time.NewTicker(interval)
|
||||
defer t.Stop()
|
||||
defer close(exited)
|
||||
var buf bytes.Buffer
|
||||
for {
|
||||
var exit bool
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
exit = true
|
||||
case <-t.C:
|
||||
case <-saveNow:
|
||||
}
|
||||
buf.Reset()
|
||||
d.mu.Lock()
|
||||
if !d.dirty {
|
||||
d.mu.Unlock()
|
||||
if exit {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
d.Saved = UTCNow()
|
||||
err := d.serialize(&buf)
|
||||
if d.debug {
|
||||
console.Debugf(color.Green("dataUpdateTracker:")+" Saving: %v bytes, Current idx: %v\n", buf.Len(), d.Current.idx)
|
||||
}
|
||||
d.dirty = false
|
||||
d.mu.Unlock()
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err, "Error serializing usage tracker data")
|
||||
if exit {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
if buf.Len() == 0 {
|
||||
logger.LogIf(ctx, errors.New("zero sized output, skipping save"))
|
||||
continue
|
||||
}
|
||||
for _, drive := range drives {
|
||||
cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
|
||||
err := os.WriteFile(cacheFormatPath, buf.Bytes(), os.ModePerm)
|
||||
if err != nil {
|
||||
if osIsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
logger.LogIf(ctx, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if exit {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// serialize all data in d to dst.
|
||||
// Caller should hold lock if d is expected to be shared.
|
||||
// If an error is returned, there will likely be partial data written to dst.
|
||||
func (d *dataUpdateTracker) serialize(dst io.Writer) (err error) {
|
||||
ctx := GlobalContext
|
||||
var tmp [8]byte
|
||||
o := bufio.NewWriter(dst)
|
||||
defer func() {
|
||||
if err == nil {
|
||||
err = o.Flush()
|
||||
}
|
||||
}()
|
||||
|
||||
// Version
|
||||
if err := o.WriteByte(dataUpdateTrackerVersion); err != nil {
|
||||
if d.debug {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
// Timestamp.
|
||||
binary.LittleEndian.PutUint64(tmp[:], uint64(d.Saved.Unix()))
|
||||
if _, err := o.Write(tmp[:]); err != nil {
|
||||
if d.debug {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Current
|
||||
binary.LittleEndian.PutUint64(tmp[:], d.Current.idx)
|
||||
if _, err := o.Write(tmp[:]); err != nil {
|
||||
if d.debug {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := d.Current.bf.WriteTo(o); err != nil {
|
||||
if d.debug {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// History
|
||||
binary.LittleEndian.PutUint64(tmp[:], uint64(len(d.History)))
|
||||
if _, err := o.Write(tmp[:]); err != nil {
|
||||
if d.debug {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
for _, bf := range d.History {
|
||||
// Current
|
||||
binary.LittleEndian.PutUint64(tmp[:], bf.idx)
|
||||
if _, err := o.Write(tmp[:]); err != nil {
|
||||
if d.debug {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := bf.bf.WriteTo(o); err != nil {
|
||||
if d.debug {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// deserialize will deserialize the supplied input if the input is newer than the supplied time.
|
||||
func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) error {
|
||||
ctx := GlobalContext
|
||||
var dst dataUpdateTracker
|
||||
var tmp [8]byte
|
||||
|
||||
// Version
|
||||
if _, err := io.ReadFull(src, tmp[:1]); err != nil {
|
||||
if d.debug {
|
||||
if err != io.EOF {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
switch tmp[0] {
|
||||
case 1, 2, 3, 4, 5, 6:
|
||||
if intDataUpdateTracker.debug {
|
||||
console.Debugln(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
|
||||
}
|
||||
return nil
|
||||
case dataUpdateTrackerVersion:
|
||||
default:
|
||||
return errors.New("dataUpdateTracker: Unknown data version")
|
||||
}
|
||||
// Timestamp.
|
||||
if _, err := io.ReadFull(src, tmp[:8]); err != nil {
|
||||
if d.debug {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
t := time.Unix(int64(binary.LittleEndian.Uint64(tmp[:])), 0)
|
||||
if !t.After(newerThan) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Current
|
||||
if _, err := io.ReadFull(src, tmp[:8]); err != nil {
|
||||
if d.debug {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
dst.Current.idx = binary.LittleEndian.Uint64(tmp[:])
|
||||
dst.Current.bf = emptyBloomFilter()
|
||||
if _, err := dst.Current.bf.ReadFrom(src); err != nil {
|
||||
if d.debug {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// History
|
||||
if _, err := io.ReadFull(src, tmp[:8]); err != nil {
|
||||
if d.debug {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
n := binary.LittleEndian.Uint64(tmp[:])
|
||||
dst.History = make(dataUpdateTrackerHistory, int(n))
|
||||
for i, e := range dst.History {
|
||||
if _, err := io.ReadFull(src, tmp[:8]); err != nil {
|
||||
if d.debug {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
e.idx = binary.LittleEndian.Uint64(tmp[:])
|
||||
e.bf = emptyBloomFilter()
|
||||
if _, err := e.bf.ReadFrom(src); err != nil {
|
||||
if d.debug {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
dst.History[i] = e
|
||||
}
|
||||
// Ignore what remains on the stream.
|
||||
// Update d:
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.Current = dst.Current
|
||||
d.History = dst.History
|
||||
d.Saved = dst.Saved
|
||||
return nil
|
||||
}
|
||||
|
||||
// start a collector that picks up entries from objectUpdatedCh
|
||||
// and adds them to the current bloom filter.
|
||||
func (d *dataUpdateTracker) startCollector(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case in := <-d.input:
|
||||
bucket, _ := path2BucketObjectWithBasePath("", in)
|
||||
if bucket == "" {
|
||||
if d.debug && len(in) > 0 {
|
||||
console.Debugf(color.Green("dataUpdateTracker:")+" no bucket (%s)\n", in)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if isReservedOrInvalidBucket(bucket, false) {
|
||||
continue
|
||||
}
|
||||
split := splitPathDeterministic(in)
|
||||
|
||||
// Add all paths until done.
|
||||
d.mu.Lock()
|
||||
for i := range split {
|
||||
d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
|
||||
}
|
||||
d.dirty = d.dirty || len(split) > 0
|
||||
d.mu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// markDirty adds the supplied path to the current bloom filter.
|
||||
func (d *dataUpdateTracker) markDirty(bucket, prefix string) {
|
||||
dateUpdateTrackerLogPrefix := color.Green("dataUpdateTracker:")
|
||||
if bucket == "" && d.debug {
|
||||
console.Debugf(dateUpdateTrackerLogPrefix + " no bucket specified\n")
|
||||
return
|
||||
}
|
||||
|
||||
if isReservedOrInvalidBucket(bucket, false) && d.debug {
|
||||
return
|
||||
}
|
||||
split := splitPathDeterministic(pathJoin(bucket, prefix))
|
||||
|
||||
// Add all paths until done.
|
||||
d.mu.Lock()
|
||||
for i := range split {
|
||||
d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
|
||||
}
|
||||
d.dirty = d.dirty || len(split) > 0
|
||||
d.mu.Unlock()
|
||||
}
|
||||
|
||||
// find entry with specified index.
|
||||
// Returns nil if not found.
|
||||
func (d dataUpdateTrackerHistory) find(idx uint64) *dataUpdateFilter {
|
||||
for _, f := range d {
|
||||
if f.idx == idx {
|
||||
return &f
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// filterFrom will return a combined bloom filter.
|
||||
func (d *dataUpdateTracker) filterFrom(ctx context.Context, oldest, newest uint64) *bloomFilterResponse {
|
||||
bf := d.newBloomFilter()
|
||||
bfr := bloomFilterResponse{
|
||||
OldestIdx: oldest,
|
||||
CurrentIdx: d.Current.idx,
|
||||
Complete: true,
|
||||
}
|
||||
// Loop through each index requested.
|
||||
for idx := oldest; idx <= newest; idx++ {
|
||||
v := d.History.find(idx)
|
||||
if v == nil {
|
||||
if d.Current.idx == idx {
|
||||
// Merge current.
|
||||
err := bf.Merge(d.Current.bf.BloomFilter)
|
||||
logger.LogIf(ctx, err)
|
||||
if err != nil {
|
||||
bfr.Complete = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
bfr.Complete = false
|
||||
bfr.OldestIdx = idx + 1
|
||||
continue
|
||||
}
|
||||
|
||||
err := bf.Merge(v.bf.BloomFilter)
|
||||
if err != nil {
|
||||
bfr.Complete = false
|
||||
logger.LogIf(ctx, err)
|
||||
continue
|
||||
}
|
||||
bfr.NewestIdx = idx
|
||||
}
|
||||
var dst bytes.Buffer
|
||||
_, err := bf.WriteTo(&dst)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return nil
|
||||
}
|
||||
bfr.Filter = dst.Bytes()
|
||||
|
||||
return &bfr
|
||||
}
|
||||
|
||||
// cycleFilter will cycle the bloom filter to start recording to index y if not already.
|
||||
// The response will contain a bloom filter starting at index x up to, but not including index y.
|
||||
// If y is 0, the response will not update y, but return the currently recorded information
|
||||
// from the oldest (unless 0, then it will be all) until and including current y.
|
||||
func (d *dataUpdateTracker) cycleFilter(ctx context.Context, req bloomFilterRequest) (*bloomFilterResponse, error) {
|
||||
if req.OldestClean != "" {
|
||||
return &bloomFilterResponse{OldestIdx: d.latestWithDir(req.OldestClean)}, nil
|
||||
}
|
||||
current := req.Current
|
||||
oldest := req.Oldest
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if current == 0 {
|
||||
if len(d.History) == 0 {
|
||||
return d.filterFrom(ctx, d.Current.idx, d.Current.idx), nil
|
||||
}
|
||||
d.History.sort()
|
||||
if oldest == 0 {
|
||||
oldest = d.History[len(d.History)-1].idx
|
||||
}
|
||||
return d.filterFrom(ctx, oldest, d.Current.idx), nil
|
||||
}
|
||||
|
||||
// Move current to history if new one requested
|
||||
if d.Current.idx != current {
|
||||
d.dirty = true
|
||||
if d.debug {
|
||||
console.Debugf(color.Green("dataUpdateTracker:")+" cycle bloom filter: %v -> %v\n", d.Current.idx, current)
|
||||
}
|
||||
|
||||
d.History = append(d.History, d.Current)
|
||||
d.Current.idx = current
|
||||
d.Current.bf = d.newBloomFilter()
|
||||
select {
|
||||
case d.save <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
d.History.removeOlderThan(oldest)
|
||||
return d.filterFrom(ctx, oldest, current), nil
|
||||
}
|
||||
|
||||
// splitPathDeterministic will split the provided relative path
|
||||
// deterministically and return up to the first 3 elements of the path.
|
||||
// slash and dot prefixes are removed.
|
||||
// Trailing slashes are removed.
|
||||
// Returns 0 length if no parts are found after trimming.
|
||||
func splitPathDeterministic(in string) []string {
|
||||
split := strings.Split(decodeDirObject(in), SlashSeparator)
|
||||
|
||||
// Trim empty start/end
|
||||
for len(split) > 0 {
|
||||
if len(split[0]) > 0 && split[0] != "." {
|
||||
break
|
||||
}
|
||||
split = split[1:]
|
||||
}
|
||||
for len(split) > 0 {
|
||||
if len(split[len(split)-1]) > 0 {
|
||||
break
|
||||
}
|
||||
split = split[:len(split)-1]
|
||||
}
|
||||
|
||||
return split
|
||||
}
|
||||
|
||||
// bloomFilterRequest request bloom filters.
|
||||
// Current index will be updated to current and entries back to Oldest is returned.
|
||||
type bloomFilterRequest struct {
|
||||
Oldest uint64
|
||||
Current uint64
|
||||
// If set the oldest clean version will be returned in OldestIdx
|
||||
// and the rest of the request will be ignored.
|
||||
OldestClean string
|
||||
}
|
||||
|
||||
type bloomFilterResponse struct {
|
||||
// Current index being written to.
|
||||
CurrentIdx uint64
|
||||
// Oldest index in the returned bloom filter.
|
||||
OldestIdx uint64
|
||||
// Newest Index in the returned bloom filter.
|
||||
NewestIdx uint64
|
||||
// Are all indexes between oldest and newest filled?
|
||||
Complete bool
|
||||
// Binary data of the bloom filter.
|
||||
Filter []byte
|
||||
}
|
||||
|
||||
// NSUpdated indicates namespace has been updated.
|
||||
// The function will block until the entry has been picked up.
|
||||
func NSUpdated(bucket, prefix string) {
|
||||
if intDataUpdateTracker != nil {
|
||||
intDataUpdateTracker.markDirty(bucket, prefix)
|
||||
}
|
||||
}
|
||||
@@ -1,299 +0,0 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/logger/target/types"
|
||||
"github.com/minio/pkg/logger/message/log"
|
||||
)
|
||||
|
||||
type testLoggerI interface {
|
||||
Helper()
|
||||
Log(args ...interface{})
|
||||
}
|
||||
|
||||
type testingLogger struct {
|
||||
mu sync.Mutex
|
||||
t testLoggerI
|
||||
}
|
||||
|
||||
func (t *testingLogger) Endpoint() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (t *testingLogger) String() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (t *testingLogger) Init() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *testingLogger) Cancel() {
|
||||
}
|
||||
|
||||
func (t *testingLogger) Type() types.TargetType {
|
||||
return types.TargetHTTP
|
||||
}
|
||||
|
||||
func (t *testingLogger) IsOnline() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Stats returns the target statistics.
|
||||
func (t *testingLogger) Stats() types.TargetStats {
|
||||
return types.TargetStats{}
|
||||
}
|
||||
|
||||
func (t *testingLogger) Send(entry interface{}) error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.t == nil {
|
||||
return nil
|
||||
}
|
||||
e, ok := entry.(log.Entry)
|
||||
if !ok {
|
||||
return fmt.Errorf("unexpected log entry structure %#v", entry)
|
||||
}
|
||||
|
||||
t.t.Helper()
|
||||
t.t.Log(e.Level, ":", e.Message)
|
||||
return nil
|
||||
}
|
||||
|
||||
func addTestingLogging(t testLoggerI) func() {
|
||||
tl := &testingLogger{t: t}
|
||||
logger.AddSystemTarget(tl)
|
||||
return func() {
|
||||
tl.mu.Lock()
|
||||
defer tl.mu.Unlock()
|
||||
tl.t = nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestDataUpdateTracker(t *testing.T) {
|
||||
dut := newDataUpdateTracker()
|
||||
// Change some defaults.
|
||||
dut.debug = testing.Verbose()
|
||||
dut.input = make(chan string)
|
||||
dut.save = make(chan struct{})
|
||||
|
||||
defer addTestingLogging(t)()
|
||||
|
||||
dut.Current.bf = dut.newBloomFilter()
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
err := os.MkdirAll(filepath.Dir(filepath.Join(tmpDir, dataUpdateTrackerFilename)), os.ModePerm)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
dut.start(ctx, tmpDir)
|
||||
|
||||
tests := []struct {
|
||||
in string
|
||||
check []string // if not empty, check against these instead.
|
||||
exist bool
|
||||
}{
|
||||
{
|
||||
in: "bucket/directory/file.txt",
|
||||
check: []string{"bucket", "bucket/", "/bucket", "bucket/directory", "bucket/directory/", "bucket/directory/file.txt", "/bucket/directory/file.txt"},
|
||||
exist: true,
|
||||
},
|
||||
{
|
||||
// System bucket
|
||||
in: ".minio.sys/ignoreme/pls",
|
||||
exist: false,
|
||||
},
|
||||
{
|
||||
// Not a valid bucket
|
||||
in: "./bucket/okfile.txt",
|
||||
check: []string{"./bucket/okfile.txt", "/bucket/okfile.txt", "bucket/okfile.txt"},
|
||||
exist: false,
|
||||
},
|
||||
{
|
||||
// Not a valid bucket
|
||||
in: "æ/okfile.txt",
|
||||
check: []string{"æ/okfile.txt", "æ/okfile.txt", "æ"},
|
||||
exist: false,
|
||||
},
|
||||
{
|
||||
in: "/bucket2/okfile2.txt",
|
||||
check: []string{"./bucket2/okfile2.txt", "/bucket2/okfile2.txt", "bucket2/okfile2.txt", "bucket2"},
|
||||
exist: true,
|
||||
},
|
||||
{
|
||||
in: "/bucket3/prefix/okfile2.txt",
|
||||
check: []string{"./bucket3/prefix/okfile2.txt", "/bucket3/prefix/okfile2.txt", "bucket3/prefix/okfile2.txt", "bucket3/prefix", "bucket3"},
|
||||
exist: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.in, func(t *testing.T) {
|
||||
dut.input <- tt.in
|
||||
dut.input <- "" // Sending empty string ensures the previous is added to filter.
|
||||
dut.mu.Lock()
|
||||
defer dut.mu.Unlock()
|
||||
if len(tt.check) == 0 {
|
||||
got := dut.Current.bf.containsDir(tt.in)
|
||||
if got != tt.exist {
|
||||
// For unlimited tests this could lead to false positives,
|
||||
// but it should be deterministic.
|
||||
t.Errorf("entry %q, got: %v, want %v", tt.in, got, tt.exist)
|
||||
}
|
||||
return
|
||||
}
|
||||
for _, check := range tt.check {
|
||||
got := dut.Current.bf.containsDir(check)
|
||||
if got != tt.exist {
|
||||
// For unlimited tests this could lead to false positives,
|
||||
// but it should be deterministic.
|
||||
t.Errorf("entry %q, check: %q, got: %v, want %v", tt.in, check, got, tt.exist)
|
||||
}
|
||||
continue
|
||||
}
|
||||
})
|
||||
}
|
||||
// Cycle to history
|
||||
req := bloomFilterRequest{
|
||||
Oldest: 1,
|
||||
Current: 2,
|
||||
}
|
||||
|
||||
_, err = dut.cycleFilter(ctx, req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dut.input <- "cycle2/file.txt"
|
||||
dut.input <- "" // Sending empty string ensures the previous is added to filter.
|
||||
|
||||
tests = append(tests, struct {
|
||||
in string
|
||||
check []string
|
||||
exist bool
|
||||
}{in: "cycle2/file.txt", exist: true})
|
||||
|
||||
// Shut down
|
||||
cancel()
|
||||
<-dut.saveExited
|
||||
|
||||
if dut.current() != 2 {
|
||||
t.Fatal("wrong current idx after save. want 2, got:", dut.current())
|
||||
}
|
||||
|
||||
ctx, cancel = context.WithCancel(context.Background())
|
||||
// Reload...
|
||||
dut = newDataUpdateTracker()
|
||||
dut.start(ctx, tmpDir)
|
||||
defer func() {
|
||||
cancel()
|
||||
<-dut.saveExited
|
||||
}()
|
||||
|
||||
if dut.current() != 2 {
|
||||
t.Fatal("current idx after load not preserved. want 2, got:", dut.current())
|
||||
}
|
||||
req = bloomFilterRequest{
|
||||
Oldest: 1,
|
||||
Current: 3,
|
||||
}
|
||||
bfr2, err := dut.cycleFilter(ctx, req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bfr2.Complete {
|
||||
t.Fatal("Wanted complete, didn't get it")
|
||||
}
|
||||
if bfr2.CurrentIdx != 3 {
|
||||
t.Fatal("wanted index 3, got", bfr2.CurrentIdx)
|
||||
}
|
||||
if bfr2.OldestIdx != 1 {
|
||||
t.Fatal("wanted oldest index 3, got", bfr2.OldestIdx)
|
||||
}
|
||||
|
||||
t.Logf("Size of filter %d bytes, M: %d, K:%d", len(bfr2.Filter), dut.Current.bf.Cap(), dut.Current.bf.K())
|
||||
// Rerun test with returned bfr2
|
||||
bf := dut.newBloomFilter()
|
||||
_, err = bf.ReadFrom(bytes.NewReader(bfr2.Filter))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.in+"-reloaded", func(t *testing.T) {
|
||||
if len(tt.check) == 0 {
|
||||
got := bf.containsDir(tt.in)
|
||||
if got != tt.exist {
|
||||
// For unlimited tests this could lead to false positives,
|
||||
// but it should be deterministic.
|
||||
t.Errorf("entry %q, got: %v, want %v", tt.in, got, tt.exist)
|
||||
}
|
||||
return
|
||||
}
|
||||
for _, check := range tt.check {
|
||||
got := bf.containsDir(check)
|
||||
if got != tt.exist {
|
||||
// For unlimited tests this could lead to false positives,
|
||||
// but it should be deterministic.
|
||||
t.Errorf("entry %q, check: %q, got: %v, want %v", tt.in, check, got, tt.exist)
|
||||
}
|
||||
continue
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDataUpdateTracker(b *testing.B) {
|
||||
dut := newDataUpdateTracker()
|
||||
// Change some defaults.
|
||||
dut.debug = false
|
||||
dut.input = make(chan string)
|
||||
dut.save = make(chan struct{})
|
||||
|
||||
defer addTestingLogging(b)()
|
||||
|
||||
dut.Current.bf = dut.newBloomFilter()
|
||||
// We do this unbuffered. This will very significantly reduce throughput, so this is a worst case.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go dut.startCollector(ctx)
|
||||
input := make([]string, 1000)
|
||||
rng := rand.New(rand.NewSource(0xabad1dea))
|
||||
tmp := []string{"bucket", "aprefix", "nextprefixlevel", "maybeobjname", "evendeeper", "ok-one-morelevel", "final.object"}
|
||||
for i := range input {
|
||||
tmp := tmp[:1+rng.Intn(cap(tmp)-1)]
|
||||
input[i] = path.Join(tmp...)
|
||||
}
|
||||
b.SetBytes(1)
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
dut.input <- input[rng.Intn(len(input))]
|
||||
}
|
||||
}
|
||||
+71
-26
@@ -23,6 +23,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"path"
|
||||
"path/filepath"
|
||||
@@ -47,6 +48,9 @@ type dataUsageHash string
|
||||
// sizeHistogram is a size histogram.
|
||||
type sizeHistogram [dataUsageBucketLen]uint64
|
||||
|
||||
// versionsHistogram is a histogram of number of versions in an object.
|
||||
type versionsHistogram [dataUsageVersionLen]uint64
|
||||
|
||||
type dataUsageEntry struct {
|
||||
Children dataUsageHashMap `msg:"ch"`
|
||||
// These fields do no include any children.
|
||||
@@ -54,6 +58,7 @@ type dataUsageEntry struct {
|
||||
Objects uint64 `msg:"os"`
|
||||
Versions uint64 `msg:"vs"` // Versions that are not delete markers.
|
||||
ObjSizes sizeHistogram `msg:"szs"`
|
||||
ObjVersions versionsHistogram `msg:"vh"`
|
||||
ReplicationStats *replicationAllStats `msg:"rs,omitempty"`
|
||||
AllTierStats *allTierStats `msg:"ats,omitempty"`
|
||||
Compacted bool `msg:"c"`
|
||||
@@ -276,7 +281,6 @@ type dataUsageCacheInfo struct {
|
||||
// indicates if the disk is being healed and scanner
|
||||
// should skip healing the disk
|
||||
SkipHealing bool
|
||||
BloomFilter []byte `msg:"BloomFilter,omitempty"`
|
||||
|
||||
// Active lifecycle, if any on the bucket
|
||||
lifeCycle *lifecycle.Lifecycle `msg:"-"`
|
||||
@@ -292,6 +296,7 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) {
|
||||
e.Size += summary.totalSize
|
||||
e.Versions += summary.versions
|
||||
e.ObjSizes.add(summary.totalSize)
|
||||
e.ObjVersions.add(summary.versions)
|
||||
|
||||
if e.ReplicationStats == nil {
|
||||
e.ReplicationStats = &replicationAllStats{
|
||||
@@ -352,6 +357,10 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
|
||||
e.ObjSizes[i] += v
|
||||
}
|
||||
|
||||
for i, v := range other.ObjVersions[:] {
|
||||
e.ObjVersions[i] += v
|
||||
}
|
||||
|
||||
if other.AllTierStats != nil {
|
||||
if e.AllTierStats == nil {
|
||||
e.AllTierStats = newAllTierStats()
|
||||
@@ -649,10 +658,7 @@ func (d *dataUsageCache) reduceChildrenOf(path dataUsageHash, limit int, compact
|
||||
// StringAll returns a detailed string representation of all entries in the cache.
|
||||
func (d *dataUsageCache) StringAll() string {
|
||||
// Remove bloom filter from print.
|
||||
bf := d.Info.BloomFilter
|
||||
d.Info.BloomFilter = nil
|
||||
s := fmt.Sprintf("info:%+v\n", d.Info)
|
||||
d.Info.BloomFilter = bf
|
||||
for k, v := range d.Cache {
|
||||
s += fmt.Sprintf("\t%v: %+v\n", k, v)
|
||||
}
|
||||
@@ -698,7 +704,7 @@ func (d *dataUsageCache) flatten(root dataUsageEntry) dataUsageEntry {
|
||||
func (h *sizeHistogram) add(size int64) {
|
||||
// Fetch the histogram interval corresponding
|
||||
// to the passed object size.
|
||||
for i, interval := range ObjectsHistogramIntervals {
|
||||
for i, interval := range ObjectsHistogramIntervals[:] {
|
||||
if size >= interval.start && size <= interval.end {
|
||||
h[i]++
|
||||
break
|
||||
@@ -715,6 +721,27 @@ func (h *sizeHistogram) toMap() map[string]uint64 {
|
||||
return res
|
||||
}
|
||||
|
||||
// add a version count to the histogram.
|
||||
func (h *versionsHistogram) add(versions uint64) {
|
||||
// Fetch the histogram interval corresponding
|
||||
// to the passed object size.
|
||||
for i, interval := range ObjectsVersionCountIntervals[:] {
|
||||
if versions >= uint64(interval.start) && versions <= uint64(interval.end) {
|
||||
h[i]++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// toMap returns the map to a map[string]uint64.
|
||||
func (h *versionsHistogram) toMap() map[string]uint64 {
|
||||
res := make(map[string]uint64, dataUsageVersionLen)
|
||||
for i, count := range h {
|
||||
res[ObjectsVersionCountIntervals[i].name] = count
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (d *dataUsageCache) tiersUsageInfo(buckets []BucketInfo) *allTierStats {
|
||||
dst := newAllTierStats()
|
||||
for _, bucket := range buckets {
|
||||
@@ -745,10 +772,11 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
|
||||
}
|
||||
flat := d.flatten(*e)
|
||||
bui := BucketUsageInfo{
|
||||
Size: uint64(flat.Size),
|
||||
VersionsCount: flat.Versions,
|
||||
ObjectsCount: flat.Objects,
|
||||
ObjectSizesHistogram: flat.ObjSizes.toMap(),
|
||||
Size: uint64(flat.Size),
|
||||
VersionsCount: flat.Versions,
|
||||
ObjectsCount: flat.Objects,
|
||||
ObjectSizesHistogram: flat.ObjSizes.toMap(),
|
||||
ObjectVersionsHistogram: flat.ObjVersions.toMap(),
|
||||
}
|
||||
if flat.ReplicationStats != nil {
|
||||
bui.ReplicaSize = flat.ReplicationStats.ReplicaSize
|
||||
@@ -851,37 +879,54 @@ type objectIO interface {
|
||||
|
||||
// load the cache content with name from minioMetaBackgroundOpsBucket.
|
||||
// Only backend errors are returned as errors.
|
||||
// The loader is optimistic and has no locking, but tries 5 times before giving up.
|
||||
// If the object is not found or unable to deserialize d is cleared and nil error is returned.
|
||||
func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string) error {
|
||||
// Abandon if more than 5 minutes, so we don't hold up scanner.
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
r, err := store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, readLock, ObjectOptions{})
|
||||
if err != nil {
|
||||
switch err.(type) {
|
||||
case ObjectNotFound:
|
||||
case BucketNotFound:
|
||||
case InsufficientReadQuorum:
|
||||
case StorageErr:
|
||||
default:
|
||||
return toObjectErr(err, dataUsageBucket, name)
|
||||
// Caches are read+written without locks,
|
||||
retries := 0
|
||||
for retries < 5 {
|
||||
r, err := store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, noLock, ObjectOptions{NoLock: true})
|
||||
if err != nil {
|
||||
switch err.(type) {
|
||||
case ObjectNotFound, BucketNotFound:
|
||||
case InsufficientReadQuorum, StorageErr:
|
||||
retries++
|
||||
time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
|
||||
continue
|
||||
default:
|
||||
return toObjectErr(err, dataUsageBucket, name)
|
||||
}
|
||||
*d = dataUsageCache{}
|
||||
return nil
|
||||
}
|
||||
*d = dataUsageCache{}
|
||||
if err := d.deserialize(r); err != nil {
|
||||
r.Close()
|
||||
retries++
|
||||
time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
|
||||
continue
|
||||
}
|
||||
r.Close()
|
||||
return nil
|
||||
}
|
||||
defer r.Close()
|
||||
if err := d.deserialize(r); err != nil {
|
||||
*d = dataUsageCache{}
|
||||
logger.LogOnceIf(ctx, err, err.Error())
|
||||
}
|
||||
*d = dataUsageCache{}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Maximum running concurrent saves on server.
|
||||
var maxConcurrentScannerSaves = make(chan struct{}, 4)
|
||||
|
||||
// save the content of the cache to minioMetaBackgroundOpsBucket with the provided name.
|
||||
// Note that no locking is done when saving.
|
||||
func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string) error {
|
||||
var r io.Reader
|
||||
|
||||
maxConcurrentScannerSaves <- struct{}{}
|
||||
defer func() {
|
||||
<-maxConcurrentScannerSaves
|
||||
}()
|
||||
// If big, do streaming...
|
||||
size := int64(-1)
|
||||
if len(d.Cache) > 10000 {
|
||||
@@ -913,7 +958,7 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
|
||||
dataUsageBucket,
|
||||
name,
|
||||
NewPutObjReader(hr),
|
||||
ObjectOptions{})
|
||||
ObjectOptions{NoLock: true})
|
||||
if isErrBucketNotFound(err) {
|
||||
return nil
|
||||
}
|
||||
|
||||
+155
-74
@@ -629,12 +629,6 @@ func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
err = msgp.WrapError(err, "SkipHealing")
|
||||
return
|
||||
}
|
||||
case "BloomFilter":
|
||||
z.BloomFilter, err = dc.ReadBytes(z.BloomFilter)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "BloomFilter")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
@@ -648,24 +642,9 @@ func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// omitempty: check for empty values
|
||||
zb0001Len := uint32(5)
|
||||
var zb0001Mask uint8 /* 5 bits */
|
||||
_ = zb0001Mask
|
||||
if z.BloomFilter == nil {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x10
|
||||
}
|
||||
// variable map header, size zb0001Len
|
||||
err = en.Append(0x80 | uint8(zb0001Len))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if zb0001Len == 0 {
|
||||
return
|
||||
}
|
||||
// map header, size 4
|
||||
// write "Name"
|
||||
err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
err = en.Append(0x84, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -704,39 +683,15 @@ func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
err = msgp.WrapError(err, "SkipHealing")
|
||||
return
|
||||
}
|
||||
if (zb0001Mask & 0x10) == 0 { // if not empty
|
||||
// write "BloomFilter"
|
||||
err = en.Append(0xab, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteBytes(z.BloomFilter)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "BloomFilter")
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// omitempty: check for empty values
|
||||
zb0001Len := uint32(5)
|
||||
var zb0001Mask uint8 /* 5 bits */
|
||||
_ = zb0001Mask
|
||||
if z.BloomFilter == nil {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x10
|
||||
}
|
||||
// variable map header, size zb0001Len
|
||||
o = append(o, 0x80|uint8(zb0001Len))
|
||||
if zb0001Len == 0 {
|
||||
return
|
||||
}
|
||||
// map header, size 4
|
||||
// string "Name"
|
||||
o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
o = append(o, 0x84, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
o = msgp.AppendString(o, z.Name)
|
||||
// string "NextCycle"
|
||||
o = append(o, 0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65)
|
||||
@@ -747,11 +702,6 @@ func (z *dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
// string "SkipHealing"
|
||||
o = append(o, 0xab, 0x53, 0x6b, 0x69, 0x70, 0x48, 0x65, 0x61, 0x6c, 0x69, 0x6e, 0x67)
|
||||
o = msgp.AppendBool(o, z.SkipHealing)
|
||||
if (zb0001Mask & 0x10) == 0 { // if not empty
|
||||
// string "BloomFilter"
|
||||
o = append(o, 0xab, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
|
||||
o = msgp.AppendBytes(o, z.BloomFilter)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -797,12 +747,6 @@ func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
err = msgp.WrapError(err, "SkipHealing")
|
||||
return
|
||||
}
|
||||
case "BloomFilter":
|
||||
z.BloomFilter, bts, err = msgp.ReadBytesBytes(bts, z.BloomFilter)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "BloomFilter")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
@@ -817,7 +761,7 @@ func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *dataUsageCacheInfo) Msgsize() (s int) {
|
||||
s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 10 + msgp.Uint32Size + 11 + msgp.TimeSize + 12 + msgp.BoolSize + 12 + msgp.BytesPrefixSize + len(z.BloomFilter)
|
||||
s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 10 + msgp.Uint32Size + 11 + msgp.TimeSize + 12 + msgp.BoolSize
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1596,6 +1540,24 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
return
|
||||
}
|
||||
}
|
||||
case "vh":
|
||||
var zb0003 uint32
|
||||
zb0003, err = dc.ReadArrayHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjVersions")
|
||||
return
|
||||
}
|
||||
if zb0003 != uint32(dataUsageVersionLen) {
|
||||
err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
|
||||
return
|
||||
}
|
||||
for za0002 := range z.ObjVersions {
|
||||
z.ObjVersions[za0002], err = dc.ReadUint64()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjVersions", za0002)
|
||||
return
|
||||
}
|
||||
}
|
||||
case "rs":
|
||||
if dc.IsNil() {
|
||||
err = dc.ReadNil()
|
||||
@@ -1652,16 +1614,16 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// omitempty: check for empty values
|
||||
zb0001Len := uint32(8)
|
||||
var zb0001Mask uint8 /* 8 bits */
|
||||
zb0001Len := uint32(9)
|
||||
var zb0001Mask uint16 /* 9 bits */
|
||||
_ = zb0001Mask
|
||||
if z.ReplicationStats == nil {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x20
|
||||
zb0001Mask |= 0x40
|
||||
}
|
||||
if z.AllTierStats == nil {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x40
|
||||
zb0001Mask |= 0x80
|
||||
}
|
||||
// variable map header, size zb0001Len
|
||||
err = en.Append(0x80 | uint8(zb0001Len))
|
||||
@@ -1728,7 +1690,24 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if (zb0001Mask & 0x20) == 0 { // if not empty
|
||||
// write "vh"
|
||||
err = en.Append(0xa2, 0x76, 0x68)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteArrayHeader(uint32(dataUsageVersionLen))
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjVersions")
|
||||
return
|
||||
}
|
||||
for za0002 := range z.ObjVersions {
|
||||
err = en.WriteUint64(z.ObjVersions[za0002])
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjVersions", za0002)
|
||||
return
|
||||
}
|
||||
}
|
||||
if (zb0001Mask & 0x40) == 0 { // if not empty
|
||||
// write "rs"
|
||||
err = en.Append(0xa2, 0x72, 0x73)
|
||||
if err != nil {
|
||||
@@ -1747,7 +1726,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if (zb0001Mask & 0x40) == 0 { // if not empty
|
||||
if (zb0001Mask & 0x80) == 0 { // if not empty
|
||||
// write "ats"
|
||||
err = en.Append(0xa3, 0x61, 0x74, 0x73)
|
||||
if err != nil {
|
||||
@@ -1783,16 +1762,16 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// omitempty: check for empty values
|
||||
zb0001Len := uint32(8)
|
||||
var zb0001Mask uint8 /* 8 bits */
|
||||
zb0001Len := uint32(9)
|
||||
var zb0001Mask uint16 /* 9 bits */
|
||||
_ = zb0001Mask
|
||||
if z.ReplicationStats == nil {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x20
|
||||
zb0001Mask |= 0x40
|
||||
}
|
||||
if z.AllTierStats == nil {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x40
|
||||
zb0001Mask |= 0x80
|
||||
}
|
||||
// variable map header, size zb0001Len
|
||||
o = append(o, 0x80|uint8(zb0001Len))
|
||||
@@ -1821,7 +1800,13 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
for za0001 := range z.ObjSizes {
|
||||
o = msgp.AppendUint64(o, z.ObjSizes[za0001])
|
||||
}
|
||||
if (zb0001Mask & 0x20) == 0 { // if not empty
|
||||
// string "vh"
|
||||
o = append(o, 0xa2, 0x76, 0x68)
|
||||
o = msgp.AppendArrayHeader(o, uint32(dataUsageVersionLen))
|
||||
for za0002 := range z.ObjVersions {
|
||||
o = msgp.AppendUint64(o, z.ObjVersions[za0002])
|
||||
}
|
||||
if (zb0001Mask & 0x40) == 0 { // if not empty
|
||||
// string "rs"
|
||||
o = append(o, 0xa2, 0x72, 0x73)
|
||||
if z.ReplicationStats == nil {
|
||||
@@ -1834,7 +1819,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if (zb0001Mask & 0x40) == 0 { // if not empty
|
||||
if (zb0001Mask & 0x80) == 0 { // if not empty
|
||||
// string "ats"
|
||||
o = append(o, 0xa3, 0x61, 0x74, 0x73)
|
||||
if z.AllTierStats == nil {
|
||||
@@ -1913,6 +1898,24 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
return
|
||||
}
|
||||
}
|
||||
case "vh":
|
||||
var zb0003 uint32
|
||||
zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjVersions")
|
||||
return
|
||||
}
|
||||
if zb0003 != uint32(dataUsageVersionLen) {
|
||||
err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
|
||||
return
|
||||
}
|
||||
for za0002 := range z.ObjVersions {
|
||||
z.ObjVersions[za0002], bts, err = msgp.ReadUint64Bytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjVersions", za0002)
|
||||
return
|
||||
}
|
||||
}
|
||||
case "rs":
|
||||
if msgp.IsNil(bts) {
|
||||
bts, err = msgp.ReadNilBytes(bts)
|
||||
@@ -1967,7 +1970,7 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *dataUsageEntry) Msgsize() (s int) {
|
||||
s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3
|
||||
s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 3
|
||||
if z.ReplicationStats == nil {
|
||||
s += msgp.NilSize
|
||||
} else {
|
||||
@@ -3760,3 +3763,81 @@ func (z tierStats) Msgsize() (s int) {
|
||||
s = 1 + 3 + msgp.Uint64Size + 3 + msgp.IntSize + 3 + msgp.IntSize
|
||||
return
|
||||
}
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *versionsHistogram) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
var zb0001 uint32
|
||||
zb0001, err = dc.ReadArrayHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
if zb0001 != uint32(dataUsageVersionLen) {
|
||||
err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0001}
|
||||
return
|
||||
}
|
||||
for za0001 := range z {
|
||||
z[za0001], err = dc.ReadUint64()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *versionsHistogram) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
err = en.WriteArrayHeader(uint32(dataUsageVersionLen))
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
for za0001 := range z {
|
||||
err = en.WriteUint64(z[za0001])
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *versionsHistogram) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
o = msgp.AppendArrayHeader(o, uint32(dataUsageVersionLen))
|
||||
for za0001 := range z {
|
||||
o = msgp.AppendUint64(o, z[za0001])
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (z *versionsHistogram) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
var zb0001 uint32
|
||||
zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
if zb0001 != uint32(dataUsageVersionLen) {
|
||||
err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0001}
|
||||
return
|
||||
}
|
||||
for za0001 := range z {
|
||||
z[za0001], bts, err = msgp.ReadUint64Bytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
o = bts
|
||||
return
|
||||
}
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *versionsHistogram) Msgsize() (s int) {
|
||||
s = msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1196,3 +1196,116 @@ func BenchmarkDecodetierStats(b *testing.B) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalversionsHistogram(t *testing.T) {
|
||||
v := versionsHistogram{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgversionsHistogram(b *testing.B) {
|
||||
v := versionsHistogram{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgversionsHistogram(b *testing.B) {
|
||||
v := versionsHistogram{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalversionsHistogram(b *testing.B) {
|
||||
v := versionsHistogram{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeversionsHistogram(t *testing.T) {
|
||||
v := versionsHistogram{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeversionsHistogram Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := versionsHistogram{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeversionsHistogram(b *testing.B) {
|
||||
v := versionsHistogram{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeversionsHistogram(b *testing.B) {
|
||||
v := versionsHistogram{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,11 +58,12 @@ type BucketUsageInfo struct {
|
||||
// Total number of objects that failed replication
|
||||
ReplicationFailedCountV1 uint64 `json:"objectsFailedReplicationCount"`
|
||||
|
||||
ObjectsCount uint64 `json:"objectsCount"`
|
||||
ObjectSizesHistogram map[string]uint64 `json:"objectsSizesHistogram"`
|
||||
VersionsCount uint64 `json:"versionsCount"`
|
||||
ReplicaSize uint64 `json:"objectReplicaTotalSize"`
|
||||
ReplicationInfo map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
|
||||
ObjectsCount uint64 `json:"objectsCount"`
|
||||
ObjectSizesHistogram map[string]uint64 `json:"objectsSizesHistogram"`
|
||||
ObjectVersionsHistogram map[string]uint64 `json:"objectsVersionsHistogram"`
|
||||
VersionsCount uint64 `json:"versionsCount"`
|
||||
ReplicaSize uint64 `json:"objectReplicaTotalSize"`
|
||||
ReplicationInfo map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
|
||||
}
|
||||
|
||||
// DataUsageInfo represents data usage stats of the underlying Object API
|
||||
|
||||
+15
-4
@@ -25,7 +25,6 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
@@ -35,6 +34,7 @@ import (
|
||||
"github.com/minio/minio/internal/hash"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/sync/errgroup"
|
||||
xnet "github.com/minio/pkg/net"
|
||||
"github.com/minio/pkg/wildcard"
|
||||
)
|
||||
|
||||
@@ -448,7 +448,10 @@ func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string,
|
||||
return cachedObjInfo, nil
|
||||
}
|
||||
c.cacheStats.incMiss()
|
||||
return ObjectInfo{}, BackendDown{}
|
||||
if xnet.IsNetworkOrHostDown(err, false) {
|
||||
return ObjectInfo{}, BackendDown{Err: err.Error()}
|
||||
}
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
// Reaching here implies cache miss
|
||||
c.cacheStats.incMiss()
|
||||
@@ -893,8 +896,16 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
|
||||
cacheDiskStats[i].UsageSize = info.Used
|
||||
cacheDiskStats[i].TotalCapacity = info.Total
|
||||
cacheDiskStats[i].Dir = dcache.stats.Dir
|
||||
atomic.StoreInt32(&cacheDiskStats[i].UsageState, atomic.LoadInt32(&dcache.stats.UsageState))
|
||||
atomic.StoreUint64(&cacheDiskStats[i].UsagePercent, atomic.LoadUint64(&dcache.stats.UsagePercent))
|
||||
if info.Total != 0 {
|
||||
// UsageState
|
||||
gcTriggerPct := dcache.quotaPct * dcache.highWatermark / 100
|
||||
usedPercent := float64(info.Used) * 100 / float64(info.Total)
|
||||
if usedPercent >= float64(gcTriggerPct) {
|
||||
cacheDiskStats[i].UsageState = 1
|
||||
}
|
||||
// UsagePercent
|
||||
cacheDiskStats[i].UsagePercent = uint64(usedPercent)
|
||||
}
|
||||
}
|
||||
}
|
||||
return cacheDiskStats
|
||||
|
||||
@@ -110,7 +110,7 @@ func kmsKeyIDFromMetadata(metadata map[string]string) string {
|
||||
//
|
||||
// DecryptETags uses a KMS bulk decryption API, if available, which
|
||||
// is more efficient than decrypting ETags sequentually.
|
||||
func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo) error {
|
||||
func DecryptETags(ctx context.Context, k kms.KMS, objects []ObjectInfo) error {
|
||||
const BatchSize = 250 // We process the objects in batches - 250 is a reasonable default.
|
||||
var (
|
||||
metadata = make([]map[string]string, 0, BatchSize)
|
||||
@@ -170,7 +170,7 @@ func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo) error
|
||||
// For all SSE-S3 single-part objects we have to
|
||||
// fetch their decryption keys. We do this using
|
||||
// a Bulk-Decryption API call, if available.
|
||||
keys, err := crypto.S3.UnsealObjectKeys(ctx, KMS, metadata, buckets, names)
|
||||
keys, err := crypto.S3.UnsealObjectKeys(ctx, k, metadata, buckets, names)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -62,13 +62,14 @@ func TestNewEndpoint(t *testing.T) {
|
||||
err = endpoint.UpdateIsLocal()
|
||||
}
|
||||
|
||||
if test.expectedErr == nil {
|
||||
switch {
|
||||
case test.expectedErr == nil:
|
||||
if err != nil {
|
||||
t.Errorf("error: expected = <nil>, got = %v", err)
|
||||
}
|
||||
} else if err == nil {
|
||||
case err == nil:
|
||||
t.Errorf("error: expected = %v, got = <nil>", test.expectedErr)
|
||||
} else if test.expectedErr.Error() != err.Error() {
|
||||
case test.expectedErr.Error() != err.Error():
|
||||
t.Errorf("error: expected = %v, got = %v", test.expectedErr, err)
|
||||
}
|
||||
|
||||
@@ -115,13 +116,14 @@ func TestNewEndpoints(t *testing.T) {
|
||||
|
||||
for _, testCase := range testCases {
|
||||
_, err := NewEndpoints(testCase.args...)
|
||||
if testCase.expectedErr == nil {
|
||||
switch {
|
||||
case testCase.expectedErr == nil:
|
||||
if err != nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
}
|
||||
} else if err == nil {
|
||||
case err == nil:
|
||||
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
||||
} else if testCase.expectedErr.Error() != err.Error() {
|
||||
case testCase.expectedErr.Error() != err.Error():
|
||||
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ import (
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/dustin/go-humanize"
|
||||
)
|
||||
|
||||
type badDisk struct{ StorageAPI }
|
||||
@@ -41,10 +41,6 @@ func (a badDisk) ReadFileStream(ctx context.Context, volume, path string, offset
|
||||
return nil, errFaultyDisk
|
||||
}
|
||||
|
||||
func (a badDisk) UpdateBloomFilter(ctx context.Context, oldest, current uint64) (*bloomFilterResponse, error) {
|
||||
return nil, errFaultyDisk
|
||||
}
|
||||
|
||||
func (a badDisk) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
|
||||
return errFaultyDisk
|
||||
}
|
||||
|
||||
+9
-12
@@ -72,10 +72,6 @@ func (fi FileInfo) DataShardFixed() bool {
|
||||
func (er erasureObjects) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (
|
||||
result madmin.HealResultItem, err error,
|
||||
) {
|
||||
if !opts.DryRun {
|
||||
defer NSUpdated(bucket, slashSeparator)
|
||||
}
|
||||
|
||||
storageDisks := er.getDisks()
|
||||
storageEndpoints := er.getEndpoints()
|
||||
|
||||
@@ -754,7 +750,6 @@ func (er *erasureObjects) healObjectDir(ctx context.Context, bucket, object stri
|
||||
}(index, disk)
|
||||
}
|
||||
wg.Wait()
|
||||
NSUpdated(bucket, object)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -915,13 +910,14 @@ func isObjectDirDangling(errs []error) (ok bool) {
|
||||
var foundNotEmpty int
|
||||
var otherFound int
|
||||
for _, readErr := range errs {
|
||||
if readErr == nil {
|
||||
switch {
|
||||
case readErr == nil:
|
||||
found++
|
||||
} else if readErr == errFileNotFound || readErr == errVolumeNotFound {
|
||||
case readErr == errFileNotFound || readErr == errVolumeNotFound:
|
||||
notFound++
|
||||
} else if readErr == errVolumeNotEmpty {
|
||||
case readErr == errVolumeNotEmpty:
|
||||
foundNotEmpty++
|
||||
} else {
|
||||
default:
|
||||
otherFound++
|
||||
}
|
||||
}
|
||||
@@ -943,11 +939,12 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
|
||||
diskNotFoundCount int
|
||||
)
|
||||
for _, readErr := range cerrs {
|
||||
if errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound) {
|
||||
switch {
|
||||
case errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound):
|
||||
notFoundCount++
|
||||
} else if errors.Is(readErr, errFileCorrupt) {
|
||||
case errors.Is(readErr, errFileCorrupt):
|
||||
corruptedCount++
|
||||
} else if errors.Is(readErr, errDiskNotFound) {
|
||||
case errors.Is(readErr, errDiskNotFound):
|
||||
diskNotFoundCount++
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1621,10 +1621,10 @@ func TestHealLastDataShard(t *testing.T) {
|
||||
}
|
||||
|
||||
firstGr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, noLock, ObjectOptions{})
|
||||
defer firstGr.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer firstGr.Close()
|
||||
|
||||
firstHealedH := sha256.New()
|
||||
_, err = io.Copy(firstHealedH, firstGr)
|
||||
@@ -1651,10 +1651,10 @@ func TestHealLastDataShard(t *testing.T) {
|
||||
}
|
||||
|
||||
secondGr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, noLock, ObjectOptions{})
|
||||
defer secondGr.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer secondGr.Close()
|
||||
|
||||
secondHealedH := sha256.New()
|
||||
_, err = io.Copy(secondHealedH, secondGr)
|
||||
|
||||
@@ -1216,7 +1216,6 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
if err != nil {
|
||||
return oi, toObjectErr(err, bucket, object)
|
||||
}
|
||||
defer NSUpdated(bucket, object)
|
||||
|
||||
if !opts.Speedtest && versionsDisparity {
|
||||
listAndHeal(ctx, bucket, object, &er, healObjectVersionsDisparity)
|
||||
|
||||
+1
-13
@@ -72,8 +72,6 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
|
||||
return oi, NotImplemented{}
|
||||
}
|
||||
|
||||
defer NSUpdated(dstBucket, dstObject)
|
||||
|
||||
if !dstOpts.NoLock {
|
||||
lk := er.NewNSLock(dstBucket, dstObject)
|
||||
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||
@@ -483,7 +481,6 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
|
||||
if opts.VersionID != "" {
|
||||
err = errFileVersionNotFound
|
||||
}
|
||||
defer NSUpdated(bucket, object)
|
||||
|
||||
fi := FileInfo{
|
||||
VersionID: m.VersionID,
|
||||
@@ -1271,8 +1268,6 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
return ObjectInfo{}, toObjectErr(err, bucket, object)
|
||||
}
|
||||
|
||||
defer NSUpdated(bucket, object)
|
||||
|
||||
for i := 0; i < len(onlineDisks); i++ {
|
||||
if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
|
||||
// Object info is the same in all disks, so we can pick
|
||||
@@ -1475,8 +1470,6 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
|
||||
} else {
|
||||
errs[objIndex] = toObjectErr(err, bucket, objects[objIndex].ObjectName)
|
||||
}
|
||||
|
||||
defer NSUpdated(bucket, objects[objIndex].ObjectName)
|
||||
}
|
||||
|
||||
// Check failed deletes across multiple objects
|
||||
@@ -1568,16 +1561,12 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
|
||||
}
|
||||
}
|
||||
|
||||
objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
|
||||
defer NSUpdated(bucket, object)
|
||||
|
||||
storageDisks := er.getDisks()
|
||||
versionFound := true
|
||||
objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
|
||||
goi, _, gerr := er.getObjectInfoAndQuorum(ctx, bucket, object, opts)
|
||||
if gerr != nil && goi.Name == "" {
|
||||
switch gerr.(type) {
|
||||
case InsufficientReadQuorum:
|
||||
if _, ok := gerr.(InsufficientReadQuorum); ok {
|
||||
return objInfo, InsufficientWriteQuorum{}
|
||||
}
|
||||
// For delete marker replication, versionID being replicated will not exist on disk
|
||||
@@ -1940,7 +1929,6 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
|
||||
if fi.TransitionStatus == lifecycle.TransitionComplete {
|
||||
return nil
|
||||
}
|
||||
defer NSUpdated(bucket, object)
|
||||
|
||||
if fi.XLV1 {
|
||||
if _, err = er.HealObject(ctx, bucket, object, "", madmin.HealOpts{NoLock: true}); err != nil {
|
||||
|
||||
@@ -32,7 +32,6 @@ import (
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/console"
|
||||
@@ -735,17 +734,15 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
|
||||
}
|
||||
versioned := vc != nil && vc.Versioned(object)
|
||||
objInfo := fi.ToObjectInfo(bucket, object, versioned)
|
||||
|
||||
evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
|
||||
switch evt.Action {
|
||||
case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
|
||||
globalExpiryState.enqueueByDays(objInfo, false, evt.Action == lifecycle.DeleteVersionAction)
|
||||
// Skip this entry.
|
||||
return true
|
||||
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
|
||||
globalExpiryState.enqueueByDays(objInfo, true, evt.Action == lifecycle.DeleteRestoredVersionAction)
|
||||
// Skip this entry.
|
||||
return true
|
||||
if evt.Action.Delete() {
|
||||
globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
|
||||
if !evt.Action.DeleteRestored() {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
@@ -32,7 +32,6 @@ import (
|
||||
|
||||
"github.com/lithammer/shortuuid/v4"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/env"
|
||||
@@ -463,17 +462,13 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
|
||||
}
|
||||
versioned := vc != nil && vc.Versioned(object)
|
||||
objInfo := fi.ToObjectInfo(bucket, object, versioned)
|
||||
event := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
|
||||
switch action := event.Action; action {
|
||||
case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
|
||||
globalExpiryState.enqueueByDays(objInfo, false, action == lifecycle.DeleteVersionAction)
|
||||
// Skip this entry.
|
||||
return true
|
||||
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
|
||||
globalExpiryState.enqueueByDays(objInfo, true, action == lifecycle.DeleteRestoredVersionAction)
|
||||
// Skip this entry.
|
||||
|
||||
evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
|
||||
if evt.Action.Delete() {
|
||||
globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
+28
-42
@@ -37,7 +37,6 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/s3utils"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/sync/errgroup"
|
||||
@@ -56,9 +55,6 @@ type erasureServerPools struct {
|
||||
|
||||
serverPools []*erasureSets
|
||||
|
||||
// Shut down async operations
|
||||
shutdown context.CancelFunc
|
||||
|
||||
// Active decommission canceler
|
||||
decommissionCancelers []context.CancelFunc
|
||||
|
||||
@@ -160,14 +156,9 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
|
||||
break
|
||||
}
|
||||
|
||||
drives := make([]string, 0, len(localDrives))
|
||||
for _, localDrive := range localDrives {
|
||||
drives = append(drives, localDrive.Endpoint().Path)
|
||||
}
|
||||
|
||||
globalLocalDrivesMu.Lock()
|
||||
globalLocalDrives = localDrives
|
||||
ctx, z.shutdown = context.WithCancel(ctx)
|
||||
go intDataUpdateTracker.start(ctx, drives...)
|
||||
defer globalLocalDrivesMu.Unlock()
|
||||
return z, nil
|
||||
}
|
||||
|
||||
@@ -431,6 +422,7 @@ func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bu
|
||||
return mtime1.After(mtime2)
|
||||
})
|
||||
|
||||
defPool := PoolObjInfo{Index: -1}
|
||||
for _, pinfo := range poolObjInfos {
|
||||
// skip all objects from suspended pools if asked by the
|
||||
// caller.
|
||||
@@ -445,14 +437,13 @@ func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bu
|
||||
if pinfo.Err != nil && !isErrObjectNotFound(pinfo.Err) {
|
||||
return pinfo, pinfo.Err
|
||||
}
|
||||
|
||||
defPool = pinfo
|
||||
if isErrObjectNotFound(pinfo.Err) {
|
||||
// No object exists or its a delete marker,
|
||||
// check objInfo to confirm.
|
||||
if pinfo.ObjInfo.DeleteMarker && pinfo.ObjInfo.Name != "" {
|
||||
return pinfo, nil
|
||||
}
|
||||
|
||||
// objInfo is not valid, truly the object doesn't
|
||||
// exist proceed to next pool.
|
||||
continue
|
||||
@@ -460,7 +451,12 @@ func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bu
|
||||
|
||||
return pinfo, nil
|
||||
}
|
||||
|
||||
if opts.ReplicationRequest && opts.DeleteMarker && defPool.Index >= 0 {
|
||||
// If the request is a delete marker replication request, return a default pool
|
||||
// in cases where the object does not exist.
|
||||
// This is to ensure that the delete marker is replicated to the destination.
|
||||
return defPool, nil
|
||||
}
|
||||
return PoolObjInfo{}, toObjectErr(errFileNotFound, bucket, object)
|
||||
}
|
||||
|
||||
@@ -524,8 +520,6 @@ func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object stri
|
||||
}
|
||||
|
||||
func (z *erasureServerPools) Shutdown(ctx context.Context) error {
|
||||
defer z.shutdown()
|
||||
|
||||
g := errgroup.WithNErrs(len(z.serverPools))
|
||||
|
||||
for index := range z.serverPools {
|
||||
@@ -594,7 +588,7 @@ func (z *erasureServerPools) StorageInfo(ctx context.Context) StorageInfo {
|
||||
return globalNotificationSys.StorageInfo(z)
|
||||
}
|
||||
|
||||
func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32, healScanMode madmin.HealScanMode) error {
|
||||
func (z *erasureServerPools) NSScanner(ctx context.Context, updates chan<- DataUsageInfo, wantCycle uint32, healScanMode madmin.HealScanMode) error {
|
||||
// Updates must be closed before we return.
|
||||
defer close(updates)
|
||||
|
||||
@@ -639,7 +633,7 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
|
||||
}
|
||||
}()
|
||||
// Start scanner. Blocks until done.
|
||||
err := erObj.nsScanner(ctx, allBuckets, bf, wantCycle, updates, healScanMode)
|
||||
err := erObj.nsScanner(ctx, allBuckets, wantCycle, updates, healScanMode)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
mu.Lock()
|
||||
@@ -701,9 +695,11 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
|
||||
case updateCloser <- ch:
|
||||
<-ch
|
||||
case <-ctx.Done():
|
||||
mu.Lock()
|
||||
if firstErr == nil {
|
||||
firstErr = ctx.Err()
|
||||
}
|
||||
defer mu.Unlock()
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
@@ -712,8 +708,6 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
|
||||
// even if one of the sets fail to create buckets, we proceed all the successful
|
||||
// operations.
|
||||
func (z *erasureServerPools) MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error {
|
||||
defer NSUpdated(bucket, slashSeparator)
|
||||
|
||||
// Verify if bucket is valid.
|
||||
if !isMinioMetaBucketName(bucket) {
|
||||
if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
|
||||
@@ -999,8 +993,7 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
||||
gopts.NoLock = true
|
||||
pinfo, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, gopts)
|
||||
if err != nil {
|
||||
switch err.(type) {
|
||||
case InsufficientReadQuorum:
|
||||
if _, ok := err.(InsufficientReadQuorum); ok {
|
||||
return objInfo, InsufficientWriteQuorum{}
|
||||
}
|
||||
return objInfo, err
|
||||
@@ -1313,11 +1306,12 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
|
||||
if err == nil {
|
||||
if opts.Lifecycle != nil {
|
||||
evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, objInfo)
|
||||
switch evt.Action {
|
||||
case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
|
||||
fallthrough
|
||||
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
|
||||
return loi, nil
|
||||
if evt.Action.Delete() {
|
||||
globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
|
||||
if !evt.Action.DeleteRestored() {
|
||||
// Skip entry if ILM action was DeleteVersionAction or DeleteAction
|
||||
return loi, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
loi.Objects = append(loi.Objects, objInfo)
|
||||
@@ -1466,8 +1460,7 @@ func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object,
|
||||
if err == nil {
|
||||
return pi, nil
|
||||
}
|
||||
switch err.(type) {
|
||||
case InvalidUploadID:
|
||||
if _, ok := err.(InvalidUploadID); ok {
|
||||
// Look for information on the next pool
|
||||
continue
|
||||
}
|
||||
@@ -1498,8 +1491,7 @@ func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, objec
|
||||
if err == nil {
|
||||
return mi, nil
|
||||
}
|
||||
switch err.(type) {
|
||||
case InvalidUploadID:
|
||||
if _, ok := err.(InvalidUploadID); ok {
|
||||
// upload id not found, continue to the next pool.
|
||||
continue
|
||||
}
|
||||
@@ -1530,8 +1522,7 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
switch err.(type) {
|
||||
case InvalidUploadID:
|
||||
if _, ok := err.(InvalidUploadID); ok {
|
||||
continue
|
||||
}
|
||||
return ListPartsInfo{}, err
|
||||
@@ -1561,8 +1552,7 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
switch err.(type) {
|
||||
case InvalidUploadID:
|
||||
if _, ok := err.(InvalidUploadID); ok {
|
||||
// upload id not found move to next pool
|
||||
continue
|
||||
}
|
||||
@@ -1593,8 +1583,7 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
|
||||
if err == nil {
|
||||
return objInfo, nil
|
||||
}
|
||||
switch err.(type) {
|
||||
case InvalidUploadID:
|
||||
if _, ok := err.(InvalidUploadID); ok {
|
||||
// upload id not found move to next pool
|
||||
continue
|
||||
}
|
||||
@@ -1637,7 +1626,6 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
|
||||
return BucketNameInvalid{Bucket: bucket}
|
||||
}
|
||||
|
||||
defer NSUpdated(bucket, slashSeparator)
|
||||
if !opts.NoLock {
|
||||
// Lock the bucket name before creating.
|
||||
lk := z.NewNSLock(minioMetaTmpBucket, bucket+".lck")
|
||||
@@ -1652,8 +1640,7 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
|
||||
err := z.s3Peer.DeleteBucket(ctx, bucket, opts)
|
||||
if err == nil || errors.Is(err, errVolumeNotFound) {
|
||||
// If site replication is configured, hold on to deleted bucket state until sites sync
|
||||
switch opts.SRDeleteOp {
|
||||
case MarkDelete:
|
||||
if opts.SRDeleteOp == MarkDelete {
|
||||
z.s3Peer.MakeBucket(context.Background(), pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, bucket), MakeBucketOptions{})
|
||||
}
|
||||
}
|
||||
@@ -1757,8 +1744,7 @@ func (z *erasureServerPools) HealBucket(ctx context.Context, bucket string, opts
|
||||
for _, pool := range z.serverPools {
|
||||
result, err := pool.HealBucket(ctx, bucket, opts)
|
||||
if err != nil {
|
||||
switch err.(type) {
|
||||
case BucketNotFound:
|
||||
if _, ok := err.(BucketNotFound); ok {
|
||||
continue
|
||||
}
|
||||
return result, err
|
||||
|
||||
+12
-17
@@ -33,7 +33,6 @@ import (
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/sync/errgroup"
|
||||
"github.com/minio/pkg/console"
|
||||
)
|
||||
|
||||
// list all errors that can be ignore in a bucket operation.
|
||||
@@ -351,7 +350,7 @@ func (er erasureObjects) cleanupDeletedObjects(ctx context.Context) {
|
||||
|
||||
// nsScanner will start scanning buckets and send updated totals as they are traversed.
|
||||
// Updates are sent on a regular basis and the caller *must* consume them.
|
||||
func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, wantCycle uint32, updates chan<- dataUsageCache, healScanMode madmin.HealScanMode) error {
|
||||
func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wantCycle uint32, updates chan<- dataUsageCache, healScanMode madmin.HealScanMode) error {
|
||||
if len(buckets) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -377,7 +376,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
|
||||
},
|
||||
Cache: make(map[string]dataUsageEntry, len(oldCache.Cache)),
|
||||
}
|
||||
bloom := bf.bytes()
|
||||
|
||||
// Put all buckets into channel.
|
||||
bucketCh := make(chan BucketInfo, len(buckets))
|
||||
@@ -414,9 +412,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Return without saving.
|
||||
return
|
||||
case <-t.C:
|
||||
if cache.Info.LastUpdate.Equal(lastSave) {
|
||||
continue
|
||||
@@ -454,6 +449,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
|
||||
// Start one scanner per disk
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(len(disks))
|
||||
|
||||
for i := range disks {
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
@@ -473,7 +469,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
|
||||
if cache.Info.Name == "" {
|
||||
cache.Info.Name = bucket.Name
|
||||
}
|
||||
cache.Info.BloomFilter = bloom
|
||||
cache.Info.SkipHealing = healing
|
||||
cache.Info.NextCycle = wantCycle
|
||||
if cache.Info.Name != bucket.Name {
|
||||
@@ -491,13 +486,13 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
|
||||
go func(name string) {
|
||||
defer wg.Done()
|
||||
for update := range updates {
|
||||
bucketResults <- dataUsageEntryInfo{
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case bucketResults <- dataUsageEntryInfo{
|
||||
Name: name,
|
||||
Parent: dataUsageRoot,
|
||||
Entry: update,
|
||||
}
|
||||
if intDataUpdateTracker.debug {
|
||||
console.Debugln("z:", er.poolIndex, "s:", er.setIndex, "bucket", name, "got update", update)
|
||||
}:
|
||||
}
|
||||
}
|
||||
}(cache.Info.Name)
|
||||
@@ -505,7 +500,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
|
||||
before := cache.Info.LastUpdate
|
||||
var err error
|
||||
cache, err = disk.NSScanner(ctx, cache, updates, healScanMode)
|
||||
cache.Info.BloomFilter = nil
|
||||
if err != nil {
|
||||
if !cache.Info.LastUpdate.IsZero() && cache.Info.LastUpdate.After(before) {
|
||||
logger.LogIf(ctx, cache.save(ctx, er, cacheName))
|
||||
@@ -525,15 +519,16 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
|
||||
if r := cache.root(); r != nil {
|
||||
root = cache.flatten(*r)
|
||||
}
|
||||
t := time.Now()
|
||||
bucketResults <- dataUsageEntryInfo{
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case bucketResults <- dataUsageEntryInfo{
|
||||
Name: cache.Info.Name,
|
||||
Parent: dataUsageRoot,
|
||||
Entry: root,
|
||||
}:
|
||||
}
|
||||
// We want to avoid synchronizing up all writes in case
|
||||
// the results are piled up.
|
||||
time.Sleep(time.Duration(float64(time.Since(t)) * rand.Float64()))
|
||||
|
||||
// Save cache
|
||||
logger.LogIf(ctx, cache.save(ctx, er, cacheName))
|
||||
}
|
||||
|
||||
@@ -101,7 +101,7 @@ func (evnot *EventNotifier) InitBucketTargets(ctx context.Context, objAPI Object
|
||||
return errServerNotInitialized
|
||||
}
|
||||
|
||||
if err := evnot.targetList.Add(globalConfigTargetList.Targets()...); err != nil {
|
||||
if err := evnot.targetList.Add(globalNotifyTargetList.Targets()...); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
+25
-5
@@ -269,7 +269,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
|
||||
}()
|
||||
|
||||
// Note: updates from healEntry to tracker must be sent on results channel.
|
||||
healEntry := func(entry metaCacheEntry) {
|
||||
healEntry := func(bucket string, entry metaCacheEntry) {
|
||||
if entry.name == "" && len(entry.metadata) == 0 {
|
||||
// ignore entries that don't have metadata.
|
||||
return
|
||||
@@ -278,6 +278,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
|
||||
// ignore healing entry.name's with `/` suffix.
|
||||
return
|
||||
}
|
||||
|
||||
// We might land at .metacache, .trash, .multipart
|
||||
// no need to heal them skip, only when bucket
|
||||
// is '.minio.sys'
|
||||
@@ -302,6 +303,11 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
|
||||
versionID: "",
|
||||
}, madmin.HealItemObject)
|
||||
if err != nil {
|
||||
if isErrObjectNotFound(err) {
|
||||
// queueing happens across namespace, ignore
|
||||
// objects that are not found.
|
||||
return
|
||||
}
|
||||
result = healEntryFailure(0)
|
||||
logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, entry.name, err))
|
||||
} else {
|
||||
@@ -317,12 +323,19 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
|
||||
return
|
||||
}
|
||||
|
||||
var versionNotFound int
|
||||
for _, version := range fivs.Versions {
|
||||
if err := bgSeq.queueHealTask(healSource{
|
||||
bucket: bucket,
|
||||
object: version.Name,
|
||||
versionID: version.VersionID,
|
||||
}, madmin.HealItemObject); err != nil {
|
||||
if isErrObjectNotFound(err) {
|
||||
// queueing happens across namespace, ignore
|
||||
// objects that are not found.
|
||||
versionNotFound++
|
||||
continue
|
||||
}
|
||||
// If not deleted, assume they failed.
|
||||
result = healEntryFailure(uint64(version.Size))
|
||||
if version.VersionID != "" {
|
||||
@@ -341,6 +354,10 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
|
||||
case results <- result:
|
||||
}
|
||||
}
|
||||
// All versions resulted in 'ObjectNotFound'
|
||||
if versionNotFound == len(fivs.Versions) {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
@@ -351,22 +368,25 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
|
||||
waitForLowHTTPReq()
|
||||
}
|
||||
|
||||
actualBucket, prefix := path2BucketObject(bucket)
|
||||
|
||||
// How to resolve partial results.
|
||||
resolver := metadataResolutionParams{
|
||||
dirQuorum: 1,
|
||||
objQuorum: 1,
|
||||
bucket: bucket,
|
||||
bucket: actualBucket,
|
||||
}
|
||||
|
||||
err := listPathRaw(ctx, listPathRawOptions{
|
||||
disks: disks,
|
||||
bucket: bucket,
|
||||
bucket: actualBucket,
|
||||
path: prefix,
|
||||
recursive: true,
|
||||
forwardTo: forwardTo,
|
||||
minDisks: 1,
|
||||
reportNotFound: false,
|
||||
agreed: func(entry metaCacheEntry) {
|
||||
healEntry(entry)
|
||||
healEntry(actualBucket, entry)
|
||||
},
|
||||
partial: func(entries metaCacheEntries, _ []error) {
|
||||
entry, ok := entries.resolve(&resolver)
|
||||
@@ -375,7 +395,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
|
||||
// proceed to heal nonetheless.
|
||||
entry, _ = entries.firstFound()
|
||||
}
|
||||
healEntry(*entry)
|
||||
healEntry(actualBucket, *entry)
|
||||
},
|
||||
finished: nil,
|
||||
})
|
||||
|
||||
+9
-11
@@ -42,16 +42,14 @@ import (
|
||||
"github.com/minio/minio/internal/config/callhome"
|
||||
"github.com/minio/minio/internal/config/compress"
|
||||
"github.com/minio/minio/internal/config/dns"
|
||||
xldap "github.com/minio/minio/internal/config/identity/ldap"
|
||||
"github.com/minio/minio/internal/config/identity/openid"
|
||||
idplugin "github.com/minio/minio/internal/config/identity/plugin"
|
||||
xtls "github.com/minio/minio/internal/config/identity/tls"
|
||||
polplugin "github.com/minio/minio/internal/config/policy/plugin"
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
"github.com/minio/minio/internal/config/subnet"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
etcd "go.etcd.io/etcd/client/v3"
|
||||
|
||||
levent "github.com/minio/minio/internal/config/lambda/event"
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/pubsub"
|
||||
"github.com/minio/pkg/certs"
|
||||
@@ -177,7 +175,8 @@ var (
|
||||
globalMinioConsoleHost = ""
|
||||
|
||||
// Holds the possible host endpoint.
|
||||
globalMinioEndpoint = ""
|
||||
globalMinioEndpoint = ""
|
||||
globalMinioEndpointURL *xnet.URL
|
||||
|
||||
// globalConfigSys server config system.
|
||||
globalConfigSys *ConfigSys
|
||||
@@ -185,7 +184,8 @@ var (
|
||||
globalNotificationSys *NotificationSys
|
||||
|
||||
globalEventNotifier *EventNotifier
|
||||
globalConfigTargetList *event.TargetList
|
||||
globalNotifyTargetList *event.TargetList
|
||||
globalLambdaTargetList *levent.TargetList
|
||||
|
||||
globalBucketMetadataSys *BucketMetadataSys
|
||||
globalBucketMonitor *bandwidth.Monitor
|
||||
@@ -201,10 +201,6 @@ var (
|
||||
|
||||
globalStorageClass storageclass.Config
|
||||
|
||||
globalLDAPConfig xldap.Config
|
||||
globalOpenIDConfig openid.Config
|
||||
globalSTSTLSConfig xtls.Config
|
||||
|
||||
globalAuthNPlugin *idplugin.AuthNPlugin
|
||||
|
||||
// CA root certificates, a nil value means system certs pool will be used
|
||||
@@ -357,8 +353,10 @@ var (
|
||||
globalServiceFreezeCnt int32
|
||||
globalServiceFreezeMu sync.Mutex // Updates.
|
||||
|
||||
// List of local drives to this node, this is only set during server startup.
|
||||
globalLocalDrives []StorageAPI
|
||||
// List of local drives to this node, this is only set during server startup,
|
||||
// and should never be mutated. Hold globalLocalDrivesMu to access.
|
||||
globalLocalDrives []StorageAPI
|
||||
globalLocalDrivesMu sync.RWMutex
|
||||
|
||||
// Is MINIO_CI_CD set?
|
||||
globalIsCICD bool
|
||||
|
||||
@@ -227,6 +227,13 @@ func (t *apiConfig) getClusterDeadline() time.Duration {
|
||||
return t.clusterDeadline
|
||||
}
|
||||
|
||||
func (t *apiConfig) getRequestsPoolCapacity() int {
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
|
||||
return cap(t.requestsPool)
|
||||
}
|
||||
|
||||
func (t *apiConfig) getRequestsPool() (chan struct{}, time.Duration) {
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
|
||||
@@ -58,6 +58,10 @@ func parseLocationConstraint(r *http.Request) (location string, s3Error APIError
|
||||
if location == "" {
|
||||
location = globalSite.Region
|
||||
}
|
||||
if !isValidLocation(location) {
|
||||
return location, ErrInvalidRegion
|
||||
}
|
||||
|
||||
return location, ErrNone
|
||||
}
|
||||
|
||||
|
||||
@@ -106,6 +106,17 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set(xhttp.MinIOServerStatus, unavailable)
|
||||
}
|
||||
|
||||
if int(globalHTTPStats.loadRequestsInQueue()) > globalAPIConfig.getRequestsPoolCapacity() {
|
||||
apiErr := getAPIError(ErrBusy)
|
||||
switch r.Method {
|
||||
case http.MethodHead:
|
||||
writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
|
||||
case http.MethodGet:
|
||||
writeErrorResponse(r.Context(), w, apiErr, r.URL)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Verify if KMS is reachable if its configured
|
||||
if GlobalKMS != nil {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), time.Minute)
|
||||
|
||||
@@ -255,6 +255,10 @@ type HTTPStats struct {
|
||||
totalS3Canceled HTTPAPIStats
|
||||
}
|
||||
|
||||
func (st *HTTPStats) loadRequestsInQueue() int32 {
|
||||
return atomic.LoadInt32(&st.s3RequestsInQueue)
|
||||
}
|
||||
|
||||
func (st *HTTPStats) addRequestsInQueue(i int32) {
|
||||
atomic.AddInt32(&st.s3RequestsInQueue, i)
|
||||
}
|
||||
|
||||
@@ -245,6 +245,13 @@ func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMU
|
||||
if u.Credentials.AccessKey == "" {
|
||||
u.Credentials.AccessKey = user
|
||||
}
|
||||
if u.Credentials.SessionToken != "" {
|
||||
jwtClaims, err := extractJWTClaims(u)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
u.Credentials.Claims = jwtClaims.Map()
|
||||
}
|
||||
m[user] = u
|
||||
return nil
|
||||
}
|
||||
|
||||
+11
-1
@@ -184,6 +184,14 @@ func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType
|
||||
u.Credentials.AccessKey = user
|
||||
}
|
||||
|
||||
if u.Credentials.SessionToken != "" {
|
||||
jwtClaims, err := extractJWTClaims(u)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
u.Credentials.Claims = jwtClaims.Map()
|
||||
}
|
||||
|
||||
m[user] = u
|
||||
return nil
|
||||
}
|
||||
@@ -341,7 +349,9 @@ func (iamOS *IAMObjectStore) listAllIAMConfigItems(ctx context.Context) (map[str
|
||||
// Assumes cache is locked by caller.
|
||||
func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iamCache) error {
|
||||
bootstrapTrace("loading all IAM items")
|
||||
|
||||
if iamOS.objAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
listedConfigItems, err := iamOS.listAllIAMConfigItems(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+72
-22
@@ -33,6 +33,7 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/config/identity/openid"
|
||||
"github.com/minio/minio/internal/jwt"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
)
|
||||
@@ -76,8 +77,13 @@ const (
|
||||
iamFormatFile = "format.json"
|
||||
|
||||
iamFormatVersion1 = 1
|
||||
|
||||
minServiceAccountExpiry time.Duration = 15 * time.Minute
|
||||
maxServiceAccountExpiry time.Duration = 365 * 24 * time.Hour
|
||||
)
|
||||
|
||||
var errInvalidSvcAcctExpiration = errors.New("invalid service account expiration")
|
||||
|
||||
type iamFormat struct {
|
||||
Version int `json:"version"`
|
||||
}
|
||||
@@ -394,6 +400,19 @@ func (c *iamCache) policyDBGet(mode UsersSysType, name string, isGroup bool) ([]
|
||||
return policies, mp.UpdatedAt, nil
|
||||
}
|
||||
|
||||
func (c *iamCache) updateUserWithClaims(key string, u UserIdentity) error {
|
||||
if u.Credentials.SessionToken != "" {
|
||||
jwtClaims, err := extractJWTClaims(u)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
u.Credentials.Claims = jwtClaims.Map()
|
||||
}
|
||||
c.iamUsersMap[key] = u
|
||||
c.updatedAt = time.Now()
|
||||
return nil
|
||||
}
|
||||
|
||||
// IAMStorageAPI defines an interface for the IAM persistence layer
|
||||
type IAMStorageAPI interface {
|
||||
// The role of the read-write lock is to prevent go routines from
|
||||
@@ -918,8 +937,7 @@ func (store *IAMStoreSys) PolicyDBUpdate(ctx context.Context, name string, isGro
|
||||
}
|
||||
|
||||
if g.Status == statusDisabled {
|
||||
// TODO: return an error?
|
||||
return updatedAt, nil, nil
|
||||
return updatedAt, nil, errGroupDisabled
|
||||
}
|
||||
}
|
||||
mp = cache.iamGroupPolicyMap[name]
|
||||
@@ -1749,14 +1767,15 @@ func (store *IAMStoreSys) DeleteUser(ctx context.Context, accessKey string, user
|
||||
if userType == regUser {
|
||||
for _, ui := range cache.iamUsersMap {
|
||||
u := ui.Credentials
|
||||
if u.IsServiceAccount() && u.ParentUser == accessKey {
|
||||
_ = store.deleteUserIdentity(ctx, u.AccessKey, svcUser)
|
||||
delete(cache.iamUsersMap, u.AccessKey)
|
||||
}
|
||||
// Delete any associated STS users.
|
||||
if u.IsTemp() && u.ParentUser == accessKey {
|
||||
_ = store.deleteUserIdentity(ctx, u.AccessKey, stsUser)
|
||||
delete(cache.iamUsersMap, u.AccessKey)
|
||||
if u.ParentUser == accessKey {
|
||||
switch {
|
||||
case u.IsServiceAccount():
|
||||
_ = store.deleteUserIdentity(ctx, u.AccessKey, svcUser)
|
||||
delete(cache.iamUsersMap, u.AccessKey)
|
||||
case u.IsTemp():
|
||||
_ = store.deleteUserIdentity(ctx, u.AccessKey, stsUser)
|
||||
delete(cache.iamUsersMap, u.AccessKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2106,8 +2125,9 @@ func (store *IAMStoreSys) SetUserStatus(ctx context.Context, accessKey string, s
|
||||
return updatedAt, err
|
||||
}
|
||||
|
||||
cache.iamUsersMap[accessKey] = uinfo
|
||||
cache.updatedAt = time.Now()
|
||||
if err := cache.updateUserWithClaims(accessKey, uinfo); err != nil {
|
||||
return updatedAt, err
|
||||
}
|
||||
|
||||
return uinfo.UpdatedAt, nil
|
||||
}
|
||||
@@ -2142,8 +2162,7 @@ func (store *IAMStoreSys) AddServiceAccount(ctx context.Context, cred auth.Crede
|
||||
return updatedAt, err
|
||||
}
|
||||
|
||||
cache.iamUsersMap[u.Credentials.AccessKey] = u
|
||||
cache.updatedAt = time.Now()
|
||||
cache.updateUserWithClaims(u.Credentials.AccessKey, u)
|
||||
|
||||
return u.UpdatedAt, nil
|
||||
}
|
||||
@@ -2170,6 +2189,14 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
|
||||
cr.Comment = opts.comment
|
||||
}
|
||||
|
||||
if opts.expiration != nil {
|
||||
expirationInUTC := opts.expiration.UTC()
|
||||
if err := validateSvcExpirationInUTC(expirationInUTC); err != nil {
|
||||
return updatedAt, err
|
||||
}
|
||||
cr.Expiration = expirationInUTC
|
||||
}
|
||||
|
||||
switch opts.status {
|
||||
// The caller did not ask to update status account, do nothing
|
||||
case "":
|
||||
@@ -2229,8 +2256,9 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
|
||||
return updatedAt, err
|
||||
}
|
||||
|
||||
cache.iamUsersMap[u.Credentials.AccessKey] = u
|
||||
cache.updatedAt = time.Now()
|
||||
if err := cache.updateUserWithClaims(u.Credentials.AccessKey, u); err != nil {
|
||||
return updatedAt, err
|
||||
}
|
||||
|
||||
return u.UpdatedAt, nil
|
||||
}
|
||||
@@ -2331,8 +2359,9 @@ func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, ureq ma
|
||||
if err := store.saveUserIdentity(ctx, accessKey, regUser, u); err != nil {
|
||||
return updatedAt, err
|
||||
}
|
||||
|
||||
cache.iamUsersMap[accessKey] = u
|
||||
if err := cache.updateUserWithClaims(accessKey, u); err != nil {
|
||||
return updatedAt, err
|
||||
}
|
||||
|
||||
return u.UpdatedAt, nil
|
||||
}
|
||||
@@ -2355,8 +2384,7 @@ func (store *IAMStoreSys) UpdateUserSecretKey(ctx context.Context, accessKey, se
|
||||
return err
|
||||
}
|
||||
|
||||
cache.iamUsersMap[accessKey] = u
|
||||
return nil
|
||||
return cache.updateUserWithClaims(accessKey, u)
|
||||
}
|
||||
|
||||
// GetSTSAndServiceAccounts - returns all STS and Service account credentials.
|
||||
@@ -2393,8 +2421,8 @@ func (store *IAMStoreSys) UpdateUserIdentity(ctx context.Context, cred auth.Cred
|
||||
if err := store.saveUserIdentity(ctx, cred.AccessKey, userType, ui); err != nil {
|
||||
return err
|
||||
}
|
||||
cache.iamUsersMap[cred.AccessKey] = ui
|
||||
return nil
|
||||
|
||||
return cache.updateUserWithClaims(cred.AccessKey, ui)
|
||||
}
|
||||
|
||||
// LoadUser - attempts to load user info from storage and updates cache.
|
||||
@@ -2437,3 +2465,25 @@ func (store *IAMStoreSys) LoadUser(ctx context.Context, accessKey string) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func extractJWTClaims(u UserIdentity) (*jwt.MapClaims, error) {
|
||||
jwtClaims, err := auth.ExtractClaims(u.Credentials.SessionToken, u.Credentials.SecretKey)
|
||||
if err != nil {
|
||||
// Session tokens for STS creds will be generated with root secret
|
||||
jwtClaims, err = auth.ExtractClaims(u.Credentials.SessionToken, globalActiveCred.SecretKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return jwtClaims, nil
|
||||
}
|
||||
|
||||
func validateSvcExpirationInUTC(expirationInUTC time.Time) error {
|
||||
currentTime := time.Now().UTC()
|
||||
minExpiration := currentTime.Add(minServiceAccountExpiry)
|
||||
maxExpiration := currentTime.Add(maxServiceAccountExpiry)
|
||||
if expirationInUTC.Before(minExpiration) || expirationInUTC.After(maxExpiration) {
|
||||
return errInvalidSvcAcctExpiration
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
+48
-30
@@ -42,6 +42,7 @@ import (
|
||||
xldap "github.com/minio/minio/internal/config/identity/ldap"
|
||||
"github.com/minio/minio/internal/config/identity/openid"
|
||||
idplugin "github.com/minio/minio/internal/config/identity/plugin"
|
||||
xtls "github.com/minio/minio/internal/config/identity/tls"
|
||||
"github.com/minio/minio/internal/config/policy/opa"
|
||||
polplugin "github.com/minio/minio/internal/config/policy/plugin"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
@@ -87,8 +88,10 @@ type IAMSys struct {
|
||||
sync.Mutex
|
||||
|
||||
iamRefreshInterval time.Duration
|
||||
ldapConfig xldap.Config // only valid if usersSysType is LDAPUsers
|
||||
openIDConfig openid.Config // only valid if OpenID is configured
|
||||
|
||||
LDAPConfig xldap.Config // only valid if usersSysType is LDAPUsers
|
||||
OpenIDConfig openid.Config // only valid if OpenID is configured
|
||||
STSTLSConfig xtls.Config // only valid if STS TLS is configured
|
||||
|
||||
usersSysType UsersSysType
|
||||
|
||||
@@ -163,7 +166,7 @@ func (sys *IAMSys) LoadServiceAccount(ctx context.Context, accessKey string) err
|
||||
|
||||
// initStore initializes IAM stores
|
||||
func (sys *IAMSys) initStore(objAPI ObjectLayer, etcdClient *etcd.Client) {
|
||||
if sys.ldapConfig.Enabled() {
|
||||
if sys.LDAPConfig.Enabled() {
|
||||
sys.SetUsersSysType(LDAPUsersSysType)
|
||||
}
|
||||
|
||||
@@ -213,26 +216,34 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
|
||||
s := globalServerConfig
|
||||
globalServerConfigMu.RUnlock()
|
||||
|
||||
var err error
|
||||
globalOpenIDConfig, err = openid.LookupConfig(s,
|
||||
openidConfig, err := openid.LookupConfig(s,
|
||||
NewHTTPTransport(), xhttp.DrainBody, globalSite.Region)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err))
|
||||
}
|
||||
|
||||
// Initialize if LDAP is enabled
|
||||
globalLDAPConfig, err = xldap.Lookup(s, globalRootCAs)
|
||||
ldapConfig, err := xldap.Lookup(s, globalRootCAs)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to parse LDAP configuration: %w", err))
|
||||
}
|
||||
|
||||
stsTLSConfig, err := xtls.Lookup(s[config.IdentityTLSSubSys][config.Default])
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize X.509/TLS STS API: %w", err))
|
||||
}
|
||||
|
||||
if stsTLSConfig.InsecureSkipVerify {
|
||||
logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
|
||||
}
|
||||
|
||||
authNPluginCfg, err := idplugin.LookupConfig(s[config.IdentityPluginSubSys][config.Default],
|
||||
NewHTTPTransport(), xhttp.DrainBody, globalSite.Region)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthNPlugin: %w", err))
|
||||
}
|
||||
|
||||
setGlobalAuthNPlugin(idplugin.New(authNPluginCfg))
|
||||
setGlobalAuthNPlugin(idplugin.New(GlobalContext, authNPluginCfg))
|
||||
|
||||
authZPluginCfg, err := polplugin.LookupConfig(s, GetDefaultConnSettings(), xhttp.DrainBody)
|
||||
if err != nil {
|
||||
@@ -257,8 +268,10 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
|
||||
sys.Lock()
|
||||
defer sys.Unlock()
|
||||
|
||||
sys.ldapConfig = globalLDAPConfig.Clone()
|
||||
sys.openIDConfig = globalOpenIDConfig.Clone()
|
||||
sys.LDAPConfig = ldapConfig
|
||||
sys.OpenIDConfig = openidConfig
|
||||
sys.STSTLSConfig = stsTLSConfig
|
||||
|
||||
sys.iamRefreshInterval = iamRefreshInterval
|
||||
|
||||
// Initialize IAM store
|
||||
@@ -322,7 +335,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
|
||||
|
||||
// Set up polling for expired accounts and credentials purging.
|
||||
switch {
|
||||
case sys.openIDConfig.ProviderEnabled():
|
||||
case sys.OpenIDConfig.ProviderEnabled():
|
||||
go func() {
|
||||
timer := time.NewTimer(refreshInterval)
|
||||
defer timer.Stop()
|
||||
@@ -337,7 +350,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
|
||||
}
|
||||
}
|
||||
}()
|
||||
case sys.ldapConfig.Enabled():
|
||||
case sys.LDAPConfig.Enabled():
|
||||
go func() {
|
||||
timer := time.NewTimer(refreshInterval)
|
||||
defer timer.Stop()
|
||||
@@ -363,7 +376,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
|
||||
sys.rolesMap = make(map[arn.ARN]string)
|
||||
|
||||
// From OpenID
|
||||
if riMap := globalOpenIDConfig.GetRoleInfo(); riMap != nil {
|
||||
if riMap := sys.OpenIDConfig.GetRoleInfo(); riMap != nil {
|
||||
sys.validateAndAddRolePolicyMappings(ctx, riMap)
|
||||
}
|
||||
|
||||
@@ -800,7 +813,7 @@ func (sys *IAMSys) QueryLDAPPolicyEntities(ctx context.Context, q madmin.PolicyE
|
||||
|
||||
select {
|
||||
case <-sys.configLoaded:
|
||||
pe := sys.store.ListLDAPPolicyMappings(q, sys.ldapConfig.IsLDAPUserDN, sys.ldapConfig.IsLDAPGroupDN)
|
||||
pe := sys.store.ListLDAPPolicyMappings(q, sys.LDAPConfig.IsLDAPUserDN, sys.LDAPConfig.IsLDAPGroupDN)
|
||||
pe.Timestamp = UTCNow()
|
||||
return &pe, nil
|
||||
case <-ctx.Done():
|
||||
@@ -920,6 +933,7 @@ type newServiceAccountOpts struct {
|
||||
accessKey string
|
||||
secretKey string
|
||||
comment string
|
||||
expiration *time.Time
|
||||
|
||||
claims map[string]interface{}
|
||||
}
|
||||
@@ -992,6 +1006,14 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
|
||||
cred.Status = string(auth.AccountOn)
|
||||
cred.Comment = opts.comment
|
||||
|
||||
if opts.expiration != nil {
|
||||
expirationInUTC := opts.expiration.UTC()
|
||||
if err := validateSvcExpirationInUTC(expirationInUTC); err != nil {
|
||||
return auth.Credentials{}, time.Time{}, err
|
||||
}
|
||||
cred.Expiration = expirationInUTC
|
||||
}
|
||||
|
||||
updatedAt, err := sys.store.AddServiceAccount(ctx, cred)
|
||||
if err != nil {
|
||||
return auth.Credentials{}, time.Time{}, err
|
||||
@@ -1006,6 +1028,7 @@ type updateServiceAccountOpts struct {
|
||||
secretKey string
|
||||
status string
|
||||
comment string
|
||||
expiration *time.Time
|
||||
}
|
||||
|
||||
// UpdateServiceAccount - edit a service account
|
||||
@@ -1145,12 +1168,9 @@ func (sys *IAMSys) getAccountWithClaims(ctx context.Context, accessKey string) (
|
||||
return UserIdentity{}, nil, errNoSuchAccount
|
||||
}
|
||||
|
||||
jwtClaims, err := auth.ExtractClaims(acc.Credentials.SessionToken, acc.Credentials.SecretKey)
|
||||
jwtClaims, err := extractJWTClaims(acc)
|
||||
if err != nil {
|
||||
jwtClaims, err = auth.ExtractClaims(acc.Credentials.SessionToken, globalActiveCred.SecretKey)
|
||||
if err != nil {
|
||||
return UserIdentity{}, nil, err
|
||||
}
|
||||
return UserIdentity{}, nil, err
|
||||
}
|
||||
|
||||
return acc, jwtClaims, nil
|
||||
@@ -1171,13 +1191,11 @@ func (sys *IAMSys) GetClaimsForSvcAcc(ctx context.Context, accessKey string) (ma
|
||||
return nil, errNoSuchServiceAccount
|
||||
}
|
||||
|
||||
jwtClaims, err := auth.ExtractClaims(sa.Credentials.SessionToken, sa.Credentials.SecretKey)
|
||||
jwtClaims, err := extractJWTClaims(sa)
|
||||
if err != nil {
|
||||
jwtClaims, err = auth.ExtractClaims(sa.Credentials.SessionToken, globalActiveCred.SecretKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return jwtClaims.Map(), nil
|
||||
}
|
||||
|
||||
@@ -1280,7 +1298,7 @@ func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) {
|
||||
continue
|
||||
}
|
||||
roleArn = roleArns[0]
|
||||
u, err := sys.openIDConfig.LookupUser(roleArn, puInfo.subClaimValue)
|
||||
u, err := sys.OpenIDConfig.LookupUser(roleArn, puInfo.subClaimValue)
|
||||
if err != nil {
|
||||
logger.LogIf(GlobalContext, err)
|
||||
continue
|
||||
@@ -1302,14 +1320,14 @@ func (sys *IAMSys) purgeExpiredCredentialsForLDAP(ctx context.Context) {
|
||||
parentUsers := sys.store.GetAllParentUsers()
|
||||
var allDistNames []string
|
||||
for parentUser := range parentUsers {
|
||||
if !sys.ldapConfig.IsLDAPUserDN(parentUser) {
|
||||
if !sys.LDAPConfig.IsLDAPUserDN(parentUser) {
|
||||
continue
|
||||
}
|
||||
|
||||
allDistNames = append(allDistNames, parentUser)
|
||||
}
|
||||
|
||||
expiredUsers, err := sys.ldapConfig.GetNonEligibleUserDistNames(allDistNames)
|
||||
expiredUsers, err := sys.LDAPConfig.GetNonEligibleUserDistNames(allDistNames)
|
||||
if err != nil {
|
||||
// Log and return on error - perhaps it'll work the next time.
|
||||
logger.LogIf(GlobalContext, err)
|
||||
@@ -1331,7 +1349,7 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
|
||||
// DN to ldap username mapping for each LDAP user
|
||||
parentUserToLDAPUsernameMap := make(map[string]string)
|
||||
for _, cred := range allCreds {
|
||||
if !sys.ldapConfig.IsLDAPUserDN(cred.ParentUser) {
|
||||
if !sys.LDAPConfig.IsLDAPUserDN(cred.ParentUser) {
|
||||
continue
|
||||
}
|
||||
// Check if this is the first time we are
|
||||
@@ -1379,7 +1397,7 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
|
||||
}
|
||||
|
||||
// 2. Query LDAP server for groups of the LDAP users collected.
|
||||
updatedGroups, err := sys.ldapConfig.LookupGroupMemberships(parentUsers, parentUserToLDAPUsernameMap)
|
||||
updatedGroups, err := sys.LDAPConfig.LookupGroupMemberships(parentUsers, parentUserToLDAPUsernameMap)
|
||||
if err != nil {
|
||||
// Log and return on error - perhaps it'll work the next time.
|
||||
logger.LogIf(GlobalContext, err)
|
||||
@@ -1566,7 +1584,7 @@ func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool,
|
||||
var dn string
|
||||
var isGroup bool
|
||||
if r.User != "" {
|
||||
dn, err = globalLDAPConfig.DoesUsernameExist(r.User)
|
||||
dn, err = sys.LDAPConfig.DoesUsernameExist(r.User)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return updatedAt, nil, err
|
||||
@@ -1576,7 +1594,7 @@ func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool,
|
||||
}
|
||||
isGroup = false
|
||||
} else {
|
||||
if exists, err := globalLDAPConfig.DoesGroupDNExist(r.Group); err != nil {
|
||||
if exists, err := sys.LDAPConfig.DoesGroupDNExist(r.Group); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return updatedAt, nil, err
|
||||
} else if !exists {
|
||||
|
||||
@@ -419,7 +419,7 @@ func Test_localLocker_RUnlock(t *testing.T) {
|
||||
}
|
||||
start = time.Now()
|
||||
for _, lock := range toUnLock {
|
||||
ok, err := l.RUnlock(nil, lock)
|
||||
ok, err := l.RUnlock(context.TODO(), lock)
|
||||
if err != nil || !ok {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -816,15 +816,16 @@ func (m *metaCacheEntriesSorted) merge(other metaCacheEntriesSorted, limit int)
|
||||
a := m.entries()
|
||||
b := other.entries()
|
||||
for len(a) > 0 && len(b) > 0 {
|
||||
if a[0].name == b[0].name && bytes.Equal(a[0].metadata, b[0].metadata) {
|
||||
switch {
|
||||
case a[0].name == b[0].name && bytes.Equal(a[0].metadata, b[0].metadata):
|
||||
// Same, discard one.
|
||||
merged = append(merged, a[0])
|
||||
a = a[1:]
|
||||
b = b[1:]
|
||||
} else if a[0].name < b[0].name {
|
||||
case a[0].name < b[0].name:
|
||||
merged = append(merged, a[0])
|
||||
a = a[1:]
|
||||
} else {
|
||||
default:
|
||||
merged = append(merged, b[0])
|
||||
b = b[1:]
|
||||
}
|
||||
|
||||
@@ -28,7 +28,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
)
|
||||
|
||||
@@ -341,7 +340,6 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
|
||||
func applyBucketActions(ctx context.Context, o listPathOptions, in <-chan metaCacheEntry, out chan<- metaCacheEntry) {
|
||||
defer close(out)
|
||||
|
||||
vcfg, _ := globalBucketVersioningSys.Get(o.Bucket)
|
||||
for {
|
||||
var obj metaCacheEntry
|
||||
var ok bool
|
||||
@@ -354,37 +352,61 @@ func applyBucketActions(ctx context.Context, o listPathOptions, in <-chan metaCa
|
||||
}
|
||||
}
|
||||
|
||||
fi, err := obj.fileInfo(o.Bucket)
|
||||
var skip bool
|
||||
|
||||
versioned := o.Versioning != nil && o.Versioning.Versioned(obj.name)
|
||||
|
||||
// skip latest object from listing only for regular
|
||||
// listObjects calls, versioned based listing cannot
|
||||
// filter out between versions 'obj' cannot be truncated
|
||||
// in such a manner, so look for skipping an object only
|
||||
// for regular ListObjects() call only.
|
||||
if !o.Versioned {
|
||||
fi, err := obj.fileInfo(o.Bucket)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
objInfo := fi.ToObjectInfo(o.Bucket, obj.name, versioned)
|
||||
if o.Lifecycle != nil {
|
||||
act := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, objInfo).Action
|
||||
skip = act.Delete()
|
||||
if act.DeleteRestored() {
|
||||
// do not skip DeleteRestored* actions
|
||||
skip = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Skip entry only if needed via ILM, skipping is never true for versioned listing.
|
||||
if !skip {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case out <- obj:
|
||||
}
|
||||
}
|
||||
|
||||
fiv, err := obj.fileInfoVersions(o.Bucket)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
versioned := vcfg != nil && vcfg.Versioned(obj.name)
|
||||
// Expire all versions if needed, if not attempt to queue for replication.
|
||||
for _, version := range fiv.Versions {
|
||||
objInfo := version.ToObjectInfo(o.Bucket, obj.name, versioned)
|
||||
|
||||
objInfo := fi.ToObjectInfo(o.Bucket, obj.name, versioned)
|
||||
if o.Lifecycle != nil {
|
||||
evt := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, objInfo)
|
||||
switch evt.Action {
|
||||
case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
|
||||
globalExpiryState.enqueueByDays(objInfo, false, evt.Action == lifecycle.DeleteVersionAction)
|
||||
// Skip this entry.
|
||||
continue
|
||||
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
|
||||
globalExpiryState.enqueueByDays(objInfo, true, evt.Action == lifecycle.DeleteRestoredVersionAction)
|
||||
// Skip this entry.
|
||||
continue
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case out <- obj:
|
||||
if fiv, err := obj.fileInfoVersions(o.Bucket); err == nil {
|
||||
for _, version := range fiv.Versions {
|
||||
objInfo := version.ToObjectInfo(o.Bucket, obj.name, versioned)
|
||||
queueReplicationHeal(ctx, o.Bucket, objInfo, o.Replication)
|
||||
if o.Lifecycle != nil {
|
||||
evt := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, objInfo)
|
||||
if evt.Action.Delete() {
|
||||
globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
|
||||
if !evt.Action.DeleteRestored() {
|
||||
continue
|
||||
} // queue version for replication upon expired restored copies if needed.
|
||||
}
|
||||
}
|
||||
|
||||
queueReplicationHeal(ctx, o.Bucket, objInfo, o.Replication)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ import (
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
"github.com/minio/minio/internal/bucket/object/lock"
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
"github.com/minio/minio/internal/color"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
@@ -95,6 +96,10 @@ type listPathOptions struct {
|
||||
// Versioned is this a ListObjectVersions call.
|
||||
Versioned bool
|
||||
|
||||
// Versioning config is used for if the path
|
||||
// has versioning enabled.
|
||||
Versioning *versioning.Versioning
|
||||
|
||||
// Lifecycle performs filtering based on lifecycle.
|
||||
// This will filter out objects if the most recent version should be deleted by lifecycle.
|
||||
// Is not transferred across request calls.
|
||||
@@ -119,12 +124,14 @@ func init() {
|
||||
|
||||
func (o *listPathOptions) setBucketMeta(ctx context.Context) {
|
||||
lc, _ := globalLifecycleSys.Get(o.Bucket)
|
||||
vc, _ := globalBucketVersioningSys.Get(o.Bucket)
|
||||
|
||||
// Check if bucket is object locked.
|
||||
rcfg, _ := globalBucketObjectLockSys.Get(o.Bucket)
|
||||
replCfg, _, _ := globalBucketMetadataSys.GetReplicationConfig(ctx, o.Bucket)
|
||||
tgts, _ := globalBucketTargetSys.ListBucketTargets(ctx, o.Bucket)
|
||||
o.Lifecycle = lc
|
||||
o.Versioning = vc
|
||||
o.Replication = replicationConfig{
|
||||
Config: replCfg,
|
||||
remotes: tgts,
|
||||
@@ -870,8 +877,7 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
|
||||
var fdMu sync.Mutex
|
||||
fds := opts.fallbackDisks
|
||||
fallback := func(err error) StorageAPI {
|
||||
switch err.(type) {
|
||||
case StorageErr:
|
||||
if _, ok := err.(StorageErr); ok {
|
||||
// Attempt to grab a fallback disk
|
||||
fdMu.Lock()
|
||||
defer fdMu.Unlock()
|
||||
|
||||
@@ -562,8 +562,7 @@ func (r *metacacheReader) readAll(ctx context.Context, dst chan<- metaCacheEntry
|
||||
}
|
||||
for {
|
||||
if more, err := r.mr.ReadBool(); !more {
|
||||
switch err {
|
||||
case io.EOF:
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
r.err = err
|
||||
|
||||
+125
-10
@@ -131,6 +131,7 @@ const (
|
||||
iamSubsystem MetricSubsystem = "iam"
|
||||
kmsSubsystem MetricSubsystem = "kms"
|
||||
notifySubsystem MetricSubsystem = "notify"
|
||||
lambdaSubsystem MetricSubsystem = "lambda"
|
||||
auditSubsystem MetricSubsystem = "audit"
|
||||
)
|
||||
|
||||
@@ -182,8 +183,9 @@ const (
|
||||
usageInfo MetricName = "usage_info"
|
||||
versionInfo MetricName = "version_info"
|
||||
|
||||
sizeDistribution = "size_distribution"
|
||||
ttfbDistribution = "ttfb_seconds_distribution"
|
||||
sizeDistribution = "size_distribution"
|
||||
versionDistribution = "version_distribution"
|
||||
ttfbDistribution = "ttfb_seconds_distribution"
|
||||
|
||||
lastActivityTime = "last_activity_nano_seconds"
|
||||
startTime = "starttime_seconds"
|
||||
@@ -564,6 +566,16 @@ func getBucketObjectDistributionMD() MetricDescription {
|
||||
}
|
||||
}
|
||||
|
||||
func getBucketObjectVersionsMD() MetricDescription {
|
||||
return MetricDescription{
|
||||
Namespace: bucketMetricNamespace,
|
||||
Subsystem: objectsSubsystem,
|
||||
Name: versionDistribution,
|
||||
Help: "Distribution of object sizes in the bucket, includes label for the bucket name",
|
||||
Type: histogramMetric,
|
||||
}
|
||||
}
|
||||
|
||||
func getInternodeFailedRequests() MetricDescription {
|
||||
return MetricDescription{
|
||||
Namespace: interNodeMetricNamespace,
|
||||
@@ -1385,6 +1397,7 @@ func getIAMNodeMetrics() *MetricsGroup {
|
||||
sinceLastSyncMillis = (uint64(time.Now().UnixNano()) - lastSyncTime) / uint64(time.Millisecond)
|
||||
}
|
||||
|
||||
pluginAuthNMetrics := globalAuthNPlugin.Metrics()
|
||||
metrics = []Metric{
|
||||
{
|
||||
Description: MetricDescription{
|
||||
@@ -1426,7 +1439,68 @@ func getIAMNodeMetrics() *MetricsGroup {
|
||||
},
|
||||
Value: float64(atomic.LoadUint64(&globalIAMSys.TotalRefreshFailures)),
|
||||
},
|
||||
{
|
||||
Description: MetricDescription{
|
||||
Namespace: nodeMetricNamespace,
|
||||
Subsystem: iamSubsystem,
|
||||
Name: "plugin_authn_service_last_succ_seconds",
|
||||
Help: "When plugin authentication is configured, returns time (in seconds) since the last successful request to the service",
|
||||
Type: gaugeMetric,
|
||||
},
|
||||
Value: pluginAuthNMetrics.LastReachableSecs,
|
||||
},
|
||||
{
|
||||
Description: MetricDescription{
|
||||
Namespace: nodeMetricNamespace,
|
||||
Subsystem: iamSubsystem,
|
||||
Name: "plugin_authn_service_last_fail_seconds",
|
||||
Help: "When plugin authentication is configured, returns time (in seconds) since the last failed request to the service",
|
||||
Type: gaugeMetric,
|
||||
},
|
||||
Value: pluginAuthNMetrics.LastUnreachableSecs,
|
||||
},
|
||||
{
|
||||
Description: MetricDescription{
|
||||
Namespace: nodeMetricNamespace,
|
||||
Subsystem: iamSubsystem,
|
||||
Name: "plugin_authn_service_total_requests_minute",
|
||||
Help: "When plugin authentication is configured, returns total requests count in the last full minute",
|
||||
Type: gaugeMetric,
|
||||
},
|
||||
Value: float64(pluginAuthNMetrics.TotalRequests),
|
||||
},
|
||||
{
|
||||
Description: MetricDescription{
|
||||
Namespace: nodeMetricNamespace,
|
||||
Subsystem: iamSubsystem,
|
||||
Name: "plugin_authn_service_failed_requests_minute",
|
||||
Help: "When plugin authentication is configured, returns failed requests count in the last full minute",
|
||||
Type: gaugeMetric,
|
||||
},
|
||||
Value: float64(pluginAuthNMetrics.FailedRequests),
|
||||
},
|
||||
{
|
||||
Description: MetricDescription{
|
||||
Namespace: nodeMetricNamespace,
|
||||
Subsystem: iamSubsystem,
|
||||
Name: "plugin_authn_service_succ_avg_rtt_ms_minute",
|
||||
Help: "When plugin authentication is configured, returns average round-trip-time of successful requests in the last full minute",
|
||||
Type: gaugeMetric,
|
||||
},
|
||||
Value: pluginAuthNMetrics.AvgSuccRTTMs,
|
||||
},
|
||||
{
|
||||
Description: MetricDescription{
|
||||
Namespace: nodeMetricNamespace,
|
||||
Subsystem: iamSubsystem,
|
||||
Name: "plugin_authn_service_succ_max_rtt_ms_minute",
|
||||
Help: "When plugin authentication is configured, returns maximum round-trip-time of successful requests in the last full minute",
|
||||
Type: gaugeMetric,
|
||||
},
|
||||
Value: pluginAuthNMetrics.MaxSuccRTTMs,
|
||||
},
|
||||
}
|
||||
|
||||
return metrics
|
||||
})
|
||||
return mg
|
||||
@@ -1594,8 +1668,8 @@ func getNotificationMetrics() *MetricsGroup {
|
||||
cacheInterval: 10 * time.Second,
|
||||
}
|
||||
mg.RegisterRead(func(ctx context.Context) []Metric {
|
||||
stats := globalConfigTargetList.Stats()
|
||||
metrics := make([]Metric, 0, 1+len(stats.TargetStats))
|
||||
nstats := globalNotifyTargetList.Stats()
|
||||
metrics := make([]Metric, 0, 1+len(nstats.TargetStats))
|
||||
metrics = append(metrics, Metric{
|
||||
Description: MetricDescription{
|
||||
Namespace: minioNamespace,
|
||||
@@ -1604,9 +1678,9 @@ func getNotificationMetrics() *MetricsGroup {
|
||||
Help: "Number of concurrent async Send calls active to all targets",
|
||||
Type: gaugeMetric,
|
||||
},
|
||||
Value: float64(stats.CurrentSendCalls),
|
||||
Value: float64(nstats.CurrentSendCalls),
|
||||
})
|
||||
for _, st := range stats.TargetStats {
|
||||
for _, st := range nstats.TargetStats {
|
||||
metrics = append(metrics, Metric{
|
||||
Description: MetricDescription{
|
||||
Namespace: minioNamespace,
|
||||
@@ -1619,6 +1693,43 @@ func getNotificationMetrics() *MetricsGroup {
|
||||
Value: float64(st.CurrentQueue),
|
||||
})
|
||||
}
|
||||
|
||||
lstats := globalLambdaTargetList.Stats()
|
||||
for _, st := range lstats.TargetStats {
|
||||
metrics = append(metrics, Metric{
|
||||
Description: MetricDescription{
|
||||
Namespace: minioNamespace,
|
||||
Subsystem: lambdaSubsystem,
|
||||
Name: "active_requests",
|
||||
Help: "Number of in progress requests",
|
||||
},
|
||||
VariableLabels: map[string]string{"target_id": st.ID.ID, "target_name": st.ID.Name},
|
||||
Value: float64(st.ActiveRequests),
|
||||
})
|
||||
metrics = append(metrics, Metric{
|
||||
Description: MetricDescription{
|
||||
Namespace: minioNamespace,
|
||||
Subsystem: lambdaSubsystem,
|
||||
Name: "total_requests",
|
||||
Help: "Total number of requests sent since start",
|
||||
Type: counterMetric,
|
||||
},
|
||||
VariableLabels: map[string]string{"target_id": st.ID.ID, "target_name": st.ID.Name},
|
||||
Value: float64(st.TotalRequests),
|
||||
})
|
||||
metrics = append(metrics, Metric{
|
||||
Description: MetricDescription{
|
||||
Namespace: minioNamespace,
|
||||
Subsystem: lambdaSubsystem,
|
||||
Name: "failed_requests",
|
||||
Help: "Total number of requests that failed to send since start",
|
||||
Type: counterMetric,
|
||||
},
|
||||
VariableLabels: map[string]string{"target_id": st.ID.ID, "target_name": st.ID.Name},
|
||||
Value: float64(st.FailedRequests),
|
||||
})
|
||||
}
|
||||
|
||||
// Audit and system:
|
||||
audit := logger.CurrentStats()
|
||||
for id, st := range audit {
|
||||
@@ -1899,6 +2010,12 @@ func getBucketUsageMetrics() *MetricsGroup {
|
||||
HistogramBucketLabel: "range",
|
||||
VariableLabels: map[string]string{"bucket": bucket},
|
||||
})
|
||||
metrics = append(metrics, Metric{
|
||||
Description: getBucketObjectVersionsMD(),
|
||||
Histogram: usage.ObjectVersionsHistogram,
|
||||
HistogramBucketLabel: "range",
|
||||
VariableLabels: map[string]string{"bucket": bucket},
|
||||
})
|
||||
}
|
||||
return
|
||||
})
|
||||
@@ -2260,8 +2377,7 @@ func (c *minioClusterCollector) Collect(out chan<- prometheus.Metric) {
|
||||
continue
|
||||
}
|
||||
metricType := prometheus.GaugeValue
|
||||
switch metric.Description.Type {
|
||||
case counterMetric:
|
||||
if metric.Description.Type == counterMetric {
|
||||
metricType = prometheus.CounterValue
|
||||
}
|
||||
toPost := prometheus.MustNewConstMetric(
|
||||
@@ -2369,8 +2485,7 @@ func (c *minioNodeCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
}
|
||||
|
||||
metricType := prometheus.GaugeValue
|
||||
switch metric.Description.Type {
|
||||
case counterMetric:
|
||||
if metric.Description.Type == counterMetric {
|
||||
metricType = prometheus.CounterValue
|
||||
}
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
|
||||
@@ -441,6 +441,18 @@ func bucketUsageMetricsPrometheus(ch chan<- prometheus.Metric) {
|
||||
k,
|
||||
)
|
||||
}
|
||||
for k, v := range usageInfo.ObjectVersionsHistogram {
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName(bucketNamespace, "objects", "histogram"),
|
||||
"Total number of versions of objects in a bucket",
|
||||
[]string{"bucket", "object_versions"}, nil),
|
||||
prometheus.GaugeValue,
|
||||
float64(v),
|
||||
bucket,
|
||||
k,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+6
-10
@@ -170,9 +170,8 @@ func (di *distLockInstance) GetLock(ctx context.Context, timeout *dynamicTimeout
|
||||
RetryInterval: timeout.RetryInterval(),
|
||||
}) {
|
||||
timeout.LogFailure()
|
||||
cancel()
|
||||
switch err := newCtx.Err(); err {
|
||||
case context.Canceled:
|
||||
defer cancel()
|
||||
if err := newCtx.Err(); err == context.Canceled {
|
||||
return LockContext{ctx: ctx, cancel: func() {}}, err
|
||||
}
|
||||
return LockContext{ctx: ctx, cancel: func() {}}, OperationTimedOut{}
|
||||
@@ -200,9 +199,8 @@ func (di *distLockInstance) GetRLock(ctx context.Context, timeout *dynamicTimeou
|
||||
RetryInterval: timeout.RetryInterval(),
|
||||
}) {
|
||||
timeout.LogFailure()
|
||||
cancel()
|
||||
switch err := newCtx.Err(); err {
|
||||
case context.Canceled:
|
||||
defer cancel()
|
||||
if err := newCtx.Err(); err == context.Canceled {
|
||||
return LockContext{ctx: ctx, cancel: func() {}}, err
|
||||
}
|
||||
return LockContext{ctx: ctx, cancel: func() {}}, OperationTimedOut{}
|
||||
@@ -257,8 +255,7 @@ func (li *localLockInstance) GetLock(ctx context.Context, timeout *dynamicTimeou
|
||||
li.ns.unlock(li.volume, li.paths[si], readLock)
|
||||
}
|
||||
}
|
||||
switch err := ctx.Err(); err {
|
||||
case context.Canceled:
|
||||
if err := ctx.Err(); err == context.Canceled {
|
||||
return LockContext{}, err
|
||||
}
|
||||
return LockContext{}, OperationTimedOut{}
|
||||
@@ -294,8 +291,7 @@ func (li *localLockInstance) GetRLock(ctx context.Context, timeout *dynamicTimeo
|
||||
li.ns.unlock(li.volume, li.paths[si], readLock)
|
||||
}
|
||||
}
|
||||
switch err := ctx.Err(); err {
|
||||
case context.Canceled:
|
||||
if err := ctx.Err(); err == context.Canceled {
|
||||
return LockContext{}, err
|
||||
}
|
||||
return LockContext{}, OperationTimedOut{}
|
||||
|
||||
+12
-9
@@ -138,13 +138,14 @@ func TestGetHostIP(t *testing.T) {
|
||||
|
||||
for _, testCase := range testCases {
|
||||
ipList, err := getHostIP(testCase.host)
|
||||
if testCase.expectedErr == nil {
|
||||
switch {
|
||||
case testCase.expectedErr == nil:
|
||||
if err != nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
}
|
||||
} else if err == nil {
|
||||
case err == nil:
|
||||
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
||||
} else if testCase.expectedErr.Error() != err.Error() {
|
||||
case testCase.expectedErr.Error() != err.Error():
|
||||
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
||||
}
|
||||
|
||||
@@ -221,13 +222,14 @@ func TestCheckPortAvailability(t *testing.T) {
|
||||
}
|
||||
|
||||
err := checkPortAvailability(testCase.host, testCase.port)
|
||||
if testCase.expectedErr == nil {
|
||||
switch {
|
||||
case testCase.expectedErr == nil:
|
||||
if err != nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
}
|
||||
} else if err == nil {
|
||||
case err == nil:
|
||||
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
||||
} else if testCase.expectedErr.Error() != err.Error() {
|
||||
case testCase.expectedErr.Error() != err.Error():
|
||||
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
||||
}
|
||||
}
|
||||
@@ -252,13 +254,14 @@ func TestCheckLocalServerAddr(t *testing.T) {
|
||||
testCase := testCase
|
||||
t.Run("", func(t *testing.T) {
|
||||
err := CheckLocalServerAddr(testCase.serverAddr)
|
||||
if testCase.expectedErr == nil {
|
||||
switch {
|
||||
case testCase.expectedErr == nil:
|
||||
if err != nil {
|
||||
t.Errorf("error: expected = <nil>, got = %v", err)
|
||||
}
|
||||
} else if err == nil {
|
||||
case err == nil:
|
||||
t.Errorf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
||||
} else if testCase.expectedErr.Error() != err.Error() {
|
||||
case testCase.expectedErr.Error() != err.Error():
|
||||
t.Errorf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
||||
}
|
||||
})
|
||||
|
||||
+26
-89
@@ -18,9 +18,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -29,7 +27,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/bits-and-blooms/bloom/v3"
|
||||
"github.com/cespare/xxhash/v2"
|
||||
"github.com/klauspost/compress/zip"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
@@ -59,14 +56,23 @@ type NotificationPeerErr struct {
|
||||
//
|
||||
// A zero NotificationGroup is valid and does not cancel on error.
|
||||
type NotificationGroup struct {
|
||||
wg sync.WaitGroup
|
||||
errs []NotificationPeerErr
|
||||
wg sync.WaitGroup
|
||||
errs []NotificationPeerErr
|
||||
retryCount int
|
||||
}
|
||||
|
||||
// WithNPeers returns a new NotificationGroup with length of errs slice upto nerrs,
|
||||
// upon Wait() errors are returned collected from all tasks.
|
||||
func WithNPeers(nerrs int) *NotificationGroup {
|
||||
return &NotificationGroup{errs: make([]NotificationPeerErr, nerrs)}
|
||||
return &NotificationGroup{errs: make([]NotificationPeerErr, nerrs), retryCount: 3}
|
||||
}
|
||||
|
||||
// WithRetries sets the retry count for all function calls from the Go method.
|
||||
func (g *NotificationGroup) WithRetries(retryCount int) *NotificationGroup {
|
||||
if g != nil {
|
||||
g.retryCount = retryCount
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
// Wait blocks until all function calls from the Go method have returned, then
|
||||
@@ -88,17 +94,17 @@ func (g *NotificationGroup) Go(ctx context.Context, f func() error, index int, a
|
||||
g.errs[index] = NotificationPeerErr{
|
||||
Host: addr,
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
for i := 0; i < g.retryCount; i++ {
|
||||
if err := f(); err != nil {
|
||||
g.errs[index].Err = err
|
||||
// Last iteration log the error.
|
||||
if i == 2 {
|
||||
if i == g.retryCount-1 {
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
|
||||
ctx := logger.SetReqInfo(ctx, reqInfo)
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
// Wait for one second and no need wait after last attempt.
|
||||
if i < 2 {
|
||||
if i < g.retryCount-1 {
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
continue
|
||||
@@ -110,7 +116,7 @@ func (g *NotificationGroup) Go(ctx context.Context, f func() error, index int, a
|
||||
|
||||
// DeletePolicy - deletes policy across all peers.
|
||||
func (sys *NotificationSys) DeletePolicy(policyName string) []NotificationPeerErr {
|
||||
ng := WithNPeers(len(sys.peerClients))
|
||||
ng := WithNPeers(len(sys.peerClients)).WithRetries(1)
|
||||
for idx, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
@@ -125,7 +131,7 @@ func (sys *NotificationSys) DeletePolicy(policyName string) []NotificationPeerEr
|
||||
|
||||
// LoadPolicy - reloads a specific modified policy across all peers
|
||||
func (sys *NotificationSys) LoadPolicy(policyName string) []NotificationPeerErr {
|
||||
ng := WithNPeers(len(sys.peerClients))
|
||||
ng := WithNPeers(len(sys.peerClients)).WithRetries(1)
|
||||
for idx, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
@@ -140,7 +146,7 @@ func (sys *NotificationSys) LoadPolicy(policyName string) []NotificationPeerErr
|
||||
|
||||
// LoadPolicyMapping - reloads a policy mapping across all peers
|
||||
func (sys *NotificationSys) LoadPolicyMapping(userOrGroup string, userType IAMUserType, isGroup bool) []NotificationPeerErr {
|
||||
ng := WithNPeers(len(sys.peerClients))
|
||||
ng := WithNPeers(len(sys.peerClients)).WithRetries(1)
|
||||
for idx, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
@@ -155,7 +161,7 @@ func (sys *NotificationSys) LoadPolicyMapping(userOrGroup string, userType IAMUs
|
||||
|
||||
// DeleteUser - deletes a specific user across all peers
|
||||
func (sys *NotificationSys) DeleteUser(accessKey string) []NotificationPeerErr {
|
||||
ng := WithNPeers(len(sys.peerClients))
|
||||
ng := WithNPeers(len(sys.peerClients)).WithRetries(1)
|
||||
for idx, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
@@ -170,7 +176,7 @@ func (sys *NotificationSys) DeleteUser(accessKey string) []NotificationPeerErr {
|
||||
|
||||
// LoadUser - reloads a specific user across all peers
|
||||
func (sys *NotificationSys) LoadUser(accessKey string, temp bool) []NotificationPeerErr {
|
||||
ng := WithNPeers(len(sys.peerClients))
|
||||
ng := WithNPeers(len(sys.peerClients)).WithRetries(1)
|
||||
for idx, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
@@ -185,7 +191,7 @@ func (sys *NotificationSys) LoadUser(accessKey string, temp bool) []Notification
|
||||
|
||||
// LoadGroup - loads a specific group on all peers.
|
||||
func (sys *NotificationSys) LoadGroup(group string) []NotificationPeerErr {
|
||||
ng := WithNPeers(len(sys.peerClients))
|
||||
ng := WithNPeers(len(sys.peerClients)).WithRetries(1)
|
||||
for idx, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
@@ -198,7 +204,7 @@ func (sys *NotificationSys) LoadGroup(group string) []NotificationPeerErr {
|
||||
|
||||
// DeleteServiceAccount - deletes a specific service account across all peers
|
||||
func (sys *NotificationSys) DeleteServiceAccount(accessKey string) []NotificationPeerErr {
|
||||
ng := WithNPeers(len(sys.peerClients))
|
||||
ng := WithNPeers(len(sys.peerClients)).WithRetries(1)
|
||||
for idx, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
@@ -213,7 +219,7 @@ func (sys *NotificationSys) DeleteServiceAccount(accessKey string) []Notificatio
|
||||
|
||||
// LoadServiceAccount - reloads a specific service account across all peers
|
||||
func (sys *NotificationSys) LoadServiceAccount(accessKey string) []NotificationPeerErr {
|
||||
ng := WithNPeers(len(sys.peerClients))
|
||||
ng := WithNPeers(len(sys.peerClients)).WithRetries(1)
|
||||
for idx, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
@@ -384,75 +390,6 @@ func (sys *NotificationSys) SignalService(sig serviceSignal) []NotificationPeerE
|
||||
return ng.Wait()
|
||||
}
|
||||
|
||||
// updateBloomFilter will cycle all servers to the current index and
|
||||
// return a merged bloom filter if a complete one can be retrieved.
|
||||
func (sys *NotificationSys) updateBloomFilter(ctx context.Context, current uint64) (*bloomFilter, error) {
|
||||
req := bloomFilterRequest{
|
||||
Current: current,
|
||||
Oldest: current - dataUsageUpdateDirCycles,
|
||||
}
|
||||
if current < dataUsageUpdateDirCycles {
|
||||
req.Oldest = 0
|
||||
}
|
||||
|
||||
// Load initial state from local...
|
||||
var bf *bloomFilter
|
||||
bfr, err := intDataUpdateTracker.cycleFilter(ctx, req)
|
||||
logger.LogIf(ctx, err)
|
||||
if err == nil && bfr.Complete {
|
||||
nbf := intDataUpdateTracker.newBloomFilter()
|
||||
bf = &nbf
|
||||
_, err = bf.ReadFrom(bytes.NewReader(bfr.Filter))
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
g := errgroup.WithNErrs(len(sys.peerClients))
|
||||
for idx, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
}
|
||||
client := client
|
||||
g.Go(func() error {
|
||||
serverBF, err := client.cycleServerBloomFilter(ctx, req)
|
||||
if false && intDataUpdateTracker.debug {
|
||||
b, _ := json.MarshalIndent(serverBF, "", " ")
|
||||
logger.Info("Drive %v, Bloom filter: %v", client.host.Name, string(b))
|
||||
}
|
||||
// Keep lock while checking result.
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
if err != nil || !serverBF.Complete || bf == nil {
|
||||
logger.LogOnceIf(ctx, err, client.host.String(), client.cycleServerBloomFilter)
|
||||
bf = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
var tmp bloom.BloomFilter
|
||||
_, err = tmp.ReadFrom(bytes.NewReader(serverBF.Filter))
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
bf = nil
|
||||
return nil
|
||||
}
|
||||
if bf.BloomFilter == nil {
|
||||
bf.BloomFilter = &tmp
|
||||
} else {
|
||||
err = bf.Merge(&tmp)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
bf = nil
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, idx)
|
||||
}
|
||||
g.Wait()
|
||||
return bf, nil
|
||||
}
|
||||
|
||||
var errPeerNotReachable = errors.New("peer is not reachable")
|
||||
|
||||
// GetLocks - makes GetLocks RPC call on all peers.
|
||||
@@ -540,7 +477,7 @@ func (sys *NotificationSys) DeleteBucketMetadata(ctx context.Context, bucketName
|
||||
|
||||
// GetClusterAllBucketStats - returns bucket stats for all buckets from all remote peers.
|
||||
func (sys *NotificationSys) GetClusterAllBucketStats(ctx context.Context) []BucketStatsMap {
|
||||
ng := WithNPeers(len(sys.peerClients))
|
||||
ng := WithNPeers(len(sys.peerClients)).WithRetries(1)
|
||||
replicationStats := make([]BucketStatsMap, len(sys.peerClients))
|
||||
for index, client := range sys.peerClients {
|
||||
index := index
|
||||
@@ -581,7 +518,7 @@ func (sys *NotificationSys) GetClusterAllBucketStats(ctx context.Context) []Buck
|
||||
|
||||
// GetClusterBucketStats - calls GetClusterBucketStats call on all peers for a cluster statistics view.
|
||||
func (sys *NotificationSys) GetClusterBucketStats(ctx context.Context, bucketName string) []BucketStats {
|
||||
ng := WithNPeers(len(sys.peerClients))
|
||||
ng := WithNPeers(len(sys.peerClients)).WithRetries(1)
|
||||
bucketStats := make([]BucketStats, len(sys.peerClients))
|
||||
for index, client := range sys.peerClients {
|
||||
index := index
|
||||
@@ -1440,7 +1377,7 @@ func (sys *NotificationSys) GetLastDayTierStats(ctx context.Context) DailyAllTie
|
||||
merged := globalTransitionState.getDailyAllTierStats()
|
||||
for i, stat := range lastDayStats {
|
||||
if errs[i] != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("failed to fetch last day tier stats: %w", errs[i]))
|
||||
logger.LogOnceIf(ctx, fmt.Errorf("failed to fetch last day tier stats: %w", errs[i]), sys.peerClients[i].host.String())
|
||||
continue
|
||||
}
|
||||
merged.merge(stat)
|
||||
|
||||
@@ -54,12 +54,13 @@ type objectHistogramInterval struct {
|
||||
|
||||
const (
|
||||
// dataUsageBucketLen must be length of ObjectsHistogramIntervals
|
||||
dataUsageBucketLen = 7
|
||||
dataUsageBucketLen = 7
|
||||
dataUsageVersionLen = 7
|
||||
)
|
||||
|
||||
// ObjectsHistogramIntervals is the list of all intervals
|
||||
// of object sizes to be included in objects histogram.
|
||||
var ObjectsHistogramIntervals = []objectHistogramInterval{
|
||||
var ObjectsHistogramIntervals = [dataUsageBucketLen]objectHistogramInterval{
|
||||
{"LESS_THAN_1024_B", 0, humanize.KiByte - 1},
|
||||
{"BETWEEN_1024_B_AND_1_MB", humanize.KiByte, humanize.MiByte - 1},
|
||||
{"BETWEEN_1_MB_AND_10_MB", humanize.MiByte, humanize.MiByte*10 - 1},
|
||||
@@ -69,6 +70,18 @@ var ObjectsHistogramIntervals = []objectHistogramInterval{
|
||||
{"GREATER_THAN_512_MB", humanize.MiByte * 512, math.MaxInt64},
|
||||
}
|
||||
|
||||
// ObjectsVersionCountIntervals is the list of all intervals
|
||||
// of object version count to be included in objects histogram.
|
||||
var ObjectsVersionCountIntervals = [dataUsageVersionLen]objectHistogramInterval{
|
||||
{"UNVERSIONED", 0, 0},
|
||||
{"SINGLE_VERSION", 1, 1},
|
||||
{"BETWEEN_2_AND_10", 2, 9},
|
||||
{"BETWEEN_10_AND_100", 10, 99},
|
||||
{"BETWEEN_100_AND_1000", 100, 999},
|
||||
{"BETWEEN_1000_AND_10000", 1000, 9999},
|
||||
{"GREATER_THAN_10000", 10000, math.MaxInt64},
|
||||
}
|
||||
|
||||
// BucketInfo - represents bucket metadata.
|
||||
type BucketInfo struct {
|
||||
// Name of the bucket.
|
||||
|
||||
@@ -195,7 +195,7 @@ type ObjectLayer interface {
|
||||
|
||||
// Storage operations.
|
||||
Shutdown(context.Context) error
|
||||
NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32, scanMode madmin.HealScanMode) error
|
||||
NSScanner(ctx context.Context, updates chan<- DataUsageInfo, wantCycle uint32, scanMode madmin.HealScanMode) error
|
||||
BackendInfo() madmin.BackendInfo
|
||||
StorageInfo(ctx context.Context) StorageInfo
|
||||
LocalStorageInfo(ctx context.Context) StorageInfo
|
||||
|
||||
+21
-30
@@ -472,22 +472,18 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
|
||||
}
|
||||
|
||||
if !proxy.Proxy { // apply lifecycle rules only for local requests
|
||||
// Automatically remove the object/version is an expiry lifecycle rule can be applied
|
||||
// Automatically remove the object/version if an expiry lifecycle rule can be applied
|
||||
if lc, err := globalLifecycleSys.Get(bucket); err == nil {
|
||||
rcfg, _ := globalBucketObjectLockSys.Get(bucket)
|
||||
evt := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo)
|
||||
var success bool
|
||||
switch evt.Action {
|
||||
case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
|
||||
success = applyExpiryRule(objInfo, false, evt.Action == lifecycle.DeleteVersionAction)
|
||||
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
|
||||
// Restored object delete would be still allowed to proceed as success
|
||||
// since transition behavior is slightly different.
|
||||
applyExpiryRule(objInfo, true, evt.Action == lifecycle.DeleteRestoredVersionAction)
|
||||
}
|
||||
if success {
|
||||
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey))
|
||||
return
|
||||
act := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo).Action
|
||||
if act.Delete() {
|
||||
// apply whatever the expiry rule is.
|
||||
applyExpiryRule(objInfo, act.DeleteRestored(), act.DeleteVersioned())
|
||||
if !act.DeleteRestored() {
|
||||
// If the ILM action is not on restored object return error.
|
||||
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey))
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -729,22 +725,18 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
|
||||
}
|
||||
|
||||
if !proxy.Proxy { // apply lifecycle rules only locally not for proxied requests
|
||||
// Automatically remove the object/version is an expiry lifecycle rule can be applied
|
||||
// Automatically remove the object/version if an expiry lifecycle rule can be applied
|
||||
if lc, err := globalLifecycleSys.Get(bucket); err == nil {
|
||||
rcfg, _ := globalBucketObjectLockSys.Get(bucket)
|
||||
evt := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo)
|
||||
var success bool
|
||||
switch evt.Action {
|
||||
case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
|
||||
success = applyExpiryRule(objInfo, false, evt.Action == lifecycle.DeleteVersionAction)
|
||||
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
|
||||
// Restored object delete would be still allowed to proceed as success
|
||||
// since transition behavior is slightly different.
|
||||
applyExpiryRule(objInfo, true, evt.Action == lifecycle.DeleteRestoredVersionAction)
|
||||
}
|
||||
if success {
|
||||
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey))
|
||||
return
|
||||
act := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo).Action
|
||||
if act.Delete() {
|
||||
// apply whatever the expiry rule is.
|
||||
applyExpiryRule(objInfo, act.DeleteRestored(), act.DeleteVersioned())
|
||||
if !act.DeleteRestored() {
|
||||
// If the ILM action is not on restored object return error.
|
||||
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey))
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
QueueReplicationHeal(ctx, bucket, objInfo)
|
||||
@@ -2368,8 +2360,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html
|
||||
objInfo, err := deleteObject(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
switch err.(type) {
|
||||
case BucketNotFound:
|
||||
if _, ok := err.(BucketNotFound); ok {
|
||||
// When bucket doesn't exist specially handle it.
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
|
||||
+13
-15
@@ -1197,17 +1197,18 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam
|
||||
rec := httptest.NewRecorder()
|
||||
// construct HTTP request for Put Object end point.
|
||||
var req *http.Request
|
||||
if testCase.fault == chunkDateMismatch {
|
||||
switch {
|
||||
case testCase.fault == chunkDateMismatch:
|
||||
req, err = newTestStreamingSignedBadChunkDateRequest(http.MethodPut,
|
||||
getPutObjectURL("", testCase.bucketName, testCase.objectName),
|
||||
int64(testCase.dataLen), testCase.chunkSize, bytes.NewReader(testCase.data),
|
||||
testCase.accessKey, testCase.secretKey)
|
||||
} else if testCase.contentEncoding == "" {
|
||||
case testCase.contentEncoding == "":
|
||||
req, err = newTestStreamingSignedRequest(http.MethodPut,
|
||||
getPutObjectURL("", testCase.bucketName, testCase.objectName),
|
||||
int64(testCase.dataLen), testCase.chunkSize, bytes.NewReader(testCase.data),
|
||||
testCase.accessKey, testCase.secretKey)
|
||||
} else if testCase.contentEncoding != "" {
|
||||
case testCase.contentEncoding != "":
|
||||
req, err = newTestStreamingSignedCustomEncodingRequest(http.MethodPut,
|
||||
getPutObjectURL("", testCase.bucketName, testCase.objectName),
|
||||
int64(testCase.dataLen), testCase.chunkSize, bytes.NewReader(testCase.data),
|
||||
@@ -2045,12 +2046,13 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
|
||||
var req *http.Request
|
||||
// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
|
||||
rec := httptest.NewRecorder()
|
||||
if !testCase.invalidPartNumber || !testCase.maximumPartNumber {
|
||||
switch {
|
||||
case !testCase.invalidPartNumber || !testCase.maximumPartNumber:
|
||||
// construct HTTP request for copy object.
|
||||
req, err = newTestSignedRequestV4(http.MethodPut, getCopyObjectPartURL("", testCase.bucketName, testObject, testCase.uploadID, "1"), 0, nil, testCase.accessKey, testCase.secretKey, nil)
|
||||
} else if testCase.invalidPartNumber {
|
||||
case testCase.invalidPartNumber:
|
||||
req, err = newTestSignedRequestV4(http.MethodPut, getCopyObjectPartURL("", testCase.bucketName, testObject, testCase.uploadID, "abc"), 0, nil, testCase.accessKey, testCase.secretKey, nil)
|
||||
} else if testCase.maximumPartNumber {
|
||||
case testCase.maximumPartNumber:
|
||||
req, err = newTestSignedRequestV4(http.MethodPut, getCopyObjectPartURL("", testCase.bucketName, testObject, testCase.uploadID, "99999"), 0, nil, testCase.accessKey, testCase.secretKey, nil)
|
||||
}
|
||||
if err != nil {
|
||||
@@ -3545,11 +3547,9 @@ func testAPIPutObjectPartHandlerStreaming(obj ObjectLayer, instanceType, bucketN
|
||||
t.Errorf("Test %d %s expected to fail with error %s, but received %s", i+1, instanceType,
|
||||
test.expectedErr.Code, errXML.Code)
|
||||
}
|
||||
} else {
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Errorf("Test %d %s expected to succeed, but failed with HTTP status code %d",
|
||||
i+1, instanceType, rec.Code)
|
||||
}
|
||||
} else if rec.Code != http.StatusOK {
|
||||
t.Errorf("Test %d %s expected to succeed, but failed with HTTP status code %d",
|
||||
i+1, instanceType, rec.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4119,10 +4119,8 @@ func testAPIListObjectPartsHandler(obj ObjectLayer, instanceType, bucketName str
|
||||
reqType, test.expectedErr.Code, errXML.Code)
|
||||
}
|
||||
// in case error is not expected response status should be 200OK.
|
||||
} else {
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Errorf("%s, Expected to succeed with response HTTP status 200OK, but failed with HTTP status code %d.", reqType, rec.Code)
|
||||
}
|
||||
} else if rec.Code != http.StatusOK {
|
||||
t.Errorf("%s, Expected to succeed with response HTTP status 200OK, but failed with HTTP status code %d.", reqType, rec.Code)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
@@ -0,0 +1,291 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"crypto/subtle"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/gzhttp"
|
||||
"github.com/lithammer/shortuuid/v4"
|
||||
miniogo "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
levent "github.com/minio/minio/internal/config/lambda/event"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
)
|
||||
|
||||
func getLambdaEventData(bucket, object string, cred auth.Credentials, r *http.Request) (levent.Event, error) {
|
||||
host := globalLocalNodeName
|
||||
secure := globalIsTLS
|
||||
if globalMinioEndpointURL != nil {
|
||||
host = globalMinioEndpointURL.Host
|
||||
secure = globalMinioEndpointURL.Scheme == "https"
|
||||
}
|
||||
|
||||
duration := time.Since(cred.Expiration)
|
||||
if cred.Expiration.IsZero() {
|
||||
duration = time.Hour
|
||||
}
|
||||
|
||||
clnt, err := miniogo.New(host, &miniogo.Options{
|
||||
Creds: credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
|
||||
Secure: secure,
|
||||
Transport: globalRemoteTargetTransport,
|
||||
Region: globalSite.Region,
|
||||
})
|
||||
if err != nil {
|
||||
return levent.Event{}, err
|
||||
}
|
||||
|
||||
reqParams := url.Values{}
|
||||
if partNumberStr := r.Form.Get("partNumber"); partNumberStr != "" {
|
||||
reqParams.Set("partNumber", partNumberStr)
|
||||
}
|
||||
for k := range supportedHeadGetReqParams {
|
||||
if v := r.Form.Get(k); v != "" {
|
||||
reqParams.Set(k, v)
|
||||
}
|
||||
}
|
||||
extraHeaders := http.Header{}
|
||||
if rng := r.Header.Get(xhttp.Range); rng != "" {
|
||||
extraHeaders.Set(xhttp.Range, r.Header.Get(xhttp.Range))
|
||||
}
|
||||
|
||||
u, err := clnt.PresignHeader(r.Context(), http.MethodGet, bucket, object, duration, reqParams, extraHeaders)
|
||||
if err != nil {
|
||||
return levent.Event{}, err
|
||||
}
|
||||
|
||||
token, err := authenticateNode(cred.AccessKey, cred.SecretKey, u.RawQuery)
|
||||
if err != nil {
|
||||
return levent.Event{}, err
|
||||
}
|
||||
|
||||
eventData := levent.Event{
|
||||
GetObjectContext: &levent.GetObjectContext{
|
||||
InputS3URL: u.String(),
|
||||
OutputRoute: shortuuid.New(),
|
||||
OutputToken: token,
|
||||
},
|
||||
UserRequest: levent.UserRequest{
|
||||
URL: r.URL.String(),
|
||||
Headers: r.Header.Clone(),
|
||||
},
|
||||
UserIdentity: levent.Identity{
|
||||
Type: "IAMUser",
|
||||
PrincipalID: cred.AccessKey,
|
||||
AccessKeyID: cred.SecretKey,
|
||||
},
|
||||
}
|
||||
return eventData, nil
|
||||
}
|
||||
|
||||
var statusTextToCode = map[string]int{
|
||||
"Continue": http.StatusContinue,
|
||||
"Switching Protocols": http.StatusSwitchingProtocols,
|
||||
"Processing": http.StatusProcessing,
|
||||
"Early Hints": http.StatusEarlyHints,
|
||||
"OK": http.StatusOK,
|
||||
"Created": http.StatusCreated,
|
||||
"Accepted": http.StatusAccepted,
|
||||
"Non-Authoritative Information": http.StatusNonAuthoritativeInfo,
|
||||
"No Content": http.StatusNoContent,
|
||||
"Reset Content": http.StatusResetContent,
|
||||
"Partial Content": http.StatusPartialContent,
|
||||
"Multi-Status": http.StatusMultiStatus,
|
||||
"Already Reported": http.StatusAlreadyReported,
|
||||
"IM Used": http.StatusIMUsed,
|
||||
"Multiple Choices": http.StatusMultipleChoices,
|
||||
"Moved Permanently": http.StatusMovedPermanently,
|
||||
"Found": http.StatusFound,
|
||||
"See Other": http.StatusSeeOther,
|
||||
"Not Modified": http.StatusNotModified,
|
||||
"Use Proxy": http.StatusUseProxy,
|
||||
"Temporary Redirect": http.StatusTemporaryRedirect,
|
||||
"Permanent Redirect": http.StatusPermanentRedirect,
|
||||
"Bad Request": http.StatusBadRequest,
|
||||
"Unauthorized": http.StatusUnauthorized,
|
||||
"Payment Required": http.StatusPaymentRequired,
|
||||
"Forbidden": http.StatusForbidden,
|
||||
"Not Found": http.StatusNotFound,
|
||||
"Method Not Allowed": http.StatusMethodNotAllowed,
|
||||
"Not Acceptable": http.StatusNotAcceptable,
|
||||
"Proxy Authentication Required": http.StatusProxyAuthRequired,
|
||||
"Request Timeout": http.StatusRequestTimeout,
|
||||
"Conflict": http.StatusConflict,
|
||||
"Gone": http.StatusGone,
|
||||
"Length Required": http.StatusLengthRequired,
|
||||
"Precondition Failed": http.StatusPreconditionFailed,
|
||||
"Request Entity Too Large": http.StatusRequestEntityTooLarge,
|
||||
"Request URI Too Long": http.StatusRequestURITooLong,
|
||||
"Unsupported Media Type": http.StatusUnsupportedMediaType,
|
||||
"Requested Range Not Satisfiable": http.StatusRequestedRangeNotSatisfiable,
|
||||
"Expectation Failed": http.StatusExpectationFailed,
|
||||
"I'm a teapot": http.StatusTeapot,
|
||||
"Misdirected Request": http.StatusMisdirectedRequest,
|
||||
"Unprocessable Entity": http.StatusUnprocessableEntity,
|
||||
"Locked": http.StatusLocked,
|
||||
"Failed Dependency": http.StatusFailedDependency,
|
||||
"Too Early": http.StatusTooEarly,
|
||||
"Upgrade Required": http.StatusUpgradeRequired,
|
||||
"Precondition Required": http.StatusPreconditionRequired,
|
||||
"Too Many Requests": http.StatusTooManyRequests,
|
||||
"Request Header Fields Too Large": http.StatusRequestHeaderFieldsTooLarge,
|
||||
"Unavailable For Legal Reasons": http.StatusUnavailableForLegalReasons,
|
||||
"Internal Server Error": http.StatusInternalServerError,
|
||||
"Not Implemented": http.StatusNotImplemented,
|
||||
"Bad Gateway": http.StatusBadGateway,
|
||||
"Service Unavailable": http.StatusServiceUnavailable,
|
||||
"Gateway Timeout": http.StatusGatewayTimeout,
|
||||
"HTTP Version Not Supported": http.StatusHTTPVersionNotSupported,
|
||||
"Variant Also Negotiates": http.StatusVariantAlsoNegotiates,
|
||||
"Insufficient Storage": http.StatusInsufficientStorage,
|
||||
"Loop Detected": http.StatusLoopDetected,
|
||||
"Not Extended": http.StatusNotExtended,
|
||||
"Network Authentication Required": http.StatusNetworkAuthenticationRequired,
|
||||
}
|
||||
|
||||
// StatusCode returns a HTTP Status code for the HTTP text. It returns -1
|
||||
// if the text is unknown.
|
||||
func StatusCode(text string) int {
|
||||
if code, ok := statusTextToCode[text]; ok {
|
||||
return code
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func fwdHeadersToS3(h http.Header, w http.ResponseWriter) {
|
||||
const trim = "x-amz-fwd-header-"
|
||||
for k, v := range h {
|
||||
if strings.HasPrefix(strings.ToLower(k), trim) {
|
||||
w.Header()[k[len(trim):]] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func fwdStatusToAPIError(resp *http.Response) *APIError {
|
||||
if status := resp.Header.Get(xhttp.AmzFwdStatus); status != "" && StatusCode(status) > -1 {
|
||||
apiErr := &APIError{
|
||||
HTTPStatusCode: StatusCode(status),
|
||||
Description: resp.Header.Get(xhttp.AmzFwdErrorMessage),
|
||||
Code: resp.Header.Get(xhttp.AmzFwdErrorCode),
|
||||
}
|
||||
if apiErr.HTTPStatusCode == http.StatusOK {
|
||||
return nil
|
||||
}
|
||||
return apiErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetObjectLamdbaHandler - GET Object with transformed data via lambda functions
|
||||
// ----------
|
||||
// This implementation of the GET operation applies lambda functions and returns the
|
||||
// response generated via the lambda functions. To use this API, you must have READ access
|
||||
// to the object.
|
||||
func (api objectAPIHandlers) GetObjectLambdaHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetObjectLambda")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
object, err := unescapePath(vars["object"])
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Check for auth type to return S3 compatible error.
|
||||
cred, _, s3Error := checkRequestAuthTypeCredential(ctx, r, policy.GetObjectAction)
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
target, err := globalLambdaTargetList.Lookup(r.Form.Get("lambdaArn"))
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
eventData, err := getLambdaEventData(bucket, object, cred, r)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
resp, err := target.Send(eventData)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if eventData.GetObjectContext.OutputRoute != resp.Header.Get(xhttp.AmzRequestRoute) {
|
||||
tokenErr := errorCodes.ToAPIErr(ErrInvalidRequest)
|
||||
tokenErr.Description = "The request route included in the request is invalid"
|
||||
writeErrorResponse(ctx, w, tokenErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if subtle.ConstantTimeCompare([]byte(resp.Header.Get(xhttp.AmzRequestToken)), []byte(eventData.GetObjectContext.OutputToken)) != 1 {
|
||||
tokenErr := errorCodes.ToAPIErr(ErrInvalidToken)
|
||||
tokenErr.Description = "The request token included in the request is invalid"
|
||||
writeErrorResponse(ctx, w, tokenErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Set all the relevant lambda forward headers if found.
|
||||
fwdHeadersToS3(resp.Header, w)
|
||||
|
||||
if apiErr := fwdStatusToAPIError(resp); apiErr != nil {
|
||||
writeErrorResponse(ctx, w, *apiErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
writeErrorResponse(ctx, w, APIError{
|
||||
Code: "LambdaFunctionError",
|
||||
HTTPStatusCode: resp.StatusCode,
|
||||
Description: "unexpected failure reported from lambda function",
|
||||
}, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if !globalAPIConfig.shouldGzipObjects() {
|
||||
w.Header().Set(gzhttp.HeaderNoCompression, "true")
|
||||
}
|
||||
|
||||
io.Copy(w, resp.Body)
|
||||
}
|
||||
+4
-19
@@ -57,6 +57,10 @@ func (client *peerRESTClient) call(method string, values url.Values, body io.Rea
|
||||
// permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
|
||||
// after verifying format.json
|
||||
func (client *peerRESTClient) callWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
|
||||
if client == nil || !client.IsOnline() {
|
||||
return nil, errPeerNotReachable
|
||||
}
|
||||
|
||||
if values == nil {
|
||||
values = make(url.Values)
|
||||
}
|
||||
@@ -309,25 +313,6 @@ func (client *peerRESTClient) DeleteBucketMetadata(bucket string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// cycleServerBloomFilter will cycle the bloom filter to start recording to index y if not already.
|
||||
// The response will contain a bloom filter starting at index x up to, but not including index y.
|
||||
// If y is 0, the response will not update y, but return the currently recorded information
|
||||
// from the current x to y-1.
|
||||
func (client *peerRESTClient) cycleServerBloomFilter(ctx context.Context, req bloomFilterRequest) (*bloomFilterResponse, error) {
|
||||
var reader bytes.Buffer
|
||||
err := gob.NewEncoder(&reader).Encode(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
respBody, err := client.callWithContext(ctx, peerRESTMethodCycleBloom, nil, &reader, -1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var resp bloomFilterResponse
|
||||
defer xhttp.DrainBody(respBody)
|
||||
return &resp, gob.NewDecoder(respBody).Decode(&resp)
|
||||
}
|
||||
|
||||
// DeletePolicy - delete a specific canned policy.
|
||||
func (client *peerRESTClient) DeletePolicy(policyName string) (err error) {
|
||||
values := make(url.Values)
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
package cmd
|
||||
|
||||
const (
|
||||
peerRESTVersion = "v29" // Added LocalStorageInfo peer API
|
||||
peerRESTVersion = "v30" // Removed bloom filter
|
||||
|
||||
peerRESTVersionPrefix = SlashSeparator + peerRESTVersion
|
||||
peerRESTPrefix = minioReservedBucketPath + "/peer"
|
||||
|
||||
@@ -631,30 +631,6 @@ func (s *peerRESTServer) LoadBucketMetadataHandler(w http.ResponseWriter, r *htt
|
||||
}
|
||||
}
|
||||
|
||||
// CycleServerBloomFilterHandler cycles bloom filter on server.
|
||||
func (s *peerRESTServer) CycleServerBloomFilterHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.IsValid(w, r) {
|
||||
s.writeErrorResponse(w, errors.New("Invalid request"))
|
||||
return
|
||||
}
|
||||
|
||||
ctx := newContext(r, w, "CycleServerBloomFilter")
|
||||
|
||||
var req bloomFilterRequest
|
||||
err := gob.NewDecoder(r.Body).Decode(&req)
|
||||
if err != nil {
|
||||
s.writeErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
bf, err := intDataUpdateTracker.cycleFilter(ctx, req)
|
||||
if err != nil {
|
||||
s.writeErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
logger.LogIf(ctx, gob.NewEncoder(w).Encode(bf))
|
||||
}
|
||||
|
||||
func (s *peerRESTServer) GetMetacacheListingHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.IsValid(w, r) {
|
||||
s.writeErrorResponse(w, errors.New("Invalid request"))
|
||||
@@ -1390,7 +1366,6 @@ func registerPeerRESTHandlers(router *mux.Router) {
|
||||
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodOsInfo).HandlerFunc(httpTraceHdrs(server.GetOSInfoHandler))
|
||||
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDiskHwInfo).HandlerFunc(httpTraceHdrs(server.GetPartitionsHandler))
|
||||
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCPUInfo).HandlerFunc(httpTraceHdrs(server.GetCPUsHandler))
|
||||
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCycleBloom).HandlerFunc(httpTraceHdrs(server.CycleServerBloomFilterHandler))
|
||||
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetAllBucketStats).HandlerFunc(httpTraceHdrs(server.GetAllBucketStatsHandler))
|
||||
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeleteBucketMetadata).HandlerFunc(httpTraceHdrs(server.DeleteBucketMetadataHandler)).Queries(restQueries(peerRESTBucket)...)
|
||||
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadBucketMetadata).HandlerFunc(httpTraceHdrs(server.LoadBucketMetadataHandler)).Queries(restQueries(peerRESTBucket)...)
|
||||
|
||||
@@ -80,6 +80,10 @@ func (s *peerS3Server) HealthHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func listBucketsLocal(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) {
|
||||
globalLocalDrivesMu.RLock()
|
||||
globalLocalDrives := globalLocalDrives
|
||||
globalLocalDrivesMu.RUnlock()
|
||||
|
||||
quorum := (len(globalLocalDrives) / 2)
|
||||
|
||||
buckets = make([]BucketInfo, 0, 32)
|
||||
@@ -128,6 +132,10 @@ func listBucketsLocal(ctx context.Context, opts BucketOptions) (buckets []Bucket
|
||||
}
|
||||
|
||||
func getBucketInfoLocal(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) {
|
||||
globalLocalDrivesMu.RLock()
|
||||
globalLocalDrives := globalLocalDrives
|
||||
globalLocalDrivesMu.RUnlock()
|
||||
|
||||
g := errgroup.WithNErrs(len(globalLocalDrives)).WithConcurrency(32)
|
||||
bucketsInfo := make([]BucketInfo, len(globalLocalDrives))
|
||||
|
||||
@@ -173,6 +181,10 @@ func getBucketInfoLocal(ctx context.Context, bucket string, opts BucketOptions)
|
||||
}
|
||||
|
||||
func deleteBucketLocal(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
|
||||
globalLocalDrivesMu.RLock()
|
||||
globalLocalDrives := globalLocalDrives
|
||||
globalLocalDrivesMu.RUnlock()
|
||||
|
||||
g := errgroup.WithNErrs(len(globalLocalDrives)).WithConcurrency(32)
|
||||
|
||||
// Make a volume entry on all underlying storage disks.
|
||||
@@ -208,6 +220,10 @@ func deleteBucketLocal(ctx context.Context, bucket string, opts DeleteBucketOpti
|
||||
}
|
||||
|
||||
func makeBucketLocal(ctx context.Context, bucket string, opts MakeBucketOptions) error {
|
||||
globalLocalDrivesMu.RLock()
|
||||
globalLocalDrives := globalLocalDrives
|
||||
globalLocalDrivesMu.RUnlock()
|
||||
|
||||
g := errgroup.WithNErrs(len(globalLocalDrives)).WithConcurrency(32)
|
||||
|
||||
// Make a volume entry on all underlying storage disks.
|
||||
|
||||
@@ -136,6 +136,7 @@ func printServerCommonMsg(apiEndpoints []string) {
|
||||
}
|
||||
}
|
||||
printEventNotifiers()
|
||||
printLambdaTargets()
|
||||
|
||||
if globalBrowserEnabled {
|
||||
consoleEndpointStr := strings.Join(stripStandardPorts(getConsoleEndpoints(), globalMinioConsoleHost), " ")
|
||||
@@ -152,6 +153,18 @@ func printObjectAPIMsg() {
|
||||
logger.Info(color.Blue("\nDocumentation: ") + "https://min.io/docs/minio/linux/index.html")
|
||||
}
|
||||
|
||||
func printLambdaTargets() {
|
||||
if globalLambdaTargetList == nil || globalLambdaTargetList.Empty() {
|
||||
return
|
||||
}
|
||||
|
||||
arnMsg := color.Blue("Object Lambda ARNs: ")
|
||||
for _, arn := range globalLambdaTargetList.List(globalSite.Region) {
|
||||
arnMsg += color.Bold(fmt.Sprintf("%s ", arn))
|
||||
}
|
||||
logger.Info(arnMsg + "\n")
|
||||
}
|
||||
|
||||
// Prints bucket notification configurations.
|
||||
func printEventNotifiers() {
|
||||
if globalNotificationSys == nil {
|
||||
@@ -168,7 +181,7 @@ func printEventNotifiers() {
|
||||
arnMsg += color.Bold(fmt.Sprintf("%s ", arn))
|
||||
}
|
||||
|
||||
logger.Info(arnMsg)
|
||||
logger.Info(arnMsg + "\n")
|
||||
}
|
||||
|
||||
// Prints startup message for command line access. Prints link to our documentation
|
||||
|
||||
@@ -109,8 +109,7 @@ func parseCredentialHeader(credElement string, region string, stype serviceType)
|
||||
return ch, ErrAuthorizationHeaderMalformed
|
||||
}
|
||||
if credElements[2] != string(stype) {
|
||||
switch stype {
|
||||
case serviceSTS:
|
||||
if stype == serviceSTS {
|
||||
return ch, ErrInvalidServiceSTS
|
||||
}
|
||||
return ch, ErrInvalidServiceS3
|
||||
|
||||
@@ -630,9 +630,8 @@ func TestDoesV4PresignParamsExist(t *testing.T) {
|
||||
// TestParsePreSignV4 - Validates the parsing logic of Presignied v4 request from its url query values.
|
||||
func TestParsePreSignV4(t *testing.T) {
|
||||
// converts the duration in seconds into string format.
|
||||
getDurationStr := func(expires int) string {
|
||||
return strconv.Itoa(expires)
|
||||
}
|
||||
getDurationStr := strconv.Itoa
|
||||
|
||||
// used in expected preSignValues, preSignValues.Date is of type time.Time .
|
||||
queryTime := UTCNow()
|
||||
|
||||
|
||||
@@ -150,10 +150,8 @@ func TestSkipContentSha256Cksum(t *testing.T) {
|
||||
q.Add(testCase.inputHeaderKey, testCase.inputHeaderValue)
|
||||
}
|
||||
inputReq.URL.RawQuery = q.Encode()
|
||||
} else {
|
||||
if testCase.inputHeaderKey != "" {
|
||||
inputReq.Header.Set(testCase.inputHeaderKey, testCase.inputHeaderValue)
|
||||
}
|
||||
} else if testCase.inputHeaderKey != "" {
|
||||
inputReq.Header.Set(testCase.inputHeaderKey, testCase.inputHeaderValue)
|
||||
}
|
||||
inputReq.ParseForm()
|
||||
|
||||
|
||||
+24
-12
@@ -599,13 +599,13 @@ func (c *SiteReplicationSys) PeerJoinReq(ctx context.Context, arg madmin.SRPeerJ
|
||||
func (c *SiteReplicationSys) GetIDPSettings(ctx context.Context) madmin.IDPSettings {
|
||||
s := madmin.IDPSettings{}
|
||||
s.LDAP = madmin.LDAPSettings{
|
||||
IsLDAPEnabled: globalLDAPConfig.Enabled(),
|
||||
LDAPUserDNSearchBase: globalLDAPConfig.LDAP.UserDNSearchBaseDistName,
|
||||
LDAPUserDNSearchFilter: globalLDAPConfig.LDAP.UserDNSearchFilter,
|
||||
LDAPGroupSearchBase: globalLDAPConfig.LDAP.GroupSearchBaseDistName,
|
||||
LDAPGroupSearchFilter: globalLDAPConfig.LDAP.GroupSearchFilter,
|
||||
IsLDAPEnabled: globalIAMSys.LDAPConfig.Enabled(),
|
||||
LDAPUserDNSearchBase: globalIAMSys.LDAPConfig.LDAP.UserDNSearchBaseDistName,
|
||||
LDAPUserDNSearchFilter: globalIAMSys.LDAPConfig.LDAP.UserDNSearchFilter,
|
||||
LDAPGroupSearchBase: globalIAMSys.LDAPConfig.LDAP.GroupSearchBaseDistName,
|
||||
LDAPGroupSearchFilter: globalIAMSys.LDAPConfig.LDAP.GroupSearchFilter,
|
||||
}
|
||||
s.OpenID = globalOpenIDConfig.GetSettings()
|
||||
s.OpenID = globalIAMSys.OpenIDConfig.GetSettings()
|
||||
if s.OpenID.Enabled {
|
||||
s.OpenID.Region = globalSite.Region
|
||||
}
|
||||
@@ -1219,6 +1219,7 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
|
||||
sessionPolicy: sp,
|
||||
claims: change.Create.Claims,
|
||||
comment: change.Create.Comment,
|
||||
expiration: change.Create.Expiration,
|
||||
}
|
||||
_, _, err = globalIAMSys.NewServiceAccount(ctx, change.Create.Parent, change.Create.Groups, opts)
|
||||
if err != nil {
|
||||
@@ -1245,6 +1246,7 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
|
||||
status: change.Update.Status,
|
||||
comment: change.Update.Comment,
|
||||
sessionPolicy: sp,
|
||||
expiration: change.Update.Expiration,
|
||||
}
|
||||
|
||||
_, err = globalIAMSys.UpdateServiceAccount(ctx, change.Update.AccessKey, opts)
|
||||
@@ -1325,10 +1327,9 @@ func (c *SiteReplicationSys) PeerSTSAccHandler(ctx context.Context, stsCred *mad
|
||||
|
||||
// Extract the username and lookup DN and groups in LDAP.
|
||||
ldapUser, isLDAPSTS := claims.Lookup(ldapUserN)
|
||||
switch {
|
||||
case isLDAPSTS:
|
||||
if isLDAPSTS {
|
||||
// Need to lookup the groups from LDAP.
|
||||
_, ldapGroups, err := globalLDAPConfig.LookupUserDN(ldapUser)
|
||||
_, ldapGroups, err := globalIAMSys.LDAPConfig.LookupUserDN(ldapUser)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to query LDAP server for %s: %w", ldapUser, err)
|
||||
}
|
||||
@@ -1848,6 +1849,7 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
|
||||
SessionPolicy: json.RawMessage(policyJSON),
|
||||
Status: acc.Credentials.Status,
|
||||
Comment: acc.Credentials.Comment,
|
||||
Expiration: &acc.Credentials.Expiration,
|
||||
},
|
||||
},
|
||||
UpdatedAt: acc.UpdatedAt,
|
||||
@@ -4716,6 +4718,7 @@ func (c *SiteReplicationSys) healUsers(ctx context.Context, objAPI ObjectLayer,
|
||||
SessionPolicy: json.RawMessage(policyJSON),
|
||||
Status: creds.Status,
|
||||
Comment: creds.Comment,
|
||||
Expiration: &creds.Expiration,
|
||||
},
|
||||
},
|
||||
UpdatedAt: lastUpdate,
|
||||
@@ -4725,10 +4728,19 @@ func (c *SiteReplicationSys) healUsers(ctx context.Context, objAPI ObjectLayer,
|
||||
continue
|
||||
}
|
||||
if creds.IsTemp() && !creds.IsExpired() {
|
||||
var parentPolicy string
|
||||
u, err := globalIAMSys.GetUserInfo(ctx, creds.ParentUser)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to heal temporary credentials %s from peer site %s -> %s : %w", user, latestPeerName, peerName, err))
|
||||
continue
|
||||
// Parent may be "virtual" (for ldap, oidc, client tls auth,
|
||||
// custom auth plugin), so in such cases we apply no parent
|
||||
// policy. The session token will contain info about policy to
|
||||
// be applied.
|
||||
if !errors.Is(err, errNoSuchUser) {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to heal temporary credentials %s from peer site %s -> %s : %w", user, latestPeerName, peerName, err))
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
parentPolicy = u.PolicyName
|
||||
}
|
||||
// Call hook for site replication.
|
||||
if err := c.IAMChangeHook(ctx, madmin.SRIAMItem{
|
||||
@@ -4738,7 +4750,7 @@ func (c *SiteReplicationSys) healUsers(ctx context.Context, objAPI ObjectLayer,
|
||||
SecretKey: creds.SecretKey,
|
||||
SessionToken: creds.SessionToken,
|
||||
ParentUser: creds.ParentUser,
|
||||
ParentPolicyMapping: u.PolicyName,
|
||||
ParentPolicyMapping: parentPolicy,
|
||||
},
|
||||
UpdatedAt: lastUpdate,
|
||||
}); err != nil {
|
||||
|
||||
+6
-6
@@ -361,7 +361,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
|
||||
// Validate JWT; check clientID in claims matches the one associated with the roleArn
|
||||
if err := globalOpenIDConfig.Validate(r.Context(), roleArn, token, accessToken, r.Form.Get(stsDurationSeconds), claims); err != nil {
|
||||
if err := globalIAMSys.OpenIDConfig.Validate(r.Context(), roleArn, token, accessToken, r.Form.Get(stsDurationSeconds), claims); err != nil {
|
||||
switch err {
|
||||
case openid.ErrTokenExpired:
|
||||
switch action {
|
||||
@@ -598,7 +598,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
|
||||
}
|
||||
}
|
||||
|
||||
ldapUserDN, groupDistNames, err := globalLDAPConfig.Bind(ldapUsername, ldapPassword)
|
||||
ldapUserDN, groupDistNames, err := globalIAMSys.LDAPConfig.Bind(ldapUsername, ldapPassword)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("LDAP server error: %w", err)
|
||||
writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
|
||||
@@ -614,7 +614,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
|
||||
return
|
||||
}
|
||||
|
||||
expiryDur, err := globalLDAPConfig.GetExpiryDuration(r.Form.Get(stsDurationSeconds))
|
||||
expiryDur, err := globalIAMSys.LDAPConfig.GetExpiryDuration(r.Form.Get(stsDurationSeconds))
|
||||
if err != nil {
|
||||
writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
|
||||
return
|
||||
@@ -686,7 +686,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
|
||||
claims := make(map[string]interface{})
|
||||
defer logger.AuditLog(ctx, w, r, claims)
|
||||
|
||||
if !globalSTSTLSConfig.Enabled {
|
||||
if !globalIAMSys.STSTLSConfig.Enabled {
|
||||
writeSTSErrorResponse(ctx, w, true, ErrSTSNotInitialized, errors.New("STS API 'AssumeRoleWithCertificate' is disabled"))
|
||||
return
|
||||
}
|
||||
@@ -727,7 +727,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
|
||||
}
|
||||
|
||||
certificate := r.TLS.PeerCertificates[0]
|
||||
if !globalSTSTLSConfig.InsecureSkipVerify { // Verify whether the client certificate has been issued by a trusted CA.
|
||||
if !globalIAMSys.STSTLSConfig.InsecureSkipVerify { // Verify whether the client certificate has been issued by a trusted CA.
|
||||
_, err := certificate.Verify(x509.VerifyOptions{
|
||||
KeyUsages: []x509.ExtKeyUsage{
|
||||
x509.ExtKeyUsageClientAuth,
|
||||
@@ -776,7 +776,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
|
||||
expiry, err := globalSTSTLSConfig.GetExpiryDuration(r.Form.Get(stsDurationSeconds))
|
||||
expiry, err := globalIAMSys.STSTLSConfig.GetExpiryDuration(r.Form.Get(stsDurationSeconds))
|
||||
if err != nil {
|
||||
writeSTSErrorResponse(ctx, w, true, ErrSTSMissingParameter, err)
|
||||
return
|
||||
|
||||
@@ -627,6 +627,11 @@ func (s *TestSuiteIAM) TestSTSForRoot(c *check) {
|
||||
if !gotBuckets.Equals(shouldHaveBuckets) {
|
||||
c.Fatalf("root user should have access to all buckets")
|
||||
}
|
||||
|
||||
// This must fail.
|
||||
if err := userAdmClient.AddUser(ctx, globalActiveCred.AccessKey, globalActiveCred.SecretKey); err == nil {
|
||||
c.Fatal("AddUser() for root credential must fail via root STS creds")
|
||||
}
|
||||
}
|
||||
|
||||
// SetUpLDAP - expects to setup an LDAP test server using the test LDAP
|
||||
|
||||
@@ -1477,9 +1477,9 @@ func getListenNotificationURL(endPoint, bucketName string, prefixes, suffixes, e
|
||||
}
|
||||
|
||||
// getRandomDisks - Creates a slice of N random disks, each of the form - minio-XXX
|
||||
func getRandomDisks(N int) ([]string, error) {
|
||||
func getRandomDisks(n int) ([]string, error) {
|
||||
var erasureDisks []string
|
||||
for i := 0; i < N; i++ {
|
||||
for i := 0; i < n; i++ {
|
||||
path, err := os.MkdirTemp(globalTestTmpDir, "minio-")
|
||||
if err != nil {
|
||||
// Remove directories created so far.
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"github.com/minio/madmin-go/v2"
|
||||
@@ -94,6 +95,11 @@ func (api adminAPIHandlers) AddTierHandler(w http.ResponseWriter, r *http.Reques
|
||||
return
|
||||
}
|
||||
|
||||
var ignoreInUse bool
|
||||
if forceStr := r.Form.Get("force"); forceStr != "" {
|
||||
ignoreInUse, _ = strconv.ParseBool(forceStr)
|
||||
}
|
||||
|
||||
// Disallow remote tiers with internal storage class names
|
||||
switch cfg.Name {
|
||||
case storageclass.STANDARD, storageclass.RRS:
|
||||
@@ -107,7 +113,7 @@ func (api adminAPIHandlers) AddTierHandler(w http.ResponseWriter, r *http.Reques
|
||||
return
|
||||
}
|
||||
|
||||
err = globalTierConfigMgr.Add(ctx, cfg)
|
||||
err = globalTierConfigMgr.Add(ctx, cfg, ignoreInUse)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
|
||||
+11
-8
@@ -103,7 +103,7 @@ func (config *TierConfigMgr) isTierNameInUse(tierName string) (madmin.TierType,
|
||||
}
|
||||
|
||||
// Add adds tier to config if it passes all validations.
|
||||
func (config *TierConfigMgr) Add(ctx context.Context, tier madmin.TierConfig) error {
|
||||
func (config *TierConfigMgr) Add(ctx context.Context, tier madmin.TierConfig, ignoreInUse bool) error {
|
||||
config.Lock()
|
||||
defer config.Unlock()
|
||||
|
||||
@@ -122,13 +122,16 @@ func (config *TierConfigMgr) Add(ctx context.Context, tier madmin.TierConfig) er
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Check if warmbackend is in use by other MinIO tenants
|
||||
inUse, err := d.InUse(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if inUse {
|
||||
return errTierBackendInUse
|
||||
|
||||
if !ignoreInUse {
|
||||
// Check if warmbackend is in use by other MinIO tenants
|
||||
inUse, err := d.InUse(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if inUse {
|
||||
return errTierBackendInUse
|
||||
}
|
||||
}
|
||||
|
||||
config.Tiers[tierName] = tier
|
||||
|
||||
@@ -88,6 +88,9 @@ var errNoPolicyToAttachOrDetach = errors.New("Specified policy update has no net
|
||||
// deleted.
|
||||
var errGroupNotEmpty = errors.New("Specified group is not empty - cannot remove it")
|
||||
|
||||
// error returned in IAM subsystem when a group is disabled
|
||||
var errGroupDisabled = errors.New("Specified group is disabled")
|
||||
|
||||
// error returned in IAM subsystem when policy doesn't exist.
|
||||
var errNoSuchPolicy = errors.New("Specified canned policy does not exist")
|
||||
|
||||
|
||||
+1
-1
@@ -243,7 +243,7 @@ func untar(ctx context.Context, r io.Reader, putObject func(reader io.Reader, in
|
||||
rc.Close()
|
||||
<-asyncWriters
|
||||
wg.Done()
|
||||
//lint:ignore SA6002 we are fine with the tiny alloc
|
||||
//nolint:staticcheck // SA6002 we are fine with the tiny alloc
|
||||
poolBuf128k.Put(b)
|
||||
}()
|
||||
if err := putObject(&rc, fi, name); err != nil {
|
||||
|
||||
+4
-3
@@ -300,13 +300,14 @@ func TestDownloadReleaseData(t *testing.T) {
|
||||
}
|
||||
|
||||
result, err := downloadReleaseURL(u, 1*time.Second, "")
|
||||
if testCase.expectedErr == nil {
|
||||
switch {
|
||||
case testCase.expectedErr == nil:
|
||||
if err != nil {
|
||||
t.Fatalf("error: expected: %v, got: %v", testCase.expectedErr, err)
|
||||
}
|
||||
} else if err == nil {
|
||||
case err == nil:
|
||||
t.Fatalf("error: expected: %v, got: %v", testCase.expectedErr, err)
|
||||
} else if testCase.expectedErr.Error() != err.Error() {
|
||||
case testCase.expectedErr.Error() != err.Error():
|
||||
t.Fatalf("error: expected: %v, got: %v", testCase.expectedErr, err)
|
||||
}
|
||||
|
||||
|
||||
+11
-11
@@ -159,8 +159,7 @@ func ErrorRespToObjectError(err error, params ...string) error {
|
||||
err = PartTooSmall{}
|
||||
}
|
||||
|
||||
switch minioErr.StatusCode {
|
||||
case http.StatusMethodNotAllowed:
|
||||
if minioErr.StatusCode == http.StatusMethodNotAllowed {
|
||||
err = toObjectErr(errMethodNotAllowed, bucket, object)
|
||||
}
|
||||
return err
|
||||
@@ -908,19 +907,20 @@ func lcp(strs []string, pre bool) string {
|
||||
|
||||
// Returns the mode in which MinIO is running
|
||||
func getMinioMode() string {
|
||||
mode := globalMinioModeFS
|
||||
if globalIsDistErasure {
|
||||
mode = globalMinioModeDistErasure
|
||||
} else if globalIsErasure {
|
||||
mode = globalMinioModeErasure
|
||||
} else if globalIsErasureSD {
|
||||
mode = globalMinioModeErasureSD
|
||||
switch {
|
||||
case globalIsDistErasure:
|
||||
return globalMinioModeDistErasure
|
||||
case globalIsErasure:
|
||||
return globalMinioModeErasure
|
||||
case globalIsErasureSD:
|
||||
return globalMinioModeErasureSD
|
||||
default:
|
||||
return globalMinioModeFS
|
||||
}
|
||||
return mode
|
||||
}
|
||||
|
||||
func iamPolicyClaimNameOpenID() string {
|
||||
return globalOpenIDConfig.GetIAMPolicyClaimName()
|
||||
return globalIAMSys.OpenIDConfig.GetIAMPolicyClaimName()
|
||||
}
|
||||
|
||||
func iamPolicyClaimNameSA() string {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user