mirror of
https://github.com/pgsty/minio.git
synced 2026-08-09 07:43:29 +03:00
Compare commits
88 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 90f5e1e5f6 | |||
| aeabac9181 | |||
| b312f13473 | |||
| 727a803bc0 | |||
| d0e443172d | |||
| 60446e7ac0 | |||
| b8544266e5 | |||
| 437dd4e32a | |||
| 447054b841 | |||
| 9bf43e54cd | |||
| 60f8423157 | |||
| 4355ea3c3f | |||
| e30f1ad7bd | |||
| f00c8c4cce | |||
| 703f51164d | |||
| b8dde47d4e | |||
| 7fa3e39f85 | |||
| 4df7a3aa8f | |||
| 64a8f2e554 | |||
| f4fd4ea66d | |||
| 712fe1a8df | |||
| 4a319bedc9 | |||
| bdb3db6dad | |||
| abb385af41 | |||
| 4ee62606e4 | |||
| 079d64c801 | |||
| dcc000ae2c | |||
| c5d19ecebb | |||
| ed29a525b3 | |||
| 020c46cd3c | |||
| 827004cd6d | |||
| 779ec8f0d4 | |||
| 3d0f513ee2 | |||
| 4b6eadbd80 | |||
| 6f47414b23 | |||
| 224a27992a | |||
| 232544e1d8 | |||
| dbcb71828d | |||
| b9196757fd | |||
| b4ac53d157 | |||
| 4952bdb770 | |||
| 00b2ef2932 | |||
| 4536ecfaa4 | |||
| 43a7402968 | |||
| 330dca9a35 | |||
| ddd137d317 | |||
| 06ddd8770e | |||
| 16f8cf1c52 | |||
| 01e520eb23 | |||
| 02f770a0c0 | |||
| 2f4c79bc0f | |||
| 969ee7dfbe | |||
| 5f0b086b05 | |||
| 68b004a48f | |||
| 54ecce66f0 | |||
| 2b008c598b | |||
| 86d02b17cf | |||
| c1a95a70ac | |||
| f246c9053f | |||
| 9cdd204ae4 | |||
| 7b3eb9f7f8 | |||
| d56ef8dbe1 | |||
| a248ed5ff5 | |||
| 5bb31e4883 | |||
| aff2a76d80 | |||
| eddbe6bca2 | |||
| 734d1e320a | |||
| b8dab7b1a9 | |||
| abd6bf060d | |||
| f0d4ef604c | |||
| 2712f75762 | |||
| 4c46668da8 | |||
| 02e93fd6ba | |||
| 366876e98b | |||
| d202fdd022 | |||
| c07e5b49d4 | |||
| 9a39f8ad4d | |||
| 7e0c1c9413 | |||
| 485d833cd7 | |||
| e8a476ef5a | |||
| 267f0ecea2 | |||
| 4ee3434854 | |||
| 0e9854372e | |||
| b5177993b3 | |||
| 55f5c18fd9 | |||
| 8ce101c174 | |||
| 4972735507 | |||
| e6ca6de194 |
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.22.x]
|
||||
go-version: [1.23.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.22.x]
|
||||
go-version: [1.23.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.22.x]
|
||||
go-version: [1.23.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.22.x]
|
||||
go-version: [1.23.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
name: Resiliency Functional Tests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.23.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
check-latest: true
|
||||
- name: Build on ${{ matrix.os }}
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
env:
|
||||
CGO_ENABLED: 0
|
||||
GO111MODULE: on
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make test-resiliency
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.22.x]
|
||||
go-version: [1.23.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -39,3 +39,4 @@ jobs:
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make verify
|
||||
make test-timeout
|
||||
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
# are turned off - i.e. if ldap="", then ldap server is not enabled for
|
||||
# the tests.
|
||||
matrix:
|
||||
go-version: [1.22.x]
|
||||
go-version: [1.23.x]
|
||||
ldap: ["", "localhost:389"]
|
||||
etcd: ["", "http://localhost:2379"]
|
||||
openid: ["", "http://127.0.0.1:5556/dex"]
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
- name: setup-go-step
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 1.22.x
|
||||
go-version: 1.23.x
|
||||
|
||||
- name: github sha short
|
||||
id: vars
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.22.x]
|
||||
go-version: [1.23.x]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.22.x]
|
||||
go-version: [1.23.x]
|
||||
os: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
|
||||
@@ -15,6 +15,9 @@ docker volume rm $(docker volume ls -f dangling=true) || true
|
||||
## change working directory
|
||||
cd .github/workflows/mint
|
||||
|
||||
## always pull latest
|
||||
docker pull docker.io/minio/mint:edge
|
||||
|
||||
docker-compose -f minio-${MODE}.yaml up -d
|
||||
sleep 1m
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.22.x]
|
||||
go-version: [1.23.x]
|
||||
os: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 1.22.7
|
||||
go-version: 1.23.5
|
||||
- name: Get official govulncheck
|
||||
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
shell: bash
|
||||
|
||||
+10
-1
@@ -21,16 +21,25 @@ linters:
|
||||
- misspell
|
||||
- revive
|
||||
- staticcheck
|
||||
- tenv
|
||||
- typecheck
|
||||
- unconvert
|
||||
- unused
|
||||
- usetesting
|
||||
- forcetypeassert
|
||||
- whitespace
|
||||
|
||||
issues:
|
||||
exclude-use-default: false
|
||||
max-issues-per-linter: 100
|
||||
max-same-issues: 100
|
||||
exclude:
|
||||
- "empty-block:"
|
||||
- "unused-parameter:"
|
||||
- "dot-imports:"
|
||||
- should have a package comment
|
||||
- error strings should not be capitalized or end with punctuation or a newline
|
||||
exclude-rules:
|
||||
# Exclude some linters from running on tests files.
|
||||
- path: _test\.go
|
||||
linters:
|
||||
- forcetypeassert
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.22-alpine as build
|
||||
FROM golang:1.23-alpine as build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.22-alpine AS build
|
||||
FROM golang:1.23-alpine AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
|
||||
@@ -1,69 +0,0 @@
|
||||
FROM golang:1.22-alpine AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
|
||||
ENV GOPATH=/go
|
||||
ENV CGO_ENABLED=0
|
||||
|
||||
# Install curl and minisign
|
||||
RUN apk add -U --no-cache ca-certificates && \
|
||||
apk add -U --no-cache curl && \
|
||||
go install aead.dev/minisign/cmd/minisign@v0.2.1
|
||||
|
||||
# Download minio binary and signature files
|
||||
RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /go/bin/minio && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /go/bin/minio.minisig && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.sha256sum -o /go/bin/minio.sha256sum && \
|
||||
chmod +x /go/bin/minio
|
||||
|
||||
# Download mc binary and signature files
|
||||
RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.fips -o /go/bin/mc && \
|
||||
curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.fips.minisig -o /go/bin/mc.minisig && \
|
||||
curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.fips.sha256sum -o /go/bin/mc.sha256sum && \
|
||||
chmod +x /go/bin/mc
|
||||
|
||||
RUN if [ "$TARGETARCH" = "amd64" ]; then \
|
||||
curl -L -s -q https://github.com/moparisthebest/static-curl/releases/latest/download/curl-${TARGETARCH} -o /go/bin/curl; \
|
||||
chmod +x /go/bin/curl; \
|
||||
fi
|
||||
|
||||
# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
|
||||
RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
|
||||
|
||||
FROM registry.access.redhat.com/ubi9/ubi-micro:latest
|
||||
|
||||
ARG RELEASE
|
||||
|
||||
LABEL name="MinIO" \
|
||||
vendor="MinIO Inc <dev@min.io>" \
|
||||
maintainer="MinIO Inc <dev@min.io>" \
|
||||
version="${RELEASE}" \
|
||||
release="${RELEASE}" \
|
||||
summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \
|
||||
description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads."
|
||||
|
||||
ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_SECRET_KEY_FILE=secret_key \
|
||||
MINIO_ROOT_USER_FILE=access_key \
|
||||
MINIO_ROOT_PASSWORD_FILE=secret_key \
|
||||
MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
|
||||
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
|
||||
MINIO_CONFIG_ENV_FILE=config.env
|
||||
|
||||
RUN chmod -R 777 /usr/bin
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /go/bin/minio* /usr/bin/
|
||||
COPY --from=build /go/bin/mc* /usr/bin/
|
||||
COPY --from=build /go/bin/cur* /usr/bin/
|
||||
|
||||
COPY CREDITS /licenses/CREDITS
|
||||
COPY LICENSE /licenses/LICENSE
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
|
||||
EXPOSE 9000
|
||||
VOLUME ["/data"]
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
CMD ["minio"]
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.22-alpine AS build
|
||||
FROM golang:1.23-alpine AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
|
||||
@@ -2,8 +2,8 @@ PWD := $(shell pwd)
|
||||
GOPATH := $(shell go env GOPATH)
|
||||
LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
|
||||
|
||||
GOARCH := $(shell go env GOARCH)
|
||||
GOOS := $(shell go env GOOS)
|
||||
GOOS ?= $(shell go env GOOS)
|
||||
GOARCH ?= $(shell go env GOARCH)
|
||||
|
||||
VERSION ?= $(shell git describe --tags)
|
||||
REPO ?= quay.io/minio
|
||||
@@ -24,7 +24,7 @@ help: ## print this help
|
||||
getdeps: ## fetch necessary dependencies
|
||||
@mkdir -p ${GOPATH}/bin
|
||||
@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR)
|
||||
@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.2.3-0.20241022140105-4558fbf3a223
|
||||
@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.2.5
|
||||
@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
|
||||
|
||||
crosscompile: ## cross compile minio
|
||||
@@ -149,6 +149,10 @@ test-multipart: install-race ## test multipart
|
||||
@echo "Test multipart behavior when part files are missing"
|
||||
@(env bash $(PWD)/buildscripts/multipart-quorum-test.sh)
|
||||
|
||||
test-timeout: install-race ## test multipart
|
||||
@echo "Test server timeout"
|
||||
@(env bash $(PWD)/buildscripts/test-timeout.sh)
|
||||
|
||||
verify: install-race ## verify minio various setups
|
||||
@echo "Verifying build with race"
|
||||
@(env bash $(PWD)/buildscripts/verify-build.sh)
|
||||
@@ -176,7 +180,7 @@ build-debugging:
|
||||
|
||||
build: checks build-debugging ## builds minio to $(PWD)
|
||||
@echo "Building minio binary to './minio'"
|
||||
@CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
||||
@CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
||||
|
||||
hotfix-vars:
|
||||
$(eval LDFLAGS := $(shell MINIO_RELEASE="RELEASE" MINIO_HOTFIX="hotfix.$(shell git rev-parse --short HEAD)" go run buildscripts/gen-ldflags.go $(shell git describe --tags --abbrev=0 | \
|
||||
@@ -184,9 +188,9 @@ hotfix-vars:
|
||||
$(eval VERSION := $(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD))
|
||||
|
||||
hotfix: hotfix-vars clean install ## builds minio binary with hotfix tags
|
||||
@wget -q -c https://github.com/minio/pkger/releases/download/v2.3.1/pkger_2.3.1_linux_amd64.deb
|
||||
@wget -q -c https://raw.githubusercontent.com/minio/minio-service/v1.0.1/linux-systemd/distributed/minio.service
|
||||
@sudo apt install ./pkger_2.3.1_linux_amd64.deb --yes
|
||||
@wget -q -c https://github.com/minio/pkger/releases/download/v2.3.10/pkger_2.3.10_linux_amd64.deb
|
||||
@wget -q -c https://raw.githubusercontent.com/minio/minio-service/v1.1.0/linux-systemd/distributed/minio.service
|
||||
@sudo apt install ./pkger_2.3.10_linux_amd64.deb --yes
|
||||
@mkdir -p minio-release/$(GOOS)-$(GOARCH)/archive
|
||||
@cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio
|
||||
@cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION)
|
||||
@@ -196,11 +200,11 @@ hotfix: hotfix-vars clean install ## builds minio binary with hotfix tags
|
||||
@pkger -r $(VERSION) --ignore
|
||||
|
||||
hotfix-push: hotfix
|
||||
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-amd64/
|
||||
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive
|
||||
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-amd64/
|
||||
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive
|
||||
@echo "Published new hotfix binaries at https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.$(VERSION)"
|
||||
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-$(GOOS)/
|
||||
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-$(GOOS)/archive
|
||||
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-$(GOOS)/
|
||||
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-$(GOOS)/archive
|
||||
@echo "Published new hotfix binaries at https://dl.min.io/server/minio/hotfixes/linux-$(GOOS)/archive/minio.$(VERSION)"
|
||||
|
||||
docker-hotfix-push: docker-hotfix
|
||||
@docker push -q $(TAG) && echo "Published new container $(TAG)"
|
||||
@@ -213,6 +217,10 @@ docker: build ## builds minio docker container
|
||||
@echo "Building minio docker image '$(TAG)'"
|
||||
@docker build -q --no-cache -t $(TAG) . -f Dockerfile
|
||||
|
||||
test-resiliency: build
|
||||
@echo "Running resiliency tests"
|
||||
@(DOCKER_COMPOSE_FILE=$(PWD)/docs/resiliency/docker-compose.yaml env bash $(PWD)/docs/resiliency/resiliency-tests.sh)
|
||||
|
||||
install-race: checks build-debugging ## builds minio to $(PWD)
|
||||
@echo "Building minio binary with -race to './minio'"
|
||||
@GORACE=history_size=7 CGO_ENABLED=1 go build -tags kqueue,dev -race -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
[](https://min.io)
|
||||
|
||||
MinIO is a High Performance Object Storage released under GNU Affero General Public License v3.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads.
|
||||
MinIO is a High Performance Object Storage released under GNU Affero General Public License v3.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads. To learn more about what MinIO is doing for AI storage, go to [AI storage documentation](https://min.io/solutions/object-storage-for-ai).
|
||||
|
||||
This README provides quickstart instructions on running MinIO on bare metal hardware, including container-based installations. For Kubernetes environments, use the [MinIO Kubernetes Operator](https://github.com/minio/operator/blob/master/README.md).
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
#
|
||||
# This script assumes that LDAP server is at:
|
||||
#
|
||||
# `localhost:1389`
|
||||
# `localhost:389`
|
||||
#
|
||||
# if this is not the case, set the environment variable
|
||||
# `_MINIO_LDAP_TEST_SERVER`.
|
||||
@@ -41,7 +41,7 @@ __init__() {
|
||||
fi
|
||||
|
||||
if [ -z "$_MINIO_LDAP_TEST_SERVER" ]; then
|
||||
export _MINIO_LDAP_TEST_SERVER=localhost:1389
|
||||
export _MINIO_LDAP_TEST_SERVER=localhost:389
|
||||
echo "Using default LDAP endpoint: $_MINIO_LDAP_TEST_SERVER"
|
||||
fi
|
||||
|
||||
@@ -58,7 +58,7 @@ create_iam_content_in_old_minio() {
|
||||
mc alias set old-minio http://localhost:9000 minioadmin minioadmin
|
||||
mc ready old-minio
|
||||
mc idp ldap add old-minio \
|
||||
server_addr=localhost:1389 \
|
||||
server_addr=localhost:389 \
|
||||
server_insecure=on \
|
||||
lookup_bind_dn=cn=admin,dc=min,dc=io \
|
||||
lookup_bind_password=admin \
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ -n "$TEST_DEBUG" ]; then
|
||||
set -x
|
||||
fi
|
||||
|
||||
WORK_DIR="$PWD/.verify-$RANDOM"
|
||||
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
|
||||
MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
|
||||
|
||||
if [ ! -x "$PWD/minio" ]; then
|
||||
echo "minio executable binary not found in current directory"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -x "$PWD/minio" ]; then
|
||||
echo "minio executable binary not found in current directory"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
trap 'catch $LINENO' ERR
|
||||
|
||||
function purge() {
|
||||
rm -rf "$1"
|
||||
}
|
||||
|
||||
# shellcheck disable=SC2120
|
||||
catch() {
|
||||
if [ $# -ne 0 ]; then
|
||||
echo "error on line $1"
|
||||
fi
|
||||
|
||||
echo "Cleaning up instances of MinIO"
|
||||
pkill minio || true
|
||||
pkill -9 minio || true
|
||||
purge "$WORK_DIR"
|
||||
if [ $# -ne 0 ]; then
|
||||
exit $#
|
||||
fi
|
||||
}
|
||||
|
||||
catch
|
||||
|
||||
function gen_put_request() {
|
||||
hdr_sleep=$1
|
||||
body_sleep=$2
|
||||
|
||||
echo "PUT /testbucket/testobject HTTP/1.1"
|
||||
sleep $hdr_sleep
|
||||
echo "Host: foo-header"
|
||||
echo "User-Agent: curl/8.2.1"
|
||||
echo "Accept: */*"
|
||||
echo "Content-Length: 30"
|
||||
echo ""
|
||||
|
||||
sleep $body_sleep
|
||||
echo "random line 0"
|
||||
echo "random line 1"
|
||||
echo ""
|
||||
echo ""
|
||||
}
|
||||
|
||||
function send_put_object_request() {
|
||||
hdr_timeout=$1
|
||||
body_timeout=$2
|
||||
|
||||
start=$(date +%s)
|
||||
timeout 5m bash -c "gen_put_request $hdr_timeout $body_timeout | netcat 127.0.0.1 $start_port | read" || return -1
|
||||
[ $(($(date +%s) - start)) -gt $((srv_hdr_timeout + srv_idle_timeout + 1)) ] && return -1
|
||||
return 0
|
||||
}
|
||||
|
||||
function test_minio_with_timeout() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ROOT_USER=minio
|
||||
export MINIO_ROOT_PASSWORD=minio123
|
||||
export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
mkdir ${WORK_DIR}
|
||||
C_PWD=${PWD}
|
||||
if [ ! -x "$PWD/mc" ]; then
|
||||
MC_BUILD_DIR="mc-$RANDOM"
|
||||
if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
|
||||
echo "failed to download https://github.com/minio/mc"
|
||||
purge "${MC_BUILD_DIR}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
(cd "${MC_BUILD_DIR}" && go build -o "$C_PWD/mc")
|
||||
|
||||
# remove mc source.
|
||||
purge "${MC_BUILD_DIR}"
|
||||
fi
|
||||
|
||||
"${MINIO[@]}" --address ":$start_port" --read-header-timeout ${srv_hdr_timeout}s --idle-timeout ${srv_idle_timeout}s "${WORK_DIR}/disk/" >"${WORK_DIR}/server1.log" 2>&1 &
|
||||
pid=$!
|
||||
disown $pid
|
||||
sleep 1
|
||||
|
||||
if ! ps -p ${pid} 1>&2 >/dev/null; then
|
||||
echo "server1 log:"
|
||||
cat "${WORK_DIR}/server1.log"
|
||||
echo "FAILED"
|
||||
purge "$WORK_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -e
|
||||
|
||||
"${PWD}/mc" mb minio/testbucket
|
||||
"${PWD}/mc" anonymous set public minio/testbucket
|
||||
|
||||
# slow header writing
|
||||
send_put_object_request 20 0 && exit -1
|
||||
"${PWD}/mc" stat minio/testbucket/testobject && exit -1
|
||||
|
||||
# quick header write and slow bodywrite
|
||||
send_put_object_request 0 40 && exit -1
|
||||
"${PWD}/mc" stat minio/testbucket/testobject && exit -1
|
||||
|
||||
# quick header and body write
|
||||
send_put_object_request 1 1 || exit -1
|
||||
"${PWD}/mc" stat minio/testbucket/testobject || exit -1
|
||||
}
|
||||
|
||||
function main() {
|
||||
export start_port=$(shuf -i 10000-65000 -n 1)
|
||||
export srv_hdr_timeout=5
|
||||
export srv_idle_timeout=5
|
||||
export -f gen_put_request
|
||||
|
||||
test_minio_with_timeout ${start_port}
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -38,6 +38,7 @@ import (
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
"github.com/minio/minio/internal/event"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
@@ -980,7 +981,6 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
rpt.SetStatus(bucket, "", err)
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
rptData, err := json.Marshal(rpt.BucketMetaImportErrs)
|
||||
@@ -1039,7 +1039,7 @@ func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.
|
||||
}
|
||||
if len(diffCh) == 0 {
|
||||
// Flush if nothing is queued
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
}
|
||||
case <-keepAliveTicker.C:
|
||||
if len(diffCh) > 0 {
|
||||
@@ -1048,7 +1048,7 @@ func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.
|
||||
if _, err := w.Write([]byte(" ")); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
@@ -1098,7 +1098,7 @@ func (a adminAPIHandlers) ReplicationMRFHandler(w http.ResponseWriter, r *http.R
|
||||
}
|
||||
if len(mrfCh) == 0 {
|
||||
// Flush if nothing is queued
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
}
|
||||
case <-keepAliveTicker.C:
|
||||
if len(mrfCh) > 0 {
|
||||
@@ -1107,7 +1107,7 @@ func (a adminAPIHandlers) ReplicationMRFHandler(w http.ResponseWriter, r *http.R
|
||||
if _, err := w.Write([]byte(" ")); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
|
||||
@@ -125,7 +125,6 @@ func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.R
|
||||
}
|
||||
|
||||
if err = validateConfig(ctx, cfg, subSys); err != nil {
|
||||
|
||||
var validationErr ldap.Validation
|
||||
if errors.As(err, &validationErr) {
|
||||
// If we got an LDAP validation error, we need to send appropriate
|
||||
@@ -416,7 +415,6 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
|
||||
return
|
||||
}
|
||||
if err = validateConfig(ctx, cfg, subSys); err != nil {
|
||||
|
||||
var validationErr ldap.Validation
|
||||
if errors.As(err, &validationErr) {
|
||||
// If we got an LDAP validation error, we need to send appropriate
|
||||
|
||||
@@ -259,7 +259,7 @@ func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request)
|
||||
if ep := globalEndpoints[0].Endpoints[0]; !ep.IsLocal {
|
||||
for nodeIdx, proxyEp := range globalProxyEndpoints {
|
||||
if proxyEp.Endpoint.Host == ep.Host {
|
||||
if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
|
||||
if proxied, success := proxyRequestByNodeIndex(ctx, w, r, nodeIdx, false); proxied && success {
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -330,7 +330,7 @@ func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request
|
||||
if ep := globalEndpoints[0].Endpoints[0]; !ep.IsLocal {
|
||||
for nodeIdx, proxyEp := range globalProxyEndpoints {
|
||||
if proxyEp.Endpoint.Host == ep.Host {
|
||||
if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
|
||||
if proxied, success := proxyRequestByNodeIndex(ctx, w, r, nodeIdx, false); proxied && success {
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -384,7 +384,7 @@ func proxyDecommissionRequest(ctx context.Context, defaultEndPoint Endpoint, w h
|
||||
}
|
||||
for nodeIdx, proxyEp := range globalProxyEndpoints {
|
||||
if proxyEp.Endpoint.Host == host && !proxyEp.IsLocal {
|
||||
if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
|
||||
if proxied, success := proxyRequestByNodeIndex(ctx, w, r, nodeIdx, false); proxied && success {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
+36
-48
@@ -37,6 +37,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/config/dns"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
xldap "github.com/minio/pkg/v3/ldap"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
@@ -65,6 +66,17 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// This API only supports removal of internal users not service accounts.
|
||||
ok, _, err = globalIAMSys.IsServiceAccount(accessKey)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if ok {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// When the user is root credential you are not allowed to
|
||||
// remove the root user. Also you cannot delete yourself.
|
||||
if accessKey == globalActiveCred.AccessKey || accessKey == cred.AccessKey {
|
||||
@@ -1475,8 +1487,8 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
effectivePolicy = globalIAMSys.GetCombinedPolicy(policies...)
|
||||
|
||||
}
|
||||
|
||||
buf, err = json.MarshalIndent(effectivePolicy, "", " ")
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
@@ -1568,6 +1580,7 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errTooManyPolicies), r.URL)
|
||||
return
|
||||
}
|
||||
setReqInfoPolicyName(ctx, name)
|
||||
|
||||
policyDoc, err := globalIAMSys.InfoPolicy(name)
|
||||
if err != nil {
|
||||
@@ -1671,6 +1684,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
|
||||
|
||||
vars := mux.Vars(r)
|
||||
policyName := vars["name"]
|
||||
setReqInfoPolicyName(ctx, policyName)
|
||||
|
||||
if err := globalIAMSys.DeletePolicy(ctx, policyName, true); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
@@ -1703,6 +1717,13 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
|
||||
return
|
||||
}
|
||||
setReqInfoPolicyName(ctx, policyName)
|
||||
|
||||
// Reject policy names with commas.
|
||||
if strings.Contains(policyName, ",") {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrPolicyInvalidName), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Error out if Content-Length is missing.
|
||||
if r.ContentLength <= 0 {
|
||||
@@ -1768,6 +1789,7 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
|
||||
policyName := vars["policyName"]
|
||||
entityName := vars["userOrGroup"]
|
||||
isGroup := vars["isGroup"] == "true"
|
||||
setReqInfoPolicyName(ctx, policyName)
|
||||
|
||||
if !isGroup {
|
||||
ok, _, err := globalIAMSys.IsTempUser(entityName)
|
||||
@@ -1853,7 +1875,7 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
|
||||
}))
|
||||
}
|
||||
|
||||
// ListPolicyMappingEntities - GET /minio/admin/v3/idp/builtin/polciy-entities?policy=xxx&user=xxx&group=xxx
|
||||
// ListPolicyMappingEntities - GET /minio/admin/v3/idp/builtin/policy-entities?policy=xxx&user=xxx&group=xxx
|
||||
func (a adminAPIHandlers) ListPolicyMappingEntities(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
@@ -1955,6 +1977,7 @@ func (a adminAPIHandlers) AttachDetachPolicyBuiltin(w http.ResponseWriter, r *ht
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
setReqInfoPolicyName(ctx, strings.Join(addedOrRemoved, ","))
|
||||
|
||||
respBody := madmin.PolicyAssociationResp{
|
||||
UpdatedAt: updatedAt,
|
||||
@@ -2009,6 +2032,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ExportIAMAction)
|
||||
if objectAPI == nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
// Initialize a zip writer which will provide a zipped content
|
||||
@@ -2231,17 +2255,13 @@ func (a adminAPIHandlers) ImportIAMV2(w http.ResponseWriter, r *http.Request) {
|
||||
func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiVer string) {
|
||||
ctx := r.Context()
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
// Validate signature, permissions and get current object layer instance.
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ImportIAMAction)
|
||||
if objectAPI == nil || globalNotificationSys == nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
cred, owner, s3Err := validateAdminSignature(ctx, r, "")
|
||||
if s3Err != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||
@@ -2259,7 +2279,6 @@ func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiV
|
||||
|
||||
// import policies first
|
||||
{
|
||||
|
||||
f, err := zr.Open(pathJoin(iamAssetsDir, allPoliciesFile))
|
||||
switch {
|
||||
case errors.Is(err, os.ErrNotExist):
|
||||
@@ -2331,44 +2350,17 @@ func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiV
|
||||
return
|
||||
}
|
||||
|
||||
if (cred.IsTemp() || cred.IsServiceAccount()) && cred.ParentUser == accessKey {
|
||||
// Incoming access key matches parent user then we should
|
||||
// reject password change requests.
|
||||
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAddUserInvalidArgument, err, allUsersFile, accessKey), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if accessKey has beginning and end space characters, this only applies to new users.
|
||||
if !exists && hasSpaceBE(accessKey) {
|
||||
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminResourceInvalidArgument, err, allUsersFile, accessKey), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
checkDenyOnly := false
|
||||
if accessKey == cred.AccessKey {
|
||||
// Check that there is no explicit deny - otherwise it's allowed
|
||||
// to change one's own password.
|
||||
checkDenyOnly = true
|
||||
}
|
||||
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.CreateUserAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: checkDenyOnly,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAccessDenied, err, allUsersFile, accessKey), r.URL)
|
||||
return
|
||||
}
|
||||
if _, err = globalIAMSys.CreateUser(ctx, accessKey, ureq); err != nil {
|
||||
failed.Users = append(failed.Users, madmin.IAMErrEntity{Name: accessKey, Error: err})
|
||||
} else {
|
||||
added.Users = append(added.Users, accessKey)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2462,17 +2454,6 @@ func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiV
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
|
||||
return
|
||||
}
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.CreateServiceAccountAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAccessDenied, err, allSvcAcctsFile, user), r.URL)
|
||||
return
|
||||
}
|
||||
updateReq := true
|
||||
_, _, err = globalIAMSys.GetServiceAccount(ctx, svcAcctReq.AccessKey)
|
||||
if err != nil {
|
||||
@@ -2801,3 +2782,10 @@ func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.
|
||||
|
||||
return ctx, cred, opts, createReq, targetUser, APIError{}
|
||||
}
|
||||
|
||||
// setReqInfoPolicyName will set the given policyName as a tag on the context's request info,
|
||||
// so that it appears in audit logs.
|
||||
func setReqInfoPolicyName(ctx context.Context, policyName string) {
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
reqInfo.SetTags("policyName", policyName)
|
||||
}
|
||||
|
||||
@@ -716,6 +716,12 @@ func (s *TestSuiteIAM) TestCannedPolicies(c *check) {
|
||||
c.Fatalf("policy info err: %v", err)
|
||||
}
|
||||
|
||||
// Check that policy with comma is rejected.
|
||||
err = s.adm.AddCannedPolicy(ctx, "invalid,policy", policyBytes)
|
||||
if err == nil {
|
||||
c.Fatalf("invalid policy created successfully")
|
||||
}
|
||||
|
||||
infoStr := string(info)
|
||||
if !strings.Contains(infoStr, `"s3:PutObject"`) || !strings.Contains(infoStr, ":"+bucket+"/") {
|
||||
c.Fatalf("policy contains unexpected content!")
|
||||
|
||||
+15
-16
@@ -829,7 +829,7 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
|
||||
// Flush before waiting for next...
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
|
||||
select {
|
||||
case <-ticker.C:
|
||||
@@ -1320,7 +1320,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// Analyze the heal token and route the request accordingly
|
||||
token, success := proxyRequestByToken(ctx, w, r, hip.clientToken)
|
||||
token, _, success := proxyRequestByToken(ctx, w, r, hip.clientToken, false)
|
||||
if success {
|
||||
return
|
||||
}
|
||||
@@ -1359,7 +1359,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if _, err := w.Write([]byte(" ")); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
case hr := <-respCh:
|
||||
switch hr.apiErr {
|
||||
case noError:
|
||||
@@ -1367,7 +1367,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if _, err := w.Write(hr.respBytes); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
} else {
|
||||
writeSuccessResponseJSON(w, hr.respBytes)
|
||||
}
|
||||
@@ -1394,7 +1394,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if _, err := w.Write(errorRespJSON); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
}
|
||||
break forLoop
|
||||
}
|
||||
@@ -1611,7 +1611,6 @@ func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request)
|
||||
if err != nil || ctx.Err() != nil || totalRx > 100*humanize.GiByte {
|
||||
break
|
||||
}
|
||||
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
@@ -1840,7 +1839,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
case result, ok := <-ch:
|
||||
if !ok {
|
||||
return
|
||||
@@ -1849,7 +1848,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
prevResult = result
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1958,7 +1957,7 @@ func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.R
|
||||
if err := enc.Encode(madmin.DriveSpeedTestResult{}); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
case result, ok := <-ch:
|
||||
if !ok {
|
||||
return
|
||||
@@ -1966,7 +1965,7 @@ func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.R
|
||||
if err := enc.Encode(result); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2083,7 +2082,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
|
||||
grid.PutByteBuffer(entry)
|
||||
if len(traceCh) == 0 {
|
||||
// Flush if nothing is queued
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
}
|
||||
case <-keepAliveTicker.C:
|
||||
if len(traceCh) > 0 {
|
||||
@@ -2092,7 +2091,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if _, err := w.Write([]byte(" ")); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
@@ -2184,7 +2183,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
|
||||
grid.PutByteBuffer(log)
|
||||
if len(logCh) == 0 {
|
||||
// Flush if nothing is queued
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
}
|
||||
case <-keepAliveTicker.C:
|
||||
if len(logCh) > 0 {
|
||||
@@ -2193,7 +2192,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
|
||||
if _, err := w.Write([]byte(" ")); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
@@ -2963,13 +2962,13 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
if len(healthInfoCh) == 0 {
|
||||
// Flush if nothing is queued
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
}
|
||||
case <-ticker.C:
|
||||
if _, err := w.Write([]byte(" ")); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
case <-healthCtx.Done():
|
||||
return
|
||||
}
|
||||
|
||||
+1
-10
@@ -841,6 +841,7 @@ func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) f
|
||||
// NOTE: Healing on meta is run regardless
|
||||
// of any bucket being selected, this is to ensure that
|
||||
// meta are always upto date and correct.
|
||||
h.settings.Recursive = true
|
||||
return objAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string, scanMode madmin.HealScanMode) error {
|
||||
if h.isQuitting() {
|
||||
return errHealStopSignalled
|
||||
@@ -896,16 +897,6 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly
|
||||
return nil
|
||||
}
|
||||
|
||||
if !h.settings.Recursive {
|
||||
if h.object != "" {
|
||||
if err := h.healObject(bucket, h.object, "", h.settings.ScanMode); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := objAPI.HealObjects(h.ctx, bucket, h.object, h.settings, h.healObject); err != nil {
|
||||
return errFnHealFromAPIErr(h.ctx, err)
|
||||
}
|
||||
|
||||
@@ -32,6 +32,8 @@ type DeletedObject struct {
|
||||
DeleteMarkerMTime DeleteMarkerMTime `xml:"-"`
|
||||
// MinIO extensions to support delete marker replication
|
||||
ReplicationState ReplicationState `xml:"-"`
|
||||
|
||||
found bool // the object was found during deletion
|
||||
}
|
||||
|
||||
// DeleteMarkerMTime is an embedded type containing time.Time for XML marshal
|
||||
|
||||
+9
-1
@@ -213,6 +213,7 @@ const (
|
||||
ErrPolicyAlreadyAttached
|
||||
ErrPolicyNotAttached
|
||||
ErrExcessData
|
||||
ErrPolicyInvalidName
|
||||
// Add new error codes here.
|
||||
|
||||
// SSE-S3/SSE-KMS related API errors
|
||||
@@ -561,6 +562,11 @@ var errorCodes = errorCodeMap{
|
||||
Description: "More data provided than indicated content length",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrPolicyInvalidName: {
|
||||
Code: "PolicyInvalidName",
|
||||
Description: "Policy name may not contain comma",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrPolicyTooLarge: {
|
||||
Code: "PolicyTooLarge",
|
||||
Description: "Policy exceeds the maximum allowed document size.",
|
||||
@@ -623,7 +629,7 @@ var errorCodes = errorCodeMap{
|
||||
},
|
||||
ErrMissingContentMD5: {
|
||||
Code: "MissingContentMD5",
|
||||
Description: "Missing required header for this request: Content-Md5.",
|
||||
Description: "Missing or invalid required header for this request: Content-Md5 or Amz-Content-Checksum",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrMissingSecurityHeader: {
|
||||
@@ -2248,6 +2254,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrServerNotInitialized
|
||||
case errBucketMetadataNotInitialized:
|
||||
apiErr = ErrBucketMetadataNotInitialized
|
||||
case hash.ErrInvalidChecksum:
|
||||
apiErr = ErrInvalidChecksum
|
||||
}
|
||||
|
||||
// Compression errors
|
||||
|
||||
+21
-15
@@ -166,10 +166,11 @@ type Part struct {
|
||||
Size int64
|
||||
|
||||
// Checksum values
|
||||
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
|
||||
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
|
||||
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
|
||||
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
|
||||
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
|
||||
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
|
||||
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
|
||||
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
|
||||
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||
}
|
||||
|
||||
// ListPartsResponse - format for list parts response.
|
||||
@@ -192,6 +193,8 @@ type ListPartsResponse struct {
|
||||
IsTruncated bool
|
||||
|
||||
ChecksumAlgorithm string
|
||||
ChecksumType string
|
||||
|
||||
// List of parts.
|
||||
Parts []Part `xml:"Part"`
|
||||
}
|
||||
@@ -413,10 +416,11 @@ type CompleteMultipartUploadResponse struct {
|
||||
Key string
|
||||
ETag string
|
||||
|
||||
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
|
||||
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
|
||||
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
|
||||
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
|
||||
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
|
||||
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
|
||||
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
|
||||
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
|
||||
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||
}
|
||||
|
||||
// DeleteError structure.
|
||||
@@ -516,7 +520,6 @@ func cleanReservedKeys(metadata map[string]string) map[string]string {
|
||||
}
|
||||
case crypto.SSEC:
|
||||
m[xhttp.AmzServerSideEncryptionCustomerAlgorithm] = xhttp.AmzEncryptionAES
|
||||
|
||||
}
|
||||
|
||||
var toRemove []string
|
||||
@@ -787,17 +790,18 @@ func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) Initi
|
||||
|
||||
// generates CompleteMultipartUploadResponse for given bucket, key, location and ETag.
|
||||
func generateCompleteMultipartUploadResponse(bucket, key, location string, oi ObjectInfo, h http.Header) CompleteMultipartUploadResponse {
|
||||
cs := oi.decryptChecksums(0, h)
|
||||
cs, _ := oi.decryptChecksums(0, h)
|
||||
c := CompleteMultipartUploadResponse{
|
||||
Location: location,
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
// AWS S3 quotes the ETag in XML, make sure we are compatible here.
|
||||
ETag: "\"" + oi.ETag + "\"",
|
||||
ChecksumSHA1: cs[hash.ChecksumSHA1.String()],
|
||||
ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
|
||||
ChecksumCRC32: cs[hash.ChecksumCRC32.String()],
|
||||
ChecksumCRC32C: cs[hash.ChecksumCRC32C.String()],
|
||||
ETag: "\"" + oi.ETag + "\"",
|
||||
ChecksumSHA1: cs[hash.ChecksumSHA1.String()],
|
||||
ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
|
||||
ChecksumCRC32: cs[hash.ChecksumCRC32.String()],
|
||||
ChecksumCRC32C: cs[hash.ChecksumCRC32C.String()],
|
||||
ChecksumCRC64NVME: cs[hash.ChecksumCRC64NVME.String()],
|
||||
}
|
||||
return c
|
||||
}
|
||||
@@ -825,6 +829,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
|
||||
listPartsResponse.IsTruncated = partsInfo.IsTruncated
|
||||
listPartsResponse.NextPartNumberMarker = partsInfo.NextPartNumberMarker
|
||||
listPartsResponse.ChecksumAlgorithm = partsInfo.ChecksumAlgorithm
|
||||
listPartsResponse.ChecksumType = partsInfo.ChecksumType
|
||||
|
||||
listPartsResponse.Parts = make([]Part, len(partsInfo.Parts))
|
||||
for index, part := range partsInfo.Parts {
|
||||
@@ -837,6 +842,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
|
||||
newPart.ChecksumCRC32C = part.ChecksumCRC32C
|
||||
newPart.ChecksumSHA1 = part.ChecksumSHA1
|
||||
newPart.ChecksumSHA256 = part.ChecksumSHA256
|
||||
newPart.ChecksumCRC64NVME = part.ChecksumCRC64NVME
|
||||
listPartsResponse.Parts[index] = newPart
|
||||
}
|
||||
return listPartsResponse
|
||||
|
||||
+200
-199
File diff suppressed because one or more lines are too long
@@ -162,7 +162,6 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
|
||||
s3Err := ErrAccessDenied
|
||||
if _, ok := r.Header[xhttp.AmzContentSha256]; ok &&
|
||||
getRequestAuthType(r) == authTypeSigned {
|
||||
|
||||
// Get credential information from the request.
|
||||
cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
|
||||
if s3Err != ErrNone {
|
||||
|
||||
+14
-11
@@ -195,8 +195,8 @@ func (ef BatchJobExpireFilter) Matches(obj ObjectInfo, now time.Time) bool {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if len(ef.Metadata) > 0 && !obj.DeleteMarker {
|
||||
for _, kv := range ef.Metadata {
|
||||
// Object (version) must match all x-amz-meta and
|
||||
@@ -289,6 +289,16 @@ type BatchJobExpire struct {
|
||||
|
||||
var _ yaml.Unmarshaler = &BatchJobExpire{}
|
||||
|
||||
// RedactSensitive will redact any sensitive information in b.
|
||||
func (r *BatchJobExpire) RedactSensitive() {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
if r.NotificationCfg.Token != "" {
|
||||
r.NotificationCfg.Token = redactedText
|
||||
}
|
||||
}
|
||||
|
||||
// UnmarshalYAML - BatchJobExpire extends default unmarshal to extract line, col information.
|
||||
func (r *BatchJobExpire) UnmarshalYAML(val *yaml.Node) error {
|
||||
type expireJob BatchJobExpire
|
||||
@@ -435,14 +445,14 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
|
||||
oiCache.Add(od, &exp.ObjectInfo)
|
||||
}
|
||||
|
||||
var done bool
|
||||
// DeleteObject(deletePrefix: true) to expire all versions of an object
|
||||
for _, exp := range toExpireAll {
|
||||
var success bool
|
||||
for attempts := 1; attempts <= retryAttempts; attempts++ {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
done = true
|
||||
ri.trackMultipleObjectVersions(exp, success)
|
||||
return
|
||||
default:
|
||||
}
|
||||
stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
|
||||
@@ -459,14 +469,7 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
|
||||
break
|
||||
}
|
||||
}
|
||||
ri.trackMultipleObjectVersions(r.Bucket, exp, success)
|
||||
if done {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if done {
|
||||
return
|
||||
ri.trackMultipleObjectVersions(exp, success)
|
||||
}
|
||||
|
||||
// DeleteMultiple objects
|
||||
|
||||
+37
-12
@@ -61,6 +61,8 @@ var globalBatchConfig batch.Config
|
||||
const (
|
||||
// Keep the completed/failed job stats 3 days before removing it
|
||||
oldJobsExpiration = 3 * 24 * time.Hour
|
||||
|
||||
redactedText = "**REDACTED**"
|
||||
)
|
||||
|
||||
// BatchJobRequest this is an internal data structure not for external consumption.
|
||||
@@ -74,6 +76,29 @@ type BatchJobRequest struct {
|
||||
ctx context.Context `msg:"-"`
|
||||
}
|
||||
|
||||
// RedactSensitive will redact any sensitive information in b.
|
||||
func (j *BatchJobRequest) RedactSensitive() {
|
||||
j.Replicate.RedactSensitive()
|
||||
j.Expire.RedactSensitive()
|
||||
j.KeyRotate.RedactSensitive()
|
||||
}
|
||||
|
||||
// RedactSensitive will redact any sensitive information in b.
|
||||
func (r *BatchJobReplicateV1) RedactSensitive() {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
if r.Target.Creds.SecretKey != "" {
|
||||
r.Target.Creds.SecretKey = redactedText
|
||||
}
|
||||
if r.Target.Creds.SessionToken != "" {
|
||||
r.Target.Creds.SessionToken = redactedText
|
||||
}
|
||||
}
|
||||
|
||||
// RedactSensitive will redact any sensitive information in b.
|
||||
func (r *BatchJobKeyRotateV1) RedactSensitive() {}
|
||||
|
||||
func notifyEndpoint(ctx context.Context, ri *batchJobInfo, endpoint, token string) error {
|
||||
if endpoint == "" {
|
||||
return nil
|
||||
@@ -597,12 +622,12 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa
|
||||
},
|
||||
}
|
||||
|
||||
opts, err := batchReplicationOpts(ctx, "", gr.ObjInfo)
|
||||
opts, _, err := batchReplicationOpts(ctx, "", gr.ObjInfo)
|
||||
if err != nil {
|
||||
batchLogIf(ctx, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// TODO: I am not sure we read it back, but we aren't sending whether checksums are single/multipart.
|
||||
for k, vals := range opts.Header() {
|
||||
for _, v := range vals {
|
||||
snowballObj.Headers.Add(k, v)
|
||||
@@ -687,14 +712,14 @@ func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectL
|
||||
return err
|
||||
}
|
||||
|
||||
putOpts, err := batchReplicationOpts(ctx, "", objInfo)
|
||||
putOpts, isMP, err := batchReplicationOpts(ctx, "", objInfo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
|
||||
putOpts.Internal = miniogo.AdvancedPutOptions{}
|
||||
}
|
||||
if objInfo.isMultipart() {
|
||||
if isMP {
|
||||
if err := replicateObjectWithMultipart(ctx, c, tgtBucket, pathJoin(tgtPrefix, objInfo.Name), rd, objInfo, putOpts); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -971,7 +996,7 @@ func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, durati
|
||||
// Note: to be used only with batch jobs that affect multiple versions through
|
||||
// a single action. e.g batch-expire has an option to expire all versions of an
|
||||
// object which matches the given filters.
|
||||
func (ri *batchJobInfo) trackMultipleObjectVersions(bucket string, info ObjectInfo, success bool) {
|
||||
func (ri *batchJobInfo) trackMultipleObjectVersions(info ObjectInfo, success bool) {
|
||||
if success {
|
||||
ri.Objects += int64(info.NumVersions)
|
||||
} else {
|
||||
@@ -1425,7 +1450,6 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
|
||||
cred = r.Source.Creds
|
||||
remoteBkt = r.Source.Bucket
|
||||
pathStyle = r.Source.Path
|
||||
|
||||
}
|
||||
|
||||
u, err := url.Parse(remoteEp)
|
||||
@@ -1551,11 +1575,11 @@ func (j *BatchJobRequest) load(ctx context.Context, api ObjectLayer, name string
|
||||
return err
|
||||
}
|
||||
|
||||
func batchReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions, err error) {
|
||||
func batchReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions, isMP bool, err error) {
|
||||
// TODO: support custom storage class for remote replication
|
||||
putOpts, err = putReplicationOpts(ctx, "", objInfo, 0)
|
||||
putOpts, isMP, err = putReplicationOpts(ctx, "", objInfo)
|
||||
if err != nil {
|
||||
return putOpts, err
|
||||
return putOpts, isMP, err
|
||||
}
|
||||
putOpts.Internal = miniogo.AdvancedPutOptions{
|
||||
SourceVersionID: objInfo.VersionID,
|
||||
@@ -1563,7 +1587,7 @@ func batchReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (p
|
||||
SourceETag: objInfo.ETag,
|
||||
ReplicationRequest: true,
|
||||
}
|
||||
return putOpts, nil
|
||||
return putOpts, isMP, nil
|
||||
}
|
||||
|
||||
// ListBatchJobs - lists all currently active batch jobs, optionally takes {jobType}
|
||||
@@ -1695,6 +1719,8 @@ func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Reques
|
||||
return
|
||||
}
|
||||
|
||||
// Remove sensitive fields.
|
||||
req.RedactSensitive()
|
||||
buf, err := yaml.Marshal(req)
|
||||
if err != nil {
|
||||
batchLogIf(ctx, err)
|
||||
@@ -1802,7 +1828,7 @@ func (a adminAPIHandlers) CancelBatchJob(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
|
||||
if _, success := proxyRequestByToken(ctx, w, r, jobID); success {
|
||||
if _, proxied, _ := proxyRequestByToken(ctx, w, r, jobID, true); proxied {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -2283,7 +2309,6 @@ func lookupStyle(s string) miniogo.BucketLookupType {
|
||||
lookup = miniogo.BucketLookupDNS
|
||||
default:
|
||||
lookup = miniogo.BucketLookupAuto
|
||||
|
||||
}
|
||||
return lookup
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ package cmd
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"hash"
|
||||
"io"
|
||||
"sync"
|
||||
@@ -37,12 +38,22 @@ type streamingBitrotWriter struct {
|
||||
shardSize int64
|
||||
canClose *sync.WaitGroup
|
||||
byteBuf []byte
|
||||
finished bool
|
||||
}
|
||||
|
||||
func (b *streamingBitrotWriter) Write(p []byte) (int, error) {
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if b.finished {
|
||||
return 0, errors.New("bitrot write not allowed")
|
||||
}
|
||||
if int64(len(p)) > b.shardSize {
|
||||
return 0, errors.New("unexpected bitrot buffer size")
|
||||
}
|
||||
if int64(len(p)) < b.shardSize {
|
||||
b.finished = true
|
||||
}
|
||||
b.h.Reset()
|
||||
b.h.Write(p)
|
||||
hashBytes := b.h.Sum(nil)
|
||||
|
||||
+1
-1
@@ -178,7 +178,7 @@ func bitrotVerify(r io.Reader, wantSize, partSize int64, algo BitrotAlgorithm, w
|
||||
return errFileCorrupt
|
||||
}
|
||||
|
||||
bufp := xioutil.ODirectPoolSmall.Get().(*[]byte)
|
||||
bufp := xioutil.ODirectPoolSmall.Get()
|
||||
defer xioutil.ODirectPoolSmall.Put(bufp)
|
||||
|
||||
for left > 0 {
|
||||
|
||||
@@ -344,11 +344,9 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
Created: dnsRecords[0].CreationDate,
|
||||
})
|
||||
}
|
||||
|
||||
sort.Slice(bucketsInfo, func(i, j int) bool {
|
||||
return bucketsInfo[i].Name < bucketsInfo[j].Name
|
||||
})
|
||||
|
||||
} else {
|
||||
// Invoke the list buckets.
|
||||
var err error
|
||||
@@ -429,7 +427,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
|
||||
// Content-Md5 is required should be set
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
|
||||
if _, ok := r.Header[xhttp.ContentMD5]; !ok {
|
||||
if !validateLengthAndChecksum(r) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -841,7 +839,6 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
|
||||
}
|
||||
apiErr := ErrBucketAlreadyExists
|
||||
if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() {
|
||||
@@ -1338,6 +1335,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
}
|
||||
opts.EncryptFn = metadataEncrypter(objectEncryptionKey)
|
||||
pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
@@ -1809,6 +1807,10 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
|
||||
return
|
||||
}
|
||||
|
||||
// Audit log tags.
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
reqInfo.SetTags("retention", config.String())
|
||||
|
||||
configData, err := xml.Marshal(config)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
|
||||
@@ -188,7 +188,6 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
|
||||
if errorResponse.Code != testCase.errorResponse.Code {
|
||||
t.Errorf("Test %d: %s: Expected the error code to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Code, errorResponse.Code)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Test for Anonymous/unsigned http request.
|
||||
@@ -290,7 +289,6 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
|
||||
if recV2.Code != testCase.expectedRespStatus {
|
||||
t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Test for Anonymous/unsigned http request.
|
||||
|
||||
@@ -19,7 +19,6 @@ package cmd
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
@@ -53,7 +52,7 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
|
||||
bucket := vars["bucket"]
|
||||
|
||||
// PutBucketLifecycle always needs a Content-Md5
|
||||
if _, ok := r.Header[xhttp.ContentMD5]; !ok {
|
||||
if !validateLengthAndChecksum(r) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -70,7 +69,7 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
bucketLifecycle, err := lifecycle.ParseLifecycleConfigWithID(io.LimitReader(r.Body, r.ContentLength))
|
||||
bucketLifecycle, err := lifecycle.ParseLifecycleConfigWithID(r.Body)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
|
||||
@@ -243,26 +243,26 @@ func parseRequestToken(token string) (subToken string, nodeIndex int) {
|
||||
return subToken, nodeIndex
|
||||
}
|
||||
|
||||
func proxyRequestByToken(ctx context.Context, w http.ResponseWriter, r *http.Request, token string) (string, bool) {
|
||||
subToken, nodeIndex := parseRequestToken(token)
|
||||
if nodeIndex > 0 {
|
||||
return subToken, proxyRequestByNodeIndex(ctx, w, r, nodeIndex)
|
||||
func proxyRequestByToken(ctx context.Context, w http.ResponseWriter, r *http.Request, token string, returnErr bool) (subToken string, proxied bool, success bool) {
|
||||
var nodeIndex int
|
||||
if subToken, nodeIndex = parseRequestToken(token); nodeIndex >= 0 {
|
||||
proxied, success = proxyRequestByNodeIndex(ctx, w, r, nodeIndex, returnErr)
|
||||
}
|
||||
return subToken, false
|
||||
return
|
||||
}
|
||||
|
||||
func proxyRequestByNodeIndex(ctx context.Context, w http.ResponseWriter, r *http.Request, index int) (success bool) {
|
||||
func proxyRequestByNodeIndex(ctx context.Context, w http.ResponseWriter, r *http.Request, index int, returnErr bool) (proxied, success bool) {
|
||||
if len(globalProxyEndpoints) == 0 {
|
||||
return false
|
||||
return
|
||||
}
|
||||
if index < 0 || index >= len(globalProxyEndpoints) {
|
||||
return false
|
||||
return
|
||||
}
|
||||
ep := globalProxyEndpoints[index]
|
||||
if ep.IsLocal {
|
||||
return false
|
||||
return
|
||||
}
|
||||
return proxyRequest(ctx, w, r, ep)
|
||||
return true, proxyRequest(ctx, w, r, ep, returnErr)
|
||||
}
|
||||
|
||||
// ListObjectsV1Handler - GET Bucket (List Objects) Version 1.
|
||||
|
||||
@@ -112,7 +112,6 @@ func (r *ReplicationStats) collectWorkerMetrics(ctx context.Context) {
|
||||
r.wlock.Lock()
|
||||
r.workers.update()
|
||||
r.wlock.Unlock()
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+95
-38
@@ -49,12 +49,11 @@ import (
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/once"
|
||||
"github.com/tinylib/msgp/msgp"
|
||||
"github.com/zeebo/xxh3"
|
||||
"golang.org/x/exp/maps"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -147,7 +146,7 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
|
||||
if errInt == nil {
|
||||
err = nil
|
||||
} else {
|
||||
err = errInt.(error)
|
||||
err, _ = errInt.(error)
|
||||
}
|
||||
}
|
||||
switch err.(type) {
|
||||
@@ -775,13 +774,14 @@ func (m caseInsensitiveMap) Lookup(key string) (string, bool) {
|
||||
return "", false
|
||||
}
|
||||
|
||||
func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, partNum int) (putOpts minio.PutObjectOptions, err error) {
|
||||
func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts minio.PutObjectOptions, isMP bool, err error) {
|
||||
meta := make(map[string]string)
|
||||
isSSEC := crypto.SSEC.IsEncrypted(objInfo.UserDefined)
|
||||
|
||||
for k, v := range objInfo.UserDefined {
|
||||
_, isValidSSEHeader := validSSEReplicationHeaders[k]
|
||||
// In case of SSE-C objects copy the allowed internal headers as well
|
||||
if !isSSEC || !slices.Contains(maps.Keys(validSSEReplicationHeaders), k) {
|
||||
if !isSSEC || !isValidSSEHeader {
|
||||
if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
|
||||
continue
|
||||
}
|
||||
@@ -789,24 +789,28 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
|
||||
continue
|
||||
}
|
||||
}
|
||||
if slices.Contains(maps.Keys(validSSEReplicationHeaders), k) {
|
||||
if isValidSSEHeader {
|
||||
meta[validSSEReplicationHeaders[k]] = v
|
||||
} else {
|
||||
meta[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
isMP = objInfo.isMultipart()
|
||||
if len(objInfo.Checksum) > 0 {
|
||||
// Add encrypted CRC to metadata for SSE-C objects.
|
||||
if isSSEC {
|
||||
meta[ReplicationSsecChecksumHeader] = base64.StdEncoding.EncodeToString(objInfo.Checksum)
|
||||
} else {
|
||||
for _, pi := range objInfo.Parts {
|
||||
if pi.Number == partNum {
|
||||
for k, v := range pi.Checksums {
|
||||
meta[k] = v
|
||||
}
|
||||
}
|
||||
cs, mp := getCRCMeta(objInfo, 0, nil)
|
||||
// Set object checksum.
|
||||
for k, v := range cs {
|
||||
meta[k] = v
|
||||
}
|
||||
isMP = mp
|
||||
if !objInfo.isMultipart() && cs[xhttp.AmzChecksumType] == xhttp.AmzChecksumTypeFullObject {
|
||||
// For objects where checksum is full object, it will be the same.
|
||||
// Therefore, we use the cheaper PutObject replication.
|
||||
isMP = false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -837,7 +841,7 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
|
||||
if tagTmstampStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]; ok {
|
||||
tagTimestamp, err = time.Parse(time.RFC3339Nano, tagTmstampStr)
|
||||
if err != nil {
|
||||
return putOpts, err
|
||||
return putOpts, false, err
|
||||
}
|
||||
}
|
||||
putOpts.Internal.TaggingTimestamp = tagTimestamp
|
||||
@@ -861,7 +865,7 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
|
||||
if retainDateStr, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok {
|
||||
rdate, err := amztime.ISO8601Parse(retainDateStr)
|
||||
if err != nil {
|
||||
return putOpts, err
|
||||
return putOpts, false, err
|
||||
}
|
||||
putOpts.RetainUntilDate = rdate
|
||||
// set retention timestamp in opts
|
||||
@@ -869,7 +873,7 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
|
||||
if retainTmstampStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp]; ok {
|
||||
retTimestamp, err = time.Parse(time.RFC3339Nano, retainTmstampStr)
|
||||
if err != nil {
|
||||
return putOpts, err
|
||||
return putOpts, false, err
|
||||
}
|
||||
}
|
||||
putOpts.Internal.RetentionTimestamp = retTimestamp
|
||||
@@ -881,7 +885,7 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
|
||||
if lholdTmstampStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp]; ok {
|
||||
lholdTimestamp, err = time.Parse(time.RFC3339Nano, lholdTmstampStr)
|
||||
if err != nil {
|
||||
return putOpts, err
|
||||
return putOpts, false, err
|
||||
}
|
||||
}
|
||||
putOpts.Internal.LegalholdTimestamp = lholdTimestamp
|
||||
@@ -891,9 +895,19 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
|
||||
}
|
||||
|
||||
if crypto.S3KMS.IsEncrypted(objInfo.UserDefined) {
|
||||
sseEnc, err := encrypt.NewSSEKMS(objInfo.KMSKeyID(), nil)
|
||||
// If KMS key ID replication is enabled (as by default)
|
||||
// we include the object's KMS key ID. In any case, we
|
||||
// always set the SSE-KMS header. If no KMS key ID is
|
||||
// specified, MinIO is supposed to use whatever default
|
||||
// config applies on the site or bucket.
|
||||
var keyID string
|
||||
if kms.ReplicateKeyID() {
|
||||
keyID = objInfo.KMSKeyID()
|
||||
}
|
||||
|
||||
sseEnc, err := encrypt.NewSSEKMS(keyID, nil)
|
||||
if err != nil {
|
||||
return putOpts, err
|
||||
return putOpts, false, err
|
||||
}
|
||||
putOpts.ServerSideEncryption = sseEnc
|
||||
}
|
||||
@@ -954,7 +968,11 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
|
||||
}
|
||||
|
||||
t, _ := tags.ParseObjectTags(oi1.UserTags)
|
||||
if (oi2.UserTagCount > 0 && !reflect.DeepEqual(oi2.UserTags, t.ToMap())) || (oi2.UserTagCount != len(t.ToMap())) {
|
||||
oi2Map := make(map[string]string)
|
||||
for k, v := range oi2.UserTags {
|
||||
oi2Map[k] = v
|
||||
}
|
||||
if (oi2.UserTagCount > 0 && !reflect.DeepEqual(oi2Map, t.ToMap())) || (oi2.UserTagCount != len(t.ToMap())) {
|
||||
return replicateMetadata
|
||||
}
|
||||
|
||||
@@ -1282,7 +1300,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
|
||||
// use core client to avoid doing multipart on PUT
|
||||
c := &minio.Core{Client: tgt.Client}
|
||||
|
||||
putOpts, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo, 0)
|
||||
putOpts, isMP, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
|
||||
if err != nil {
|
||||
replLogIf(ctx, fmt.Errorf("failure setting options for replication bucket:%s err:%w", bucket, err))
|
||||
sendEvent(eventArgs{
|
||||
@@ -1314,7 +1332,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
|
||||
defer cancel()
|
||||
}
|
||||
r := bandwidth.NewMonitoredReader(newCtx, globalBucketMonitor, gr, opts)
|
||||
if objInfo.isMultipart() {
|
||||
if isMP {
|
||||
rinfo.Err = replicateObjectWithMultipart(ctx, c, tgt.Bucket, object, r, objInfo, putOpts)
|
||||
} else {
|
||||
_, rinfo.Err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts)
|
||||
@@ -1443,13 +1461,14 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
|
||||
}
|
||||
rinfo.Duration = time.Since(startTime)
|
||||
}()
|
||||
|
||||
oi, cerr := tgt.StatObject(ctx, tgt.Bucket, object, minio.StatObjectOptions{
|
||||
sOpts := minio.StatObjectOptions{
|
||||
VersionID: objInfo.VersionID,
|
||||
Internal: minio.AdvancedGetOptions{
|
||||
ReplicationProxyRequest: "false",
|
||||
},
|
||||
})
|
||||
}
|
||||
sOpts.Set(xhttp.AmzTagDirective, "ACCESS")
|
||||
oi, cerr := tgt.StatObject(ctx, tgt.Bucket, object, sOpts)
|
||||
if cerr == nil {
|
||||
rAction = getReplicationAction(objInfo, oi, ri.OpType)
|
||||
rinfo.ReplicationStatus = replication.Completed
|
||||
@@ -1534,19 +1553,30 @@ applyAction:
|
||||
ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside
|
||||
},
|
||||
}
|
||||
if tagTmStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]; ok {
|
||||
// default timestamps to ModTime unless present in metadata
|
||||
lkMap := caseInsensitiveMap(objInfo.UserDefined)
|
||||
if _, ok := lkMap.Lookup(xhttp.AmzObjectLockLegalHold); ok {
|
||||
dstOpts.Internal.LegalholdTimestamp = objInfo.ModTime
|
||||
}
|
||||
if _, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok {
|
||||
dstOpts.Internal.RetentionTimestamp = objInfo.ModTime
|
||||
}
|
||||
if objInfo.UserTags != "" {
|
||||
dstOpts.Internal.TaggingTimestamp = objInfo.ModTime
|
||||
}
|
||||
if tagTmStr, ok := lkMap.Lookup(ReservedMetadataPrefixLower + TaggingTimestamp); ok {
|
||||
ondiskTimestamp, err := time.Parse(time.RFC3339, tagTmStr)
|
||||
if err == nil {
|
||||
dstOpts.Internal.TaggingTimestamp = ondiskTimestamp
|
||||
}
|
||||
}
|
||||
if retTmStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp]; ok {
|
||||
if retTmStr, ok := lkMap.Lookup(ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp); ok {
|
||||
ondiskTimestamp, err := time.Parse(time.RFC3339, retTmStr)
|
||||
if err == nil {
|
||||
dstOpts.Internal.RetentionTimestamp = ondiskTimestamp
|
||||
}
|
||||
}
|
||||
if lholdTmStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp]; ok {
|
||||
if lholdTmStr, ok := lkMap.Lookup(ReservedMetadataPrefixLower + ObjectLockLegalHoldTimestamp); ok {
|
||||
ondiskTimestamp, err := time.Parse(time.RFC3339, lholdTmStr)
|
||||
if err == nil {
|
||||
dstOpts.Internal.LegalholdTimestamp = ondiskTimestamp
|
||||
@@ -1557,8 +1587,7 @@ applyAction:
|
||||
replLogIf(ctx, fmt.Errorf("unable to replicate metadata for object %s/%s(%s) to target %s: %w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err))
|
||||
}
|
||||
} else {
|
||||
var putOpts minio.PutObjectOptions
|
||||
putOpts, err = putReplicationOpts(ctx, tgt.StorageClass, objInfo, 0)
|
||||
putOpts, isMP, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
|
||||
if err != nil {
|
||||
replLogIf(ctx, fmt.Errorf("failed to set replicate options for object %s/%s(%s) (target %s) err:%w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), err))
|
||||
sendEvent(eventArgs{
|
||||
@@ -1589,7 +1618,7 @@ applyAction:
|
||||
defer cancel()
|
||||
}
|
||||
r := bandwidth.NewMonitoredReader(newCtx, globalBucketMonitor, gr, opts)
|
||||
if objInfo.isMultipart() {
|
||||
if isMP {
|
||||
rinfo.Err = replicateObjectWithMultipart(ctx, c, tgt.Bucket, object, r, objInfo, putOpts)
|
||||
} else {
|
||||
_, rinfo.Err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts)
|
||||
@@ -1667,7 +1696,8 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
|
||||
cHeader := http.Header{}
|
||||
cHeader.Add(xhttp.MinIOSourceReplicationRequest, "true")
|
||||
if !isSSEC {
|
||||
for k, v := range partInfo.Checksums {
|
||||
cs, _ := getCRCMeta(objInfo, partInfo.Number, nil)
|
||||
for k, v := range cs {
|
||||
cHeader.Add(k, v)
|
||||
}
|
||||
}
|
||||
@@ -1691,12 +1721,13 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
|
||||
return fmt.Errorf("ssec(%t): Part size mismatch: got %d, want %d", isSSEC, pInfo.Size, size)
|
||||
}
|
||||
uploadedParts = append(uploadedParts, minio.CompletePart{
|
||||
PartNumber: pInfo.PartNumber,
|
||||
ETag: pInfo.ETag,
|
||||
ChecksumCRC32: pInfo.ChecksumCRC32,
|
||||
ChecksumCRC32C: pInfo.ChecksumCRC32C,
|
||||
ChecksumSHA1: pInfo.ChecksumSHA1,
|
||||
ChecksumSHA256: pInfo.ChecksumSHA256,
|
||||
PartNumber: pInfo.PartNumber,
|
||||
ETag: pInfo.ETag,
|
||||
ChecksumCRC32: pInfo.ChecksumCRC32,
|
||||
ChecksumCRC32C: pInfo.ChecksumCRC32C,
|
||||
ChecksumSHA1: pInfo.ChecksumSHA1,
|
||||
ChecksumSHA256: pInfo.ChecksumSHA256,
|
||||
ChecksumCRC64NVME: pInfo.ChecksumCRC64NVME,
|
||||
})
|
||||
}
|
||||
userMeta := map[string]string{
|
||||
@@ -1709,6 +1740,13 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
|
||||
// really big value but its okay on heavily loaded systems. This is just tail end timeout.
|
||||
cctx, ccancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||
defer ccancel()
|
||||
|
||||
if len(objInfo.Checksum) > 0 {
|
||||
cs, _ := getCRCMeta(objInfo, 0, nil)
|
||||
for k, v := range cs {
|
||||
userMeta[k] = strings.Split(v, "-")[0]
|
||||
}
|
||||
}
|
||||
_, err = c.CompleteMultipartUpload(cctx, bucket, object, uploadID, uploadedParts, minio.PutObjectOptions{
|
||||
UserMetadata: userMeta,
|
||||
Internal: minio.AdvancedPutOptions{
|
||||
@@ -3754,3 +3792,22 @@ type validateReplicationDestinationOptions struct {
|
||||
|
||||
checkReadyErr sync.Map
|
||||
}
|
||||
|
||||
func getCRCMeta(oi ObjectInfo, partNum int, h http.Header) (cs map[string]string, isMP bool) {
|
||||
meta := make(map[string]string)
|
||||
cs, isMP = oi.decryptChecksums(partNum, h)
|
||||
for k, v := range cs {
|
||||
cksum := hash.NewChecksumString(k, v)
|
||||
if cksum == nil {
|
||||
continue
|
||||
}
|
||||
if cksum.Valid() {
|
||||
meta[cksum.Type.Key()] = v
|
||||
}
|
||||
if isMP && partNum == 0 {
|
||||
meta[xhttp.AmzChecksumType] = cksum.Type.ObjType()
|
||||
meta[xhttp.AmzChecksumAlgo] = cksum.Type.String()
|
||||
}
|
||||
}
|
||||
return meta, isMP
|
||||
}
|
||||
|
||||
+7
-7
@@ -108,18 +108,18 @@ func (l ReplicationLastHour) merge(o ReplicationLastHour) (merged ReplicationLas
|
||||
|
||||
// Add a new duration data
|
||||
func (l *ReplicationLastHour) addsize(sz int64) {
|
||||
min := time.Now().Unix() / 60
|
||||
l.forwardTo(min)
|
||||
winIdx := min % 60
|
||||
l.Totals[winIdx].merge(AccElem{Total: min, Size: sz, N: 1})
|
||||
l.LastMin = min
|
||||
minutes := time.Now().Unix() / 60
|
||||
l.forwardTo(minutes)
|
||||
winIdx := minutes % 60
|
||||
l.Totals[winIdx].merge(AccElem{Total: minutes, Size: sz, N: 1})
|
||||
l.LastMin = minutes
|
||||
}
|
||||
|
||||
// Merge all recorded counts of last hour into one
|
||||
func (l *ReplicationLastHour) getTotal() AccElem {
|
||||
var res AccElem
|
||||
min := time.Now().Unix() / 60
|
||||
l.forwardTo(min)
|
||||
minutes := time.Now().Unix() / 60
|
||||
l.forwardTo(minutes)
|
||||
for _, elem := range l.Totals[:] {
|
||||
res.merge(elem)
|
||||
}
|
||||
|
||||
@@ -45,7 +45,7 @@ func Test_readFromSecret(t *testing.T) {
|
||||
for _, testCase := range testCases {
|
||||
testCase := testCase
|
||||
t.Run("", func(t *testing.T) {
|
||||
tmpfile, err := os.CreateTemp("", "testfile")
|
||||
tmpfile, err := os.CreateTemp(t.TempDir(), "testfile")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
@@ -157,7 +157,7 @@ MINIO_ROOT_PASSWORD=minio123`,
|
||||
for _, testCase := range testCases {
|
||||
testCase := testCase
|
||||
t.Run("", func(t *testing.T) {
|
||||
tmpfile, err := os.CreateTemp("", "testfile")
|
||||
tmpfile, err := os.CreateTemp(t.TempDir(), "testfile")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
+1
-2
@@ -858,8 +858,8 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if compact {
|
||||
stop := globalScannerMetrics.log(scannerMetricCompactFolder, folder.name)
|
||||
f.newCache.deleteRecursive(thisHash)
|
||||
@@ -873,7 +873,6 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
|
||||
}
|
||||
stop(total)
|
||||
}
|
||||
|
||||
}
|
||||
// Compact if too many children...
|
||||
if !into.Compacted {
|
||||
|
||||
+571
-177
@@ -1721,11 +1721,93 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
if z.AllTierStats == nil {
|
||||
z.AllTierStats = new(allTierStats)
|
||||
}
|
||||
err = z.AllTierStats.DecodeMsg(dc)
|
||||
var zb0004 uint32
|
||||
zb0004, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats")
|
||||
return
|
||||
}
|
||||
for zb0004 > 0 {
|
||||
zb0004--
|
||||
field, err = dc.ReadMapKeyPtr()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats")
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "ts":
|
||||
var zb0005 uint32
|
||||
zb0005, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers")
|
||||
return
|
||||
}
|
||||
if z.AllTierStats.Tiers == nil {
|
||||
z.AllTierStats.Tiers = make(map[string]tierStats, zb0005)
|
||||
} else if len(z.AllTierStats.Tiers) > 0 {
|
||||
for key := range z.AllTierStats.Tiers {
|
||||
delete(z.AllTierStats.Tiers, key)
|
||||
}
|
||||
}
|
||||
for zb0005 > 0 {
|
||||
zb0005--
|
||||
var za0003 string
|
||||
var za0004 tierStats
|
||||
za0003, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers")
|
||||
return
|
||||
}
|
||||
var zb0006 uint32
|
||||
zb0006, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
|
||||
return
|
||||
}
|
||||
for zb0006 > 0 {
|
||||
zb0006--
|
||||
field, err = dc.ReadMapKeyPtr()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "ts":
|
||||
za0004.TotalSize, err = dc.ReadUint64()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "TotalSize")
|
||||
return
|
||||
}
|
||||
case "nv":
|
||||
za0004.NumVersions, err = dc.ReadInt()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumVersions")
|
||||
return
|
||||
}
|
||||
case "no":
|
||||
za0004.NumObjects, err = dc.ReadInt()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumObjects")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
z.AllTierStats.Tiers[za0003] = za0004
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
zb0001Mask |= 0x1
|
||||
case "c":
|
||||
@@ -1743,11 +1825,10 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
}
|
||||
}
|
||||
// Clear omitted fields.
|
||||
if zb0001Mask != 0x1 {
|
||||
if (zb0001Mask & 0x1) == 0 {
|
||||
z.AllTierStats = nil
|
||||
}
|
||||
if (zb0001Mask & 0x1) == 0 {
|
||||
z.AllTierStats = nil
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1766,121 +1847,166 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if zb0001Len == 0 {
|
||||
return
|
||||
}
|
||||
// write "ch"
|
||||
err = en.Append(0xa2, 0x63, 0x68)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = z.Children.EncodeMsg(en)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Children")
|
||||
return
|
||||
}
|
||||
// write "sz"
|
||||
err = en.Append(0xa2, 0x73, 0x7a)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteInt64(z.Size)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Size")
|
||||
return
|
||||
}
|
||||
// write "os"
|
||||
err = en.Append(0xa2, 0x6f, 0x73)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteUint64(z.Objects)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Objects")
|
||||
return
|
||||
}
|
||||
// write "vs"
|
||||
err = en.Append(0xa2, 0x76, 0x73)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteUint64(z.Versions)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Versions")
|
||||
return
|
||||
}
|
||||
// write "dms"
|
||||
err = en.Append(0xa3, 0x64, 0x6d, 0x73)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteUint64(z.DeleteMarkers)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "DeleteMarkers")
|
||||
return
|
||||
}
|
||||
// write "szs"
|
||||
err = en.Append(0xa3, 0x73, 0x7a, 0x73)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteArrayHeader(uint32(dataUsageBucketLen))
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjSizes")
|
||||
return
|
||||
}
|
||||
for za0001 := range z.ObjSizes {
|
||||
err = en.WriteUint64(z.ObjSizes[za0001])
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjSizes", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
// write "vh"
|
||||
err = en.Append(0xa2, 0x76, 0x68)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteArrayHeader(uint32(dataUsageVersionLen))
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjVersions")
|
||||
return
|
||||
}
|
||||
for za0002 := range z.ObjVersions {
|
||||
err = en.WriteUint64(z.ObjVersions[za0002])
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjVersions", za0002)
|
||||
return
|
||||
}
|
||||
}
|
||||
if (zb0001Mask & 0x80) == 0 { // if not omitted
|
||||
// write "ats"
|
||||
err = en.Append(0xa3, 0x61, 0x74, 0x73)
|
||||
|
||||
// skip if no fields are to be emitted
|
||||
if zb0001Len != 0 {
|
||||
// write "ch"
|
||||
err = en.Append(0xa2, 0x63, 0x68)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if z.AllTierStats == nil {
|
||||
err = en.WriteNil()
|
||||
err = z.Children.EncodeMsg(en)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Children")
|
||||
return
|
||||
}
|
||||
// write "sz"
|
||||
err = en.Append(0xa2, 0x73, 0x7a)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteInt64(z.Size)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Size")
|
||||
return
|
||||
}
|
||||
// write "os"
|
||||
err = en.Append(0xa2, 0x6f, 0x73)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteUint64(z.Objects)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Objects")
|
||||
return
|
||||
}
|
||||
// write "vs"
|
||||
err = en.Append(0xa2, 0x76, 0x73)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteUint64(z.Versions)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Versions")
|
||||
return
|
||||
}
|
||||
// write "dms"
|
||||
err = en.Append(0xa3, 0x64, 0x6d, 0x73)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteUint64(z.DeleteMarkers)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "DeleteMarkers")
|
||||
return
|
||||
}
|
||||
// write "szs"
|
||||
err = en.Append(0xa3, 0x73, 0x7a, 0x73)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteArrayHeader(uint32(dataUsageBucketLen))
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjSizes")
|
||||
return
|
||||
}
|
||||
for za0001 := range z.ObjSizes {
|
||||
err = en.WriteUint64(z.ObjSizes[za0001])
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
err = z.AllTierStats.EncodeMsg(en)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats")
|
||||
err = msgp.WrapError(err, "ObjSizes", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
// write "c"
|
||||
err = en.Append(0xa1, 0x63)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteBool(z.Compacted)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Compacted")
|
||||
return
|
||||
// write "vh"
|
||||
err = en.Append(0xa2, 0x76, 0x68)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteArrayHeader(uint32(dataUsageVersionLen))
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjVersions")
|
||||
return
|
||||
}
|
||||
for za0002 := range z.ObjVersions {
|
||||
err = en.WriteUint64(z.ObjVersions[za0002])
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjVersions", za0002)
|
||||
return
|
||||
}
|
||||
}
|
||||
if (zb0001Mask & 0x80) == 0 { // if not omitted
|
||||
// write "ats"
|
||||
err = en.Append(0xa3, 0x61, 0x74, 0x73)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if z.AllTierStats == nil {
|
||||
err = en.WriteNil()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
// map header, size 1
|
||||
// write "ts"
|
||||
err = en.Append(0x81, 0xa2, 0x74, 0x73)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteMapHeader(uint32(len(z.AllTierStats.Tiers)))
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers")
|
||||
return
|
||||
}
|
||||
for za0003, za0004 := range z.AllTierStats.Tiers {
|
||||
err = en.WriteString(za0003)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers")
|
||||
return
|
||||
}
|
||||
// map header, size 3
|
||||
// write "ts"
|
||||
err = en.Append(0x83, 0xa2, 0x74, 0x73)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteUint64(za0004.TotalSize)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "TotalSize")
|
||||
return
|
||||
}
|
||||
// write "nv"
|
||||
err = en.Append(0xa2, 0x6e, 0x76)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteInt(za0004.NumVersions)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumVersions")
|
||||
return
|
||||
}
|
||||
// write "no"
|
||||
err = en.Append(0xa2, 0x6e, 0x6f)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteInt(za0004.NumObjects)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumObjects")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// write "c"
|
||||
err = en.Append(0xa1, 0x63)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteBool(z.Compacted)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Compacted")
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -1898,56 +2024,69 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
}
|
||||
// variable map header, size zb0001Len
|
||||
o = append(o, 0x80|uint8(zb0001Len))
|
||||
if zb0001Len == 0 {
|
||||
return
|
||||
}
|
||||
// string "ch"
|
||||
o = append(o, 0xa2, 0x63, 0x68)
|
||||
o, err = z.Children.MarshalMsg(o)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Children")
|
||||
return
|
||||
}
|
||||
// string "sz"
|
||||
o = append(o, 0xa2, 0x73, 0x7a)
|
||||
o = msgp.AppendInt64(o, z.Size)
|
||||
// string "os"
|
||||
o = append(o, 0xa2, 0x6f, 0x73)
|
||||
o = msgp.AppendUint64(o, z.Objects)
|
||||
// string "vs"
|
||||
o = append(o, 0xa2, 0x76, 0x73)
|
||||
o = msgp.AppendUint64(o, z.Versions)
|
||||
// string "dms"
|
||||
o = append(o, 0xa3, 0x64, 0x6d, 0x73)
|
||||
o = msgp.AppendUint64(o, z.DeleteMarkers)
|
||||
// string "szs"
|
||||
o = append(o, 0xa3, 0x73, 0x7a, 0x73)
|
||||
o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen))
|
||||
for za0001 := range z.ObjSizes {
|
||||
o = msgp.AppendUint64(o, z.ObjSizes[za0001])
|
||||
}
|
||||
// string "vh"
|
||||
o = append(o, 0xa2, 0x76, 0x68)
|
||||
o = msgp.AppendArrayHeader(o, uint32(dataUsageVersionLen))
|
||||
for za0002 := range z.ObjVersions {
|
||||
o = msgp.AppendUint64(o, z.ObjVersions[za0002])
|
||||
}
|
||||
if (zb0001Mask & 0x80) == 0 { // if not omitted
|
||||
// string "ats"
|
||||
o = append(o, 0xa3, 0x61, 0x74, 0x73)
|
||||
if z.AllTierStats == nil {
|
||||
o = msgp.AppendNil(o)
|
||||
} else {
|
||||
o, err = z.AllTierStats.MarshalMsg(o)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats")
|
||||
return
|
||||
|
||||
// skip if no fields are to be emitted
|
||||
if zb0001Len != 0 {
|
||||
// string "ch"
|
||||
o = append(o, 0xa2, 0x63, 0x68)
|
||||
o, err = z.Children.MarshalMsg(o)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Children")
|
||||
return
|
||||
}
|
||||
// string "sz"
|
||||
o = append(o, 0xa2, 0x73, 0x7a)
|
||||
o = msgp.AppendInt64(o, z.Size)
|
||||
// string "os"
|
||||
o = append(o, 0xa2, 0x6f, 0x73)
|
||||
o = msgp.AppendUint64(o, z.Objects)
|
||||
// string "vs"
|
||||
o = append(o, 0xa2, 0x76, 0x73)
|
||||
o = msgp.AppendUint64(o, z.Versions)
|
||||
// string "dms"
|
||||
o = append(o, 0xa3, 0x64, 0x6d, 0x73)
|
||||
o = msgp.AppendUint64(o, z.DeleteMarkers)
|
||||
// string "szs"
|
||||
o = append(o, 0xa3, 0x73, 0x7a, 0x73)
|
||||
o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen))
|
||||
for za0001 := range z.ObjSizes {
|
||||
o = msgp.AppendUint64(o, z.ObjSizes[za0001])
|
||||
}
|
||||
// string "vh"
|
||||
o = append(o, 0xa2, 0x76, 0x68)
|
||||
o = msgp.AppendArrayHeader(o, uint32(dataUsageVersionLen))
|
||||
for za0002 := range z.ObjVersions {
|
||||
o = msgp.AppendUint64(o, z.ObjVersions[za0002])
|
||||
}
|
||||
if (zb0001Mask & 0x80) == 0 { // if not omitted
|
||||
// string "ats"
|
||||
o = append(o, 0xa3, 0x61, 0x74, 0x73)
|
||||
if z.AllTierStats == nil {
|
||||
o = msgp.AppendNil(o)
|
||||
} else {
|
||||
// map header, size 1
|
||||
// string "ts"
|
||||
o = append(o, 0x81, 0xa2, 0x74, 0x73)
|
||||
o = msgp.AppendMapHeader(o, uint32(len(z.AllTierStats.Tiers)))
|
||||
for za0003, za0004 := range z.AllTierStats.Tiers {
|
||||
o = msgp.AppendString(o, za0003)
|
||||
// map header, size 3
|
||||
// string "ts"
|
||||
o = append(o, 0x83, 0xa2, 0x74, 0x73)
|
||||
o = msgp.AppendUint64(o, za0004.TotalSize)
|
||||
// string "nv"
|
||||
o = append(o, 0xa2, 0x6e, 0x76)
|
||||
o = msgp.AppendInt(o, za0004.NumVersions)
|
||||
// string "no"
|
||||
o = append(o, 0xa2, 0x6e, 0x6f)
|
||||
o = msgp.AppendInt(o, za0004.NumObjects)
|
||||
}
|
||||
}
|
||||
}
|
||||
// string "c"
|
||||
o = append(o, 0xa1, 0x63)
|
||||
o = msgp.AppendBool(o, z.Compacted)
|
||||
}
|
||||
// string "c"
|
||||
o = append(o, 0xa1, 0x63)
|
||||
o = msgp.AppendBool(o, z.Compacted)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -2048,11 +2187,93 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
if z.AllTierStats == nil {
|
||||
z.AllTierStats = new(allTierStats)
|
||||
}
|
||||
bts, err = z.AllTierStats.UnmarshalMsg(bts)
|
||||
var zb0004 uint32
|
||||
zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats")
|
||||
return
|
||||
}
|
||||
for zb0004 > 0 {
|
||||
zb0004--
|
||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats")
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "ts":
|
||||
var zb0005 uint32
|
||||
zb0005, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers")
|
||||
return
|
||||
}
|
||||
if z.AllTierStats.Tiers == nil {
|
||||
z.AllTierStats.Tiers = make(map[string]tierStats, zb0005)
|
||||
} else if len(z.AllTierStats.Tiers) > 0 {
|
||||
for key := range z.AllTierStats.Tiers {
|
||||
delete(z.AllTierStats.Tiers, key)
|
||||
}
|
||||
}
|
||||
for zb0005 > 0 {
|
||||
var za0003 string
|
||||
var za0004 tierStats
|
||||
zb0005--
|
||||
za0003, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers")
|
||||
return
|
||||
}
|
||||
var zb0006 uint32
|
||||
zb0006, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
|
||||
return
|
||||
}
|
||||
for zb0006 > 0 {
|
||||
zb0006--
|
||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "ts":
|
||||
za0004.TotalSize, bts, err = msgp.ReadUint64Bytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "TotalSize")
|
||||
return
|
||||
}
|
||||
case "nv":
|
||||
za0004.NumVersions, bts, err = msgp.ReadIntBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumVersions")
|
||||
return
|
||||
}
|
||||
case "no":
|
||||
za0004.NumObjects, bts, err = msgp.ReadIntBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumObjects")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
z.AllTierStats.Tiers[za0003] = za0004
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
zb0001Mask |= 0x1
|
||||
case "c":
|
||||
@@ -2070,11 +2291,10 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
}
|
||||
}
|
||||
// Clear omitted fields.
|
||||
if zb0001Mask != 0x1 {
|
||||
if (zb0001Mask & 0x1) == 0 {
|
||||
z.AllTierStats = nil
|
||||
}
|
||||
if (zb0001Mask & 0x1) == 0 {
|
||||
z.AllTierStats = nil
|
||||
}
|
||||
|
||||
o = bts
|
||||
return
|
||||
}
|
||||
@@ -2085,7 +2305,13 @@ func (z *dataUsageEntry) Msgsize() (s int) {
|
||||
if z.AllTierStats == nil {
|
||||
s += msgp.NilSize
|
||||
} else {
|
||||
s += z.AllTierStats.Msgsize()
|
||||
s += 1 + 3 + msgp.MapHeaderSize
|
||||
if z.AllTierStats.Tiers != nil {
|
||||
for za0003, za0004 := range z.AllTierStats.Tiers {
|
||||
_ = za0004
|
||||
s += msgp.StringPrefixSize + len(za0003) + 1 + 3 + msgp.Uint64Size + 3 + msgp.IntSize + 3 + msgp.IntSize
|
||||
}
|
||||
}
|
||||
}
|
||||
s += 2 + msgp.BoolSize
|
||||
return
|
||||
@@ -2734,11 +2960,93 @@ func (z *dataUsageEntryV7) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
if z.AllTierStats == nil {
|
||||
z.AllTierStats = new(allTierStats)
|
||||
}
|
||||
err = z.AllTierStats.DecodeMsg(dc)
|
||||
var zb0004 uint32
|
||||
zb0004, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats")
|
||||
return
|
||||
}
|
||||
for zb0004 > 0 {
|
||||
zb0004--
|
||||
field, err = dc.ReadMapKeyPtr()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats")
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "ts":
|
||||
var zb0005 uint32
|
||||
zb0005, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers")
|
||||
return
|
||||
}
|
||||
if z.AllTierStats.Tiers == nil {
|
||||
z.AllTierStats.Tiers = make(map[string]tierStats, zb0005)
|
||||
} else if len(z.AllTierStats.Tiers) > 0 {
|
||||
for key := range z.AllTierStats.Tiers {
|
||||
delete(z.AllTierStats.Tiers, key)
|
||||
}
|
||||
}
|
||||
for zb0005 > 0 {
|
||||
zb0005--
|
||||
var za0003 string
|
||||
var za0004 tierStats
|
||||
za0003, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers")
|
||||
return
|
||||
}
|
||||
var zb0006 uint32
|
||||
zb0006, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
|
||||
return
|
||||
}
|
||||
for zb0006 > 0 {
|
||||
zb0006--
|
||||
field, err = dc.ReadMapKeyPtr()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "ts":
|
||||
za0004.TotalSize, err = dc.ReadUint64()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "TotalSize")
|
||||
return
|
||||
}
|
||||
case "nv":
|
||||
za0004.NumVersions, err = dc.ReadInt()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumVersions")
|
||||
return
|
||||
}
|
||||
case "no":
|
||||
za0004.NumObjects, err = dc.ReadInt()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumObjects")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
z.AllTierStats.Tiers[za0003] = za0004
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
zb0001Mask |= 0x1
|
||||
case "c":
|
||||
@@ -2756,11 +3064,10 @@ func (z *dataUsageEntryV7) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
}
|
||||
}
|
||||
// Clear omitted fields.
|
||||
if zb0001Mask != 0x1 {
|
||||
if (zb0001Mask & 0x1) == 0 {
|
||||
z.AllTierStats = nil
|
||||
}
|
||||
if (zb0001Mask & 0x1) == 0 {
|
||||
z.AllTierStats = nil
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -2861,11 +3168,93 @@ func (z *dataUsageEntryV7) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
if z.AllTierStats == nil {
|
||||
z.AllTierStats = new(allTierStats)
|
||||
}
|
||||
bts, err = z.AllTierStats.UnmarshalMsg(bts)
|
||||
var zb0004 uint32
|
||||
zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats")
|
||||
return
|
||||
}
|
||||
for zb0004 > 0 {
|
||||
zb0004--
|
||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats")
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "ts":
|
||||
var zb0005 uint32
|
||||
zb0005, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers")
|
||||
return
|
||||
}
|
||||
if z.AllTierStats.Tiers == nil {
|
||||
z.AllTierStats.Tiers = make(map[string]tierStats, zb0005)
|
||||
} else if len(z.AllTierStats.Tiers) > 0 {
|
||||
for key := range z.AllTierStats.Tiers {
|
||||
delete(z.AllTierStats.Tiers, key)
|
||||
}
|
||||
}
|
||||
for zb0005 > 0 {
|
||||
var za0003 string
|
||||
var za0004 tierStats
|
||||
zb0005--
|
||||
za0003, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers")
|
||||
return
|
||||
}
|
||||
var zb0006 uint32
|
||||
zb0006, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
|
||||
return
|
||||
}
|
||||
for zb0006 > 0 {
|
||||
zb0006--
|
||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "ts":
|
||||
za0004.TotalSize, bts, err = msgp.ReadUint64Bytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "TotalSize")
|
||||
return
|
||||
}
|
||||
case "nv":
|
||||
za0004.NumVersions, bts, err = msgp.ReadIntBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumVersions")
|
||||
return
|
||||
}
|
||||
case "no":
|
||||
za0004.NumObjects, bts, err = msgp.ReadIntBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumObjects")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
z.AllTierStats.Tiers[za0003] = za0004
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AllTierStats")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
zb0001Mask |= 0x1
|
||||
case "c":
|
||||
@@ -2883,11 +3272,10 @@ func (z *dataUsageEntryV7) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
}
|
||||
}
|
||||
// Clear omitted fields.
|
||||
if zb0001Mask != 0x1 {
|
||||
if (zb0001Mask & 0x1) == 0 {
|
||||
z.AllTierStats = nil
|
||||
}
|
||||
if (zb0001Mask & 0x1) == 0 {
|
||||
z.AllTierStats = nil
|
||||
}
|
||||
|
||||
o = bts
|
||||
return
|
||||
}
|
||||
@@ -2898,7 +3286,13 @@ func (z *dataUsageEntryV7) Msgsize() (s int) {
|
||||
if z.AllTierStats == nil {
|
||||
s += msgp.NilSize
|
||||
} else {
|
||||
s += z.AllTierStats.Msgsize()
|
||||
s += 1 + 3 + msgp.MapHeaderSize
|
||||
if z.AllTierStats.Tiers != nil {
|
||||
for za0003, za0004 := range z.AllTierStats.Tiers {
|
||||
_ = za0004
|
||||
s += msgp.StringPrefixSize + len(za0003) + 1 + 3 + msgp.Uint64Size + 3 + msgp.IntSize + 3 + msgp.IntSize
|
||||
}
|
||||
}
|
||||
}
|
||||
s += 2 + msgp.BoolSize
|
||||
return
|
||||
|
||||
@@ -98,7 +98,6 @@ func (dt *dynamicTimeout) logEntry(duration time.Duration) {
|
||||
|
||||
// We leak entries while we copy
|
||||
if entries == dynamicTimeoutLogSize {
|
||||
|
||||
// Make copy on stack in order to call adjust()
|
||||
logCopy := [dynamicTimeoutLogSize]time.Duration{}
|
||||
copy(logCopy[:], dt.log[:])
|
||||
@@ -117,12 +116,12 @@ func (dt *dynamicTimeout) logEntry(duration time.Duration) {
|
||||
// adjust changes the value of the dynamic timeout based on the
|
||||
// previous results
|
||||
func (dt *dynamicTimeout) adjust(entries [dynamicTimeoutLogSize]time.Duration) {
|
||||
failures, max := 0, time.Duration(0)
|
||||
failures, maxDur := 0, time.Duration(0)
|
||||
for _, dur := range entries[:] {
|
||||
if dur == maxDuration {
|
||||
failures++
|
||||
} else if dur > max {
|
||||
max = dur
|
||||
} else if dur > maxDur {
|
||||
maxDur = dur
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,12 +143,12 @@ func (dt *dynamicTimeout) adjust(entries [dynamicTimeoutLogSize]time.Duration) {
|
||||
} else if failPct < dynamicTimeoutDecreaseThresholdPct {
|
||||
// We are hitting the timeout relatively few times,
|
||||
// so decrease the timeout towards 25 % of maximum time spent.
|
||||
max = max * 125 / 100
|
||||
maxDur = maxDur * 125 / 100
|
||||
|
||||
timeout := atomic.LoadInt64(&dt.timeout)
|
||||
if max < time.Duration(timeout) {
|
||||
if maxDur < time.Duration(timeout) {
|
||||
// Move 50% toward the max.
|
||||
timeout = (int64(max) + timeout) / 2
|
||||
timeout = (int64(maxDur) + timeout) / 2
|
||||
}
|
||||
if timeout < dt.minimum {
|
||||
timeout = dt.minimum
|
||||
|
||||
@@ -167,7 +167,6 @@ func testDynamicTimeoutAdjust(t *testing.T, timeout *dynamicTimeout, f func() fl
|
||||
const successTimeout = 20 * time.Second
|
||||
|
||||
for i := 0; i < dynamicTimeoutLogSize; i++ {
|
||||
|
||||
rnd := f()
|
||||
duration := time.Duration(float64(successTimeout) * rnd)
|
||||
|
||||
|
||||
@@ -347,8 +347,8 @@ func rotateKey(ctx context.Context, oldKey []byte, newKeyID string, newKey []byt
|
||||
return errInvalidSSEParameters // AWS returns special error for equal but invalid keys.
|
||||
}
|
||||
return crypto.ErrInvalidCustomerKey // To provide strict AWS S3 compatibility we return: access denied.
|
||||
|
||||
}
|
||||
|
||||
if subtle.ConstantTimeCompare(oldKey, newKey) == 1 && sealedKey.Algorithm == crypto.SealAlgorithm {
|
||||
return nil // don't rotate on equal keys if seal algorithm is latest
|
||||
}
|
||||
@@ -1155,16 +1155,17 @@ func (o *ObjectInfo) metadataEncryptFn(headers http.Header) (objectMetaEncryptFn
|
||||
|
||||
// decryptChecksums will attempt to decode checksums and return it/them if set.
|
||||
// if part > 0, and we have the checksum for the part that will be returned.
|
||||
func (o *ObjectInfo) decryptChecksums(part int, h http.Header) map[string]string {
|
||||
// Returns whether the checksum (main part 0) is a multipart checksum.
|
||||
func (o *ObjectInfo) decryptChecksums(part int, h http.Header) (cs map[string]string, isMP bool) {
|
||||
data := o.Checksum
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
return nil, false
|
||||
}
|
||||
if part > 0 && !crypto.SSEC.IsEncrypted(o.UserDefined) {
|
||||
// already decrypted in ToObjectInfo for multipart objects
|
||||
for _, pi := range o.Parts {
|
||||
if pi.Number == part {
|
||||
return pi.Checksums
|
||||
return pi.Checksums, true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1174,7 +1175,7 @@ func (o *ObjectInfo) decryptChecksums(part int, h http.Header) map[string]string
|
||||
if err != crypto.ErrSecretKeyMismatch {
|
||||
encLogIf(GlobalContext, err)
|
||||
}
|
||||
return nil
|
||||
return nil, part > 0
|
||||
}
|
||||
data = decrypted
|
||||
}
|
||||
|
||||
@@ -362,7 +362,6 @@ func TestGetDecryptedRange(t *testing.T) {
|
||||
t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Multipart object tests
|
||||
@@ -538,7 +537,6 @@ func TestGetDecryptedRange(t *testing.T) {
|
||||
i, o, l, skip, sn, ps, oRef, lRef, skipRef, snRef, psRef)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+3
-4
@@ -26,6 +26,7 @@ import (
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -38,7 +39,6 @@ import (
|
||||
"github.com/minio/minio/internal/mountinfo"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
// EndpointType - enum for endpoint type.
|
||||
@@ -213,7 +213,6 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
|
||||
u.Path = u.Path[1:]
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
// Only check if the arg is an ip address and ask for scheme since its absent.
|
||||
// localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as
|
||||
@@ -1194,7 +1193,7 @@ func GetProxyEndpointLocalIndex(proxyEps []ProxyEndpoint) int {
|
||||
}
|
||||
|
||||
// GetProxyEndpoints - get all endpoints that can be used to proxy list request.
|
||||
func GetProxyEndpoints(endpointServerPools EndpointServerPools) []ProxyEndpoint {
|
||||
func GetProxyEndpoints(endpointServerPools EndpointServerPools, transport http.RoundTripper) []ProxyEndpoint {
|
||||
var proxyEps []ProxyEndpoint
|
||||
|
||||
proxyEpSet := set.NewStringSet()
|
||||
@@ -1213,7 +1212,7 @@ func GetProxyEndpoints(endpointServerPools EndpointServerPools) []ProxyEndpoint
|
||||
|
||||
proxyEps = append(proxyEps, ProxyEndpoint{
|
||||
Endpoint: endpoint,
|
||||
Transport: globalRemoteTargetTransport,
|
||||
Transport: transport,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -201,7 +201,6 @@ func erasureSelfTest() {
|
||||
ok = false
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
|
||||
@@ -277,23 +277,21 @@ func partNeedsHealing(partErrs []int) bool {
|
||||
return slices.IndexFunc(partErrs, func(i int) bool { return i != checkPartSuccess && i != checkPartUnknown }) > -1
|
||||
}
|
||||
|
||||
func hasPartErr(partErrs []int) bool {
|
||||
return slices.IndexFunc(partErrs, func(i int) bool { return i != checkPartSuccess }) > -1
|
||||
func countPartNotSuccess(partErrs []int) (c int) {
|
||||
for _, pe := range partErrs {
|
||||
if pe != checkPartSuccess {
|
||||
c++
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// disksWithAllParts - This function needs to be called with
|
||||
// []StorageAPI returned by listOnlineDisks. Returns,
|
||||
//
|
||||
// - disks which have all parts specified in the latest xl.meta.
|
||||
//
|
||||
// - slice of errors about the state of data files on disk - can have
|
||||
// a not-found error or a hash-mismatch error.
|
||||
func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []FileInfo,
|
||||
// checkObjectWithAllParts sets partsMetadata and onlineDisks when xl.meta is inexistant/corrupted or outdated
|
||||
// it also checks if the status of each part (corrupted, missing, ok) in each drive
|
||||
func checkObjectWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []FileInfo,
|
||||
errs []error, latestMeta FileInfo, filterByETag bool, bucket, object string,
|
||||
scanMode madmin.HealScanMode,
|
||||
) (availableDisks []StorageAPI, dataErrsByDisk map[int][]int, dataErrsByPart map[int][]int) {
|
||||
availableDisks = make([]StorageAPI, len(onlineDisks))
|
||||
|
||||
) (dataErrsByDisk map[int][]int, dataErrsByPart map[int][]int) {
|
||||
dataErrsByDisk = make(map[int][]int, len(onlineDisks))
|
||||
for i := range onlineDisks {
|
||||
dataErrsByDisk[i] = make([]int, len(latestMeta.Parts))
|
||||
@@ -334,12 +332,12 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
|
||||
|
||||
metaErrs := make([]error, len(errs))
|
||||
|
||||
for i, onlineDisk := range onlineDisks {
|
||||
for i := range onlineDisks {
|
||||
if errs[i] != nil {
|
||||
metaErrs[i] = errs[i]
|
||||
continue
|
||||
}
|
||||
if onlineDisk == OfflineDisk {
|
||||
if onlineDisks[i] == OfflineDisk {
|
||||
metaErrs[i] = errDiskNotFound
|
||||
continue
|
||||
}
|
||||
@@ -355,6 +353,7 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
|
||||
if corrupted {
|
||||
metaErrs[i] = errFileCorrupt
|
||||
partsMetadata[i] = FileInfo{}
|
||||
onlineDisks[i] = nil
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -362,6 +361,7 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
|
||||
if !meta.IsValid() {
|
||||
partsMetadata[i] = FileInfo{}
|
||||
metaErrs[i] = errFileCorrupt
|
||||
onlineDisks[i] = nil
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -372,6 +372,7 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
|
||||
// might have the right erasure distribution.
|
||||
partsMetadata[i] = FileInfo{}
|
||||
metaErrs[i] = errFileCorrupt
|
||||
onlineDisks[i] = nil
|
||||
continue
|
||||
}
|
||||
}
|
||||
@@ -440,20 +441,5 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
|
||||
dataErrsByDisk[disk][part] = dataErrsByPart[part][disk]
|
||||
}
|
||||
}
|
||||
|
||||
for i, onlineDisk := range onlineDisks {
|
||||
if metaErrs[i] == nil {
|
||||
meta := partsMetadata[i]
|
||||
if meta.Deleted || meta.IsRemote() || !hasPartErr(dataErrsByDisk[i]) {
|
||||
// All parts verified, mark it as all data available.
|
||||
availableDisks[i] = onlineDisk
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// upon errors just make that disk's fileinfo invalid
|
||||
partsMetadata[i] = FileInfo{}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -304,7 +304,6 @@ func TestListOnlineDisks(t *testing.T) {
|
||||
f.Close()
|
||||
break
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
rQuorum := len(errs) - z.serverPools[0].sets[0].defaultParityCount
|
||||
@@ -313,11 +312,11 @@ func TestListOnlineDisks(t *testing.T) {
|
||||
t.Fatalf("Expected modTime to be equal to %v but was found to be %v",
|
||||
test.expectedTime, modTime)
|
||||
}
|
||||
availableDisks, _, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
|
||||
_, _ = checkObjectWithAllParts(ctx, onlineDisks, partsMetadata,
|
||||
test.errs, fi, false, bucket, object, madmin.HealDeepScan)
|
||||
|
||||
if test._tamperBackend != noTamper {
|
||||
if tamperedIndex != -1 && availableDisks[tamperedIndex] != nil {
|
||||
if tamperedIndex != -1 && onlineDisks[tamperedIndex] != nil {
|
||||
t.Fatalf("Drive (%v) with part.1 missing is not a drive with available data",
|
||||
erasureDisks[tamperedIndex])
|
||||
}
|
||||
@@ -485,7 +484,6 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
|
||||
f.Close()
|
||||
break
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
rQuorum := len(errs) - z.serverPools[0].sets[0].defaultParityCount
|
||||
@@ -495,11 +493,11 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
|
||||
test.expectedTime, modTime)
|
||||
}
|
||||
|
||||
availableDisks, _, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
|
||||
_, _ = checkObjectWithAllParts(ctx, onlineDisks, partsMetadata,
|
||||
test.errs, fi, false, bucket, object, madmin.HealDeepScan)
|
||||
|
||||
if test._tamperBackend != noTamper {
|
||||
if tamperedIndex != -1 && availableDisks[tamperedIndex] != nil {
|
||||
if tamperedIndex != -1 && onlineDisks[tamperedIndex] != nil {
|
||||
t.Fatalf("Drive (%v) with part.1 missing is not a drive with available data",
|
||||
erasureDisks[tamperedIndex])
|
||||
}
|
||||
@@ -545,6 +543,7 @@ func TestDisksWithAllParts(t *testing.T) {
|
||||
|
||||
// Test 1: Test that all disks are returned without any failures with
|
||||
// unmodified meta data
|
||||
erasureDisks = s.getDisks()
|
||||
partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read xl meta data %v", err)
|
||||
@@ -557,14 +556,10 @@ func TestDisksWithAllParts(t *testing.T) {
|
||||
|
||||
erasureDisks, _, _ = listOnlineDisks(erasureDisks, partsMetadata, errs, readQuorum)
|
||||
|
||||
filteredDisks, _, dataErrsPerDisk := disksWithAllParts(ctx, erasureDisks, partsMetadata,
|
||||
dataErrsPerDisk, _ := checkObjectWithAllParts(ctx, erasureDisks, partsMetadata,
|
||||
errs, fi, false, bucket, object, madmin.HealDeepScan)
|
||||
|
||||
if len(filteredDisks) != len(erasureDisks) {
|
||||
t.Errorf("Unexpected number of drives: %d", len(filteredDisks))
|
||||
}
|
||||
|
||||
for diskIndex, disk := range filteredDisks {
|
||||
for diskIndex, disk := range erasureDisks {
|
||||
if partNeedsHealing(dataErrsPerDisk[diskIndex]) {
|
||||
t.Errorf("Unexpected error: %v", dataErrsPerDisk[diskIndex])
|
||||
}
|
||||
@@ -575,17 +570,15 @@ func TestDisksWithAllParts(t *testing.T) {
|
||||
}
|
||||
|
||||
// Test 2: Not synchronized modtime
|
||||
erasureDisks = s.getDisks()
|
||||
partsMetadataBackup := partsMetadata[0]
|
||||
partsMetadata[0].ModTime = partsMetadata[0].ModTime.Add(-1 * time.Hour)
|
||||
|
||||
errs = make([]error, len(erasureDisks))
|
||||
filteredDisks, _, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
|
||||
_, _ = checkObjectWithAllParts(ctx, erasureDisks, partsMetadata,
|
||||
errs, fi, false, bucket, object, madmin.HealDeepScan)
|
||||
|
||||
if len(filteredDisks) != len(erasureDisks) {
|
||||
t.Errorf("Unexpected number of drives: %d", len(filteredDisks))
|
||||
}
|
||||
for diskIndex, disk := range filteredDisks {
|
||||
for diskIndex, disk := range erasureDisks {
|
||||
if diskIndex == 0 && disk != nil {
|
||||
t.Errorf("Drive not filtered as expected, drive: %d", diskIndex)
|
||||
}
|
||||
@@ -596,17 +589,15 @@ func TestDisksWithAllParts(t *testing.T) {
|
||||
partsMetadata[0] = partsMetadataBackup // Revert before going to the next test
|
||||
|
||||
// Test 3: Not synchronized DataDir
|
||||
erasureDisks = s.getDisks()
|
||||
partsMetadataBackup = partsMetadata[1]
|
||||
partsMetadata[1].DataDir = "foo-random"
|
||||
|
||||
errs = make([]error, len(erasureDisks))
|
||||
filteredDisks, _, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
|
||||
_, _ = checkObjectWithAllParts(ctx, erasureDisks, partsMetadata,
|
||||
errs, fi, false, bucket, object, madmin.HealDeepScan)
|
||||
|
||||
if len(filteredDisks) != len(erasureDisks) {
|
||||
t.Errorf("Unexpected number of drives: %d", len(filteredDisks))
|
||||
}
|
||||
for diskIndex, disk := range filteredDisks {
|
||||
for diskIndex, disk := range erasureDisks {
|
||||
if diskIndex == 1 && disk != nil {
|
||||
t.Errorf("Drive not filtered as expected, drive: %d", diskIndex)
|
||||
}
|
||||
@@ -617,6 +608,7 @@ func TestDisksWithAllParts(t *testing.T) {
|
||||
partsMetadata[1] = partsMetadataBackup // Revert before going to the next test
|
||||
|
||||
// Test 4: key = disk index, value = part name with hash mismatch
|
||||
erasureDisks = s.getDisks()
|
||||
diskFailures := make(map[int]string)
|
||||
diskFailures[0] = "part.1"
|
||||
diskFailures[3] = "part.1"
|
||||
@@ -637,29 +629,18 @@ func TestDisksWithAllParts(t *testing.T) {
|
||||
}
|
||||
|
||||
errs = make([]error, len(erasureDisks))
|
||||
filteredDisks, dataErrsPerDisk, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
|
||||
dataErrsPerDisk, _ = checkObjectWithAllParts(ctx, erasureDisks, partsMetadata,
|
||||
errs, fi, false, bucket, object, madmin.HealDeepScan)
|
||||
|
||||
if len(filteredDisks) != len(erasureDisks) {
|
||||
t.Errorf("Unexpected number of drives: %d", len(filteredDisks))
|
||||
}
|
||||
|
||||
for diskIndex, disk := range filteredDisks {
|
||||
for diskIndex := range erasureDisks {
|
||||
if _, ok := diskFailures[diskIndex]; ok {
|
||||
if disk != nil {
|
||||
t.Errorf("Drive not filtered as expected, drive: %d", diskIndex)
|
||||
}
|
||||
if !partNeedsHealing(dataErrsPerDisk[diskIndex]) {
|
||||
t.Errorf("Disk expected to be healed, driveIndex: %d", diskIndex)
|
||||
}
|
||||
} else {
|
||||
if disk == nil {
|
||||
t.Errorf("Drive erroneously filtered, driveIndex: %d", diskIndex)
|
||||
}
|
||||
if partNeedsHealing(dataErrsPerDisk[diskIndex]) {
|
||||
t.Errorf("Disk not expected to be healed, driveIndex: %d", diskIndex)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+100
-80
@@ -32,7 +32,7 @@ import (
|
||||
"github.com/minio/minio/internal/grid"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"golang.org/x/exp/slices"
|
||||
"github.com/puzpuzpuz/xsync/v3"
|
||||
)
|
||||
|
||||
//go:generate stringer -type=healingMetric -trimprefix=healingMetric $GOFILE
|
||||
@@ -45,7 +45,8 @@ const (
|
||||
healingMetricCheckAbandonedParts
|
||||
)
|
||||
|
||||
func (er erasureObjects) listAndHeal(ctx context.Context, bucket, prefix string, scanMode madmin.HealScanMode, healEntry func(string, metaCacheEntry, madmin.HealScanMode) error) error {
|
||||
// List a prefix or a single object versions and heal
|
||||
func (er erasureObjects) listAndHeal(ctx context.Context, bucket, prefix string, recursive bool, scanMode madmin.HealScanMode, healEntry func(string, metaCacheEntry, madmin.HealScanMode) error) error {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
@@ -78,11 +79,14 @@ func (er erasureObjects) listAndHeal(ctx context.Context, bucket, prefix string,
|
||||
bucket: bucket,
|
||||
path: path,
|
||||
filterPrefix: filterPrefix,
|
||||
recursive: true,
|
||||
recursive: recursive,
|
||||
forwardTo: "",
|
||||
minDisks: 1,
|
||||
reportNotFound: false,
|
||||
agreed: func(entry metaCacheEntry) {
|
||||
if !recursive && prefix != entry.name {
|
||||
return
|
||||
}
|
||||
if err := healEntry(bucket, entry, scanMode); err != nil {
|
||||
cancel()
|
||||
}
|
||||
@@ -94,7 +98,9 @@ func (er erasureObjects) listAndHeal(ctx context.Context, bucket, prefix string,
|
||||
// proceed to heal nonetheless.
|
||||
entry, _ = entries.firstFound()
|
||||
}
|
||||
|
||||
if !recursive && prefix != entry.name {
|
||||
return
|
||||
}
|
||||
if err := healEntry(bucket, *entry, scanMode); err != nil {
|
||||
cancel()
|
||||
return
|
||||
@@ -112,9 +118,8 @@ func (er erasureObjects) listAndHeal(ctx context.Context, bucket, prefix string,
|
||||
|
||||
// listAllBuckets lists all buckets from all disks. It also
|
||||
// returns the occurrence of each buckets in all disks
|
||||
func listAllBuckets(ctx context.Context, storageDisks []StorageAPI, healBuckets map[string]VolInfo, readQuorum int) error {
|
||||
func listAllBuckets(ctx context.Context, storageDisks []StorageAPI, healBuckets *xsync.MapOf[string, VolInfo], readQuorum int) error {
|
||||
g := errgroup.WithNErrs(len(storageDisks))
|
||||
var mu sync.Mutex
|
||||
for index := range storageDisks {
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
@@ -126,6 +131,7 @@ func listAllBuckets(ctx context.Context, storageDisks []StorageAPI, healBuckets
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, volInfo := range volsInfo {
|
||||
// StorageAPI can send volume names which are
|
||||
// incompatible with buckets - these are
|
||||
@@ -133,53 +139,75 @@ func listAllBuckets(ctx context.Context, storageDisks []StorageAPI, healBuckets
|
||||
if isReservedOrInvalidBucket(volInfo.Name, false) {
|
||||
continue
|
||||
}
|
||||
mu.Lock()
|
||||
if _, ok := healBuckets[volInfo.Name]; !ok {
|
||||
healBuckets[volInfo.Name] = volInfo
|
||||
}
|
||||
mu.Unlock()
|
||||
|
||||
healBuckets.Compute(volInfo.Name, func(oldValue VolInfo, loaded bool) (newValue VolInfo, del bool) {
|
||||
if loaded {
|
||||
newValue = oldValue
|
||||
newValue.count = oldValue.count + 1
|
||||
return newValue, false
|
||||
}
|
||||
return VolInfo{
|
||||
Name: volInfo.Name,
|
||||
Created: volInfo.Created,
|
||||
count: 1,
|
||||
}, false
|
||||
})
|
||||
}
|
||||
|
||||
return nil
|
||||
}, index)
|
||||
}
|
||||
return reduceReadQuorumErrs(ctx, g.Wait(), bucketMetadataOpIgnoredErrs, readQuorum)
|
||||
|
||||
if err := reduceReadQuorumErrs(ctx, g.Wait(), bucketMetadataOpIgnoredErrs, readQuorum); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
healBuckets.Range(func(volName string, volInfo VolInfo) bool {
|
||||
if volInfo.count < readQuorum {
|
||||
healBuckets.Delete(volName)
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var errLegacyXLMeta = errors.New("legacy XL meta")
|
||||
|
||||
var errOutdatedXLMeta = errors.New("outdated XL meta")
|
||||
|
||||
var errPartMissingOrCorrupt = errors.New("part missing or corrupt")
|
||||
var (
|
||||
errLegacyXLMeta = errors.New("legacy XL meta")
|
||||
errOutdatedXLMeta = errors.New("outdated XL meta")
|
||||
errPartCorrupt = errors.New("part corrupt")
|
||||
errPartMissing = errors.New("part missing")
|
||||
)
|
||||
|
||||
// Only heal on disks where we are sure that healing is needed. We can expand
|
||||
// this list as and when we figure out more errors can be added to this list safely.
|
||||
func shouldHealObjectOnDisk(erErr error, partsErrs []int, meta FileInfo, latestMeta FileInfo) (bool, error) {
|
||||
func shouldHealObjectOnDisk(erErr error, partsErrs []int, meta FileInfo, latestMeta FileInfo) (bool, bool, error) {
|
||||
if errors.Is(erErr, errFileNotFound) || errors.Is(erErr, errFileVersionNotFound) || errors.Is(erErr, errFileCorrupt) {
|
||||
return true, erErr
|
||||
return true, true, erErr
|
||||
}
|
||||
if erErr == nil {
|
||||
if meta.XLV1 {
|
||||
// Legacy means heal always
|
||||
// always check first.
|
||||
return true, errLegacyXLMeta
|
||||
return true, true, errLegacyXLMeta
|
||||
}
|
||||
if !latestMeta.Equals(meta) {
|
||||
return true, errOutdatedXLMeta
|
||||
return true, true, errOutdatedXLMeta
|
||||
}
|
||||
if !meta.Deleted && !meta.IsRemote() {
|
||||
// If xl.meta was read fine but there may be problem with the part.N files.
|
||||
for _, partErr := range partsErrs {
|
||||
if slices.Contains([]int{
|
||||
checkPartFileNotFound,
|
||||
checkPartFileCorrupt,
|
||||
}, partErr) {
|
||||
return true, errPartMissingOrCorrupt
|
||||
if partErr == checkPartFileNotFound {
|
||||
return true, false, errPartMissing
|
||||
}
|
||||
if partErr == checkPartFileCorrupt {
|
||||
return true, false, errPartCorrupt
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
return false, false, nil
|
||||
}
|
||||
return false, erErr
|
||||
return false, false, erErr
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -254,6 +282,21 @@ func (er *erasureObjects) auditHealObject(ctx context.Context, bucket, object, v
|
||||
auditLogInternal(ctx, opts)
|
||||
}
|
||||
|
||||
func objectErrToDriveState(reason error) string {
|
||||
switch {
|
||||
case reason == nil:
|
||||
return madmin.DriveStateOk
|
||||
case IsErr(reason, errDiskNotFound):
|
||||
return madmin.DriveStateOffline
|
||||
case IsErr(reason, errFileNotFound, errFileVersionNotFound, errVolumeNotFound, errPartMissing, errOutdatedXLMeta, errLegacyXLMeta):
|
||||
return madmin.DriveStateMissing
|
||||
case IsErr(reason, errFileCorrupt, errPartCorrupt):
|
||||
return madmin.DriveStateCorrupt
|
||||
default:
|
||||
return fmt.Sprintf("%s (%s)", madmin.DriveStateUnknown, reason.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// Heals an object by re-writing corrupt/missing erasure blocks.
|
||||
func (er *erasureObjects) healObject(ctx context.Context, bucket string, object string, versionID string, opts madmin.HealOpts) (result madmin.HealResultItem, err error) {
|
||||
dryRun := opts.DryRun
|
||||
@@ -340,16 +383,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
|
||||
// No modtime quorum
|
||||
filterDisksByETag := quorumETag != ""
|
||||
|
||||
// List of disks having all parts as per latest metadata.
|
||||
// NOTE: do not pass in latestDisks to diskWithAllParts since
|
||||
// the diskWithAllParts needs to reach the drive to ensure
|
||||
// validity of the metadata content, we should make sure that
|
||||
// we pass in disks as is for it to be verified. Once verified
|
||||
// the disksWithAllParts() returns the actual disks that can be
|
||||
// used here for reconstruction. This is done to ensure that
|
||||
// we do not skip drives that have inconsistent metadata to be
|
||||
// skipped from purging when they are stale.
|
||||
availableDisks, dataErrsByDisk, dataErrsByPart := disksWithAllParts(ctx, onlineDisks, partsMetadata,
|
||||
dataErrsByDisk, dataErrsByPart := checkObjectWithAllParts(ctx, onlineDisks, partsMetadata,
|
||||
errs, latestMeta, filterDisksByETag, bucket, object, scanMode)
|
||||
|
||||
var erasure Erasure
|
||||
@@ -371,26 +405,18 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
|
||||
// data state and a list of outdated disks on which data needs
|
||||
// to be healed.
|
||||
outDatedDisks := make([]StorageAPI, len(storageDisks))
|
||||
disksToHealCount := 0
|
||||
for i := range availableDisks {
|
||||
yes, reason := shouldHealObjectOnDisk(errs[i], dataErrsByDisk[i], partsMetadata[i], latestMeta)
|
||||
disksToHealCount, xlMetaToHealCount := 0, 0
|
||||
for i := range onlineDisks {
|
||||
yes, isMeta, reason := shouldHealObjectOnDisk(errs[i], dataErrsByDisk[i], partsMetadata[i], latestMeta)
|
||||
if yes {
|
||||
outDatedDisks[i] = storageDisks[i]
|
||||
disksToHealCount++
|
||||
if isMeta {
|
||||
xlMetaToHealCount++
|
||||
}
|
||||
}
|
||||
|
||||
driveState := ""
|
||||
switch {
|
||||
case reason == nil:
|
||||
driveState = madmin.DriveStateOk
|
||||
case IsErr(reason, errDiskNotFound):
|
||||
driveState = madmin.DriveStateOffline
|
||||
case IsErr(reason, errFileNotFound, errFileVersionNotFound, errVolumeNotFound, errPartMissingOrCorrupt, errOutdatedXLMeta, errLegacyXLMeta):
|
||||
driveState = madmin.DriveStateMissing
|
||||
default:
|
||||
// all remaining cases imply corrupt data/metadata
|
||||
driveState = madmin.DriveStateCorrupt
|
||||
}
|
||||
driveState := objectErrToDriveState(reason)
|
||||
|
||||
result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
|
||||
UUID: "",
|
||||
@@ -404,16 +430,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
|
||||
})
|
||||
}
|
||||
|
||||
if isAllNotFound(errs) {
|
||||
// File is fully gone, fileInfo is empty.
|
||||
err := errFileNotFound
|
||||
if versionID != "" {
|
||||
err = errFileVersionNotFound
|
||||
}
|
||||
return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints, errs,
|
||||
bucket, object, versionID), err
|
||||
}
|
||||
|
||||
if disksToHealCount == 0 {
|
||||
// Nothing to heal!
|
||||
return result, nil
|
||||
@@ -425,13 +441,23 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
|
||||
return result, nil
|
||||
}
|
||||
|
||||
cannotHeal := !latestMeta.XLV1 && !latestMeta.Deleted && disksToHealCount > latestMeta.Erasure.ParityBlocks
|
||||
cannotHeal := !latestMeta.XLV1 && !latestMeta.Deleted && xlMetaToHealCount > latestMeta.Erasure.ParityBlocks
|
||||
if cannotHeal && quorumETag != "" {
|
||||
// This is an object that is supposed to be removed by the dangling code
|
||||
// but we noticed that ETag is the same for all objects, let's give it a shot
|
||||
cannotHeal = false
|
||||
}
|
||||
|
||||
if !latestMeta.Deleted && !latestMeta.IsRemote() {
|
||||
// check if there is a part that lost its quorum
|
||||
for _, partErrs := range dataErrsByPart {
|
||||
if countPartNotSuccess(partErrs) > latestMeta.Erasure.ParityBlocks {
|
||||
cannotHeal = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if cannotHeal {
|
||||
// Allow for dangling deletes, on versions that have DataDir missing etc.
|
||||
// this would end up restoring the correct readable versions.
|
||||
@@ -470,16 +496,15 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
|
||||
tmpID := mustGetUUID()
|
||||
migrateDataDir := mustGetUUID()
|
||||
|
||||
// Reorder so that we have data disks first and parity disks next.
|
||||
if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(availableDisks) {
|
||||
err := fmt.Errorf("unexpected file distribution (%v) from available disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
|
||||
latestMeta.Erasure.Distribution, availableDisks, bucket, object, versionID)
|
||||
healingLogOnceIf(ctx, err, "heal-object-available-disks")
|
||||
if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(onlineDisks) {
|
||||
err := fmt.Errorf("unexpected file distribution (%v) from online disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
|
||||
latestMeta.Erasure.Distribution, onlineDisks, bucket, object, versionID)
|
||||
healingLogOnceIf(ctx, err, "heal-object-online-disks")
|
||||
return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
|
||||
bucket, object, versionID), err
|
||||
}
|
||||
|
||||
latestDisks := shuffleDisks(availableDisks, latestMeta.Erasure.Distribution)
|
||||
latestDisks := shuffleDisks(onlineDisks, latestMeta.Erasure.Distribution)
|
||||
|
||||
if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(outDatedDisks) {
|
||||
err := fmt.Errorf("unexpected file distribution (%v) from outdated disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
|
||||
@@ -550,12 +575,15 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
|
||||
if disk == OfflineDisk {
|
||||
continue
|
||||
}
|
||||
thisPartErrs := shuffleCheckParts(dataErrsByPart[partIndex], latestMeta.Erasure.Distribution)
|
||||
if thisPartErrs[i] != checkPartSuccess {
|
||||
continue
|
||||
}
|
||||
checksumInfo := copyPartsMetadata[i].Erasure.GetChecksumInfo(partNumber)
|
||||
partPath := pathJoin(object, srcDataDir, fmt.Sprintf("part.%d", partNumber))
|
||||
readers[i] = newBitrotReader(disk, copyPartsMetadata[i].Data, bucket, partPath, tillOffset, checksumAlgo,
|
||||
checksumInfo.Hash, erasure.ShardSize())
|
||||
prefer[i] = disk.Hostname() == ""
|
||||
|
||||
}
|
||||
writers := make([]io.Writer, len(outDatedDisks))
|
||||
for i, disk := range outDatedDisks {
|
||||
@@ -614,9 +642,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
|
||||
if disksToHealCount == 0 {
|
||||
return result, fmt.Errorf("all drives had write errors, unable to heal %s/%s", bucket, object)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
defer er.deleteAll(context.Background(), minioMetaTmpBucket, tmpID)
|
||||
@@ -817,13 +843,7 @@ func (er *erasureObjects) defaultHealResult(lfi FileInfo, storageDisks []Storage
|
||||
})
|
||||
continue
|
||||
}
|
||||
driveState := madmin.DriveStateCorrupt
|
||||
switch errs[index] {
|
||||
case errFileNotFound, errVolumeNotFound:
|
||||
driveState = madmin.DriveStateMissing
|
||||
case nil:
|
||||
driveState = madmin.DriveStateOk
|
||||
}
|
||||
driveState := objectErrToDriveState(errs[index])
|
||||
result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
|
||||
UUID: "",
|
||||
Endpoint: storageEndpoints[index].String(),
|
||||
|
||||
@@ -733,7 +733,7 @@ func TestHealingDanglingObject(t *testing.T) {
|
||||
t.Fatalf("Expected versions 1, got %d", fileInfoPreHeal.NumVersions)
|
||||
}
|
||||
|
||||
if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Remove: true},
|
||||
if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Recursive: true, Remove: true},
|
||||
func(bucket, object, vid string, scanMode madmin.HealScanMode) error {
|
||||
_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{ScanMode: scanMode, Remove: true})
|
||||
return err
|
||||
@@ -780,7 +780,7 @@ func TestHealingDanglingObject(t *testing.T) {
|
||||
t.Fatalf("Expected versions 1, got %d", fileInfoPreHeal.NumVersions)
|
||||
}
|
||||
|
||||
if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Remove: true},
|
||||
if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Recursive: true, Remove: true},
|
||||
func(bucket, object, vid string, scanMode madmin.HealScanMode) error {
|
||||
_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{ScanMode: scanMode, Remove: true})
|
||||
return err
|
||||
@@ -829,7 +829,7 @@ func TestHealingDanglingObject(t *testing.T) {
|
||||
t.Fatalf("Expected versions 3, got %d", fileInfoPreHeal.NumVersions)
|
||||
}
|
||||
|
||||
if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Remove: true},
|
||||
if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Recursive: true, Remove: true},
|
||||
func(bucket, object, vid string, scanMode madmin.HealScanMode) error {
|
||||
_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{ScanMode: scanMode, Remove: true})
|
||||
return err
|
||||
|
||||
@@ -52,15 +52,15 @@ func reduceCommonVersions(diskVersions [][]byte, writeQuorum int) (versions []by
|
||||
}
|
||||
|
||||
var commonVersions uint64
|
||||
max := 0
|
||||
maxCnt := 0
|
||||
for versions, count := range diskVersionsCount {
|
||||
if max < count {
|
||||
max = count
|
||||
if maxCnt < count {
|
||||
maxCnt = count
|
||||
commonVersions = versions
|
||||
}
|
||||
}
|
||||
|
||||
if max >= writeQuorum {
|
||||
if maxCnt >= writeQuorum {
|
||||
for _, versions := range diskVersions {
|
||||
if binary.BigEndian.Uint64(versions) == commonVersions {
|
||||
return versions
|
||||
@@ -80,15 +80,15 @@ func reduceCommonDataDir(dataDirs []string, writeQuorum int) (dataDir string) {
|
||||
dataDirsCount[ddir]++
|
||||
}
|
||||
|
||||
max := 0
|
||||
maxCnt := 0
|
||||
for ddir, count := range dataDirsCount {
|
||||
if max < count {
|
||||
max = count
|
||||
if maxCnt < count {
|
||||
maxCnt = count
|
||||
dataDir = ddir
|
||||
}
|
||||
}
|
||||
|
||||
if max >= writeQuorum {
|
||||
if maxCnt >= writeQuorum {
|
||||
return dataDir
|
||||
}
|
||||
|
||||
@@ -115,20 +115,20 @@ func reduceErrs(errs []error, ignoredErrs []error) (maxCount int, maxErr error)
|
||||
errorCounts[err]++
|
||||
}
|
||||
|
||||
max := 0
|
||||
maxCnt := 0
|
||||
for err, count := range errorCounts {
|
||||
switch {
|
||||
case max < count:
|
||||
max = count
|
||||
case maxCnt < count:
|
||||
maxCnt = count
|
||||
maxErr = err
|
||||
|
||||
// Prefer `nil` over other error values with the same
|
||||
// number of occurrences.
|
||||
case max == count && err == nil:
|
||||
case maxCnt == count && err == nil:
|
||||
maxErr = err
|
||||
}
|
||||
}
|
||||
return max, maxErr
|
||||
return maxCnt, maxErr
|
||||
}
|
||||
|
||||
// reduceQuorumErrs behaves like reduceErrs by only for returning
|
||||
@@ -295,34 +295,34 @@ func shuffleDisksAndPartsMetadata(disks []StorageAPI, partsMetadata []FileInfo,
|
||||
return shuffledDisks, shuffledPartsMetadata
|
||||
}
|
||||
|
||||
// Return shuffled partsMetadata depending on distribution.
|
||||
func shufflePartsMetadata(partsMetadata []FileInfo, distribution []int) (shuffledPartsMetadata []FileInfo) {
|
||||
func shuffleWithDist[T any](input []T, distribution []int) []T {
|
||||
if distribution == nil {
|
||||
return partsMetadata
|
||||
return input
|
||||
}
|
||||
shuffledPartsMetadata = make([]FileInfo, len(partsMetadata))
|
||||
// Shuffle slice xl metadata for expected distribution.
|
||||
for index := range partsMetadata {
|
||||
shuffled := make([]T, len(input))
|
||||
for index := range input {
|
||||
blockIndex := distribution[index]
|
||||
shuffledPartsMetadata[blockIndex-1] = partsMetadata[index]
|
||||
shuffled[blockIndex-1] = input[index]
|
||||
}
|
||||
return shuffledPartsMetadata
|
||||
return shuffled
|
||||
}
|
||||
|
||||
// Return shuffled partsMetadata depending on distribution.
|
||||
func shufflePartsMetadata(partsMetadata []FileInfo, distribution []int) []FileInfo {
|
||||
return shuffleWithDist[FileInfo](partsMetadata, distribution)
|
||||
}
|
||||
|
||||
// shuffleCheckParts - shuffle CheckParts slice depending on the
|
||||
// erasure distribution.
|
||||
func shuffleCheckParts(parts []int, distribution []int) []int {
|
||||
return shuffleWithDist[int](parts, distribution)
|
||||
}
|
||||
|
||||
// shuffleDisks - shuffle input disks slice depending on the
|
||||
// erasure distribution. Return shuffled slice of disks with
|
||||
// their expected distribution.
|
||||
func shuffleDisks(disks []StorageAPI, distribution []int) (shuffledDisks []StorageAPI) {
|
||||
if distribution == nil {
|
||||
return disks
|
||||
}
|
||||
shuffledDisks = make([]StorageAPI, len(disks))
|
||||
// Shuffle disks for expected distribution.
|
||||
for index := range disks {
|
||||
blockIndex := distribution[index]
|
||||
shuffledDisks[blockIndex-1] = disks[index]
|
||||
}
|
||||
return shuffledDisks
|
||||
func shuffleDisks(disks []StorageAPI, distribution []int) []StorageAPI {
|
||||
return shuffleWithDist[StorageAPI](disks, distribution)
|
||||
}
|
||||
|
||||
// evalDisks - returns a new slice of disks where nil is set if
|
||||
|
||||
+75
-39
@@ -480,6 +480,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
|
||||
|
||||
if opts.WantChecksum != nil && opts.WantChecksum.Type.IsSet() {
|
||||
userDefined[hash.MinIOMultipartChecksum] = opts.WantChecksum.Type.String()
|
||||
userDefined[hash.MinIOMultipartChecksumType] = opts.WantChecksum.Type.ObjType()
|
||||
}
|
||||
|
||||
modTime := opts.MTime
|
||||
@@ -508,6 +509,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
|
||||
return &NewMultipartUploadResult{
|
||||
UploadID: uploadID,
|
||||
ChecksumAlgo: userDefined[hash.MinIOMultipartChecksum],
|
||||
ChecksumType: userDefined[hash.MinIOMultipartChecksumType],
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -626,17 +628,13 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
|
||||
switch size := data.Size(); {
|
||||
case size == 0:
|
||||
buffer = make([]byte, 1) // Allocate at least a byte to reach EOF
|
||||
case size == -1:
|
||||
if size := data.ActualSize(); size > 0 && size < fi.Erasure.BlockSize {
|
||||
// Account for padding and forced compression overhead and encryption.
|
||||
buffer = make([]byte, data.ActualSize()+256+32+32, data.ActualSize()*2+512)
|
||||
case size >= fi.Erasure.BlockSize || size == -1:
|
||||
if int64(globalBytePoolCap.Load().Width()) < fi.Erasure.BlockSize {
|
||||
buffer = make([]byte, fi.Erasure.BlockSize, 2*fi.Erasure.BlockSize)
|
||||
} else {
|
||||
buffer = globalBytePoolCap.Load().Get()
|
||||
defer globalBytePoolCap.Load().Put(buffer)
|
||||
}
|
||||
case size >= fi.Erasure.BlockSize:
|
||||
buffer = globalBytePoolCap.Load().Get()
|
||||
defer globalBytePoolCap.Load().Put(buffer)
|
||||
case size < fi.Erasure.BlockSize:
|
||||
// No need to allocate fully fi.Erasure.BlockSize buffer if the incoming data is smaller.
|
||||
buffer = make([]byte, size, 2*size+int64(fi.Erasure.ParityBlocks+fi.Erasure.DataBlocks-1))
|
||||
@@ -744,6 +742,15 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
|
||||
ctx = plkctx.Context()
|
||||
defer partIDLock.Unlock(plkctx)
|
||||
|
||||
// Read lock for upload id, only held while reading the upload metadata.
|
||||
uploadIDRLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
|
||||
rlkctx, err := uploadIDRLock.GetRLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
return PartInfo{}, err
|
||||
}
|
||||
ctx = rlkctx.Context()
|
||||
defer uploadIDRLock.RUnlock(rlkctx)
|
||||
|
||||
onlineDisks, err = er.renamePart(ctx, onlineDisks, minioMetaTmpBucket, tmpPartPath, minioMetaMultipartBucket, partPath, partFI, writeQuorum)
|
||||
if err != nil {
|
||||
if errors.Is(err, errFileNotFound) {
|
||||
@@ -760,15 +767,16 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
|
||||
|
||||
// Return success.
|
||||
return PartInfo{
|
||||
PartNumber: partInfo.Number,
|
||||
ETag: partInfo.ETag,
|
||||
LastModified: partInfo.ModTime,
|
||||
Size: partInfo.Size,
|
||||
ActualSize: partInfo.ActualSize,
|
||||
ChecksumCRC32: partInfo.Checksums["CRC32"],
|
||||
ChecksumCRC32C: partInfo.Checksums["CRC32C"],
|
||||
ChecksumSHA1: partInfo.Checksums["SHA1"],
|
||||
ChecksumSHA256: partInfo.Checksums["SHA256"],
|
||||
PartNumber: partInfo.Number,
|
||||
ETag: partInfo.ETag,
|
||||
LastModified: partInfo.ModTime,
|
||||
Size: partInfo.Size,
|
||||
ActualSize: partInfo.ActualSize,
|
||||
ChecksumCRC32: partInfo.Checksums["CRC32"],
|
||||
ChecksumCRC32C: partInfo.Checksums["CRC32C"],
|
||||
ChecksumSHA1: partInfo.Checksums["SHA1"],
|
||||
ChecksumSHA256: partInfo.Checksums["SHA256"],
|
||||
ChecksumCRC64NVME: partInfo.Checksums["CRC64NVME"],
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -890,6 +898,7 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
|
||||
result.PartNumberMarker = partNumberMarker
|
||||
result.UserDefined = cloneMSS(fi.Metadata)
|
||||
result.ChecksumAlgorithm = fi.Metadata[hash.MinIOMultipartChecksum]
|
||||
result.ChecksumType = fi.Metadata[hash.MinIOMultipartChecksumType]
|
||||
|
||||
if maxParts == 0 {
|
||||
return result, nil
|
||||
@@ -936,15 +945,16 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
|
||||
count := maxParts
|
||||
for _, objPart := range objParts {
|
||||
result.Parts = append(result.Parts, PartInfo{
|
||||
PartNumber: objPart.Number,
|
||||
LastModified: objPart.ModTime,
|
||||
ETag: objPart.ETag,
|
||||
Size: objPart.Size,
|
||||
ActualSize: objPart.ActualSize,
|
||||
ChecksumCRC32: objPart.Checksums["CRC32"],
|
||||
ChecksumCRC32C: objPart.Checksums["CRC32C"],
|
||||
ChecksumSHA1: objPart.Checksums["SHA1"],
|
||||
ChecksumSHA256: objPart.Checksums["SHA256"],
|
||||
PartNumber: objPart.Number,
|
||||
LastModified: objPart.ModTime,
|
||||
ETag: objPart.ETag,
|
||||
Size: objPart.Size,
|
||||
ActualSize: objPart.ActualSize,
|
||||
ChecksumCRC32: objPart.Checksums["CRC32"],
|
||||
ChecksumCRC32C: objPart.Checksums["CRC32C"],
|
||||
ChecksumSHA1: objPart.Checksums["SHA1"],
|
||||
ChecksumSHA256: objPart.Checksums["SHA256"],
|
||||
ChecksumCRC64NVME: objPart.Checksums["CRC64NVME"],
|
||||
})
|
||||
count--
|
||||
if count == 0 {
|
||||
@@ -1039,7 +1049,6 @@ func readParts(ctx context.Context, disks []StorageAPI, bucket string, partMetaP
|
||||
PartNumber: partNumbers[pidx],
|
||||
}.Error(),
|
||||
}
|
||||
|
||||
}
|
||||
return partInfosInQuorum, nil
|
||||
}
|
||||
@@ -1126,12 +1135,12 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
// Checksum type set when upload started.
|
||||
var checksumType hash.ChecksumType
|
||||
if cs := fi.Metadata[hash.MinIOMultipartChecksum]; cs != "" {
|
||||
checksumType = hash.NewChecksumType(cs)
|
||||
checksumType = hash.NewChecksumType(cs, fi.Metadata[hash.MinIOMultipartChecksumType])
|
||||
if opts.WantChecksum != nil && !opts.WantChecksum.Type.Is(checksumType) {
|
||||
return oi, InvalidArgument{
|
||||
Bucket: bucket,
|
||||
Object: fi.Name,
|
||||
Err: fmt.Errorf("checksum type mismatch"),
|
||||
Err: fmt.Errorf("checksum type mismatch. got %q (%s) expected %q (%s)", checksumType.String(), checksumType.ObjType(), opts.WantChecksum.Type.String(), opts.WantChecksum.Type.ObjType()),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1211,6 +1220,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
// Allocate parts similar to incoming slice.
|
||||
fi.Parts = make([]ObjectPartInfo, len(parts))
|
||||
|
||||
var checksum hash.Checksum
|
||||
checksum.Type = checksumType
|
||||
|
||||
// Validate each part and then commit to disk.
|
||||
for i, part := range parts {
|
||||
partIdx := objectPartIndex(currentFI.Parts, part.PartNumber)
|
||||
@@ -1244,10 +1256,11 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
}
|
||||
}
|
||||
wantCS := map[string]string{
|
||||
hash.ChecksumCRC32.String(): part.ChecksumCRC32,
|
||||
hash.ChecksumCRC32C.String(): part.ChecksumCRC32C,
|
||||
hash.ChecksumSHA1.String(): part.ChecksumSHA1,
|
||||
hash.ChecksumSHA256.String(): part.ChecksumSHA256,
|
||||
hash.ChecksumCRC32.String(): part.ChecksumCRC32,
|
||||
hash.ChecksumCRC32C.String(): part.ChecksumCRC32C,
|
||||
hash.ChecksumSHA1.String(): part.ChecksumSHA1,
|
||||
hash.ChecksumSHA256.String(): part.ChecksumSHA256,
|
||||
hash.ChecksumCRC64NVME.String(): part.ChecksumCRC64NVME,
|
||||
}
|
||||
if wantCS[checksumType.String()] != crc {
|
||||
return oi, InvalidPart{
|
||||
@@ -1262,6 +1275,15 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
PartNumber: part.PartNumber,
|
||||
}
|
||||
}
|
||||
if checksumType.FullObjectRequested() {
|
||||
if err := checksum.AddPart(*cs, expPart.ActualSize); err != nil {
|
||||
return oi, InvalidPart{
|
||||
PartNumber: part.PartNumber,
|
||||
ExpETag: "<nil>",
|
||||
GotETag: err.Error(),
|
||||
}
|
||||
}
|
||||
}
|
||||
checksumCombined = append(checksumCombined, cs.Raw...)
|
||||
}
|
||||
|
||||
@@ -1292,9 +1314,19 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
}
|
||||
|
||||
if opts.WantChecksum != nil {
|
||||
err := opts.WantChecksum.Matches(checksumCombined, len(parts))
|
||||
if err != nil {
|
||||
return oi, err
|
||||
if checksumType.FullObjectRequested() {
|
||||
if opts.WantChecksum.Encoded != checksum.Encoded {
|
||||
err := hash.ChecksumMismatch{
|
||||
Want: opts.WantChecksum.Encoded,
|
||||
Got: checksum.Encoded,
|
||||
}
|
||||
return oi, err
|
||||
}
|
||||
} else {
|
||||
err := opts.WantChecksum.Matches(checksumCombined, len(parts))
|
||||
if err != nil {
|
||||
return oi, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1308,14 +1340,18 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
|
||||
if checksumType.IsSet() {
|
||||
checksumType |= hash.ChecksumMultipart | hash.ChecksumIncludesMultipart
|
||||
var cs *hash.Checksum
|
||||
cs = hash.NewChecksumFromData(checksumType, checksumCombined)
|
||||
fi.Checksum = cs.AppendTo(nil, checksumCombined)
|
||||
checksum.Type = checksumType
|
||||
if !checksumType.FullObjectRequested() {
|
||||
checksum = *hash.NewChecksumFromData(checksumType, checksumCombined)
|
||||
}
|
||||
fi.Checksum = checksum.AppendTo(nil, checksumCombined)
|
||||
if opts.EncryptFn != nil {
|
||||
fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
|
||||
}
|
||||
}
|
||||
delete(fi.Metadata, hash.MinIOMultipartChecksum) // Not needed in final object.
|
||||
// Remove superfluous internal headers.
|
||||
delete(fi.Metadata, hash.MinIOMultipartChecksum)
|
||||
delete(fi.Metadata, hash.MinIOMultipartChecksumType)
|
||||
|
||||
// Save the final object size and modtime.
|
||||
fi.Size = objectSize
|
||||
|
||||
+24
-8
@@ -27,6 +27,8 @@ import (
|
||||
"net/http"
|
||||
"path"
|
||||
"runtime"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -1706,8 +1708,21 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
|
||||
}
|
||||
|
||||
dedupVersions := make([]FileInfoVersions, 0, len(versionsMap))
|
||||
for _, version := range versionsMap {
|
||||
dedupVersions = append(dedupVersions, version)
|
||||
for _, fivs := range versionsMap {
|
||||
// Removal of existing versions and adding a delete marker in the same
|
||||
// request is supported. At the same time, we cannot allow adding
|
||||
// two delete markers on top of any object. To avoid this situation,
|
||||
// we will sort deletions to execute existing deletion first,
|
||||
// then add only one delete marker if requested
|
||||
sort.SliceStable(fivs.Versions, func(i, j int) bool {
|
||||
return !fivs.Versions[i].Deleted
|
||||
})
|
||||
if idx := slices.IndexFunc(fivs.Versions, func(fi FileInfo) bool {
|
||||
return fi.Deleted
|
||||
}); idx > -1 {
|
||||
fivs.Versions = fivs.Versions[:idx+1]
|
||||
}
|
||||
dedupVersions = append(dedupVersions, fivs)
|
||||
}
|
||||
|
||||
// Initialize list of errors.
|
||||
@@ -1732,12 +1747,6 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
|
||||
continue
|
||||
}
|
||||
for _, v := range dedupVersions[i].Versions {
|
||||
if err == errFileNotFound || err == errFileVersionNotFound {
|
||||
if !dobjects[v.Idx].DeleteMarker {
|
||||
// Not delete marker, if not found, ok.
|
||||
continue
|
||||
}
|
||||
}
|
||||
delObjErrs[index][v.Idx] = err
|
||||
}
|
||||
}
|
||||
@@ -1757,6 +1766,13 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
|
||||
}
|
||||
}
|
||||
err := reduceWriteQuorumErrs(ctx, diskErrs, objectOpIgnoredErrs, writeQuorums[objIndex])
|
||||
if err == nil {
|
||||
dobjects[objIndex].found = true
|
||||
} else if isErrVersionNotFound(err) || isErrObjectNotFound(err) {
|
||||
if !dobjects[objIndex].DeleteMarker {
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
if objects[objIndex].VersionID != "" {
|
||||
errs[objIndex] = toObjectErr(err, bucket, objects[objIndex].ObjectName, objects[objIndex].VersionID)
|
||||
} else {
|
||||
|
||||
@@ -131,6 +131,75 @@ func TestErasureDeleteObjectBasic(t *testing.T) {
|
||||
removeRoots(fsDirs)
|
||||
}
|
||||
|
||||
func TestDeleteObjectsVersionedTwoPools(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
obj, fsDirs, err := prepareErasurePools()
|
||||
if err != nil {
|
||||
t.Fatal("Unable to initialize 'Erasure' object layer.", err)
|
||||
}
|
||||
// Remove all dirs.
|
||||
for _, dir := range fsDirs {
|
||||
defer os.RemoveAll(dir)
|
||||
}
|
||||
|
||||
bucketName := "bucket"
|
||||
objectName := "myobject"
|
||||
err = obj.MakeBucket(ctx, bucketName, MakeBucketOptions{
|
||||
VersioningEnabled: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
z, ok := obj.(*erasureServerPools)
|
||||
if !ok {
|
||||
t.Fatal("unexpected object layer type")
|
||||
}
|
||||
|
||||
versions := make([]string, 2)
|
||||
for i := range z.serverPools {
|
||||
objInfo, err := z.serverPools[i].PutObject(ctx, bucketName, objectName,
|
||||
mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{
|
||||
Versioned: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Erasure Object upload failed: <ERROR> %s", err)
|
||||
}
|
||||
versions[i] = objInfo.VersionID
|
||||
}
|
||||
|
||||
// Remove and check the version in the second pool, then
|
||||
// remove and check the version in the first pool
|
||||
for testIdx, vid := range []string{versions[1], versions[0]} {
|
||||
names := []ObjectToDelete{
|
||||
{
|
||||
ObjectV: ObjectV{
|
||||
ObjectName: objectName,
|
||||
VersionID: vid,
|
||||
},
|
||||
},
|
||||
}
|
||||
_, delErrs := obj.DeleteObjects(ctx, bucketName, names, ObjectOptions{
|
||||
Versioned: true,
|
||||
})
|
||||
for i := range delErrs {
|
||||
if delErrs[i] != nil {
|
||||
t.Errorf("Test %d: Failed to remove object `%v` with the error: `%v`", testIdx, names[i], delErrs[i])
|
||||
}
|
||||
_, statErr := obj.GetObjectInfo(ctx, bucketName, objectName, ObjectOptions{
|
||||
VersionID: names[i].ObjectV.VersionID,
|
||||
})
|
||||
switch statErr.(type) {
|
||||
case VersionNotFound:
|
||||
default:
|
||||
t.Errorf("Test %d: Object %s is not removed", testIdx, objectName)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteObjectsVersioned(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
@@ -177,7 +246,6 @@ func TestDeleteObjectsVersioned(t *testing.T) {
|
||||
VersionID: objInfo.VersionID,
|
||||
},
|
||||
}
|
||||
|
||||
}
|
||||
names = append(names, ObjectToDelete{
|
||||
ObjectV: ObjectV{
|
||||
|
||||
@@ -845,7 +845,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
|
||||
// to decommission, just skip it, this also includes
|
||||
// any other versions that have already expired.
|
||||
remainingVersions := len(fivs.Versions) - expired
|
||||
if version.Deleted && remainingVersions == 1 {
|
||||
if version.Deleted && remainingVersions == 1 && rcfg == nil {
|
||||
decommissioned++
|
||||
stopFn(version.Size, errors.New("DELETE marked object with no other non-current versions will be skipped"))
|
||||
continue
|
||||
|
||||
@@ -32,9 +32,9 @@ func prepareErasurePools() (ObjectLayer, []string, error) {
|
||||
pools := mustGetPoolEndpoints(0, fsDirs[:16]...)
|
||||
pools = append(pools, mustGetPoolEndpoints(1, fsDirs[16:]...)...)
|
||||
|
||||
// Everything is fine, should return nil
|
||||
objLayer, err := newErasureServerPools(context.Background(), pools)
|
||||
objLayer, _, err := initObjectLayer(context.Background(), pools)
|
||||
if err != nil {
|
||||
removeRoots(fsDirs)
|
||||
return nil, nil, err
|
||||
}
|
||||
return objLayer, fsDirs, nil
|
||||
|
||||
+42
-91
@@ -256,7 +256,7 @@ func (z *erasureServerPools) NewNSLock(bucket string, objects ...string) RWLocke
|
||||
|
||||
// GetDisksID will return disks by their ID.
|
||||
func (z *erasureServerPools) GetDisksID(ids ...string) []StorageAPI {
|
||||
idMap := make(map[string]struct{})
|
||||
idMap := make(map[string]struct{}, len(ids))
|
||||
for _, id := range ids {
|
||||
idMap[id] = struct{}{}
|
||||
}
|
||||
@@ -357,7 +357,7 @@ func (p serverPoolsAvailableSpace) TotalAvailable() uint64 {
|
||||
|
||||
// FilterMaxUsed will filter out any pools that has used percent bigger than max,
|
||||
// unless all have that, in which case all are preserved.
|
||||
func (p serverPoolsAvailableSpace) FilterMaxUsed(max int) {
|
||||
func (p serverPoolsAvailableSpace) FilterMaxUsed(maxUsed int) {
|
||||
// We aren't modifying p, only entries in it, so we don't need to receive a pointer.
|
||||
if len(p) <= 1 {
|
||||
// Nothing to do.
|
||||
@@ -365,7 +365,7 @@ func (p serverPoolsAvailableSpace) FilterMaxUsed(max int) {
|
||||
}
|
||||
var ok bool
|
||||
for _, z := range p {
|
||||
if z.Available > 0 && z.MaxUsedPct < max {
|
||||
if z.Available > 0 && z.MaxUsedPct < maxUsed {
|
||||
ok = true
|
||||
break
|
||||
}
|
||||
@@ -378,7 +378,7 @@ func (p serverPoolsAvailableSpace) FilterMaxUsed(max int) {
|
||||
|
||||
// Remove entries that are above.
|
||||
for i, z := range p {
|
||||
if z.Available > 0 && z.MaxUsedPct < max {
|
||||
if z.Available > 0 && z.MaxUsedPct < maxUsed {
|
||||
continue
|
||||
}
|
||||
p[i].Available = 0
|
||||
@@ -1248,89 +1248,42 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
|
||||
ctx = lkctx.Context()
|
||||
defer multiDeleteLock.Unlock(lkctx)
|
||||
|
||||
// Fetch location of up to 10 objects concurrently.
|
||||
poolObjIdxMap := map[int][]ObjectToDelete{}
|
||||
origIndexMap := map[int][]int{}
|
||||
dObjectsByPool := make([][]DeletedObject, len(z.serverPools))
|
||||
dErrsByPool := make([][]error, len(z.serverPools))
|
||||
|
||||
// Always perform 1/10th of the number of objects per delete
|
||||
concurrent := len(objects) / 10
|
||||
if concurrent <= 10 {
|
||||
// if we cannot get 1/10th then choose the number of
|
||||
// objects as concurrent.
|
||||
concurrent = len(objects)
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
eg := errgroup.WithNErrs(len(objects)).WithConcurrency(concurrent)
|
||||
for j, obj := range objects {
|
||||
j := j
|
||||
obj := obj
|
||||
eg := errgroup.WithNErrs(len(z.serverPools)).WithConcurrency(len(z.serverPools))
|
||||
for i, pool := range z.serverPools {
|
||||
i := i
|
||||
pool := pool
|
||||
eg.Go(func() error {
|
||||
pinfo, _, err := z.getPoolInfoExistingWithOpts(ctx, bucket, obj.ObjectName, ObjectOptions{
|
||||
NoLock: true,
|
||||
})
|
||||
if err != nil {
|
||||
if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
derrs[j] = err
|
||||
}
|
||||
dobjects[j] = DeletedObject{
|
||||
ObjectName: decodeDirObject(obj.ObjectName),
|
||||
VersionID: obj.VersionID,
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete marker already present we are not going to create new delete markers.
|
||||
if pinfo.ObjInfo.DeleteMarker && obj.VersionID == "" {
|
||||
dobjects[j] = DeletedObject{
|
||||
DeleteMarker: pinfo.ObjInfo.DeleteMarker,
|
||||
DeleteMarkerVersionID: pinfo.ObjInfo.VersionID,
|
||||
DeleteMarkerMTime: DeleteMarkerMTime{pinfo.ObjInfo.ModTime},
|
||||
ObjectName: decodeDirObject(pinfo.ObjInfo.Name),
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
idx := pinfo.Index
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
poolObjIdxMap[idx] = append(poolObjIdxMap[idx], obj)
|
||||
origIndexMap[idx] = append(origIndexMap[idx], j)
|
||||
dObjectsByPool[i], dErrsByPool[i] = pool.DeleteObjects(ctx, bucket, objects, opts)
|
||||
return nil
|
||||
}, j)
|
||||
}, i)
|
||||
}
|
||||
|
||||
eg.Wait() // wait to check all the pools.
|
||||
|
||||
if len(poolObjIdxMap) > 0 {
|
||||
// Delete concurrently in all server pools.
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(len(z.serverPools))
|
||||
for idx, pool := range z.serverPools {
|
||||
go func(idx int, pool *erasureSets) {
|
||||
defer wg.Done()
|
||||
objs := poolObjIdxMap[idx]
|
||||
if len(objs) == 0 {
|
||||
return
|
||||
}
|
||||
orgIndexes := origIndexMap[idx]
|
||||
deletedObjects, errs := pool.DeleteObjects(ctx, bucket, objs, opts)
|
||||
mu.Lock()
|
||||
for i, derr := range errs {
|
||||
if derr != nil {
|
||||
derrs[orgIndexes[i]] = derr
|
||||
}
|
||||
deletedObjects[i].ObjectName = decodeDirObject(deletedObjects[i].ObjectName)
|
||||
dobjects[orgIndexes[i]] = deletedObjects[i]
|
||||
}
|
||||
mu.Unlock()
|
||||
}(idx, pool)
|
||||
for i := range dobjects {
|
||||
// Iterate over pools
|
||||
for pool := range z.serverPools {
|
||||
if dErrsByPool[pool][i] == nil && dObjectsByPool[pool][i].found {
|
||||
// A fast exit when the object is found and removed
|
||||
dobjects[i] = dObjectsByPool[pool][i]
|
||||
derrs[i] = nil
|
||||
break
|
||||
}
|
||||
if derrs[i] == nil {
|
||||
// No error related to this object is found, assign this pool result
|
||||
// whether it is nil because there is no object found or because of
|
||||
// some other errors such erasure quorum errors.
|
||||
dobjects[i] = dObjectsByPool[pool][i]
|
||||
derrs[i] = dErrsByPool[pool][i]
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
for i := range dobjects {
|
||||
dobjects[i].ObjectName = decodeDirObject(dobjects[i].ObjectName)
|
||||
}
|
||||
return dobjects, derrs
|
||||
}
|
||||
|
||||
@@ -1858,16 +1811,6 @@ func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object,
|
||||
return PartInfo{}, err
|
||||
}
|
||||
|
||||
// Read lock for upload id.
|
||||
// Only held while reading the upload metadata.
|
||||
uploadIDRLock := z.NewNSLock(bucket, pathJoin(object, uploadID))
|
||||
rlkctx, err := uploadIDRLock.GetRLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
return PartInfo{}, err
|
||||
}
|
||||
ctx = rlkctx.Context()
|
||||
defer uploadIDRLock.RUnlock(rlkctx)
|
||||
|
||||
if z.SinglePool() {
|
||||
return z.serverPools[0].PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
|
||||
}
|
||||
@@ -2478,6 +2421,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
|
||||
// HealObjectFn closure function heals the object.
|
||||
type HealObjectFn func(bucket, object, versionID string, scanMode madmin.HealScanMode) error
|
||||
|
||||
// List a prefix or a single object versions and heal
|
||||
func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObjectFn HealObjectFn) error {
|
||||
healEntry := func(bucket string, entry metaCacheEntry, scanMode madmin.HealScanMode) error {
|
||||
if entry.isDir() {
|
||||
@@ -2541,7 +2485,7 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
|
||||
go func(idx int, set *erasureObjects) {
|
||||
defer wg.Done()
|
||||
|
||||
errs[idx] = set.listAndHeal(ctx, bucket, prefix, opts.ScanMode, healEntry)
|
||||
errs[idx] = set.listAndHeal(ctx, bucket, prefix, opts.Recursive, opts.ScanMode, healEntry)
|
||||
}(idx, set)
|
||||
}
|
||||
wg.Wait()
|
||||
@@ -2593,15 +2537,22 @@ func (z *erasureServerPools) HealObject(ctx context.Context, bucket, object, ver
|
||||
}
|
||||
}
|
||||
|
||||
hr := madmin.HealResultItem{
|
||||
Type: madmin.HealItemObject,
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
VersionID: versionID,
|
||||
}
|
||||
|
||||
// At this stage, all errors are 'not found'
|
||||
if versionID != "" {
|
||||
return madmin.HealResultItem{}, VersionNotFound{
|
||||
return hr, VersionNotFound{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
VersionID: versionID,
|
||||
}
|
||||
}
|
||||
return madmin.HealResultItem{}, ObjectNotFound{
|
||||
return hr, ObjectNotFound{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
}
|
||||
|
||||
+3
-7
@@ -39,6 +39,7 @@ import (
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/console"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/puzpuzpuz/xsync/v3"
|
||||
)
|
||||
|
||||
// setsDsyncLockers is encapsulated type for Close()
|
||||
@@ -701,9 +702,8 @@ func (s *erasureSets) getHashedSet(input string) (set *erasureObjects) {
|
||||
}
|
||||
|
||||
// listDeletedBuckets lists deleted buckets from all disks.
|
||||
func listDeletedBuckets(ctx context.Context, storageDisks []StorageAPI, delBuckets map[string]VolInfo, readQuorum int) error {
|
||||
func listDeletedBuckets(ctx context.Context, storageDisks []StorageAPI, delBuckets *xsync.MapOf[string, VolInfo], readQuorum int) error {
|
||||
g := errgroup.WithNErrs(len(storageDisks))
|
||||
var mu sync.Mutex
|
||||
for index := range storageDisks {
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
@@ -722,11 +722,7 @@ func listDeletedBuckets(ctx context.Context, storageDisks []StorageAPI, delBucke
|
||||
vi, err := storageDisks[index].StatVol(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, volName))
|
||||
if err == nil {
|
||||
vi.Name = strings.TrimSuffix(volName, SlashSeparator)
|
||||
mu.Lock()
|
||||
if _, ok := delBuckets[volName]; !ok {
|
||||
delBuckets[volName] = vi
|
||||
}
|
||||
mu.Unlock()
|
||||
delBuckets.Store(volName, vi)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -24,6 +24,8 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
@@ -286,6 +288,10 @@ func (driver *ftpDriver) CheckPasswd(c *ftp.Context, username, password string)
|
||||
}
|
||||
|
||||
func (driver *ftpDriver) getMinIOClient(ctx *ftp.Context) (*minio.Client, error) {
|
||||
tr := http.RoundTripper(globalRemoteFTPClientTransport)
|
||||
if host, _, err := net.SplitHostPort(ctx.Sess.RemoteAddr().String()); err == nil {
|
||||
tr = forwardForTransport{tr: tr, fwd: host}
|
||||
}
|
||||
ui, ok := globalIAMSys.GetUser(context.Background(), ctx.Sess.LoginUser())
|
||||
if !ok && !globalIAMSys.LDAPConfig.Enabled() {
|
||||
return nil, errNoSuchUser
|
||||
@@ -361,9 +367,10 @@ func (driver *ftpDriver) getMinIOClient(ctx *ftp.Context) (*minio.Client, error)
|
||||
}
|
||||
|
||||
return minio.New(driver.endpoint, &minio.Options{
|
||||
Creds: mcreds,
|
||||
Secure: globalIsTLS,
|
||||
Transport: globalRemoteFTPClientTransport,
|
||||
Creds: mcreds,
|
||||
Secure: globalIsTLS,
|
||||
Transport: tr,
|
||||
TrailingHeaders: true,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -375,9 +382,10 @@ func (driver *ftpDriver) getMinIOClient(ctx *ftp.Context) (*minio.Client, error)
|
||||
}
|
||||
|
||||
return minio.New(driver.endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(ui.Credentials.AccessKey, ui.Credentials.SecretKey, ""),
|
||||
Secure: globalIsTLS,
|
||||
Transport: globalRemoteFTPClientTransport,
|
||||
Creds: credentials.NewStaticV4(ui.Credentials.AccessKey, ui.Credentials.SecretKey, ""),
|
||||
Secure: globalIsTLS,
|
||||
Transport: tr,
|
||||
TrailingHeaders: true,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -546,6 +554,7 @@ func (driver *ftpDriver) PutFile(ctx *ftp.Context, objPath string, data io.Reade
|
||||
info, err := clnt.PutObject(context.Background(), bucket, object, data, -1, minio.PutObjectOptions{
|
||||
ContentType: mimedb.TypeByExtension(path.Ext(object)),
|
||||
DisableContentSha256: true,
|
||||
Checksum: minio.ChecksumFullObjectCRC32C,
|
||||
})
|
||||
n = info.Size
|
||||
return n, err
|
||||
|
||||
+32
-20
@@ -22,19 +22,17 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/minio-go/v7/pkg/s3utils"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio/internal/grid"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
"golang.org/x/exp/maps"
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/minio/minio/internal/amztime"
|
||||
"github.com/minio/minio/internal/config/dns"
|
||||
@@ -75,7 +73,7 @@ const (
|
||||
// and must not set by clients
|
||||
func containsReservedMetadata(header http.Header) bool {
|
||||
for key := range header {
|
||||
if slices.Contains(maps.Keys(validSSEReplicationHeaders), key) {
|
||||
if _, ok := validSSEReplicationHeaders[key]; ok {
|
||||
return false
|
||||
}
|
||||
if stringsHasPrefixFold(key, ReservedMetadataPrefix) {
|
||||
@@ -295,12 +293,6 @@ func parseAmzDateHeader(req *http.Request) (time.Time, APIErrorCode) {
|
||||
return time.Time{}, ErrMissingDateHeader
|
||||
}
|
||||
|
||||
// Bad path components to be rejected by the path validity handler.
|
||||
const (
|
||||
dotdotComponent = ".."
|
||||
dotComponent = "."
|
||||
)
|
||||
|
||||
func hasBadHost(host string) error {
|
||||
if globalIsCICD && strings.TrimSpace(host) == "" {
|
||||
// under CI/CD test setups ignore empty hosts as invalid hosts
|
||||
@@ -313,21 +305,41 @@ func hasBadHost(host string) error {
|
||||
// Check if the incoming path has bad path components,
|
||||
// such as ".." and "."
|
||||
func hasBadPathComponent(path string) bool {
|
||||
if len(path) > 4096 {
|
||||
// path cannot be greater than Linux PATH_MAX
|
||||
// this is to avoid a busy loop, that can happen
|
||||
// if the caller sends path of following style
|
||||
// a/a/a/a/a/a/a/a...
|
||||
n := len(path)
|
||||
if n > 32<<10 {
|
||||
// At 32K we are beyond reasonable.
|
||||
return true
|
||||
}
|
||||
path = filepath.ToSlash(strings.TrimSpace(path)) // For windows '\' must be converted to '/'
|
||||
for _, p := range strings.Split(path, SlashSeparator) {
|
||||
switch strings.TrimSpace(p) {
|
||||
case dotdotComponent:
|
||||
i := 0
|
||||
// Skip leading slashes (for sake of Windows \ is included as well)
|
||||
for i < n && (path[i] == SlashSeparatorChar || path[i] == '\\') {
|
||||
i++
|
||||
}
|
||||
|
||||
for i < n {
|
||||
// Find the next segment
|
||||
start := i
|
||||
for i < n && path[i] != SlashSeparatorChar && path[i] != '\\' {
|
||||
i++
|
||||
}
|
||||
|
||||
// Trim whitespace of segment
|
||||
segmentStart, segmentEnd := start, i
|
||||
for segmentStart < segmentEnd && unicode.IsSpace(rune(path[segmentStart])) {
|
||||
segmentStart++
|
||||
}
|
||||
for segmentEnd > segmentStart && unicode.IsSpace(rune(path[segmentEnd-1])) {
|
||||
segmentEnd--
|
||||
}
|
||||
|
||||
// Check for ".." or "."
|
||||
switch {
|
||||
case segmentEnd-segmentStart == 2 && path[segmentStart] == '.' && path[segmentStart+1] == '.':
|
||||
return true
|
||||
case dotComponent:
|
||||
case segmentEnd-segmentStart == 1 && path[segmentStart] == '.':
|
||||
return true
|
||||
}
|
||||
i++
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
@@ -184,3 +185,27 @@ func TestSSETLSHandler(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func Benchmark_hasBadPathComponent(t *testing.B) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want bool
|
||||
}{
|
||||
{name: "empty", input: "", want: false},
|
||||
{name: "backslashes", input: `\a\a\ \\ \\\\\\\`, want: false},
|
||||
{name: "long", input: strings.Repeat("a/", 2000), want: false},
|
||||
{name: "long-fail", input: strings.Repeat("a/", 2000) + "../..", want: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(b *testing.B) {
|
||||
b.SetBytes(int64(len(tt.input)))
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
if got := hasBadPathComponent(tt.input); got != tt.want {
|
||||
t.Fatalf("hasBadPathComponent() = %v, want %v", got, tt.want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -366,7 +366,6 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
|
||||
writeErrorResponse(ctx, w,
|
||||
errorCodes.ToAPIErr(ErrTooManyRequests),
|
||||
r.URL)
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -447,7 +447,7 @@ func getHostName(r *http.Request) (hostName string) {
|
||||
}
|
||||
|
||||
// Proxy any request to an endpoint.
|
||||
func proxyRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, ep ProxyEndpoint) (success bool) {
|
||||
func proxyRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, ep ProxyEndpoint, returnErr bool) (success bool) {
|
||||
success = true
|
||||
|
||||
// Make sure we remove any existing headers before
|
||||
@@ -462,7 +462,10 @@ func proxyRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, e
|
||||
ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
|
||||
success = false
|
||||
if err != nil && !errors.Is(err, context.Canceled) {
|
||||
replLogIf(GlobalContext, err)
|
||||
proxyLogIf(GlobalContext, err)
|
||||
}
|
||||
if returnErr {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
@@ -493,7 +493,6 @@ func (ies *IAMEtcdStore) watch(ctx context.Context, keyPath string) <-chan iamWa
|
||||
keyPath: string(event.Kv.Key),
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -278,7 +278,6 @@ func (iamOS *IAMObjectStore) loadUserIdentity(ctx context.Context, user string,
|
||||
iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false))
|
||||
}
|
||||
return u, errNoSuchUser
|
||||
|
||||
}
|
||||
u.Credentials.Claims = jwtClaims.Map()
|
||||
}
|
||||
@@ -793,7 +792,10 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
|
||||
for _, item := range listedConfigItems[stsListKey] {
|
||||
userName := path.Dir(item)
|
||||
// loadUser() will delete expired user during the load.
|
||||
iamLogIf(ctx, iamOS.loadUser(ctx, userName, stsUser, stsAccountsFromStore))
|
||||
err := iamOS.loadUser(ctx, userName, stsUser, stsAccountsFromStore)
|
||||
if err != nil && !errors.Is(err, errNoSuchUser) {
|
||||
iamLogIf(ctx, err)
|
||||
}
|
||||
// No need to return errors for failed expiration of STS users
|
||||
}
|
||||
|
||||
@@ -801,7 +803,10 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
|
||||
// (removed during loadUser() in the loop above) are removed from memory.
|
||||
for _, item := range listedConfigItems[policyDBSTSUsersListKey] {
|
||||
stsName := strings.TrimSuffix(item, ".json")
|
||||
iamLogIf(ctx, iamOS.loadMappedPolicy(ctx, stsName, stsUser, false, stsAccPoliciesFromStore))
|
||||
err := iamOS.loadMappedPolicy(ctx, stsName, stsUser, false, stsAccPoliciesFromStore)
|
||||
if err != nil && !errors.Is(err, errNoSuchPolicy) {
|
||||
iamLogIf(ctx, err)
|
||||
}
|
||||
// No need to return errors for failed expiration of STS users
|
||||
}
|
||||
|
||||
|
||||
+96
-15
@@ -26,6 +26,7 @@ import (
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
@@ -37,6 +38,7 @@ import (
|
||||
"github.com/minio/minio/internal/jwt"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/puzpuzpuz/xsync/v3"
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
@@ -357,6 +359,68 @@ func (c *iamCache) removeGroupFromMembershipsMap(group string) {
|
||||
}
|
||||
}
|
||||
|
||||
func (c *iamCache) policyDBGetGroups(store *IAMStoreSys, userPolicyPresent bool, groups ...string) ([]string, error) {
|
||||
var policies []string
|
||||
for _, group := range groups {
|
||||
if store.getUsersSysType() == MinIOUsersSysType {
|
||||
g, ok := c.iamGroupsMap[group]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
// Group is disabled, so we return no policy - this
|
||||
// ensures the request is denied.
|
||||
if g.Status == statusDisabled {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
policy, ok := c.iamGroupPolicyMap.Load(group)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
policies = append(policies, policy.toSlice()...)
|
||||
}
|
||||
|
||||
found := len(policies) > 0
|
||||
if found {
|
||||
return policies, nil
|
||||
}
|
||||
|
||||
if userPolicyPresent {
|
||||
// if user mapping present and no group policies found
|
||||
// rely on user policy for access, instead of fallback.
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
|
||||
// no mappings found, fallback for all groups.
|
||||
g := errgroup.WithNErrs(len(groups)).WithConcurrency(10) // load like 10 groups at a time.
|
||||
|
||||
for index := range groups {
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
err := store.loadMappedPolicy(context.TODO(), groups[index], regUser, true, c.iamGroupPolicyMap)
|
||||
if err != nil && !errors.Is(err, errNoSuchPolicy) {
|
||||
return err
|
||||
}
|
||||
if errors.Is(err, errNoSuchPolicy) {
|
||||
return nil
|
||||
}
|
||||
policy, _ := c.iamGroupPolicyMap.Load(groups[index])
|
||||
mu.Lock()
|
||||
policies = append(policies, policy.toSlice()...)
|
||||
mu.Unlock()
|
||||
return nil
|
||||
}, index)
|
||||
}
|
||||
|
||||
err := errors.Join(g.Wait()...)
|
||||
return policies, err
|
||||
}
|
||||
|
||||
// policyDBGet - lower-level helper; does not take locks.
|
||||
//
|
||||
// If a group is passed, it returns policies associated with the group.
|
||||
@@ -676,7 +740,8 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context, firstTime bool) erro
|
||||
type IAMStoreSys struct {
|
||||
IAMStorageAPI
|
||||
|
||||
group *singleflight.Group
|
||||
group *singleflight.Group
|
||||
policy *singleflight.Group
|
||||
}
|
||||
|
||||
// HasWatcher - returns if the storage system has a watcher.
|
||||
@@ -757,21 +822,36 @@ func (store *IAMStoreSys) PolicyDBGet(name string, groups ...string) ([]string,
|
||||
cache := store.rlock()
|
||||
defer store.runlock()
|
||||
|
||||
policies, _, err := cache.policyDBGet(store, name, false, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
userPolicyPresent := len(policies) > 0
|
||||
for _, group := range groups {
|
||||
ps, _, err := cache.policyDBGet(store, group, true, userPolicyPresent)
|
||||
getPolicies := func() ([]string, error) {
|
||||
policies, _, err := cache.policyDBGet(store, name, false, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
policies = append(policies, ps...)
|
||||
}
|
||||
|
||||
return policies, nil
|
||||
userPolicyPresent := len(policies) > 0
|
||||
|
||||
groupPolicies, err := cache.policyDBGetGroups(store, userPolicyPresent, groups...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
policies = append(policies, groupPolicies...)
|
||||
return policies, nil
|
||||
}
|
||||
if store.policy != nil {
|
||||
val, err, _ := store.policy.Do(name, func() (interface{}, error) {
|
||||
return getPolicies()
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res, ok := val.([]string)
|
||||
if !ok {
|
||||
return nil, errors.New("unexpected policy type")
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
return getPolicies()
|
||||
}
|
||||
|
||||
// AddUsersToGroup - adds users to group, creating the group if needed.
|
||||
@@ -1142,7 +1222,6 @@ func (store *IAMStoreSys) PolicyDBUpdate(ctx context.Context, name string, isGro
|
||||
cache.iamGroupPolicyMap.Delete(name)
|
||||
}
|
||||
} else {
|
||||
|
||||
if err = store.saveMappedPolicy(ctx, name, userType, isGroup, newPolicyMapping); err != nil {
|
||||
return
|
||||
}
|
||||
@@ -1544,7 +1623,6 @@ func (store *IAMStoreSys) MergePolicies(policyName string) (string, policy.Polic
|
||||
policies = append(policies, policy)
|
||||
toMerge = append(toMerge, p.Policy)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return strings.Join(policies, ","), policy.MergePolicies(toMerge...)
|
||||
@@ -2841,7 +2919,10 @@ func (store *IAMStoreSys) LoadUser(ctx context.Context, accessKey string) error
|
||||
return err
|
||||
}
|
||||
|
||||
newCache := val.(*iamCache)
|
||||
newCache, ok := val.(*iamCache)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
cache := store.lock()
|
||||
defer store.unlock()
|
||||
|
||||
+6
-2
@@ -178,13 +178,18 @@ func (sys *IAMSys) initStore(objAPI ObjectLayer, etcdClient *etcd.Client) {
|
||||
}
|
||||
|
||||
if etcdClient == nil {
|
||||
var group *singleflight.Group
|
||||
var (
|
||||
group *singleflight.Group
|
||||
policy *singleflight.Group
|
||||
)
|
||||
if env.Get("_MINIO_IAM_SINGLE_FLIGHT", config.EnableOn) == config.EnableOn {
|
||||
group = &singleflight.Group{}
|
||||
policy = &singleflight.Group{}
|
||||
}
|
||||
sys.store = &IAMStoreSys{
|
||||
IAMStorageAPI: newIAMObjectStore(objAPI, sys.usersSysType),
|
||||
group: group,
|
||||
policy: policy,
|
||||
}
|
||||
} else {
|
||||
sys.store = &IAMStoreSys{IAMStorageAPI: newIAMEtcdStore(etcdClient, sys.usersSysType)}
|
||||
@@ -2281,7 +2286,6 @@ func (sys *IAMSys) IsAllowedSTS(args policy.Args, parentUser string) bool {
|
||||
}
|
||||
policies = policySet.ToSlice()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Defensive code: Do not allow any operation if no policy is found in the session token
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/grid"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/pubsub"
|
||||
"github.com/minio/mux"
|
||||
@@ -200,19 +201,19 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
|
||||
}
|
||||
if len(mergeCh) == 0 {
|
||||
// Flush if nothing is queued
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
}
|
||||
grid.PutByteBuffer(ev)
|
||||
case <-emptyEventTicker:
|
||||
if err := enc.Encode(struct{ Records []event.Event }{}); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
case <-keepAliveTicker:
|
||||
if _, err := w.Write([]byte(" ")); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
xhttp.Flush(w)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
|
||||
+152
-111
@@ -24,11 +24,20 @@ import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
)
|
||||
|
||||
// Reject new lock requests immediately when this many are queued
|
||||
// for the local lock mutex.
|
||||
// We do not block unlocking or maintenance, but they add to the count.
|
||||
// The limit is set to allow for bursty behavior,
|
||||
// but prevent requests to overload the server completely.
|
||||
// Rejected clients are expected to retry.
|
||||
const lockMutexWaitLimit = 1000
|
||||
|
||||
// lockRequesterInfo stores various info from the client for each lock that is requested.
|
||||
type lockRequesterInfo struct {
|
||||
Name string // name of the resource lock was requested for
|
||||
@@ -52,9 +61,25 @@ func isWriteLock(lri []lockRequesterInfo) bool {
|
||||
//
|
||||
//msgp:ignore localLocker
|
||||
type localLocker struct {
|
||||
mutex sync.Mutex
|
||||
lockMap map[string][]lockRequesterInfo
|
||||
lockUID map[string]string // UUID -> resource map.
|
||||
mutex sync.Mutex
|
||||
waitMutex atomic.Int32
|
||||
lockMap map[string][]lockRequesterInfo
|
||||
lockUID map[string]string // UUID -> resource map.
|
||||
|
||||
// the following are updated on every cleanup defined in lockValidityDuration
|
||||
readers atomic.Int32
|
||||
writers atomic.Int32
|
||||
lastCleanup atomic.Pointer[time.Time]
|
||||
locksOverloaded atomic.Int64
|
||||
}
|
||||
|
||||
// getMutex will lock the mutex.
|
||||
// Call the returned function to unlock.
|
||||
func (l *localLocker) getMutex() func() {
|
||||
l.waitMutex.Add(1)
|
||||
l.mutex.Lock()
|
||||
l.waitMutex.Add(-1)
|
||||
return l.mutex.Unlock
|
||||
}
|
||||
|
||||
func (l *localLocker) String() string {
|
||||
@@ -76,9 +101,16 @@ func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool
|
||||
return false, fmt.Errorf("internal error: localLocker.Lock called with more than %d resources", maxDeleteList)
|
||||
}
|
||||
|
||||
l.mutex.Lock()
|
||||
defer l.mutex.Unlock()
|
||||
|
||||
// If we have too many waiting, reject this at once.
|
||||
if l.waitMutex.Load() > lockMutexWaitLimit {
|
||||
l.locksOverloaded.Add(1)
|
||||
return false, nil
|
||||
}
|
||||
// Wait for mutex
|
||||
defer l.getMutex()()
|
||||
if ctx.Err() != nil {
|
||||
return false, ctx.Err()
|
||||
}
|
||||
if !l.canTakeLock(args.Resources...) {
|
||||
// Not all locks can be taken on resources,
|
||||
// reject it completely.
|
||||
@@ -117,9 +149,7 @@ func (l *localLocker) Unlock(_ context.Context, args dsync.LockArgs) (reply bool
|
||||
return false, fmt.Errorf("internal error: localLocker.Unlock called with more than %d resources", maxDeleteList)
|
||||
}
|
||||
|
||||
l.mutex.Lock()
|
||||
defer l.mutex.Unlock()
|
||||
err = nil
|
||||
defer l.getMutex()()
|
||||
|
||||
for _, resource := range args.Resources {
|
||||
lri, ok := l.lockMap[resource]
|
||||
@@ -164,9 +194,17 @@ func (l *localLocker) RLock(ctx context.Context, args dsync.LockArgs) (reply boo
|
||||
if len(args.Resources) != 1 {
|
||||
return false, fmt.Errorf("internal error: localLocker.RLock called with more than one resource")
|
||||
}
|
||||
// If we have too many waiting, reject this at once.
|
||||
if l.waitMutex.Load() > lockMutexWaitLimit {
|
||||
l.locksOverloaded.Add(1)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
l.mutex.Lock()
|
||||
defer l.mutex.Unlock()
|
||||
// Wait for mutex
|
||||
defer l.getMutex()()
|
||||
if ctx.Err() != nil {
|
||||
return false, ctx.Err()
|
||||
}
|
||||
resource := args.Resources[0]
|
||||
now := UTCNow()
|
||||
lrInfo := lockRequesterInfo{
|
||||
@@ -200,8 +238,7 @@ func (l *localLocker) RUnlock(_ context.Context, args dsync.LockArgs) (reply boo
|
||||
return false, fmt.Errorf("internal error: localLocker.RUnlock called with more than one resource")
|
||||
}
|
||||
|
||||
l.mutex.Lock()
|
||||
defer l.mutex.Unlock()
|
||||
defer l.getMutex()()
|
||||
var lri []lockRequesterInfo
|
||||
|
||||
resource := args.Resources[0]
|
||||
@@ -218,35 +255,28 @@ func (l *localLocker) RUnlock(_ context.Context, args dsync.LockArgs) (reply boo
|
||||
}
|
||||
|
||||
type lockStats struct {
|
||||
Total int
|
||||
Writes int
|
||||
Reads int
|
||||
Total int
|
||||
Writes int
|
||||
Reads int
|
||||
LockQueue int
|
||||
LocksAbandoned int
|
||||
LastCleanup *time.Time
|
||||
}
|
||||
|
||||
func (l *localLocker) stats() lockStats {
|
||||
l.mutex.Lock()
|
||||
defer l.mutex.Unlock()
|
||||
|
||||
st := lockStats{Total: len(l.lockMap)}
|
||||
for _, v := range l.lockMap {
|
||||
if len(v) == 0 {
|
||||
continue
|
||||
}
|
||||
entry := v[0]
|
||||
if entry.Writer {
|
||||
st.Writes++
|
||||
} else {
|
||||
st.Reads += len(v)
|
||||
}
|
||||
return lockStats{
|
||||
Total: len(l.lockMap),
|
||||
Reads: int(l.readers.Load()),
|
||||
Writes: int(l.writers.Load()),
|
||||
LockQueue: int(l.waitMutex.Load()),
|
||||
LastCleanup: l.lastCleanup.Load(),
|
||||
}
|
||||
return st
|
||||
}
|
||||
|
||||
type localLockMap map[string][]lockRequesterInfo
|
||||
|
||||
func (l *localLocker) DupLockMap() localLockMap {
|
||||
l.mutex.Lock()
|
||||
defer l.mutex.Unlock()
|
||||
defer l.getMutex()()
|
||||
|
||||
lockCopy := make(map[string][]lockRequesterInfo, len(l.lockMap))
|
||||
for k, v := range l.lockMap {
|
||||
@@ -274,108 +304,110 @@ func (l *localLocker) IsLocal() bool {
|
||||
}
|
||||
|
||||
func (l *localLocker) ForceUnlock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if ctx.Err() != nil {
|
||||
return false, ctx.Err()
|
||||
default:
|
||||
l.mutex.Lock()
|
||||
defer l.mutex.Unlock()
|
||||
if len(args.UID) == 0 {
|
||||
for _, resource := range args.Resources {
|
||||
lris, ok := l.lockMap[resource]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
// Collect uids, so we don't mutate while we delete
|
||||
uids := make([]string, 0, len(lris))
|
||||
for _, lri := range lris {
|
||||
uids = append(uids, lri.UID)
|
||||
}
|
||||
}
|
||||
|
||||
// Delete collected uids:
|
||||
for _, uid := range uids {
|
||||
lris, ok := l.lockMap[resource]
|
||||
if !ok {
|
||||
// Just to be safe, delete uuids.
|
||||
for idx := 0; idx < maxDeleteList; idx++ {
|
||||
mapID := formatUUID(uid, idx)
|
||||
if _, ok := l.lockUID[mapID]; !ok {
|
||||
break
|
||||
}
|
||||
delete(l.lockUID, mapID)
|
||||
}
|
||||
continue
|
||||
}
|
||||
l.removeEntry(resource, dsync.LockArgs{UID: uid}, &lris)
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
idx := 0
|
||||
for {
|
||||
mapID := formatUUID(args.UID, idx)
|
||||
resource, ok := l.lockUID[mapID]
|
||||
if !ok {
|
||||
return idx > 0, nil
|
||||
}
|
||||
defer l.getMutex()()
|
||||
if ctx.Err() != nil {
|
||||
return false, ctx.Err()
|
||||
}
|
||||
if len(args.UID) == 0 {
|
||||
for _, resource := range args.Resources {
|
||||
lris, ok := l.lockMap[resource]
|
||||
if !ok {
|
||||
// Unexpected inconsistency, delete.
|
||||
delete(l.lockUID, mapID)
|
||||
idx++
|
||||
continue
|
||||
}
|
||||
reply = true
|
||||
l.removeEntry(resource, dsync.LockArgs{UID: args.UID}, &lris)
|
||||
idx++
|
||||
// Collect uids, so we don't mutate while we delete
|
||||
uids := make([]string, 0, len(lris))
|
||||
for _, lri := range lris {
|
||||
uids = append(uids, lri.UID)
|
||||
}
|
||||
|
||||
// Delete collected uids:
|
||||
for _, uid := range uids {
|
||||
lris, ok := l.lockMap[resource]
|
||||
if !ok {
|
||||
// Just to be safe, delete uuids.
|
||||
for idx := 0; idx < maxDeleteList; idx++ {
|
||||
mapID := formatUUID(uid, idx)
|
||||
if _, ok := l.lockUID[mapID]; !ok {
|
||||
break
|
||||
}
|
||||
delete(l.lockUID, mapID)
|
||||
}
|
||||
continue
|
||||
}
|
||||
l.removeEntry(resource, dsync.LockArgs{UID: uid}, &lris)
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
idx := 0
|
||||
for {
|
||||
mapID := formatUUID(args.UID, idx)
|
||||
resource, ok := l.lockUID[mapID]
|
||||
if !ok {
|
||||
return idx > 0, nil
|
||||
}
|
||||
lris, ok := l.lockMap[resource]
|
||||
if !ok {
|
||||
// Unexpected inconsistency, delete.
|
||||
delete(l.lockUID, mapID)
|
||||
idx++
|
||||
continue
|
||||
}
|
||||
reply = true
|
||||
l.removeEntry(resource, dsync.LockArgs{UID: args.UID}, &lris)
|
||||
idx++
|
||||
}
|
||||
}
|
||||
|
||||
func (l *localLocker) Refresh(ctx context.Context, args dsync.LockArgs) (refreshed bool, err error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if ctx.Err() != nil {
|
||||
return false, ctx.Err()
|
||||
default:
|
||||
l.mutex.Lock()
|
||||
defer l.mutex.Unlock()
|
||||
}
|
||||
|
||||
// Check whether uid is still active.
|
||||
resource, ok := l.lockUID[formatUUID(args.UID, 0)]
|
||||
defer l.getMutex()()
|
||||
if ctx.Err() != nil {
|
||||
return false, ctx.Err()
|
||||
}
|
||||
|
||||
// Check whether uid is still active.
|
||||
resource, ok := l.lockUID[formatUUID(args.UID, 0)]
|
||||
if !ok {
|
||||
return false, nil
|
||||
}
|
||||
idx := 0
|
||||
for {
|
||||
lris, ok := l.lockMap[resource]
|
||||
if !ok {
|
||||
return false, nil
|
||||
// Inconsistent. Delete UID.
|
||||
delete(l.lockUID, formatUUID(args.UID, idx))
|
||||
return idx > 0, nil
|
||||
}
|
||||
idx := 0
|
||||
for {
|
||||
lris, ok := l.lockMap[resource]
|
||||
if !ok {
|
||||
// Inconsistent. Delete UID.
|
||||
delete(l.lockUID, formatUUID(args.UID, idx))
|
||||
return idx > 0, nil
|
||||
}
|
||||
now := UTCNow()
|
||||
for i := range lris {
|
||||
if lris[i].UID == args.UID {
|
||||
lris[i].TimeLastRefresh = now.UnixNano()
|
||||
}
|
||||
}
|
||||
idx++
|
||||
resource, ok = l.lockUID[formatUUID(args.UID, idx)]
|
||||
if !ok {
|
||||
// No more resources for UID, but we did update at least one.
|
||||
return true, nil
|
||||
now := UTCNow()
|
||||
for i := range lris {
|
||||
if lris[i].UID == args.UID {
|
||||
lris[i].TimeLastRefresh = now.UnixNano()
|
||||
}
|
||||
}
|
||||
idx++
|
||||
resource, ok = l.lockUID[formatUUID(args.UID, idx)]
|
||||
if !ok {
|
||||
// No more resources for UID, but we did update at least one.
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Similar to removeEntry but only removes an entry only if the lock entry exists in map.
|
||||
// Caller must hold 'l.mutex' lock.
|
||||
func (l *localLocker) expireOldLocks(interval time.Duration) {
|
||||
l.mutex.Lock()
|
||||
defer l.mutex.Unlock()
|
||||
defer l.getMutex()()
|
||||
|
||||
var readers, writers int32
|
||||
for k, lris := range l.lockMap {
|
||||
modified := false
|
||||
for i := 0; i < len(lris); {
|
||||
@@ -393,6 +425,11 @@ func (l *localLocker) expireOldLocks(interval time.Duration) {
|
||||
lris = append(lris[:i], lris[i+1:]...)
|
||||
// Check same i
|
||||
} else {
|
||||
if lri.Writer {
|
||||
writers++
|
||||
} else {
|
||||
readers++
|
||||
}
|
||||
// Move to next
|
||||
i++
|
||||
}
|
||||
@@ -401,6 +438,10 @@ func (l *localLocker) expireOldLocks(interval time.Duration) {
|
||||
l.lockMap[k] = lris
|
||||
}
|
||||
}
|
||||
t := time.Now()
|
||||
l.lastCleanup.Store(&t)
|
||||
l.readers.Store(readers)
|
||||
l.writers.Store(writers)
|
||||
}
|
||||
|
||||
func newLocker() *localLocker {
|
||||
|
||||
+124
-8
@@ -3,6 +3,8 @@ package cmd
|
||||
// Code generated by github.com/tinylib/msgp DO NOT EDIT.
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/tinylib/msgp/msgp"
|
||||
)
|
||||
|
||||
@@ -506,6 +508,36 @@ func (z *lockStats) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
err = msgp.WrapError(err, "Reads")
|
||||
return
|
||||
}
|
||||
case "LockQueue":
|
||||
z.LockQueue, err = dc.ReadInt()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "LockQueue")
|
||||
return
|
||||
}
|
||||
case "LocksAbandoned":
|
||||
z.LocksAbandoned, err = dc.ReadInt()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "LocksAbandoned")
|
||||
return
|
||||
}
|
||||
case "LastCleanup":
|
||||
if dc.IsNil() {
|
||||
err = dc.ReadNil()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "LastCleanup")
|
||||
return
|
||||
}
|
||||
z.LastCleanup = nil
|
||||
} else {
|
||||
if z.LastCleanup == nil {
|
||||
z.LastCleanup = new(time.Time)
|
||||
}
|
||||
*z.LastCleanup, err = dc.ReadTime()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "LastCleanup")
|
||||
return
|
||||
}
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
@@ -518,10 +550,10 @@ func (z *lockStats) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
}
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z lockStats) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// map header, size 3
|
||||
func (z *lockStats) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// map header, size 6
|
||||
// write "Total"
|
||||
err = en.Append(0x83, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
|
||||
err = en.Append(0x86, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -550,15 +582,52 @@ func (z lockStats) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
err = msgp.WrapError(err, "Reads")
|
||||
return
|
||||
}
|
||||
// write "LockQueue"
|
||||
err = en.Append(0xa9, 0x4c, 0x6f, 0x63, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteInt(z.LockQueue)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "LockQueue")
|
||||
return
|
||||
}
|
||||
// write "LocksAbandoned"
|
||||
err = en.Append(0xae, 0x4c, 0x6f, 0x63, 0x6b, 0x73, 0x41, 0x62, 0x61, 0x6e, 0x64, 0x6f, 0x6e, 0x65, 0x64)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteInt(z.LocksAbandoned)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "LocksAbandoned")
|
||||
return
|
||||
}
|
||||
// write "LastCleanup"
|
||||
err = en.Append(0xab, 0x4c, 0x61, 0x73, 0x74, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if z.LastCleanup == nil {
|
||||
err = en.WriteNil()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
err = en.WriteTime(*z.LastCleanup)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "LastCleanup")
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z lockStats) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
func (z *lockStats) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// map header, size 3
|
||||
// map header, size 6
|
||||
// string "Total"
|
||||
o = append(o, 0x83, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
|
||||
o = append(o, 0x86, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
|
||||
o = msgp.AppendInt(o, z.Total)
|
||||
// string "Writes"
|
||||
o = append(o, 0xa6, 0x57, 0x72, 0x69, 0x74, 0x65, 0x73)
|
||||
@@ -566,6 +635,19 @@ func (z lockStats) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
// string "Reads"
|
||||
o = append(o, 0xa5, 0x52, 0x65, 0x61, 0x64, 0x73)
|
||||
o = msgp.AppendInt(o, z.Reads)
|
||||
// string "LockQueue"
|
||||
o = append(o, 0xa9, 0x4c, 0x6f, 0x63, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65)
|
||||
o = msgp.AppendInt(o, z.LockQueue)
|
||||
// string "LocksAbandoned"
|
||||
o = append(o, 0xae, 0x4c, 0x6f, 0x63, 0x6b, 0x73, 0x41, 0x62, 0x61, 0x6e, 0x64, 0x6f, 0x6e, 0x65, 0x64)
|
||||
o = msgp.AppendInt(o, z.LocksAbandoned)
|
||||
// string "LastCleanup"
|
||||
o = append(o, 0xab, 0x4c, 0x61, 0x73, 0x74, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70)
|
||||
if z.LastCleanup == nil {
|
||||
o = msgp.AppendNil(o)
|
||||
} else {
|
||||
o = msgp.AppendTime(o, *z.LastCleanup)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -605,6 +687,35 @@ func (z *lockStats) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
err = msgp.WrapError(err, "Reads")
|
||||
return
|
||||
}
|
||||
case "LockQueue":
|
||||
z.LockQueue, bts, err = msgp.ReadIntBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "LockQueue")
|
||||
return
|
||||
}
|
||||
case "LocksAbandoned":
|
||||
z.LocksAbandoned, bts, err = msgp.ReadIntBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "LocksAbandoned")
|
||||
return
|
||||
}
|
||||
case "LastCleanup":
|
||||
if msgp.IsNil(bts) {
|
||||
bts, err = msgp.ReadNilBytes(bts)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
z.LastCleanup = nil
|
||||
} else {
|
||||
if z.LastCleanup == nil {
|
||||
z.LastCleanup = new(time.Time)
|
||||
}
|
||||
*z.LastCleanup, bts, err = msgp.ReadTimeBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "LastCleanup")
|
||||
return
|
||||
}
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
@@ -618,7 +729,12 @@ func (z *lockStats) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
}
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z lockStats) Msgsize() (s int) {
|
||||
s = 1 + 6 + msgp.IntSize + 7 + msgp.IntSize + 6 + msgp.IntSize
|
||||
func (z *lockStats) Msgsize() (s int) {
|
||||
s = 1 + 6 + msgp.IntSize + 7 + msgp.IntSize + 6 + msgp.IntSize + 10 + msgp.IntSize + 15 + msgp.IntSize + 12
|
||||
if z.LastCleanup == nil {
|
||||
s += msgp.NilSize
|
||||
} else {
|
||||
s += msgp.TimeSize
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -136,7 +136,6 @@ func TestLocalLockerUnlock(t *testing.T) {
|
||||
}
|
||||
wResources[i] = names
|
||||
wUIDs[i] = uid
|
||||
|
||||
}
|
||||
for i := range rResources {
|
||||
name := mustGetUUID()
|
||||
|
||||
+13
-3
@@ -33,8 +33,12 @@ type lockRESTServer struct {
|
||||
|
||||
// RefreshHandler - refresh the current lock
|
||||
func (l *lockRESTServer) RefreshHandler(args *dsync.LockArgs) (*dsync.LockResp, *grid.RemoteErr) {
|
||||
// Add a timeout similar to what we expect upstream.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), dsync.DefaultTimeouts.RefreshCall)
|
||||
defer cancel()
|
||||
|
||||
resp := lockRPCRefresh.NewResponse()
|
||||
refreshed, err := l.ll.Refresh(context.Background(), *args)
|
||||
refreshed, err := l.ll.Refresh(ctx, *args)
|
||||
if err != nil {
|
||||
return l.makeResp(resp, err)
|
||||
}
|
||||
@@ -46,8 +50,11 @@ func (l *lockRESTServer) RefreshHandler(args *dsync.LockArgs) (*dsync.LockResp,
|
||||
|
||||
// LockHandler - Acquires a lock.
|
||||
func (l *lockRESTServer) LockHandler(args *dsync.LockArgs) (*dsync.LockResp, *grid.RemoteErr) {
|
||||
// Add a timeout similar to what we expect upstream.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), dsync.DefaultTimeouts.Acquire)
|
||||
defer cancel()
|
||||
resp := lockRPCLock.NewResponse()
|
||||
success, err := l.ll.Lock(context.Background(), *args)
|
||||
success, err := l.ll.Lock(ctx, *args)
|
||||
if err == nil && !success {
|
||||
return l.makeResp(resp, errLockConflict)
|
||||
}
|
||||
@@ -65,8 +72,11 @@ func (l *lockRESTServer) UnlockHandler(args *dsync.LockArgs) (*dsync.LockResp, *
|
||||
|
||||
// RLockHandler - Acquires an RLock.
|
||||
func (l *lockRESTServer) RLockHandler(args *dsync.LockArgs) (*dsync.LockResp, *grid.RemoteErr) {
|
||||
// Add a timeout similar to what we expect upstream.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), dsync.DefaultTimeouts.Acquire)
|
||||
defer cancel()
|
||||
resp := lockRPCRLock.NewResponse()
|
||||
success, err := l.ll.RLock(context.Background(), *args)
|
||||
success, err := l.ll.RLock(ctx, *args)
|
||||
if err == nil && !success {
|
||||
err = errLockConflict
|
||||
}
|
||||
|
||||
+12
-16
@@ -29,6 +29,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
@@ -182,8 +183,7 @@ func (o *listPathOptions) gatherResults(ctx context.Context, in <-chan metaCache
|
||||
resultsDone := make(chan metaCacheEntriesSorted)
|
||||
// Copy so we can mutate
|
||||
resCh := resultsDone
|
||||
var done bool
|
||||
var mu sync.Mutex
|
||||
var done atomic.Bool
|
||||
resErr := io.EOF
|
||||
|
||||
go func() {
|
||||
@@ -194,9 +194,7 @@ func (o *listPathOptions) gatherResults(ctx context.Context, in <-chan metaCache
|
||||
// past limit
|
||||
continue
|
||||
}
|
||||
mu.Lock()
|
||||
returned = done
|
||||
mu.Unlock()
|
||||
returned = done.Load()
|
||||
if returned {
|
||||
resCh = nil
|
||||
continue
|
||||
@@ -250,9 +248,7 @@ func (o *listPathOptions) gatherResults(ctx context.Context, in <-chan metaCache
|
||||
return func() (metaCacheEntriesSorted, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
mu.Lock()
|
||||
done = true
|
||||
mu.Unlock()
|
||||
done.Store(true)
|
||||
return metaCacheEntriesSorted{}, ctx.Err()
|
||||
case r := <-resultsDone:
|
||||
return r, resErr
|
||||
@@ -626,18 +622,18 @@ func calcCommonWritesDeletes(infos []DiskInfo, readQuorum int) (commonWrite, com
|
||||
}
|
||||
|
||||
filter := func(list []uint64) (commonCount uint64) {
|
||||
max := 0
|
||||
maxCnt := 0
|
||||
signatureMap := map[uint64]int{}
|
||||
for _, v := range list {
|
||||
signatureMap[v]++
|
||||
}
|
||||
for ops, count := range signatureMap {
|
||||
if max < count && commonCount < ops {
|
||||
max = count
|
||||
if maxCnt < count && commonCount < ops {
|
||||
maxCnt = count
|
||||
commonCount = ops
|
||||
}
|
||||
}
|
||||
if max < readQuorum {
|
||||
if maxCnt < readQuorum {
|
||||
return 0
|
||||
}
|
||||
return commonCount
|
||||
@@ -650,7 +646,7 @@ func calcCommonWritesDeletes(infos []DiskInfo, readQuorum int) (commonWrite, com
|
||||
|
||||
func calcCommonCounter(infos []DiskInfo, readQuorum int) (commonCount uint64) {
|
||||
filter := func() (commonCount uint64) {
|
||||
max := 0
|
||||
maxCnt := 0
|
||||
signatureMap := map[uint64]int{}
|
||||
for _, info := range infos {
|
||||
if info.Error != "" {
|
||||
@@ -660,12 +656,12 @@ func calcCommonCounter(infos []DiskInfo, readQuorum int) (commonCount uint64) {
|
||||
signatureMap[mutations]++
|
||||
}
|
||||
for ops, count := range signatureMap {
|
||||
if max < count && commonCount < ops {
|
||||
max = count
|
||||
if maxCnt < count && commonCount < ops {
|
||||
maxCnt = count
|
||||
commonCount = ops
|
||||
}
|
||||
}
|
||||
if max < readQuorum {
|
||||
if maxCnt < readQuorum {
|
||||
return 0
|
||||
}
|
||||
return commonCount
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"github.com/klauspost/compress/s2"
|
||||
"github.com/minio/minio/internal/bpool"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/tinylib/msgp/msgp"
|
||||
"github.com/valyala/bytebufferpool"
|
||||
@@ -236,7 +237,7 @@ func (w *metacacheWriter) Reset(out io.Writer) {
|
||||
}
|
||||
}
|
||||
|
||||
var s2DecPool = sync.Pool{New: func() interface{} {
|
||||
var s2DecPool = bpool.Pool[*s2.Reader]{New: func() *s2.Reader {
|
||||
// Default alloc block for network transfer.
|
||||
return s2.NewReader(nil, s2.ReaderAllocBlock(16<<10))
|
||||
}}
|
||||
@@ -253,7 +254,7 @@ type metacacheReader struct {
|
||||
// newMetacacheReader creates a new cache reader.
|
||||
// Nothing will be read from the stream yet.
|
||||
func newMetacacheReader(r io.Reader) *metacacheReader {
|
||||
dec := s2DecPool.Get().(*s2.Reader)
|
||||
dec := s2DecPool.Get()
|
||||
dec.Reset(r)
|
||||
mr := msgpNewReader(dec)
|
||||
return &metacacheReader{
|
||||
|
||||
+17
-4
@@ -685,6 +685,16 @@ func getUsageLastScanActivityMD() MetricDescription {
|
||||
}
|
||||
}
|
||||
|
||||
func getBucketUsageLastScanActivityMD() MetricDescription {
|
||||
return MetricDescription{
|
||||
Namespace: bucketMetricNamespace,
|
||||
Subsystem: usageSubsystem,
|
||||
Name: lastActivityTime,
|
||||
Help: "Time elapsed (in nano seconds) since last scan activity",
|
||||
Type: gaugeMetric,
|
||||
}
|
||||
}
|
||||
|
||||
func getBucketUsageQuotaTotalBytesMD() MetricDescription {
|
||||
return MetricDescription{
|
||||
Namespace: bucketMetricNamespace,
|
||||
@@ -1875,13 +1885,17 @@ func getHistogramMetrics(hist *prometheus.HistogramVec, desc MetricDescription,
|
||||
// add metrics with +Inf label
|
||||
labels1 := make(map[string]string)
|
||||
for _, lp := range dtoMetric.GetLabel() {
|
||||
labels1[*lp.Name] = *lp.Value
|
||||
if *lp.Name == "api" && toLowerAPILabels {
|
||||
labels1[*lp.Name] = strings.ToLower(*lp.Value)
|
||||
} else {
|
||||
labels1[*lp.Name] = *lp.Value
|
||||
}
|
||||
}
|
||||
labels1["le"] = fmt.Sprintf("%.3f", math.Inf(+1))
|
||||
metrics = append(metrics, MetricV2{
|
||||
Description: desc,
|
||||
VariableLabels: labels1,
|
||||
Value: dtoMetric.Counter.GetValue(),
|
||||
Value: float64(dtoMetric.Histogram.GetSampleCount()),
|
||||
})
|
||||
}
|
||||
return metrics
|
||||
@@ -2460,7 +2474,6 @@ func getReplicationNodeMetrics(opts MetricsGroupOpts) *MetricsGroupV2 {
|
||||
}
|
||||
downtimeDuration.Value = float64(dwntime / time.Second)
|
||||
ml = append(ml, downtimeDuration)
|
||||
|
||||
}
|
||||
return ml
|
||||
})
|
||||
@@ -3239,7 +3252,7 @@ func getBucketUsageMetrics(opts MetricsGroupOpts) *MetricsGroupV2 {
|
||||
}
|
||||
|
||||
metrics = append(metrics, MetricV2{
|
||||
Description: getUsageLastScanActivityMD(),
|
||||
Description: getBucketUsageLastScanActivityMD(),
|
||||
Value: float64(time.Since(dataUsageInfo.LastUpdate)),
|
||||
})
|
||||
|
||||
|
||||
+133
-2
@@ -18,13 +18,15 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
func TestGetHistogramMetrics(t *testing.T) {
|
||||
func TestGetHistogramMetrics_BucketCount(t *testing.T) {
|
||||
histBuckets := []float64{0.05, 0.1, 0.25, 0.5, 0.75}
|
||||
labels := []string{"GetObject", "PutObject", "CopyObject", "CompleteMultipartUpload"}
|
||||
ttfbHist := prometheus.NewHistogramVec(
|
||||
@@ -83,6 +85,135 @@ func TestGetHistogramMetrics(t *testing.T) {
|
||||
metrics := getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), false)
|
||||
// additional labels for +Inf for all histogram metrics
|
||||
if expPoints := len(labels) * (len(histBuckets) + 1); expPoints != len(metrics) {
|
||||
t.Fatalf("Expected %v data points but got %v", expPoints, len(metrics))
|
||||
t.Fatalf("Expected %v data points when toLowerAPILabels=false but got %v", expPoints, len(metrics))
|
||||
}
|
||||
|
||||
metrics = getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), true)
|
||||
// additional labels for +Inf for all histogram metrics
|
||||
if expPoints := len(labels) * (len(histBuckets) + 1); expPoints != len(metrics) {
|
||||
t.Fatalf("Expected %v data points when toLowerAPILabels=true but got %v", expPoints, len(metrics))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetHistogramMetrics_Values(t *testing.T) {
|
||||
histBuckets := []float64{0.50, 5.00}
|
||||
labels := []string{"PutObject", "CopyObject"}
|
||||
ttfbHist := prometheus.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
Name: "s3_ttfb_seconds",
|
||||
Help: "Time taken by requests served by current MinIO server instance",
|
||||
Buckets: histBuckets,
|
||||
},
|
||||
[]string{"api"},
|
||||
)
|
||||
observations := []struct {
|
||||
val float64
|
||||
label string
|
||||
}{
|
||||
{
|
||||
val: 0.02,
|
||||
label: labels[0],
|
||||
},
|
||||
{
|
||||
val: 0.19,
|
||||
label: labels[1],
|
||||
},
|
||||
{
|
||||
val: 0.31,
|
||||
label: labels[1],
|
||||
},
|
||||
{
|
||||
val: 0.61,
|
||||
label: labels[0],
|
||||
},
|
||||
{
|
||||
val: 6.79,
|
||||
label: labels[1],
|
||||
},
|
||||
}
|
||||
ticker := time.NewTicker(1 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for _, obs := range observations {
|
||||
// Send observations once every 1ms, to simulate delay between
|
||||
// observations. This is to test the channel based
|
||||
// synchronization used internally.
|
||||
select {
|
||||
case <-ticker.C:
|
||||
ttfbHist.With(prometheus.Labels{"api": obs.label}).Observe(obs.val)
|
||||
}
|
||||
}
|
||||
|
||||
// Accumulate regular-cased API label metrics for 'PutObject' for deeper verification
|
||||
metrics := getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), false)
|
||||
capitalPutObjects := make([]MetricV2, 0, len(histBuckets)+1)
|
||||
for _, metric := range metrics {
|
||||
if value := metric.VariableLabels["api"]; value == "PutObject" {
|
||||
capitalPutObjects = append(capitalPutObjects, metric)
|
||||
}
|
||||
}
|
||||
if expMetricsPerAPI := len(histBuckets) + 1; expMetricsPerAPI != len(capitalPutObjects) {
|
||||
t.Fatalf("Expected %d api=PutObject metrics but got %d", expMetricsPerAPI, len(capitalPutObjects))
|
||||
}
|
||||
|
||||
// Deterministic ordering
|
||||
slices.SortFunc(capitalPutObjects, func(a MetricV2, b MetricV2) int {
|
||||
le1 := a.VariableLabels["le"]
|
||||
le2 := a.VariableLabels["le"]
|
||||
return strings.Compare(le1, le2)
|
||||
})
|
||||
if le := capitalPutObjects[0].VariableLabels["le"]; le != "0.500" {
|
||||
t.Errorf("Expected le='0.050' api=PutObject metrics but got '%v'", le)
|
||||
}
|
||||
if value := capitalPutObjects[0].Value; value != 1 {
|
||||
t.Errorf("Expected le='0.050' api=PutObject value to be 1 but got '%v'", value)
|
||||
}
|
||||
if le := capitalPutObjects[1].VariableLabels["le"]; le != "5.000" {
|
||||
t.Errorf("Expected le='5.000' api=PutObject metrics but got '%v'", le)
|
||||
}
|
||||
if value := capitalPutObjects[1].Value; value != 2 {
|
||||
t.Errorf("Expected le='5.000' api=PutObject value to be 2 but got '%v'", value)
|
||||
}
|
||||
if le := capitalPutObjects[2].VariableLabels["le"]; le != "+Inf" {
|
||||
t.Errorf("Expected le='+Inf' api=PutObject metrics but got '%v'", le)
|
||||
}
|
||||
if value := capitalPutObjects[2].Value; value != 2 {
|
||||
t.Errorf("Expected le='+Inf' api=PutObject value to be 2 but got '%v'", value)
|
||||
}
|
||||
|
||||
// Accumulate lower-cased API label metrics for 'copyobject' for deeper verification
|
||||
metrics = getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), true)
|
||||
lowerCopyObjects := make([]MetricV2, 0, len(histBuckets)+1)
|
||||
for _, metric := range metrics {
|
||||
if value := metric.VariableLabels["api"]; value == "copyobject" {
|
||||
lowerCopyObjects = append(lowerCopyObjects, metric)
|
||||
}
|
||||
}
|
||||
if expMetricsPerAPI := len(histBuckets) + 1; expMetricsPerAPI != len(lowerCopyObjects) {
|
||||
t.Fatalf("Expected %d api=copyobject metrics but got %d", expMetricsPerAPI, len(lowerCopyObjects))
|
||||
}
|
||||
|
||||
// Deterministic ordering
|
||||
slices.SortFunc(lowerCopyObjects, func(a MetricV2, b MetricV2) int {
|
||||
le1 := a.VariableLabels["le"]
|
||||
le2 := a.VariableLabels["le"]
|
||||
return strings.Compare(le1, le2)
|
||||
})
|
||||
if le := lowerCopyObjects[0].VariableLabels["le"]; le != "0.500" {
|
||||
t.Errorf("Expected le='0.050' api=copyobject metrics but got '%v'", le)
|
||||
}
|
||||
if value := lowerCopyObjects[0].Value; value != 2 {
|
||||
t.Errorf("Expected le='0.050' api=copyobject value to be 2 but got '%v'", value)
|
||||
}
|
||||
if le := lowerCopyObjects[1].VariableLabels["le"]; le != "5.000" {
|
||||
t.Errorf("Expected le='5.000' api=copyobject metrics but got '%v'", le)
|
||||
}
|
||||
if value := lowerCopyObjects[1].Value; value != 2 {
|
||||
t.Errorf("Expected le='5.000' api=copyobject value to be 2 but got '%v'", value)
|
||||
}
|
||||
if le := lowerCopyObjects[2].VariableLabels["le"]; le != "+Inf" {
|
||||
t.Errorf("Expected le='+Inf' api=copyobject metrics but got '%v'", le)
|
||||
}
|
||||
if value := lowerCopyObjects[2].Value; value != 3 {
|
||||
t.Errorf("Expected le='+Inf' api=copyobject value to be 3 but got '%v'", value)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -177,7 +177,6 @@ func loadClusterUsageBucketMetrics(ctx context.Context, m MetricValues, c *metri
|
||||
for k, v := range usage.ObjectVersionsHistogram {
|
||||
m.Set(usageBucketObjectVersionCountDistribution, float64(v), "range", k, "bucket", bucket)
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
@@ -27,7 +28,6 @@ import (
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
type collectorPath string
|
||||
|
||||
@@ -1184,7 +1184,6 @@ func (sys *NotificationSys) GetPeerOnlineCount() (nodesOnline, nodesOffline int)
|
||||
defer wg.Done()
|
||||
nodesOnlineIndex[idx] = client.restClient.HealthCheckFn()
|
||||
}(idx, client)
|
||||
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
|
||||
+28
-18
@@ -419,6 +419,9 @@ type ListPartsInfo struct {
|
||||
|
||||
// ChecksumAlgorithm if set
|
||||
ChecksumAlgorithm string
|
||||
|
||||
// ChecksumType if set
|
||||
ChecksumType string
|
||||
}
|
||||
|
||||
// Lookup - returns if uploadID is valid
|
||||
@@ -597,10 +600,11 @@ type PartInfo struct {
|
||||
ActualSize int64
|
||||
|
||||
// Checksum values
|
||||
ChecksumCRC32 string
|
||||
ChecksumCRC32C string
|
||||
ChecksumSHA1 string
|
||||
ChecksumSHA256 string
|
||||
ChecksumCRC32 string
|
||||
ChecksumCRC32C string
|
||||
ChecksumSHA1 string
|
||||
ChecksumSHA256 string
|
||||
ChecksumCRC64NVME string
|
||||
}
|
||||
|
||||
// CompletePart - represents the part that was completed, this is sent by the client
|
||||
@@ -613,11 +617,14 @@ type CompletePart struct {
|
||||
// Entity tag returned when the part was uploaded.
|
||||
ETag string
|
||||
|
||||
Size int64
|
||||
|
||||
// Checksum values. Optional.
|
||||
ChecksumCRC32 string
|
||||
ChecksumCRC32C string
|
||||
ChecksumSHA1 string
|
||||
ChecksumSHA256 string
|
||||
ChecksumCRC32 string
|
||||
ChecksumCRC32C string
|
||||
ChecksumSHA1 string
|
||||
ChecksumSHA256 string
|
||||
ChecksumCRC64NVME string
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload - represents list of parts which are completed, this is sent by the
|
||||
@@ -630,6 +637,7 @@ type CompleteMultipartUpload struct {
|
||||
type NewMultipartUploadResult struct {
|
||||
UploadID string
|
||||
ChecksumAlgo string
|
||||
ChecksumType string
|
||||
}
|
||||
|
||||
type getObjectAttributesResponse struct {
|
||||
@@ -641,10 +649,11 @@ type getObjectAttributesResponse struct {
|
||||
}
|
||||
|
||||
type objectAttributesChecksum struct {
|
||||
ChecksumCRC32 string `xml:",omitempty"`
|
||||
ChecksumCRC32C string `xml:",omitempty"`
|
||||
ChecksumSHA1 string `xml:",omitempty"`
|
||||
ChecksumSHA256 string `xml:",omitempty"`
|
||||
ChecksumCRC32 string `xml:",omitempty"`
|
||||
ChecksumCRC32C string `xml:",omitempty"`
|
||||
ChecksumSHA1 string `xml:",omitempty"`
|
||||
ChecksumSHA256 string `xml:",omitempty"`
|
||||
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||
}
|
||||
|
||||
type objectAttributesParts struct {
|
||||
@@ -657,12 +666,13 @@ type objectAttributesParts struct {
|
||||
}
|
||||
|
||||
type objectAttributesPart struct {
|
||||
PartNumber int
|
||||
Size int64
|
||||
ChecksumCRC32 string `xml:",omitempty"`
|
||||
ChecksumCRC32C string `xml:",omitempty"`
|
||||
ChecksumSHA1 string `xml:",omitempty"`
|
||||
ChecksumSHA256 string `xml:",omitempty"`
|
||||
PartNumber int
|
||||
Size int64
|
||||
ChecksumCRC32 string `xml:",omitempty"`
|
||||
ChecksumCRC32C string `xml:",omitempty"`
|
||||
ChecksumSHA1 string `xml:",omitempty"`
|
||||
ChecksumSHA256 string `xml:",omitempty"`
|
||||
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||
}
|
||||
|
||||
type objectAttributesErrorResponse struct {
|
||||
|
||||
@@ -201,13 +201,16 @@ func (z *CompleteMultipartUpload) Msgsize() (s int) {
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *CompletePart) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// map header, size 6
|
||||
// map header, size 8
|
||||
// string "PartNumber"
|
||||
o = append(o, 0x86, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72)
|
||||
o = append(o, 0x88, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72)
|
||||
o = msgp.AppendInt(o, z.PartNumber)
|
||||
// string "ETag"
|
||||
o = append(o, 0xa4, 0x45, 0x54, 0x61, 0x67)
|
||||
o = msgp.AppendString(o, z.ETag)
|
||||
// string "Size"
|
||||
o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65)
|
||||
o = msgp.AppendInt64(o, z.Size)
|
||||
// string "ChecksumCRC32"
|
||||
o = append(o, 0xad, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x43, 0x52, 0x43, 0x33, 0x32)
|
||||
o = msgp.AppendString(o, z.ChecksumCRC32)
|
||||
@@ -220,6 +223,9 @@ func (z *CompletePart) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
// string "ChecksumSHA256"
|
||||
o = append(o, 0xae, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36)
|
||||
o = msgp.AppendString(o, z.ChecksumSHA256)
|
||||
// string "ChecksumCRC64NVME"
|
||||
o = append(o, 0xb1, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x43, 0x52, 0x43, 0x36, 0x34, 0x4e, 0x56, 0x4d, 0x45)
|
||||
o = msgp.AppendString(o, z.ChecksumCRC64NVME)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -253,6 +259,12 @@ func (z *CompletePart) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
err = msgp.WrapError(err, "ETag")
|
||||
return
|
||||
}
|
||||
case "Size":
|
||||
z.Size, bts, err = msgp.ReadInt64Bytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Size")
|
||||
return
|
||||
}
|
||||
case "ChecksumCRC32":
|
||||
z.ChecksumCRC32, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
@@ -277,6 +289,12 @@ func (z *CompletePart) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
err = msgp.WrapError(err, "ChecksumSHA256")
|
||||
return
|
||||
}
|
||||
case "ChecksumCRC64NVME":
|
||||
z.ChecksumCRC64NVME, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ChecksumCRC64NVME")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
@@ -291,7 +309,7 @@ func (z *CompletePart) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *CompletePart) Msgsize() (s int) {
|
||||
s = 1 + 11 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.ETag) + 14 + msgp.StringPrefixSize + len(z.ChecksumCRC32) + 15 + msgp.StringPrefixSize + len(z.ChecksumCRC32C) + 13 + msgp.StringPrefixSize + len(z.ChecksumSHA1) + 15 + msgp.StringPrefixSize + len(z.ChecksumSHA256)
|
||||
s = 1 + 11 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.ETag) + 5 + msgp.Int64Size + 14 + msgp.StringPrefixSize + len(z.ChecksumCRC32) + 15 + msgp.StringPrefixSize + len(z.ChecksumCRC32C) + 13 + msgp.StringPrefixSize + len(z.ChecksumSHA1) + 15 + msgp.StringPrefixSize + len(z.ChecksumSHA256) + 18 + msgp.StringPrefixSize + len(z.ChecksumCRC64NVME)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -956,9 +974,9 @@ func (z *ListObjectsV2Info) Msgsize() (s int) {
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *ListPartsInfo) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// map header, size 11
|
||||
// map header, size 12
|
||||
// string "Bucket"
|
||||
o = append(o, 0x8b, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
|
||||
o = append(o, 0x8c, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
|
||||
o = msgp.AppendString(o, z.Bucket)
|
||||
// string "Object"
|
||||
o = append(o, 0xa6, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74)
|
||||
@@ -1001,6 +1019,9 @@ func (z *ListPartsInfo) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
// string "ChecksumAlgorithm"
|
||||
o = append(o, 0xb1, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d)
|
||||
o = msgp.AppendString(o, z.ChecksumAlgorithm)
|
||||
// string "ChecksumType"
|
||||
o = append(o, 0xac, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x54, 0x79, 0x70, 0x65)
|
||||
o = msgp.AppendString(o, z.ChecksumType)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1125,6 +1146,12 @@ func (z *ListPartsInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
err = msgp.WrapError(err, "ChecksumAlgorithm")
|
||||
return
|
||||
}
|
||||
case "ChecksumType":
|
||||
z.ChecksumType, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ChecksumType")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
@@ -1150,7 +1177,7 @@ func (z *ListPartsInfo) Msgsize() (s int) {
|
||||
s += msgp.StringPrefixSize + len(za0002) + msgp.StringPrefixSize + len(za0003)
|
||||
}
|
||||
}
|
||||
s += 18 + msgp.StringPrefixSize + len(z.ChecksumAlgorithm)
|
||||
s += 18 + msgp.StringPrefixSize + len(z.ChecksumAlgorithm) + 13 + msgp.StringPrefixSize + len(z.ChecksumType)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1279,13 +1306,16 @@ func (z *MultipartInfo) Msgsize() (s int) {
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z NewMultipartUploadResult) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// map header, size 2
|
||||
// map header, size 3
|
||||
// string "UploadID"
|
||||
o = append(o, 0x82, 0xa8, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x44)
|
||||
o = append(o, 0x83, 0xa8, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x44)
|
||||
o = msgp.AppendString(o, z.UploadID)
|
||||
// string "ChecksumAlgo"
|
||||
o = append(o, 0xac, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x41, 0x6c, 0x67, 0x6f)
|
||||
o = msgp.AppendString(o, z.ChecksumAlgo)
|
||||
// string "ChecksumType"
|
||||
o = append(o, 0xac, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x54, 0x79, 0x70, 0x65)
|
||||
o = msgp.AppendString(o, z.ChecksumType)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1319,6 +1349,12 @@ func (z *NewMultipartUploadResult) UnmarshalMsg(bts []byte) (o []byte, err error
|
||||
err = msgp.WrapError(err, "ChecksumAlgo")
|
||||
return
|
||||
}
|
||||
case "ChecksumType":
|
||||
z.ChecksumType, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ChecksumType")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
@@ -1333,7 +1369,7 @@ func (z *NewMultipartUploadResult) UnmarshalMsg(bts []byte) (o []byte, err error
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z NewMultipartUploadResult) Msgsize() (s int) {
|
||||
s = 1 + 9 + msgp.StringPrefixSize + len(z.UploadID) + 13 + msgp.StringPrefixSize + len(z.ChecksumAlgo)
|
||||
s = 1 + 9 + msgp.StringPrefixSize + len(z.UploadID) + 13 + msgp.StringPrefixSize + len(z.ChecksumAlgo) + 13 + msgp.StringPrefixSize + len(z.ChecksumType)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1772,9 +1808,9 @@ func (z *ObjectInfo) Msgsize() (s int) {
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *PartInfo) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// map header, size 9
|
||||
// map header, size 10
|
||||
// string "PartNumber"
|
||||
o = append(o, 0x89, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72)
|
||||
o = append(o, 0x8a, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72)
|
||||
o = msgp.AppendInt(o, z.PartNumber)
|
||||
// string "LastModified"
|
||||
o = append(o, 0xac, 0x4c, 0x61, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64)
|
||||
@@ -1800,6 +1836,9 @@ func (z *PartInfo) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
// string "ChecksumSHA256"
|
||||
o = append(o, 0xae, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36)
|
||||
o = msgp.AppendString(o, z.ChecksumSHA256)
|
||||
// string "ChecksumCRC64NVME"
|
||||
o = append(o, 0xb1, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x43, 0x52, 0x43, 0x36, 0x34, 0x4e, 0x56, 0x4d, 0x45)
|
||||
o = msgp.AppendString(o, z.ChecksumCRC64NVME)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1875,6 +1914,12 @@ func (z *PartInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
err = msgp.WrapError(err, "ChecksumSHA256")
|
||||
return
|
||||
}
|
||||
case "ChecksumCRC64NVME":
|
||||
z.ChecksumCRC64NVME, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ChecksumCRC64NVME")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
@@ -1889,7 +1934,7 @@ func (z *PartInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *PartInfo) Msgsize() (s int) {
|
||||
s = 1 + 11 + msgp.IntSize + 13 + msgp.TimeSize + 5 + msgp.StringPrefixSize + len(z.ETag) + 5 + msgp.Int64Size + 11 + msgp.Int64Size + 14 + msgp.StringPrefixSize + len(z.ChecksumCRC32) + 15 + msgp.StringPrefixSize + len(z.ChecksumCRC32C) + 13 + msgp.StringPrefixSize + len(z.ChecksumSHA1) + 15 + msgp.StringPrefixSize + len(z.ChecksumSHA256)
|
||||
s = 1 + 11 + msgp.IntSize + 13 + msgp.TimeSize + 5 + msgp.StringPrefixSize + len(z.ETag) + 5 + msgp.Int64Size + 11 + msgp.Int64Size + 14 + msgp.StringPrefixSize + len(z.ChecksumCRC32) + 15 + msgp.StringPrefixSize + len(z.ChecksumCRC32C) + 13 + msgp.StringPrefixSize + len(z.ChecksumSHA1) + 15 + msgp.StringPrefixSize + len(z.ChecksumSHA256) + 18 + msgp.StringPrefixSize + len(z.ChecksumCRC64NVME)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -389,7 +389,6 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
|
||||
if err != nil {
|
||||
t.Fatalf("%s : %s", instanceType, err.Error())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Formulating the result data set to be expected from ListObjects call inside the tests,
|
||||
@@ -1014,7 +1013,6 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
|
||||
t.Errorf("Test %d: %s: Expected NextMarker to be empty since listing is not truncated, but instead found `%v`", i+1, instanceType, result.NextMarker)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1166,7 +1164,6 @@ func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHand
|
||||
if err != nil {
|
||||
t.Fatalf("%s : %s", instanceType, err.Error())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Formualting the result data set to be expected from ListObjects call inside the tests,
|
||||
@@ -1785,12 +1782,10 @@ func testListObjectsContinuation(obj ObjectLayer, instanceType string, t1 TestEr
|
||||
if err != nil {
|
||||
t.Fatalf("%s : %s", instanceType, err.Error())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Formulating the result data set to be expected from ListObjects call inside the tests,
|
||||
// This will be used in testCases and used for asserting the correctness of ListObjects output in the tests.
|
||||
|
||||
resultCases := []ListObjectsInfo{
|
||||
{
|
||||
Objects: []ObjectInfo{
|
||||
|
||||
@@ -443,7 +443,6 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
|
||||
if err != nil {
|
||||
t.Fatalf("%s : %s", instanceType, err.Error())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Expected Results set for asserting ListObjectMultipart test.
|
||||
|
||||
@@ -404,12 +404,7 @@ func putOptsFromHeaders(ctx context.Context, hdr http.Header, metadata map[strin
|
||||
metadata = make(map[string]string)
|
||||
}
|
||||
|
||||
wantCRC, err := hash.GetContentChecksum(hdr)
|
||||
if err != nil {
|
||||
return opts, fmt.Errorf("invalid/unknown checksum sent: %v", err)
|
||||
}
|
||||
etag := strings.TrimSpace(hdr.Get(xhttp.MinIOSourceETag))
|
||||
|
||||
if crypto.S3KMS.IsRequested(hdr) {
|
||||
keyID, context, err := crypto.S3KMS.ParseHTTP(hdr)
|
||||
if err != nil {
|
||||
@@ -423,7 +418,6 @@ func putOptsFromHeaders(ctx context.Context, hdr http.Header, metadata map[strin
|
||||
ServerSideEncryption: sseKms,
|
||||
UserDefined: metadata,
|
||||
MTime: mtime,
|
||||
WantChecksum: wantCRC,
|
||||
PreserveETag: etag,
|
||||
}, nil
|
||||
}
|
||||
@@ -438,7 +432,6 @@ func putOptsFromHeaders(ctx context.Context, hdr http.Header, metadata map[strin
|
||||
opts.ReplicationSourceRetentionTimestamp = retaintimestmp
|
||||
opts.ReplicationSourceTaggingTimestamp = taggingtimestmp
|
||||
opts.PreserveETag = etag
|
||||
opts.WantChecksum = wantCRC
|
||||
|
||||
return opts, nil
|
||||
}
|
||||
@@ -474,13 +467,10 @@ func completeMultipartOpts(ctx context.Context, r *http.Request, bucket, object
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
opts.WantChecksum, err = hash.GetContentChecksum(r.Header)
|
||||
if err != nil {
|
||||
return opts, InvalidArgument{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
Err: fmt.Errorf("invalid/unknown checksum sent: %v", err),
|
||||
}
|
||||
return opts, err
|
||||
}
|
||||
opts.MTime = mtime
|
||||
opts.UserDefined = make(map[string]string)
|
||||
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
"net/http"
|
||||
"path"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -52,7 +53,6 @@ import (
|
||||
"github.com/minio/pkg/v3/trie"
|
||||
"github.com/minio/pkg/v3/wildcard"
|
||||
"github.com/valyala/bytebufferpool"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -352,11 +352,11 @@ func isETagEqual(left, right string) bool {
|
||||
// setPutObjHeaders sets all the necessary headers returned back
|
||||
// upon a success Put/Copy/CompleteMultipart/Delete requests
|
||||
// to activate delete only headers set delete as true
|
||||
func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, delete bool, h http.Header) {
|
||||
func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, del bool, h http.Header) {
|
||||
// We must not use the http.Header().Set method here because some (broken)
|
||||
// clients expect the ETag header key to be literally "ETag" - not "Etag" (case-sensitive).
|
||||
// Therefore, we have to set the ETag directly as map entry.
|
||||
if objInfo.ETag != "" && !delete {
|
||||
if objInfo.ETag != "" && !del {
|
||||
w.Header()[xhttp.ETag] = []string{`"` + objInfo.ETag + `"`}
|
||||
}
|
||||
|
||||
@@ -364,17 +364,18 @@ func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, delete bool, h
|
||||
if objInfo.VersionID != "" && objInfo.VersionID != nullVersionID {
|
||||
w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
|
||||
// If version is a deleted marker, set this header as well
|
||||
if objInfo.DeleteMarker && delete { // only returned during delete object
|
||||
if objInfo.DeleteMarker && del { // only returned during delete object
|
||||
w.Header()[xhttp.AmzDeleteMarker] = []string{strconv.FormatBool(objInfo.DeleteMarker)}
|
||||
}
|
||||
}
|
||||
|
||||
if objInfo.Bucket != "" && objInfo.Name != "" {
|
||||
if lc, err := globalLifecycleSys.Get(objInfo.Bucket); err == nil && !delete {
|
||||
if lc, err := globalLifecycleSys.Get(objInfo.Bucket); err == nil && !del {
|
||||
lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts())
|
||||
}
|
||||
}
|
||||
hash.AddChecksumHeader(w, objInfo.decryptChecksums(0, h))
|
||||
cs, _ := objInfo.decryptChecksums(0, h)
|
||||
hash.AddChecksumHeader(w, cs)
|
||||
}
|
||||
|
||||
func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toDel []ObjectToDelete, lcEvent lifecycle.Event) {
|
||||
|
||||
+60
-42
@@ -521,7 +521,8 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
|
||||
|
||||
if r.Header.Get(xhttp.AmzChecksumMode) == "ENABLED" && rs == nil {
|
||||
// AWS S3 silently drops checksums on range requests.
|
||||
hash.AddChecksumHeader(w, objInfo.decryptChecksums(opts.PartNumber, r.Header))
|
||||
cs, _ := objInfo.decryptChecksums(opts.PartNumber, r.Header)
|
||||
hash.AddChecksumHeader(w, cs)
|
||||
}
|
||||
|
||||
if err = setObjectHeaders(ctx, w, objInfo, rs, opts); err != nil {
|
||||
@@ -632,14 +633,15 @@ func (api objectAPIHandlers) getObjectAttributesHandler(ctx context.Context, obj
|
||||
w.Header().Del(xhttp.ContentType)
|
||||
|
||||
if _, ok := opts.ObjectAttributes[xhttp.Checksum]; ok {
|
||||
chkSums := objInfo.decryptChecksums(0, r.Header)
|
||||
chkSums, _ := objInfo.decryptChecksums(0, r.Header)
|
||||
// AWS does not appear to append part number on this API call.
|
||||
if len(chkSums) > 0 {
|
||||
OA.Checksum = &objectAttributesChecksum{
|
||||
ChecksumCRC32: strings.Split(chkSums["CRC32"], "-")[0],
|
||||
ChecksumCRC32C: strings.Split(chkSums["CRC32C"], "-")[0],
|
||||
ChecksumSHA1: strings.Split(chkSums["SHA1"], "-")[0],
|
||||
ChecksumSHA256: strings.Split(chkSums["SHA256"], "-")[0],
|
||||
ChecksumCRC32: strings.Split(chkSums["CRC32"], "-")[0],
|
||||
ChecksumCRC32C: strings.Split(chkSums["CRC32C"], "-")[0],
|
||||
ChecksumSHA1: strings.Split(chkSums["SHA1"], "-")[0],
|
||||
ChecksumSHA256: strings.Split(chkSums["SHA256"], "-")[0],
|
||||
ChecksumCRC64NVME: strings.Split(chkSums["CRC64NVME"], "-")[0],
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -678,12 +680,13 @@ func (api objectAPIHandlers) getObjectAttributesHandler(ctx context.Context, obj
|
||||
|
||||
OA.ObjectParts.NextPartNumberMarker = v.Number
|
||||
OA.ObjectParts.Parts = append(OA.ObjectParts.Parts, &objectAttributesPart{
|
||||
ChecksumSHA1: objInfo.Parts[i].Checksums["SHA1"],
|
||||
ChecksumSHA256: objInfo.Parts[i].Checksums["SHA256"],
|
||||
ChecksumCRC32: objInfo.Parts[i].Checksums["CRC32"],
|
||||
ChecksumCRC32C: objInfo.Parts[i].Checksums["CRC32C"],
|
||||
PartNumber: objInfo.Parts[i].Number,
|
||||
Size: objInfo.Parts[i].Size,
|
||||
ChecksumSHA1: objInfo.Parts[i].Checksums["SHA1"],
|
||||
ChecksumSHA256: objInfo.Parts[i].Checksums["SHA256"],
|
||||
ChecksumCRC32: objInfo.Parts[i].Checksums["CRC32"],
|
||||
ChecksumCRC32C: objInfo.Parts[i].Checksums["CRC32C"],
|
||||
ChecksumCRC64NVME: objInfo.Parts[i].Checksums["CRC64NVME"],
|
||||
PartNumber: objInfo.Parts[i].Number,
|
||||
Size: objInfo.Parts[i].Size,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -943,7 +946,8 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
|
||||
|
||||
if r.Header.Get(xhttp.AmzChecksumMode) == "ENABLED" && rs == nil {
|
||||
// AWS S3 silently drops checksums on range requests.
|
||||
hash.AddChecksumHeader(w, objInfo.decryptChecksums(opts.PartNumber, r.Header))
|
||||
cs, _ := objInfo.decryptChecksums(opts.PartNumber, r.Header)
|
||||
hash.AddChecksumHeader(w, cs)
|
||||
}
|
||||
|
||||
// Set standard object headers.
|
||||
@@ -1191,6 +1195,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
return
|
||||
}
|
||||
|
||||
// Sanitize the source object name similar to NewMultipart and PutObject API
|
||||
srcObject = trimLeadingSlash(srcObject)
|
||||
|
||||
if vid != "" && vid != nullVersionID {
|
||||
_, err := uuid.Parse(vid)
|
||||
if err != nil {
|
||||
@@ -1531,7 +1538,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
srcInfo.UserDefined[xhttp.AmzObjectTagging] = objTags
|
||||
srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
srcInfo.UserDefined = filterReplicationStatusMetadata(srcInfo.UserDefined)
|
||||
@@ -1624,7 +1630,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
srcInfo.metadataOnly && srcOpts.VersionID == "" &&
|
||||
!crypto.Requested(r.Header) &&
|
||||
!crypto.IsSourceEncrypted(srcInfo.UserDefined) {
|
||||
|
||||
// If x-amz-metadata-directive is not set to REPLACE then we need
|
||||
// to error out if source and destination are same.
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopyDest), r.URL)
|
||||
@@ -1899,6 +1904,13 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
var reader io.Reader
|
||||
reader = rd
|
||||
|
||||
var opts ObjectOptions
|
||||
opts, err = putOptsFromReq(ctx, r, bucket, object, metadata)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
actualSize := size
|
||||
var idxCb func() []byte
|
||||
if isCompressible(r.Header, object) && size > minCompressibleSize {
|
||||
@@ -1915,6 +1927,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
|
||||
return
|
||||
}
|
||||
opts.WantChecksum = actualReader.Checksum()
|
||||
|
||||
// Set compression metrics.
|
||||
var s2c io.ReadCloser
|
||||
wantEncryption := crypto.Requested(r.Header)
|
||||
@@ -1945,22 +1959,17 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if err := hashReader.AddChecksum(r, size < 0); err != nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
|
||||
return
|
||||
if size >= 0 {
|
||||
if err := hashReader.AddChecksum(r, false); err != nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
|
||||
return
|
||||
}
|
||||
opts.WantChecksum = hashReader.Checksum()
|
||||
}
|
||||
|
||||
rawReader := hashReader
|
||||
pReader := NewPutObjReader(rawReader)
|
||||
|
||||
var opts ObjectOptions
|
||||
opts, err = putOptsFromReq(ctx, r, bucket, object, metadata)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
opts.IndexCB = idxCb
|
||||
opts.WantChecksum = hashReader.Checksum()
|
||||
|
||||
if opts.PreserveETag != "" ||
|
||||
r.Header.Get(xhttp.IfMatch) != "" ||
|
||||
@@ -2220,8 +2229,15 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
|
||||
putObject = objectAPI.PutObject
|
||||
)
|
||||
|
||||
// Check if put is allowed
|
||||
if s3Err = isPutActionAllowed(ctx, rAuthType, bucket, object, r, policy.PutObjectAction); s3Err != ErrNone {
|
||||
var opts untarOptions
|
||||
opts.ignoreDirs = strings.EqualFold(r.Header.Get(xhttp.MinIOSnowballIgnoreDirs), "true")
|
||||
opts.ignoreErrs = strings.EqualFold(r.Header.Get(xhttp.MinIOSnowballIgnoreErrors), "true")
|
||||
opts.prefixAll = r.Header.Get(xhttp.MinIOSnowballPrefix)
|
||||
if opts.prefixAll != "" {
|
||||
opts.prefixAll = trimLeadingSlash(pathJoin(opts.prefixAll, slashSeparator))
|
||||
}
|
||||
// Check if put is allow for specified prefix.
|
||||
if s3Err = isPutActionAllowed(ctx, rAuthType, bucket, opts.prefixAll, r, policy.PutObjectAction); s3Err != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -2289,7 +2305,10 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
|
||||
|
||||
putObjectTar := func(reader io.Reader, info os.FileInfo, object string) error {
|
||||
size := info.Size()
|
||||
|
||||
if s3Err = isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.PutObjectAction); s3Err != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return errors.New(errorCodes.ToAPIErr(s3Err).Code)
|
||||
}
|
||||
metadata := map[string]string{
|
||||
xhttp.AmzStorageClass: sc, // save same storage-class as incoming stream.
|
||||
}
|
||||
@@ -2325,7 +2344,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
|
||||
|
||||
if r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() {
|
||||
if s3Err = isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.ReplicateObjectAction); s3Err != ErrNone {
|
||||
return err
|
||||
return errors.New(errorCodes.ToAPIErr(s3Err).Code)
|
||||
}
|
||||
metadata[ReservedMetadataPrefixLower+ReplicaStatus] = replication.Replica.String()
|
||||
metadata[ReservedMetadataPrefixLower+ReplicaTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
@@ -2389,7 +2408,6 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
|
||||
if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(metadata, "", "", replication.ObjectReplicationType, opts)); dsc.ReplicateAny() {
|
||||
metadata[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
metadata[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus()
|
||||
|
||||
}
|
||||
|
||||
var objectEncryptionKey crypto.ObjectKey
|
||||
@@ -2472,14 +2490,6 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
|
||||
return nil
|
||||
}
|
||||
|
||||
var opts untarOptions
|
||||
opts.ignoreDirs = strings.EqualFold(r.Header.Get(xhttp.MinIOSnowballIgnoreDirs), "true")
|
||||
opts.ignoreErrs = strings.EqualFold(r.Header.Get(xhttp.MinIOSnowballIgnoreErrors), "true")
|
||||
opts.prefixAll = r.Header.Get(xhttp.MinIOSnowballPrefix)
|
||||
if opts.prefixAll != "" {
|
||||
opts.prefixAll = trimLeadingSlash(pathJoin(opts.prefixAll, slashSeparator))
|
||||
}
|
||||
|
||||
if err = untar(ctx, hreader, putObjectTar, opts); err != nil {
|
||||
apiErr := errorCodes.ToAPIErr(s3Err)
|
||||
// If not set, convert or use BadRequest
|
||||
@@ -2724,7 +2734,7 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if !hasContentMD5(r.Header) {
|
||||
if !validateLengthAndChecksum(r) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -2734,7 +2744,7 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
legalHold, err := objectlock.ParseObjectLegalHold(io.LimitReader(r.Body, r.ContentLength))
|
||||
legalHold, err := objectlock.ParseObjectLegalHold(r.Body)
|
||||
if err != nil {
|
||||
apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
|
||||
apiErr.Description = err.Error()
|
||||
@@ -2871,6 +2881,12 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
// Check permissions to perform this object retention operation
|
||||
if s3Err := checkRequestAuthType(ctx, r, policy.PutObjectRetentionAction, bucket, object); s3Err != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
cred, owner, s3Err := validateSignature(getRequestAuthType(r), r)
|
||||
if s3Err != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
@@ -2882,7 +2898,7 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
if !hasContentMD5(r.Header) {
|
||||
if !validateLengthAndChecksum(r) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -2899,6 +2915,8 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
|
||||
writeErrorResponse(ctx, w, apiErr, r.URL)
|
||||
return
|
||||
}
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
reqInfo.SetTags("retention", objRetention.String())
|
||||
|
||||
opts, err := getOpts(ctx, r, bucket, object)
|
||||
if err != nil {
|
||||
|
||||
@@ -814,7 +814,6 @@ func testAPIGetObjectWithMPHandler(obj ObjectLayer, instanceType, bucketName str
|
||||
caseNumber++
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// HTTP request for testing when `objectLayer` is set to `nil`.
|
||||
@@ -1567,8 +1566,8 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if testCase.expectedRespStatus == http.StatusOK {
|
||||
buffer := new(bytes.Buffer)
|
||||
// Fetch the object to check whether the content is same as the one uploaded via PutObject.
|
||||
@@ -3520,7 +3519,6 @@ func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string
|
||||
t.Errorf("Case %d: MinIO %s: Expected the response status to be `%d`, but instead found `%d`", i+1,
|
||||
instanceType, testCase.expectedRespStatus, recV2.Code)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Test for Anonymous/unsigned http request.
|
||||
@@ -4198,7 +4196,6 @@ func testAPIListObjectPartsHandler(obj ObjectLayer, instanceType, bucketName str
|
||||
|
||||
// validate the error response.
|
||||
if test.expectedErr != noAPIErr {
|
||||
|
||||
var errBytes []byte
|
||||
// read the response body.
|
||||
errBytes, err = io.ReadAll(rec.Result().Body)
|
||||
|
||||
@@ -93,8 +93,8 @@ func getLambdaEventData(bucket, object string, cred auth.Credentials, r *http.Re
|
||||
},
|
||||
UserIdentity: levent.Identity{
|
||||
Type: "IAMUser",
|
||||
PrincipalID: cred.AccessKey,
|
||||
AccessKeyID: cred.SecretKey,
|
||||
PrincipalID: cred.ParentUser,
|
||||
AccessKeyID: cred.AccessKey,
|
||||
},
|
||||
}
|
||||
return eventData, nil
|
||||
|
||||
@@ -214,9 +214,9 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
|
||||
}
|
||||
}
|
||||
|
||||
checksumType := hash.NewChecksumType(r.Header.Get(xhttp.AmzChecksumAlgo))
|
||||
checksumType := hash.NewChecksumHeader(r.Header)
|
||||
if checksumType.Is(hash.ChecksumInvalid) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequestParameter), r.URL)
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
|
||||
return
|
||||
} else if checksumType.IsSet() && !checksumType.Is(hash.ChecksumTrailing) {
|
||||
opts.WantChecksum = &hash.Checksum{Type: checksumType}
|
||||
@@ -233,6 +233,9 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
|
||||
response := generateInitiateMultipartUploadResponse(bucket, object, res.UploadID)
|
||||
if res.ChecksumAlgo != "" {
|
||||
w.Header().Set(xhttp.AmzChecksumAlgo, res.ChecksumAlgo)
|
||||
if res.ChecksumType != "" {
|
||||
w.Header().Set(xhttp.AmzChecksumType, res.ChecksumType)
|
||||
}
|
||||
}
|
||||
encodedSuccessResponse := encodeResponse(response)
|
||||
|
||||
@@ -1204,6 +1207,10 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
|
||||
listPartsInfo.Parts[i].ETag = tryDecryptETag(objectEncryptionKey, p.ETag, kind == crypto.S3)
|
||||
listPartsInfo.Parts[i].Size = p.ActualSize
|
||||
}
|
||||
} else if _, ok := listPartsInfo.UserDefined[ReservedMetadataPrefix+"compression"]; ok {
|
||||
for i, p := range listPartsInfo.Parts {
|
||||
listPartsInfo.Parts[i].Size = p.ActualSize
|
||||
}
|
||||
}
|
||||
|
||||
response := generateListPartsResponse(listPartsInfo, encodingType)
|
||||
|
||||
@@ -228,7 +228,6 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrH
|
||||
if objInfo.Size != int64(len(value)) {
|
||||
t.Errorf("%s: Size mismatch of the GetObject data.", instanceType)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -384,7 +383,6 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
|
||||
|
||||
// check results with Marker.
|
||||
{
|
||||
|
||||
result, err = obj.ListObjects(context.Background(), "bucket", "", "newPrefix", "", 3)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: <ERROR> %s", instanceType, err)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user