mirror of
https://github.com/pgsty/minio.git
synced 2026-08-14 10:43:15 +03:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6105cc6ab4 | |||
| 068ad33c9a | |||
| 2db5cbc7a5 |
@@ -1,12 +1,3 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: community, triage
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
<!--- Provide a general summary of the issue in the Title above -->
|
||||
|
||||
## Expected Behavior
|
||||
@@ -24,8 +15,6 @@ assignees: ''
|
||||
## Steps to Reproduce (for bugs)
|
||||
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
|
||||
<!--- reproduce this bug. Include code to reproduce, if relevant -->
|
||||
<!--- and make sure you have followed https://github.com/minio/minio/tree/release/docs/debugging to capture relevant logs -->
|
||||
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
@@ -41,6 +30,8 @@ assignees: ''
|
||||
|
||||
## Your Environment
|
||||
<!--- Include as many relevant details about the environment you experienced the bug in -->
|
||||
* Version used (`minio --version`):
|
||||
* Server setup and configuration:
|
||||
* Version used (`minio version`):
|
||||
* Environment name and version (e.g. nginx 1.9.1):
|
||||
* Server type and version:
|
||||
* Operating System and version (`uname -a`):
|
||||
* Link to your project:
|
||||
|
||||
@@ -24,8 +24,6 @@ assignees: ''
|
||||
## Steps to Reproduce (for bugs)
|
||||
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
|
||||
<!--- reproduce this bug. Include code to reproduce, if relevant -->
|
||||
<!--- and make sure you have followed https://github.com/minio/minio/tree/release/docs/debugging to capture relevant logs -->
|
||||
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
@@ -41,6 +39,8 @@ assignees: ''
|
||||
|
||||
## Your Environment
|
||||
<!--- Include as many relevant details about the environment you experienced the bug in -->
|
||||
* Version used (`minio --version`):
|
||||
* Server setup and configuration:
|
||||
* Version used (`minio version`):
|
||||
* Environment name and version (e.g. nginx 1.9.1):
|
||||
* Server type and version:
|
||||
* Operating System and version (`uname -a`):
|
||||
* Link to your project:
|
||||
|
||||
@@ -2,7 +2,7 @@ blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: MinIO Community Support
|
||||
url: https://slack.min.io
|
||||
about: Join here for Community Support
|
||||
about: Please ask and answer questions here.
|
||||
- name: MinIO SUBNET Support
|
||||
url: https://min.io/pricing
|
||||
about: Join here for Enterprise Support
|
||||
about: Join this for Enterprise Support.
|
||||
|
||||
@@ -16,3 +16,4 @@
|
||||
- [ ] Fixes a regression (If yes, please add `commit-id` or `PR #` here)
|
||||
- [ ] Documentation needed
|
||||
- [ ] Unit tests needed
|
||||
- [ ] Functional tests needed (If yes, add [mint](https://github.com/minio/mint) PR # here: )
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ skipCreatedBefore: false
|
||||
exemptLabels: []
|
||||
|
||||
# Label to add before locking, such as `outdated`. Set to `false` to disable
|
||||
lockLabel: true
|
||||
lockLabel: false
|
||||
|
||||
# Comment to post before locking. Set to `false` to disable
|
||||
lockComment: >-
|
||||
|
||||
+3
-3
@@ -1,11 +1,11 @@
|
||||
# Configuration for probot-stale - https://github.com/probot/stale
|
||||
|
||||
# Number of days of inactivity before an Issue or Pull Request becomes stale
|
||||
daysUntilStale: 30
|
||||
daysUntilStale: 90
|
||||
|
||||
# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
|
||||
# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
|
||||
daysUntilClose: 15
|
||||
daysUntilClose: 30
|
||||
|
||||
# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
|
||||
onlyLabels: []
|
||||
@@ -30,7 +30,7 @@ staleLabel: stale
|
||||
# Comment to post when marking as stale. Set to `false` to disable
|
||||
markComment: >-
|
||||
This issue has been automatically marked as stale because it has not had
|
||||
recent activity. It will be closed after 15 days if no further activity
|
||||
recent activity. It will be closed after 21 days if no further activity
|
||||
occurs. Thank you for your contributions.
|
||||
# Comment to post when removing the stale label.
|
||||
# unmarkComment: >
|
||||
|
||||
@@ -4,7 +4,9 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- release
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
||||
-167
@@ -1,167 +0,0 @@
|
||||
project_name: minio
|
||||
|
||||
release:
|
||||
name_template: "Version {{.MinIO.Version}}"
|
||||
disable: true
|
||||
github:
|
||||
owner: minio
|
||||
name: minio
|
||||
|
||||
env:
|
||||
- CGO_ENABLED=0
|
||||
- GO111MODULE=on
|
||||
|
||||
before:
|
||||
hooks:
|
||||
- make clean
|
||||
- go generate ./...
|
||||
- go mod tidy
|
||||
- go mod download
|
||||
|
||||
builds:
|
||||
-
|
||||
goos:
|
||||
- linux
|
||||
- darwin
|
||||
- windows
|
||||
- freebsd
|
||||
|
||||
goarch:
|
||||
- amd64
|
||||
- arm64
|
||||
- arm
|
||||
- ppc64le
|
||||
- s390x
|
||||
|
||||
goarm:
|
||||
- 7
|
||||
|
||||
ignore:
|
||||
- goos: darwin
|
||||
goarch: arm64
|
||||
- goos: darwin
|
||||
goarch: arm
|
||||
- goos: darwin
|
||||
goarch: ppc64le
|
||||
- goos: darwin
|
||||
goarch: s390x
|
||||
- goos: windows
|
||||
goarch: arm64
|
||||
- goos: windows
|
||||
goarch: arm
|
||||
- goos: windows
|
||||
goarch: ppc64le
|
||||
- goos: windows
|
||||
goarch: s390x
|
||||
- goos: freebsd
|
||||
goarch: arm
|
||||
- goos: freebsd
|
||||
goarch: arm64
|
||||
- goos: freebsd
|
||||
goarch: ppc64le
|
||||
- goos: freebsd
|
||||
goarch: s390x
|
||||
|
||||
flags:
|
||||
- -tags=kqueue
|
||||
- -trimpath
|
||||
|
||||
ldflags:
|
||||
- "-s -w -X github.com/minio/minio/cmd.Version={{.Version}} -X github.com/minio/minio/cmd.ReleaseTag={{.Tag}} -X github.com/minio/minio/cmd.CommitID={{.FullCommit}} -X github.com/minio/minio/cmd.ShortCommitID={{.ShortCommit}}"
|
||||
|
||||
archives:
|
||||
-
|
||||
format: binary
|
||||
name_template: "{{ .Binary }}-release/{{ .Os }}-{{ .Arch }}/{{ .Binary }}.{{ .Version }}"
|
||||
|
||||
nfpms:
|
||||
-
|
||||
id: minio
|
||||
package_name: minio
|
||||
vendor: MinIO, Inc.
|
||||
homepage: https://min.io/
|
||||
maintainer: dev@min.io
|
||||
description: MinIO is a High Performance Object Storage released under Apache License v2.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads.
|
||||
license: Apache 2.0
|
||||
bindir: /usr/bin
|
||||
formats:
|
||||
- deb
|
||||
- rpm
|
||||
overrides:
|
||||
deb:
|
||||
file_name_template: "{{ .Binary }}-release/debs/{{ .ProjectName }}-{{ .Version }}_{{ .Arch }}"
|
||||
replacements:
|
||||
arm: armv7
|
||||
files:
|
||||
"NOTICE": "/usr/share/minio/NOTICE"
|
||||
"CREDITS": "/usr/share/minio/CREDITS"
|
||||
"LICENSE": "/usr/share/minio/LICENSE"
|
||||
"README.md": "/usr/share/minio/README.md"
|
||||
rpm:
|
||||
file_name_template: "{{ .Binary }}-release/rpms/{{ .ProjectName }}-{{ .Version }}.{{ .Arch }}"
|
||||
replacements:
|
||||
amd64: x86_64
|
||||
arm64: aarch64
|
||||
arm: armv7
|
||||
files:
|
||||
"NOTICE": "/usr/share/minio/NOTICE"
|
||||
"CREDITS": "/usr/share/minio/CREDITS"
|
||||
"LICENSE": "/usr/share/minio/LICENSE"
|
||||
"README.md": "/usr/share/minio/README.md"
|
||||
|
||||
checksum:
|
||||
algorithm: sha256
|
||||
|
||||
signs:
|
||||
-
|
||||
signature: "${artifact}.asc"
|
||||
cmd: "sh"
|
||||
args:
|
||||
- '-c'
|
||||
- 'gpg --quiet --detach-sign -a ${artifact}'
|
||||
artifacts: all
|
||||
|
||||
changelog:
|
||||
sort: asc
|
||||
filters:
|
||||
exclude:
|
||||
- '^Update yaml files'
|
||||
|
||||
dockers:
|
||||
-
|
||||
goos: linux
|
||||
goarch: amd64
|
||||
dockerfile: Dockerfile.release
|
||||
image_templates:
|
||||
- minio/minio:{{ .Tag }}
|
||||
- minio/minio:latest
|
||||
|
||||
-
|
||||
goos: linux
|
||||
goarch: ppc64le
|
||||
dockerfile: Dockerfile.ppc64le.release
|
||||
image_templates:
|
||||
- minio/minio:{{ .Tag }}-ppc64le
|
||||
|
||||
-
|
||||
goos: linux
|
||||
goarch: s390x
|
||||
dockerfile: Dockerfile.s390x.release
|
||||
image_templates:
|
||||
- minio/minio:{{ .Tag }}-s390x
|
||||
|
||||
-
|
||||
goos: linux
|
||||
goarch: arm64
|
||||
goarm: ''
|
||||
dockerfile: Dockerfile.arm64.release
|
||||
image_templates:
|
||||
- minio/minio:{{ .Tag }}-arm64
|
||||
|
||||
-
|
||||
goos: linux
|
||||
goarch: arm
|
||||
goarm: '7'
|
||||
dockerfile: Dockerfile.arm.release
|
||||
image_templates:
|
||||
- minio/minio:{{ .Tag }}-arm
|
||||
+6
-6
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.13-alpine as builder
|
||||
FROM golang:1.13-alpine
|
||||
|
||||
LABEL maintainer="MinIO Inc <dev@min.io>"
|
||||
|
||||
@@ -9,9 +9,9 @@ ENV GO111MODULE on
|
||||
RUN \
|
||||
apk add --no-cache git && \
|
||||
git clone https://github.com/minio/minio && cd minio && \
|
||||
git checkout master && go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
|
||||
go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
|
||||
|
||||
FROM alpine:3.12
|
||||
FROM alpine:3.10
|
||||
|
||||
ENV MINIO_UPDATE off
|
||||
ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
@@ -21,9 +21,9 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
|
||||
EXPOSE 9000
|
||||
|
||||
COPY --from=builder /go/bin/minio /usr/bin/minio
|
||||
COPY --from=builder /go/minio/CREDITS /third_party/
|
||||
COPY --from=builder /go/minio/dockerscripts/docker-entrypoint.sh /usr/bin/
|
||||
COPY --from=0 /go/bin/minio /usr/bin/minio
|
||||
COPY --from=0 /go/minio/CREDITS /third_party/
|
||||
COPY --from=0 /go/minio/dockerscripts/docker-entrypoint.sh /usr/bin/
|
||||
|
||||
RUN \
|
||||
apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
|
||||
|
||||
+20
-8
@@ -1,9 +1,23 @@
|
||||
FROM multiarch/qemu-user-static:x86_64-arm as qemu
|
||||
FROM arm32v7/alpine:3.12
|
||||
FROM golang:1.13-alpine as builder
|
||||
|
||||
WORKDIR /home
|
||||
|
||||
ENV GOPATH /go
|
||||
ENV CGO_ENABLED 0
|
||||
ENV GO111MODULE on
|
||||
|
||||
RUN \
|
||||
apk add --no-cache git 'curl>7.61.0' && \
|
||||
git clone https://github.com/minio/minio && \
|
||||
curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static .
|
||||
|
||||
FROM arm32v7/alpine:3.10
|
||||
|
||||
LABEL maintainer="MinIO Inc <dev@min.io>"
|
||||
|
||||
COPY --from=qemu /usr/bin/qemu-arm-static /usr/bin
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/
|
||||
COPY CREDITS /third_party/
|
||||
COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static
|
||||
|
||||
ENV MINIO_UPDATE off
|
||||
ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
@@ -14,11 +28,9 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
RUN \
|
||||
apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
|
||||
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-arm/minio -o /usr/bin/minio && \
|
||||
curl -s -q https://raw.githubusercontent.com/minio/minio/release/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
|
||||
chmod +x /usr/bin/minio && \
|
||||
chmod +x /usr/bin/docker-entrypoint.sh && \
|
||||
curl -s -q -O https://raw.githubusercontent.com/minio/minio/release/CREDITS
|
||||
curl https://dl.min.io/server/minio/release/linux-arm/minio > /usr/bin/minio && \
|
||||
chmod +x /usr/bin/minio && \
|
||||
chmod +x /usr/bin/docker-entrypoint.sh
|
||||
|
||||
EXPOSE 9000
|
||||
|
||||
|
||||
@@ -1,9 +1,23 @@
|
||||
FROM multiarch/qemu-user-static:x86_64-aarch64 as qemu
|
||||
FROM arm64v8/alpine:3.12
|
||||
FROM golang:1.13-alpine as builder
|
||||
|
||||
WORKDIR /home
|
||||
|
||||
ENV GOPATH /go
|
||||
ENV CGO_ENABLED 0
|
||||
ENV GO111MODULE on
|
||||
|
||||
RUN \
|
||||
apk add --no-cache git 'curl>7.61.0' && \
|
||||
git clone https://github.com/minio/minio && \
|
||||
curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static .
|
||||
|
||||
FROM arm64v8/alpine:3.10
|
||||
|
||||
LABEL maintainer="MinIO Inc <dev@min.io>"
|
||||
|
||||
COPY --from=qemu /usr/bin/qemu-aarch64-static /usr/bin
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/
|
||||
COPY CREDITS /third_party/
|
||||
COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static
|
||||
|
||||
ENV MINIO_UPDATE off
|
||||
ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
@@ -14,11 +28,9 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
RUN \
|
||||
apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
|
||||
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-arm64/minio -o /usr/bin/minio && \
|
||||
curl -s -q https://raw.githubusercontent.com/minio/minio/release/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
|
||||
chmod +x /usr/bin/minio && \
|
||||
chmod +x /usr/bin/docker-entrypoint.sh && \
|
||||
curl -s -q -O https://raw.githubusercontent.com/minio/minio/release/CREDITS
|
||||
curl https://dl.min.io/server/minio/release/linux-arm64/minio > /usr/bin/minio && \
|
||||
chmod +x /usr/bin/minio && \
|
||||
chmod +x /usr/bin/docker-entrypoint.sh
|
||||
|
||||
EXPOSE 9000
|
||||
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
FROM alpine:3.12
|
||||
FROM alpine:3.10
|
||||
|
||||
LABEL maintainer="MinIO Inc <dev@min.io>"
|
||||
|
||||
|
||||
@@ -10,3 +10,4 @@ ENV PATH=$PATH:/root/go/bin
|
||||
|
||||
RUN go get github.com/go-bindata/go-bindata/go-bindata && \
|
||||
go get github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs
|
||||
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
FROM multiarch/qemu-user-static:x86_64-ppc64le as qemu
|
||||
FROM ppc64le/alpine:3.12
|
||||
|
||||
LABEL maintainer="MinIO Inc <dev@min.io>"
|
||||
|
||||
COPY --from=qemu /usr/bin/qemu-ppc64le-static /usr/bin
|
||||
|
||||
ENV MINIO_UPDATE off
|
||||
ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_SECRET_KEY_FILE=secret_key \
|
||||
MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
|
||||
MINIO_SSE_MASTER_KEY_FILE=sse_master_key
|
||||
|
||||
RUN \
|
||||
apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
|
||||
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-ppc64le/minio -o /usr/bin/minio && \
|
||||
curl -s -q https://raw.githubusercontent.com/minio/minio/release/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
|
||||
chmod +x /usr/bin/minio && \
|
||||
chmod +x /usr/bin/docker-entrypoint.sh && \
|
||||
curl -s -q -O https://raw.githubusercontent.com/minio/minio/release/CREDITS
|
||||
|
||||
EXPOSE 9000
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
|
||||
VOLUME ["/data"]
|
||||
|
||||
CMD ["minio"]
|
||||
+17
-6
@@ -1,7 +1,20 @@
|
||||
FROM alpine:3.12
|
||||
FROM golang:1.13-alpine
|
||||
|
||||
ENV GOPATH /go
|
||||
ENV CGO_ENABLED 0
|
||||
ENV GO111MODULE on
|
||||
|
||||
RUN \
|
||||
apk add --no-cache git && \
|
||||
git clone https://github.com/minio/minio
|
||||
|
||||
FROM alpine:3.10
|
||||
|
||||
LABEL maintainer="MinIO Inc <dev@min.io>"
|
||||
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/
|
||||
COPY CREDITS /third_party/
|
||||
|
||||
ENV MINIO_UPDATE off
|
||||
ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_SECRET_KEY_FILE=secret_key \
|
||||
@@ -11,11 +24,9 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
RUN \
|
||||
apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
|
||||
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-amd64/minio -o /usr/bin/minio && \
|
||||
curl -s -q https://raw.githubusercontent.com/minio/minio/release/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
|
||||
chmod +x /usr/bin/minio && \
|
||||
chmod +x /usr/bin/docker-entrypoint.sh && \
|
||||
curl -s -q -O https://raw.githubusercontent.com/minio/minio/release/CREDITS
|
||||
curl https://dl.min.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \
|
||||
chmod +x /usr/bin/minio && \
|
||||
chmod +x /usr/bin/docker-entrypoint.sh
|
||||
|
||||
EXPOSE 9000
|
||||
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
FROM multiarch/qemu-user-static:x86_64-s390x as qemu
|
||||
FROM s390x/alpine:3.12
|
||||
|
||||
LABEL maintainer="MinIO Inc <dev@min.io>"
|
||||
|
||||
COPY --from=qemu /usr/bin/qemu-s390x-static /usr/bin
|
||||
|
||||
ENV MINIO_UPDATE off
|
||||
ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_SECRET_KEY_FILE=secret_key \
|
||||
MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
|
||||
MINIO_SSE_MASTER_KEY_FILE=sse_master_key
|
||||
|
||||
RUN \
|
||||
apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
|
||||
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-s390x/minio -o /usr/bin/minio && \
|
||||
curl -s -q https://raw.githubusercontent.com/minio/minio/release/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
|
||||
chmod +x /usr/bin/minio && \
|
||||
chmod +x /usr/bin/docker-entrypoint.sh && \
|
||||
curl -s -q -O https://raw.githubusercontent.com/minio/minio/release/CREDITS
|
||||
|
||||
EXPOSE 9000
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
|
||||
VOLUME ["/data"]
|
||||
|
||||
CMD ["minio"]
|
||||
@@ -60,9 +60,7 @@ build: checks
|
||||
@echo "Building minio binary to './minio'"
|
||||
@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
||||
|
||||
docker: checks
|
||||
@echo "Building minio docker image '$(TAG)'"
|
||||
@GOOS=linux GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
||||
docker: build
|
||||
@docker build -t $(TAG) . -f Dockerfile.dev
|
||||
|
||||
# Builds minio and installs it to $GOPATH/bin.
|
||||
|
||||
@@ -8,18 +8,14 @@ MinIO is a High Performance Object Storage released under Apache License v2.0. I
|
||||
## Docker Container
|
||||
### Stable
|
||||
```
|
||||
docker run -p 9000:9000 \
|
||||
-e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
|
||||
-e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
|
||||
minio/minio server /data
|
||||
docker pull minio/minio
|
||||
docker run -p 9000:9000 minio/minio server /data
|
||||
```
|
||||
|
||||
### Edge
|
||||
```
|
||||
docker run -p 9000:9000 \
|
||||
-e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
|
||||
-e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
|
||||
minio/minio:edge server /data
|
||||
docker pull minio/minio:edge
|
||||
docker run -p 9000:9000 minio/minio:edge server /data
|
||||
```
|
||||
|
||||
> NOTE: Docker will not display the default keys unless you start the container with the `-it`(interactive TTY) argument. Generally, it is not recommended to use default keys with containers. Please visit MinIO Docker quickstart guide for more information [here](https://docs.min.io/docs/minio-docker-quickstart-guide)
|
||||
@@ -98,6 +94,23 @@ GO111MODULE=on go get github.com/minio/minio
|
||||
|
||||
By default MinIO uses the port 9000 to listen for incoming connections. If your platform blocks the port by default, you may need to enable access to the port.
|
||||
|
||||
### iptables
|
||||
|
||||
For hosts with iptables enabled (RHEL, CentOS, etc), you can use `iptables` command to enable all traffic coming to specific ports. Use below command to allow
|
||||
access to port 9000
|
||||
|
||||
```sh
|
||||
iptables -A INPUT -p tcp --dport 9000 -j ACCEPT
|
||||
service iptables restart
|
||||
```
|
||||
|
||||
Below command enables all incoming traffic to ports ranging from 9000 to 9010.
|
||||
|
||||
```sh
|
||||
iptables -A INPUT -p tcp --dport 9000:9010 -j ACCEPT
|
||||
service iptables restart
|
||||
```
|
||||
|
||||
### ufw
|
||||
|
||||
For hosts with ufw enabled (Debian based distros), you can use `ufw` command to allow traffic to specific ports. Use below command to allow access to port 9000
|
||||
@@ -132,23 +145,6 @@ Note that `permanent` makes sure the rules are persistent across firewall start,
|
||||
firewall-cmd --reload
|
||||
```
|
||||
|
||||
### iptables
|
||||
|
||||
For hosts with iptables enabled (RHEL, CentOS, etc), you can use `iptables` command to enable all traffic coming to specific ports. Use below command to allow
|
||||
access to port 9000
|
||||
|
||||
```sh
|
||||
iptables -A INPUT -p tcp --dport 9000 -j ACCEPT
|
||||
service iptables restart
|
||||
```
|
||||
|
||||
Below command enables all incoming traffic to ports ranging from 9000 to 9010.
|
||||
|
||||
```sh
|
||||
iptables -A INPUT -p tcp --dport 9000:9010 -j ACCEPT
|
||||
service iptables restart
|
||||
```
|
||||
|
||||
## Test using MinIO Browser
|
||||
MinIO Server comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 ensure your server has started successfully.
|
||||
|
||||
@@ -163,23 +159,20 @@ When deployed on a single drive, MinIO server lets clients access any pre-existi
|
||||
The above statement is also valid for all gateway backends.
|
||||
|
||||
## Upgrading MinIO
|
||||
MinIO server supports rolling upgrades, i.e. you can update one MinIO instance at a time in a distributed cluster. This allows upgrades with no downtime. Upgrades can be done manually by replacing the binary with the latest release and restarting all servers in a rolling fashion. However, we recommend all our users to use [`mc admin update`](https://docs.min.io/docs/minio-admin-complete-guide.html#update) from the client. This will update all the nodes in the cluster simultaneously and restart them, as shown in the following command from the MinIO client (mc):
|
||||
MinIO server supports rolling upgrades, i.e. you can update one MinIO instance at a time in a distributed cluster. This allows upgrades with no downtime. Upgrades can be done manually by replacing the binary with the latest release and restarting all servers in a rolling fashion. However, we recommend all our users to use [`mc admin update`](https://docs.min.io/docs/minio-admin-complete-guide.html#update) from the client. This will update all the nodes in the cluster and restart them, as shown in the following command from the MinIO client (mc):
|
||||
|
||||
```
|
||||
mc admin update <minio alias, e.g., myminio>
|
||||
```
|
||||
|
||||
> NOTE: some releases might not allow rolling upgrades, this is always called out in the release notes and it is generally advised to read release notes before upgrading. In such a situation `mc admin update` is the recommended upgrading mechanism to upgrade all servers at once.
|
||||
|
||||
### Important things to remember during MinIO upgrades
|
||||
**Important things to remember during upgrades**:
|
||||
|
||||
- `mc admin update` will only work if the user running MinIO has write access to the parent directory where the binary is located, for example if the current binary is at `/usr/local/bin/minio`, you would need write access to `/usr/local/bin`.
|
||||
- `mc admin update` updates and restarts all servers simultaneously, applications would retry and continue their respective operations upon upgrade.
|
||||
- `mc admin update` is disabled in kubernetes/container environments, container environments provide their own mechanisms to rollout of updates.
|
||||
- In the case of federated setups `mc admin update` should be run against each cluster individually. Avoid updating `mc` to any new releases until all clusters have been successfully updated.
|
||||
- If using `kes` as KMS with MinIO, just replace the binary and restart `kes` more information about `kes` can be found [here](https://github.com/minio/kes/wiki)x
|
||||
- If using Vault as KMS with MinIO, ensure you have followed the Vault upgrade procedure outlined here: https://www.vaultproject.io/docs/upgrading/index.html
|
||||
- If using etcd with MinIO for the federation, ensure you have followed the etcd upgrade procedure outlined here: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md
|
||||
- In the case of federated setups `mc admin update` should be run against each cluster individually. Avoid updating `mc` until all clusters have been updated.
|
||||
- If you are updating the server it is always recommended (unless explicitly mentioned in MinIO server release notes), to update `mc` once all the servers have been upgraded using `mc update`.
|
||||
- `mc admin update` is disabled in docker/container environments, container environments provide their own mechanisms for updating running containers.
|
||||
- If you are using Vault as KMS with MinIO, ensure you have followed the Vault upgrade procedure outlined here: https://www.vaultproject.io/docs/upgrading/index.html
|
||||
- If you are using etcd with MinIO for the federation, ensure you have followed the etcd upgrade procedure outlined here: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md
|
||||
|
||||
## Explore Further
|
||||
- [MinIO Erasure Code QuickStart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)
|
||||
|
||||
@@ -54,11 +54,8 @@ export const ObjectItem = ({
|
||||
href={getDataType(name, contentType) === "folder" ? name : "#"}
|
||||
onClick={e => {
|
||||
e.preventDefault()
|
||||
// onclick function is passed only when we have a prefix
|
||||
if (onClick) {
|
||||
onClick()
|
||||
} else {
|
||||
checked ? uncheckObject(name) : checkObject(name)
|
||||
}
|
||||
}}
|
||||
>
|
||||
|
||||
@@ -46,11 +46,11 @@ class PreviewObjectModal extends React.Component {
|
||||
<ModalBody>
|
||||
<div className="input-group">
|
||||
{this.state.url && (
|
||||
<object data={this.state.url}>
|
||||
<h3 style={{ textAlign: "center", display: "block", width: "100%" }}>
|
||||
Do not have read permissions to preview "{this.props.object.name}"
|
||||
</h3>
|
||||
</object>
|
||||
<img
|
||||
alt="Image broken"
|
||||
src={this.state.url}
|
||||
style={{ display: "block", width: "100%" }}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</ModalBody>
|
||||
|
||||
@@ -30,10 +30,7 @@ describe("ObjectItem", () => {
|
||||
|
||||
it("shouldn't call onClick when the object isclicked", () => {
|
||||
const onClick = jest.fn()
|
||||
const checkObject = jest.fn()
|
||||
const wrapper = shallow(
|
||||
<ObjectItem name={"test"} checkObject={checkObject} />
|
||||
)
|
||||
const wrapper = shallow(<ObjectItem name={"test"} />)
|
||||
wrapper.find("a").simulate("click", { preventDefault: jest.fn() })
|
||||
expect(onClick).not.toHaveBeenCalled()
|
||||
})
|
||||
@@ -60,15 +57,9 @@ describe("ObjectItem", () => {
|
||||
})
|
||||
|
||||
it("should call uncheckObject when the object/prefix is unchecked", () => {
|
||||
const checkObject = jest.fn()
|
||||
const uncheckObject = jest.fn()
|
||||
const wrapper = shallow(
|
||||
<ObjectItem
|
||||
name={"test"}
|
||||
checked={true}
|
||||
checkObject={checkObject}
|
||||
uncheckObject={uncheckObject}
|
||||
/>
|
||||
<ObjectItem name={"test"} checked={true} uncheckObject={uncheckObject} />
|
||||
)
|
||||
wrapper.find("input[type='checkbox']").simulate("change")
|
||||
expect(uncheckObject).toHaveBeenCalledWith("test")
|
||||
|
||||
@@ -105,7 +105,7 @@ div.fesl-row {
|
||||
|
||||
.fesl-item-name {
|
||||
a {
|
||||
cursor: pointer;
|
||||
cursor: default;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,6 +114,12 @@ div.fesl-row {
|
||||
----------------------------*/
|
||||
&[data-type=folder] {
|
||||
.list-type(#a1d6dd, '\f07b');
|
||||
|
||||
.fesl-item-name {
|
||||
a {
|
||||
cursor: pointer;
|
||||
}
|
||||
}
|
||||
}
|
||||
&[data-type=pdf] {.list-type(#fa7775, '\f1c1'); }
|
||||
&[data-type=zip] { .list-type(#427089, '\f1c6'); }
|
||||
|
||||
+26
-26
File diff suppressed because one or more lines are too long
@@ -9,7 +9,7 @@ function _init() {
|
||||
export CGO_ENABLED=0
|
||||
|
||||
## List of architectures and OS to test coss compilation.
|
||||
SUPPORTED_OSARCH="linux/ppc64le linux/arm64 linux/s390x darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64"
|
||||
SUPPORTED_OSARCH="linux/ppc64le linux/arm64 linux/s390x darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386"
|
||||
}
|
||||
|
||||
function _build() {
|
||||
|
||||
@@ -33,7 +33,6 @@ function start_minio_3_node() {
|
||||
declare -a ARGS
|
||||
export MINIO_ACCESS_KEY=minio
|
||||
export MINIO_SECRET_KEY=minio123
|
||||
export MINIO_ERASURE_SET_DRIVE_COUNT=6
|
||||
|
||||
start_port=$(shuf -i 10000-65000 -n 1)
|
||||
for i in $(seq 1 3); do
|
||||
|
||||
@@ -173,7 +173,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
|
||||
cfg := globalServerConfig
|
||||
if newObjectLayerFn() == nil {
|
||||
if globalSafeMode {
|
||||
var err error
|
||||
cfg, err = getValidConfig(objectAPI)
|
||||
if err != nil {
|
||||
|
||||
+34
-80
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
|
||||
* MinIO Cloud Storage, (C) 2016-2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -328,7 +328,7 @@ func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Re
|
||||
writeSuccessResponseJSON(w, dataUsageInfoJSON)
|
||||
}
|
||||
|
||||
func lriToLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntry {
|
||||
func newLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntry {
|
||||
entry := &madmin.LockEntry{
|
||||
Timestamp: l.Timestamp,
|
||||
Resource: resource,
|
||||
@@ -337,14 +337,14 @@ func lriToLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEn
|
||||
ID: l.UID,
|
||||
}
|
||||
if l.Writer {
|
||||
entry.Type = "WRITE"
|
||||
entry.Type = "Write"
|
||||
} else {
|
||||
entry.Type = "READ"
|
||||
entry.Type = "Read"
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
func topLockEntries(peerLocks []*PeerLocks, count int) madmin.LockEntries {
|
||||
func topLockEntries(peerLocks []*PeerLocks) madmin.LockEntries {
|
||||
entryMap := make(map[string]*madmin.LockEntry)
|
||||
for _, peerLock := range peerLocks {
|
||||
if peerLock == nil {
|
||||
@@ -356,19 +356,20 @@ func topLockEntries(peerLocks []*PeerLocks, count int) madmin.LockEntries {
|
||||
if val, ok := entryMap[lockReqInfo.UID]; ok {
|
||||
val.ServerList = append(val.ServerList, peerLock.Addr)
|
||||
} else {
|
||||
entryMap[lockReqInfo.UID] = lriToLockEntry(lockReqInfo, k, peerLock.Addr)
|
||||
entryMap[lockReqInfo.UID] = newLockEntry(lockReqInfo, k, peerLock.Addr)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
var lockEntries = make(madmin.LockEntries, 0, len(entryMap))
|
||||
var lockEntries = make(madmin.LockEntries, 0)
|
||||
for _, v := range entryMap {
|
||||
lockEntries = append(lockEntries, *v)
|
||||
}
|
||||
sort.Sort(lockEntries)
|
||||
if len(lockEntries) > count {
|
||||
lockEntries = lockEntries[:count]
|
||||
const listCount int = 10
|
||||
if len(lockEntries) > listCount {
|
||||
lockEntries = lockEntries[:listCount]
|
||||
}
|
||||
return lockEntries
|
||||
}
|
||||
@@ -390,16 +391,6 @@ func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request
|
||||
return
|
||||
}
|
||||
|
||||
count := 10 // by default list only top 10 entries
|
||||
if countStr := r.URL.Query().Get("count"); countStr != "" {
|
||||
var err error
|
||||
count, err = strconv.Atoi(countStr)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
peerLocks := globalNotificationSys.GetLocks(ctx)
|
||||
// Once we have received all the locks currently used from peers
|
||||
// add the local peer locks list as well.
|
||||
@@ -407,13 +398,12 @@ func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request
|
||||
for _, llocker := range globalLockServers {
|
||||
getRespLocks = append(getRespLocks, llocker.DupLockMap())
|
||||
}
|
||||
|
||||
peerLocks = append(peerLocks, &PeerLocks{
|
||||
Addr: getHostName(r),
|
||||
Locks: getRespLocks,
|
||||
})
|
||||
|
||||
topLocks := topLockEntries(peerLocks, count)
|
||||
topLocks := topLockEntries(peerLocks)
|
||||
|
||||
// Marshal API response
|
||||
jsonBytes, err := json.Marshal(topLocks)
|
||||
@@ -641,7 +631,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// Check if this setup has an erasure coded backend.
|
||||
if !globalIsErasure {
|
||||
if !globalIsXL {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -652,16 +642,6 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Analyze the heal token and route the request accordingly
|
||||
token, success := proxyRequestByToken(ctx, w, r, hip.clientToken)
|
||||
if success {
|
||||
return
|
||||
}
|
||||
hip.clientToken = token
|
||||
// if request was not successful, try this server locally if token
|
||||
// is not found the call will fail anyways. if token is empty
|
||||
// try this server to generate a new token.
|
||||
|
||||
type healResp struct {
|
||||
respBytes []byte
|
||||
apiErr APIError
|
||||
@@ -728,16 +708,16 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// find number of disks in the setup, ignore any errors here.
|
||||
info, _ := objectAPI.StorageInfo(ctx, false)
|
||||
numDisks := info.Backend.OfflineDisks.Sum() + info.Backend.OnlineDisks.Sum()
|
||||
|
||||
healPath := pathJoin(hip.bucket, hip.objPrefix)
|
||||
if hip.clientToken == "" && !hip.forceStart && !hip.forceStop {
|
||||
nh, exists := globalAllHealState.getHealSequence(healPath)
|
||||
if exists && !nh.hasEnded() && len(nh.currentStatus.Items) > 0 {
|
||||
clientToken := nh.clientToken
|
||||
if globalIsDistErasure {
|
||||
clientToken = fmt.Sprintf("%s@%d", nh.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
|
||||
}
|
||||
b, err := json.Marshal(madmin.HealStartSuccess{
|
||||
ClientToken: clientToken,
|
||||
ClientToken: nh.clientToken,
|
||||
ClientAddress: nh.clientAddress,
|
||||
StartTime: nh.startTime,
|
||||
})
|
||||
@@ -774,7 +754,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
respCh <- hr
|
||||
}()
|
||||
case hip.clientToken == "":
|
||||
nh := newHealSequence(GlobalContext, hip.bucket, hip.objPrefix, handlers.GetSourceIP(r), hip.hs, hip.forceStart)
|
||||
nh := newHealSequence(GlobalContext, hip.bucket, hip.objPrefix, handlers.GetSourceIP(r), numDisks, hip.hs, hip.forceStart)
|
||||
go func() {
|
||||
respBytes, apiErr, errMsg := globalAllHealState.LaunchNewHealSequence(nh)
|
||||
hr := healResp{respBytes, apiErr, errMsg}
|
||||
@@ -799,7 +779,7 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
|
||||
}
|
||||
|
||||
// Check if this setup has an erasure coded backend.
|
||||
if !globalIsErasure {
|
||||
if !globalIsXL {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -809,7 +789,7 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
|
||||
// Get local heal status first
|
||||
bgHealStates = append(bgHealStates, getLocalBackgroundHealStatus())
|
||||
|
||||
if globalIsDistErasure {
|
||||
if globalIsDistXL {
|
||||
// Get heal status from other peers
|
||||
peersHealStates := globalNotificationSys.BackgroundHealStatus()
|
||||
bgHealStates = append(bgHealStates, peersHealStates...)
|
||||
@@ -882,11 +862,11 @@ const (
|
||||
AdminUpdateApplyFailure = "XMinioAdminUpdateApplyFailure"
|
||||
)
|
||||
|
||||
// toAdminAPIErrCode - converts errErasureWriteQuorum error to admin API
|
||||
// toAdminAPIErrCode - converts errXLWriteQuorum error to admin API
|
||||
// specific error.
|
||||
func toAdminAPIErrCode(ctx context.Context, err error) APIErrorCode {
|
||||
switch err {
|
||||
case errErasureWriteQuorum:
|
||||
case errXLWriteQuorum:
|
||||
return ErrAdminConfigNoQuorum
|
||||
default:
|
||||
return toAPIErrorCode(ctx, err)
|
||||
@@ -938,12 +918,6 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
|
||||
Description: err.Error(),
|
||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||
}
|
||||
case errors.Is(err, crypto.ErrKESKeyExists):
|
||||
apiErr = APIError{
|
||||
Code: "XMinioKMSKeyExists",
|
||||
Description: err.Error(),
|
||||
HTTPStatusCode: http.StatusConflict,
|
||||
}
|
||||
default:
|
||||
apiErr = errorCodes.ToAPIErrWithErr(toAdminAPIErrCode(ctx, err), err)
|
||||
}
|
||||
@@ -1096,28 +1070,6 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
}
|
||||
|
||||
// KMSCreateKeyHandler - POST /minio/admin/v3/kms/key/create?key-id=<master-key-id>
|
||||
func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "KMSCreateKey")
|
||||
defer logger.AuditLog(w, r, "KMSCreateKey", mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSCreateKeyAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if GlobalKMS == nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err := GlobalKMS.CreateKey(r.URL.Query().Get("key-id")); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
// KMSKeyStatusHandler - GET /minio/admin/v3/kms/key/status?key-id=<master-key-id>
|
||||
func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "KMSKeyStatus")
|
||||
@@ -1136,7 +1088,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
|
||||
|
||||
keyID := r.URL.Query().Get("key-id")
|
||||
if keyID == "" {
|
||||
keyID = GlobalKMS.DefaultKeyID()
|
||||
keyID = GlobalKMS.KeyID()
|
||||
}
|
||||
var response = madmin.KMSKeyStatus{
|
||||
KeyID: keyID,
|
||||
@@ -1325,7 +1277,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
|
||||
partialWrite(obdInfo)
|
||||
}
|
||||
|
||||
if net, ok := vars["perfnet"]; ok && net == "true" && globalIsDistErasure {
|
||||
if net, ok := vars["perfnet"]; ok && net == "true" && globalIsDistXL {
|
||||
obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.NetOBDInfo(deadlinedCtx))
|
||||
partialWrite(obdInfo)
|
||||
|
||||
@@ -1432,7 +1384,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
OffDisks += v
|
||||
}
|
||||
|
||||
backend = madmin.ErasureBackend{
|
||||
backend = madmin.XLBackend{
|
||||
Type: madmin.ErasureType,
|
||||
OnlineDisks: OnDisks,
|
||||
OfflineDisks: OffDisks,
|
||||
@@ -1447,8 +1399,10 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
}
|
||||
|
||||
mode := "safemode"
|
||||
if newObjectLayerFn() != nil {
|
||||
mode := ""
|
||||
if globalSafeMode {
|
||||
mode = "safe"
|
||||
} else {
|
||||
mode = "online"
|
||||
}
|
||||
|
||||
@@ -1459,10 +1413,10 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
for _, sp := range servers {
|
||||
for i, di := range sp.Disks {
|
||||
path := ""
|
||||
if globalIsErasure {
|
||||
if globalIsXL {
|
||||
path = di.DrivePath
|
||||
}
|
||||
if globalIsDistErasure {
|
||||
if globalIsDistXL {
|
||||
path = sp.Endpoint + di.DrivePath
|
||||
}
|
||||
// For distributed
|
||||
@@ -1470,13 +1424,13 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
for b := range storageInfo.Backend.Sets[a] {
|
||||
ep := storageInfo.Backend.Sets[a][b].Endpoint
|
||||
|
||||
if globalIsDistErasure {
|
||||
if globalIsDistXL {
|
||||
if strings.Replace(ep, "http://", "", -1) == path || strings.Replace(ep, "https://", "", -1) == path {
|
||||
sp.Disks[i].State = storageInfo.Backend.Sets[a][b].State
|
||||
sp.Disks[i].UUID = storageInfo.Backend.Sets[a][b].UUID
|
||||
}
|
||||
}
|
||||
if globalIsErasure {
|
||||
if globalIsXL {
|
||||
if ep == path {
|
||||
sp.Disks[i].State = storageInfo.Backend.Sets[a][b].State
|
||||
sp.Disks[i].UUID = storageInfo.Backend.Sets[a][b].UUID
|
||||
@@ -1569,7 +1523,7 @@ func fetchVaultStatus(cfg config.Config) madmin.Vault {
|
||||
vault.Status = "disabled"
|
||||
return vault
|
||||
}
|
||||
keyID := GlobalKMS.DefaultKeyID()
|
||||
keyID := GlobalKMS.KeyID()
|
||||
kmsInfo := GlobalKMS.Info()
|
||||
|
||||
if kmsInfo.Endpoint == "" {
|
||||
|
||||
+31
-32
@@ -33,27 +33,27 @@ import (
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
)
|
||||
|
||||
// adminErasureTestBed - encapsulates subsystems that need to be setup for
|
||||
// adminXLTestBed - encapsulates subsystems that need to be setup for
|
||||
// admin-handler unit tests.
|
||||
type adminErasureTestBed struct {
|
||||
erasureDirs []string
|
||||
objLayer ObjectLayer
|
||||
router *mux.Router
|
||||
type adminXLTestBed struct {
|
||||
xlDirs []string
|
||||
objLayer ObjectLayer
|
||||
router *mux.Router
|
||||
}
|
||||
|
||||
// prepareAdminErasureTestBed - helper function that setups a single-node
|
||||
// Erasure backend for admin-handler tests.
|
||||
func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, error) {
|
||||
// prepareAdminXLTestBed - helper function that setups a single-node
|
||||
// XL backend for admin-handler tests.
|
||||
func prepareAdminXLTestBed(ctx context.Context) (*adminXLTestBed, error) {
|
||||
|
||||
// reset global variables to start afresh.
|
||||
resetTestGlobals()
|
||||
|
||||
// Set globalIsErasure to indicate that the setup uses an erasure
|
||||
// Set globalIsXL to indicate that the setup uses an erasure
|
||||
// code backend.
|
||||
globalIsErasure = true
|
||||
globalIsXL = true
|
||||
|
||||
// Initializing objectLayer for HealFormatHandler.
|
||||
objLayer, erasureDirs, xlErr := initTestErasureObjLayer(ctx)
|
||||
objLayer, xlDirs, xlErr := initTestXLObjLayer(ctx)
|
||||
if xlErr != nil {
|
||||
return nil, xlErr
|
||||
}
|
||||
@@ -66,7 +66,7 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
|
||||
// Initialize boot time
|
||||
globalBootTime = UTCNow()
|
||||
|
||||
globalEndpoints = mustGetZoneEndpoints(erasureDirs...)
|
||||
globalEndpoints = mustGetZoneEndpoints(xlDirs...)
|
||||
|
||||
newAllSubsystems()
|
||||
|
||||
@@ -76,37 +76,36 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
|
||||
adminRouter := mux.NewRouter()
|
||||
registerAdminRouter(adminRouter, true, true)
|
||||
|
||||
return &adminErasureTestBed{
|
||||
erasureDirs: erasureDirs,
|
||||
objLayer: objLayer,
|
||||
router: adminRouter,
|
||||
return &adminXLTestBed{
|
||||
xlDirs: xlDirs,
|
||||
objLayer: objLayer,
|
||||
router: adminRouter,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// TearDown - method that resets the test bed for subsequent unit
|
||||
// tests to start afresh.
|
||||
func (atb *adminErasureTestBed) TearDown() {
|
||||
removeRoots(atb.erasureDirs)
|
||||
func (atb *adminXLTestBed) TearDown() {
|
||||
removeRoots(atb.xlDirs)
|
||||
resetTestGlobals()
|
||||
}
|
||||
|
||||
// initTestObjLayer - Helper function to initialize an Erasure-based object
|
||||
// initTestObjLayer - Helper function to initialize an XL-based object
|
||||
// layer and set globalObjectAPI.
|
||||
func initTestErasureObjLayer(ctx context.Context) (ObjectLayer, []string, error) {
|
||||
erasureDirs, err := getRandomDisks(16)
|
||||
func initTestXLObjLayer(ctx context.Context) (ObjectLayer, []string, error) {
|
||||
xlDirs, err := getRandomDisks(16)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
endpoints := mustGetNewEndpoints(erasureDirs...)
|
||||
storageDisks, format, err := waitForFormatErasure(true, endpoints, 1, 1, 16, "")
|
||||
endpoints := mustGetNewEndpoints(xlDirs...)
|
||||
storageDisks, format, err := waitForFormatXL(true, endpoints, 1, 1, 16, "")
|
||||
if err != nil {
|
||||
removeRoots(erasureDirs)
|
||||
removeRoots(xlDirs)
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
globalPolicySys = NewPolicySys()
|
||||
objLayer := &erasureZones{zones: make([]*erasureSets, 1)}
|
||||
objLayer.zones[0], err = newErasureSets(ctx, endpoints, storageDisks, format)
|
||||
objLayer, err := newXLSets(ctx, endpoints, storageDisks, format)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@@ -115,7 +114,7 @@ func initTestErasureObjLayer(ctx context.Context) (ObjectLayer, []string, error)
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = objLayer
|
||||
globalObjLayerMutex.Unlock()
|
||||
return objLayer, erasureDirs, nil
|
||||
return objLayer, xlDirs, nil
|
||||
}
|
||||
|
||||
// cmdType - Represents different service subcomands like status, stop
|
||||
@@ -184,9 +183,9 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
adminTestBed, err := prepareAdminErasureTestBed(ctx)
|
||||
adminTestBed, err := prepareAdminXLTestBed(ctx)
|
||||
if err != nil {
|
||||
t.Fatal("Failed to initialize a single node Erasure backend for admin handler tests.")
|
||||
t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
|
||||
}
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
@@ -255,9 +254,9 @@ func TestAdminServerInfo(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
adminTestBed, err := prepareAdminErasureTestBed(ctx)
|
||||
adminTestBed, err := prepareAdminXLTestBed(ctx)
|
||||
if err != nil {
|
||||
t.Fatal("Failed to initialize a single node Erasure backend for admin handler tests.")
|
||||
t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
|
||||
}
|
||||
|
||||
defer adminTestBed.TearDown()
|
||||
@@ -299,7 +298,7 @@ func TestToAdminAPIErrCode(t *testing.T) {
|
||||
}{
|
||||
// 1. Server not in quorum.
|
||||
{
|
||||
err: errErasureWriteQuorum,
|
||||
err: errXLWriteQuorum,
|
||||
expectedAPIErr: ErrAdminConfigNoQuorum,
|
||||
},
|
||||
// 2. No error.
|
||||
|
||||
+49
-64
@@ -21,6 +21,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -76,6 +77,9 @@ type healSequenceStatus struct {
|
||||
FailureDetail string `json:"Detail,omitempty"`
|
||||
StartTime time.Time `json:"StartTime"`
|
||||
|
||||
// disk information
|
||||
NumDisks int `json:"NumDisks"`
|
||||
|
||||
// settings for the heal sequence
|
||||
HealSettings madmin.HealOpts `json:"Settings"`
|
||||
|
||||
@@ -91,8 +95,8 @@ type allHealState struct {
|
||||
healSeqMap map[string]*healSequence
|
||||
}
|
||||
|
||||
// newHealState - initialize global heal state management
|
||||
func newHealState() *allHealState {
|
||||
// initHealState - initialize healing apparatus
|
||||
func initHealState() *allHealState {
|
||||
healState := &allHealState{
|
||||
healSeqMap: make(map[string]*healSequence),
|
||||
}
|
||||
@@ -155,13 +159,8 @@ func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) {
|
||||
StartTime: UTCNow(),
|
||||
}
|
||||
} else {
|
||||
clientToken := he.clientToken
|
||||
if globalIsDistErasure {
|
||||
clientToken = fmt.Sprintf("%s@%d", he.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
|
||||
}
|
||||
|
||||
hsp = madmin.HealStopSuccess{
|
||||
ClientToken: clientToken,
|
||||
ClientToken: he.clientToken,
|
||||
ClientAddress: he.clientAddress,
|
||||
StartTime: he.startTime,
|
||||
}
|
||||
@@ -194,7 +193,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
|
||||
respBytes []byte, apiErr APIError, errMsg string) {
|
||||
|
||||
existsAndLive := false
|
||||
he, exists := ahs.getHealSequence(pathJoin(h.bucket, h.object))
|
||||
he, exists := ahs.getHealSequence(h.path)
|
||||
if exists {
|
||||
existsAndLive = !he.hasEnded()
|
||||
}
|
||||
@@ -221,9 +220,8 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
|
||||
|
||||
// Check if new heal sequence to be started overlaps with any
|
||||
// existing, running sequence
|
||||
hpath := pathJoin(h.bucket, h.object)
|
||||
for k, hSeq := range ahs.healSeqMap {
|
||||
if !hSeq.hasEnded() && (HasPrefix(k, hpath) || HasPrefix(hpath, k)) {
|
||||
if !hSeq.hasEnded() && (HasPrefix(k, h.path) || HasPrefix(h.path, k)) {
|
||||
|
||||
errMsg = "The provided heal sequence path overlaps with an existing " +
|
||||
fmt.Sprintf("heal path: %s", k)
|
||||
@@ -232,18 +230,13 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
|
||||
}
|
||||
|
||||
// Add heal state and start sequence
|
||||
ahs.healSeqMap[hpath] = h
|
||||
ahs.healSeqMap[h.path] = h
|
||||
|
||||
// Launch top-level background heal go-routine
|
||||
go h.healSequenceStart()
|
||||
|
||||
clientToken := h.clientToken
|
||||
if globalIsDistErasure {
|
||||
clientToken = fmt.Sprintf("%s@%d", h.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
|
||||
}
|
||||
|
||||
b, err := json.Marshal(madmin.HealStartSuccess{
|
||||
ClientToken: clientToken,
|
||||
ClientToken: h.clientToken,
|
||||
ClientAddress: h.clientAddress,
|
||||
StartTime: h.startTime,
|
||||
})
|
||||
@@ -258,11 +251,11 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
|
||||
// status results from global state and returns its JSON
|
||||
// representation. The clientToken helps ensure there aren't
|
||||
// conflicting clients fetching status.
|
||||
func (ahs *allHealState) PopHealStatusJSON(hpath string,
|
||||
func (ahs *allHealState) PopHealStatusJSON(path string,
|
||||
clientToken string) ([]byte, APIErrorCode) {
|
||||
|
||||
// fetch heal state for given path
|
||||
h, exists := ahs.getHealSequence(hpath)
|
||||
h, exists := ahs.getHealSequence(path)
|
||||
if !exists {
|
||||
// If there is no such heal sequence, return error.
|
||||
return nil, ErrHealNoSuchProcess
|
||||
@@ -303,17 +296,18 @@ func (ahs *allHealState) PopHealStatusJSON(hpath string,
|
||||
|
||||
// healSource denotes single entity and heal option.
|
||||
type healSource struct {
|
||||
bucket string
|
||||
object string
|
||||
versionID string
|
||||
opts *madmin.HealOpts // optional heal option overrides default setting
|
||||
path string // entity path (format, buckets, objects) to heal
|
||||
opts *madmin.HealOpts // optional heal option overrides default setting
|
||||
}
|
||||
|
||||
// healSequence - state for each heal sequence initiated on the
|
||||
// server.
|
||||
type healSequence struct {
|
||||
// bucket, and object on which heal seq. was initiated
|
||||
bucket, object string
|
||||
// bucket, and prefix on which heal seq. was initiated
|
||||
bucket, objPrefix string
|
||||
|
||||
// path is just pathJoin(bucket, objPrefix)
|
||||
path string
|
||||
|
||||
// A channel of entities (format, buckets, objects) to heal
|
||||
sourceCh chan healSource
|
||||
@@ -374,27 +368,27 @@ type healSequence struct {
|
||||
// NewHealSequence - creates healSettings, assumes bucket and
|
||||
// objPrefix are already validated.
|
||||
func newHealSequence(ctx context.Context, bucket, objPrefix, clientAddr string,
|
||||
hs madmin.HealOpts, forceStart bool) *healSequence {
|
||||
numDisks int, hs madmin.HealOpts, forceStart bool) *healSequence {
|
||||
|
||||
reqInfo := &logger.ReqInfo{RemoteHost: clientAddr, API: "Heal", BucketName: bucket}
|
||||
reqInfo.AppendTags("prefix", objPrefix)
|
||||
ctx, cancel := context.WithCancel(logger.SetReqInfo(ctx, reqInfo))
|
||||
|
||||
clientToken := mustGetUUID()
|
||||
|
||||
return &healSequence{
|
||||
respCh: make(chan healResult),
|
||||
bucket: bucket,
|
||||
object: objPrefix,
|
||||
objPrefix: objPrefix,
|
||||
path: pathJoin(bucket, objPrefix),
|
||||
reportProgress: true,
|
||||
startTime: UTCNow(),
|
||||
clientToken: clientToken,
|
||||
clientToken: mustGetUUID(),
|
||||
clientAddress: clientAddr,
|
||||
forceStarted: forceStart,
|
||||
settings: hs,
|
||||
currentStatus: healSequenceStatus{
|
||||
Summary: healNotStartedStatus,
|
||||
HealSettings: hs,
|
||||
NumDisks: numDisks,
|
||||
},
|
||||
traverseAndHealDoneCh: make(chan error),
|
||||
cancelCtx: cancel,
|
||||
@@ -624,9 +618,7 @@ func (h *healSequence) healSequenceStart() {
|
||||
func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItemType) error {
|
||||
// Send heal request
|
||||
task := healTask{
|
||||
bucket: source.bucket,
|
||||
object: source.object,
|
||||
versionID: source.versionID,
|
||||
path: source.path,
|
||||
opts: h.settings,
|
||||
responseCh: h.respCh,
|
||||
}
|
||||
@@ -641,7 +633,7 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
|
||||
// Object might have been deleted, by the time heal
|
||||
// was attempted, we should ignore this object and
|
||||
// return success.
|
||||
if isErrObjectNotFound(res.err) || isErrVersionNotFound(res.err) {
|
||||
if isErrObjectNotFound(res.err) {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -669,7 +661,7 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
|
||||
if res.err != nil {
|
||||
// Object might have been deleted, by the time heal
|
||||
// was attempted, we should ignore this object and return success.
|
||||
if isErrObjectNotFound(res.err) || isErrVersionNotFound(res.err) {
|
||||
if isErrObjectNotFound(res.err) {
|
||||
return nil
|
||||
}
|
||||
// Only report object error
|
||||
@@ -685,6 +677,11 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
|
||||
}
|
||||
|
||||
func (h *healSequence) healItemsFromSourceCh() error {
|
||||
bucketsOnly := true // heal buckets only, not objects.
|
||||
if err := h.healItems(bucketsOnly); err != nil {
|
||||
logger.LogIf(h.ctx, err)
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case source, ok := <-h.sourceCh:
|
||||
@@ -693,11 +690,11 @@ func (h *healSequence) healItemsFromSourceCh() error {
|
||||
}
|
||||
var itemType madmin.HealItemType
|
||||
switch {
|
||||
case source.bucket == nopHeal:
|
||||
case source.path == nopHeal:
|
||||
continue
|
||||
case source.bucket == SlashSeparator:
|
||||
case source.path == SlashSeparator:
|
||||
itemType = madmin.HealItemMetadata
|
||||
case source.bucket != "" && source.object == "":
|
||||
case !strings.Contains(source.path, SlashSeparator):
|
||||
itemType = madmin.HealItemBucket
|
||||
default:
|
||||
itemType = madmin.HealItemObject
|
||||
@@ -719,7 +716,7 @@ func (h *healSequence) healFromSourceCh() {
|
||||
h.healItemsFromSourceCh()
|
||||
}
|
||||
|
||||
func (h *healSequence) healDiskMeta() error {
|
||||
func (h *healSequence) healItems(bucketsOnly bool) error {
|
||||
// Start with format healing
|
||||
if err := h.healDiskFormat(); err != nil {
|
||||
return err
|
||||
@@ -731,11 +728,7 @@ func (h *healSequence) healDiskMeta() error {
|
||||
}
|
||||
|
||||
// Start healing the bucket config prefix.
|
||||
return h.healMinioSysMeta(bucketConfigPrefix)()
|
||||
}
|
||||
|
||||
func (h *healSequence) healItems(bucketsOnly bool) error {
|
||||
if err := h.healDiskMeta(); err != nil {
|
||||
if err := h.healMinioSysMeta(bucketConfigPrefix)(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -769,19 +762,15 @@ func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
|
||||
// NOTE: Healing on meta is run regardless
|
||||
// of any bucket being selected, this is to ensure that
|
||||
// meta are always upto date and correct.
|
||||
return objectAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string) error {
|
||||
return objectAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket string, object string) error {
|
||||
if h.isQuitting() {
|
||||
return errHealStopSignalled
|
||||
}
|
||||
|
||||
herr := h.queueHealTask(healSource{
|
||||
bucket: bucket,
|
||||
object: object,
|
||||
versionID: versionID,
|
||||
}, madmin.HealItemBucketMetadata)
|
||||
herr := h.queueHealTask(healSource{path: pathJoin(bucket, object)}, madmin.HealItemBucketMetadata)
|
||||
// Object might have been deleted, by the time heal
|
||||
// was attempted we ignore this object an move on.
|
||||
if isErrObjectNotFound(herr) || isErrVersionNotFound(herr) {
|
||||
if isErrObjectNotFound(herr) {
|
||||
return nil
|
||||
}
|
||||
return herr
|
||||
@@ -802,7 +791,7 @@ func (h *healSequence) healDiskFormat() error {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
|
||||
return h.queueHealTask(healSource{bucket: SlashSeparator}, madmin.HealItemMetadata)
|
||||
return h.queueHealTask(healSource{path: SlashSeparator}, madmin.HealItemMetadata)
|
||||
}
|
||||
|
||||
// healBuckets - check for all buckets heal or just particular bucket.
|
||||
@@ -844,7 +833,7 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
|
||||
if err := h.queueHealTask(healSource{bucket: bucket}, madmin.HealItemBucket); err != nil {
|
||||
if err := h.queueHealTask(healSource{path: bucket}, madmin.HealItemBucket); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -853,12 +842,12 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
|
||||
}
|
||||
|
||||
if !h.settings.Recursive {
|
||||
if h.object != "" {
|
||||
if h.objPrefix != "" {
|
||||
// Check if an object named as the objPrefix exists,
|
||||
// and if so heal it.
|
||||
_, err := objectAPI.GetObjectInfo(h.ctx, bucket, h.object, ObjectOptions{})
|
||||
_, err := objectAPI.GetObjectInfo(h.ctx, bucket, h.objPrefix, ObjectOptions{})
|
||||
if err == nil {
|
||||
if err = h.healObject(bucket, h.object, ""); err != nil {
|
||||
if err = h.healObject(bucket, h.objPrefix); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -867,14 +856,14 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := objectAPI.HealObjects(h.ctx, bucket, h.object, h.settings, h.healObject); err != nil {
|
||||
if err := objectAPI.HealObjects(h.ctx, bucket, h.objPrefix, h.settings, h.healObject); err != nil {
|
||||
return errFnHealFromAPIErr(h.ctx, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// healObject - heal the given object and record result
|
||||
func (h *healSequence) healObject(bucket, object, versionID string) error {
|
||||
func (h *healSequence) healObject(bucket, object string) error {
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerWithoutSafeModeFn()
|
||||
if objectAPI == nil {
|
||||
@@ -885,9 +874,5 @@ func (h *healSequence) healObject(bucket, object, versionID string) error {
|
||||
return errHealStopSignalled
|
||||
}
|
||||
|
||||
return h.queueHealTask(healSource{
|
||||
bucket: bucket,
|
||||
object: object,
|
||||
versionID: versionID,
|
||||
}, madmin.HealItemObject)
|
||||
return h.queueHealTask(healSource{path: pathJoin(bucket, object)}, madmin.HealItemObject)
|
||||
}
|
||||
|
||||
+3
-4
@@ -64,7 +64,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
|
||||
// DataUsageInfo operations
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(httpTraceAll(adminAPI.DataUsageInfoHandler))
|
||||
|
||||
if globalIsDistErasure || globalIsErasure {
|
||||
if globalIsDistXL || globalIsXL {
|
||||
/// Heal operations
|
||||
|
||||
// Heal processing endpoint.
|
||||
@@ -172,7 +172,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
|
||||
}
|
||||
|
||||
// Quota operations
|
||||
if globalIsDistErasure || globalIsErasure {
|
||||
if globalIsXL || globalIsDistXL {
|
||||
if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
|
||||
// GetBucketQuotaConfig
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
|
||||
@@ -185,7 +185,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
|
||||
|
||||
// -- Top APIs --
|
||||
// Top locks
|
||||
if globalIsDistErasure {
|
||||
if globalIsDistXL {
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/top/locks").HandlerFunc(httpTraceHdrs(adminAPI.TopLocksHandler))
|
||||
}
|
||||
|
||||
@@ -197,7 +197,6 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
|
||||
|
||||
// -- KMS APIs --
|
||||
//
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/kms/key/create").HandlerFunc(httpTraceAll(adminAPI.KMSCreateKeyHandler)).Queries("key-id", "{key-id:.*}")
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(httpTraceAll(adminAPI.KMSKeyStatusHandler))
|
||||
|
||||
if !globalIsGateway {
|
||||
|
||||
@@ -29,7 +29,7 @@ import (
|
||||
func getLocalServerProperty(endpointZones EndpointZones, r *http.Request) madmin.ServerProperties {
|
||||
var disks []madmin.Disk
|
||||
addr := r.Host
|
||||
if globalIsDistErasure {
|
||||
if globalIsDistXL {
|
||||
addr = GetLocalPeer(endpointZones)
|
||||
}
|
||||
network := make(map[string]string)
|
||||
|
||||
+3
-12
@@ -20,18 +20,9 @@ import (
|
||||
"encoding/xml"
|
||||
)
|
||||
|
||||
// DeletedObject objects deleted
|
||||
type DeletedObject struct {
|
||||
DeleteMarker bool `xml:"DeleteMarker,omitempty"`
|
||||
DeleteMarkerVersionID string `xml:"DeleteMarkerVersionId,omitempty"`
|
||||
ObjectName string `xml:"Key,omitempty"`
|
||||
VersionID string `xml:"VersionId,omitempty"`
|
||||
}
|
||||
|
||||
// ObjectToDelete carries key name for the object to delete.
|
||||
type ObjectToDelete struct {
|
||||
// ObjectIdentifier carries key name for the object to delete.
|
||||
type ObjectIdentifier struct {
|
||||
ObjectName string `xml:"Key"`
|
||||
VersionID string `xml:"VersionId"`
|
||||
}
|
||||
|
||||
// createBucketConfiguration container for bucket configuration request from client.
|
||||
@@ -46,5 +37,5 @@ type DeleteObjectsRequest struct {
|
||||
// Element to enable quiet mode for the request
|
||||
Quiet bool
|
||||
// List of objects to be deleted
|
||||
Objects []ObjectToDelete `xml:"Object"`
|
||||
Objects []ObjectIdentifier `xml:"Object"`
|
||||
}
|
||||
|
||||
+3
-20
@@ -36,7 +36,6 @@ import (
|
||||
"github.com/minio/minio/pkg/bucket/lifecycle"
|
||||
objectlock "github.com/minio/minio/pkg/bucket/object/lock"
|
||||
"github.com/minio/minio/pkg/bucket/policy"
|
||||
"github.com/minio/minio/pkg/bucket/versioning"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
@@ -539,9 +538,9 @@ var errorCodes = errorCodeMap{
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrNoSuchVersion: {
|
||||
Code: "InvalidArgument",
|
||||
Description: "Invalid version id specified",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
Code: "NoSuchVersion",
|
||||
Description: "Indicates that the version ID specified in the request does not match an existing version.",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrNotImplemented: {
|
||||
Code: "NotImplemented",
|
||||
@@ -1783,10 +1782,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrBucketAlreadyOwnedByYou
|
||||
case ObjectNotFound:
|
||||
apiErr = ErrNoSuchKey
|
||||
case MethodNotAllowed:
|
||||
apiErr = ErrMethodNotAllowed
|
||||
case VersionNotFound:
|
||||
apiErr = ErrNoSuchVersion
|
||||
case ObjectAlreadyExists:
|
||||
apiErr = ErrMethodNotAllowed
|
||||
case ObjectNameInvalid:
|
||||
@@ -1909,12 +1904,6 @@ func toAPIError(ctx context.Context, err error) APIError {
|
||||
// their internal error types. This code is only
|
||||
// useful with gateway implementations.
|
||||
switch e := err.(type) {
|
||||
case InvalidArgument:
|
||||
apiErr = APIError{
|
||||
Code: "InvalidArgument",
|
||||
Description: e.Error(),
|
||||
HTTPStatusCode: errorCodes[ErrInvalidRequest].HTTPStatusCode,
|
||||
}
|
||||
case *xml.SyntaxError:
|
||||
apiErr = APIError{
|
||||
Code: "MalformedXML",
|
||||
@@ -1929,12 +1918,6 @@ func toAPIError(ctx context.Context, err error) APIError {
|
||||
e.Error()),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
case versioning.Error:
|
||||
apiErr = APIError{
|
||||
Code: "IllegalVersioningConfigurationException",
|
||||
Description: fmt.Sprintf("Versioning configuration specified in the request is invalid. (%s)", e.Error()),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
case lifecycle.Error:
|
||||
apiErr = APIError{
|
||||
Code: "InvalidRequest",
|
||||
|
||||
@@ -29,7 +29,6 @@ import (
|
||||
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/pkg/bucket/lifecycle"
|
||||
)
|
||||
|
||||
// Returns a hexadecimal representation of time at the
|
||||
@@ -153,26 +152,5 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
|
||||
w.Header().Set(xhttp.ContentRange, contentRange)
|
||||
}
|
||||
|
||||
// Set the relevant version ID as part of the response header.
|
||||
if objInfo.VersionID != "" {
|
||||
w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
|
||||
}
|
||||
|
||||
if lc, err := globalLifecycleSys.Get(objInfo.Bucket); err == nil {
|
||||
ruleID, expiryTime := lc.PredictExpiryTime(lifecycle.ObjectOpts{
|
||||
Name: objInfo.Name,
|
||||
UserTags: objInfo.UserTags,
|
||||
VersionID: objInfo.VersionID,
|
||||
ModTime: objInfo.ModTime,
|
||||
IsLatest: objInfo.IsLatest,
|
||||
DeleteMarker: objInfo.DeleteMarker,
|
||||
})
|
||||
if !expiryTime.IsZero() {
|
||||
w.Header()[xhttp.AmzExpiration] = []string{
|
||||
fmt.Sprintf(`expiry-date="%s", rule-id="%s"`, expiryTime.Format(http.TimeFormat), ruleID),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
+12
-42
@@ -35,7 +35,7 @@ import (
|
||||
const (
|
||||
// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
|
||||
iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
|
||||
maxObjectList = 10000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
|
||||
maxObjectList = 50000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
|
||||
maxDeleteList = 10000 // Limit number of objects deleted in a delete call.
|
||||
maxUploadsList = 10000 // Limit number of uploads in a listUploadsResponse.
|
||||
maxPartsList = 10000 // Limit number of parts in a listPartsResponse.
|
||||
@@ -235,23 +235,10 @@ type Bucket struct {
|
||||
|
||||
// ObjectVersion container for object version metadata
|
||||
type ObjectVersion struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Version" json:"-"`
|
||||
Object
|
||||
IsLatest bool
|
||||
VersionID string `xml:"VersionId"`
|
||||
|
||||
isDeleteMarker bool
|
||||
}
|
||||
|
||||
// MarshalXML - marshal ObjectVersion
|
||||
func (o ObjectVersion) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
if o.isDeleteMarker {
|
||||
start.Name.Local = "DeleteMarker"
|
||||
} else {
|
||||
start.Name.Local = "Version"
|
||||
}
|
||||
|
||||
type objectVersionWrapper ObjectVersion
|
||||
return e.EncodeElement(objectVersionWrapper(o), start)
|
||||
IsLatest bool
|
||||
}
|
||||
|
||||
// StringMap is a map[string]string.
|
||||
@@ -346,10 +333,9 @@ type CompleteMultipartUploadResponse struct {
|
||||
|
||||
// DeleteError structure.
|
||||
type DeleteError struct {
|
||||
Code string
|
||||
Message string
|
||||
Key string
|
||||
VersionID string `xml:"VersionId"`
|
||||
Code string
|
||||
Message string
|
||||
Key string
|
||||
}
|
||||
|
||||
// DeleteObjectsResponse container for multiple object deletes.
|
||||
@@ -357,7 +343,7 @@ type DeleteObjectsResponse struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteResult" json:"-"`
|
||||
|
||||
// Collection of all deleted objects
|
||||
DeletedObjects []DeletedObject `xml:"Deleted,omitempty"`
|
||||
DeletedObjects []ObjectIdentifier `xml:"Deleted,omitempty"`
|
||||
|
||||
// Collection of errors deleting certain objects.
|
||||
Errors []DeleteError `xml:"Error,omitempty"`
|
||||
@@ -427,7 +413,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
|
||||
}
|
||||
|
||||
// generates an ListBucketVersions response for the said bucket with other enumerated options.
|
||||
func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo) ListVersionsResponse {
|
||||
func generateListVersionsResponse(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListVersionsResponse {
|
||||
var versions []ObjectVersion
|
||||
var prefixes []CommonPrefix
|
||||
var owner = Owner{}
|
||||
@@ -450,13 +436,10 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
|
||||
} else {
|
||||
content.StorageClass = globalMinioDefaultStorageClass
|
||||
}
|
||||
|
||||
content.Owner = owner
|
||||
content.VersionID = object.VersionID
|
||||
if content.VersionID == "" {
|
||||
content.VersionID = nullVersionID
|
||||
}
|
||||
content.IsLatest = object.IsLatest
|
||||
content.isDeleteMarker = object.DeleteMarker
|
||||
content.VersionID = "null"
|
||||
content.IsLatest = true
|
||||
versions = append(versions, content)
|
||||
}
|
||||
|
||||
@@ -469,8 +452,6 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
|
||||
data.MaxKeys = maxKeys
|
||||
|
||||
data.NextKeyMarker = s3EncodeName(resp.NextMarker, encodingType)
|
||||
data.NextVersionIDMarker = resp.NextVersionIDMarker
|
||||
data.VersionIDMarker = versionIDMarker
|
||||
data.IsTruncated = resp.IsTruncated
|
||||
|
||||
for _, prefix := range resp.Prefixes {
|
||||
@@ -685,23 +666,16 @@ func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMult
|
||||
}
|
||||
|
||||
// generate multi objects delete response.
|
||||
func generateMultiDeleteResponse(quiet bool, deletedObjects []DeletedObject, errs []DeleteError) DeleteObjectsResponse {
|
||||
func generateMultiDeleteResponse(quiet bool, deletedObjects []ObjectIdentifier, errs []DeleteError) DeleteObjectsResponse {
|
||||
deleteResp := DeleteObjectsResponse{}
|
||||
if !quiet {
|
||||
deleteResp.DeletedObjects = deletedObjects
|
||||
}
|
||||
if len(errs) == len(deletedObjects) {
|
||||
deleteResp.DeletedObjects = nil
|
||||
}
|
||||
deleteResp.Errors = errs
|
||||
return deleteResp
|
||||
}
|
||||
|
||||
func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
|
||||
if newObjectLayerFn() == nil {
|
||||
// Server still in safe mode.
|
||||
w.Header().Set(xhttp.MinIOServerStatus, "safemode")
|
||||
}
|
||||
setCommonHeaders(w)
|
||||
if mType != mimeNone {
|
||||
w.Header().Set(xhttp.ContentType, string(mType))
|
||||
@@ -764,10 +738,6 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
|
||||
// The request is from browser and also if browser
|
||||
// is enabled we need to redirect.
|
||||
if browser && globalBrowserEnabled {
|
||||
if newObjectLayerFn() == nil {
|
||||
// server still in safe mode.
|
||||
w.Header().Set(xhttp.MinIOServerStatus, "safemode")
|
||||
}
|
||||
w.Header().Set(xhttp.Location, minioReservedBucketPath+reqURL.Path)
|
||||
w.WriteHeader(http.StatusTemporaryRedirect)
|
||||
return
|
||||
|
||||
+3
-45
@@ -21,8 +21,6 @@ import (
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/pkg/wildcard"
|
||||
"github.com/rs/cors"
|
||||
)
|
||||
|
||||
func newHTTPServerFn() *xhttp.Server {
|
||||
@@ -85,6 +83,7 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
|
||||
var routers []*mux.Router
|
||||
for _, domainName := range globalDomainNames {
|
||||
routers = append(routers, apiRouter.Host("{bucket:.+}."+domainName).Subrouter())
|
||||
routers = append(routers, apiRouter.Host("{bucket:.+}."+domainName+":{port:.*}").Subrouter())
|
||||
}
|
||||
routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
|
||||
|
||||
@@ -225,9 +224,9 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
|
||||
// ListObjectsV2
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("listobjectsv2", httpTraceAll(api.ListObjectsV2Handler)))).Queries("list-type", "2")
|
||||
// ListObjectVersions
|
||||
// ListBucketVersions
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("listobjectversions", httpTraceAll(api.ListObjectVersionsHandler)))).Queries("versions", "")
|
||||
maxClients(collectAPIStats("listbucketversions", httpTraceAll(api.ListBucketObjectVersionsHandler)))).Queries("versions", "")
|
||||
// ListObjectsV1 (Legacy)
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("listobjectsv1", httpTraceAll(api.ListObjectsV1Handler))))
|
||||
@@ -296,44 +295,3 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
|
||||
apiRouter.MethodNotAllowedHandler = http.HandlerFunc(collectAPIStats("methodnotallowed", httpTraceAll(errorResponseHandler)))
|
||||
|
||||
}
|
||||
|
||||
// corsHandler handler for CORS (Cross Origin Resource Sharing)
|
||||
func corsHandler(handler http.Handler) http.Handler {
|
||||
commonS3Headers := []string{
|
||||
xhttp.Date,
|
||||
xhttp.ETag,
|
||||
xhttp.ServerInfo,
|
||||
xhttp.Connection,
|
||||
xhttp.AcceptRanges,
|
||||
xhttp.ContentRange,
|
||||
xhttp.ContentEncoding,
|
||||
xhttp.ContentLength,
|
||||
xhttp.ContentType,
|
||||
"X-Amz*",
|
||||
"x-amz*",
|
||||
"*",
|
||||
}
|
||||
|
||||
return cors.New(cors.Options{
|
||||
AllowOriginFunc: func(origin string) bool {
|
||||
for _, allowedOrigin := range globalAPIConfig.getCorsAllowOrigins() {
|
||||
if wildcard.MatchSimple(allowedOrigin, origin) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
},
|
||||
AllowedMethods: []string{
|
||||
http.MethodGet,
|
||||
http.MethodPut,
|
||||
http.MethodHead,
|
||||
http.MethodPost,
|
||||
http.MethodDelete,
|
||||
http.MethodOptions,
|
||||
http.MethodPatch,
|
||||
},
|
||||
AllowedHeaders: commonS3Headers,
|
||||
ExposedHeaders: commonS3Headers,
|
||||
AllowCredentials: true,
|
||||
}).Handler(handler)
|
||||
}
|
||||
|
||||
+28
-40
@@ -18,7 +18,6 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
@@ -30,10 +29,8 @@ import (
|
||||
// path: 'bucket/' or '/bucket/' => Heal bucket
|
||||
// path: 'bucket/object' => Heal object
|
||||
type healTask struct {
|
||||
bucket string
|
||||
object string
|
||||
versionID string
|
||||
opts madmin.HealOpts
|
||||
path string
|
||||
opts madmin.HealOpts
|
||||
// Healing response will be sent here
|
||||
responseCh chan healResult
|
||||
}
|
||||
@@ -82,18 +79,17 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
|
||||
|
||||
var res madmin.HealResultItem
|
||||
var err error
|
||||
bucket, object := path2BucketObject(task.path)
|
||||
switch {
|
||||
case task.bucket == nopHeal:
|
||||
continue
|
||||
case task.bucket == SlashSeparator:
|
||||
case bucket == "" && object == "":
|
||||
res, err = healDiskFormat(ctx, objAPI, task.opts)
|
||||
case task.bucket != "" && task.object == "":
|
||||
res, err = objAPI.HealBucket(ctx, task.bucket, task.opts.DryRun, task.opts.Remove)
|
||||
case task.bucket != "" && task.object != "":
|
||||
res, err = objAPI.HealObject(ctx, task.bucket, task.object, task.versionID, task.opts)
|
||||
case bucket != "" && object == "":
|
||||
res, err = objAPI.HealBucket(ctx, bucket, task.opts.DryRun, task.opts.Remove)
|
||||
case bucket != "" && object != "":
|
||||
res, err = objAPI.HealObject(ctx, bucket, object, task.opts)
|
||||
}
|
||||
if task.bucket != "" && task.object != "" {
|
||||
ObjectPathUpdated(path.Join(task.bucket, task.object))
|
||||
if task.path != slashSeparator && task.path != nopHeal {
|
||||
ObjectPathUpdated(task.path)
|
||||
}
|
||||
task.responseCh <- healResult{result: res, err: err}
|
||||
case <-h.doneCh:
|
||||
@@ -104,7 +100,7 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
|
||||
}
|
||||
}
|
||||
|
||||
func newHealRoutine() *healRoutine {
|
||||
func initHealRoutine() *healRoutine {
|
||||
return &healRoutine{
|
||||
tasks: make(chan healTask),
|
||||
doneCh: make(chan struct{}),
|
||||
@@ -112,24 +108,24 @@ func newHealRoutine() *healRoutine {
|
||||
|
||||
}
|
||||
|
||||
func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
|
||||
func startBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
|
||||
// Run the background healer
|
||||
globalBackgroundHealRoutine = newHealRoutine()
|
||||
globalBackgroundHealRoutine = initHealRoutine()
|
||||
go globalBackgroundHealRoutine.run(ctx, objAPI)
|
||||
|
||||
nh := newBgHealSequence()
|
||||
// Heal any disk format and metadata early, if possible.
|
||||
if err := nh.healDiskMeta(); err != nil {
|
||||
if newObjectLayerFn() != nil {
|
||||
// log only in situations, when object layer
|
||||
// has fully initialized.
|
||||
logger.LogIf(nh.ctx, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Launch the background healer sequence to track
|
||||
// background healing operations, ignore errors
|
||||
// errors are handled into offline disks already.
|
||||
info, _ := objAPI.StorageInfo(ctx, false)
|
||||
numDisks := info.Backend.OnlineDisks.Sum() + info.Backend.OfflineDisks.Sum()
|
||||
nh := newBgHealSequence(numDisks)
|
||||
globalBackgroundHealState.LaunchNewHealSequence(nh)
|
||||
}
|
||||
|
||||
func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
|
||||
go startBackgroundHealing(ctx, objAPI)
|
||||
}
|
||||
|
||||
// healDiskFormat - heals format.json, return value indicates if a
|
||||
// failure error occurred.
|
||||
func healDiskFormat(ctx context.Context, objAPI ObjectLayer, opts madmin.HealOpts) (madmin.HealResultItem, error) {
|
||||
@@ -144,20 +140,12 @@ func healDiskFormat(ctx context.Context, objAPI ObjectLayer, opts madmin.HealOpt
|
||||
// Healing succeeded notify the peers to reload format and re-initialize disks.
|
||||
// We will not notify peers if healing is not required.
|
||||
if err == nil {
|
||||
// Notify servers in background and retry if needed.
|
||||
go func() {
|
||||
retry:
|
||||
for _, nerr := range globalNotificationSys.ReloadFormat(opts.DryRun) {
|
||||
if nerr.Err != nil {
|
||||
if nerr.Err.Error() == errServerNotInitialized.Error() {
|
||||
time.Sleep(time.Second)
|
||||
goto retry
|
||||
}
|
||||
logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
|
||||
logger.LogIf(ctx, nerr.Err)
|
||||
}
|
||||
for _, nerr := range globalNotificationSys.ReloadFormat(opts.DryRun) {
|
||||
if nerr.Err != nil {
|
||||
logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
|
||||
logger.LogIf(ctx, nerr.Err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
return res, nil
|
||||
|
||||
@@ -33,7 +33,7 @@ func initLocalDisksAutoHeal(ctx context.Context, objAPI ObjectLayer) {
|
||||
// 1. Only the concerned erasure set will be listed and healed
|
||||
// 2. Only the node hosting the disk is responsible to perform the heal
|
||||
func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
|
||||
z, ok := objAPI.(*erasureZones)
|
||||
z, ok := objAPI.(*xlZones)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
@@ -84,10 +84,10 @@ func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
|
||||
}
|
||||
|
||||
// Reformat disks
|
||||
bgSeq.sourceCh <- healSource{bucket: SlashSeparator}
|
||||
bgSeq.sourceCh <- healSource{path: SlashSeparator}
|
||||
|
||||
// Ensure that reformatting disks is finished
|
||||
bgSeq.sourceCh <- healSource{bucket: nopHeal}
|
||||
bgSeq.sourceCh <- healSource{path: nopHeal}
|
||||
|
||||
var erasureSetInZoneToHeal = make([][]int, len(localDisksInZoneHeal))
|
||||
// Compute the list of erasure set to heal
|
||||
|
||||
+17
-17
@@ -35,7 +35,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// obtains random bucket name.
|
||||
bucket := getRandomBucketName()
|
||||
// create bucket.
|
||||
err = obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
|
||||
err = obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -76,7 +76,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
|
||||
object := getRandomObjectName()
|
||||
|
||||
// create bucket.
|
||||
err = obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
|
||||
err = obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -127,9 +127,9 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
|
||||
b.StopTimer()
|
||||
}
|
||||
|
||||
// creates Erasure/FS backend setup, obtains the object layer and calls the runPutObjectPartBenchmark function.
|
||||
// creates XL/FS backend setup, obtains the object layer and calls the runPutObjectPartBenchmark function.
|
||||
func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
|
||||
// create a temp Erasure/FS backend.
|
||||
// create a temp XL/FS backend.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
objLayer, disks, err := prepareTestBackend(ctx, instanceType)
|
||||
@@ -143,9 +143,9 @@ func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
|
||||
runPutObjectPartBenchmark(b, objLayer, objSize)
|
||||
}
|
||||
|
||||
// creates Erasure/FS backend setup, obtains the object layer and calls the runPutObjectBenchmark function.
|
||||
// creates XL/FS backend setup, obtains the object layer and calls the runPutObjectBenchmark function.
|
||||
func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
|
||||
// create a temp Erasure/FS backend.
|
||||
// create a temp XL/FS backend.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
objLayer, disks, err := prepareTestBackend(ctx, instanceType)
|
||||
@@ -159,9 +159,9 @@ func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
|
||||
runPutObjectBenchmark(b, objLayer, objSize)
|
||||
}
|
||||
|
||||
// creates Erasure/FS backend setup, obtains the object layer and runs parallel benchmark for put object.
|
||||
// creates XL/FS backend setup, obtains the object layer and runs parallel benchmark for put object.
|
||||
func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int) {
|
||||
// create a temp Erasure/FS backend.
|
||||
// create a temp XL/FS backend.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
objLayer, disks, err := prepareTestBackend(ctx, instanceType)
|
||||
@@ -181,7 +181,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// obtains random bucket name.
|
||||
bucket := getRandomBucketName()
|
||||
// create bucket.
|
||||
err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
|
||||
err := obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -190,7 +190,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
|
||||
// generate etag for the generated data.
|
||||
// etag of the data to written is required as input for PutObject.
|
||||
// PutObject is the functions which writes the data onto the FS/Erasure backend.
|
||||
// PutObject is the functions which writes the data onto the FS/XL backend.
|
||||
|
||||
// get text data generated for number of bytes equal to object size.
|
||||
md5hex := getMD5Hash(textData)
|
||||
@@ -240,9 +240,9 @@ func generateBytesData(size int) []byte {
|
||||
return bytes.Repeat(getRandomByte(), size)
|
||||
}
|
||||
|
||||
// creates Erasure/FS backend setup, obtains the object layer and calls the runGetObjectBenchmark function.
|
||||
// creates XL/FS backend setup, obtains the object layer and calls the runGetObjectBenchmark function.
|
||||
func benchmarkGetObject(b *testing.B, instanceType string, objSize int) {
|
||||
// create a temp Erasure/FS backend.
|
||||
// create a temp XL/FS backend.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
objLayer, disks, err := prepareTestBackend(ctx, instanceType)
|
||||
@@ -256,9 +256,9 @@ func benchmarkGetObject(b *testing.B, instanceType string, objSize int) {
|
||||
runGetObjectBenchmark(b, objLayer, objSize)
|
||||
}
|
||||
|
||||
// creates Erasure/FS backend setup, obtains the object layer and runs parallel benchmark for ObjectLayer.GetObject() .
|
||||
// creates XL/FS backend setup, obtains the object layer and runs parallel benchmark for ObjectLayer.GetObject() .
|
||||
func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int) {
|
||||
// create a temp Erasure/FS backend.
|
||||
// create a temp XL/FS backend.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
objLayer, disks, err := prepareTestBackend(ctx, instanceType)
|
||||
@@ -278,7 +278,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// obtains random bucket name.
|
||||
bucket := getRandomBucketName()
|
||||
// create bucket.
|
||||
err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
|
||||
err := obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -322,7 +322,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// obtains random bucket name.
|
||||
bucket := getRandomBucketName()
|
||||
// create bucket.
|
||||
err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
|
||||
err := obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -331,7 +331,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
textData := generateBytesData(objSize)
|
||||
// generate md5sum for the generated data.
|
||||
// md5sum of the data to written is required as input for PutObject.
|
||||
// PutObject is the functions which writes the data onto the FS/Erasure backend.
|
||||
// PutObject is the functions which writes the data onto the FS/XL backend.
|
||||
|
||||
md5hex := getMD5Hash([]byte(textData))
|
||||
sha256hex := ""
|
||||
|
||||
@@ -30,6 +30,25 @@ import (
|
||||
// magic HH-256 key as HH-256 hash of the first 100 decimals of π as utf-8 string with a zero key.
|
||||
var magicHighwayHash256Key = []byte("\x4b\xe7\x34\xfa\x8e\x23\x8a\xcd\x26\x3e\x83\xe6\xbb\x96\x85\x52\x04\x0f\x93\x5d\xa3\x9f\x44\x14\x97\xe0\x9d\x13\x22\xde\x36\xa0")
|
||||
|
||||
// BitrotAlgorithm specifies a algorithm used for bitrot protection.
|
||||
type BitrotAlgorithm uint
|
||||
|
||||
const (
|
||||
// SHA256 represents the SHA-256 hash function
|
||||
SHA256 BitrotAlgorithm = 1 + iota
|
||||
// HighwayHash256 represents the HighwayHash-256 hash function
|
||||
HighwayHash256
|
||||
// HighwayHash256S represents the Streaming HighwayHash-256 hash function
|
||||
HighwayHash256S
|
||||
// BLAKE2b512 represents the BLAKE2b-512 hash function
|
||||
BLAKE2b512
|
||||
)
|
||||
|
||||
// DefaultBitrotAlgorithm is the default algorithm used for bitrot protection.
|
||||
const (
|
||||
DefaultBitrotAlgorithm = HighwayHash256S
|
||||
)
|
||||
|
||||
var bitrotAlgorithms = map[BitrotAlgorithm]string{
|
||||
SHA256: "sha256",
|
||||
BLAKE2b512: "blake2b",
|
||||
|
||||
+1
-1
@@ -34,7 +34,7 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
|
||||
volume := "testvol"
|
||||
filePath := "testfile"
|
||||
|
||||
disk, err := newXLStorage(tmpDir, "")
|
||||
disk, err := newPosix(tmpDir, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -20,12 +20,12 @@ import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"runtime"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
@@ -43,7 +43,6 @@ const (
|
||||
)
|
||||
|
||||
const (
|
||||
bootstrapRESTMethodHealth = "/health"
|
||||
bootstrapRESTMethodVerify = "/verify"
|
||||
)
|
||||
|
||||
@@ -95,9 +94,6 @@ func getServerSystemCfg() ServerSystemConfig {
|
||||
}
|
||||
}
|
||||
|
||||
// HealthHandler returns success if request is valid
|
||||
func (b *bootstrapRESTServer) HealthHandler(w http.ResponseWriter, r *http.Request) {}
|
||||
|
||||
func (b *bootstrapRESTServer) VerifyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "VerifyHandler")
|
||||
cfg := getServerSystemCfg()
|
||||
@@ -110,9 +106,6 @@ func registerBootstrapRESTHandlers(router *mux.Router) {
|
||||
server := &bootstrapRESTServer{}
|
||||
subrouter := router.PathPrefix(bootstrapRESTPrefix).Subrouter()
|
||||
|
||||
subrouter.Methods(http.MethodPost).Path(bootstrapRESTVersionPrefix + bootstrapRESTMethodHealth).HandlerFunc(
|
||||
httpTraceHdrs(server.HealthHandler))
|
||||
|
||||
subrouter.Methods(http.MethodPost).Path(bootstrapRESTVersionPrefix + bootstrapRESTMethodVerify).HandlerFunc(
|
||||
httpTraceHdrs(server.VerifyHandler))
|
||||
}
|
||||
@@ -121,12 +114,29 @@ func registerBootstrapRESTHandlers(router *mux.Router) {
|
||||
type bootstrapRESTClient struct {
|
||||
endpoint Endpoint
|
||||
restClient *rest.Client
|
||||
connected int32
|
||||
}
|
||||
|
||||
// Reconnect to a bootstrap rest server.k
|
||||
func (client *bootstrapRESTClient) reConnect() {
|
||||
atomic.StoreInt32(&client.connected, 1)
|
||||
}
|
||||
|
||||
// Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
|
||||
// permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
|
||||
// after verifying format.json
|
||||
func (client *bootstrapRESTClient) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
|
||||
return client.callWithContext(GlobalContext, method, values, body, length)
|
||||
}
|
||||
|
||||
// Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
|
||||
// permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
|
||||
// after verifying format.json
|
||||
func (client *bootstrapRESTClient) callWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
|
||||
if !client.IsOnline() {
|
||||
client.reConnect()
|
||||
}
|
||||
|
||||
if values == nil {
|
||||
values = make(url.Values)
|
||||
}
|
||||
@@ -136,6 +146,10 @@ func (client *bootstrapRESTClient) callWithContext(ctx context.Context, method s
|
||||
return respBody, nil
|
||||
}
|
||||
|
||||
if isNetworkError(err) {
|
||||
atomic.StoreInt32(&client.connected, 0)
|
||||
}
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -144,12 +158,24 @@ func (client *bootstrapRESTClient) String() string {
|
||||
return client.endpoint.String()
|
||||
}
|
||||
|
||||
// IsOnline - returns whether RPC client failed to connect or not.
|
||||
func (client *bootstrapRESTClient) IsOnline() bool {
|
||||
return atomic.LoadInt32(&client.connected) == 1
|
||||
}
|
||||
|
||||
// Close - marks the client as closed.
|
||||
func (client *bootstrapRESTClient) Close() error {
|
||||
atomic.StoreInt32(&client.connected, 0)
|
||||
client.restClient.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Verify - fetches system server config.
|
||||
func (client *bootstrapRESTClient) Verify(ctx context.Context, srcCfg ServerSystemConfig) (err error) {
|
||||
func (client *bootstrapRESTClient) Verify(srcCfg ServerSystemConfig) (err error) {
|
||||
if newObjectLayerFn() != nil {
|
||||
return nil
|
||||
}
|
||||
respBody, err := client.callWithContext(ctx, bootstrapRESTMethodVerify, nil, nil, -1)
|
||||
respBody, err := client.call(bootstrapRESTMethodVerify, nil, nil, -1)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -161,17 +187,14 @@ func (client *bootstrapRESTClient) Verify(ctx context.Context, srcCfg ServerSyst
|
||||
return srcCfg.Diff(recvCfg)
|
||||
}
|
||||
|
||||
func verifyServerSystemConfig(ctx context.Context, endpointZones EndpointZones) error {
|
||||
func verifyServerSystemConfig(endpointZones EndpointZones) error {
|
||||
srcCfg := getServerSystemCfg()
|
||||
clnts := newBootstrapRESTClients(endpointZones)
|
||||
var onlineServers int
|
||||
var offlineEndpoints []string
|
||||
var retries int
|
||||
for onlineServers < len(clnts)/2 {
|
||||
for _, clnt := range clnts {
|
||||
if err := clnt.Verify(ctx, srcCfg); err != nil {
|
||||
if err := clnt.Verify(srcCfg); err != nil {
|
||||
if isNetworkError(err) {
|
||||
offlineEndpoints = append(offlineEndpoints, clnt.String())
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("%s as has incorrect configuration: %w", clnt.String(), err)
|
||||
@@ -181,14 +204,6 @@ func verifyServerSystemConfig(ctx context.Context, endpointZones EndpointZones)
|
||||
// Sleep for a while - so that we don't go into
|
||||
// 100% CPU when half the endpoints are offline.
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
retries++
|
||||
// after 5 retries start logging that servers are not reachable yet
|
||||
if retries >= 5 {
|
||||
logger.Info(fmt.Sprintf("Waiting for atleast %d servers to be online for bootstrap check", len(clnts)/2))
|
||||
logger.Info(fmt.Sprintf("Following servers are currently offline or unreachable %s", offlineEndpoints))
|
||||
retries = 0 // reset to log again after 5 retries.
|
||||
}
|
||||
offlineEndpoints = nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -205,7 +220,11 @@ func newBootstrapRESTClients(endpointZones EndpointZones) []*bootstrapRESTClient
|
||||
|
||||
// Only proceed for remote endpoints.
|
||||
if !endpoint.IsLocal {
|
||||
clnts = append(clnts, newBootstrapRESTClient(endpoint))
|
||||
clnt, err := newBootstrapRESTClient(endpoint)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
clnts = append(clnts, clnt)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -213,7 +232,7 @@ func newBootstrapRESTClients(endpointZones EndpointZones) []*bootstrapRESTClient
|
||||
}
|
||||
|
||||
// Returns a new bootstrap client.
|
||||
func newBootstrapRESTClient(endpoint Endpoint) *bootstrapRESTClient {
|
||||
func newBootstrapRESTClient(endpoint Endpoint) (*bootstrapRESTClient, error) {
|
||||
serverURL := &url.URL{
|
||||
Scheme: endpoint.Scheme,
|
||||
Host: endpoint.Host,
|
||||
@@ -229,17 +248,10 @@ func newBootstrapRESTClient(endpoint Endpoint) *bootstrapRESTClient {
|
||||
}
|
||||
|
||||
trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)
|
||||
restClient := rest.NewClient(serverURL, trFn, newAuthToken)
|
||||
restClient.HealthCheckFn = func() bool {
|
||||
ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
|
||||
// Instantiate a new rest client for healthcheck
|
||||
// to avoid recursive healthCheckFn()
|
||||
respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).CallWithContext(ctx, bootstrapRESTMethodHealth, nil, nil, -1)
|
||||
xhttp.DrainBody(respBody)
|
||||
cancel()
|
||||
var ne *rest.NetworkError
|
||||
return !errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &ne)
|
||||
restClient, err := rest.NewClient(serverURL, trFn, newAuthToken)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &bootstrapRESTClient{endpoint: endpoint, restClient: restClient}
|
||||
return &bootstrapRESTClient{endpoint: endpoint, restClient: restClient, connected: 1}, nil
|
||||
}
|
||||
|
||||
@@ -55,6 +55,5 @@ func validateBucketSSEConfig(r io.Reader) (*bucketsse.BucketSSEConfig, error) {
|
||||
if len(encConfig.Rules) == 1 && encConfig.Rules[0].DefaultEncryptionAction.Algorithm == bucketsse.AES256 {
|
||||
return encConfig, nil
|
||||
}
|
||||
|
||||
return nil, errors.New("Unsupported bucket encryption configuration")
|
||||
}
|
||||
|
||||
+133
-125
@@ -25,7 +25,6 @@ import (
|
||||
"net/url"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
@@ -46,8 +45,9 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
objectLockConfig = "object-lock.xml"
|
||||
bucketTaggingConfig = "tagging.xml"
|
||||
getBucketVersioningResponse = `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`
|
||||
objectLockConfig = "object-lock.xml"
|
||||
bucketTaggingConfigFile = "tagging.xml"
|
||||
)
|
||||
|
||||
// Check if there are buckets on server without corresponding entry in etcd backend and
|
||||
@@ -377,98 +377,80 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
return
|
||||
}
|
||||
|
||||
// Before proceeding validate if bucket exists.
|
||||
_, err := objectAPI.GetBucketInfo(ctx, bucket)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
deleteObjectsFn := objectAPI.DeleteObjects
|
||||
if api.CacheAPI() != nil {
|
||||
deleteObjectsFn = api.CacheAPI().DeleteObjects
|
||||
}
|
||||
|
||||
var objectsToDelete = map[ObjectToDelete]int{}
|
||||
var objectsToDelete = map[string]int{}
|
||||
getObjectInfoFn := objectAPI.GetObjectInfo
|
||||
if api.CacheAPI() != nil {
|
||||
getObjectInfoFn = api.CacheAPI().GetObjectInfo
|
||||
}
|
||||
|
||||
dErrs := make([]DeleteError, len(deleteObjects.Objects))
|
||||
var dErrs = make([]APIErrorCode, len(deleteObjects.Objects))
|
||||
|
||||
for index, object := range deleteObjects.Objects {
|
||||
if apiErrCode := checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, object.ObjectName); apiErrCode != ErrNone {
|
||||
if apiErrCode == ErrSignatureDoesNotMatch || apiErrCode == ErrInvalidAccessKeyID {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL, guessIsBrowserReq(r))
|
||||
if dErrs[index] = checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, object.ObjectName); dErrs[index] != ErrNone {
|
||||
if dErrs[index] == ErrSignatureDoesNotMatch || dErrs[index] == ErrInvalidAccessKeyID {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(dErrs[index]), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
apiErr := errorCodes.ToAPIErr(apiErrCode)
|
||||
dErrs[index] = DeleteError{
|
||||
Code: apiErr.Code,
|
||||
Message: apiErr.Description,
|
||||
Key: object.ObjectName,
|
||||
VersionID: object.VersionID,
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if object.VersionID != "" {
|
||||
if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
|
||||
if apiErrCode := enforceRetentionBypassForDelete(ctx, r, bucket, object, getObjectInfoFn); apiErrCode != ErrNone {
|
||||
apiErr := errorCodes.ToAPIErr(apiErrCode)
|
||||
dErrs[index] = DeleteError{
|
||||
Code: apiErr.Code,
|
||||
Message: apiErr.Description,
|
||||
Key: object.ObjectName,
|
||||
VersionID: object.VersionID,
|
||||
}
|
||||
continue
|
||||
}
|
||||
if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
|
||||
if apiErr := enforceRetentionBypassForDelete(ctx, r, bucket, object.ObjectName, getObjectInfoFn); apiErr != ErrNone {
|
||||
dErrs[index] = apiErr
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Avoid duplicate objects, we use map to filter them out.
|
||||
if _, ok := objectsToDelete[object]; !ok {
|
||||
objectsToDelete[object] = index
|
||||
if _, ok := objectsToDelete[object.ObjectName]; !ok {
|
||||
objectsToDelete[object.ObjectName] = index
|
||||
}
|
||||
}
|
||||
|
||||
toNames := func(input map[ObjectToDelete]int) (output []ObjectToDelete) {
|
||||
output = make([]ObjectToDelete, len(input))
|
||||
toNames := func(input map[string]int) (output []string) {
|
||||
output = make([]string, len(input))
|
||||
idx := 0
|
||||
for obj := range input {
|
||||
output[idx] = obj
|
||||
for name := range input {
|
||||
output[idx] = name
|
||||
idx++
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
deleteList := toNames(objectsToDelete)
|
||||
dObjects, errs := deleteObjectsFn(ctx, bucket, deleteList, ObjectOptions{
|
||||
Versioned: globalBucketVersioningSys.Enabled(bucket),
|
||||
})
|
||||
|
||||
deletedObjects := make([]DeletedObject, len(deleteObjects.Objects))
|
||||
for i := range errs {
|
||||
dindex := objectsToDelete[deleteList[i]]
|
||||
apiErr := toAPIError(ctx, errs[i])
|
||||
if apiErr.Code == "" || apiErr.Code == "NoSuchKey" {
|
||||
deletedObjects[dindex] = dObjects[i]
|
||||
continue
|
||||
}
|
||||
dErrs[dindex] = DeleteError{
|
||||
Code: apiErr.Code,
|
||||
Message: apiErr.Description,
|
||||
Key: deleteList[i].ObjectName,
|
||||
VersionID: deleteList[i].VersionID,
|
||||
}
|
||||
errs, err := deleteObjectsFn(ctx, bucket, deleteList)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
for i, objName := range deleteList {
|
||||
dIdx := objectsToDelete[objName]
|
||||
dErrs[dIdx] = toAPIErrorCode(ctx, errs[i])
|
||||
}
|
||||
|
||||
// Collect deleted objects and errors if any.
|
||||
var deletedObjects []ObjectIdentifier
|
||||
var deleteErrors []DeleteError
|
||||
for _, dErr := range dErrs {
|
||||
if dErr.Code != "" {
|
||||
deleteErrors = append(deleteErrors, dErr)
|
||||
for index, errCode := range dErrs {
|
||||
object := deleteObjects.Objects[index]
|
||||
// Success deleted objects are collected separately.
|
||||
if errCode == ErrNone || errCode == ErrNoSuchKey {
|
||||
deletedObjects = append(deletedObjects, object)
|
||||
continue
|
||||
}
|
||||
apiErr := getAPIError(errCode)
|
||||
// Error during delete should be collected separately.
|
||||
deleteErrors = append(deleteErrors, DeleteError{
|
||||
Code: apiErr.Code,
|
||||
Message: apiErr.Description,
|
||||
Key: object.ObjectName,
|
||||
})
|
||||
}
|
||||
|
||||
// Generate response
|
||||
@@ -480,21 +462,12 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
|
||||
// Notify deleted event for objects.
|
||||
for _, dobj := range deletedObjects {
|
||||
objInfo := ObjectInfo{
|
||||
Name: dobj.ObjectName,
|
||||
VersionID: dobj.VersionID,
|
||||
}
|
||||
if dobj.DeleteMarker {
|
||||
objInfo = ObjectInfo{
|
||||
Name: dobj.ObjectName,
|
||||
DeleteMarker: dobj.DeleteMarker,
|
||||
VersionID: dobj.DeleteMarkerVersionID,
|
||||
}
|
||||
}
|
||||
sendEvent(eventArgs{
|
||||
EventName: event.ObjectRemovedDelete,
|
||||
BucketName: bucket,
|
||||
Object: objInfo,
|
||||
EventName: event.ObjectRemovedDelete,
|
||||
BucketName: bucket,
|
||||
Object: ObjectInfo{
|
||||
Name: dobj.ObjectName,
|
||||
},
|
||||
ReqParams: extractReqParams(r),
|
||||
RespElements: extractRespElements(w),
|
||||
UserAgent: r.UserAgent(),
|
||||
@@ -549,17 +522,12 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
return
|
||||
}
|
||||
|
||||
opts := BucketOptions{
|
||||
Location: location,
|
||||
LockEnabled: objectLockEnabled,
|
||||
}
|
||||
|
||||
if globalDNSConfig != nil {
|
||||
sr, err := globalDNSConfig.Get(bucket)
|
||||
if err != nil {
|
||||
if err == dns.ErrNoEntriesFound {
|
||||
// Proceed to creating a bucket.
|
||||
if err = objectAPI.MakeBucketWithLocation(ctx, bucket, opts); err != nil {
|
||||
if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location, objectLockEnabled); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
@@ -597,7 +565,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
|
||||
// Proceed to creating a bucket.
|
||||
err := objectAPI.MakeBucketWithLocation(ctx, bucket, opts)
|
||||
err := objectAPI.MakeBucketWithLocation(ctx, bucket, location, objectLockEnabled)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
@@ -829,17 +797,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
|
||||
// We must not use the http.Header().Set method here because some (broken)
|
||||
// clients expect the ETag header key to be literally "ETag" - not "Etag" (case-sensitive).
|
||||
// Therefore, we have to set the ETag directly as map entry.
|
||||
location := getObjectLocation(r, globalDomainNames, bucket, object)
|
||||
w.Header()[xhttp.ETag] = []string{`"` + objInfo.ETag + `"`}
|
||||
|
||||
// Set the relevant version ID as part of the response header.
|
||||
if objInfo.VersionID != "" {
|
||||
w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
|
||||
}
|
||||
|
||||
w.Header().Set(xhttp.Location, getObjectLocation(r, globalDomainNames, bucket, object))
|
||||
w.Header().Set(xhttp.Location, location)
|
||||
|
||||
// Notify object created event.
|
||||
defer sendEvent(eventArgs{
|
||||
@@ -866,9 +826,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
Bucket: objInfo.Bucket,
|
||||
Key: objInfo.Name,
|
||||
ETag: `"` + objInfo.ETag + `"`,
|
||||
Location: w.Header().Get(xhttp.Location),
|
||||
Location: location,
|
||||
})
|
||||
writeResponse(w, http.StatusCreated, resp, mimeXML)
|
||||
writeResponse(w, http.StatusCreated, resp, "application/xml")
|
||||
case "200":
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
default:
|
||||
@@ -926,35 +886,34 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
|
||||
// Verify if the caller has sufficient permissions.
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
forceDelete := false
|
||||
if value := r.Header.Get(xhttp.MinIOForceDelete); value != "" {
|
||||
var err error
|
||||
forceDelete, err = strconv.ParseBool(value)
|
||||
if err != nil {
|
||||
apiErr := errorCodes.ToAPIErr(ErrInvalidRequest)
|
||||
apiErr.Description = err.Error()
|
||||
writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
|
||||
switch value {
|
||||
case "true":
|
||||
forceDelete = true
|
||||
case "false":
|
||||
default:
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// if force delete header is set, we need to evaluate the policy anyways
|
||||
// regardless of it being true or not.
|
||||
if forceDelete {
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ForceDeleteBucketAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
} else {
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if forceDelete {
|
||||
if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if forceDelete {
|
||||
if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -962,30 +921,79 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
|
||||
|
||||
// Attempt to delete bucket.
|
||||
if err := deleteBucket(ctx, bucket, forceDelete); err != nil {
|
||||
if _, ok := err.(BucketNotEmpty); ok && (globalBucketVersioningSys.Enabled(bucket) || globalBucketVersioningSys.Suspended(bucket)) {
|
||||
apiErr := toAPIError(ctx, err)
|
||||
apiErr.Description = "The bucket you tried to delete is not empty. You must delete all versions in the bucket."
|
||||
writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
|
||||
} else {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
globalNotificationSys.DeleteBucketMetadata(ctx, bucket)
|
||||
|
||||
if globalDNSConfig != nil {
|
||||
if err := globalDNSConfig.Delete(bucket); err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually using etcdctl", err))
|
||||
// Deleting DNS entry failed, attempt to create the bucket again.
|
||||
objectAPI.MakeBucketWithLocation(ctx, bucket, "", false)
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
globalNotificationSys.DeleteBucketMetadata(ctx, bucket)
|
||||
|
||||
// Write success response.
|
||||
writeSuccessNoContent(w)
|
||||
}
|
||||
|
||||
// PutBucketVersioningHandler - PUT Bucket Versioning.
|
||||
// ----------
|
||||
// No-op. Available for API compatibility.
|
||||
func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PutBucketVersioning")
|
||||
|
||||
defer logger.AuditLog(w, r, "PutBucketVersioning", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
getBucketInfo := objectAPI.GetBucketInfo
|
||||
if _, err := getBucketInfo(ctx, bucket); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Write success response.
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
// GetBucketVersioningHandler - GET Bucket Versioning.
|
||||
// ----------
|
||||
// No-op. Available for API compatibility.
|
||||
func (api objectAPIHandlers) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetBucketVersioning")
|
||||
|
||||
defer logger.AuditLog(w, r, "GetBucketVersioning", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
getBucketInfo := objectAPI.GetBucketInfo
|
||||
if _, err := getBucketInfo(ctx, bucket); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Write success response.
|
||||
writeSuccessResponseXML(w, []byte(getBucketVersioningResponse))
|
||||
}
|
||||
|
||||
// PutBucketObjectLockConfigHandler - PUT Bucket object lock configuration.
|
||||
// ----------
|
||||
// Places an Object Lock configuration on the specified bucket. The rule
|
||||
@@ -1115,7 +1123,7 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
|
||||
if err = globalBucketMetadataSys.Update(bucket, bucketTaggingConfig, configData); err != nil {
|
||||
if err = globalBucketMetadataSys.Update(bucket, bucketTaggingConfigFile, configData); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
@@ -1183,7 +1191,7 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
if err := globalBucketMetadataSys.Update(bucket, bucketTaggingConfig, nil); err != nil {
|
||||
if err := globalBucketMetadataSys.Update(bucket, bucketTaggingConfigFile, nil); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
+15
-36
@@ -19,7 +19,6 @@ package cmd
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -29,7 +28,7 @@ import (
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
)
|
||||
|
||||
// Wrapper for calling RemoveBucket HTTP handler tests for both Erasure multiple disks and single node setup.
|
||||
// Wrapper for calling RemoveBucket HTTP handler tests for both XL multiple disks and single node setup.
|
||||
func TestRemoveBucketHandler(t *testing.T) {
|
||||
ExecObjectLayerAPITest(t, testRemoveBucketHandler, []string{"RemoveBucket"})
|
||||
}
|
||||
@@ -74,7 +73,7 @@ func testRemoveBucketHandler(obj ObjectLayer, instanceType, bucketName string, a
|
||||
}
|
||||
}
|
||||
|
||||
// Wrapper for calling GetBucketPolicy HTTP handler tests for both Erasure multiple disks and single node setup.
|
||||
// Wrapper for calling GetBucketPolicy HTTP handler tests for both XL multiple disks and single node setup.
|
||||
func TestGetBucketLocationHandler(t *testing.T) {
|
||||
ExecObjectLayerAPITest(t, testGetBucketLocationHandler, []string{"GetBucketLocation"})
|
||||
}
|
||||
@@ -218,7 +217,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
|
||||
ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq)
|
||||
}
|
||||
|
||||
// Wrapper for calling HeadBucket HTTP handler tests for both Erasure multiple disks and single node setup.
|
||||
// Wrapper for calling HeadBucket HTTP handler tests for both XL multiple disks and single node setup.
|
||||
func TestHeadBucketHandler(t *testing.T) {
|
||||
ExecObjectLayerAPITest(t, testHeadBucketHandler, []string{"HeadBucket"})
|
||||
}
|
||||
@@ -323,7 +322,7 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
|
||||
ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq)
|
||||
}
|
||||
|
||||
// Wrapper for calling TestListMultipartUploadsHandler tests for both Erasure multiple disks and single node setup.
|
||||
// Wrapper for calling TestListMultipartUploadsHandler tests for both XL multiple disks and single node setup.
|
||||
func TestListMultipartUploadsHandler(t *testing.T) {
|
||||
ExecObjectLayerAPITest(t, testListMultipartUploadsHandler, []string{"ListMultipartUploads"})
|
||||
}
|
||||
@@ -560,7 +559,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
|
||||
ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq)
|
||||
}
|
||||
|
||||
// Wrapper for calling TestListBucketsHandler tests for both Erasure multiple disks and single node setup.
|
||||
// Wrapper for calling TestListBucketsHandler tests for both XL multiple disks and single node setup.
|
||||
func TestListBucketsHandler(t *testing.T) {
|
||||
ExecObjectLayerAPITest(t, testListBucketsHandler, []string{"ListBuckets"})
|
||||
}
|
||||
@@ -654,7 +653,7 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
|
||||
ExecObjectLayerAPINilTest(t, "", "", instanceType, apiRouter, nilReq)
|
||||
}
|
||||
|
||||
// Wrapper for calling DeleteMultipleObjects HTTP handler tests for both Erasure multiple disks and single node setup.
|
||||
// Wrapper for calling DeleteMultipleObjects HTTP handler tests for both XL multiple disks and single node setup.
|
||||
func TestAPIDeleteMultipleObjectsHandler(t *testing.T) {
|
||||
ExecObjectLayerAPITest(t, testAPIDeleteMultipleObjectsHandler, []string{"DeleteMultipleObjects"})
|
||||
}
|
||||
@@ -680,17 +679,14 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
|
||||
objectNames = append(objectNames, objectName)
|
||||
}
|
||||
|
||||
getObjectToDeleteList := func(objectNames []string) (objectList []ObjectToDelete) {
|
||||
getObjectIdentifierList := func(objectNames []string) (objectIdentifierList []ObjectIdentifier) {
|
||||
for _, objectName := range objectNames {
|
||||
objectList = append(objectList, ObjectToDelete{
|
||||
ObjectName: objectName,
|
||||
})
|
||||
objectIdentifierList = append(objectIdentifierList, ObjectIdentifier{objectName})
|
||||
}
|
||||
|
||||
return objectList
|
||||
return objectIdentifierList
|
||||
}
|
||||
|
||||
getDeleteErrorList := func(objects []ObjectToDelete) (deleteErrorList []DeleteError) {
|
||||
getDeleteErrorList := func(objects []ObjectIdentifier) (deleteErrorList []DeleteError) {
|
||||
for _, obj := range objects {
|
||||
deleteErrorList = append(deleteErrorList, DeleteError{
|
||||
Code: errorCodes[ErrAccessDenied].Code,
|
||||
@@ -703,38 +699,22 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
|
||||
}
|
||||
|
||||
requestList := []DeleteObjectsRequest{
|
||||
{Quiet: false, Objects: getObjectToDeleteList(objectNames[:5])},
|
||||
{Quiet: true, Objects: getObjectToDeleteList(objectNames[5:])},
|
||||
{Quiet: false, Objects: getObjectIdentifierList(objectNames[:5])},
|
||||
{Quiet: true, Objects: getObjectIdentifierList(objectNames[5:])},
|
||||
}
|
||||
|
||||
// generate multi objects delete response.
|
||||
successRequest0 := encodeResponse(requestList[0])
|
||||
|
||||
deletedObjects := make([]DeletedObject, len(requestList[0].Objects))
|
||||
for i := range requestList[0].Objects {
|
||||
deletedObjects[i] = DeletedObject{
|
||||
ObjectName: requestList[0].Objects[i].ObjectName,
|
||||
}
|
||||
}
|
||||
|
||||
successResponse0 := generateMultiDeleteResponse(requestList[0].Quiet, deletedObjects, nil)
|
||||
successResponse0 := generateMultiDeleteResponse(requestList[0].Quiet, requestList[0].Objects, nil)
|
||||
encodedSuccessResponse0 := encodeResponse(successResponse0)
|
||||
|
||||
successRequest1 := encodeResponse(requestList[1])
|
||||
|
||||
deletedObjects = make([]DeletedObject, len(requestList[1].Objects))
|
||||
for i := range requestList[0].Objects {
|
||||
deletedObjects[i] = DeletedObject{
|
||||
ObjectName: requestList[1].Objects[i].ObjectName,
|
||||
}
|
||||
}
|
||||
|
||||
successResponse1 := generateMultiDeleteResponse(requestList[1].Quiet, deletedObjects, nil)
|
||||
successResponse1 := generateMultiDeleteResponse(requestList[1].Quiet, requestList[1].Objects, nil)
|
||||
encodedSuccessResponse1 := encodeResponse(successResponse1)
|
||||
|
||||
// generate multi objects delete response for errors.
|
||||
// errorRequest := encodeResponse(requestList[1])
|
||||
errorResponse := generateMultiDeleteResponse(requestList[1].Quiet, deletedObjects, nil)
|
||||
errorResponse := generateMultiDeleteResponse(requestList[1].Quiet, requestList[1].Objects, nil)
|
||||
encodedErrorResponse := encodeResponse(errorResponse)
|
||||
|
||||
anonRequest := encodeResponse(requestList[0])
|
||||
@@ -837,7 +817,6 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
|
||||
|
||||
// Verify whether the bucket obtained object is same as the one created.
|
||||
if testCase.expectedContent != nil && !bytes.Equal(testCase.expectedContent, actualContent) {
|
||||
fmt.Println(string(testCase.expectedContent), string(actualContent))
|
||||
t.Errorf("Test %d : MinIO %s: Object content differs from expected value.", i+1, instanceType)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
// Disabled means the lifecycle rule is inactive
|
||||
Disabled = "Disabled"
|
||||
)
|
||||
|
||||
@@ -17,10 +17,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
@@ -52,13 +49,13 @@ func validateListObjectsArgs(marker, delimiter, encodingType string, maxKeys int
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
// ListObjectVersions - GET Bucket Object versions
|
||||
// ListBucketObjectVersions - GET Bucket Object versions
|
||||
// You can use the versions subresource to list metadata about all
|
||||
// of the versions of objects in a bucket.
|
||||
func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListObjectVersions")
|
||||
func (api objectAPIHandlers) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListBucketObjectVersions")
|
||||
|
||||
defer logger.AuditLog(w, r, "ListObjectVersions", mustGetClaimsFromToken(r))
|
||||
defer logger.AuditLog(w, r, "ListBucketObjectVersions", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
@@ -77,7 +74,8 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
|
||||
urlValues := r.URL.Query()
|
||||
|
||||
// Extract all the listBucketVersions query params to their native values.
|
||||
prefix, marker, delimiter, maxkeys, encodingType, versionIDMarker, errCode := getListBucketObjectVersionsArgs(urlValues)
|
||||
// versionIDMarker is ignored here.
|
||||
prefix, marker, delimiter, maxkeys, encodingType, _, errCode := getListBucketObjectVersionsArgs(urlValues)
|
||||
if errCode != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
@@ -89,29 +87,29 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
listObjectVersions := objectAPI.ListObjectVersions
|
||||
listObjects := objectAPI.ListObjects
|
||||
|
||||
// Inititate a list object versions operation based on the input params.
|
||||
// Inititate a list objects operation based on the input params.
|
||||
// On success would return back ListObjectsInfo object to be
|
||||
// marshaled into S3 compatible XML header.
|
||||
listObjectVersionsInfo, err := listObjectVersions(ctx, bucket, prefix, marker, versionIDMarker, delimiter, maxkeys)
|
||||
listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxkeys)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
for i := range listObjectVersionsInfo.Objects {
|
||||
if crypto.IsEncrypted(listObjectVersionsInfo.Objects[i].UserDefined) {
|
||||
listObjectVersionsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectVersionsInfo.Objects[i], false)
|
||||
for i := range listObjectsInfo.Objects {
|
||||
if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
|
||||
listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false)
|
||||
}
|
||||
listObjectVersionsInfo.Objects[i].Size, err = listObjectVersionsInfo.Objects[i].GetActualSize()
|
||||
listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].GetActualSize()
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
response := generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType, maxkeys, listObjectVersionsInfo)
|
||||
response := generateListVersionsResponse(bucket, prefix, marker, delimiter, encodingType, maxkeys, listObjectsInfo)
|
||||
|
||||
// Write success response.
|
||||
writeSuccessResponseXML(w, encodeResponse(response))
|
||||
@@ -160,13 +158,6 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
|
||||
return
|
||||
}
|
||||
|
||||
// Analyze continuation token and route the request accordingly
|
||||
var success bool
|
||||
token, success = proxyRequestByToken(ctx, w, r, token)
|
||||
if success {
|
||||
return
|
||||
}
|
||||
|
||||
listObjectsV2 := objectAPI.ListObjectsV2
|
||||
|
||||
// Inititate a list objects operation based on the input params.
|
||||
@@ -189,13 +180,8 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
|
||||
}
|
||||
}
|
||||
|
||||
// The next continuation token has id@node_index format to optimize paginated listing
|
||||
nextContinuationToken := listObjectsV2Info.NextContinuationToken
|
||||
if nextContinuationToken != "" && listObjectsV2Info.IsTruncated {
|
||||
nextContinuationToken = fmt.Sprintf("%s@%d", listObjectsV2Info.NextContinuationToken, getLocalNodeIndex())
|
||||
}
|
||||
|
||||
response := generateListObjectsV2Response(bucket, prefix, token, nextContinuationToken, startAfter,
|
||||
response := generateListObjectsV2Response(bucket, prefix, token,
|
||||
listObjectsV2Info.NextContinuationToken, startAfter,
|
||||
delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
|
||||
maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, true)
|
||||
|
||||
@@ -246,13 +232,6 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
return
|
||||
}
|
||||
|
||||
// Analyze continuation token and route the request accordingly
|
||||
var success bool
|
||||
token, success = proxyRequestByToken(ctx, w, r, token)
|
||||
if success {
|
||||
return
|
||||
}
|
||||
|
||||
listObjectsV2 := objectAPI.ListObjectsV2
|
||||
|
||||
// Inititate a list objects operation based on the input params.
|
||||
@@ -275,13 +254,8 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
}
|
||||
}
|
||||
|
||||
// The next continuation token has id@node_index format to optimize paginated listing
|
||||
nextContinuationToken := listObjectsV2Info.NextContinuationToken
|
||||
if nextContinuationToken != "" && listObjectsV2Info.IsTruncated {
|
||||
nextContinuationToken = fmt.Sprintf("%s@%d", listObjectsV2Info.NextContinuationToken, getLocalNodeIndex())
|
||||
}
|
||||
|
||||
response := generateListObjectsV2Response(bucket, prefix, token, nextContinuationToken, startAfter,
|
||||
response := generateListObjectsV2Response(bucket, prefix, token,
|
||||
listObjectsV2Info.NextContinuationToken, startAfter,
|
||||
delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
|
||||
maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, false)
|
||||
|
||||
@@ -289,60 +263,6 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
writeSuccessResponseXML(w, encodeResponse(response))
|
||||
}
|
||||
|
||||
func getLocalNodeIndex() int {
|
||||
if len(globalProxyEndpoints) == 0 {
|
||||
return -1
|
||||
}
|
||||
for i, ep := range globalProxyEndpoints {
|
||||
if ep.IsLocal {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func parseRequestToken(token string) (subToken string, nodeIndex int) {
|
||||
if token == "" {
|
||||
return token, -1
|
||||
}
|
||||
i := strings.Index(token, "@")
|
||||
if i < 0 {
|
||||
return token, -1
|
||||
}
|
||||
nodeIndex, err := strconv.Atoi(token[i+1:])
|
||||
if err != nil {
|
||||
return token, -1
|
||||
}
|
||||
subToken = token[:i]
|
||||
return subToken, nodeIndex
|
||||
}
|
||||
|
||||
func proxyRequestByToken(ctx context.Context, w http.ResponseWriter, r *http.Request, token string) (string, bool) {
|
||||
subToken, nodeIndex := parseRequestToken(token)
|
||||
if nodeIndex > 0 {
|
||||
return subToken, proxyRequestByNodeIndex(ctx, w, r, nodeIndex)
|
||||
}
|
||||
return subToken, false
|
||||
}
|
||||
|
||||
func proxyRequestByNodeIndex(ctx context.Context, w http.ResponseWriter, r *http.Request, index int) (success bool) {
|
||||
if len(globalProxyEndpoints) == 0 {
|
||||
return false
|
||||
}
|
||||
if index < 0 || index >= len(globalProxyEndpoints) {
|
||||
return false
|
||||
}
|
||||
ep := globalProxyEndpoints[index]
|
||||
if ep.IsLocal {
|
||||
return false
|
||||
}
|
||||
return proxyRequest(ctx, w, r, ep)
|
||||
}
|
||||
|
||||
func proxyRequestByBucket(ctx context.Context, w http.ResponseWriter, r *http.Request, bucket string) (success bool) {
|
||||
return proxyRequestByNodeIndex(ctx, w, r, crcHashMod(bucket, len(globalProxyEndpoints)))
|
||||
}
|
||||
|
||||
// ListObjectsV1Handler - GET Bucket (List Objects) Version 1.
|
||||
// --------------------------
|
||||
// This implementation of the GET operation returns some or all (up to 10000)
|
||||
@@ -381,10 +301,6 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
|
||||
return
|
||||
}
|
||||
|
||||
if proxyRequestByBucket(ctx, w, r, bucket) {
|
||||
return
|
||||
}
|
||||
|
||||
listObjects := objectAPI.ListObjects
|
||||
|
||||
// Inititate a list objects operation based on the input params.
|
||||
|
||||
@@ -28,7 +28,6 @@ import (
|
||||
"github.com/minio/minio/pkg/bucket/lifecycle"
|
||||
objectlock "github.com/minio/minio/pkg/bucket/object/lock"
|
||||
"github.com/minio/minio/pkg/bucket/policy"
|
||||
"github.com/minio/minio/pkg/bucket/versioning"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
"github.com/minio/minio/pkg/sync/errgroup"
|
||||
@@ -77,44 +76,7 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
|
||||
|
||||
if globalIsGateway {
|
||||
// This code is needed only for gateway implementations.
|
||||
switch configFile {
|
||||
case bucketSSEConfig:
|
||||
if globalGatewayName == "nas" {
|
||||
meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
meta.EncryptionConfigXML = configData
|
||||
return meta.Save(GlobalContext, objAPI)
|
||||
}
|
||||
case bucketLifecycleConfig:
|
||||
if globalGatewayName == "nas" {
|
||||
meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
meta.LifecycleConfigXML = configData
|
||||
return meta.Save(GlobalContext, objAPI)
|
||||
}
|
||||
case bucketTaggingConfig:
|
||||
if globalGatewayName == "nas" {
|
||||
meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
meta.TaggingConfigXML = configData
|
||||
return meta.Save(GlobalContext, objAPI)
|
||||
}
|
||||
case bucketNotificationConfig:
|
||||
if globalGatewayName == "nas" {
|
||||
meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
meta.NotificationConfigXML = configData
|
||||
return meta.Save(GlobalContext, objAPI)
|
||||
}
|
||||
case bucketPolicyConfig:
|
||||
if configFile == bucketPolicyConfig {
|
||||
if configData == nil {
|
||||
return objAPI.DeleteBucketPolicy(GlobalContext, bucket)
|
||||
}
|
||||
@@ -145,12 +107,10 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
|
||||
meta.LifecycleConfigXML = configData
|
||||
case bucketSSEConfig:
|
||||
meta.EncryptionConfigXML = configData
|
||||
case bucketTaggingConfig:
|
||||
case bucketTaggingConfigFile:
|
||||
meta.TaggingConfigXML = configData
|
||||
case objectLockConfig:
|
||||
meta.ObjectLockConfigXML = configData
|
||||
case bucketVersioningConfig:
|
||||
meta.VersioningConfigXML = configData
|
||||
case bucketQuotaConfigFile:
|
||||
meta.QuotaConfigJSON = configData
|
||||
default:
|
||||
@@ -187,16 +147,6 @@ func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
// GetVersioningConfig returns configured versioning config
|
||||
// The returned object may not be modified.
|
||||
func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Versioning, error) {
|
||||
meta, err := sys.GetConfig(bucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return meta.versioningConfig, nil
|
||||
}
|
||||
|
||||
// GetTaggingConfig returns configured tagging config
|
||||
// The returned object may not be modified.
|
||||
func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, error) {
|
||||
|
||||
+5
-31
@@ -32,7 +32,6 @@ import (
|
||||
"github.com/minio/minio/pkg/bucket/lifecycle"
|
||||
objectlock "github.com/minio/minio/pkg/bucket/object/lock"
|
||||
"github.com/minio/minio/pkg/bucket/policy"
|
||||
"github.com/minio/minio/pkg/bucket/versioning"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
)
|
||||
@@ -48,7 +47,6 @@ const (
|
||||
|
||||
var (
|
||||
enabledBucketObjectLockConfig = []byte(`<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><ObjectLockEnabled>Enabled</ObjectLockEnabled></ObjectLockConfiguration>`)
|
||||
enabledBucketVersioningConfig = []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status></VersioningConfiguration>`)
|
||||
)
|
||||
|
||||
//go:generate msgp -file $GOFILE
|
||||
@@ -66,7 +64,6 @@ type BucketMetadata struct {
|
||||
NotificationConfigXML []byte
|
||||
LifecycleConfigXML []byte
|
||||
ObjectLockConfigXML []byte
|
||||
VersioningConfigXML []byte
|
||||
EncryptionConfigXML []byte
|
||||
TaggingConfigXML []byte
|
||||
QuotaConfigJSON []byte
|
||||
@@ -76,7 +73,6 @@ type BucketMetadata struct {
|
||||
notificationConfig *event.Config
|
||||
lifecycleConfig *lifecycle.Lifecycle
|
||||
objectLockConfig *objectlock.Config
|
||||
versioningConfig *versioning.Versioning
|
||||
sseConfig *bucketsse.BucketSSEConfig
|
||||
taggingConfig *tags.Tags
|
||||
quotaConfig *madmin.BucketQuota
|
||||
@@ -91,9 +87,6 @@ func newBucketMetadata(name string) BucketMetadata {
|
||||
XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
|
||||
},
|
||||
quotaConfig: &madmin.BucketQuota{},
|
||||
versioningConfig: &versioning.Versioning{
|
||||
XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -186,10 +179,6 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
|
||||
b.taggingConfig = nil
|
||||
}
|
||||
|
||||
if bytes.Equal(b.ObjectLockConfigXML, enabledBucketObjectLockConfig) {
|
||||
b.VersioningConfigXML = enabledBucketVersioningConfig
|
||||
}
|
||||
|
||||
if len(b.ObjectLockConfigXML) != 0 {
|
||||
b.objectLockConfig, err = objectlock.ParseObjectLockConfig(bytes.NewReader(b.ObjectLockConfigXML))
|
||||
if err != nil {
|
||||
@@ -199,13 +188,6 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
|
||||
b.objectLockConfig = nil
|
||||
}
|
||||
|
||||
if len(b.VersioningConfigXML) != 0 {
|
||||
b.versioningConfig, err = versioning.ParseConfig(bytes.NewReader(b.VersioningConfigXML))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if len(b.QuotaConfigJSON) != 0 {
|
||||
b.quotaConfig, err = parseBucketQuota(b.Name, b.QuotaConfigJSON)
|
||||
if err != nil {
|
||||
@@ -224,7 +206,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
|
||||
bucketLifecycleConfig,
|
||||
bucketQuotaConfigFile,
|
||||
bucketSSEConfig,
|
||||
bucketTaggingConfig,
|
||||
bucketTaggingConfigFile,
|
||||
objectLockConfig,
|
||||
}
|
||||
|
||||
@@ -262,7 +244,6 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
|
||||
case legacyBucketObjectLockEnabledConfigFile:
|
||||
if string(configData) == legacyBucketObjectLockEnabledConfig {
|
||||
b.ObjectLockConfigXML = enabledBucketObjectLockConfig
|
||||
b.VersioningConfigXML = enabledBucketVersioningConfig
|
||||
b.LockEnabled = false // legacy value unset it
|
||||
// we are only interested in b.ObjectLockConfigXML
|
||||
}
|
||||
@@ -274,11 +255,10 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
|
||||
b.LifecycleConfigXML = configData
|
||||
case bucketSSEConfig:
|
||||
b.EncryptionConfigXML = configData
|
||||
case bucketTaggingConfig:
|
||||
case bucketTaggingConfigFile:
|
||||
b.TaggingConfigXML = configData
|
||||
case objectLockConfig:
|
||||
b.ObjectLockConfigXML = configData
|
||||
b.VersioningConfigXML = enabledBucketVersioningConfig
|
||||
case bucketQuotaConfigFile:
|
||||
b.QuotaConfigJSON = configData
|
||||
}
|
||||
@@ -323,15 +303,9 @@ func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
|
||||
// deleteBucketMetadata deletes bucket metadata
|
||||
// If config does not exist no error is returned.
|
||||
func deleteBucketMetadata(ctx context.Context, obj ObjectLayer, bucket string) error {
|
||||
metadataFiles := []string{
|
||||
dataUsageCacheName,
|
||||
bucketMetadataFile,
|
||||
}
|
||||
for _, metaFile := range metadataFiles {
|
||||
configFile := path.Join(bucketConfigPrefix, bucket, metaFile)
|
||||
if err := deleteConfig(ctx, obj, configFile); err != nil && err != errConfigNotFound {
|
||||
return err
|
||||
}
|
||||
configFile := path.Join(bucketConfigPrefix, bucket, bucketMetadataFile)
|
||||
if err := deleteConfig(ctx, obj, configFile); err != nil && err != errConfigNotFound {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -66,12 +66,6 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
err = msgp.WrapError(err, "ObjectLockConfigXML")
|
||||
return
|
||||
}
|
||||
case "VersioningConfigXML":
|
||||
z.VersioningConfigXML, err = dc.ReadBytes(z.VersioningConfigXML)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "VersioningConfigXML")
|
||||
return
|
||||
}
|
||||
case "EncryptionConfigXML":
|
||||
z.EncryptionConfigXML, err = dc.ReadBytes(z.EncryptionConfigXML)
|
||||
if err != nil {
|
||||
@@ -103,9 +97,9 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// map header, size 11
|
||||
// map header, size 10
|
||||
// write "Name"
|
||||
err = en.Append(0x8b, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
err = en.Append(0x8a, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -174,16 +168,6 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
err = msgp.WrapError(err, "ObjectLockConfigXML")
|
||||
return
|
||||
}
|
||||
// write "VersioningConfigXML"
|
||||
err = en.Append(0xb3, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteBytes(z.VersioningConfigXML)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "VersioningConfigXML")
|
||||
return
|
||||
}
|
||||
// write "EncryptionConfigXML"
|
||||
err = en.Append(0xb3, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
|
||||
if err != nil {
|
||||
@@ -220,9 +204,9 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// map header, size 11
|
||||
// map header, size 10
|
||||
// string "Name"
|
||||
o = append(o, 0x8b, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
o = append(o, 0x8a, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
o = msgp.AppendString(o, z.Name)
|
||||
// string "Created"
|
||||
o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
|
||||
@@ -242,9 +226,6 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
// string "ObjectLockConfigXML"
|
||||
o = append(o, 0xb3, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
|
||||
o = msgp.AppendBytes(o, z.ObjectLockConfigXML)
|
||||
// string "VersioningConfigXML"
|
||||
o = append(o, 0xb3, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
|
||||
o = msgp.AppendBytes(o, z.VersioningConfigXML)
|
||||
// string "EncryptionConfigXML"
|
||||
o = append(o, 0xb3, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
|
||||
o = msgp.AppendBytes(o, z.EncryptionConfigXML)
|
||||
@@ -317,12 +298,6 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
err = msgp.WrapError(err, "ObjectLockConfigXML")
|
||||
return
|
||||
}
|
||||
case "VersioningConfigXML":
|
||||
z.VersioningConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.VersioningConfigXML)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "VersioningConfigXML")
|
||||
return
|
||||
}
|
||||
case "EncryptionConfigXML":
|
||||
z.EncryptionConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.EncryptionConfigXML)
|
||||
if err != nil {
|
||||
@@ -355,6 +330,6 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *BucketMetadata) Msgsize() (s int) {
|
||||
s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON)
|
||||
s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON)
|
||||
return
|
||||
}
|
||||
|
||||
+104
-37
@@ -52,6 +52,79 @@ func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention,
|
||||
return config.ToRetention(), nil
|
||||
}
|
||||
|
||||
// Similar to enforceRetentionBypassForDelete but for WebUI
|
||||
func enforceRetentionBypassForDeleteWeb(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, govBypassPerms bool) APIErrorCode {
|
||||
opts, err := getOpts(ctx, r, bucket, object)
|
||||
if err != nil {
|
||||
return toAPIErrorCode(ctx, err)
|
||||
}
|
||||
|
||||
oi, err := getObjectInfoFn(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
return toAPIErrorCode(ctx, err)
|
||||
}
|
||||
|
||||
lhold := objectlock.GetObjectLegalHoldMeta(oi.UserDefined)
|
||||
if lhold.Status.Valid() && lhold.Status == objectlock.LegalHoldOn {
|
||||
return ErrObjectLocked
|
||||
}
|
||||
|
||||
ret := objectlock.GetObjectRetentionMeta(oi.UserDefined)
|
||||
if ret.Mode.Valid() {
|
||||
switch ret.Mode {
|
||||
case objectlock.RetCompliance:
|
||||
// In compliance mode, a protected object version can't be overwritten
|
||||
// or deleted by any user, including the root user in your AWS account.
|
||||
// When an object is locked in compliance mode, its retention mode can't
|
||||
// be changed, and its retention period can't be shortened. Compliance mode
|
||||
// ensures that an object version can't be overwritten or deleted for the
|
||||
// duration of the retention period.
|
||||
t, err := objectlock.UTCNowNTP()
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return ErrObjectLocked
|
||||
}
|
||||
|
||||
if !ret.RetainUntilDate.Before(t) {
|
||||
return ErrObjectLocked
|
||||
}
|
||||
return ErrNone
|
||||
case objectlock.RetGovernance:
|
||||
// In governance mode, users can't overwrite or delete an object
|
||||
// version or alter its lock settings unless they have special
|
||||
// permissions. With governance mode, you protect objects against
|
||||
// being deleted by most users, but you can still grant some users
|
||||
// permission to alter the retention settings or delete the object
|
||||
// if necessary. You can also use governance mode to test retention-period
|
||||
// settings before creating a compliance-mode retention period.
|
||||
// To override or remove governance-mode retention settings, a
|
||||
// user must have the s3:BypassGovernanceRetention permission
|
||||
// and must explicitly include x-amz-bypass-governance-retention:true
|
||||
// as a request header with any request that requires overriding
|
||||
// governance mode.
|
||||
byPassSet := govBypassPerms && objectlock.IsObjectLockGovernanceBypassSet(r.Header)
|
||||
if !byPassSet {
|
||||
t, err := objectlock.UTCNowNTP()
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return ErrObjectLocked
|
||||
}
|
||||
|
||||
if !ret.RetainUntilDate.Before(t) {
|
||||
return ErrObjectLocked
|
||||
}
|
||||
|
||||
if !govBypassPerms {
|
||||
return ErrObjectLocked
|
||||
}
|
||||
|
||||
return ErrNone
|
||||
}
|
||||
}
|
||||
}
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
// enforceRetentionForDeletion checks if it is appropriate to remove an
|
||||
// object according to locking configuration when this is lifecycle/ bucket quota asking.
|
||||
func enforceRetentionForDeletion(ctx context.Context, objInfo ObjectInfo) (locked bool) {
|
||||
@@ -80,23 +153,14 @@ func enforceRetentionForDeletion(ctx context.Context, objInfo ObjectInfo) (locke
|
||||
// For objects in "Governance" mode, overwrite is allowed if a) object retention date is past OR
|
||||
// governance bypass headers are set and user has governance bypass permissions.
|
||||
// Objects in "Compliance" mode can be overwritten only if retention date is past.
|
||||
func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucket string, object ObjectToDelete, getObjectInfoFn GetObjectInfoFn) APIErrorCode {
|
||||
opts, err := getOpts(ctx, r, bucket, object.ObjectName)
|
||||
func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn) APIErrorCode {
|
||||
opts, err := getOpts(ctx, r, bucket, object)
|
||||
if err != nil {
|
||||
return toAPIErrorCode(ctx, err)
|
||||
}
|
||||
|
||||
opts.VersionID = object.VersionID
|
||||
|
||||
oi, err := getObjectInfoFn(ctx, bucket, object.ObjectName, opts)
|
||||
oi, err := getObjectInfoFn(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
switch err.(type) {
|
||||
case MethodNotAllowed: // This happens usually for a delete marker
|
||||
if oi.DeleteMarker {
|
||||
// Delete marker should be present and valid.
|
||||
return ErrNone
|
||||
}
|
||||
}
|
||||
return toAPIErrorCode(ctx, err)
|
||||
}
|
||||
|
||||
@@ -155,8 +219,8 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
|
||||
// If you try to delete objects protected by governance mode and have s3:BypassGovernanceRetention
|
||||
// or s3:GetBucketObjectLockConfiguration permissions, the operation will succeed.
|
||||
govBypassPerms1 := checkRequestAuthType(ctx, r, policy.BypassGovernanceRetentionAction, bucket, object.ObjectName)
|
||||
govBypassPerms2 := checkRequestAuthType(ctx, r, policy.GetBucketObjectLockConfigurationAction, bucket, object.ObjectName)
|
||||
govBypassPerms1 := checkRequestAuthType(ctx, r, policy.BypassGovernanceRetentionAction, bucket, object)
|
||||
govBypassPerms2 := checkRequestAuthType(ctx, r, policy.GetBucketObjectLockConfigurationAction, bucket, object)
|
||||
if govBypassPerms1 != ErrNone && govBypassPerms2 != ErrNone {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
@@ -267,32 +331,30 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
|
||||
return mode, retainDate, legalHold, ErrNone
|
||||
}
|
||||
|
||||
var objExists bool
|
||||
opts, err := getOpts(ctx, r, bucket, object)
|
||||
if err != nil {
|
||||
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
|
||||
}
|
||||
|
||||
if opts.VersionID != "" {
|
||||
if objInfo, err := getObjectInfoFn(ctx, bucket, object, opts); err == nil {
|
||||
r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
|
||||
t, err := objectlock.UTCNowNTP()
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return mode, retainDate, legalHold, ErrObjectLocked
|
||||
}
|
||||
|
||||
t, err := objectlock.UTCNowNTP()
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return mode, retainDate, legalHold, ErrObjectLocked
|
||||
}
|
||||
|
||||
if r.Mode == objectlock.RetCompliance && r.RetainUntilDate.After(t) {
|
||||
return mode, retainDate, legalHold, ErrObjectLocked
|
||||
}
|
||||
|
||||
mode = r.Mode
|
||||
retainDate = r.RetainUntilDate
|
||||
legalHold = objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined)
|
||||
// Disallow overwriting an object on legal hold
|
||||
if legalHold.Status == objectlock.LegalHoldOn {
|
||||
return mode, retainDate, legalHold, ErrObjectLocked
|
||||
}
|
||||
if objInfo, err := getObjectInfoFn(ctx, bucket, object, opts); err == nil {
|
||||
objExists = true
|
||||
r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
|
||||
if r.Mode == objectlock.RetCompliance && r.RetainUntilDate.After(t) {
|
||||
return mode, retainDate, legalHold, ErrObjectLocked
|
||||
}
|
||||
mode = r.Mode
|
||||
retainDate = r.RetainUntilDate
|
||||
legalHold = objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined)
|
||||
// Disallow overwriting an object on legal hold
|
||||
if legalHold.Status == objectlock.LegalHoldOn {
|
||||
return mode, retainDate, legalHold, ErrObjectLocked
|
||||
}
|
||||
}
|
||||
|
||||
@@ -312,6 +374,9 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
|
||||
if err != nil {
|
||||
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
|
||||
}
|
||||
if objExists && retainDate.After(t) {
|
||||
return mode, retainDate, legalHold, ErrObjectLocked
|
||||
}
|
||||
if retentionPermErr != ErrNone {
|
||||
return mode, retainDate, legalHold, retentionPermErr
|
||||
}
|
||||
@@ -322,14 +387,16 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
|
||||
if retentionPermErr != ErrNone {
|
||||
return mode, retainDate, legalHold, retentionPermErr
|
||||
}
|
||||
|
||||
t, err := objectlock.UTCNowNTP()
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return mode, retainDate, legalHold, ErrObjectLocked
|
||||
}
|
||||
|
||||
if !legalHoldRequested && retentionCfg.LockEnabled {
|
||||
// AWS S3 just creates a new version of object when an object is being overwritten.
|
||||
if objExists && retainDate.After(t) {
|
||||
return mode, retainDate, legalHold, ErrObjectLocked
|
||||
}
|
||||
if !legalHoldRequested {
|
||||
// inherit retention from bucket configuration
|
||||
return retentionCfg.Mode, objectlock.RetentionDate{Time: t.Add(retentionCfg.Validity)}, legalHold, ErrNone
|
||||
}
|
||||
|
||||
@@ -164,7 +164,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
}
|
||||
|
||||
// Read bucket access policy.
|
||||
config, err := globalPolicySys.Get(bucket)
|
||||
config, err := globalBucketMetadataSys.GetPolicyConfig(bucket)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
|
||||
@@ -92,7 +92,7 @@ func getAnonWriteOnlyObjectPolicy(bucketName, prefix string) *policy.Policy {
|
||||
}
|
||||
}
|
||||
|
||||
// Wrapper for calling Put Bucket Policy HTTP handler tests for both Erasure multiple disks and single node setup.
|
||||
// Wrapper for calling Put Bucket Policy HTTP handler tests for both XL multiple disks and single node setup.
|
||||
func TestPutBucketPolicyHandler(t *testing.T) {
|
||||
ExecObjectLayerAPITest(t, testPutBucketPolicyHandler, []string{"PutBucketPolicy"})
|
||||
}
|
||||
@@ -102,7 +102,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
|
||||
credentials auth.Credentials, t *testing.T) {
|
||||
|
||||
bucketName1 := fmt.Sprintf("%s-1", bucketName)
|
||||
if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, BucketOptions{}); err != nil {
|
||||
if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, "", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -314,7 +314,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
|
||||
|
||||
}
|
||||
|
||||
// Wrapper for calling Get Bucket Policy HTTP handler tests for both Erasure multiple disks and single node setup.
|
||||
// Wrapper for calling Get Bucket Policy HTTP handler tests for both XL multiple disks and single node setup.
|
||||
func TestGetBucketPolicyHandler(t *testing.T) {
|
||||
ExecObjectLayerAPITest(t, testGetBucketPolicyHandler, []string{"PutBucketPolicy", "GetBucketPolicy"})
|
||||
}
|
||||
@@ -520,7 +520,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
|
||||
ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq)
|
||||
}
|
||||
|
||||
// Wrapper for calling Delete Bucket Policy HTTP handler tests for both Erasure multiple disks and single node setup.
|
||||
// Wrapper for calling Delete Bucket Policy HTTP handler tests for both XL multiple disks and single node setup.
|
||||
func TestDeleteBucketPolicyHandler(t *testing.T) {
|
||||
ExecObjectLayerAPITest(t, testDeleteBucketPolicyHandler, []string{"PutBucketPolicy", "DeleteBucketPolicy"})
|
||||
}
|
||||
|
||||
+2
-13
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2018,2020 MinIO, Inc.
|
||||
* MinIO Cloud Storage, (C) 2018 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -70,16 +70,6 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
|
||||
principalType = "User"
|
||||
}
|
||||
|
||||
vid := r.URL.Query().Get("versionId")
|
||||
if vid == "" {
|
||||
if u, err := url.Parse(r.Header.Get(xhttp.AmzCopySource)); err == nil {
|
||||
vid = u.Query().Get("versionId")
|
||||
}
|
||||
if vid == "" {
|
||||
vid = r.Header.Get(xhttp.AmzCopySourceVersionID)
|
||||
}
|
||||
}
|
||||
|
||||
args := map[string][]string{
|
||||
"CurrentTime": {currTime.Format(time.RFC3339)},
|
||||
"EpochTime": {strconv.FormatInt(currTime.Unix(), 10)},
|
||||
@@ -90,7 +80,6 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
|
||||
"principaltype": {principalType},
|
||||
"userid": {username},
|
||||
"username": {username},
|
||||
"versionid": {vid},
|
||||
}
|
||||
|
||||
if lc != "" {
|
||||
@@ -153,7 +142,7 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
|
||||
return args
|
||||
}
|
||||
|
||||
// PolicyToBucketAccessPolicy converts a MinIO policy into a minio-go policy data structure.
|
||||
// PolicyToBucketAccessPolicy - converts policy.Policy to minio-go/policy.BucketAccessPolicy.
|
||||
func PolicyToBucketAccessPolicy(bucketPolicy *policy.Policy) (*miniogopolicy.BucketAccessPolicy, error) {
|
||||
// Return empty BucketAccessPolicy for empty bucket policy.
|
||||
if bucketPolicy == nil {
|
||||
|
||||
+35
-46
@@ -138,7 +138,7 @@ func startBucketQuotaEnforcement(ctx context.Context, objAPI ObjectLayer) {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.NewTimer(bgQuotaInterval).C:
|
||||
enforceFIFOQuota(ctx, objAPI)
|
||||
logger.LogIf(ctx, enforceFIFOQuota(ctx, objAPI))
|
||||
}
|
||||
|
||||
}
|
||||
@@ -146,22 +146,20 @@ func startBucketQuotaEnforcement(ctx context.Context, objAPI ObjectLayer) {
|
||||
|
||||
// enforceFIFOQuota deletes objects in FIFO order until sufficient objects
|
||||
// have been deleted so as to bring bucket usage within quota
|
||||
func enforceFIFOQuota(ctx context.Context, objectAPI ObjectLayer) {
|
||||
func enforceFIFOQuota(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
// Turn off quota enforcement if data usage info is unavailable.
|
||||
if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOff {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
||||
buckets, err := objectAPI.ListBuckets(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return
|
||||
return err
|
||||
}
|
||||
|
||||
dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return
|
||||
return err
|
||||
}
|
||||
|
||||
for _, binfo := range buckets {
|
||||
@@ -196,12 +194,9 @@ func enforceFIFOQuota(ctx context.Context, objectAPI ObjectLayer) {
|
||||
// Allocate new results channel to receive ObjectInfo.
|
||||
objInfoCh := make(chan ObjectInfo)
|
||||
|
||||
versioned := globalBucketVersioningSys.Enabled(bucket)
|
||||
|
||||
// Walk through all objects
|
||||
if err := objectAPI.Walk(ctx, bucket, "", objInfoCh, ObjectOptions{WalkVersions: versioned}); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
continue
|
||||
if err := objectAPI.Walk(ctx, bucket, "", objInfoCh); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// reuse the fileScorer used by disk cache to score entries by
|
||||
@@ -210,59 +205,53 @@ func enforceFIFOQuota(ctx context.Context, objectAPI ObjectLayer) {
|
||||
// irrelevant.
|
||||
scorer, err := newFileScorer(toFree, time.Now().Unix(), 1)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
continue
|
||||
return err
|
||||
}
|
||||
|
||||
rcfg, _ := globalBucketObjectLockSys.Get(bucket)
|
||||
|
||||
for obj := range objInfoCh {
|
||||
if obj.DeleteMarker {
|
||||
// Delete markers are automatically added for FIFO purge.
|
||||
scorer.addFileWithObjInfo(obj, 1)
|
||||
continue
|
||||
}
|
||||
// skip objects currently under retention
|
||||
if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
|
||||
continue
|
||||
}
|
||||
scorer.addFileWithObjInfo(obj, 1)
|
||||
scorer.addFile(obj.Name, obj.ModTime, obj.Size, 1)
|
||||
}
|
||||
|
||||
var objects []ObjectToDelete
|
||||
numKeys := len(scorer.fileObjInfos())
|
||||
for i, obj := range scorer.fileObjInfos() {
|
||||
objects = append(objects, ObjectToDelete{
|
||||
ObjectName: obj.Name,
|
||||
VersionID: obj.VersionID,
|
||||
})
|
||||
var objects []string
|
||||
numKeys := len(scorer.fileNames())
|
||||
for i, key := range scorer.fileNames() {
|
||||
objects = append(objects, key)
|
||||
if len(objects) < maxDeleteList && (i < numKeys-1) {
|
||||
// skip deletion until maxDeleteList or end of slice
|
||||
// skip deletion until maxObjectList or end of slice
|
||||
continue
|
||||
}
|
||||
|
||||
if len(objects) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// Deletes a list of objects.
|
||||
_, deleteErrs := objectAPI.DeleteObjects(ctx, bucket, objects, ObjectOptions{
|
||||
Versioned: versioned,
|
||||
})
|
||||
for i := range deleteErrs {
|
||||
if deleteErrs[i] != nil {
|
||||
logger.LogIf(ctx, deleteErrs[i])
|
||||
continue
|
||||
deleteErrs, err := objectAPI.DeleteObjects(ctx, bucket, objects)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
} else {
|
||||
for i := range deleteErrs {
|
||||
if deleteErrs[i] != nil {
|
||||
logger.LogIf(ctx, deleteErrs[i])
|
||||
continue
|
||||
}
|
||||
// Notify object deleted event.
|
||||
sendEvent(eventArgs{
|
||||
EventName: event.ObjectRemovedDelete,
|
||||
BucketName: bucket,
|
||||
Object: ObjectInfo{
|
||||
Name: objects[i],
|
||||
},
|
||||
Host: "Internal: [FIFO-QUOTA-EXPIRY]",
|
||||
})
|
||||
}
|
||||
|
||||
// Notify object deleted event.
|
||||
sendEvent(eventArgs{
|
||||
EventName: event.ObjectRemovedDelete,
|
||||
BucketName: bucket,
|
||||
Object: obj,
|
||||
Host: "Internal: [FIFO-QUOTA-EXPIRY]",
|
||||
})
|
||||
objects = nil
|
||||
}
|
||||
objects = nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,130 +0,0 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2020 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/bucket/policy"
|
||||
"github.com/minio/minio/pkg/bucket/versioning"
|
||||
)
|
||||
|
||||
const (
|
||||
bucketVersioningConfig = "versioning.xml"
|
||||
|
||||
// Maximum size of bucket versioning configuration payload sent to the PutBucketVersioningHandler.
|
||||
maxBucketVersioningConfigSize = 1 * humanize.MiByte
|
||||
)
|
||||
|
||||
// PutBucketVersioningHandler - PUT Bucket Versioning.
|
||||
// ----------
|
||||
func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PutBucketVersioning")
|
||||
|
||||
defer logger.AuditLog(w, r, "PutBucketVersioning", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketVersioningAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
v, err := versioning.ParseConfig(io.LimitReader(r.Body, maxBucketVersioningConfigSize))
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && v.Suspended() {
|
||||
writeErrorResponse(ctx, w, APIError{
|
||||
Code: "InvalidBucketState",
|
||||
Description: "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.",
|
||||
HTTPStatusCode: http.StatusConflict,
|
||||
}, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
configData, err := xml.Marshal(v)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if err = globalBucketMetadataSys.Update(bucket, bucketVersioningConfig, configData); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
// GetBucketVersioningHandler - GET Bucket Versioning.
|
||||
// ----------
|
||||
func (api objectAPIHandlers) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetBucketVersioning")
|
||||
|
||||
defer logger.AuditLog(w, r, "GetBucketVersioning", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketVersioningAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
config, err := globalBucketVersioningSys.Get(bucket)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
configData, err := xml.Marshal(config)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Write bucket versioning configuration to client
|
||||
writeSuccessResponseXML(w, configData)
|
||||
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2020 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import "github.com/minio/minio/pkg/bucket/versioning"
|
||||
|
||||
// BucketVersioningSys - policy subsystem.
|
||||
type BucketVersioningSys struct{}
|
||||
|
||||
// Enabled enabled versioning?
|
||||
func (sys *BucketVersioningSys) Enabled(bucket string) bool {
|
||||
vc, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return vc.Enabled()
|
||||
}
|
||||
|
||||
// Suspended suspended versioning?
|
||||
func (sys *BucketVersioningSys) Suspended(bucket string) bool {
|
||||
vc, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return vc.Suspended()
|
||||
}
|
||||
|
||||
// Get returns stored bucket policy
|
||||
func (sys *BucketVersioningSys) Get(bucket string) (*versioning.Versioning, error) {
|
||||
if globalIsGateway {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return nil, errServerNotInitialized
|
||||
}
|
||||
return nil, NotImplemented{}
|
||||
}
|
||||
return globalBucketMetadataSys.GetVersioningConfig(bucket)
|
||||
}
|
||||
|
||||
// NewBucketVersioningSys - creates new versioning system.
|
||||
func NewBucketVersioningSys() *BucketVersioningSys {
|
||||
return &BucketVersioningSys{}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2015-2020 MinIO, Inc.
|
||||
* MinIO Cloud Storage, (C) 2015, 2016 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -25,15 +25,15 @@ var (
|
||||
// GOROOT - GOROOT value at the time of build.
|
||||
GOROOT = ""
|
||||
|
||||
// Go get development tag.
|
||||
goGetTag = "DEVELOPMENT.GOGET"
|
||||
|
||||
// Version - version time.RFC3339.
|
||||
Version = "DEVELOPMENT.GOGET"
|
||||
|
||||
Version = goGetTag
|
||||
// ReleaseTag - release tag in TAG.%Y-%m-%dT%H-%M-%SZ.
|
||||
ReleaseTag = "DEVELOPMENT.GOGET"
|
||||
|
||||
ReleaseTag = goGetTag
|
||||
// CommitID - latest commit id.
|
||||
CommitID = "DEVELOPMENT.GOGET"
|
||||
|
||||
CommitID = goGetTag
|
||||
// ShortCommitID - first 12 characters from CommitID.
|
||||
ShortCommitID = "DEVELOPMENT.GOGET"
|
||||
ShortCommitID = CommitID[:12]
|
||||
)
|
||||
|
||||
@@ -50,7 +50,7 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]b
|
||||
}
|
||||
|
||||
func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
|
||||
_, err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile, ObjectOptions{})
|
||||
err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile)
|
||||
if err != nil && isErrObjectNotFound(err) {
|
||||
return errConfigNotFound
|
||||
}
|
||||
|
||||
+29
-44
@@ -59,7 +59,7 @@ func initHelp() {
|
||||
for k, v := range notify.DefaultNotificationKVS {
|
||||
kvs[k] = v
|
||||
}
|
||||
if globalIsErasure {
|
||||
if globalIsXL {
|
||||
kvs[config.StorageClassSubSys] = storageclass.DefaultKVS
|
||||
}
|
||||
config.RegisterDefaultKVS(kvs)
|
||||
@@ -168,7 +168,7 @@ func initHelp() {
|
||||
},
|
||||
}
|
||||
|
||||
if globalIsErasure {
|
||||
if globalIsXL {
|
||||
helpSubSys = append(helpSubSys, config.HelpKV{})
|
||||
copy(helpSubSys[2:], helpSubSys[1:])
|
||||
helpSubSys[1] = config.HelpKV{
|
||||
@@ -232,9 +232,9 @@ func validateConfig(s config.Config) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if globalIsErasure {
|
||||
if globalIsXL {
|
||||
if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default],
|
||||
globalErasureSetDriveCount); err != nil {
|
||||
globalXLSetDriveCount); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -325,40 +325,13 @@ func lookupConfigs(s config.Config) {
|
||||
|
||||
etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
|
||||
if err != nil {
|
||||
if globalIsGateway {
|
||||
logger.FatalIf(err, "Unable to initialize etcd config")
|
||||
} else {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
|
||||
}
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
|
||||
}
|
||||
|
||||
if etcdCfg.Enabled {
|
||||
if globalEtcdClient == nil {
|
||||
globalEtcdClient, err = etcd.New(etcdCfg)
|
||||
if err != nil {
|
||||
if globalIsGateway {
|
||||
logger.FatalIf(err, "Unable to initialize etcd config")
|
||||
} else {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil && globalDNSConfig == nil {
|
||||
globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
|
||||
dns.DomainNames(globalDomainNames),
|
||||
dns.DomainIPs(globalDomainIPs),
|
||||
dns.DomainPort(globalMinioPort),
|
||||
dns.CoreDNSPath(etcdCfg.CoreDNSPath),
|
||||
)
|
||||
if err != nil {
|
||||
if globalIsGateway {
|
||||
logger.FatalIf(err, "Unable to initialize DNS config")
|
||||
} else {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
|
||||
globalDomainNames, err))
|
||||
}
|
||||
}
|
||||
globalEtcdClient, err = etcd.New(etcdCfg)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -369,6 +342,19 @@ func lookupConfigs(s config.Config) {
|
||||
// but not federation.
|
||||
globalBucketFederation = etcdCfg.PathPrefix == "" && etcdCfg.Enabled
|
||||
|
||||
if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
|
||||
globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
|
||||
dns.DomainNames(globalDomainNames),
|
||||
dns.DomainIPs(globalDomainIPs),
|
||||
dns.DomainPort(globalMinioPort),
|
||||
dns.CoreDNSPath(etcdCfg.CoreDNSPath),
|
||||
)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
|
||||
globalDomainNames, err))
|
||||
}
|
||||
}
|
||||
|
||||
globalServerRegion, err = config.LookupRegion(s[config.RegionSubSys][config.Default])
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Invalid region configuration: %w", err))
|
||||
@@ -381,9 +367,9 @@ func lookupConfigs(s config.Config) {
|
||||
|
||||
globalAPIConfig.init(apiConfig)
|
||||
|
||||
if globalIsErasure {
|
||||
if globalIsXL {
|
||||
globalStorageClass, err = storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default],
|
||||
globalErasureSetDriveCount)
|
||||
globalXLSetDriveCount)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
|
||||
}
|
||||
@@ -391,11 +377,7 @@ func lookupConfigs(s config.Config) {
|
||||
|
||||
globalCacheConfig, err = cache.LookupConfig(s[config.CacheSubSys][config.Default])
|
||||
if err != nil {
|
||||
if globalIsGateway {
|
||||
logger.FatalIf(err, "Unable to setup cache")
|
||||
} else {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
|
||||
}
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
|
||||
}
|
||||
|
||||
if globalCacheConfig.Enabled {
|
||||
@@ -482,12 +464,12 @@ func lookupConfigs(s config.Config) {
|
||||
}
|
||||
}
|
||||
|
||||
globalConfigTargetList, err = notify.GetNotificationTargets(s, GlobalContext.Done(), NewGatewayHTTPTransport(), false)
|
||||
globalConfigTargetList, err = notify.GetNotificationTargets(s, GlobalContext.Done(), NewGatewayHTTPTransport())
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
|
||||
}
|
||||
|
||||
globalEnvTargetList, err = notify.GetNotificationTargets(newServerConfig(), GlobalContext.Done(), NewGatewayHTTPTransport(), true)
|
||||
globalEnvTargetList, err = notify.GetNotificationTargets(newServerConfig(), GlobalContext.Done(), NewGatewayHTTPTransport())
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
|
||||
}
|
||||
@@ -579,6 +561,9 @@ func newSrvConfig(objAPI ObjectLayer) error {
|
||||
// Initialize server config.
|
||||
srvCfg := newServerConfig()
|
||||
|
||||
// Override any values from ENVs.
|
||||
lookupConfigs(srvCfg)
|
||||
|
||||
// hold the mutex lock before a new config is assigned.
|
||||
globalServerConfigMu.Lock()
|
||||
globalServerConfig = srvCfg
|
||||
|
||||
@@ -24,14 +24,17 @@ import (
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
etcd "github.com/coreos/etcd/clientv3"
|
||||
"github.com/minio/minio/cmd/config"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
etcd "go.etcd.io/etcd/v3/clientv3"
|
||||
)
|
||||
|
||||
func handleEncryptedConfigBackend(objAPI ObjectLayer) error {
|
||||
func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
|
||||
if !server {
|
||||
return nil
|
||||
}
|
||||
|
||||
encrypted, err := checkBackendEncrypted(objAPI)
|
||||
if err != nil {
|
||||
|
||||
+20
-3
@@ -23,6 +23,7 @@ import (
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"github.com/minio/minio/cmd/config"
|
||||
@@ -91,8 +92,7 @@ func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData b
|
||||
|
||||
func delServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) error {
|
||||
historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV+kvPrefix)
|
||||
_, err := objAPI.DeleteObject(ctx, minioMetaBucket, historyFile, ObjectOptions{})
|
||||
return err
|
||||
return objAPI.DeleteObject(ctx, minioMetaBucket, historyFile)
|
||||
}
|
||||
|
||||
func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) ([]byte, error) {
|
||||
@@ -149,7 +149,7 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
|
||||
if err != nil {
|
||||
// Config not found for some reason, allow things to continue
|
||||
// by initializing a new fresh config in safe mode.
|
||||
if err == errConfigNotFound && newObjectLayerFn() == nil {
|
||||
if err == errConfigNotFound && globalSafeMode {
|
||||
return newServerConfig(), nil
|
||||
}
|
||||
return nil, err
|
||||
@@ -183,6 +183,23 @@ func (sys *ConfigSys) Load(objAPI ObjectLayer) error {
|
||||
return sys.Init(objAPI)
|
||||
}
|
||||
|
||||
// WatchConfigNASDisk - watches nas disk on periodic basis.
|
||||
func (sys *ConfigSys) WatchConfigNASDisk(ctx context.Context, objAPI ObjectLayer) {
|
||||
configInterval := globalRefreshIAMInterval
|
||||
watchDisk := func() {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(configInterval):
|
||||
loadConfig(objAPI)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Refresh configSys in background for NAS gateway.
|
||||
go watchDisk()
|
||||
}
|
||||
|
||||
// Init - initializes config system from config.json.
|
||||
func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
|
||||
if objAPI == nil {
|
||||
|
||||
@@ -33,6 +33,12 @@ var (
|
||||
Optional: true,
|
||||
Type: "duration",
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: apiReadyDeadline,
|
||||
Description: `set the deadline for health check API /minio/health/ready e.g. "1m"`,
|
||||
Optional: true,
|
||||
Type: "duration",
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: apiCorsAllowOrigin,
|
||||
Description: `set comma separated list of origins allowed for CORS requests e.g. "https://example1.com,https://example2.com"`,
|
||||
|
||||
Vendored
-1
@@ -37,7 +37,6 @@ type Config struct {
|
||||
After int `json:"after"`
|
||||
WatermarkLow int `json:"watermark_low"`
|
||||
WatermarkHigh int `json:"watermark_high"`
|
||||
Range bool `json:"range"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON - implements JSON unmarshal interface for unmarshalling
|
||||
|
||||
Vendored
-6
@@ -68,11 +68,5 @@ var (
|
||||
Optional: true,
|
||||
Type: "number",
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: Range,
|
||||
Description: `set to "on" or "off" caching of independent range requests per object, defaults to "on"`,
|
||||
Optional: true,
|
||||
Type: "string",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
Vendored
-16
@@ -34,7 +34,6 @@ const (
|
||||
After = "after"
|
||||
WatermarkLow = "watermark_low"
|
||||
WatermarkHigh = "watermark_high"
|
||||
Range = "range"
|
||||
|
||||
EnvCacheDrives = "MINIO_CACHE_DRIVES"
|
||||
EnvCacheExclude = "MINIO_CACHE_EXCLUDE"
|
||||
@@ -44,7 +43,6 @@ const (
|
||||
EnvCacheAfter = "MINIO_CACHE_AFTER"
|
||||
EnvCacheWatermarkLow = "MINIO_CACHE_WATERMARK_LOW"
|
||||
EnvCacheWatermarkHigh = "MINIO_CACHE_WATERMARK_HIGH"
|
||||
EnvCacheRange = "MINIO_CACHE_RANGE"
|
||||
|
||||
EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY"
|
||||
|
||||
@@ -86,10 +84,6 @@ var (
|
||||
Key: WatermarkHigh,
|
||||
Value: DefaultWaterMarkHigh,
|
||||
},
|
||||
config.KV{
|
||||
Key: Range,
|
||||
Value: config.EnableOn,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
@@ -201,15 +195,5 @@ func LookupConfig(kvs config.KVS) (Config, error) {
|
||||
err := errors.New("config high watermark value should be greater than low watermark value")
|
||||
return cfg, config.ErrInvalidCacheWatermarkHigh(err)
|
||||
}
|
||||
|
||||
cfg.Range = true // by default range caching is enabled.
|
||||
if rangeStr := env.Get(EnvCacheRange, kvs.Get(Range)); rangeStr != "" {
|
||||
rng, err := config.ParseBool(rangeStr)
|
||||
if err != nil {
|
||||
return cfg, config.ErrInvalidCacheRange(err)
|
||||
}
|
||||
cfg.Range = rng
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
@@ -96,12 +96,6 @@ var (
|
||||
"MINIO_CACHE_ENCRYPTION_MASTER_KEY: For more information, please refer to https://docs.min.io/docs/minio-disk-cache-guide",
|
||||
)
|
||||
|
||||
ErrInvalidCacheRange = newErrFn(
|
||||
"Invalid cache range value",
|
||||
"Please check the passed value",
|
||||
"MINIO_CACHE_RANGE: Valid expected value is `on` or `off`",
|
||||
)
|
||||
|
||||
ErrInvalidRotatingCredentialsBackendEncrypted = newErrFn(
|
||||
"Invalid rotating credentials",
|
||||
"Please set correct rotating credentials in the environment for decryption",
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2018,2020 MinIO, Inc.
|
||||
* MinIO Cloud Storage, (C) 2018 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -29,7 +29,7 @@ import (
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
|
||||
"github.com/coredns/coredns/plugin/etcd/msg"
|
||||
"go.etcd.io/etcd/v3/clientv3"
|
||||
"github.com/coreos/etcd/clientv3"
|
||||
)
|
||||
|
||||
// ErrNoEntriesFound - Indicates no entries were found for the given key (directory)
|
||||
@@ -47,11 +47,6 @@ func newCoreDNSMsg(ip string, port string, ttl uint32, t time.Time) ([]byte, err
|
||||
})
|
||||
}
|
||||
|
||||
// Close closes the internal etcd client and cannot be used further
|
||||
func (c *CoreDNS) Close() {
|
||||
c.etcdClient.Close()
|
||||
}
|
||||
|
||||
// List - Retrieves list of DNS entries for the domain.
|
||||
func (c *CoreDNS) List() (map[string][]SrvRecord, error) {
|
||||
var srvRecords = map[string][]SrvRecord{}
|
||||
|
||||
@@ -22,11 +22,11 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/etcd/clientv3"
|
||||
"github.com/coreos/etcd/clientv3/namespace"
|
||||
"github.com/minio/minio/cmd/config"
|
||||
"github.com/minio/minio/pkg/env"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
"go.etcd.io/etcd/v3/clientv3"
|
||||
"go.etcd.io/etcd/v3/clientv3/namespace"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -44,12 +44,6 @@ var (
|
||||
Optional: true,
|
||||
Type: "string",
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: Scopes,
|
||||
Description: `Comma separated list of OpenID scopes for server, defaults to advertised scopes from discovery document e.g. "email,admin"`,
|
||||
Optional: true,
|
||||
Type: "csv",
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: config.Comment,
|
||||
Description: config.DefaultComment,
|
||||
|
||||
@@ -24,7 +24,6 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
jwtgo "github.com/dgrijalva/jwt-go"
|
||||
@@ -218,14 +217,12 @@ const (
|
||||
ClaimName = "claim_name"
|
||||
ClaimPrefix = "claim_prefix"
|
||||
ClientID = "client_id"
|
||||
Scopes = "scopes"
|
||||
|
||||
EnvIdentityOpenIDClientID = "MINIO_IDENTITY_OPENID_CLIENT_ID"
|
||||
EnvIdentityOpenIDJWKSURL = "MINIO_IDENTITY_OPENID_JWKS_URL"
|
||||
EnvIdentityOpenIDURL = "MINIO_IDENTITY_OPENID_CONFIG_URL"
|
||||
EnvIdentityOpenIDClaimName = "MINIO_IDENTITY_OPENID_CLAIM_NAME"
|
||||
EnvIdentityOpenIDClaimPrefix = "MINIO_IDENTITY_OPENID_CLAIM_PREFIX"
|
||||
EnvIdentityOpenIDScopes = "MINIO_IDENTITY_OPENID_SCOPES"
|
||||
)
|
||||
|
||||
// DiscoveryDoc - parses the output from openid-configuration
|
||||
@@ -290,10 +287,6 @@ var (
|
||||
Key: ClaimPrefix,
|
||||
Value: "",
|
||||
},
|
||||
config.KV{
|
||||
Key: Scopes,
|
||||
Value: "",
|
||||
},
|
||||
config.KV{
|
||||
Key: JwksURL,
|
||||
Value: "",
|
||||
@@ -338,19 +331,6 @@ func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io
|
||||
}
|
||||
}
|
||||
|
||||
if scopeList := env.Get(EnvIdentityOpenIDScopes, kvs.Get(Scopes)); scopeList != "" {
|
||||
var scopes []string
|
||||
for _, scope := range strings.Split(scopeList, ",") {
|
||||
scope = strings.TrimSpace(scope)
|
||||
if scope == "" {
|
||||
return c, config.Errorf("empty scope value is not allowed '%s', please refer to our documentation", scopeList)
|
||||
}
|
||||
scopes = append(scopes, scope)
|
||||
}
|
||||
// Replace the discovery document scopes by client customized scopes.
|
||||
c.DiscoveryDoc.ScopesSupported = scopes
|
||||
}
|
||||
|
||||
if c.ClaimName == "" {
|
||||
c.ClaimName = iampolicy.PolicyName
|
||||
}
|
||||
|
||||
@@ -60,7 +60,8 @@ func TestNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transpor
|
||||
|
||||
// GetNotificationTargets registers and initializes all notification
|
||||
// targets, returns error if any.
|
||||
func GetNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport, test bool) (*event.TargetList, error) {
|
||||
func GetNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport) (*event.TargetList, error) {
|
||||
test := false
|
||||
returnOnTargetError := false
|
||||
return RegisterNotificationTargets(cfg, doneCh, transport, nil, test, returnOnTargetError)
|
||||
}
|
||||
@@ -71,6 +72,7 @@ func GetNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport
|
||||
// * Add newly added target configuration to serverConfig.Notify.<TARGET_NAME>.
|
||||
// * Handle the configuration in this function to create/add into TargetList.
|
||||
func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport, targetIDs []event.TargetID, test bool, returnOnTargetError bool) (*event.TargetList, error) {
|
||||
|
||||
targetList, err := FetchRegisteredTargets(cfg, doneCh, transport, test, returnOnTargetError)
|
||||
if err != nil {
|
||||
return targetList, err
|
||||
|
||||
@@ -45,7 +45,7 @@ func mustGetNodeName(endpointZones EndpointZones) (nodeName string) {
|
||||
if err != nil {
|
||||
logger.FatalIf(err, "Unable to start console logging subsystem")
|
||||
}
|
||||
if globalIsDistErasure {
|
||||
if globalIsDistXL {
|
||||
nodeName = host.Name
|
||||
}
|
||||
return nodeName
|
||||
|
||||
@@ -32,9 +32,7 @@ func writeCopyPartErr(ctx context.Context, w http.ResponseWriter, err error, url
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopyPartRangeSource), url, browser)
|
||||
return
|
||||
default:
|
||||
apiErr := errorCodes.ToAPIErr(ErrInvalidCopyPartRangeSource)
|
||||
apiErr.Description = err.Error()
|
||||
writeErrorResponse(ctx, w, apiErr, url, browser)
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), url, browser)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
+33
-17
@@ -34,9 +34,9 @@ import (
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
)
|
||||
|
||||
// ErrKESKeyExists is the error returned a KES server
|
||||
// when a master key does exist.
|
||||
var ErrKESKeyExists = NewKESError(http.StatusBadRequest, "key does already exist")
|
||||
// ErrKESKeyNotFound is the error returned a KES server
|
||||
// when a master key does not exist.
|
||||
var ErrKESKeyNotFound = NewKESError(http.StatusNotFound, "key does not exist")
|
||||
|
||||
// KesConfig contains the configuration required
|
||||
// to initialize and connect to a kes server.
|
||||
@@ -134,27 +134,20 @@ func NewKes(cfg KesConfig) (KMS, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DefaultKeyID returns the default key ID that should be
|
||||
// used for SSE-S3 or SSE-KMS when the S3 client does not
|
||||
// provide an explicit key ID.
|
||||
func (kes *kesService) DefaultKeyID() string {
|
||||
// KeyID returns the default key ID.
|
||||
func (kes *kesService) KeyID() string {
|
||||
return kes.defaultKeyID
|
||||
}
|
||||
|
||||
// Info returns some information about the KES,
|
||||
// configuration - like the endpoint or authentication
|
||||
// method.
|
||||
// Info returns some status information about the KMS.
|
||||
func (kes *kesService) Info() KMSInfo {
|
||||
return KMSInfo{
|
||||
Endpoint: kes.endpoint,
|
||||
Name: kes.DefaultKeyID(),
|
||||
Name: kes.KeyID(),
|
||||
AuthType: "TLS",
|
||||
}
|
||||
}
|
||||
|
||||
// CreateKey tries to create a new master key with the given keyID.
|
||||
func (kes *kesService) CreateKey(keyID string) error { return kes.client.CreateKey(keyID) }
|
||||
|
||||
// GenerateKey returns a new plaintext key, generated by the KMS,
|
||||
// and a sealed version of this plaintext key encrypted using the
|
||||
// named key referenced by keyID. It also binds the generated key
|
||||
@@ -165,6 +158,12 @@ func (kes *kesService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
|
||||
|
||||
var plainKey []byte
|
||||
plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context.Bytes())
|
||||
if err == ErrKESKeyNotFound { // Try to create the key if it does not exist.
|
||||
if err = kes.client.CreateKey(keyID); err != nil {
|
||||
return key, nil, err
|
||||
}
|
||||
plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context.Bytes())
|
||||
}
|
||||
if err != nil {
|
||||
return key, nil, err
|
||||
}
|
||||
@@ -198,11 +197,28 @@ func (kes *kesService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (k
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// UpdateKey re-wraps the sealedKey if the master key referenced by the keyID
|
||||
// has been changed by the KMS operator - i.e. the master key has been rotated.
|
||||
// If the master key hasn't changed since the sealedKey has been created / updated
|
||||
// it may return the same sealedKey as rotatedKey.
|
||||
//
|
||||
// The context must be same context as the one provided while
|
||||
// generating the plaintext key / sealedKey.
|
||||
func (kes *kesService) UpdateKey(keyID string, sealedKey []byte, ctx Context) ([]byte, error) {
|
||||
_, err := kes.UnsealKey(keyID, sealedKey, ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Currently a kes server does not support key rotation (of the same key)
|
||||
// Therefore, we simply return the same sealedKey.
|
||||
return sealedKey, nil
|
||||
}
|
||||
|
||||
// kesClient implements the bare minimum functionality needed for
|
||||
// MinIO to talk to a KES server. In particular, it implements
|
||||
// • CreateKey (API: /v1/key/create/)
|
||||
// • GenerateDataKey (API: /v1/key/generate/)
|
||||
// • DecryptDataKey (API: /v1/key/decrypt/)
|
||||
// GenerateDataKey (API: /v1/key/generate/) and
|
||||
// DecryptDataKey (API: /v1/key/decrypt/).
|
||||
type kesClient struct {
|
||||
addr string
|
||||
httpClient http.Client
|
||||
|
||||
+26
-18
@@ -72,14 +72,8 @@ func (c Context) WriteTo(w io.Writer) (n int64, err error) {
|
||||
// data key generation and unsealing of KMS-generated
|
||||
// data keys.
|
||||
type KMS interface {
|
||||
// DefaultKeyID returns the default master key ID. It should be
|
||||
// used for SSE-S3 and whenever a S3 client requests SSE-KMS but
|
||||
// does not specify an explicit SSE-KMS key ID.
|
||||
DefaultKeyID() string
|
||||
|
||||
// CreateKey creates a new master key with the given key ID
|
||||
// at the KMS.
|
||||
CreateKey(keyID string) error
|
||||
// KeyID - returns configured KMS key id.
|
||||
KeyID() string
|
||||
|
||||
// GenerateKey generates a new random data key using
|
||||
// the master key referenced by the keyID. It returns
|
||||
@@ -96,9 +90,21 @@ type KMS interface {
|
||||
// match the context used to generate the sealed key.
|
||||
UnsealKey(keyID string, sealedKey []byte, context Context) (key [32]byte, err error)
|
||||
|
||||
// Info returns descriptive information about the KMS,
|
||||
// like the default key ID and authentication method.
|
||||
Info() KMSInfo
|
||||
// UpdateKey re-wraps the sealedKey if the master key, referenced by
|
||||
// `keyID`, has changed in the meantime. This usually happens when the
|
||||
// KMS operator performs a key-rotation operation of the master key.
|
||||
// UpdateKey fails if the provided sealedKey cannot be decrypted using
|
||||
// the master key referenced by keyID.
|
||||
//
|
||||
// UpdateKey makes no guarantees whatsoever about whether the returned
|
||||
// rotatedKey is actually different from the sealedKey. If nothing has
|
||||
// changed at the KMS or if the KMS does not support updating generated
|
||||
// keys this method may behave like a NOP and just return the sealedKey
|
||||
// itself.
|
||||
UpdateKey(keyID string, sealedKey []byte, context Context) (rotatedKey []byte, err error)
|
||||
|
||||
// Returns KMSInfo
|
||||
Info() (kmsInfo KMSInfo)
|
||||
}
|
||||
|
||||
type masterKeyKMS struct {
|
||||
@@ -106,8 +112,7 @@ type masterKeyKMS struct {
|
||||
masterKey [32]byte
|
||||
}
|
||||
|
||||
// KMSInfo contains some describing information about
|
||||
// the KMS.
|
||||
// KMSInfo stores the details of KMS
|
||||
type KMSInfo struct {
|
||||
Endpoint string
|
||||
Name string
|
||||
@@ -120,14 +125,10 @@ type KMSInfo struct {
|
||||
// to the generated keys.
|
||||
func NewMasterKey(keyID string, key [32]byte) KMS { return &masterKeyKMS{keyID: keyID, masterKey: key} }
|
||||
|
||||
func (kms *masterKeyKMS) DefaultKeyID() string {
|
||||
func (kms *masterKeyKMS) KeyID() string {
|
||||
return kms.keyID
|
||||
}
|
||||
|
||||
func (kms *masterKeyKMS) CreateKey(keyID string) error {
|
||||
return errors.New("crypto: creating keys is not supported by a static master key")
|
||||
}
|
||||
|
||||
func (kms *masterKeyKMS) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
|
||||
if _, err = io.ReadFull(rand.Reader, key[:]); err != nil {
|
||||
logger.CriticalIf(context.Background(), errOutOfEntropy)
|
||||
@@ -165,6 +166,13 @@ func (kms *masterKeyKMS) UnsealKey(keyID string, sealedKey []byte, ctx Context)
|
||||
return key, nil
|
||||
}
|
||||
|
||||
func (kms *masterKeyKMS) UpdateKey(keyID string, sealedKey []byte, ctx Context) ([]byte, error) {
|
||||
if _, err := kms.UnsealKey(keyID, sealedKey, ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return sealedKey, nil // The master key cannot update data keys -> Do nothing.
|
||||
}
|
||||
|
||||
func (kms *masterKeyKMS) deriveKey(keyID string, context Context) (key [32]byte) {
|
||||
if context == nil {
|
||||
context = Context{}
|
||||
|
||||
@@ -57,6 +57,15 @@ func TestMasterKeyKMS(t *testing.T) {
|
||||
if !test.ShouldFail && !bytes.Equal(key[:], unsealedKey[:]) {
|
||||
t.Errorf("Test %d: The generated and unsealed key differ", i)
|
||||
}
|
||||
|
||||
rotatedKey, err := kms.UpdateKey(test.UnsealKeyID, sealedKey, test.UnsealContext)
|
||||
if err == nil && test.ShouldFail {
|
||||
t.Errorf("Test %d: KMS updated the generated key successfully but should have failed", i)
|
||||
}
|
||||
if !test.ShouldFail && !bytes.Equal(rotatedKey, sealedKey[:]) {
|
||||
t.Errorf("Test %d: The updated and sealed key differ", i)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -51,8 +51,8 @@ func TestParseMasterKey(t *testing.T) {
|
||||
if !tt.success && err == nil {
|
||||
t.Error("Unexpected failure")
|
||||
}
|
||||
if err == nil && kms.DefaultKeyID() != tt.expectedKeyID {
|
||||
t.Errorf("Expected keyID %s, got %s", tt.expectedKeyID, kms.DefaultKeyID())
|
||||
if err == nil && kms.KeyID() != tt.expectedKeyID {
|
||||
t.Errorf("Expected keyID %s, got %s", tt.expectedKeyID, kms.KeyID())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+5
-20
@@ -17,7 +17,6 @@ package crypto
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -191,34 +190,20 @@ func (v *vaultService) authenticate() (err error) {
|
||||
return
|
||||
}
|
||||
|
||||
// DefaultKeyID returns the default key ID that should be
|
||||
// used for SSE-S3 or SSE-KMS when the S3 client does not
|
||||
// provide an explicit key ID.
|
||||
func (v *vaultService) DefaultKeyID() string {
|
||||
// KeyID - vault configured keyID
|
||||
func (v *vaultService) KeyID() string {
|
||||
return v.config.Key.Name
|
||||
}
|
||||
|
||||
// Info returns some information about the Vault,
|
||||
// configuration - like the endpoint or authentication
|
||||
// method.
|
||||
func (v *vaultService) Info() KMSInfo {
|
||||
// Returns - vault info
|
||||
func (v *vaultService) Info() (kmsInfo KMSInfo) {
|
||||
return KMSInfo{
|
||||
Endpoint: v.config.Endpoint,
|
||||
Name: v.DefaultKeyID(),
|
||||
Name: v.config.Key.Name,
|
||||
AuthType: v.config.Auth.Type,
|
||||
}
|
||||
}
|
||||
|
||||
// CreateKey is a stub that exists such that the Vault
|
||||
// client implements the KMS interface. It always returns
|
||||
// a not-implemented error.
|
||||
//
|
||||
// Creating keys requires a KES instance between MinIO and Vault.
|
||||
func (v *vaultService) CreateKey(keyID string) error {
|
||||
// Creating new keys requires KES.
|
||||
return errors.New("crypto: creating keys is not supported by Vault")
|
||||
}
|
||||
|
||||
// GenerateKey returns a new plaintext key, generated by the KMS,
|
||||
// and a sealed version of this plaintext key encrypted using the
|
||||
// named key referenced by keyID. It also binds the generated key
|
||||
|
||||
+17
-56
@@ -28,6 +28,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/cmd/config"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/bucket/lifecycle"
|
||||
"github.com/minio/minio/pkg/color"
|
||||
@@ -509,9 +510,9 @@ func (i *crawlItem) transformMetaDir() {
|
||||
|
||||
// actionMeta contains information used to apply actions.
|
||||
type actionMeta struct {
|
||||
oi ObjectInfo
|
||||
trustOI bool // Set true if oi can be trusted and has been read with quorum.
|
||||
numVersions int // The number of versions of this object
|
||||
oi ObjectInfo
|
||||
trustOI bool // Set true if oi can be trusted and has been read with quorum.
|
||||
meta map[string]string
|
||||
}
|
||||
|
||||
// applyActions will apply lifecycle checks on to a scanned item.
|
||||
@@ -527,22 +528,12 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
|
||||
return size
|
||||
}
|
||||
|
||||
versionID := meta.oi.VersionID
|
||||
action := i.lifeCycle.ComputeAction(
|
||||
lifecycle.ObjectOpts{
|
||||
Name: i.objectPath(),
|
||||
UserTags: meta.oi.UserTags,
|
||||
ModTime: meta.oi.ModTime,
|
||||
VersionID: meta.oi.VersionID,
|
||||
DeleteMarker: meta.oi.DeleteMarker,
|
||||
IsLatest: meta.oi.IsLatest,
|
||||
NumVersions: meta.numVersions,
|
||||
})
|
||||
action := i.lifeCycle.ComputeAction(i.objectPath(), meta.meta[xhttp.AmzObjectTagging], meta.oi.ModTime)
|
||||
if i.debug {
|
||||
logger.Info(color.Green("applyActions:")+" lifecycle: %q, Initial scan: %v", i.objectPath(), action)
|
||||
}
|
||||
switch action {
|
||||
case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
|
||||
case lifecycle.DeleteAction:
|
||||
default:
|
||||
// No action.
|
||||
return size
|
||||
@@ -551,60 +542,28 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
|
||||
// These (expensive) operations should only run on items we are likely to delete.
|
||||
// Load to ensure that we have the correct version and not an unsynced version.
|
||||
if !meta.trustOI {
|
||||
obj, err := o.GetObjectInfo(ctx, i.bucket, i.objectPath(), ObjectOptions{
|
||||
VersionID: versionID,
|
||||
})
|
||||
obj, err := o.GetObjectInfo(ctx, i.bucket, i.objectPath(), ObjectOptions{})
|
||||
if err != nil {
|
||||
switch err.(type) {
|
||||
case MethodNotAllowed: // This happens usually for a delete marker
|
||||
if !obj.DeleteMarker { // if this is not a delete marker log and return
|
||||
// Do nothing - heal in the future.
|
||||
logger.LogIf(ctx, err)
|
||||
return size
|
||||
}
|
||||
case ObjectNotFound:
|
||||
// object not found return 0
|
||||
return 0
|
||||
default:
|
||||
// All other errors proceed.
|
||||
logger.LogIf(ctx, err)
|
||||
return size
|
||||
}
|
||||
// Do nothing - heal in the future.
|
||||
logger.LogIf(ctx, err)
|
||||
return size
|
||||
}
|
||||
size = obj.Size
|
||||
|
||||
// Recalculate action.
|
||||
action = i.lifeCycle.ComputeAction(
|
||||
lifecycle.ObjectOpts{
|
||||
Name: i.objectPath(),
|
||||
UserTags: obj.UserTags,
|
||||
ModTime: obj.ModTime,
|
||||
VersionID: obj.VersionID,
|
||||
DeleteMarker: obj.DeleteMarker,
|
||||
IsLatest: obj.IsLatest,
|
||||
NumVersions: meta.numVersions,
|
||||
})
|
||||
action = i.lifeCycle.ComputeAction(i.objectPath(), obj.UserTags, obj.ModTime)
|
||||
if i.debug {
|
||||
logger.Info(color.Green("applyActions:")+" lifecycle: Secondary scan: %v", action)
|
||||
}
|
||||
versionID = obj.VersionID
|
||||
switch action {
|
||||
case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
|
||||
case lifecycle.DeleteAction:
|
||||
default:
|
||||
// No action.
|
||||
return size
|
||||
}
|
||||
}
|
||||
|
||||
opts := ObjectOptions{}
|
||||
switch action {
|
||||
case lifecycle.DeleteVersionAction:
|
||||
opts.VersionID = versionID
|
||||
case lifecycle.DeleteAction:
|
||||
opts.Versioned = globalBucketVersioningSys.Enabled(i.bucket)
|
||||
}
|
||||
|
||||
obj, err := o.DeleteObject(ctx, i.bucket, i.objectPath(), opts)
|
||||
err = o.DeleteObject(ctx, i.bucket, i.objectPath())
|
||||
if err != nil {
|
||||
// Assume it is still there.
|
||||
logger.LogIf(ctx, err)
|
||||
@@ -615,8 +574,10 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
|
||||
sendEvent(eventArgs{
|
||||
EventName: event.ObjectRemovedDelete,
|
||||
BucketName: i.bucket,
|
||||
Object: obj,
|
||||
Host: "Internal: [ILM-EXPIRY]",
|
||||
Object: ObjectInfo{
|
||||
Name: i.objectPath(),
|
||||
},
|
||||
Host: "Internal: [ILM-EXPIRY]",
|
||||
})
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -120,8 +120,8 @@ func (b bloomFilter) containsDir(in string) bool {
|
||||
}
|
||||
|
||||
// bytes returns the bloom filter serialized as a byte slice.
|
||||
func (b *bloomFilter) bytes() []byte {
|
||||
if b == nil || b.BloomFilter == nil {
|
||||
func (b bloomFilter) bytes() []byte {
|
||||
if b.BloomFilter == nil {
|
||||
return nil
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
|
||||
+40
-69
@@ -33,7 +33,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/djherbis/atime"
|
||||
"github.com/minio/minio/cmd/config/cache"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
@@ -61,7 +60,7 @@ type CacheChecksumInfoV1 struct {
|
||||
// Represents the cache metadata struct
|
||||
type cacheMeta struct {
|
||||
Version string `json:"version"`
|
||||
Stat StatInfo `json:"stat"` // Stat of the current object `cache.json`.
|
||||
Stat statInfo `json:"stat"` // Stat of the current object `cache.json`.
|
||||
|
||||
// checksums of blocks on disk.
|
||||
Checksum CacheChecksumInfoV1 `json:"checksum,omitempty"`
|
||||
@@ -126,19 +125,16 @@ func (m *cacheMeta) ToObjectInfo(bucket, object string) (o ObjectInfo) {
|
||||
|
||||
// represents disk cache struct
|
||||
type diskCache struct {
|
||||
gcCounter uint64 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
|
||||
// is set to 0 if drive is offline
|
||||
online uint32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
|
||||
purgeRunning int32
|
||||
online uint32
|
||||
|
||||
triggerGC chan struct{}
|
||||
dir string // caching directory
|
||||
stats CacheDiskStats // disk cache stats for prometheus
|
||||
quotaPct int // max usage in %
|
||||
dir string // caching directory
|
||||
quotaPct int // max usage in %
|
||||
pool sync.Pool
|
||||
after int // minimum accesses before an object is cached.
|
||||
lowWatermark int
|
||||
highWatermark int
|
||||
enableRange bool
|
||||
// nsMutex namespace lock
|
||||
nsMutex *nsLockMap
|
||||
// Object functions pointing to the corresponding functions of backend implementation.
|
||||
@@ -146,24 +142,16 @@ type diskCache struct {
|
||||
}
|
||||
|
||||
// Inits the disk cache dir if it is not initialized already.
|
||||
func newDiskCache(ctx context.Context, dir string, config cache.Config) (*diskCache, error) {
|
||||
quotaPct := config.MaxUse
|
||||
if quotaPct == 0 {
|
||||
quotaPct = config.Quota
|
||||
}
|
||||
|
||||
func newDiskCache(dir string, quotaPct, after, lowWatermark, highWatermark int) (*diskCache, error) {
|
||||
if err := os.MkdirAll(dir, 0777); err != nil {
|
||||
return nil, fmt.Errorf("Unable to initialize '%s' dir, %w", dir, err)
|
||||
}
|
||||
cache := diskCache{
|
||||
dir: dir,
|
||||
triggerGC: make(chan struct{}),
|
||||
stats: CacheDiskStats{Dir: dir},
|
||||
quotaPct: quotaPct,
|
||||
after: config.After,
|
||||
lowWatermark: config.WatermarkLow,
|
||||
highWatermark: config.WatermarkHigh,
|
||||
enableRange: config.Range,
|
||||
after: after,
|
||||
lowWatermark: lowWatermark,
|
||||
highWatermark: highWatermark,
|
||||
online: 1,
|
||||
pool: sync.Pool{
|
||||
New: func() interface{} {
|
||||
@@ -173,8 +161,6 @@ func newDiskCache(ctx context.Context, dir string, config cache.Config) (*diskCa
|
||||
},
|
||||
nsMutex: newNSLock(false),
|
||||
}
|
||||
go cache.purgeWait(ctx)
|
||||
cache.diskUsageHigh() // update if cache usage is already high.
|
||||
cache.NewNSLockFn = func(ctx context.Context, cachePath string) RWLocker {
|
||||
return cache.nsMutex.NewNSLock(ctx, nil, cachePath, "")
|
||||
}
|
||||
@@ -195,12 +181,7 @@ func (c *diskCache) diskUsageLow() bool {
|
||||
return false
|
||||
}
|
||||
usedPercent := (di.Total - di.Free) * 100 / di.Total
|
||||
low := int(usedPercent) < gcStopPct
|
||||
atomic.StoreUint64(&c.stats.UsagePercent, usedPercent)
|
||||
if low {
|
||||
atomic.StoreInt32(&c.stats.UsageState, 0)
|
||||
}
|
||||
return low
|
||||
return int(usedPercent) < gcStopPct
|
||||
}
|
||||
|
||||
// Returns if the disk usage reaches high water mark w.r.t the configured cache quota.
|
||||
@@ -215,12 +196,7 @@ func (c *diskCache) diskUsageHigh() bool {
|
||||
return false
|
||||
}
|
||||
usedPercent := (di.Total - di.Free) * 100 / di.Total
|
||||
high := int(usedPercent) >= gcTriggerPct
|
||||
atomic.StoreUint64(&c.stats.UsagePercent, usedPercent)
|
||||
if high {
|
||||
atomic.StoreInt32(&c.stats.UsageState, 1)
|
||||
}
|
||||
return high
|
||||
return int(usedPercent) >= gcTriggerPct
|
||||
}
|
||||
|
||||
// Returns if size space can be allocated without exceeding
|
||||
@@ -254,36 +230,24 @@ var (
|
||||
errDoneForNow = errors.New("done for now")
|
||||
)
|
||||
|
||||
func (c *diskCache) purgeWait(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-c.triggerGC: // wait here until someone triggers.
|
||||
c.purge(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Purge cache entries that were not accessed.
|
||||
func (c *diskCache) purge(ctx context.Context) {
|
||||
if atomic.LoadInt32(&c.purgeRunning) == 1 || c.diskUsageLow() {
|
||||
if c.diskUsageLow() {
|
||||
return
|
||||
}
|
||||
|
||||
toFree := c.toClear()
|
||||
if toFree == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
atomic.StoreInt32(&c.purgeRunning, 1) // do not run concurrent purge()
|
||||
defer atomic.StoreInt32(&c.purgeRunning, 0)
|
||||
|
||||
// expiry for cleaning up old cache.json files that
|
||||
// need to be cleaned up.
|
||||
expiry := UTCNow().Add(-cacheExpiryDays)
|
||||
// defaulting max hits count to 100
|
||||
// ignore error we know what value we are passing.
|
||||
scorer, _ := newFileScorer(toFree, time.Now().Unix(), 100)
|
||||
scorer, err := newFileScorer(toFree, time.Now().Unix(), 100)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return
|
||||
}
|
||||
|
||||
// this function returns FileInfo for cached range files and cache data file.
|
||||
fiStatFn := func(ranges map[string]string, dataFile, pathPrefix string) map[string]os.FileInfo {
|
||||
@@ -362,20 +326,27 @@ func (c *diskCache) purge(ctx context.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
scorer.purgeFunc(func(qfile queuedFile) {
|
||||
fileName := qfile.name
|
||||
removeAll(fileName)
|
||||
slashIdx := strings.LastIndex(fileName, SlashSeparator)
|
||||
if slashIdx >= 0 {
|
||||
fileNamePrefix := fileName[0:slashIdx]
|
||||
fname := fileName[slashIdx+1:]
|
||||
if fname == cacheDataFile {
|
||||
removeAll(fileNamePrefix)
|
||||
}
|
||||
for _, path := range scorer.fileNames() {
|
||||
removeAll(path)
|
||||
slashIdx := strings.LastIndex(path, SlashSeparator)
|
||||
pathPrefix := path[0:slashIdx]
|
||||
fname := path[slashIdx+1:]
|
||||
if fname == cacheDataFile {
|
||||
removeAll(pathPrefix)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
scorer.reset()
|
||||
func (c *diskCache) incGCCounter() {
|
||||
atomic.AddUint64(&c.gcCounter, 1)
|
||||
}
|
||||
|
||||
func (c *diskCache) resetGCCounter() {
|
||||
atomic.StoreUint64(&c.gcCounter, 0)
|
||||
}
|
||||
|
||||
func (c *diskCache) gcCount() uint64 {
|
||||
return atomic.LoadUint64(&c.gcCounter)
|
||||
}
|
||||
|
||||
// sets cache drive status
|
||||
@@ -582,7 +553,7 @@ func (c *diskCache) bitrotWriteToCache(cachePath, fileName string, reader io.Rea
|
||||
}
|
||||
f, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return 0, osErrToFileErr(err)
|
||||
return 0, osErrToFSFileErr(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
@@ -640,14 +611,14 @@ func newCacheEncryptMetadata(bucket, object string, metadata map[string]string)
|
||||
if globalCacheKMS == nil {
|
||||
return nil, errKMSNotConfigured
|
||||
}
|
||||
key, encKey, err := globalCacheKMS.GenerateKey(globalCacheKMS.DefaultKeyID(), crypto.Context{bucket: pathJoin(bucket, object)})
|
||||
key, encKey, err := globalCacheKMS.GenerateKey(globalCacheKMS.KeyID(), crypto.Context{bucket: pathJoin(bucket, object)})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
objectKey := crypto.GenerateKey(key, rand.Reader)
|
||||
sealedKey = objectKey.Seal(key, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
|
||||
crypto.S3.CreateMetadata(metadata, globalCacheKMS.DefaultKeyID(), encKey, sealedKey)
|
||||
crypto.S3.CreateMetadata(metadata, globalCacheKMS.KeyID(), encKey, sealedKey)
|
||||
|
||||
if etag, ok := metadata["etag"]; ok {
|
||||
metadata["etag"] = hex.EncodeToString(objectKey.SealETag([]byte(etag)))
|
||||
@@ -659,7 +630,7 @@ func newCacheEncryptMetadata(bucket, object string, metadata map[string]string)
|
||||
// Caches the object to disk
|
||||
func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions, incHitsOnly bool) error {
|
||||
if c.diskUsageHigh() {
|
||||
c.triggerGC <- struct{}{}
|
||||
c.incGCCounter()
|
||||
io.Copy(ioutil.Discard, data)
|
||||
return errDiskFull
|
||||
}
|
||||
|
||||
+5
-18
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2019, 2020 MinIO, Inc.
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -16,27 +16,14 @@
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// CacheDiskStats represents cache disk statistics
|
||||
// such as current disk usage and available.
|
||||
type CacheDiskStats struct {
|
||||
// indicates if usage is high or low, if high value is '1', if low its '0'
|
||||
UsageState int32
|
||||
// indicates the current usage percentage of this cache disk
|
||||
UsagePercent uint64
|
||||
Dir string
|
||||
}
|
||||
import "sync/atomic"
|
||||
|
||||
// CacheStats - represents bytes served from cache,
|
||||
// cache hits and cache misses.
|
||||
type CacheStats struct {
|
||||
BytesServed uint64
|
||||
Hits uint64
|
||||
Misses uint64
|
||||
GetDiskStats func() []CacheDiskStats
|
||||
BytesServed uint64
|
||||
Hits uint64
|
||||
Misses uint64
|
||||
}
|
||||
|
||||
// Increase total bytes served from cache
|
||||
|
||||
+10
-53
@@ -187,12 +187,12 @@ func readCacheFileStream(filePath string, offset, length int64) (io.ReadCloser,
|
||||
|
||||
fr, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return nil, osErrToFileErr(err)
|
||||
return nil, osErrToFSFileErr(err)
|
||||
}
|
||||
// Stat to get the size of the file at path.
|
||||
st, err := fr.Stat()
|
||||
if err != nil {
|
||||
err = osErrToFileErr(err)
|
||||
err = osErrToFSFileErr(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -298,10 +298,9 @@ type fileScorer struct {
|
||||
}
|
||||
|
||||
type queuedFile struct {
|
||||
name string
|
||||
versionID string
|
||||
size uint64
|
||||
score float64
|
||||
name string
|
||||
size uint64
|
||||
score float64
|
||||
}
|
||||
|
||||
// newFileScorer allows to collect files to save a specific number of bytes.
|
||||
@@ -322,33 +321,15 @@ func newFileScorer(saveBytes uint64, now int64, maxHits int) (*fileScorer, error
|
||||
return &f, nil
|
||||
}
|
||||
|
||||
func (f *fileScorer) addFile(name string, accTime time.Time, size int64, hits int) {
|
||||
f.addFileWithObjInfo(ObjectInfo{
|
||||
Name: name,
|
||||
AccTime: accTime,
|
||||
Size: size,
|
||||
}, hits)
|
||||
}
|
||||
|
||||
func (f *fileScorer) addFileWithObjInfo(objInfo ObjectInfo, hits int) {
|
||||
func (f *fileScorer) addFile(name string, lastAccess time.Time, size int64, hits int) {
|
||||
// Calculate how much we want to delete this object.
|
||||
file := queuedFile{
|
||||
name: objInfo.Name,
|
||||
versionID: objInfo.VersionID,
|
||||
size: uint64(objInfo.Size),
|
||||
name: name,
|
||||
size: uint64(size),
|
||||
}
|
||||
|
||||
var score float64
|
||||
if objInfo.ModTime.IsZero() {
|
||||
// Mod time is not available with disk cache use atime.
|
||||
score = float64(f.now - objInfo.AccTime.Unix())
|
||||
} else {
|
||||
// if not used mod time when mod time is available.
|
||||
score = float64(f.now - objInfo.ModTime.Unix())
|
||||
}
|
||||
|
||||
score := float64(f.now - lastAccess.Unix())
|
||||
// Size as fraction of how much we want to save, 0->1.
|
||||
szWeight := math.Max(0, (math.Min(1, float64(file.size)*f.sizeMult)))
|
||||
szWeight := math.Max(0, (math.Min(1, float64(size)*f.sizeMult)))
|
||||
// 0 at f.maxHits, 1 at 0.
|
||||
hitsWeight := (1.0 - math.Max(0, math.Min(1.0, float64(hits)/float64(f.maxHits))))
|
||||
file.score = score * (1 + 0.25*szWeight + 0.25*hitsWeight)
|
||||
@@ -423,30 +404,6 @@ func (f *fileScorer) trimQueue() {
|
||||
}
|
||||
}
|
||||
|
||||
// fileObjInfos returns all queued file object infos
|
||||
func (f *fileScorer) fileObjInfos() []ObjectInfo {
|
||||
res := make([]ObjectInfo, 0, f.queue.Len())
|
||||
e := f.queue.Front()
|
||||
for e != nil {
|
||||
qfile := e.Value.(queuedFile)
|
||||
res = append(res, ObjectInfo{
|
||||
Name: qfile.name,
|
||||
Size: int64(qfile.size),
|
||||
VersionID: qfile.versionID,
|
||||
})
|
||||
e = e.Next()
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (f *fileScorer) purgeFunc(p func(qfile queuedFile)) {
|
||||
e := f.queue.Front()
|
||||
for e != nil {
|
||||
p(e.Value.(queuedFile))
|
||||
e = e.Next()
|
||||
}
|
||||
}
|
||||
|
||||
// fileNames returns all queued file names.
|
||||
func (f *fileScorer) fileNames() []string {
|
||||
res := make([]string, 0, f.queue.Len())
|
||||
|
||||
+38
-53
@@ -24,7 +24,6 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/cmd/config/cache"
|
||||
@@ -52,8 +51,8 @@ type CacheObjectLayer interface {
|
||||
// Object operations.
|
||||
GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
|
||||
GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
|
||||
DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error)
|
||||
DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error)
|
||||
DeleteObject(ctx context.Context, bucket, object string) error
|
||||
DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error)
|
||||
PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
|
||||
CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error)
|
||||
// Storage operations.
|
||||
@@ -79,7 +78,8 @@ type cacheObjects struct {
|
||||
|
||||
GetObjectNInfoFn func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
|
||||
GetObjectInfoFn func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
|
||||
DeleteObjectFn func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
|
||||
DeleteObjectFn func(ctx context.Context, bucket, object string) error
|
||||
DeleteObjectsFn func(ctx context.Context, bucket string, objects []string) ([]error, error)
|
||||
PutObjectFn func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
|
||||
CopyObjectFn func(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error)
|
||||
}
|
||||
@@ -120,8 +120,8 @@ func (c *cacheObjects) updateMetadataIfChanged(ctx context.Context, dcache *disk
|
||||
}
|
||||
|
||||
// DeleteObject clears cache entry if backend delete operation succeeds
|
||||
func (c *cacheObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
|
||||
if objInfo, err = c.DeleteObjectFn(ctx, bucket, object, opts); err != nil {
|
||||
func (c *cacheObjects) DeleteObject(ctx context.Context, bucket, object string) (err error) {
|
||||
if err = c.DeleteObjectFn(ctx, bucket, object); err != nil {
|
||||
return
|
||||
}
|
||||
if c.isCacheExclude(bucket, object) || c.skipCache() {
|
||||
@@ -130,38 +130,19 @@ func (c *cacheObjects) DeleteObject(ctx context.Context, bucket, object string,
|
||||
|
||||
dcache, cerr := c.getCacheLoc(bucket, object)
|
||||
if cerr != nil {
|
||||
return objInfo, cerr
|
||||
return
|
||||
}
|
||||
dcache.Delete(ctx, bucket, object)
|
||||
return
|
||||
}
|
||||
|
||||
// DeleteObjects batch deletes objects in slice, and clears any cached entries
|
||||
func (c *cacheObjects) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
|
||||
func (c *cacheObjects) DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error) {
|
||||
errs := make([]error, len(objects))
|
||||
objInfos := make([]ObjectInfo, len(objects))
|
||||
for idx, object := range objects {
|
||||
opts.VersionID = object.VersionID
|
||||
objInfos[idx], errs[idx] = c.DeleteObject(ctx, bucket, object.ObjectName, opts)
|
||||
errs[idx] = c.DeleteObject(ctx, bucket, object)
|
||||
}
|
||||
deletedObjects := make([]DeletedObject, len(objInfos))
|
||||
for idx := range errs {
|
||||
if errs[idx] != nil {
|
||||
continue
|
||||
}
|
||||
if objInfos[idx].DeleteMarker {
|
||||
deletedObjects[idx] = DeletedObject{
|
||||
DeleteMarker: objInfos[idx].DeleteMarker,
|
||||
DeleteMarkerVersionID: objInfos[idx].VersionID,
|
||||
}
|
||||
continue
|
||||
}
|
||||
deletedObjects[idx] = DeletedObject{
|
||||
ObjectName: objInfos[idx].Name,
|
||||
VersionID: objInfos[idx].VersionID,
|
||||
}
|
||||
}
|
||||
return deletedObjects, errs
|
||||
return errs, nil
|
||||
}
|
||||
|
||||
// construct a metadata k-v map
|
||||
@@ -283,11 +264,10 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
|
||||
|
||||
// Reaching here implies cache miss
|
||||
c.cacheStats.incMiss()
|
||||
|
||||
// Since we got here, we are serving the request from backend,
|
||||
// and also adding the object to the cache.
|
||||
if dcache.diskUsageHigh() {
|
||||
dcache.triggerGC <- struct{}{} // this is non-blocking
|
||||
dcache.incGCCounter()
|
||||
}
|
||||
|
||||
bkReader, bkErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
|
||||
@@ -312,10 +292,6 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
|
||||
|
||||
if rs != nil {
|
||||
go func() {
|
||||
// if range caching is disabled, download entire object.
|
||||
if !dcache.enableRange {
|
||||
rs = nil
|
||||
}
|
||||
// fill cache in the background for range GET requests
|
||||
bReader, bErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
|
||||
if bErr != nil {
|
||||
@@ -325,9 +301,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
|
||||
oi, _, _, err := dcache.statRange(ctx, bucket, object, rs)
|
||||
// avoid cache overwrite if another background routine filled cache
|
||||
if err != nil || oi.ETag != bReader.ObjInfo.ETag {
|
||||
// use a new context to avoid locker prematurely timing out operation when the GetObjectNInfo returns.
|
||||
dcache.Put(context.Background(), bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false)
|
||||
return
|
||||
dcache.Put(ctx, bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false)
|
||||
}
|
||||
}()
|
||||
return bkReader, bkErr
|
||||
@@ -548,7 +522,11 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
|
||||
return nil, false, errors.New("Atime support required for disk caching")
|
||||
}
|
||||
|
||||
cache, err := newDiskCache(ctx, dir, config)
|
||||
quota := config.MaxUse
|
||||
if quota == 0 {
|
||||
quota = config.Quota
|
||||
}
|
||||
cache, err := newDiskCache(dir, quota, config.After, config.WatermarkLow, config.WatermarkHigh)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
@@ -671,8 +649,15 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
|
||||
GetObjectNInfoFn: func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
|
||||
return newObjectLayerFn().GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
|
||||
},
|
||||
DeleteObjectFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
|
||||
return newObjectLayerFn().DeleteObject(ctx, bucket, object, opts)
|
||||
DeleteObjectFn: func(ctx context.Context, bucket, object string) error {
|
||||
return newObjectLayerFn().DeleteObject(ctx, bucket, object)
|
||||
},
|
||||
DeleteObjectsFn: func(ctx context.Context, bucket string, objects []string) ([]error, error) {
|
||||
errs := make([]error, len(objects))
|
||||
for idx, object := range objects {
|
||||
errs[idx] = newObjectLayerFn().DeleteObject(ctx, bucket, object)
|
||||
}
|
||||
return errs, nil
|
||||
},
|
||||
PutObjectFn: func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
|
||||
return newObjectLayerFn().PutObject(ctx, bucket, object, data, opts)
|
||||
@@ -681,17 +666,7 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
|
||||
return newObjectLayerFn().CopyObject(ctx, srcBucket, srcObject, destBucket, destObject, srcInfo, srcOpts, dstOpts)
|
||||
},
|
||||
}
|
||||
c.cacheStats.GetDiskStats = func() []CacheDiskStats {
|
||||
cacheDiskStats := make([]CacheDiskStats, len(c.cache))
|
||||
for i := range c.cache {
|
||||
cacheDiskStats[i] = CacheDiskStats{
|
||||
Dir: c.cache[i].stats.Dir,
|
||||
}
|
||||
atomic.StoreInt32(&cacheDiskStats[i].UsageState, atomic.LoadInt32(&c.cache[i].stats.UsageState))
|
||||
atomic.StoreUint64(&cacheDiskStats[i].UsagePercent, atomic.LoadUint64(&c.cache[i].stats.UsagePercent))
|
||||
}
|
||||
return cacheDiskStats
|
||||
}
|
||||
|
||||
if migrateSw {
|
||||
go c.migrateCacheFromV1toV2(ctx)
|
||||
}
|
||||
@@ -711,9 +686,19 @@ func (c *cacheObjects) gc(ctx context.Context) {
|
||||
if c.migrating {
|
||||
continue
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for _, dcache := range c.cache {
|
||||
dcache.triggerGC <- struct{}{}
|
||||
if dcache.gcCount() == 0 {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(d *diskCache) {
|
||||
defer wg.Done()
|
||||
d.resetGCCounter()
|
||||
d.purge(ctx)
|
||||
}(dcache)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+138
-6
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2017-2020 MinIO, Inc.
|
||||
* MinIO Cloud Storage, (C) 2017, 2018 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -18,6 +18,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/rand"
|
||||
"crypto/subtle"
|
||||
@@ -30,6 +31,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go/v6/pkg/encrypt"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
sha256 "github.com/minio/sha256-simd"
|
||||
@@ -80,7 +82,7 @@ func isEncryptedMultipart(objInfo ObjectInfo) bool {
|
||||
}
|
||||
}
|
||||
// Further check if this object is uploaded using multipart mechanism
|
||||
// by the user and it is not about Erasure internally splitting the
|
||||
// by the user and it is not about XL internally splitting the
|
||||
// object into parts in PutObject()
|
||||
return !(objInfo.backendType == BackendErasure && len(objInfo.ETag) == 32)
|
||||
}
|
||||
@@ -157,12 +159,12 @@ func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map
|
||||
return err
|
||||
}
|
||||
|
||||
newKey, encKey, err := GlobalKMS.GenerateKey(GlobalKMS.DefaultKeyID(), crypto.Context{bucket: path.Join(bucket, object)})
|
||||
newKey, encKey, err := GlobalKMS.GenerateKey(GlobalKMS.KeyID(), crypto.Context{bucket: path.Join(bucket, object)})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sealedKey = objectKey.Seal(newKey, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
|
||||
crypto.S3.CreateMetadata(metadata, GlobalKMS.DefaultKeyID(), encKey, sealedKey)
|
||||
crypto.S3.CreateMetadata(metadata, GlobalKMS.KeyID(), encKey, sealedKey)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -173,14 +175,14 @@ func newEncryptMetadata(key []byte, bucket, object string, metadata map[string]s
|
||||
if GlobalKMS == nil {
|
||||
return crypto.ObjectKey{}, errKMSNotConfigured
|
||||
}
|
||||
key, encKey, err := GlobalKMS.GenerateKey(GlobalKMS.DefaultKeyID(), crypto.Context{bucket: path.Join(bucket, object)})
|
||||
key, encKey, err := GlobalKMS.GenerateKey(GlobalKMS.KeyID(), crypto.Context{bucket: path.Join(bucket, object)})
|
||||
if err != nil {
|
||||
return crypto.ObjectKey{}, err
|
||||
}
|
||||
|
||||
objectKey := crypto.GenerateKey(key, rand.Reader)
|
||||
sealedKey = objectKey.Seal(key, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
|
||||
crypto.S3.CreateMetadata(metadata, GlobalKMS.DefaultKeyID(), encKey, sealedKey)
|
||||
crypto.S3.CreateMetadata(metadata, GlobalKMS.KeyID(), encKey, sealedKey)
|
||||
return objectKey, nil
|
||||
}
|
||||
var extKey [32]byte
|
||||
@@ -851,3 +853,133 @@ func deriveClientKey(clientKey [32]byte, bucket, object string) [32]byte {
|
||||
mac.Sum(key[:0])
|
||||
return key
|
||||
}
|
||||
|
||||
// set encryption options for pass through to backend in the case of gateway and UserDefined metadata
|
||||
func getDefaultOpts(header http.Header, copySource bool, metadata map[string]string) (opts ObjectOptions, err error) {
|
||||
var clientKey [32]byte
|
||||
var sse encrypt.ServerSide
|
||||
|
||||
if copySource {
|
||||
if crypto.SSECopy.IsRequested(header) {
|
||||
clientKey, err = crypto.SSECopy.ParseHTTP(header)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil {
|
||||
return
|
||||
}
|
||||
return ObjectOptions{ServerSideEncryption: encrypt.SSECopy(sse), UserDefined: metadata}, nil
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if crypto.SSEC.IsRequested(header) {
|
||||
clientKey, err = crypto.SSEC.ParseHTTP(header)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil {
|
||||
return
|
||||
}
|
||||
return ObjectOptions{ServerSideEncryption: sse, UserDefined: metadata}, nil
|
||||
}
|
||||
if crypto.S3.IsRequested(header) || (metadata != nil && crypto.S3.IsEncrypted(metadata)) {
|
||||
return ObjectOptions{ServerSideEncryption: encrypt.NewSSE(), UserDefined: metadata}, nil
|
||||
}
|
||||
return ObjectOptions{UserDefined: metadata}, nil
|
||||
}
|
||||
|
||||
// get ObjectOptions for GET calls from encryption headers
|
||||
func getOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) {
|
||||
var (
|
||||
encryption encrypt.ServerSide
|
||||
opts ObjectOptions
|
||||
)
|
||||
|
||||
var partNumber int
|
||||
var err error
|
||||
if pn := r.URL.Query().Get("partNumber"); pn != "" {
|
||||
partNumber, err = strconv.Atoi(pn)
|
||||
if err != nil {
|
||||
return opts, err
|
||||
}
|
||||
if partNumber <= 0 {
|
||||
return opts, errInvalidArgument
|
||||
}
|
||||
}
|
||||
|
||||
if GlobalGatewaySSE.SSEC() && crypto.SSEC.IsRequested(r.Header) {
|
||||
key, err := crypto.SSEC.ParseHTTP(r.Header)
|
||||
if err != nil {
|
||||
return opts, err
|
||||
}
|
||||
derivedKey := deriveClientKey(key, bucket, object)
|
||||
encryption, err = encrypt.NewSSEC(derivedKey[:])
|
||||
logger.CriticalIf(ctx, err)
|
||||
return ObjectOptions{ServerSideEncryption: encryption, PartNumber: partNumber}, nil
|
||||
}
|
||||
|
||||
// default case of passing encryption headers to backend
|
||||
opts, err = getDefaultOpts(r.Header, false, nil)
|
||||
if err != nil {
|
||||
return opts, err
|
||||
}
|
||||
opts.PartNumber = partNumber
|
||||
return opts, nil
|
||||
}
|
||||
|
||||
// get ObjectOptions for PUT calls from encryption headers and metadata
|
||||
func putOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
|
||||
// In the case of multipart custom format, the metadata needs to be checked in addition to header to see if it
|
||||
// is SSE-S3 encrypted, primarily because S3 protocol does not require SSE-S3 headers in PutObjectPart calls
|
||||
if GlobalGatewaySSE.SSES3() && (crypto.S3.IsRequested(r.Header) || crypto.S3.IsEncrypted(metadata)) {
|
||||
return ObjectOptions{ServerSideEncryption: encrypt.NewSSE(), UserDefined: metadata}, nil
|
||||
}
|
||||
if GlobalGatewaySSE.SSEC() && crypto.SSEC.IsRequested(r.Header) {
|
||||
opts, err = getOpts(ctx, r, bucket, object)
|
||||
opts.UserDefined = metadata
|
||||
return
|
||||
}
|
||||
if crypto.S3KMS.IsRequested(r.Header) {
|
||||
keyID, context, err := crypto.S3KMS.ParseHTTP(r.Header)
|
||||
if err != nil {
|
||||
return ObjectOptions{}, err
|
||||
}
|
||||
sseKms, err := encrypt.NewSSEKMS(keyID, context)
|
||||
if err != nil {
|
||||
return ObjectOptions{}, err
|
||||
}
|
||||
return ObjectOptions{ServerSideEncryption: sseKms, UserDefined: metadata}, nil
|
||||
}
|
||||
// default case of passing encryption headers and UserDefined metadata to backend
|
||||
return getDefaultOpts(r.Header, false, metadata)
|
||||
}
|
||||
|
||||
// get ObjectOptions for Copy calls with encryption headers provided on the target side and source side metadata
|
||||
func copyDstOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
|
||||
return putOpts(ctx, r, bucket, object, metadata)
|
||||
}
|
||||
|
||||
// get ObjectOptions for Copy calls with encryption headers provided on the source side
|
||||
func copySrcOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) {
|
||||
var (
|
||||
ssec encrypt.ServerSide
|
||||
opts ObjectOptions
|
||||
)
|
||||
|
||||
if GlobalGatewaySSE.SSEC() && crypto.SSECopy.IsRequested(r.Header) {
|
||||
key, err := crypto.SSECopy.ParseHTTP(r.Header)
|
||||
if err != nil {
|
||||
return opts, err
|
||||
}
|
||||
derivedKey := deriveClientKey(key, bucket, object)
|
||||
ssec, err = encrypt.NewSSEC(derivedKey[:])
|
||||
if err != nil {
|
||||
return opts, err
|
||||
}
|
||||
return ObjectOptions{ServerSideEncryption: encrypt.SSECopy(ssec)}, nil
|
||||
}
|
||||
|
||||
// default case of passing encryption headers to backend
|
||||
return getDefaultOpts(r.Header, true, nil)
|
||||
}
|
||||
|
||||
+10
-62
@@ -17,10 +17,8 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"path/filepath"
|
||||
@@ -33,7 +31,6 @@ import (
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
"github.com/minio/minio/cmd/config"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/cmd/rest"
|
||||
"github.com/minio/minio/pkg/env"
|
||||
"github.com/minio/minio/pkg/mountinfo"
|
||||
)
|
||||
@@ -47,14 +44,9 @@ const (
|
||||
|
||||
// URLEndpointType - URL style endpoint type enum.
|
||||
URLEndpointType
|
||||
)
|
||||
|
||||
// ProxyEndpoint - endpoint used for proxy redirects
|
||||
// See proxyRequest() for details.
|
||||
type ProxyEndpoint struct {
|
||||
Endpoint
|
||||
Transport *http.Transport
|
||||
}
|
||||
retryInterval = 5 // In Seconds.
|
||||
)
|
||||
|
||||
// Endpoint - any type of endpoint.
|
||||
type Endpoint struct {
|
||||
@@ -310,7 +302,7 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
|
||||
resolvedList := make([]bool, len(endpoints))
|
||||
// Mark the starting time
|
||||
startTime := time.Now()
|
||||
keepAliveTicker := time.NewTicker(10 * time.Millisecond)
|
||||
keepAliveTicker := time.NewTicker(retryInterval * time.Second)
|
||||
defer keepAliveTicker.Stop()
|
||||
for {
|
||||
// Break if the local endpoint is found already Or all the endpoints are resolved.
|
||||
@@ -555,9 +547,9 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
|
||||
return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg("invalid number of endpoints")
|
||||
}
|
||||
|
||||
// Return Erasure setup when all endpoints are path style.
|
||||
// Return XL setup when all endpoints are path style.
|
||||
if endpoints[0].Type() == PathEndpointType {
|
||||
setupType = ErasureSetupType
|
||||
setupType = XLSetupType
|
||||
return endpoints, setupType, nil
|
||||
}
|
||||
|
||||
@@ -622,18 +614,18 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
|
||||
|
||||
// All endpoints are pointing to local host
|
||||
if len(endpoints) == localEndpointCount {
|
||||
// If all endpoints have same port number, Just treat it as distErasure setup
|
||||
// If all endpoints have same port number, Just treat it as distXL setup
|
||||
// using URL style endpoints.
|
||||
if len(localPortSet) == 1 {
|
||||
if len(localServerHostSet) > 1 {
|
||||
return endpoints, setupType,
|
||||
config.ErrInvalidErasureEndpoints(nil).Msg("all local endpoints should not have different hostnames/ips")
|
||||
}
|
||||
return endpoints, DistErasureSetupType, nil
|
||||
return endpoints, DistXLSetupType, nil
|
||||
}
|
||||
|
||||
// Even though all endpoints are local, but those endpoints use different ports.
|
||||
// This means it is DistErasure setup.
|
||||
// This means it is DistXL setup.
|
||||
}
|
||||
|
||||
// Add missing port in all endpoints.
|
||||
@@ -653,7 +645,7 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
|
||||
}
|
||||
|
||||
// Error out if we have less than 2 unique servers.
|
||||
if len(uniqueArgs.ToSlice()) < 2 && setupType == DistErasureSetupType {
|
||||
if len(uniqueArgs.ToSlice()) < 2 && setupType == DistXLSetupType {
|
||||
err := fmt.Errorf("Unsupported number of endpoints (%s), minimum number of servers cannot be less than 2 in distributed setup", endpoints)
|
||||
return endpoints, setupType, err
|
||||
}
|
||||
@@ -663,7 +655,7 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
|
||||
updateDomainIPs(uniqueArgs)
|
||||
}
|
||||
|
||||
setupType = DistErasureSetupType
|
||||
setupType = DistXLSetupType
|
||||
return endpoints, setupType, nil
|
||||
}
|
||||
|
||||
@@ -718,50 +710,6 @@ func GetRemotePeers(endpointZones EndpointZones) []string {
|
||||
return peerSet.ToSlice()
|
||||
}
|
||||
|
||||
// GetProxyEndpointLocalIndex returns index of the local proxy endpoint
|
||||
func GetProxyEndpointLocalIndex(proxyEps []ProxyEndpoint) int {
|
||||
for i, pep := range proxyEps {
|
||||
if pep.IsLocal {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// GetProxyEndpoints - get all endpoints that can be used to proxy list request.
|
||||
func GetProxyEndpoints(endpointZones EndpointZones) ([]ProxyEndpoint, error) {
|
||||
var proxyEps []ProxyEndpoint
|
||||
|
||||
proxyEpSet := set.NewStringSet()
|
||||
|
||||
for _, ep := range endpointZones {
|
||||
for _, endpoint := range ep.Endpoints {
|
||||
if endpoint.Type() != URLEndpointType {
|
||||
continue
|
||||
}
|
||||
|
||||
host := endpoint.Host
|
||||
if proxyEpSet.Contains(host) {
|
||||
continue
|
||||
}
|
||||
proxyEpSet.Add(host)
|
||||
|
||||
var tlsConfig *tls.Config
|
||||
if globalIsSSL {
|
||||
tlsConfig = &tls.Config{
|
||||
ServerName: endpoint.Hostname(),
|
||||
RootCAs: globalRootCAs,
|
||||
}
|
||||
}
|
||||
proxyEps = append(proxyEps, ProxyEndpoint{
|
||||
Endpoint: endpoint,
|
||||
Transport: newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)(),
|
||||
})
|
||||
}
|
||||
}
|
||||
return proxyEps, nil
|
||||
}
|
||||
|
||||
func updateDomainIPs(endPoints set.StringSet) {
|
||||
ipList := set.NewStringSet()
|
||||
for e := range endPoints {
|
||||
|
||||
+13
-13
@@ -232,71 +232,71 @@ func TestCreateEndpoints(t *testing.T) {
|
||||
{"localhost:10000", [][]string{{"/d1"}}, "localhost:10000", Endpoints{Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true}}, FSSetupType, nil},
|
||||
{"localhost:9000", [][]string{{"https://127.0.0.1:9000/d1", "https://localhost:9001/d1", "https://example.com/d1", "https://example.com/d2"}}, "", Endpoints{}, -1, fmt.Errorf("path '/d1' can not be served by different port on same address")},
|
||||
|
||||
// Erasure Setup with PathEndpointType
|
||||
// XL Setup with PathEndpointType
|
||||
{":1234", [][]string{{"/d1", "/d2", "/d3", "/d4"}}, ":1234",
|
||||
Endpoints{
|
||||
Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: mustAbs("/d2")}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: mustAbs("/d3")}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: mustAbs("/d4")}, IsLocal: true},
|
||||
}, ErasureSetupType, nil},
|
||||
// DistErasure Setup with URLEndpointType
|
||||
}, XLSetupType, nil},
|
||||
// DistXL Setup with URLEndpointType
|
||||
{":9000", [][]string{{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}}, ":9000", Endpoints{
|
||||
Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d1"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d2"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d3"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d4"}, IsLocal: true},
|
||||
}, DistErasureSetupType, nil},
|
||||
// DistErasure Setup with URLEndpointType having mixed naming to local host.
|
||||
}, DistXLSetupType, nil},
|
||||
// DistXL Setup with URLEndpointType having mixed naming to local host.
|
||||
{"127.0.0.1:10000", [][]string{{"http://localhost/d1", "http://localhost/d2", "http://127.0.0.1/d3", "http://127.0.0.1/d4"}}, "", Endpoints{}, -1, fmt.Errorf("all local endpoints should not have different hostnames/ips")},
|
||||
|
||||
{":9001", [][]string{{"http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export", "http://" + nonLoopBackIP + ":9001/export", "http://10.0.0.2:9001/export"}}, "", Endpoints{}, -1, fmt.Errorf("path '/export' can not be served by different port on same address")},
|
||||
|
||||
{":9000", [][]string{{"http://127.0.0.1:9000/export", "http://" + nonLoopBackIP + ":9000/export", "http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export"}}, "", Endpoints{}, -1, fmt.Errorf("path '/export' cannot be served by different address on same server")},
|
||||
|
||||
// DistErasure type
|
||||
// DistXL type
|
||||
{"127.0.0.1:10000", [][]string{{case1Endpoint1, case1Endpoint2, "http://example.org/d3", "http://example.com/d4"}}, "127.0.0.1:10000", Endpoints{
|
||||
Endpoint{URL: case1URLs[0], IsLocal: case1LocalFlags[0]},
|
||||
Endpoint{URL: case1URLs[1], IsLocal: case1LocalFlags[1]},
|
||||
Endpoint{URL: case1URLs[2], IsLocal: case1LocalFlags[2]},
|
||||
Endpoint{URL: case1URLs[3], IsLocal: case1LocalFlags[3]},
|
||||
}, DistErasureSetupType, nil},
|
||||
}, DistXLSetupType, nil},
|
||||
|
||||
{"127.0.0.1:10000", [][]string{{case2Endpoint1, case2Endpoint2, "http://example.org/d3", "http://example.com/d4"}}, "127.0.0.1:10000", Endpoints{
|
||||
Endpoint{URL: case2URLs[0], IsLocal: case2LocalFlags[0]},
|
||||
Endpoint{URL: case2URLs[1], IsLocal: case2LocalFlags[1]},
|
||||
Endpoint{URL: case2URLs[2], IsLocal: case2LocalFlags[2]},
|
||||
Endpoint{URL: case2URLs[3], IsLocal: case2LocalFlags[3]},
|
||||
}, DistErasureSetupType, nil},
|
||||
}, DistXLSetupType, nil},
|
||||
|
||||
{":80", [][]string{{case3Endpoint1, "http://example.org:9000/d2", "http://example.com/d3", "http://example.net/d4"}}, ":80", Endpoints{
|
||||
Endpoint{URL: case3URLs[0], IsLocal: case3LocalFlags[0]},
|
||||
Endpoint{URL: case3URLs[1], IsLocal: case3LocalFlags[1]},
|
||||
Endpoint{URL: case3URLs[2], IsLocal: case3LocalFlags[2]},
|
||||
Endpoint{URL: case3URLs[3], IsLocal: case3LocalFlags[3]},
|
||||
}, DistErasureSetupType, nil},
|
||||
}, DistXLSetupType, nil},
|
||||
|
||||
{":9000", [][]string{{case4Endpoint1, "http://example.org/d2", "http://example.com/d3", "http://example.net/d4"}}, ":9000", Endpoints{
|
||||
Endpoint{URL: case4URLs[0], IsLocal: case4LocalFlags[0]},
|
||||
Endpoint{URL: case4URLs[1], IsLocal: case4LocalFlags[1]},
|
||||
Endpoint{URL: case4URLs[2], IsLocal: case4LocalFlags[2]},
|
||||
Endpoint{URL: case4URLs[3], IsLocal: case4LocalFlags[3]},
|
||||
}, DistErasureSetupType, nil},
|
||||
}, DistXLSetupType, nil},
|
||||
|
||||
{":9000", [][]string{{case5Endpoint1, case5Endpoint2, case5Endpoint3, case5Endpoint4}}, ":9000", Endpoints{
|
||||
Endpoint{URL: case5URLs[0], IsLocal: case5LocalFlags[0]},
|
||||
Endpoint{URL: case5URLs[1], IsLocal: case5LocalFlags[1]},
|
||||
Endpoint{URL: case5URLs[2], IsLocal: case5LocalFlags[2]},
|
||||
Endpoint{URL: case5URLs[3], IsLocal: case5LocalFlags[3]},
|
||||
}, DistErasureSetupType, nil},
|
||||
}, DistXLSetupType, nil},
|
||||
|
||||
// DistErasure Setup using only local host.
|
||||
// DistXL Setup using only local host.
|
||||
{":9003", [][]string{{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://127.0.0.1:9002/d3", case6Endpoint}}, ":9003", Endpoints{
|
||||
Endpoint{URL: case6URLs[0], IsLocal: case6LocalFlags[0]},
|
||||
Endpoint{URL: case6URLs[1], IsLocal: case6LocalFlags[1]},
|
||||
Endpoint{URL: case6URLs[2], IsLocal: case6LocalFlags[2]},
|
||||
Endpoint{URL: case6URLs[3], IsLocal: case6LocalFlags[3]},
|
||||
}, DistErasureSetupType, nil},
|
||||
}, DistXLSetupType, nil},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
|
||||
@@ -1,143 +0,0 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2017 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/klauspost/reedsolomon"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
)
|
||||
|
||||
// Erasure - erasure encoding details.
|
||||
type Erasure struct {
|
||||
encoder func() reedsolomon.Encoder
|
||||
dataBlocks, parityBlocks int
|
||||
blockSize int64
|
||||
}
|
||||
|
||||
// NewErasure creates a new ErasureStorage.
|
||||
func NewErasure(ctx context.Context, dataBlocks, parityBlocks int, blockSize int64) (e Erasure, err error) {
|
||||
// Check the parameters for sanity now.
|
||||
if dataBlocks <= 0 || parityBlocks <= 0 {
|
||||
return e, reedsolomon.ErrInvShardNum
|
||||
}
|
||||
|
||||
if dataBlocks+parityBlocks > 256 {
|
||||
return e, reedsolomon.ErrMaxShardNum
|
||||
}
|
||||
|
||||
e = Erasure{
|
||||
dataBlocks: dataBlocks,
|
||||
parityBlocks: parityBlocks,
|
||||
blockSize: blockSize,
|
||||
}
|
||||
|
||||
// Encoder when needed.
|
||||
var enc reedsolomon.Encoder
|
||||
var once sync.Once
|
||||
e.encoder = func() reedsolomon.Encoder {
|
||||
once.Do(func() {
|
||||
e, err := reedsolomon.New(dataBlocks, parityBlocks, reedsolomon.WithAutoGoroutines(int(e.ShardSize())))
|
||||
if err != nil {
|
||||
// Error conditions should be checked above.
|
||||
panic(err)
|
||||
}
|
||||
enc = e
|
||||
})
|
||||
return enc
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// EncodeData encodes the given data and returns the erasure-coded data.
|
||||
// It returns an error if the erasure coding failed.
|
||||
func (e *Erasure) EncodeData(ctx context.Context, data []byte) ([][]byte, error) {
|
||||
if len(data) == 0 {
|
||||
return make([][]byte, e.dataBlocks+e.parityBlocks), nil
|
||||
}
|
||||
encoded, err := e.encoder().Split(data)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return nil, err
|
||||
}
|
||||
if err = e.encoder().Encode(encoded); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return nil, err
|
||||
}
|
||||
return encoded, nil
|
||||
}
|
||||
|
||||
// DecodeDataBlocks decodes the given erasure-coded data.
|
||||
// It only decodes the data blocks but does not verify them.
|
||||
// It returns an error if the decoding failed.
|
||||
func (e *Erasure) DecodeDataBlocks(data [][]byte) error {
|
||||
var isZero = 0
|
||||
for _, b := range data[:] {
|
||||
if len(b) == 0 {
|
||||
isZero++
|
||||
break
|
||||
}
|
||||
}
|
||||
if isZero == 0 || isZero == len(data) {
|
||||
// If all are zero, payload is 0 bytes.
|
||||
return nil
|
||||
}
|
||||
return e.encoder().ReconstructData(data)
|
||||
}
|
||||
|
||||
// DecodeDataAndParityBlocks decodes the given erasure-coded data and verifies it.
|
||||
// It returns an error if the decoding failed.
|
||||
func (e *Erasure) DecodeDataAndParityBlocks(ctx context.Context, data [][]byte) error {
|
||||
if err := e.encoder().Reconstruct(data); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ShardSize - returns actual shared size from erasure blockSize.
|
||||
func (e *Erasure) ShardSize() int64 {
|
||||
return ceilFrac(e.blockSize, int64(e.dataBlocks))
|
||||
}
|
||||
|
||||
// ShardFileSize - returns final erasure size from original size.
|
||||
func (e *Erasure) ShardFileSize(totalLength int64) int64 {
|
||||
if totalLength == 0 {
|
||||
return 0
|
||||
}
|
||||
if totalLength == -1 {
|
||||
return -1
|
||||
}
|
||||
numShards := totalLength / e.blockSize
|
||||
lastBlockSize := totalLength % int64(e.blockSize)
|
||||
lastShardSize := ceilFrac(lastBlockSize, int64(e.dataBlocks))
|
||||
return numShards*e.ShardSize() + lastShardSize
|
||||
}
|
||||
|
||||
// ShardFileOffset - returns the effective offset where erasure reading begins.
|
||||
func (e *Erasure) ShardFileOffset(startOffset, length, totalLength int64) int64 {
|
||||
shardSize := e.ShardSize()
|
||||
shardFileSize := e.ShardFileSize(totalLength)
|
||||
endShard := (startOffset + int64(length)) / e.blockSize
|
||||
tillOffset := endShard*shardSize + shardSize
|
||||
if tillOffset > shardFileSize {
|
||||
tillOffset = shardFileSize
|
||||
}
|
||||
return tillOffset
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
|
||||
* MinIO Cloud Storage, (C) 2016 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -145,9 +145,9 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
rr := p.readers[i]
|
||||
if rr == nil {
|
||||
// Since reader is nil, trigger another read.
|
||||
disk := p.readers[i]
|
||||
if disk == nil {
|
||||
// Since disk is nil, trigger another read.
|
||||
readTriggerCh <- true
|
||||
return
|
||||
}
|
||||
@@ -160,7 +160,7 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
|
||||
// For the last shard, the shardsize might be less than previous shard sizes.
|
||||
// Hence the following statement ensures that the buffer size is reset to the right size.
|
||||
p.buf[bufIdx] = p.buf[bufIdx][:p.shardSize]
|
||||
_, err := rr.ReadAt(p.buf[bufIdx], p.offset)
|
||||
_, err := disk.ReadAt(p.buf[bufIdx], p.offset)
|
||||
if err != nil {
|
||||
if _, ok := err.(*errHashMismatch); ok {
|
||||
atomic.StoreInt32(&healRequired, 1)
|
||||
@@ -191,7 +191,7 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
|
||||
return newBuf, nil
|
||||
}
|
||||
|
||||
return nil, errErasureReadQuorum
|
||||
return nil, errXLReadQuorum
|
||||
}
|
||||
|
||||
type errDecodeHealRequired struct {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
|
||||
* MinIO Cloud Storage, (C) 2016, 2017 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -132,7 +132,7 @@ func TestErasureDecode(t *testing.T) {
|
||||
if disk == OfflineDisk {
|
||||
continue
|
||||
}
|
||||
tillOffset := erasure.ShardFileOffset(test.offset, test.length, test.data)
|
||||
tillOffset := erasure.ShardFileTillOffset(test.offset, test.length, test.data)
|
||||
|
||||
bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", tillOffset, writeAlgorithm, bitrotWriterSum(writers[index]), erasure.ShardSize())
|
||||
}
|
||||
@@ -163,7 +163,7 @@ func TestErasureDecode(t *testing.T) {
|
||||
if disk == OfflineDisk {
|
||||
continue
|
||||
}
|
||||
tillOffset := erasure.ShardFileOffset(test.offset, test.length, test.data)
|
||||
tillOffset := erasure.ShardFileTillOffset(test.offset, test.length, test.data)
|
||||
bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", tillOffset, writeAlgorithm, bitrotWriterSum(writers[index]), erasure.ShardSize())
|
||||
}
|
||||
for j := range disks[:test.offDisks] {
|
||||
@@ -268,7 +268,7 @@ func TestErasureDecodeRandomOffsetLength(t *testing.T) {
|
||||
if disk == OfflineDisk {
|
||||
continue
|
||||
}
|
||||
tillOffset := erasure.ShardFileOffset(offset, readLen, length)
|
||||
tillOffset := erasure.ShardFileTillOffset(offset, readLen, length)
|
||||
bitrotReaders[index] = newStreamingBitrotReader(disk, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
|
||||
}
|
||||
err = erasure.Decode(context.Background(), buf, bitrotReaders, offset, readLen, length, nil)
|
||||
@@ -330,7 +330,7 @@ func benchmarkErasureDecode(data, parity, dataDown, parityDown int, size int64,
|
||||
if writers[index] == nil {
|
||||
continue
|
||||
}
|
||||
tillOffset := erasure.ShardFileOffset(0, size, size)
|
||||
tillOffset := erasure.ShardFileTillOffset(0, size, size)
|
||||
bitrotReaders[index] = newStreamingBitrotReader(disk, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
|
||||
}
|
||||
if err = erasure.Decode(context.Background(), bytes.NewBuffer(content[:0]), bitrotReaders, 0, size, size, nil); err != nil {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
|
||||
* MinIO Cloud Storage, (C) 2016 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
|
||||
* MinIO Cloud Storage, (C) 2016 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
|
||||
* MinIO Cloud Storage, (C) 2016 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
|
||||
* MinIO Cloud Storage, (C) 2016 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -71,7 +71,7 @@ func TestErasureHeal(t *testing.T) {
|
||||
// create some test data
|
||||
setup, err := newErasureTestSetup(test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: failed to setup Erasure environment: %v", i, err)
|
||||
t.Fatalf("Test %d: failed to setup XL environment: %v", i, err)
|
||||
}
|
||||
disks := setup.disks
|
||||
erasure, err := NewErasure(context.Background(), test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
)
|
||||
|
||||
// ListObjectVersions - This is not implemented, look for erasure-zones.ListObjectVersions()
|
||||
func (er erasureObjects) ListObjectVersions(ctx context.Context, bucket, prefix, marker, versionMarker, delimiter string, maxKeys int) (loi ListObjectVersionsInfo, e error) {
|
||||
return loi, NotImplemented{}
|
||||
}
|
||||
|
||||
// ListObjectsV2 - This is not implemented/needed anymore, look for erasure-zones.ListObjectsV2()
|
||||
func (er erasureObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi ListObjectsV2Info, e error) {
|
||||
return loi, NotImplemented{}
|
||||
}
|
||||
|
||||
// ListObjects - This is not implemented/needed anymore, look for erasure-zones.ListObjects()
|
||||
func (er erasureObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
|
||||
return loi, NotImplemented{}
|
||||
}
|
||||
|
||||
// ListBucketsHeal - This is not implemented/needed anymore, look for erasure-zones.ListBucketHeal()
|
||||
func (er erasureObjects) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error) {
|
||||
return nil, NotImplemented{}
|
||||
}
|
||||
|
||||
// ListObjectsHeal - This is not implemented, look for erasure-zones.ListObjectsHeal()
|
||||
func (er erasureObjects) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) {
|
||||
return ListObjectsInfo{}, NotImplemented{}
|
||||
}
|
||||
|
||||
// HealObjects - This is not implemented/needed anymore, look for erasure-zones.HealObjects()
|
||||
func (er erasureObjects) HealObjects(ctx context.Context, bucket, prefix string, _ madmin.HealOpts, _ HealObjectFn) (e error) {
|
||||
return NotImplemented{}
|
||||
}
|
||||
|
||||
// Walk - This is not implemented/needed anymore, look for erasure-zones.Walk()
|
||||
func (er erasureObjects) Walk(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo, _ ObjectOptions) error {
|
||||
return NotImplemented{}
|
||||
}
|
||||
@@ -1,201 +0,0 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2015, 2016, 2017 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Tests caclculating disk count.
|
||||
func TestDiskCount(t *testing.T) {
|
||||
testCases := []struct {
|
||||
disks []StorageAPI
|
||||
diskCount int
|
||||
}{
|
||||
// Test case - 1
|
||||
{
|
||||
disks: []StorageAPI{&xlStorage{}, &xlStorage{}, &xlStorage{}, &xlStorage{}},
|
||||
diskCount: 4,
|
||||
},
|
||||
// Test case - 2
|
||||
{
|
||||
disks: []StorageAPI{nil, &xlStorage{}, &xlStorage{}, &xlStorage{}},
|
||||
diskCount: 3,
|
||||
},
|
||||
}
|
||||
for i, testCase := range testCases {
|
||||
cdiskCount := diskCount(testCase.disks)
|
||||
if cdiskCount != testCase.diskCount {
|
||||
t.Errorf("Test %d: Expected %d, got %d", i+1, testCase.diskCount, cdiskCount)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test for reduceErrs, reduceErr reduces collection
|
||||
// of errors into a single maximal error with in the list.
|
||||
func TestReduceErrs(t *testing.T) {
|
||||
// List all of all test cases to validate various cases of reduce errors.
|
||||
testCases := []struct {
|
||||
errs []error
|
||||
ignoredErrs []error
|
||||
err error
|
||||
}{
|
||||
// Validate if have reduced properly.
|
||||
{[]error{
|
||||
errDiskNotFound,
|
||||
errDiskNotFound,
|
||||
errDiskFull,
|
||||
}, []error{}, errErasureReadQuorum},
|
||||
// Validate if have no consensus.
|
||||
{[]error{
|
||||
errDiskFull,
|
||||
errDiskNotFound,
|
||||
nil, nil,
|
||||
}, []error{}, errErasureReadQuorum},
|
||||
// Validate if have consensus and errors ignored.
|
||||
{[]error{
|
||||
errVolumeNotFound,
|
||||
errVolumeNotFound,
|
||||
errVolumeNotFound,
|
||||
errVolumeNotFound,
|
||||
errVolumeNotFound,
|
||||
errDiskNotFound,
|
||||
errDiskNotFound,
|
||||
}, []error{errDiskNotFound}, errVolumeNotFound},
|
||||
{[]error{}, []error{}, errErasureReadQuorum},
|
||||
{[]error{errFileNotFound, errFileNotFound, errFileNotFound,
|
||||
errFileNotFound, errFileNotFound, nil, nil, nil, nil, nil},
|
||||
nil, nil},
|
||||
}
|
||||
// Validates list of all the testcases for returning valid errors.
|
||||
for i, testCase := range testCases {
|
||||
gotErr := reduceReadQuorumErrs(context.Background(), testCase.errs, testCase.ignoredErrs, 5)
|
||||
if gotErr != testCase.err {
|
||||
t.Errorf("Test %d : expected %s, got %s", i+1, testCase.err, gotErr)
|
||||
}
|
||||
gotNewErr := reduceWriteQuorumErrs(context.Background(), testCase.errs, testCase.ignoredErrs, 6)
|
||||
if gotNewErr != errErasureWriteQuorum {
|
||||
t.Errorf("Test %d : expected %s, got %s", i+1, errErasureWriteQuorum, gotErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestHashOrder - test order of ints in array
|
||||
func TestHashOrder(t *testing.T) {
|
||||
testCases := []struct {
|
||||
objectName string
|
||||
hashedOrder []int
|
||||
}{
|
||||
// cases which should pass the test.
|
||||
// passing in valid object name.
|
||||
{"object", []int{14, 15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}},
|
||||
{"The Shining Script <v1>.pdf", []int{16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}},
|
||||
{"Cost Benefit Analysis (2009-2010).pptx", []int{15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}},
|
||||
{"117Gn8rfHL2ACARPAhaFd0AGzic9pUbIA/5OCn5A", []int{3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2}},
|
||||
{"SHØRT", []int{11, 12, 13, 14, 15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}},
|
||||
{"There are far too many object names, and far too few bucket names!", []int{15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}},
|
||||
{"a/b/c/", []int{3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2}},
|
||||
{"/a/b/c", []int{6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2, 3, 4, 5}},
|
||||
{string([]byte{0xff, 0xfe, 0xfd}), []int{15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}},
|
||||
}
|
||||
|
||||
// Tests hashing order to be consistent.
|
||||
for i, testCase := range testCases {
|
||||
hashedOrder := hashOrder(testCase.objectName, 16)
|
||||
if !reflect.DeepEqual(testCase.hashedOrder, hashedOrder) {
|
||||
t.Errorf("Test case %d: Expected \"%v\" but failed \"%v\"", i+1, testCase.hashedOrder, hashedOrder)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests hashing order to fail for when order is '-1'.
|
||||
if hashedOrder := hashOrder("This will fail", -1); hashedOrder != nil {
|
||||
t.Errorf("Test: Expect \"nil\" but failed \"%#v\"", hashedOrder)
|
||||
}
|
||||
|
||||
if hashedOrder := hashOrder("This will fail", 0); hashedOrder != nil {
|
||||
t.Errorf("Test: Expect \"nil\" but failed \"%#v\"", hashedOrder)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShuffleDisks(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
nDisks := 16
|
||||
disks, err := getRandomDisks(nDisks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
objLayer, _, err := initObjectLayer(ctx, mustGetZoneEndpoints(disks...))
|
||||
if err != nil {
|
||||
removeRoots(disks)
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer removeRoots(disks)
|
||||
z := objLayer.(*erasureZones)
|
||||
testShuffleDisks(t, z)
|
||||
}
|
||||
|
||||
// Test shuffleDisks which returns shuffled slice of disks for their actual distribution.
|
||||
func testShuffleDisks(t *testing.T, z *erasureZones) {
|
||||
disks := z.zones[0].GetDisks(0)()
|
||||
distribution := []int{16, 14, 12, 10, 8, 6, 4, 2, 1, 3, 5, 7, 9, 11, 13, 15}
|
||||
shuffledDisks := shuffleDisks(disks, distribution)
|
||||
// From the "distribution" above you can notice that:
|
||||
// 1st data block is in the 9th disk (i.e distribution index 8)
|
||||
// 2nd data block is in the 8th disk (i.e distribution index 7) and so on.
|
||||
if shuffledDisks[0] != disks[8] ||
|
||||
shuffledDisks[1] != disks[7] ||
|
||||
shuffledDisks[2] != disks[9] ||
|
||||
shuffledDisks[3] != disks[6] ||
|
||||
shuffledDisks[4] != disks[10] ||
|
||||
shuffledDisks[5] != disks[5] ||
|
||||
shuffledDisks[6] != disks[11] ||
|
||||
shuffledDisks[7] != disks[4] ||
|
||||
shuffledDisks[8] != disks[12] ||
|
||||
shuffledDisks[9] != disks[3] ||
|
||||
shuffledDisks[10] != disks[13] ||
|
||||
shuffledDisks[11] != disks[2] ||
|
||||
shuffledDisks[12] != disks[14] ||
|
||||
shuffledDisks[13] != disks[1] ||
|
||||
shuffledDisks[14] != disks[15] ||
|
||||
shuffledDisks[15] != disks[0] {
|
||||
t.Errorf("shuffleDisks returned incorrect order.")
|
||||
}
|
||||
}
|
||||
|
||||
// TestEvalDisks tests the behavior of evalDisks
|
||||
func TestEvalDisks(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
nDisks := 16
|
||||
disks, err := getRandomDisks(nDisks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
objLayer, _, err := initObjectLayer(ctx, mustGetZoneEndpoints(disks...))
|
||||
if err != nil {
|
||||
removeRoots(disks)
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer removeRoots(disks)
|
||||
z := objLayer.(*erasureZones)
|
||||
testShuffleDisks(t, z)
|
||||
}
|
||||
@@ -1,326 +0,0 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2016-2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/sync/errgroup"
|
||||
"github.com/minio/sha256-simd"
|
||||
)
|
||||
|
||||
const erasureAlgorithm = "rs-vandermonde"
|
||||
|
||||
// byObjectPartNumber is a collection satisfying sort.Interface.
|
||||
type byObjectPartNumber []ObjectPartInfo
|
||||
|
||||
func (t byObjectPartNumber) Len() int { return len(t) }
|
||||
func (t byObjectPartNumber) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
|
||||
func (t byObjectPartNumber) Less(i, j int) bool { return t[i].Number < t[j].Number }
|
||||
|
||||
// AddChecksumInfo adds a checksum of a part.
|
||||
func (e *ErasureInfo) AddChecksumInfo(ckSumInfo ChecksumInfo) {
|
||||
for i, sum := range e.Checksums {
|
||||
if sum.PartNumber == ckSumInfo.PartNumber {
|
||||
e.Checksums[i] = ckSumInfo
|
||||
return
|
||||
}
|
||||
}
|
||||
e.Checksums = append(e.Checksums, ckSumInfo)
|
||||
}
|
||||
|
||||
// GetChecksumInfo - get checksum of a part.
|
||||
func (e ErasureInfo) GetChecksumInfo(partNumber int) (ckSum ChecksumInfo) {
|
||||
for _, sum := range e.Checksums {
|
||||
if sum.PartNumber == partNumber {
|
||||
// Return the checksum
|
||||
return sum
|
||||
}
|
||||
}
|
||||
return ChecksumInfo{}
|
||||
}
|
||||
|
||||
// ShardFileSize - returns final erasure size from original size.
|
||||
func (e ErasureInfo) ShardFileSize(totalLength int64) int64 {
|
||||
if totalLength == 0 {
|
||||
return 0
|
||||
}
|
||||
if totalLength == -1 {
|
||||
return -1
|
||||
}
|
||||
numShards := totalLength / e.BlockSize
|
||||
lastBlockSize := totalLength % e.BlockSize
|
||||
lastShardSize := ceilFrac(lastBlockSize, int64(e.DataBlocks))
|
||||
return numShards*e.ShardSize() + lastShardSize
|
||||
}
|
||||
|
||||
// ShardSize - returns actual shared size from erasure blockSize.
|
||||
func (e ErasureInfo) ShardSize() int64 {
|
||||
return ceilFrac(e.BlockSize, int64(e.DataBlocks))
|
||||
}
|
||||
|
||||
// IsValid - tells if erasure info fields are valid.
|
||||
func (fi FileInfo) IsValid() bool {
|
||||
if fi.Deleted {
|
||||
// Delete marker has no data, no need to check
|
||||
// for erasure coding information
|
||||
return true
|
||||
}
|
||||
data := fi.Erasure.DataBlocks
|
||||
parity := fi.Erasure.ParityBlocks
|
||||
return ((data >= parity) && (data != 0) && (parity != 0))
|
||||
}
|
||||
|
||||
// ToObjectInfo - Converts metadata to object info.
|
||||
func (fi FileInfo) ToObjectInfo(bucket, object string) ObjectInfo {
|
||||
if HasSuffix(object, SlashSeparator) {
|
||||
return ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: object,
|
||||
IsDir: true,
|
||||
}
|
||||
}
|
||||
objInfo := ObjectInfo{
|
||||
IsDir: false,
|
||||
Bucket: bucket,
|
||||
Name: object,
|
||||
VersionID: fi.VersionID,
|
||||
IsLatest: fi.IsLatest,
|
||||
DeleteMarker: fi.Deleted,
|
||||
Size: fi.Size,
|
||||
ModTime: fi.ModTime,
|
||||
ContentType: fi.Metadata["content-type"],
|
||||
ContentEncoding: fi.Metadata["content-encoding"],
|
||||
}
|
||||
// Update expires
|
||||
var (
|
||||
t time.Time
|
||||
e error
|
||||
)
|
||||
if exp, ok := fi.Metadata["expires"]; ok {
|
||||
if t, e = time.Parse(http.TimeFormat, exp); e == nil {
|
||||
objInfo.Expires = t.UTC()
|
||||
}
|
||||
}
|
||||
objInfo.backendType = BackendErasure
|
||||
|
||||
// Extract etag from metadata.
|
||||
objInfo.ETag = extractETag(fi.Metadata)
|
||||
|
||||
// Add user tags to the object info
|
||||
objInfo.UserTags = fi.Metadata[xhttp.AmzObjectTagging]
|
||||
|
||||
// etag/md5Sum has already been extracted. We need to
|
||||
// remove to avoid it from appearing as part of
|
||||
// response headers. e.g, X-Minio-* or X-Amz-*.
|
||||
// Tags have also been extracted, we remove that as well.
|
||||
objInfo.UserDefined = cleanMetadata(fi.Metadata)
|
||||
|
||||
// All the parts per object.
|
||||
objInfo.Parts = fi.Parts
|
||||
|
||||
// Update storage class
|
||||
if sc, ok := fi.Metadata[xhttp.AmzStorageClass]; ok {
|
||||
objInfo.StorageClass = sc
|
||||
} else {
|
||||
objInfo.StorageClass = globalMinioDefaultStorageClass
|
||||
}
|
||||
|
||||
// Success.
|
||||
return objInfo
|
||||
}
|
||||
|
||||
// objectPartIndex - returns the index of matching object part number.
|
||||
func objectPartIndex(parts []ObjectPartInfo, partNumber int) int {
|
||||
for i, part := range parts {
|
||||
if partNumber == part.Number {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// AddObjectPart - add a new object part in order.
|
||||
func (fi *FileInfo) AddObjectPart(partNumber int, partETag string, partSize int64, actualSize int64) {
|
||||
partInfo := ObjectPartInfo{
|
||||
Number: partNumber,
|
||||
ETag: partETag,
|
||||
Size: partSize,
|
||||
ActualSize: actualSize,
|
||||
}
|
||||
|
||||
// Update part info if it already exists.
|
||||
for i, part := range fi.Parts {
|
||||
if partNumber == part.Number {
|
||||
fi.Parts[i] = partInfo
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Proceed to include new part info.
|
||||
fi.Parts = append(fi.Parts, partInfo)
|
||||
|
||||
// Parts in FileInfo should be in sorted order by part number.
|
||||
sort.Sort(byObjectPartNumber(fi.Parts))
|
||||
}
|
||||
|
||||
// ObjectToPartOffset - translate offset of an object to offset of its individual part.
|
||||
func (fi FileInfo) ObjectToPartOffset(ctx context.Context, offset int64) (partIndex int, partOffset int64, err error) {
|
||||
if offset == 0 {
|
||||
// Special case - if offset is 0, then partIndex and partOffset are always 0.
|
||||
return 0, 0, nil
|
||||
}
|
||||
partOffset = offset
|
||||
// Seek until object offset maps to a particular part offset.
|
||||
for i, part := range fi.Parts {
|
||||
partIndex = i
|
||||
// Offset is smaller than size we have reached the proper part offset.
|
||||
if partOffset < part.Size {
|
||||
return partIndex, partOffset, nil
|
||||
}
|
||||
// Continue to towards the next part.
|
||||
partOffset -= part.Size
|
||||
}
|
||||
logger.LogIf(ctx, InvalidRange{})
|
||||
// Offset beyond the size of the object return InvalidRange.
|
||||
return 0, 0, InvalidRange{}
|
||||
}
|
||||
|
||||
func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.Time, quorum int) (xmv FileInfo, e error) {
|
||||
metaHashes := make([]string, len(metaArr))
|
||||
for i, meta := range metaArr {
|
||||
if meta.IsValid() && meta.ModTime.Equal(modTime) {
|
||||
h := sha256.New()
|
||||
for _, part := range meta.Parts {
|
||||
h.Write([]byte(fmt.Sprintf("part.%d", part.Number)))
|
||||
}
|
||||
metaHashes[i] = hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
}
|
||||
|
||||
metaHashCountMap := make(map[string]int)
|
||||
for _, hash := range metaHashes {
|
||||
if hash == "" {
|
||||
continue
|
||||
}
|
||||
metaHashCountMap[hash]++
|
||||
}
|
||||
|
||||
maxHash := ""
|
||||
maxCount := 0
|
||||
for hash, count := range metaHashCountMap {
|
||||
if count > maxCount {
|
||||
maxCount = count
|
||||
maxHash = hash
|
||||
}
|
||||
}
|
||||
|
||||
if maxCount < quorum {
|
||||
return FileInfo{}, errErasureReadQuorum
|
||||
}
|
||||
|
||||
for i, hash := range metaHashes {
|
||||
if hash == maxHash {
|
||||
return metaArr[i], nil
|
||||
}
|
||||
}
|
||||
|
||||
return FileInfo{}, errErasureReadQuorum
|
||||
}
|
||||
|
||||
// pickValidFileInfo - picks one valid FileInfo content and returns from a
|
||||
// slice of FileInfo.
|
||||
func pickValidFileInfo(ctx context.Context, metaArr []FileInfo, modTime time.Time, quorum int) (xmv FileInfo, e error) {
|
||||
return findFileInfoInQuorum(ctx, metaArr, modTime, quorum)
|
||||
}
|
||||
|
||||
// Rename metadata content to destination location for each disk concurrently.
|
||||
func renameFileInfo(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, quorum int) ([]StorageAPI, error) {
|
||||
ignoredErr := []error{errFileNotFound}
|
||||
|
||||
g := errgroup.WithNErrs(len(disks))
|
||||
|
||||
// Rename file on all underlying storage disks.
|
||||
for index := range disks {
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
if disks[index] == nil {
|
||||
return errDiskNotFound
|
||||
}
|
||||
if err := disks[index].RenameData(srcBucket, srcEntry, "", dstBucket, dstEntry); err != nil {
|
||||
if !IsErrIgnored(err, ignoredErr...) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, index)
|
||||
}
|
||||
|
||||
// Wait for all renames to finish.
|
||||
errs := g.Wait()
|
||||
|
||||
// We can safely allow RenameData errors up to len(er.getDisks()) - writeQuorum
|
||||
// otherwise return failure. Cleanup successful renames.
|
||||
err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, quorum)
|
||||
return evalDisks(disks, errs), err
|
||||
}
|
||||
|
||||
// writeUniqueFileInfo - writes unique `xl.meta` content for each disk concurrently.
|
||||
func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, bucket, prefix string, files []FileInfo, quorum int) ([]StorageAPI, error) {
|
||||
g := errgroup.WithNErrs(len(disks))
|
||||
|
||||
// Start writing `xl.meta` to all disks in parallel.
|
||||
for index := range disks {
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
if disks[index] == nil {
|
||||
return errDiskNotFound
|
||||
}
|
||||
// Pick one FileInfo for a disk at index.
|
||||
files[index].Erasure.Index = index + 1
|
||||
return disks[index].WriteMetadata(bucket, prefix, files[index])
|
||||
}, index)
|
||||
}
|
||||
|
||||
// Wait for all the routines.
|
||||
mErrs := g.Wait()
|
||||
|
||||
err := reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, quorum)
|
||||
return evalDisks(disks, mErrs), err
|
||||
}
|
||||
|
||||
// Returns per object readQuorum and writeQuorum
|
||||
// readQuorum is the min required disks to read data.
|
||||
// writeQuorum is the min required disks to write data.
|
||||
func objectQuorumFromMeta(ctx context.Context, er erasureObjects, partsMetaData []FileInfo, errs []error) (objectReadQuorum, objectWriteQuorum int, err error) {
|
||||
// get the latest updated Metadata and a count of all the latest updated FileInfo(s)
|
||||
latestFileInfo, err := getLatestFileInfo(ctx, partsMetaData, errs)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
// Since all the valid erasure code meta updated at the same time are equivalent, pass dataBlocks
|
||||
// from latestFileInfo to get the quorum
|
||||
return latestFileInfo.Erasure.DataBlocks, latestFileInfo.Erasure.DataBlocks + 1, nil
|
||||
}
|
||||
@@ -1,153 +0,0 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2015, 2016, 2017 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
)
|
||||
|
||||
const ActualSize = 1000
|
||||
|
||||
// Test FileInfo.AddObjectPart()
|
||||
func TestAddObjectPart(t *testing.T) {
|
||||
testCases := []struct {
|
||||
partNum int
|
||||
expectedIndex int
|
||||
}{
|
||||
{1, 0},
|
||||
{2, 1},
|
||||
{4, 2},
|
||||
{5, 3},
|
||||
{7, 4},
|
||||
// Insert part.
|
||||
{3, 2},
|
||||
// Replace existing part.
|
||||
{4, 3},
|
||||
// Missing part.
|
||||
{6, -1},
|
||||
}
|
||||
|
||||
// Setup.
|
||||
fi := newFileInfo("test-object", 8, 8)
|
||||
if !fi.IsValid() {
|
||||
t.Fatalf("unable to get xl meta")
|
||||
}
|
||||
|
||||
// Test them.
|
||||
for _, testCase := range testCases {
|
||||
if testCase.expectedIndex > -1 {
|
||||
partNumString := strconv.Itoa(testCase.partNum)
|
||||
fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize)
|
||||
}
|
||||
|
||||
if index := objectPartIndex(fi.Parts, testCase.partNum); index != testCase.expectedIndex {
|
||||
t.Fatalf("%+v: expected = %d, got: %d", testCase, testCase.expectedIndex, index)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test objectPartIndex(). generates a sample FileInfo data and asserts
|
||||
// the output of objectPartIndex() with the expected value.
|
||||
func TestObjectPartIndex(t *testing.T) {
|
||||
testCases := []struct {
|
||||
partNum int
|
||||
expectedIndex int
|
||||
}{
|
||||
{2, 1},
|
||||
{1, 0},
|
||||
{5, 3},
|
||||
{4, 2},
|
||||
{7, 4},
|
||||
}
|
||||
|
||||
// Setup.
|
||||
fi := newFileInfo("test-object", 8, 8)
|
||||
if !fi.IsValid() {
|
||||
t.Fatalf("unable to get xl meta")
|
||||
}
|
||||
|
||||
// Add some parts for testing.
|
||||
for _, testCase := range testCases {
|
||||
partNumString := strconv.Itoa(testCase.partNum)
|
||||
fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize)
|
||||
}
|
||||
|
||||
// Add failure test case.
|
||||
testCases = append(testCases, struct {
|
||||
partNum int
|
||||
expectedIndex int
|
||||
}{6, -1})
|
||||
|
||||
// Test them.
|
||||
for _, testCase := range testCases {
|
||||
if index := objectPartIndex(fi.Parts, testCase.partNum); index != testCase.expectedIndex {
|
||||
t.Fatalf("%+v: expected = %d, got: %d", testCase, testCase.expectedIndex, index)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test FileInfo.ObjectToPartOffset().
|
||||
func TestObjectToPartOffset(t *testing.T) {
|
||||
// Setup.
|
||||
fi := newFileInfo("test-object", 8, 8)
|
||||
if !fi.IsValid() {
|
||||
t.Fatalf("unable to get xl meta")
|
||||
}
|
||||
|
||||
// Add some parts for testing.
|
||||
// Total size of all parts is 5,242,899 bytes.
|
||||
for _, partNum := range []int{1, 2, 4, 5, 7} {
|
||||
partNumString := strconv.Itoa(partNum)
|
||||
fi.AddObjectPart(partNum, "etag."+partNumString, int64(partNum+humanize.MiByte), ActualSize)
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
offset int64
|
||||
expectedIndex int
|
||||
expectedOffset int64
|
||||
expectedErr error
|
||||
}{
|
||||
{0, 0, 0, nil},
|
||||
{1 * humanize.MiByte, 0, 1 * humanize.MiByte, nil},
|
||||
{1 + humanize.MiByte, 1, 0, nil},
|
||||
{2 + humanize.MiByte, 1, 1, nil},
|
||||
// Its valid for zero sized object.
|
||||
{-1, 0, -1, nil},
|
||||
// Max fffset is always (size - 1).
|
||||
{(1 + 2 + 4 + 5 + 7) + (5 * humanize.MiByte) - 1, 4, 1048582, nil},
|
||||
// Error if offset is size.
|
||||
{(1 + 2 + 4 + 5 + 7) + (5 * humanize.MiByte), 0, 0, InvalidRange{}},
|
||||
}
|
||||
|
||||
// Test them.
|
||||
for _, testCase := range testCases {
|
||||
index, offset, err := fi.ObjectToPartOffset(context.Background(), testCase.offset)
|
||||
if err != testCase.expectedErr {
|
||||
t.Fatalf("%+v: expected = %s, got: %s", testCase, testCase.expectedErr, err)
|
||||
}
|
||||
if index != testCase.expectedIndex {
|
||||
t.Fatalf("%+v: index: expected = %d, got: %d", testCase, testCase.expectedIndex, index)
|
||||
}
|
||||
if offset != testCase.expectedOffset {
|
||||
t.Fatalf("%+v: offset: expected = %d, got: %d", testCase, testCase.expectedOffset, offset)
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user