fix: avoid broken link when preview image (#10021)

closes #9589

.github/ISSUE_TEMPLATE.md

@@ -1,3 +1,12 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: community, triage
+assignees: ''
+
+---
+
 <!--- Provide a general summary of the issue in the Title above -->
 
 ## Expected Behavior
@@ -15,6 +24,8 @@
 ## Steps to Reproduce (for bugs)
 <!--- Provide a link to a live example, or an unambiguous set of steps to -->
 <!--- reproduce this bug. Include code to reproduce, if relevant -->
+<!--- and make sure you have followed https://github.com/minio/minio/tree/release/docs/debugging to capture relevant logs -->
+
 1.
 2.
 3.
@@ -30,8 +41,6 @@
 
 ## Your Environment
 <!--- Include as many relevant details about the environment you experienced the bug in -->
-* Version used (`minio version`):
-* Environment name and version (e.g. nginx 1.9.1):
-* Server type and version:
+* Version used (`minio --version`):
+* Server setup and configuration:
 * Operating System and version (`uname -a`):
-* Link to your project:

.github/ISSUE_TEMPLATE/bug_report.md

@@ -24,6 +24,8 @@ assignees: ''
 ## Steps to Reproduce (for bugs)
 <!--- Provide a link to a live example, or an unambiguous set of steps to -->
 <!--- reproduce this bug. Include code to reproduce, if relevant -->
+<!--- and make sure you have followed https://github.com/minio/minio/tree/release/docs/debugging to capture relevant logs -->
+
 1.
 2.
 3.
@@ -39,8 +41,6 @@ assignees: ''
 
 ## Your Environment
 <!--- Include as many relevant details about the environment you experienced the bug in -->
-* Version used (`minio version`):
-* Environment name and version (e.g. nginx 1.9.1):
-* Server type and version:
+* Version used (`minio --version`):
+* Server setup and configuration:
 * Operating System and version (`uname -a`):
-* Link to your project:

.github/ISSUE_TEMPLATE/config.yml

@@ -2,7 +2,7 @@ blank_issues_enabled: false
 contact_links:
   - name: MinIO Community Support
     url: https://slack.min.io
-    about: Please ask and answer questions here.
+    about: Join here for Community Support
   - name: MinIO SUBNET Support
     url: https://min.io/pricing
-    about: Join this for Enterprise Support.
+    about: Join here for Enterprise Support

.github/PULL_REQUEST_TEMPLATE.md

@@ -16,4 +16,3 @@
 - [ ] Fixes a regression (If yes, please add `commit-id` or `PR #` here)
 - [ ] Documentation needed
 - [ ] Unit tests needed
-- [ ] Functional tests needed (If yes, add [mint](https://github.com/minio/mint) PR # here: )

.github/lock.yml

@@ -11,7 +11,7 @@ skipCreatedBefore: false
 exemptLabels: []
 
 # Label to add before locking, such as `outdated`. Set to `false` to disable
-lockLabel: false
+lockLabel: true
 
 # Comment to post before locking. Set to `false` to disable
 lockComment: >-

.github/stale.yml

@@ -1,11 +1,11 @@
 # Configuration for probot-stale - https://github.com/probot/stale
 
 # Number of days of inactivity before an Issue or Pull Request becomes stale
-daysUntilStale: 90
+daysUntilStale: 30
 
 # Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
 # Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
-daysUntilClose: 30
+daysUntilClose: 15
 
 # Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
 onlyLabels: []
@@ -30,7 +30,7 @@ staleLabel: stale
 # Comment to post when marking as stale. Set to `false` to disable
 markComment: >-
   This issue has been automatically marked as stale because it has not had
-  recent activity. It will be closed after 21 days if no further activity
+  recent activity. It will be closed after 15 days if no further activity
   occurs. Thank you for your contributions.
 # Comment to post when removing the stale label.
 # unmarkComment: >

.github/workflows/go.yml

@@ -4,9 +4,7 @@ on:
   pull_request:
     branches:
     - master
-  push:
-    branches:
-    - master
+    - release
 
 jobs:
   build:

.goreleaser.yml

@@ -0,0 +1,167 @@
+project_name: minio
+
+release:
+   name_template: "Version {{.MinIO.Version}}"
+   disable: true
+   github:
+    owner: minio
+    name: minio
+
+env:
+  - CGO_ENABLED=0
+  - GO111MODULE=on
+
+before:
+  hooks:
+    - make clean
+    - go generate ./...
+    - go mod tidy
+    - go mod download
+
+builds:
+  -
+    goos:
+      - linux
+      - darwin
+      - windows
+      - freebsd
+
+    goarch:
+      - amd64
+      - arm64
+      - arm
+      - ppc64le
+      - s390x
+
+    goarm:
+      - 7
+
+    ignore:
+      - goos: darwin
+        goarch: arm64
+      - goos: darwin
+        goarch: arm
+      - goos: darwin
+        goarch: ppc64le
+      - goos: darwin
+        goarch: s390x
+      - goos: windows
+        goarch: arm64
+      - goos: windows
+        goarch: arm
+      - goos: windows
+        goarch: ppc64le
+      - goos: windows
+        goarch: s390x
+      - goos: freebsd
+        goarch: arm
+      - goos: freebsd
+        goarch: arm64
+      - goos: freebsd
+        goarch: ppc64le
+      - goos: freebsd
+        goarch: s390x
+
+    flags:
+      - -tags=kqueue
+      - -trimpath
+
+    ldflags:
+      - "-s -w -X github.com/minio/minio/cmd.Version={{.Version}} -X github.com/minio/minio/cmd.ReleaseTag={{.Tag}} -X github.com/minio/minio/cmd.CommitID={{.FullCommit}} -X github.com/minio/minio/cmd.ShortCommitID={{.ShortCommit}}"
+
+archives:
+  -
+    format: binary
+    name_template: "{{ .Binary }}-release/{{ .Os }}-{{ .Arch }}/{{ .Binary }}.{{ .Version }}"
+
+nfpms:
+  -
+    id: minio
+    package_name: minio
+    vendor: MinIO, Inc.
+    homepage: https://min.io/
+    maintainer: dev@min.io
+    description: MinIO is a High Performance Object Storage released under Apache License v2.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads.
+    license: Apache 2.0
+    bindir: /usr/bin
+    formats:
+      - deb
+      - rpm
+    overrides:
+      deb:
+        file_name_template: "{{ .Binary }}-release/debs/{{ .ProjectName }}-{{ .Version }}_{{ .Arch }}"
+        replacements:
+          arm: armv7
+        files:
+          "NOTICE": "/usr/share/minio/NOTICE"
+          "CREDITS": "/usr/share/minio/CREDITS"
+          "LICENSE": "/usr/share/minio/LICENSE"
+          "README.md": "/usr/share/minio/README.md"
+      rpm:
+        file_name_template: "{{ .Binary }}-release/rpms/{{ .ProjectName }}-{{ .Version }}.{{ .Arch }}"
+        replacements:
+          amd64: x86_64
+          arm64: aarch64
+          arm: armv7
+        files:
+          "NOTICE": "/usr/share/minio/NOTICE"
+          "CREDITS": "/usr/share/minio/CREDITS"
+          "LICENSE": "/usr/share/minio/LICENSE"
+          "README.md": "/usr/share/minio/README.md"
+
+checksum:
+  algorithm: sha256
+
+signs:
+  -
+    signature: "${artifact}.asc"
+    cmd: "sh"
+    args:
+      - '-c'
+      - 'gpg --quiet --detach-sign -a ${artifact}'
+    artifacts: all
+
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^Update yaml files'
+
+dockers:
+  -
+    goos: linux
+    goarch: amd64
+    dockerfile: Dockerfile.release
+    image_templates:
+      - minio/minio:{{ .Tag }}
+      - minio/minio:latest
+
+  -
+    goos: linux
+    goarch: ppc64le
+    dockerfile: Dockerfile.ppc64le.release
+    image_templates:
+      - minio/minio:{{ .Tag }}-ppc64le
+
+  -
+    goos: linux
+    goarch: s390x
+    dockerfile: Dockerfile.s390x.release
+    image_templates:
+      - minio/minio:{{ .Tag }}-s390x
+
+  -
+    goos: linux
+    goarch: arm64
+    goarm: ''
+    dockerfile: Dockerfile.arm64.release
+    image_templates:
+      - minio/minio:{{ .Tag }}-arm64
+
+  -
+    goos: linux
+    goarch: arm
+    goarm: '7'
+    dockerfile: Dockerfile.arm.release
+    image_templates:
+      - minio/minio:{{ .Tag }}-arm

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13-alpine
+FROM golang:1.13-alpine as builder
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
@@ -9,9 +9,9 @@ ENV GO111MODULE on
 RUN  \
      apk add --no-cache git && \
      git clone https://github.com/minio/minio && cd minio && \
-     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
+     git checkout master && go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
 
-FROM alpine:3.10
+FROM alpine:3.12
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
@@ -21,9 +21,9 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 
 EXPOSE 9000
 
-COPY --from=0 /go/bin/minio /usr/bin/minio
-COPY --from=0 /go/minio/CREDITS /third_party/
-COPY --from=0 /go/minio/dockerscripts/docker-entrypoint.sh /usr/bin/
+COPY --from=builder /go/bin/minio /usr/bin/minio
+COPY --from=builder /go/minio/CREDITS /third_party/
+COPY --from=builder /go/minio/dockerscripts/docker-entrypoint.sh /usr/bin/
 
 RUN  \
      apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \

Dockerfile.arm.release

@@ -1,23 +1,9 @@
-FROM golang:1.13-alpine as builder
-
-WORKDIR /home
-
-ENV GOPATH /go
-ENV CGO_ENABLED 0
-ENV GO111MODULE on
-
-RUN  \
-     apk add --no-cache git 'curl>7.61.0' && \
-     git clone https://github.com/minio/minio && \
-     curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static . 
-
-FROM arm32v7/alpine:3.10
+FROM multiarch/qemu-user-static:x86_64-arm as qemu
+FROM arm32v7/alpine:3.12
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
-COPY dockerscripts/docker-entrypoint.sh /usr/bin/
-COPY CREDITS /third_party/
-COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static
+COPY --from=qemu /usr/bin/qemu-arm-static /usr/bin
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
@@ -28,9 +14,11 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 RUN \
      apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
-     curl https://dl.min.io/server/minio/release/linux-arm/minio > /usr/bin/minio && \
-     chmod +x /usr/bin/minio  && \
-     chmod +x /usr/bin/docker-entrypoint.sh
+     curl -s -q https://dl.min.io/server/minio/release/linux-arm/minio -o /usr/bin/minio && \
+     curl -s -q https://raw.githubusercontent.com/minio/minio/release/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
+     chmod +x /usr/bin/minio && \
+     chmod +x /usr/bin/docker-entrypoint.sh && \
+     curl -s -q -O https://raw.githubusercontent.com/minio/minio/release/CREDITS
 
 EXPOSE 9000
 

Dockerfile.arm64.release

@@ -1,23 +1,9 @@
-FROM golang:1.13-alpine as builder
-
-WORKDIR /home
-
-ENV GOPATH /go
-ENV CGO_ENABLED 0
-ENV GO111MODULE on
-
-RUN  \
-     apk add --no-cache git 'curl>7.61.0' && \
-     git clone https://github.com/minio/minio && \
-     curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static . 
-
-FROM arm64v8/alpine:3.10
+FROM multiarch/qemu-user-static:x86_64-aarch64 as qemu
+FROM arm64v8/alpine:3.12
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
-COPY dockerscripts/docker-entrypoint.sh /usr/bin/
-COPY CREDITS /third_party/
-COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static
+COPY --from=qemu /usr/bin/qemu-aarch64-static /usr/bin
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
@@ -28,9 +14,11 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 RUN \
      apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
-     curl https://dl.min.io/server/minio/release/linux-arm64/minio > /usr/bin/minio && \
-     chmod +x /usr/bin/minio  && \
-     chmod +x /usr/bin/docker-entrypoint.sh
+     curl -s -q https://dl.min.io/server/minio/release/linux-arm64/minio -o /usr/bin/minio && \
+     curl -s -q https://raw.githubusercontent.com/minio/minio/release/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
+     chmod +x /usr/bin/minio && \
+     chmod +x /usr/bin/docker-entrypoint.sh && \
+     curl -s -q -O https://raw.githubusercontent.com/minio/minio/release/CREDITS
 
 EXPOSE 9000
 

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM alpine:3.10
+FROM alpine:3.12
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 

Dockerfile.dev.browser

@@ -10,4 +10,3 @@ ENV PATH=$PATH:/root/go/bin
 
 RUN go get github.com/go-bindata/go-bindata/go-bindata && \
     go get github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs
-

Dockerfile.ppc64le.release

@@ -0,0 +1,29 @@
+FROM multiarch/qemu-user-static:x86_64-ppc64le as qemu
+FROM ppc64le/alpine:3.12
+
+LABEL maintainer="MinIO Inc <dev@min.io>"
+
+COPY --from=qemu /usr/bin/qemu-ppc64le-static /usr/bin
+
+ENV MINIO_UPDATE off
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+
+RUN \
+     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-ppc64le/minio -o /usr/bin/minio && \
+     curl -s -q https://raw.githubusercontent.com/minio/minio/release/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
+     chmod +x /usr/bin/minio && \
+     chmod +x /usr/bin/docker-entrypoint.sh && \
+     curl -s -q -O https://raw.githubusercontent.com/minio/minio/release/CREDITS
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/data"]
+
+CMD ["minio"]

Dockerfile.release

@@ -1,20 +1,7 @@
-FROM golang:1.13-alpine
-
-ENV GOPATH /go
-ENV CGO_ENABLED 0
-ENV GO111MODULE on
-
-RUN  \
-     apk add --no-cache git && \
-     git clone https://github.com/minio/minio
-
-FROM alpine:3.10
+FROM alpine:3.12
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
-COPY dockerscripts/docker-entrypoint.sh /usr/bin/
-COPY CREDITS /third_party/
-
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
@@ -24,9 +11,11 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 RUN \
      apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
-     curl https://dl.min.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \
-     chmod +x /usr/bin/minio  && \
-     chmod +x /usr/bin/docker-entrypoint.sh
+     curl -s -q https://dl.min.io/server/minio/release/linux-amd64/minio -o /usr/bin/minio && \
+     curl -s -q https://raw.githubusercontent.com/minio/minio/release/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
+     chmod +x /usr/bin/minio && \
+     chmod +x /usr/bin/docker-entrypoint.sh && \
+     curl -s -q -O https://raw.githubusercontent.com/minio/minio/release/CREDITS
 
 EXPOSE 9000
 

Dockerfile.s390x.release

@@ -0,0 +1,29 @@
+FROM multiarch/qemu-user-static:x86_64-s390x as qemu
+FROM s390x/alpine:3.12
+
+LABEL maintainer="MinIO Inc <dev@min.io>"
+
+COPY --from=qemu /usr/bin/qemu-s390x-static /usr/bin
+
+ENV MINIO_UPDATE off
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+
+RUN \
+     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-s390x/minio -o /usr/bin/minio && \
+     curl -s -q https://raw.githubusercontent.com/minio/minio/release/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
+     chmod +x /usr/bin/minio && \
+     chmod +x /usr/bin/docker-entrypoint.sh && \
+     curl -s -q -O https://raw.githubusercontent.com/minio/minio/release/CREDITS
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/data"]
+
+CMD ["minio"]

Makefile

@@ -60,7 +60,9 @@ build: checks
 	@echo "Building minio binary to './minio'"
 	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 
-docker: build
+docker: checks
+	@echo "Building minio docker image '$(TAG)'"
+	@GOOS=linux GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@docker build -t $(TAG) . -f Dockerfile.dev
 
 # Builds minio and installs it to $GOPATH/bin.

README.md

@@ -8,14 +8,18 @@ MinIO is a High Performance Object Storage released under Apache License v2.0. I
 ## Docker Container
 ### Stable
 ```
-docker pull minio/minio
-docker run -p 9000:9000 minio/minio server /data
+docker run -p 9000:9000 \
+  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
+  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
+  minio/minio server /data
 ```
 
 ### Edge
 ```
-docker pull minio/minio:edge
-docker run -p 9000:9000 minio/minio:edge server /data
+docker run -p 9000:9000 \
+  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
+  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
+  minio/minio:edge server /data
 ```
 
 > NOTE: Docker will not display the default keys unless you start the container with the `-it`(interactive TTY) argument. Generally, it is not recommended to use default keys with containers. Please visit MinIO Docker quickstart guide for more information [here](https://docs.min.io/docs/minio-docker-quickstart-guide)
@@ -94,23 +98,6 @@ GO111MODULE=on go get github.com/minio/minio
 
 By default MinIO uses the port 9000 to listen for incoming connections. If your platform blocks the port by default, you may need to enable access to the port.
 
-### iptables
-
-For hosts with iptables enabled (RHEL, CentOS, etc), you can use `iptables` command to enable all traffic coming to specific ports. Use below command to allow
-access to port 9000
-
-```sh
-iptables -A INPUT -p tcp --dport 9000 -j ACCEPT
-service iptables restart
-```
-
-Below command enables all incoming traffic to ports ranging from 9000 to 9010.
-
-```sh
-iptables -A INPUT -p tcp --dport 9000:9010 -j ACCEPT
-service iptables restart
-```
-
 ### ufw
 
 For hosts with ufw enabled (Debian based distros), you can use `ufw` command to allow traffic to specific ports. Use below command to allow access to port 9000
@@ -145,6 +132,23 @@ Note that `permanent` makes sure the rules are persistent across firewall start,
 firewall-cmd --reload
 ```
 
+### iptables
+
+For hosts with iptables enabled (RHEL, CentOS, etc), you can use `iptables` command to enable all traffic coming to specific ports. Use below command to allow
+access to port 9000
+
+```sh
+iptables -A INPUT -p tcp --dport 9000 -j ACCEPT
+service iptables restart
+```
+
+Below command enables all incoming traffic to ports ranging from 9000 to 9010.
+
+```sh
+iptables -A INPUT -p tcp --dport 9000:9010 -j ACCEPT
+service iptables restart
+```
+
 ## Test using MinIO Browser
 MinIO Server comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 ensure your server has started successfully.
 
@@ -159,20 +163,23 @@ When deployed on a single drive, MinIO server lets clients access any pre-existi
 The above statement is also valid for all gateway backends.
 
 ## Upgrading MinIO
-MinIO server supports rolling upgrades, i.e. you can update one MinIO instance at a time in a distributed cluster. This allows upgrades with no downtime. Upgrades can be done manually by replacing the binary with the latest release and restarting all servers in a rolling fashion. However, we recommend all our users to use [`mc admin update`](https://docs.min.io/docs/minio-admin-complete-guide.html#update) from the client. This will update all the nodes in the cluster and restart them, as shown in the following command from the MinIO client (mc):
+MinIO server supports rolling upgrades, i.e. you can update one MinIO instance at a time in a distributed cluster. This allows upgrades with no downtime. Upgrades can be done manually by replacing the binary with the latest release and restarting all servers in a rolling fashion. However, we recommend all our users to use [`mc admin update`](https://docs.min.io/docs/minio-admin-complete-guide.html#update) from the client. This will update all the nodes in the cluster simultaneously and restart them, as shown in the following command from the MinIO client (mc):
 
 ```
 mc admin update <minio alias, e.g., myminio>
 ```
 
-**Important things to remember during upgrades**:
+> NOTE: some releases might not allow rolling upgrades, this is always called out in the release notes and it is generally advised to read release notes before upgrading. In such a situation `mc admin update` is the recommended upgrading mechanism to upgrade all servers at once.
+
+### Important things to remember during MinIO upgrades
 
 - `mc admin update` will only work if the user running MinIO has write access to the parent directory where the binary is located, for example if the current binary is at `/usr/local/bin/minio`, you would need write access to `/usr/local/bin`.
-- In the case of federated setups `mc admin update` should be run against each cluster individually. Avoid updating `mc` until all clusters have been updated.
-- If you are updating the server it is always recommended (unless explicitly mentioned in MinIO server release notes), to update `mc` once all the servers have been upgraded using `mc update`.
-- `mc admin update` is disabled in docker/container environments, container environments provide their own mechanisms for updating running containers.
-- If you are using Vault as KMS with MinIO, ensure you have followed the Vault upgrade procedure outlined here: https://www.vaultproject.io/docs/upgrading/index.html
-- If you are using etcd with MinIO for the federation, ensure you have followed the etcd upgrade procedure outlined here: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md
+- `mc admin update` updates and restarts all servers simultaneously, applications would retry and continue their respective operations upon upgrade.
+- `mc admin update` is disabled in kubernetes/container environments, container environments provide their own mechanisms to rollout of updates.
+- In the case of federated setups `mc admin update` should be run against each cluster individually. Avoid updating `mc` to any new releases until all clusters have been successfully updated.
+- If using `kes` as KMS with MinIO, just replace the binary and restart `kes` more information about `kes` can be found [here](https://github.com/minio/kes/wiki)x
+- If using Vault as KMS with MinIO, ensure you have followed the Vault upgrade procedure outlined here: https://www.vaultproject.io/docs/upgrading/index.html
+- If using etcd with MinIO for the federation, ensure you have followed the etcd upgrade procedure outlined here: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md
 
 ## Explore Further
 - [MinIO Erasure Code QuickStart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)

browser/app/js/objects/ObjectItem.js

@@ -54,8 +54,11 @@ export const ObjectItem = ({
           href={getDataType(name, contentType) === "folder" ? name : "#"}
           onClick={e => {
             e.preventDefault()
+            // onclick function is passed only when we have a prefix
             if (onClick) {
               onClick()
+            } else {
+              checked ? uncheckObject(name) : checkObject(name)
             }
           }}
         >

browser/app/js/objects/PreviewObjectModal.js

@@ -46,11 +46,11 @@ class PreviewObjectModal extends React.Component {
         <ModalBody>
           <div className="input-group">
             {this.state.url && (
-              <img
-                alt="Image broken"
-                src={this.state.url}
-                style={{ display: "block", width: "100%" }}
-              />
+              <object data={this.state.url}>
+                <h3 style={{ textAlign: "center", display: "block", width: "100%" }}>
+                  Do not have read permissions to preview "{this.props.object.name}"
+                </h3>
+              </object>
             )}
           </div>
         </ModalBody>

browser/app/js/objects/__tests__/ObjectItem.test.js

@@ -30,7 +30,10 @@ describe("ObjectItem", () => {
 
   it("shouldn't call onClick when the object isclicked", () => {
     const onClick = jest.fn()
-    const wrapper = shallow(<ObjectItem name={"test"} />)
+    const checkObject = jest.fn()
+    const wrapper = shallow(
+      <ObjectItem name={"test"} checkObject={checkObject} />
+    )
     wrapper.find("a").simulate("click", { preventDefault: jest.fn() })
     expect(onClick).not.toHaveBeenCalled()
   })
@@ -57,9 +60,15 @@ describe("ObjectItem", () => {
   })
 
   it("should call uncheckObject when the object/prefix is unchecked", () => {
+    const checkObject = jest.fn()
     const uncheckObject = jest.fn()
     const wrapper = shallow(
-      <ObjectItem name={"test"} checked={true} uncheckObject={uncheckObject} />
+      <ObjectItem
+        name={"test"}
+        checked={true}
+        checkObject={checkObject}
+        uncheckObject={uncheckObject}
+      />
     )
     wrapper.find("input[type='checkbox']").simulate("change")
     expect(uncheckObject).toHaveBeenCalledWith("test")

browser/app/less/inc/list.less

@@ -105,7 +105,7 @@ div.fesl-row {
 
     .fesl-item-name {
         a {
-            cursor: default;
+            cursor: pointer;
         }
     }
 
@@ -114,12 +114,6 @@ div.fesl-row {
     ----------------------------*/
     &[data-type=folder] {
         .list-type(#a1d6dd, '\f07b');
-
-        .fesl-item-name {
-            a {
-                cursor: pointer;
-            }
-        }
     }
     &[data-type=pdf] {.list-type(#fa7775, '\f1c1'); }
     &[data-type=zip] { .list-type(#427089, '\f1c6'); }

buildscripts/cross-compile.sh

@@ -9,7 +9,7 @@ function _init() {
     export CGO_ENABLED=0
 
     ## List of architectures and OS to test coss compilation.
-    SUPPORTED_OSARCH="linux/ppc64le linux/arm64 linux/s390x darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386"
+    SUPPORTED_OSARCH="linux/ppc64le linux/arm64 linux/s390x darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64"
 }
 
 function _build() {

buildscripts/verify-healing.sh

@@ -33,6 +33,7 @@ function start_minio_3_node() {
     declare -a ARGS
     export MINIO_ACCESS_KEY=minio
     export MINIO_SECRET_KEY=minio123
+    export MINIO_ERASURE_SET_DRIVE_COUNT=6
 
     start_port=$(shuf -i 10000-65000 -n 1)
     for i in $(seq 1 3); do

cmd/admin-handlers-config-kv.go

@@ -173,7 +173,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	}
 
 	cfg := globalServerConfig
-	if globalSafeMode {
+	if newObjectLayerFn() == nil {
 		var err error
 		cfg, err = getValidConfig(objectAPI)
 		if err != nil {

cmd/admin-handlers.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016-2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -328,7 +328,7 @@ func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Re
 	writeSuccessResponseJSON(w, dataUsageInfoJSON)
 }
 
-func newLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntry {
+func lriToLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntry {
 	entry := &madmin.LockEntry{
 		Timestamp:  l.Timestamp,
 		Resource:   resource,
@@ -337,14 +337,14 @@ func newLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntr
 		ID:         l.UID,
 	}
 	if l.Writer {
-		entry.Type = "Write"
+		entry.Type = "WRITE"
 	} else {
-		entry.Type = "Read"
+		entry.Type = "READ"
 	}
 	return entry
 }
 
-func topLockEntries(peerLocks []*PeerLocks) madmin.LockEntries {
+func topLockEntries(peerLocks []*PeerLocks, count int) madmin.LockEntries {
 	entryMap := make(map[string]*madmin.LockEntry)
 	for _, peerLock := range peerLocks {
 		if peerLock == nil {
@@ -356,20 +356,19 @@ func topLockEntries(peerLocks []*PeerLocks) madmin.LockEntries {
 					if val, ok := entryMap[lockReqInfo.UID]; ok {
 						val.ServerList = append(val.ServerList, peerLock.Addr)
 					} else {
-						entryMap[lockReqInfo.UID] = newLockEntry(lockReqInfo, k, peerLock.Addr)
+						entryMap[lockReqInfo.UID] = lriToLockEntry(lockReqInfo, k, peerLock.Addr)
 					}
 				}
 			}
 		}
 	}
-	var lockEntries = make(madmin.LockEntries, 0)
+	var lockEntries = make(madmin.LockEntries, 0, len(entryMap))
 	for _, v := range entryMap {
 		lockEntries = append(lockEntries, *v)
 	}
 	sort.Sort(lockEntries)
-	const listCount int = 10
-	if len(lockEntries) > listCount {
-		lockEntries = lockEntries[:listCount]
+	if len(lockEntries) > count {
+		lockEntries = lockEntries[:count]
 	}
 	return lockEntries
 }
@@ -391,6 +390,16 @@ func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request
 		return
 	}
 
+	count := 10 // by default list only top 10 entries
+	if countStr := r.URL.Query().Get("count"); countStr != "" {
+		var err error
+		count, err = strconv.Atoi(countStr)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+
 	peerLocks := globalNotificationSys.GetLocks(ctx)
 	// Once we have received all the locks currently used from peers
 	// add the local peer locks list as well.
@@ -398,12 +407,13 @@ func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request
 	for _, llocker := range globalLockServers {
 		getRespLocks = append(getRespLocks, llocker.DupLockMap())
 	}
+
 	peerLocks = append(peerLocks, &PeerLocks{
 		Addr:  getHostName(r),
 		Locks: getRespLocks,
 	})
 
-	topLocks := topLockEntries(peerLocks)
+	topLocks := topLockEntries(peerLocks, count)
 
 	// Marshal API response
 	jsonBytes, err := json.Marshal(topLocks)
@@ -631,7 +641,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Check if this setup has an erasure coded backend.
-	if !globalIsXL {
+	if !globalIsErasure {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL)
 		return
 	}
@@ -642,6 +652,16 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Analyze the heal token and route the request accordingly
+	token, success := proxyRequestByToken(ctx, w, r, hip.clientToken)
+	if success {
+		return
+	}
+	hip.clientToken = token
+	// if request was not successful, try this server locally if token
+	// is not found the call will fail anyways. if token is empty
+	// try this server to generate a new token.
+
 	type healResp struct {
 		respBytes []byte
 		apiErr    APIError
@@ -708,16 +728,16 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	// find number of disks in the setup, ignore any errors here.
-	info, _ := objectAPI.StorageInfo(ctx, false)
-	numDisks := info.Backend.OfflineDisks.Sum() + info.Backend.OnlineDisks.Sum()
-
 	healPath := pathJoin(hip.bucket, hip.objPrefix)
 	if hip.clientToken == "" && !hip.forceStart && !hip.forceStop {
 		nh, exists := globalAllHealState.getHealSequence(healPath)
 		if exists && !nh.hasEnded() && len(nh.currentStatus.Items) > 0 {
+			clientToken := nh.clientToken
+			if globalIsDistErasure {
+				clientToken = fmt.Sprintf("%s@%d", nh.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
+			}
 			b, err := json.Marshal(madmin.HealStartSuccess{
-				ClientToken:   nh.clientToken,
+				ClientToken:   clientToken,
 				ClientAddress: nh.clientAddress,
 				StartTime:     nh.startTime,
 			})
@@ -754,7 +774,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 			respCh <- hr
 		}()
 	case hip.clientToken == "":
-		nh := newHealSequence(GlobalContext, hip.bucket, hip.objPrefix, handlers.GetSourceIP(r), numDisks, hip.hs, hip.forceStart)
+		nh := newHealSequence(GlobalContext, hip.bucket, hip.objPrefix, handlers.GetSourceIP(r), hip.hs, hip.forceStart)
 		go func() {
 			respBytes, apiErr, errMsg := globalAllHealState.LaunchNewHealSequence(nh)
 			hr := healResp{respBytes, apiErr, errMsg}
@@ -779,7 +799,7 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 	}
 
 	// Check if this setup has an erasure coded backend.
-	if !globalIsXL {
+	if !globalIsErasure {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL)
 		return
 	}
@@ -789,7 +809,7 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 	// Get local heal status first
 	bgHealStates = append(bgHealStates, getLocalBackgroundHealStatus())
 
-	if globalIsDistXL {
+	if globalIsDistErasure {
 		// Get heal status from other peers
 		peersHealStates := globalNotificationSys.BackgroundHealStatus()
 		bgHealStates = append(bgHealStates, peersHealStates...)
@@ -862,11 +882,11 @@ const (
 	AdminUpdateApplyFailure      = "XMinioAdminUpdateApplyFailure"
 )
 
-// toAdminAPIErrCode - converts errXLWriteQuorum error to admin API
+// toAdminAPIErrCode - converts errErasureWriteQuorum error to admin API
 // specific error.
 func toAdminAPIErrCode(ctx context.Context, err error) APIErrorCode {
 	switch err {
-	case errXLWriteQuorum:
+	case errErasureWriteQuorum:
 		return ErrAdminConfigNoQuorum
 	default:
 		return toAPIErrorCode(ctx, err)
@@ -918,6 +938,12 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 				Description:    err.Error(),
 				HTTPStatusCode: http.StatusServiceUnavailable,
 			}
+		case errors.Is(err, crypto.ErrKESKeyExists):
+			apiErr = APIError{
+				Code:           "XMinioKMSKeyExists",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusConflict,
+			}
 		default:
 			apiErr = errorCodes.ToAPIErrWithErr(toAdminAPIErrCode(ctx, err), err)
 		}
@@ -1070,6 +1096,28 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 	}
 }
 
+// KMSCreateKeyHandler - POST /minio/admin/v3/kms/key/create?key-id=<master-key-id>
+func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "KMSCreateKey")
+	defer logger.AuditLog(w, r, "KMSCreateKey", mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSCreateKeyAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	if GlobalKMS == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
+		return
+	}
+
+	if err := GlobalKMS.CreateKey(r.URL.Query().Get("key-id")); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	writeSuccessResponseHeadersOnly(w)
+}
+
 // KMSKeyStatusHandler - GET /minio/admin/v3/kms/key/status?key-id=<master-key-id>
 func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSKeyStatus")
@@ -1088,7 +1136,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 
 	keyID := r.URL.Query().Get("key-id")
 	if keyID == "" {
-		keyID = GlobalKMS.KeyID()
+		keyID = GlobalKMS.DefaultKeyID()
 	}
 	var response = madmin.KMSKeyStatus{
 		KeyID: keyID,
@@ -1277,7 +1325,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			partialWrite(obdInfo)
 		}
 
-		if net, ok := vars["perfnet"]; ok && net == "true" && globalIsDistXL {
+		if net, ok := vars["perfnet"]; ok && net == "true" && globalIsDistErasure {
 			obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.NetOBDInfo(deadlinedCtx))
 			partialWrite(obdInfo)
 
@@ -1384,7 +1432,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 			OffDisks += v
 		}
 
-		backend = madmin.XLBackend{
+		backend = madmin.ErasureBackend{
 			Type:             madmin.ErasureType,
 			OnlineDisks:      OnDisks,
 			OfflineDisks:     OffDisks,
@@ -1399,10 +1447,8 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		}
 	}
 
-	mode := ""
-	if globalSafeMode {
-		mode = "safe"
-	} else {
+	mode := "safemode"
+	if newObjectLayerFn() != nil {
 		mode = "online"
 	}
 
@@ -1413,10 +1459,10 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	for _, sp := range servers {
 		for i, di := range sp.Disks {
 			path := ""
-			if globalIsXL {
+			if globalIsErasure {
 				path = di.DrivePath
 			}
-			if globalIsDistXL {
+			if globalIsDistErasure {
 				path = sp.Endpoint + di.DrivePath
 			}
 			// For distributed
@@ -1424,13 +1470,13 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 				for b := range storageInfo.Backend.Sets[a] {
 					ep := storageInfo.Backend.Sets[a][b].Endpoint
 
-					if globalIsDistXL {
+					if globalIsDistErasure {
 						if strings.Replace(ep, "http://", "", -1) == path || strings.Replace(ep, "https://", "", -1) == path {
 							sp.Disks[i].State = storageInfo.Backend.Sets[a][b].State
 							sp.Disks[i].UUID = storageInfo.Backend.Sets[a][b].UUID
 						}
 					}
-					if globalIsXL {
+					if globalIsErasure {
 						if ep == path {
 							sp.Disks[i].State = storageInfo.Backend.Sets[a][b].State
 							sp.Disks[i].UUID = storageInfo.Backend.Sets[a][b].UUID
@@ -1523,7 +1569,7 @@ func fetchVaultStatus(cfg config.Config) madmin.Vault {
 		vault.Status = "disabled"
 		return vault
 	}
-	keyID := GlobalKMS.KeyID()
+	keyID := GlobalKMS.DefaultKeyID()
 	kmsInfo := GlobalKMS.Info()
 
 	if kmsInfo.Endpoint == "" {

cmd/admin-handlers_test.go

@@ -33,27 +33,27 @@ import (
 	"github.com/minio/minio/pkg/madmin"
 )
 
-// adminXLTestBed - encapsulates subsystems that need to be setup for
+// adminErasureTestBed - encapsulates subsystems that need to be setup for
 // admin-handler unit tests.
-type adminXLTestBed struct {
-	xlDirs   []string
-	objLayer ObjectLayer
-	router   *mux.Router
+type adminErasureTestBed struct {
+	erasureDirs []string
+	objLayer    ObjectLayer
+	router      *mux.Router
 }
 
-// prepareAdminXLTestBed - helper function that setups a single-node
-// XL backend for admin-handler tests.
-func prepareAdminXLTestBed(ctx context.Context) (*adminXLTestBed, error) {
+// prepareAdminErasureTestBed - helper function that setups a single-node
+// Erasure backend for admin-handler tests.
+func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, error) {
 
 	// reset global variables to start afresh.
 	resetTestGlobals()
 
-	// Set globalIsXL to indicate that the setup uses an erasure
+	// Set globalIsErasure to indicate that the setup uses an erasure
 	// code backend.
-	globalIsXL = true
+	globalIsErasure = true
 
 	// Initializing objectLayer for HealFormatHandler.
-	objLayer, xlDirs, xlErr := initTestXLObjLayer(ctx)
+	objLayer, erasureDirs, xlErr := initTestErasureObjLayer(ctx)
 	if xlErr != nil {
 		return nil, xlErr
 	}
@@ -66,7 +66,7 @@ func prepareAdminXLTestBed(ctx context.Context) (*adminXLTestBed, error) {
 	// Initialize boot time
 	globalBootTime = UTCNow()
 
-	globalEndpoints = mustGetZoneEndpoints(xlDirs...)
+	globalEndpoints = mustGetZoneEndpoints(erasureDirs...)
 
 	newAllSubsystems()
 
@@ -76,36 +76,37 @@ func prepareAdminXLTestBed(ctx context.Context) (*adminXLTestBed, error) {
 	adminRouter := mux.NewRouter()
 	registerAdminRouter(adminRouter, true, true)
 
-	return &adminXLTestBed{
-		xlDirs:   xlDirs,
-		objLayer: objLayer,
-		router:   adminRouter,
+	return &adminErasureTestBed{
+		erasureDirs: erasureDirs,
+		objLayer:    objLayer,
+		router:      adminRouter,
 	}, nil
 }
 
 // TearDown - method that resets the test bed for subsequent unit
 // tests to start afresh.
-func (atb *adminXLTestBed) TearDown() {
-	removeRoots(atb.xlDirs)
+func (atb *adminErasureTestBed) TearDown() {
+	removeRoots(atb.erasureDirs)
 	resetTestGlobals()
 }
 
-// initTestObjLayer - Helper function to initialize an XL-based object
+// initTestObjLayer - Helper function to initialize an Erasure-based object
 // layer and set globalObjectAPI.
-func initTestXLObjLayer(ctx context.Context) (ObjectLayer, []string, error) {
-	xlDirs, err := getRandomDisks(16)
+func initTestErasureObjLayer(ctx context.Context) (ObjectLayer, []string, error) {
+	erasureDirs, err := getRandomDisks(16)
 	if err != nil {
 		return nil, nil, err
 	}
-	endpoints := mustGetNewEndpoints(xlDirs...)
-	storageDisks, format, err := waitForFormatXL(true, endpoints, 1, 1, 16, "")
+	endpoints := mustGetNewEndpoints(erasureDirs...)
+	storageDisks, format, err := waitForFormatErasure(true, endpoints, 1, 1, 16, "")
 	if err != nil {
-		removeRoots(xlDirs)
+		removeRoots(erasureDirs)
 		return nil, nil, err
 	}
 
 	globalPolicySys = NewPolicySys()
-	objLayer, err := newXLSets(ctx, endpoints, storageDisks, format)
+	objLayer := &erasureZones{zones: make([]*erasureSets, 1)}
+	objLayer.zones[0], err = newErasureSets(ctx, endpoints, storageDisks, format)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -114,7 +115,7 @@ func initTestXLObjLayer(ctx context.Context) (ObjectLayer, []string, error) {
 	globalObjLayerMutex.Lock()
 	globalObjectAPI = objLayer
 	globalObjLayerMutex.Unlock()
-	return objLayer, xlDirs, nil
+	return objLayer, erasureDirs, nil
 }
 
 // cmdType - Represents different service subcomands like status, stop
@@ -183,9 +184,9 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	adminTestBed, err := prepareAdminXLTestBed(ctx)
+	adminTestBed, err := prepareAdminErasureTestBed(ctx)
 	if err != nil {
-		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
+		t.Fatal("Failed to initialize a single node Erasure backend for admin handler tests.")
 	}
 	defer adminTestBed.TearDown()
 
@@ -254,9 +255,9 @@ func TestAdminServerInfo(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	adminTestBed, err := prepareAdminXLTestBed(ctx)
+	adminTestBed, err := prepareAdminErasureTestBed(ctx)
 	if err != nil {
-		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
+		t.Fatal("Failed to initialize a single node Erasure backend for admin handler tests.")
 	}
 
 	defer adminTestBed.TearDown()
@@ -298,7 +299,7 @@ func TestToAdminAPIErrCode(t *testing.T) {
 	}{
 		// 1. Server not in quorum.
 		{
-			err:            errXLWriteQuorum,
+			err:            errErasureWriteQuorum,
 			expectedAPIErr: ErrAdminConfigNoQuorum,
 		},
 		// 2. No error.

cmd/admin-heal-ops.go

@@ -21,7 +21,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"strings"
 	"sync"
 	"time"
 
@@ -77,9 +76,6 @@ type healSequenceStatus struct {
 	FailureDetail string            `json:"Detail,omitempty"`
 	StartTime     time.Time         `json:"StartTime"`
 
-	// disk information
-	NumDisks int `json:"NumDisks"`
-
 	// settings for the heal sequence
 	HealSettings madmin.HealOpts `json:"Settings"`
 
@@ -95,8 +91,8 @@ type allHealState struct {
 	healSeqMap map[string]*healSequence
 }
 
-// initHealState - initialize healing apparatus
-func initHealState() *allHealState {
+// newHealState - initialize global heal state management
+func newHealState() *allHealState {
 	healState := &allHealState{
 		healSeqMap: make(map[string]*healSequence),
 	}
@@ -159,8 +155,13 @@ func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) {
 			StartTime:   UTCNow(),
 		}
 	} else {
+		clientToken := he.clientToken
+		if globalIsDistErasure {
+			clientToken = fmt.Sprintf("%s@%d", he.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
+		}
+
 		hsp = madmin.HealStopSuccess{
-			ClientToken:   he.clientToken,
+			ClientToken:   clientToken,
 			ClientAddress: he.clientAddress,
 			StartTime:     he.startTime,
 		}
@@ -193,7 +194,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 	respBytes []byte, apiErr APIError, errMsg string) {
 
 	existsAndLive := false
-	he, exists := ahs.getHealSequence(h.path)
+	he, exists := ahs.getHealSequence(pathJoin(h.bucket, h.object))
 	if exists {
 		existsAndLive = !he.hasEnded()
 	}
@@ -220,8 +221,9 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 
 	// Check if new heal sequence to be started overlaps with any
 	// existing, running sequence
+	hpath := pathJoin(h.bucket, h.object)
 	for k, hSeq := range ahs.healSeqMap {
-		if !hSeq.hasEnded() && (HasPrefix(k, h.path) || HasPrefix(h.path, k)) {
+		if !hSeq.hasEnded() && (HasPrefix(k, hpath) || HasPrefix(hpath, k)) {
 
 			errMsg = "The provided heal sequence path overlaps with an existing " +
 				fmt.Sprintf("heal path: %s", k)
@@ -230,13 +232,18 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 	}
 
 	// Add heal state and start sequence
-	ahs.healSeqMap[h.path] = h
+	ahs.healSeqMap[hpath] = h
 
 	// Launch top-level background heal go-routine
 	go h.healSequenceStart()
 
+	clientToken := h.clientToken
+	if globalIsDistErasure {
+		clientToken = fmt.Sprintf("%s@%d", h.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
+	}
+
 	b, err := json.Marshal(madmin.HealStartSuccess{
-		ClientToken:   h.clientToken,
+		ClientToken:   clientToken,
 		ClientAddress: h.clientAddress,
 		StartTime:     h.startTime,
 	})
@@ -251,11 +258,11 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 // status results from global state and returns its JSON
 // representation. The clientToken helps ensure there aren't
 // conflicting clients fetching status.
-func (ahs *allHealState) PopHealStatusJSON(path string,
+func (ahs *allHealState) PopHealStatusJSON(hpath string,
 	clientToken string) ([]byte, APIErrorCode) {
 
 	// fetch heal state for given path
-	h, exists := ahs.getHealSequence(path)
+	h, exists := ahs.getHealSequence(hpath)
 	if !exists {
 		// If there is no such heal sequence, return error.
 		return nil, ErrHealNoSuchProcess
@@ -296,18 +303,17 @@ func (ahs *allHealState) PopHealStatusJSON(path string,
 
 // healSource denotes single entity and heal option.
 type healSource struct {
-	path string           // entity path (format, buckets, objects) to heal
-	opts *madmin.HealOpts // optional heal option overrides default setting
+	bucket    string
+	object    string
+	versionID string
+	opts      *madmin.HealOpts // optional heal option overrides default setting
 }
 
 // healSequence - state for each heal sequence initiated on the
 // server.
 type healSequence struct {
-	// bucket, and prefix on which heal seq. was initiated
-	bucket, objPrefix string
-
-	// path is just pathJoin(bucket, objPrefix)
-	path string
+	// bucket, and object on which heal seq. was initiated
+	bucket, object string
 
 	// A channel of entities (format, buckets, objects) to heal
 	sourceCh chan healSource
@@ -368,27 +374,27 @@ type healSequence struct {
 // NewHealSequence - creates healSettings, assumes bucket and
 // objPrefix are already validated.
 func newHealSequence(ctx context.Context, bucket, objPrefix, clientAddr string,
-	numDisks int, hs madmin.HealOpts, forceStart bool) *healSequence {
+	hs madmin.HealOpts, forceStart bool) *healSequence {
 
 	reqInfo := &logger.ReqInfo{RemoteHost: clientAddr, API: "Heal", BucketName: bucket}
 	reqInfo.AppendTags("prefix", objPrefix)
 	ctx, cancel := context.WithCancel(logger.SetReqInfo(ctx, reqInfo))
 
+	clientToken := mustGetUUID()
+
 	return &healSequence{
 		respCh:         make(chan healResult),
 		bucket:         bucket,
-		objPrefix:      objPrefix,
-		path:           pathJoin(bucket, objPrefix),
+		object:         objPrefix,
 		reportProgress: true,
 		startTime:      UTCNow(),
-		clientToken:    mustGetUUID(),
+		clientToken:    clientToken,
 		clientAddress:  clientAddr,
 		forceStarted:   forceStart,
 		settings:       hs,
 		currentStatus: healSequenceStatus{
 			Summary:      healNotStartedStatus,
 			HealSettings: hs,
-			NumDisks:     numDisks,
 		},
 		traverseAndHealDoneCh: make(chan error),
 		cancelCtx:             cancel,
@@ -618,7 +624,9 @@ func (h *healSequence) healSequenceStart() {
 func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItemType) error {
 	// Send heal request
 	task := healTask{
-		path:       source.path,
+		bucket:     source.bucket,
+		object:     source.object,
+		versionID:  source.versionID,
 		opts:       h.settings,
 		responseCh: h.respCh,
 	}
@@ -633,7 +641,7 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 			// Object might have been deleted, by the time heal
 			// was attempted, we should ignore this object and
 			// return success.
-			if isErrObjectNotFound(res.err) {
+			if isErrObjectNotFound(res.err) || isErrVersionNotFound(res.err) {
 				return nil
 			}
 
@@ -661,7 +669,7 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 		if res.err != nil {
 			// Object might have been deleted, by the time heal
 			// was attempted, we should ignore this object and return success.
-			if isErrObjectNotFound(res.err) {
+			if isErrObjectNotFound(res.err) || isErrVersionNotFound(res.err) {
 				return nil
 			}
 			// Only report object error
@@ -677,11 +685,6 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 }
 
 func (h *healSequence) healItemsFromSourceCh() error {
-	bucketsOnly := true // heal buckets only, not objects.
-	if err := h.healItems(bucketsOnly); err != nil {
-		logger.LogIf(h.ctx, err)
-	}
-
 	for {
 		select {
 		case source, ok := <-h.sourceCh:
@@ -690,11 +693,11 @@ func (h *healSequence) healItemsFromSourceCh() error {
 			}
 			var itemType madmin.HealItemType
 			switch {
-			case source.path == nopHeal:
+			case source.bucket == nopHeal:
 				continue
-			case source.path == SlashSeparator:
+			case source.bucket == SlashSeparator:
 				itemType = madmin.HealItemMetadata
-			case !strings.Contains(source.path, SlashSeparator):
+			case source.bucket != "" && source.object == "":
 				itemType = madmin.HealItemBucket
 			default:
 				itemType = madmin.HealItemObject
@@ -716,7 +719,7 @@ func (h *healSequence) healFromSourceCh() {
 	h.healItemsFromSourceCh()
 }
 
-func (h *healSequence) healItems(bucketsOnly bool) error {
+func (h *healSequence) healDiskMeta() error {
 	// Start with format healing
 	if err := h.healDiskFormat(); err != nil {
 		return err
@@ -728,7 +731,11 @@ func (h *healSequence) healItems(bucketsOnly bool) error {
 	}
 
 	// Start healing the bucket config prefix.
-	if err := h.healMinioSysMeta(bucketConfigPrefix)(); err != nil {
+	return h.healMinioSysMeta(bucketConfigPrefix)()
+}
+
+func (h *healSequence) healItems(bucketsOnly bool) error {
+	if err := h.healDiskMeta(); err != nil {
 		return err
 	}
 
@@ -762,15 +769,19 @@ func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
 		// NOTE: Healing on meta is run regardless
 		// of any bucket being selected, this is to ensure that
 		// meta are always upto date and correct.
-		return objectAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket string, object string) error {
+		return objectAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string) error {
 			if h.isQuitting() {
 				return errHealStopSignalled
 			}
 
-			herr := h.queueHealTask(healSource{path: pathJoin(bucket, object)}, madmin.HealItemBucketMetadata)
+			herr := h.queueHealTask(healSource{
+				bucket:    bucket,
+				object:    object,
+				versionID: versionID,
+			}, madmin.HealItemBucketMetadata)
 			// Object might have been deleted, by the time heal
 			// was attempted we ignore this object an move on.
-			if isErrObjectNotFound(herr) {
+			if isErrObjectNotFound(herr) || isErrVersionNotFound(herr) {
 				return nil
 			}
 			return herr
@@ -791,7 +802,7 @@ func (h *healSequence) healDiskFormat() error {
 		return errServerNotInitialized
 	}
 
-	return h.queueHealTask(healSource{path: SlashSeparator}, madmin.HealItemMetadata)
+	return h.queueHealTask(healSource{bucket: SlashSeparator}, madmin.HealItemMetadata)
 }
 
 // healBuckets - check for all buckets heal or just particular bucket.
@@ -833,7 +844,7 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 		return errServerNotInitialized
 	}
 
-	if err := h.queueHealTask(healSource{path: bucket}, madmin.HealItemBucket); err != nil {
+	if err := h.queueHealTask(healSource{bucket: bucket}, madmin.HealItemBucket); err != nil {
 		return err
 	}
 
@@ -842,12 +853,12 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 	}
 
 	if !h.settings.Recursive {
-		if h.objPrefix != "" {
+		if h.object != "" {
 			// Check if an object named as the objPrefix exists,
 			// and if so heal it.
-			_, err := objectAPI.GetObjectInfo(h.ctx, bucket, h.objPrefix, ObjectOptions{})
+			_, err := objectAPI.GetObjectInfo(h.ctx, bucket, h.object, ObjectOptions{})
 			if err == nil {
-				if err = h.healObject(bucket, h.objPrefix); err != nil {
+				if err = h.healObject(bucket, h.object, ""); err != nil {
 					return err
 				}
 			}
@@ -856,14 +867,14 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 		return nil
 	}
 
-	if err := objectAPI.HealObjects(h.ctx, bucket, h.objPrefix, h.settings, h.healObject); err != nil {
+	if err := objectAPI.HealObjects(h.ctx, bucket, h.object, h.settings, h.healObject); err != nil {
 		return errFnHealFromAPIErr(h.ctx, err)
 	}
 	return nil
 }
 
 // healObject - heal the given object and record result
-func (h *healSequence) healObject(bucket, object string) error {
+func (h *healSequence) healObject(bucket, object, versionID string) error {
 	// Get current object layer instance.
 	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
@@ -874,5 +885,9 @@ func (h *healSequence) healObject(bucket, object string) error {
 		return errHealStopSignalled
 	}
 
-	return h.queueHealTask(healSource{path: pathJoin(bucket, object)}, madmin.HealItemObject)
+	return h.queueHealTask(healSource{
+		bucket:    bucket,
+		object:    object,
+		versionID: versionID,
+	}, madmin.HealItemObject)
 }

cmd/admin-router.go

@@ -64,7 +64,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		// DataUsageInfo operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(httpTraceAll(adminAPI.DataUsageInfoHandler))
 
-		if globalIsDistXL || globalIsXL {
+		if globalIsDistErasure || globalIsErasure {
 			/// Heal operations
 
 			// Heal processing endpoint.
@@ -172,7 +172,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		}
 
 		// Quota operations
-		if globalIsXL || globalIsDistXL {
+		if globalIsDistErasure || globalIsErasure {
 			if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
 				// GetBucketQuotaConfig
 				adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
@@ -185,7 +185,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 		// -- Top APIs --
 		// Top locks
-		if globalIsDistXL {
+		if globalIsDistErasure {
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/top/locks").HandlerFunc(httpTraceHdrs(adminAPI.TopLocksHandler))
 		}
 
@@ -197,6 +197,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 		// -- KMS APIs --
 		//
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/kms/key/create").HandlerFunc(httpTraceAll(adminAPI.KMSCreateKeyHandler)).Queries("key-id", "{key-id:.*}")
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(httpTraceAll(adminAPI.KMSKeyStatusHandler))
 
 		if !globalIsGateway {

cmd/admin-server-info.go

@@ -29,7 +29,7 @@ import (
 func getLocalServerProperty(endpointZones EndpointZones, r *http.Request) madmin.ServerProperties {
 	var disks []madmin.Disk
 	addr := r.Host
-	if globalIsDistXL {
+	if globalIsDistErasure {
 		addr = GetLocalPeer(endpointZones)
 	}
 	network := make(map[string]string)

cmd/api-datatypes.go

@@ -20,9 +20,18 @@ import (
 	"encoding/xml"
 )
 
-// ObjectIdentifier carries key name for the object to delete.
-type ObjectIdentifier struct {
+// DeletedObject objects deleted
+type DeletedObject struct {
+	DeleteMarker          bool   `xml:"DeleteMarker,omitempty"`
+	DeleteMarkerVersionID string `xml:"DeleteMarkerVersionId,omitempty"`
+	ObjectName            string `xml:"Key,omitempty"`
+	VersionID             string `xml:"VersionId,omitempty"`
+}
+
+// ObjectToDelete carries key name for the object to delete.
+type ObjectToDelete struct {
 	ObjectName string `xml:"Key"`
+	VersionID  string `xml:"VersionId"`
 }
 
 // createBucketConfiguration container for bucket configuration request from client.
@@ -37,5 +46,5 @@ type DeleteObjectsRequest struct {
 	// Element to enable quiet mode for the request
 	Quiet bool
 	// List of objects to be deleted
-	Objects []ObjectIdentifier `xml:"Object"`
+	Objects []ObjectToDelete `xml:"Object"`
 }

cmd/api-errors.go

@@ -36,6 +36,7 @@ import (
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/bucket/versioning"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
 )
@@ -538,9 +539,9 @@ var errorCodes = errorCodeMap{
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrNoSuchVersion: {
-		Code:           "NoSuchVersion",
-		Description:    "Indicates that the version ID specified in the request does not match an existing version.",
-		HTTPStatusCode: http.StatusNotFound,
+		Code:           "InvalidArgument",
+		Description:    "Invalid version id specified",
+		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrNotImplemented: {
 		Code:           "NotImplemented",
@@ -1782,6 +1783,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrBucketAlreadyOwnedByYou
 	case ObjectNotFound:
 		apiErr = ErrNoSuchKey
+	case MethodNotAllowed:
+		apiErr = ErrMethodNotAllowed
+	case VersionNotFound:
+		apiErr = ErrNoSuchVersion
 	case ObjectAlreadyExists:
 		apiErr = ErrMethodNotAllowed
 	case ObjectNameInvalid:
@@ -1904,6 +1909,12 @@ func toAPIError(ctx context.Context, err error) APIError {
 		// their internal error types. This code is only
 		// useful with gateway implementations.
 		switch e := err.(type) {
+		case InvalidArgument:
+			apiErr = APIError{
+				Code:           "InvalidArgument",
+				Description:    e.Error(),
+				HTTPStatusCode: errorCodes[ErrInvalidRequest].HTTPStatusCode,
+			}
 		case *xml.SyntaxError:
 			apiErr = APIError{
 				Code: "MalformedXML",
@@ -1918,6 +1929,12 @@ func toAPIError(ctx context.Context, err error) APIError {
 					e.Error()),
 				HTTPStatusCode: http.StatusBadRequest,
 			}
+		case versioning.Error:
+			apiErr = APIError{
+				Code:           "IllegalVersioningConfigurationException",
+				Description:    fmt.Sprintf("Versioning configuration specified in the request is invalid. (%s)", e.Error()),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case lifecycle.Error:
 			apiErr = APIError{
 				Code:           "InvalidRequest",

cmd/api-headers.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
 )
 
 // Returns a hexadecimal representation of time at the
@@ -152,5 +153,26 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		w.Header().Set(xhttp.ContentRange, contentRange)
 	}
 
+	// Set the relevant version ID as part of the response header.
+	if objInfo.VersionID != "" {
+		w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
+	}
+
+	if lc, err := globalLifecycleSys.Get(objInfo.Bucket); err == nil {
+		ruleID, expiryTime := lc.PredictExpiryTime(lifecycle.ObjectOpts{
+			Name:         objInfo.Name,
+			UserTags:     objInfo.UserTags,
+			VersionID:    objInfo.VersionID,
+			ModTime:      objInfo.ModTime,
+			IsLatest:     objInfo.IsLatest,
+			DeleteMarker: objInfo.DeleteMarker,
+		})
+		if !expiryTime.IsZero() {
+			w.Header()[xhttp.AmzExpiration] = []string{
+				fmt.Sprintf(`expiry-date="%s", rule-id="%s"`, expiryTime.Format(http.TimeFormat), ruleID),
+			}
+		}
+	}
+
 	return nil
 }

cmd/api-response.go

@@ -35,7 +35,7 @@ import (
 const (
 	// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
 	iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
-	maxObjectList     = 50000                      // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
+	maxObjectList     = 10000                      // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
 	maxDeleteList     = 10000                      // Limit number of objects deleted in a delete call.
 	maxUploadsList    = 10000                      // Limit number of uploads in a listUploadsResponse.
 	maxPartsList      = 10000                      // Limit number of parts in a listPartsResponse.
@@ -235,10 +235,23 @@ type Bucket struct {
 
 // ObjectVersion container for object version metadata
 type ObjectVersion struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Version" json:"-"`
 	Object
-	VersionID string `xml:"VersionId"`
 	IsLatest  bool
+	VersionID string `xml:"VersionId"`
+
+	isDeleteMarker bool
+}
+
+// MarshalXML - marshal ObjectVersion
+func (o ObjectVersion) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	if o.isDeleteMarker {
+		start.Name.Local = "DeleteMarker"
+	} else {
+		start.Name.Local = "Version"
+	}
+
+	type objectVersionWrapper ObjectVersion
+	return e.EncodeElement(objectVersionWrapper(o), start)
 }
 
 // StringMap is a map[string]string.
@@ -333,9 +346,10 @@ type CompleteMultipartUploadResponse struct {
 
 // DeleteError structure.
 type DeleteError struct {
-	Code    string
-	Message string
-	Key     string
+	Code      string
+	Message   string
+	Key       string
+	VersionID string `xml:"VersionId"`
 }
 
 // DeleteObjectsResponse container for multiple object deletes.
@@ -343,7 +357,7 @@ type DeleteObjectsResponse struct {
 	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteResult" json:"-"`
 
 	// Collection of all deleted objects
-	DeletedObjects []ObjectIdentifier `xml:"Deleted,omitempty"`
+	DeletedObjects []DeletedObject `xml:"Deleted,omitempty"`
 
 	// Collection of errors deleting certain objects.
 	Errors []DeleteError `xml:"Error,omitempty"`
@@ -413,7 +427,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 }
 
 // generates an ListBucketVersions response for the said bucket with other enumerated options.
-func generateListVersionsResponse(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListVersionsResponse {
+func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo) ListVersionsResponse {
 	var versions []ObjectVersion
 	var prefixes []CommonPrefix
 	var owner = Owner{}
@@ -436,10 +450,13 @@ func generateListVersionsResponse(bucket, prefix, marker, delimiter, encodingTyp
 		} else {
 			content.StorageClass = globalMinioDefaultStorageClass
 		}
-
 		content.Owner = owner
-		content.VersionID = "null"
-		content.IsLatest = true
+		content.VersionID = object.VersionID
+		if content.VersionID == "" {
+			content.VersionID = nullVersionID
+		}
+		content.IsLatest = object.IsLatest
+		content.isDeleteMarker = object.DeleteMarker
 		versions = append(versions, content)
 	}
 
@@ -452,6 +469,8 @@ func generateListVersionsResponse(bucket, prefix, marker, delimiter, encodingTyp
 	data.MaxKeys = maxKeys
 
 	data.NextKeyMarker = s3EncodeName(resp.NextMarker, encodingType)
+	data.NextVersionIDMarker = resp.NextVersionIDMarker
+	data.VersionIDMarker = versionIDMarker
 	data.IsTruncated = resp.IsTruncated
 
 	for _, prefix := range resp.Prefixes {
@@ -666,16 +685,23 @@ func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMult
 }
 
 // generate multi objects delete response.
-func generateMultiDeleteResponse(quiet bool, deletedObjects []ObjectIdentifier, errs []DeleteError) DeleteObjectsResponse {
+func generateMultiDeleteResponse(quiet bool, deletedObjects []DeletedObject, errs []DeleteError) DeleteObjectsResponse {
 	deleteResp := DeleteObjectsResponse{}
 	if !quiet {
 		deleteResp.DeletedObjects = deletedObjects
 	}
+	if len(errs) == len(deletedObjects) {
+		deleteResp.DeletedObjects = nil
+	}
 	deleteResp.Errors = errs
 	return deleteResp
 }
 
 func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
+	if newObjectLayerFn() == nil {
+		// Server still in safe mode.
+		w.Header().Set(xhttp.MinIOServerStatus, "safemode")
+	}
 	setCommonHeaders(w)
 	if mType != mimeNone {
 		w.Header().Set(xhttp.ContentType, string(mType))
@@ -738,6 +764,10 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 		// The request is from browser and also if browser
 		// is enabled we need to redirect.
 		if browser && globalBrowserEnabled {
+			if newObjectLayerFn() == nil {
+				// server still in safe mode.
+				w.Header().Set(xhttp.MinIOServerStatus, "safemode")
+			}
 			w.Header().Set(xhttp.Location, minioReservedBucketPath+reqURL.Path)
 			w.WriteHeader(http.StatusTemporaryRedirect)
 			return

cmd/api-router.go

@@ -21,6 +21,8 @@ import (
 
 	"github.com/gorilla/mux"
 	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/pkg/wildcard"
+	"github.com/rs/cors"
 )
 
 func newHTTPServerFn() *xhttp.Server {
@@ -83,7 +85,6 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 	var routers []*mux.Router
 	for _, domainName := range globalDomainNames {
 		routers = append(routers, apiRouter.Host("{bucket:.+}."+domainName).Subrouter())
-		routers = append(routers, apiRouter.Host("{bucket:.+}."+domainName+":{port:.*}").Subrouter())
 	}
 	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 
@@ -224,9 +225,9 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		// ListObjectsV2
 		bucket.Methods(http.MethodGet).HandlerFunc(
 			maxClients(collectAPIStats("listobjectsv2", httpTraceAll(api.ListObjectsV2Handler)))).Queries("list-type", "2")
-		// ListBucketVersions
+		// ListObjectVersions
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("listbucketversions", httpTraceAll(api.ListBucketObjectVersionsHandler)))).Queries("versions", "")
+			maxClients(collectAPIStats("listobjectversions", httpTraceAll(api.ListObjectVersionsHandler)))).Queries("versions", "")
 		// ListObjectsV1 (Legacy)
 		bucket.Methods(http.MethodGet).HandlerFunc(
 			maxClients(collectAPIStats("listobjectsv1", httpTraceAll(api.ListObjectsV1Handler))))
@@ -295,3 +296,44 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 	apiRouter.MethodNotAllowedHandler = http.HandlerFunc(collectAPIStats("methodnotallowed", httpTraceAll(errorResponseHandler)))
 
 }
+
+// corsHandler handler for CORS (Cross Origin Resource Sharing)
+func corsHandler(handler http.Handler) http.Handler {
+	commonS3Headers := []string{
+		xhttp.Date,
+		xhttp.ETag,
+		xhttp.ServerInfo,
+		xhttp.Connection,
+		xhttp.AcceptRanges,
+		xhttp.ContentRange,
+		xhttp.ContentEncoding,
+		xhttp.ContentLength,
+		xhttp.ContentType,
+		"X-Amz*",
+		"x-amz*",
+		"*",
+	}
+
+	return cors.New(cors.Options{
+		AllowOriginFunc: func(origin string) bool {
+			for _, allowedOrigin := range globalAPIConfig.getCorsAllowOrigins() {
+				if wildcard.MatchSimple(allowedOrigin, origin) {
+					return true
+				}
+			}
+			return false
+		},
+		AllowedMethods: []string{
+			http.MethodGet,
+			http.MethodPut,
+			http.MethodHead,
+			http.MethodPost,
+			http.MethodDelete,
+			http.MethodOptions,
+			http.MethodPatch,
+		},
+		AllowedHeaders:   commonS3Headers,
+		ExposedHeaders:   commonS3Headers,
+		AllowCredentials: true,
+	}).Handler(handler)
+}

cmd/background-heal-ops.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"path"
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
@@ -29,8 +30,10 @@ import (
 //   path: 'bucket/' or '/bucket/' => Heal bucket
 //   path: 'bucket/object' => Heal object
 type healTask struct {
-	path string
-	opts madmin.HealOpts
+	bucket    string
+	object    string
+	versionID string
+	opts      madmin.HealOpts
 	// Healing response will be sent here
 	responseCh chan healResult
 }
@@ -79,17 +82,18 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 
 			var res madmin.HealResultItem
 			var err error
-			bucket, object := path2BucketObject(task.path)
 			switch {
-			case bucket == "" && object == "":
+			case task.bucket == nopHeal:
+				continue
+			case task.bucket == SlashSeparator:
 				res, err = healDiskFormat(ctx, objAPI, task.opts)
-			case bucket != "" && object == "":
-				res, err = objAPI.HealBucket(ctx, bucket, task.opts.DryRun, task.opts.Remove)
-			case bucket != "" && object != "":
-				res, err = objAPI.HealObject(ctx, bucket, object, task.opts)
+			case task.bucket != "" && task.object == "":
+				res, err = objAPI.HealBucket(ctx, task.bucket, task.opts.DryRun, task.opts.Remove)
+			case task.bucket != "" && task.object != "":
+				res, err = objAPI.HealObject(ctx, task.bucket, task.object, task.versionID, task.opts)
 			}
-			if task.path != slashSeparator && task.path != nopHeal {
-				ObjectPathUpdated(task.path)
+			if task.bucket != "" && task.object != "" {
+				ObjectPathUpdated(path.Join(task.bucket, task.object))
 			}
 			task.responseCh <- healResult{result: res, err: err}
 		case <-h.doneCh:
@@ -100,7 +104,7 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 	}
 }
 
-func initHealRoutine() *healRoutine {
+func newHealRoutine() *healRoutine {
 	return &healRoutine{
 		tasks:  make(chan healTask),
 		doneCh: make(chan struct{}),
@@ -108,22 +112,22 @@ func initHealRoutine() *healRoutine {
 
 }
 
-func startBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
+func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
 	// Run the background healer
-	globalBackgroundHealRoutine = initHealRoutine()
+	globalBackgroundHealRoutine = newHealRoutine()
 	go globalBackgroundHealRoutine.run(ctx, objAPI)
 
-	// Launch the background healer sequence to track
-	// background healing operations, ignore errors
-	// errors are handled into offline disks already.
-	info, _ := objAPI.StorageInfo(ctx, false)
-	numDisks := info.Backend.OnlineDisks.Sum() + info.Backend.OfflineDisks.Sum()
-	nh := newBgHealSequence(numDisks)
-	globalBackgroundHealState.LaunchNewHealSequence(nh)
-}
+	nh := newBgHealSequence()
+	// Heal any disk format and metadata early, if possible.
+	if err := nh.healDiskMeta(); err != nil {
+		if newObjectLayerFn() != nil {
+			// log only in situations, when object layer
+			// has fully initialized.
+			logger.LogIf(nh.ctx, err)
+		}
+	}
 
-func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
-	go startBackgroundHealing(ctx, objAPI)
+	globalBackgroundHealState.LaunchNewHealSequence(nh)
 }
 
 // healDiskFormat - heals format.json, return value indicates if a
@@ -140,12 +144,20 @@ func healDiskFormat(ctx context.Context, objAPI ObjectLayer, opts madmin.HealOpt
 	// Healing succeeded notify the peers to reload format and re-initialize disks.
 	// We will not notify peers if healing is not required.
 	if err == nil {
-		for _, nerr := range globalNotificationSys.ReloadFormat(opts.DryRun) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
+		// Notify servers in background and retry if needed.
+		go func() {
+		retry:
+			for _, nerr := range globalNotificationSys.ReloadFormat(opts.DryRun) {
+				if nerr.Err != nil {
+					if nerr.Err.Error() == errServerNotInitialized.Error() {
+						time.Sleep(time.Second)
+						goto retry
+					}
+					logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+					logger.LogIf(ctx, nerr.Err)
+				}
 			}
-		}
+		}()
 	}
 
 	return res, nil

cmd/background-newdisks-heal-ops.go

@@ -33,7 +33,7 @@ func initLocalDisksAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 //  1. Only the concerned erasure set will be listed and healed
 //  2. Only the node hosting the disk is responsible to perform the heal
 func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
-	z, ok := objAPI.(*xlZones)
+	z, ok := objAPI.(*erasureZones)
 	if !ok {
 		return
 	}
@@ -84,10 +84,10 @@ func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
 			}
 
 			// Reformat disks
-			bgSeq.sourceCh <- healSource{path: SlashSeparator}
+			bgSeq.sourceCh <- healSource{bucket: SlashSeparator}
 
 			// Ensure that reformatting disks is finished
-			bgSeq.sourceCh <- healSource{path: nopHeal}
+			bgSeq.sourceCh <- healSource{bucket: nopHeal}
 
 			var erasureSetInZoneToHeal = make([][]int, len(localDisksInZoneHeal))
 			// Compute the list of erasure set to heal

cmd/benchmark-utils_test.go

@@ -35,7 +35,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
+	err = obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -76,7 +76,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 	object := getRandomObjectName()
 
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
+	err = obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -127,9 +127,9 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 	b.StopTimer()
 }
 
-// creates XL/FS backend setup, obtains the object layer and calls the runPutObjectPartBenchmark function.
+// creates Erasure/FS backend setup, obtains the object layer and calls the runPutObjectPartBenchmark function.
 func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
-	// create a temp XL/FS backend.
+	// create a temp Erasure/FS backend.
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	objLayer, disks, err := prepareTestBackend(ctx, instanceType)
@@ -143,9 +143,9 @@ func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
 	runPutObjectPartBenchmark(b, objLayer, objSize)
 }
 
-// creates XL/FS backend setup, obtains the object layer and calls the runPutObjectBenchmark function.
+// creates Erasure/FS backend setup, obtains the object layer and calls the runPutObjectBenchmark function.
 func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
-	// create a temp XL/FS backend.
+	// create a temp Erasure/FS backend.
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	objLayer, disks, err := prepareTestBackend(ctx, instanceType)
@@ -159,9 +159,9 @@ func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
 	runPutObjectBenchmark(b, objLayer, objSize)
 }
 
-// creates XL/FS backend setup, obtains the object layer and runs parallel benchmark for put object.
+// creates Erasure/FS backend setup, obtains the object layer and runs parallel benchmark for put object.
 func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int) {
-	// create a temp XL/FS backend.
+	// create a temp Erasure/FS backend.
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	objLayer, disks, err := prepareTestBackend(ctx, instanceType)
@@ -181,7 +181,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -190,7 +190,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 
 	// generate etag for the generated data.
 	// etag of the data to written is required as input for PutObject.
-	// PutObject is the functions which writes the data onto the FS/XL backend.
+	// PutObject is the functions which writes the data onto the FS/Erasure backend.
 
 	// get text data generated for number of bytes equal to object size.
 	md5hex := getMD5Hash(textData)
@@ -240,9 +240,9 @@ func generateBytesData(size int) []byte {
 	return bytes.Repeat(getRandomByte(), size)
 }
 
-// creates XL/FS backend setup, obtains the object layer and calls the runGetObjectBenchmark function.
+// creates Erasure/FS backend setup, obtains the object layer and calls the runGetObjectBenchmark function.
 func benchmarkGetObject(b *testing.B, instanceType string, objSize int) {
-	// create a temp XL/FS backend.
+	// create a temp Erasure/FS backend.
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	objLayer, disks, err := prepareTestBackend(ctx, instanceType)
@@ -256,9 +256,9 @@ func benchmarkGetObject(b *testing.B, instanceType string, objSize int) {
 	runGetObjectBenchmark(b, objLayer, objSize)
 }
 
-// creates XL/FS backend setup, obtains the object layer and runs parallel benchmark for ObjectLayer.GetObject() .
+// creates Erasure/FS backend setup, obtains the object layer and runs parallel benchmark for ObjectLayer.GetObject() .
 func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int) {
-	// create a temp XL/FS backend.
+	// create a temp Erasure/FS backend.
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	objLayer, disks, err := prepareTestBackend(ctx, instanceType)
@@ -278,7 +278,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -322,7 +322,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -331,7 +331,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	textData := generateBytesData(objSize)
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
-	// PutObject is the functions which writes the data onto the FS/XL backend.
+	// PutObject is the functions which writes the data onto the FS/Erasure backend.
 
 	md5hex := getMD5Hash([]byte(textData))
 	sha256hex := ""

cmd/bitrot.go

@@ -30,25 +30,6 @@ import (
 // magic HH-256 key as HH-256 hash of the first 100 decimals of π as utf-8 string with a zero key.
 var magicHighwayHash256Key = []byte("\x4b\xe7\x34\xfa\x8e\x23\x8a\xcd\x26\x3e\x83\xe6\xbb\x96\x85\x52\x04\x0f\x93\x5d\xa3\x9f\x44\x14\x97\xe0\x9d\x13\x22\xde\x36\xa0")
 
-// BitrotAlgorithm specifies a algorithm used for bitrot protection.
-type BitrotAlgorithm uint
-
-const (
-	// SHA256 represents the SHA-256 hash function
-	SHA256 BitrotAlgorithm = 1 + iota
-	// HighwayHash256 represents the HighwayHash-256 hash function
-	HighwayHash256
-	// HighwayHash256S represents the Streaming HighwayHash-256 hash function
-	HighwayHash256S
-	// BLAKE2b512 represents the BLAKE2b-512 hash function
-	BLAKE2b512
-)
-
-// DefaultBitrotAlgorithm is the default algorithm used for bitrot protection.
-const (
-	DefaultBitrotAlgorithm = HighwayHash256S
-)
-
 var bitrotAlgorithms = map[BitrotAlgorithm]string{
 	SHA256:          "sha256",
 	BLAKE2b512:      "blake2b",

cmd/bitrot_test.go

@@ -34,7 +34,7 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 	volume := "testvol"
 	filePath := "testfile"
 
-	disk, err := newPosix(tmpDir, "")
+	disk, err := newXLStorage(tmpDir, "")
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/bootstrap-peer-server.go

@@ -20,12 +20,12 @@ import (
 	"context"
 	"crypto/tls"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"runtime"
-	"sync/atomic"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -43,6 +43,7 @@ const (
 )
 
 const (
+	bootstrapRESTMethodHealth = "/health"
 	bootstrapRESTMethodVerify = "/verify"
 )
 
@@ -94,6 +95,9 @@ func getServerSystemCfg() ServerSystemConfig {
 	}
 }
 
+// HealthHandler returns success if request is valid
+func (b *bootstrapRESTServer) HealthHandler(w http.ResponseWriter, r *http.Request) {}
+
 func (b *bootstrapRESTServer) VerifyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "VerifyHandler")
 	cfg := getServerSystemCfg()
@@ -106,6 +110,9 @@ func registerBootstrapRESTHandlers(router *mux.Router) {
 	server := &bootstrapRESTServer{}
 	subrouter := router.PathPrefix(bootstrapRESTPrefix).Subrouter()
 
+	subrouter.Methods(http.MethodPost).Path(bootstrapRESTVersionPrefix + bootstrapRESTMethodHealth).HandlerFunc(
+		httpTraceHdrs(server.HealthHandler))
+
 	subrouter.Methods(http.MethodPost).Path(bootstrapRESTVersionPrefix + bootstrapRESTMethodVerify).HandlerFunc(
 		httpTraceHdrs(server.VerifyHandler))
 }
@@ -114,29 +121,12 @@ func registerBootstrapRESTHandlers(router *mux.Router) {
 type bootstrapRESTClient struct {
 	endpoint   Endpoint
 	restClient *rest.Client
-	connected  int32
-}
-
-// Reconnect to a bootstrap rest server.k
-func (client *bootstrapRESTClient) reConnect() {
-	atomic.StoreInt32(&client.connected, 1)
-}
-
-// Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
-// permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
-// after verifying format.json
-func (client *bootstrapRESTClient) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
-	return client.callWithContext(GlobalContext, method, values, body, length)
 }
 
 // Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
 // permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
 // after verifying format.json
 func (client *bootstrapRESTClient) callWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
-	if !client.IsOnline() {
-		client.reConnect()
-	}
-
 	if values == nil {
 		values = make(url.Values)
 	}
@@ -146,10 +136,6 @@ func (client *bootstrapRESTClient) callWithContext(ctx context.Context, method s
 		return respBody, nil
 	}
 
-	if isNetworkError(err) {
-		atomic.StoreInt32(&client.connected, 0)
-	}
-
 	return nil, err
 }
 
@@ -158,24 +144,12 @@ func (client *bootstrapRESTClient) String() string {
 	return client.endpoint.String()
 }
 
-// IsOnline - returns whether RPC client failed to connect or not.
-func (client *bootstrapRESTClient) IsOnline() bool {
-	return atomic.LoadInt32(&client.connected) == 1
-}
-
-// Close - marks the client as closed.
-func (client *bootstrapRESTClient) Close() error {
-	atomic.StoreInt32(&client.connected, 0)
-	client.restClient.Close()
-	return nil
-}
-
 // Verify - fetches system server config.
-func (client *bootstrapRESTClient) Verify(srcCfg ServerSystemConfig) (err error) {
+func (client *bootstrapRESTClient) Verify(ctx context.Context, srcCfg ServerSystemConfig) (err error) {
 	if newObjectLayerFn() != nil {
 		return nil
 	}
-	respBody, err := client.call(bootstrapRESTMethodVerify, nil, nil, -1)
+	respBody, err := client.callWithContext(ctx, bootstrapRESTMethodVerify, nil, nil, -1)
 	if err != nil {
 		return
 	}
@@ -187,14 +161,17 @@ func (client *bootstrapRESTClient) Verify(srcCfg ServerSystemConfig) (err error)
 	return srcCfg.Diff(recvCfg)
 }
 
-func verifyServerSystemConfig(endpointZones EndpointZones) error {
+func verifyServerSystemConfig(ctx context.Context, endpointZones EndpointZones) error {
 	srcCfg := getServerSystemCfg()
 	clnts := newBootstrapRESTClients(endpointZones)
 	var onlineServers int
+	var offlineEndpoints []string
+	var retries int
 	for onlineServers < len(clnts)/2 {
 		for _, clnt := range clnts {
-			if err := clnt.Verify(srcCfg); err != nil {
+			if err := clnt.Verify(ctx, srcCfg); err != nil {
 				if isNetworkError(err) {
+					offlineEndpoints = append(offlineEndpoints, clnt.String())
 					continue
 				}
 				return fmt.Errorf("%s as has incorrect configuration: %w", clnt.String(), err)
@@ -204,6 +181,14 @@ func verifyServerSystemConfig(endpointZones EndpointZones) error {
 		// Sleep for a while - so that we don't go into
 		// 100% CPU when half the endpoints are offline.
 		time.Sleep(500 * time.Millisecond)
+		retries++
+		// after 5 retries start logging that servers are not reachable yet
+		if retries >= 5 {
+			logger.Info(fmt.Sprintf("Waiting for atleast %d servers to be online for bootstrap check", len(clnts)/2))
+			logger.Info(fmt.Sprintf("Following servers are currently offline or unreachable %s", offlineEndpoints))
+			retries = 0 // reset to log again after 5 retries.
+		}
+		offlineEndpoints = nil
 	}
 	return nil
 }
@@ -220,11 +205,7 @@ func newBootstrapRESTClients(endpointZones EndpointZones) []*bootstrapRESTClient
 
 			// Only proceed for remote endpoints.
 			if !endpoint.IsLocal {
-				clnt, err := newBootstrapRESTClient(endpoint)
-				if err != nil {
-					continue
-				}
-				clnts = append(clnts, clnt)
+				clnts = append(clnts, newBootstrapRESTClient(endpoint))
 			}
 		}
 	}
@@ -232,7 +213,7 @@ func newBootstrapRESTClients(endpointZones EndpointZones) []*bootstrapRESTClient
 }
 
 // Returns a new bootstrap client.
-func newBootstrapRESTClient(endpoint Endpoint) (*bootstrapRESTClient, error) {
+func newBootstrapRESTClient(endpoint Endpoint) *bootstrapRESTClient {
 	serverURL := &url.URL{
 		Scheme: endpoint.Scheme,
 		Host:   endpoint.Host,
@@ -248,10 +229,17 @@ func newBootstrapRESTClient(endpoint Endpoint) (*bootstrapRESTClient, error) {
 	}
 
 	trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)
-	restClient, err := rest.NewClient(serverURL, trFn, newAuthToken)
-	if err != nil {
-		return nil, err
+	restClient := rest.NewClient(serverURL, trFn, newAuthToken)
+	restClient.HealthCheckFn = func() bool {
+		ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
+		// Instantiate a new rest client for healthcheck
+		// to avoid recursive healthCheckFn()
+		respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).CallWithContext(ctx, bootstrapRESTMethodHealth, nil, nil, -1)
+		xhttp.DrainBody(respBody)
+		cancel()
+		var ne *rest.NetworkError
+		return !errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &ne)
 	}
 
-	return &bootstrapRESTClient{endpoint: endpoint, restClient: restClient, connected: 1}, nil
+	return &bootstrapRESTClient{endpoint: endpoint, restClient: restClient}
 }

cmd/bucket-encryption.go

@@ -55,5 +55,6 @@ func validateBucketSSEConfig(r io.Reader) (*bucketsse.BucketSSEConfig, error) {
 	if len(encConfig.Rules) == 1 && encConfig.Rules[0].DefaultEncryptionAction.Algorithm == bucketsse.AES256 {
 		return encConfig, nil
 	}
+
 	return nil, errors.New("Unsupported bucket encryption configuration")
 }

cmd/bucket-handlers.go

@@ -25,6 +25,7 @@ import (
 	"net/url"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	"github.com/gorilla/mux"
@@ -45,9 +46,8 @@ import (
 )
 
 const (
-	getBucketVersioningResponse = `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`
-	objectLockConfig            = "object-lock.xml"
-	bucketTaggingConfigFile     = "tagging.xml"
+	objectLockConfig    = "object-lock.xml"
+	bucketTaggingConfig = "tagging.xml"
 )
 
 // Check if there are buckets on server without corresponding entry in etcd backend and
@@ -377,80 +377,98 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
+	// Before proceeding validate if bucket exists.
+	_, err := objectAPI.GetBucketInfo(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
 	deleteObjectsFn := objectAPI.DeleteObjects
 	if api.CacheAPI() != nil {
 		deleteObjectsFn = api.CacheAPI().DeleteObjects
 	}
 
-	var objectsToDelete = map[string]int{}
+	var objectsToDelete = map[ObjectToDelete]int{}
 	getObjectInfoFn := objectAPI.GetObjectInfo
 	if api.CacheAPI() != nil {
 		getObjectInfoFn = api.CacheAPI().GetObjectInfo
 	}
 
-	var dErrs = make([]APIErrorCode, len(deleteObjects.Objects))
-
+	dErrs := make([]DeleteError, len(deleteObjects.Objects))
 	for index, object := range deleteObjects.Objects {
-		if dErrs[index] = checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, object.ObjectName); dErrs[index] != ErrNone {
-			if dErrs[index] == ErrSignatureDoesNotMatch || dErrs[index] == ErrInvalidAccessKeyID {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(dErrs[index]), r.URL, guessIsBrowserReq(r))
+		if apiErrCode := checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, object.ObjectName); apiErrCode != ErrNone {
+			if apiErrCode == ErrSignatureDoesNotMatch || apiErrCode == ErrInvalidAccessKeyID {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL, guessIsBrowserReq(r))
 				return
 			}
+			apiErr := errorCodes.ToAPIErr(apiErrCode)
+			dErrs[index] = DeleteError{
+				Code:      apiErr.Code,
+				Message:   apiErr.Description,
+				Key:       object.ObjectName,
+				VersionID: object.VersionID,
+			}
 			continue
 		}
 
-		if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
-			if apiErr := enforceRetentionBypassForDelete(ctx, r, bucket, object.ObjectName, getObjectInfoFn); apiErr != ErrNone {
-				dErrs[index] = apiErr
-				continue
+		if object.VersionID != "" {
+			if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
+				if apiErrCode := enforceRetentionBypassForDelete(ctx, r, bucket, object, getObjectInfoFn); apiErrCode != ErrNone {
+					apiErr := errorCodes.ToAPIErr(apiErrCode)
+					dErrs[index] = DeleteError{
+						Code:      apiErr.Code,
+						Message:   apiErr.Description,
+						Key:       object.ObjectName,
+						VersionID: object.VersionID,
+					}
+					continue
+				}
 			}
 		}
 
 		// Avoid duplicate objects, we use map to filter them out.
-		if _, ok := objectsToDelete[object.ObjectName]; !ok {
-			objectsToDelete[object.ObjectName] = index
+		if _, ok := objectsToDelete[object]; !ok {
+			objectsToDelete[object] = index
 		}
 	}
 
-	toNames := func(input map[string]int) (output []string) {
-		output = make([]string, len(input))
+	toNames := func(input map[ObjectToDelete]int) (output []ObjectToDelete) {
+		output = make([]ObjectToDelete, len(input))
 		idx := 0
-		for name := range input {
-			output[idx] = name
+		for obj := range input {
+			output[idx] = obj
 			idx++
 		}
 		return
 	}
 
 	deleteList := toNames(objectsToDelete)
-	errs, err := deleteObjectsFn(ctx, bucket, deleteList)
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
-	}
+	dObjects, errs := deleteObjectsFn(ctx, bucket, deleteList, ObjectOptions{
+		Versioned: globalBucketVersioningSys.Enabled(bucket),
+	})
 
-	for i, objName := range deleteList {
-		dIdx := objectsToDelete[objName]
-		dErrs[dIdx] = toAPIErrorCode(ctx, errs[i])
-	}
-
-	// Collect deleted objects and errors if any.
-	var deletedObjects []ObjectIdentifier
-	var deleteErrors []DeleteError
-	for index, errCode := range dErrs {
-		object := deleteObjects.Objects[index]
-		// Success deleted objects are collected separately.
-		if errCode == ErrNone || errCode == ErrNoSuchKey {
-			deletedObjects = append(deletedObjects, object)
+	deletedObjects := make([]DeletedObject, len(deleteObjects.Objects))
+	for i := range errs {
+		dindex := objectsToDelete[deleteList[i]]
+		apiErr := toAPIError(ctx, errs[i])
+		if apiErr.Code == "" || apiErr.Code == "NoSuchKey" {
+			deletedObjects[dindex] = dObjects[i]
 			continue
 		}
-		apiErr := getAPIError(errCode)
-		// Error during delete should be collected separately.
-		deleteErrors = append(deleteErrors, DeleteError{
-			Code:    apiErr.Code,
-			Message: apiErr.Description,
-			Key:     object.ObjectName,
-		})
+		dErrs[dindex] = DeleteError{
+			Code:      apiErr.Code,
+			Message:   apiErr.Description,
+			Key:       deleteList[i].ObjectName,
+			VersionID: deleteList[i].VersionID,
+		}
+	}
+
+	var deleteErrors []DeleteError
+	for _, dErr := range dErrs {
+		if dErr.Code != "" {
+			deleteErrors = append(deleteErrors, dErr)
+		}
 	}
 
 	// Generate response
@@ -462,12 +480,21 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	// Notify deleted event for objects.
 	for _, dobj := range deletedObjects {
+		objInfo := ObjectInfo{
+			Name:      dobj.ObjectName,
+			VersionID: dobj.VersionID,
+		}
+		if dobj.DeleteMarker {
+			objInfo = ObjectInfo{
+				Name:         dobj.ObjectName,
+				DeleteMarker: dobj.DeleteMarker,
+				VersionID:    dobj.DeleteMarkerVersionID,
+			}
+		}
 		sendEvent(eventArgs{
-			EventName:  event.ObjectRemovedDelete,
-			BucketName: bucket,
-			Object: ObjectInfo{
-				Name: dobj.ObjectName,
-			},
+			EventName:    event.ObjectRemovedDelete,
+			BucketName:   bucket,
+			Object:       objInfo,
 			ReqParams:    extractReqParams(r),
 			RespElements: extractRespElements(w),
 			UserAgent:    r.UserAgent(),
@@ -522,12 +549,17 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
+	opts := BucketOptions{
+		Location:    location,
+		LockEnabled: objectLockEnabled,
+	}
+
 	if globalDNSConfig != nil {
 		sr, err := globalDNSConfig.Get(bucket)
 		if err != nil {
 			if err == dns.ErrNoEntriesFound {
 				// Proceed to creating a bucket.
-				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location, objectLockEnabled); err != nil {
+				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, opts); err != nil {
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 					return
 				}
@@ -565,7 +597,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// Proceed to creating a bucket.
-	err := objectAPI.MakeBucketWithLocation(ctx, bucket, location, objectLockEnabled)
+	err := objectAPI.MakeBucketWithLocation(ctx, bucket, opts)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
@@ -797,9 +829,17 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	location := getObjectLocation(r, globalDomainNames, bucket, object)
+	// We must not use the http.Header().Set method here because some (broken)
+	// clients expect the ETag header key to be literally "ETag" - not "Etag" (case-sensitive).
+	// Therefore, we have to set the ETag directly as map entry.
 	w.Header()[xhttp.ETag] = []string{`"` + objInfo.ETag + `"`}
-	w.Header().Set(xhttp.Location, location)
+
+	// Set the relevant version ID as part of the response header.
+	if objInfo.VersionID != "" {
+		w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
+	}
+
+	w.Header().Set(xhttp.Location, getObjectLocation(r, globalDomainNames, bucket, object))
 
 	// Notify object created event.
 	defer sendEvent(eventArgs{
@@ -826,9 +866,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 			Bucket:   objInfo.Bucket,
 			Key:      objInfo.Name,
 			ETag:     `"` + objInfo.ETag + `"`,
-			Location: location,
+			Location: w.Header().Get(xhttp.Location),
 		})
-		writeResponse(w, http.StatusCreated, resp, "application/xml")
+		writeResponse(w, http.StatusCreated, resp, mimeXML)
 	case "200":
 		writeSuccessResponseHeadersOnly(w)
 	default:
@@ -886,34 +926,35 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		return
 	}
 
-	forceDelete := false
-	if value := r.Header.Get(xhttp.MinIOForceDelete); value != "" {
-		switch value {
-		case "true":
-			forceDelete = true
-		case "false":
-		default:
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL, guessIsBrowserReq(r))
-			return
-		}
+	// Verify if the caller has sufficient permissions.
+	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
 	}
 
-	if forceDelete {
+	forceDelete := false
+	if value := r.Header.Get(xhttp.MinIOForceDelete); value != "" {
+		var err error
+		forceDelete, err = strconv.ParseBool(value)
+		if err != nil {
+			apiErr := errorCodes.ToAPIErr(ErrInvalidRequest)
+			apiErr.Description = err.Error()
+			writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
+			return
+		}
+
+		// if force delete header is set, we need to evaluate the policy anyways
+		// regardless of it being true or not.
 		if s3Error := checkRequestAuthType(ctx, r, policy.ForceDeleteBucketAction, bucket, ""); s3Error != ErrNone {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 			return
 		}
-	} else {
-		if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
-			return
-		}
-	}
 
-	if forceDelete {
-		if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
-			return
+		if forceDelete {
+			if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
+				return
+			}
 		}
 	}
 
@@ -921,79 +962,30 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 
 	// Attempt to delete bucket.
 	if err := deleteBucket(ctx, bucket, forceDelete); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		if _, ok := err.(BucketNotEmpty); ok && (globalBucketVersioningSys.Enabled(bucket) || globalBucketVersioningSys.Suspended(bucket)) {
+			apiErr := toAPIError(ctx, err)
+			apiErr.Description = "The bucket you tried to delete is not empty. You must delete all versions in the bucket."
+			writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
+		} else {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		}
 		return
 	}
 
+	globalNotificationSys.DeleteBucketMetadata(ctx, bucket)
+
 	if globalDNSConfig != nil {
 		if err := globalDNSConfig.Delete(bucket); err != nil {
-			// Deleting DNS entry failed, attempt to create the bucket again.
-			objectAPI.MakeBucketWithLocation(ctx, bucket, "", false)
+			logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually using etcdctl", err))
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 
-	globalNotificationSys.DeleteBucketMetadata(ctx, bucket)
-
 	// Write success response.
 	writeSuccessNoContent(w)
 }
 
-// PutBucketVersioningHandler - PUT Bucket Versioning.
-// ----------
-// No-op. Available for API compatibility.
-func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "PutBucketVersioning")
-
-	defer logger.AuditLog(w, r, "PutBucketVersioning", mustGetClaimsFromToken(r))
-
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	objectAPI := api.ObjectAPI()
-	if objectAPI == nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	getBucketInfo := objectAPI.GetBucketInfo
-	if _, err := getBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	// Write success response.
-	writeSuccessResponseHeadersOnly(w)
-}
-
-// GetBucketVersioningHandler - GET Bucket Versioning.
-// ----------
-// No-op. Available for API compatibility.
-func (api objectAPIHandlers) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetBucketVersioning")
-
-	defer logger.AuditLog(w, r, "GetBucketVersioning", mustGetClaimsFromToken(r))
-
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	objectAPI := api.ObjectAPI()
-	if objectAPI == nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	getBucketInfo := objectAPI.GetBucketInfo
-	if _, err := getBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	// Write success response.
-	writeSuccessResponseXML(w, []byte(getBucketVersioningResponse))
-}
-
 // PutBucketObjectLockConfigHandler - PUT Bucket object lock configuration.
 // ----------
 // Places an Object Lock configuration on the specified bucket. The rule
@@ -1123,7 +1115,7 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(bucket, bucketTaggingConfigFile, configData); err != nil {
+	if err = globalBucketMetadataSys.Update(bucket, bucketTaggingConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -1191,7 +1183,7 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err := globalBucketMetadataSys.Update(bucket, bucketTaggingConfigFile, nil); err != nil {
+	if err := globalBucketMetadataSys.Update(bucket, bucketTaggingConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}

cmd/bucket-handlers_test.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"bytes"
 	"encoding/xml"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -28,7 +29,7 @@ import (
 	"github.com/minio/minio/pkg/auth"
 )
 
-// Wrapper for calling RemoveBucket HTTP handler tests for both XL multiple disks and single node setup.
+// Wrapper for calling RemoveBucket HTTP handler tests for both Erasure multiple disks and single node setup.
 func TestRemoveBucketHandler(t *testing.T) {
 	ExecObjectLayerAPITest(t, testRemoveBucketHandler, []string{"RemoveBucket"})
 }
@@ -73,7 +74,7 @@ func testRemoveBucketHandler(obj ObjectLayer, instanceType, bucketName string, a
 	}
 }
 
-// Wrapper for calling GetBucketPolicy HTTP handler tests for both XL multiple disks and single node setup.
+// Wrapper for calling GetBucketPolicy HTTP handler tests for both Erasure multiple disks and single node setup.
 func TestGetBucketLocationHandler(t *testing.T) {
 	ExecObjectLayerAPITest(t, testGetBucketLocationHandler, []string{"GetBucketLocation"})
 }
@@ -217,7 +218,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 	ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq)
 }
 
-// Wrapper for calling HeadBucket HTTP handler tests for both XL multiple disks and single node setup.
+// Wrapper for calling HeadBucket HTTP handler tests for both Erasure multiple disks and single node setup.
 func TestHeadBucketHandler(t *testing.T) {
 	ExecObjectLayerAPITest(t, testHeadBucketHandler, []string{"HeadBucket"})
 }
@@ -322,7 +323,7 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 	ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq)
 }
 
-// Wrapper for calling TestListMultipartUploadsHandler tests for both XL multiple disks and single node setup.
+// Wrapper for calling TestListMultipartUploadsHandler tests for both Erasure multiple disks and single node setup.
 func TestListMultipartUploadsHandler(t *testing.T) {
 	ExecObjectLayerAPITest(t, testListMultipartUploadsHandler, []string{"ListMultipartUploads"})
 }
@@ -559,7 +560,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 	ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq)
 }
 
-// Wrapper for calling TestListBucketsHandler tests for both XL multiple disks and single node setup.
+// Wrapper for calling TestListBucketsHandler tests for both Erasure multiple disks and single node setup.
 func TestListBucketsHandler(t *testing.T) {
 	ExecObjectLayerAPITest(t, testListBucketsHandler, []string{"ListBuckets"})
 }
@@ -653,7 +654,7 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 	ExecObjectLayerAPINilTest(t, "", "", instanceType, apiRouter, nilReq)
 }
 
-// Wrapper for calling DeleteMultipleObjects HTTP handler tests for both XL multiple disks and single node setup.
+// Wrapper for calling DeleteMultipleObjects HTTP handler tests for both Erasure multiple disks and single node setup.
 func TestAPIDeleteMultipleObjectsHandler(t *testing.T) {
 	ExecObjectLayerAPITest(t, testAPIDeleteMultipleObjectsHandler, []string{"DeleteMultipleObjects"})
 }
@@ -679,14 +680,17 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		objectNames = append(objectNames, objectName)
 	}
 
-	getObjectIdentifierList := func(objectNames []string) (objectIdentifierList []ObjectIdentifier) {
+	getObjectToDeleteList := func(objectNames []string) (objectList []ObjectToDelete) {
 		for _, objectName := range objectNames {
-			objectIdentifierList = append(objectIdentifierList, ObjectIdentifier{objectName})
+			objectList = append(objectList, ObjectToDelete{
+				ObjectName: objectName,
+			})
 		}
 
-		return objectIdentifierList
+		return objectList
 	}
-	getDeleteErrorList := func(objects []ObjectIdentifier) (deleteErrorList []DeleteError) {
+
+	getDeleteErrorList := func(objects []ObjectToDelete) (deleteErrorList []DeleteError) {
 		for _, obj := range objects {
 			deleteErrorList = append(deleteErrorList, DeleteError{
 				Code:    errorCodes[ErrAccessDenied].Code,
@@ -699,22 +703,38 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	}
 
 	requestList := []DeleteObjectsRequest{
-		{Quiet: false, Objects: getObjectIdentifierList(objectNames[:5])},
-		{Quiet: true, Objects: getObjectIdentifierList(objectNames[5:])},
+		{Quiet: false, Objects: getObjectToDeleteList(objectNames[:5])},
+		{Quiet: true, Objects: getObjectToDeleteList(objectNames[5:])},
 	}
 
 	// generate multi objects delete response.
 	successRequest0 := encodeResponse(requestList[0])
-	successResponse0 := generateMultiDeleteResponse(requestList[0].Quiet, requestList[0].Objects, nil)
+
+	deletedObjects := make([]DeletedObject, len(requestList[0].Objects))
+	for i := range requestList[0].Objects {
+		deletedObjects[i] = DeletedObject{
+			ObjectName: requestList[0].Objects[i].ObjectName,
+		}
+	}
+
+	successResponse0 := generateMultiDeleteResponse(requestList[0].Quiet, deletedObjects, nil)
 	encodedSuccessResponse0 := encodeResponse(successResponse0)
 
 	successRequest1 := encodeResponse(requestList[1])
-	successResponse1 := generateMultiDeleteResponse(requestList[1].Quiet, requestList[1].Objects, nil)
+
+	deletedObjects = make([]DeletedObject, len(requestList[1].Objects))
+	for i := range requestList[0].Objects {
+		deletedObjects[i] = DeletedObject{
+			ObjectName: requestList[1].Objects[i].ObjectName,
+		}
+	}
+
+	successResponse1 := generateMultiDeleteResponse(requestList[1].Quiet, deletedObjects, nil)
 	encodedSuccessResponse1 := encodeResponse(successResponse1)
 
 	// generate multi objects delete response for errors.
 	// errorRequest := encodeResponse(requestList[1])
-	errorResponse := generateMultiDeleteResponse(requestList[1].Quiet, requestList[1].Objects, nil)
+	errorResponse := generateMultiDeleteResponse(requestList[1].Quiet, deletedObjects, nil)
 	encodedErrorResponse := encodeResponse(errorResponse)
 
 	anonRequest := encodeResponse(requestList[0])
@@ -817,6 +837,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 
 		// Verify whether the bucket obtained object is same as the one created.
 		if testCase.expectedContent != nil && !bytes.Equal(testCase.expectedContent, actualContent) {
+			fmt.Println(string(testCase.expectedContent), string(actualContent))
 			t.Errorf("Test %d : MinIO %s: Object content differs from expected value.", i+1, instanceType)
 		}
 	}

cmd/bucket-lifecycle.go

@@ -21,7 +21,6 @@ import (
 )
 
 const (
-
 	// Disabled means the lifecycle rule is inactive
 	Disabled = "Disabled"
 )

cmd/bucket-listobjects-handlers.go

@@ -17,7 +17,10 @@
 package cmd
 
 import (
+	"context"
+	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 
 	"github.com/gorilla/mux"
@@ -49,13 +52,13 @@ func validateListObjectsArgs(marker, delimiter, encodingType string, maxKeys int
 	return ErrNone
 }
 
-// ListBucketObjectVersions - GET Bucket Object versions
+// ListObjectVersions - GET Bucket Object versions
 // You can use the versions subresource to list metadata about all
 // of the versions of objects in a bucket.
-func (api objectAPIHandlers) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListBucketObjectVersions")
+func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListObjectVersions")
 
-	defer logger.AuditLog(w, r, "ListBucketObjectVersions", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(w, r, "ListObjectVersions", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -74,8 +77,7 @@ func (api objectAPIHandlers) ListBucketObjectVersionsHandler(w http.ResponseWrit
 	urlValues := r.URL.Query()
 
 	// Extract all the listBucketVersions query params to their native values.
-	// versionIDMarker is ignored here.
-	prefix, marker, delimiter, maxkeys, encodingType, _, errCode := getListBucketObjectVersionsArgs(urlValues)
+	prefix, marker, delimiter, maxkeys, encodingType, versionIDMarker, errCode := getListBucketObjectVersionsArgs(urlValues)
 	if errCode != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
 		return
@@ -87,29 +89,29 @@ func (api objectAPIHandlers) ListBucketObjectVersionsHandler(w http.ResponseWrit
 		return
 	}
 
-	listObjects := objectAPI.ListObjects
+	listObjectVersions := objectAPI.ListObjectVersions
 
-	// Inititate a list objects operation based on the input params.
+	// Inititate a list object versions operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
 	// marshaled into S3 compatible XML header.
-	listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxkeys)
+	listObjectVersionsInfo, err := listObjectVersions(ctx, bucket, prefix, marker, versionIDMarker, delimiter, maxkeys)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	for i := range listObjectsInfo.Objects {
-		if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
-			listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false)
+	for i := range listObjectVersionsInfo.Objects {
+		if crypto.IsEncrypted(listObjectVersionsInfo.Objects[i].UserDefined) {
+			listObjectVersionsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectVersionsInfo.Objects[i], false)
 		}
-		listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].GetActualSize()
+		listObjectVersionsInfo.Objects[i].Size, err = listObjectVersionsInfo.Objects[i].GetActualSize()
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 
-	response := generateListVersionsResponse(bucket, prefix, marker, delimiter, encodingType, maxkeys, listObjectsInfo)
+	response := generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType, maxkeys, listObjectVersionsInfo)
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodeResponse(response))
@@ -158,6 +160,13 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
+	// Analyze continuation token and route the request accordingly
+	var success bool
+	token, success = proxyRequestByToken(ctx, w, r, token)
+	if success {
+		return
+	}
+
 	listObjectsV2 := objectAPI.ListObjectsV2
 
 	// Inititate a list objects operation based on the input params.
@@ -180,8 +189,13 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 		}
 	}
 
-	response := generateListObjectsV2Response(bucket, prefix, token,
-		listObjectsV2Info.NextContinuationToken, startAfter,
+	// The next continuation token has id@node_index format to optimize paginated listing
+	nextContinuationToken := listObjectsV2Info.NextContinuationToken
+	if nextContinuationToken != "" && listObjectsV2Info.IsTruncated {
+		nextContinuationToken = fmt.Sprintf("%s@%d", listObjectsV2Info.NextContinuationToken, getLocalNodeIndex())
+	}
+
+	response := generateListObjectsV2Response(bucket, prefix, token, nextContinuationToken, startAfter,
 		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
 		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, true)
 
@@ -232,6 +246,13 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
+	// Analyze continuation token and route the request accordingly
+	var success bool
+	token, success = proxyRequestByToken(ctx, w, r, token)
+	if success {
+		return
+	}
+
 	listObjectsV2 := objectAPI.ListObjectsV2
 
 	// Inititate a list objects operation based on the input params.
@@ -254,8 +275,13 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		}
 	}
 
-	response := generateListObjectsV2Response(bucket, prefix, token,
-		listObjectsV2Info.NextContinuationToken, startAfter,
+	// The next continuation token has id@node_index format to optimize paginated listing
+	nextContinuationToken := listObjectsV2Info.NextContinuationToken
+	if nextContinuationToken != "" && listObjectsV2Info.IsTruncated {
+		nextContinuationToken = fmt.Sprintf("%s@%d", listObjectsV2Info.NextContinuationToken, getLocalNodeIndex())
+	}
+
+	response := generateListObjectsV2Response(bucket, prefix, token, nextContinuationToken, startAfter,
 		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
 		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, false)
 
@@ -263,6 +289,60 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	writeSuccessResponseXML(w, encodeResponse(response))
 }
 
+func getLocalNodeIndex() int {
+	if len(globalProxyEndpoints) == 0 {
+		return -1
+	}
+	for i, ep := range globalProxyEndpoints {
+		if ep.IsLocal {
+			return i
+		}
+	}
+	return -1
+}
+
+func parseRequestToken(token string) (subToken string, nodeIndex int) {
+	if token == "" {
+		return token, -1
+	}
+	i := strings.Index(token, "@")
+	if i < 0 {
+		return token, -1
+	}
+	nodeIndex, err := strconv.Atoi(token[i+1:])
+	if err != nil {
+		return token, -1
+	}
+	subToken = token[:i]
+	return subToken, nodeIndex
+}
+
+func proxyRequestByToken(ctx context.Context, w http.ResponseWriter, r *http.Request, token string) (string, bool) {
+	subToken, nodeIndex := parseRequestToken(token)
+	if nodeIndex > 0 {
+		return subToken, proxyRequestByNodeIndex(ctx, w, r, nodeIndex)
+	}
+	return subToken, false
+}
+
+func proxyRequestByNodeIndex(ctx context.Context, w http.ResponseWriter, r *http.Request, index int) (success bool) {
+	if len(globalProxyEndpoints) == 0 {
+		return false
+	}
+	if index < 0 || index >= len(globalProxyEndpoints) {
+		return false
+	}
+	ep := globalProxyEndpoints[index]
+	if ep.IsLocal {
+		return false
+	}
+	return proxyRequest(ctx, w, r, ep)
+}
+
+func proxyRequestByBucket(ctx context.Context, w http.ResponseWriter, r *http.Request, bucket string) (success bool) {
+	return proxyRequestByNodeIndex(ctx, w, r, crcHashMod(bucket, len(globalProxyEndpoints)))
+}
+
 // ListObjectsV1Handler - GET Bucket (List Objects) Version 1.
 // --------------------------
 // This implementation of the GET operation returns some or all (up to 10000)
@@ -301,6 +381,10 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 		return
 	}
 
+	if proxyRequestByBucket(ctx, w, r, bucket) {
+		return
+	}
+
 	listObjects := objectAPI.ListObjects
 
 	// Inititate a list objects operation based on the input params.

cmd/bucket-metadata-sys.go

@@ -28,6 +28,7 @@ import (
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/bucket/versioning"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/sync/errgroup"
@@ -76,7 +77,44 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 
 	if globalIsGateway {
 		// This code is needed only for gateway implementations.
-		if configFile == bucketPolicyConfig {
+		switch configFile {
+		case bucketSSEConfig:
+			if globalGatewayName == "nas" {
+				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+				if err != nil {
+					return err
+				}
+				meta.EncryptionConfigXML = configData
+				return meta.Save(GlobalContext, objAPI)
+			}
+		case bucketLifecycleConfig:
+			if globalGatewayName == "nas" {
+				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+				if err != nil {
+					return err
+				}
+				meta.LifecycleConfigXML = configData
+				return meta.Save(GlobalContext, objAPI)
+			}
+		case bucketTaggingConfig:
+			if globalGatewayName == "nas" {
+				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+				if err != nil {
+					return err
+				}
+				meta.TaggingConfigXML = configData
+				return meta.Save(GlobalContext, objAPI)
+			}
+		case bucketNotificationConfig:
+			if globalGatewayName == "nas" {
+				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+				if err != nil {
+					return err
+				}
+				meta.NotificationConfigXML = configData
+				return meta.Save(GlobalContext, objAPI)
+			}
+		case bucketPolicyConfig:
 			if configData == nil {
 				return objAPI.DeleteBucketPolicy(GlobalContext, bucket)
 			}
@@ -107,10 +145,12 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 		meta.LifecycleConfigXML = configData
 	case bucketSSEConfig:
 		meta.EncryptionConfigXML = configData
-	case bucketTaggingConfigFile:
+	case bucketTaggingConfig:
 		meta.TaggingConfigXML = configData
 	case objectLockConfig:
 		meta.ObjectLockConfigXML = configData
+	case bucketVersioningConfig:
+		meta.VersioningConfigXML = configData
 	case bucketQuotaConfigFile:
 		meta.QuotaConfigJSON = configData
 	default:
@@ -147,6 +187,16 @@ func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
 	return meta, nil
 }
 
+// GetVersioningConfig returns configured versioning config
+// The returned object may not be modified.
+func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Versioning, error) {
+	meta, err := sys.GetConfig(bucket)
+	if err != nil {
+		return nil, err
+	}
+	return meta.versioningConfig, nil
+}
+
 // GetTaggingConfig returns configured tagging config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, error) {

cmd/bucket-metadata.go

@@ -32,6 +32,7 @@ import (
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/bucket/versioning"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/madmin"
 )
@@ -47,6 +48,7 @@ const (
 
 var (
 	enabledBucketObjectLockConfig = []byte(`<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><ObjectLockEnabled>Enabled</ObjectLockEnabled></ObjectLockConfiguration>`)
+	enabledBucketVersioningConfig = []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status></VersioningConfiguration>`)
 )
 
 //go:generate msgp -file $GOFILE
@@ -64,6 +66,7 @@ type BucketMetadata struct {
 	NotificationConfigXML []byte
 	LifecycleConfigXML    []byte
 	ObjectLockConfigXML   []byte
+	VersioningConfigXML   []byte
 	EncryptionConfigXML   []byte
 	TaggingConfigXML      []byte
 	QuotaConfigJSON       []byte
@@ -73,6 +76,7 @@ type BucketMetadata struct {
 	notificationConfig *event.Config
 	lifecycleConfig    *lifecycle.Lifecycle
 	objectLockConfig   *objectlock.Config
+	versioningConfig   *versioning.Versioning
 	sseConfig          *bucketsse.BucketSSEConfig
 	taggingConfig      *tags.Tags
 	quotaConfig        *madmin.BucketQuota
@@ -87,6 +91,9 @@ func newBucketMetadata(name string) BucketMetadata {
 			XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
 		},
 		quotaConfig: &madmin.BucketQuota{},
+		versioningConfig: &versioning.Versioning{
+			XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
+		},
 	}
 }
 
@@ -179,6 +186,10 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
 		b.taggingConfig = nil
 	}
 
+	if bytes.Equal(b.ObjectLockConfigXML, enabledBucketObjectLockConfig) {
+		b.VersioningConfigXML = enabledBucketVersioningConfig
+	}
+
 	if len(b.ObjectLockConfigXML) != 0 {
 		b.objectLockConfig, err = objectlock.ParseObjectLockConfig(bytes.NewReader(b.ObjectLockConfigXML))
 		if err != nil {
@@ -188,6 +199,13 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
 		b.objectLockConfig = nil
 	}
 
+	if len(b.VersioningConfigXML) != 0 {
+		b.versioningConfig, err = versioning.ParseConfig(bytes.NewReader(b.VersioningConfigXML))
+		if err != nil {
+			return err
+		}
+	}
+
 	if len(b.QuotaConfigJSON) != 0 {
 		b.quotaConfig, err = parseBucketQuota(b.Name, b.QuotaConfigJSON)
 		if err != nil {
@@ -206,7 +224,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 		bucketLifecycleConfig,
 		bucketQuotaConfigFile,
 		bucketSSEConfig,
-		bucketTaggingConfigFile,
+		bucketTaggingConfig,
 		objectLockConfig,
 	}
 
@@ -244,6 +262,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 		case legacyBucketObjectLockEnabledConfigFile:
 			if string(configData) == legacyBucketObjectLockEnabledConfig {
 				b.ObjectLockConfigXML = enabledBucketObjectLockConfig
+				b.VersioningConfigXML = enabledBucketVersioningConfig
 				b.LockEnabled = false // legacy value unset it
 				// we are only interested in b.ObjectLockConfigXML
 			}
@@ -255,10 +274,11 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 			b.LifecycleConfigXML = configData
 		case bucketSSEConfig:
 			b.EncryptionConfigXML = configData
-		case bucketTaggingConfigFile:
+		case bucketTaggingConfig:
 			b.TaggingConfigXML = configData
 		case objectLockConfig:
 			b.ObjectLockConfigXML = configData
+			b.VersioningConfigXML = enabledBucketVersioningConfig
 		case bucketQuotaConfigFile:
 			b.QuotaConfigJSON = configData
 		}
@@ -303,9 +323,15 @@ func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
 // deleteBucketMetadata deletes bucket metadata
 // If config does not exist no error is returned.
 func deleteBucketMetadata(ctx context.Context, obj ObjectLayer, bucket string) error {
-	configFile := path.Join(bucketConfigPrefix, bucket, bucketMetadataFile)
-	if err := deleteConfig(ctx, obj, configFile); err != nil && err != errConfigNotFound {
-		return err
+	metadataFiles := []string{
+		dataUsageCacheName,
+		bucketMetadataFile,
+	}
+	for _, metaFile := range metadataFiles {
+		configFile := path.Join(bucketConfigPrefix, bucket, metaFile)
+		if err := deleteConfig(ctx, obj, configFile); err != nil && err != errConfigNotFound {
+			return err
+		}
 	}
 	return nil
 }

cmd/bucket-metadata_gen.go

@@ -66,6 +66,12 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "ObjectLockConfigXML")
 				return
 			}
+		case "VersioningConfigXML":
+			z.VersioningConfigXML, err = dc.ReadBytes(z.VersioningConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "VersioningConfigXML")
+				return
+			}
 		case "EncryptionConfigXML":
 			z.EncryptionConfigXML, err = dc.ReadBytes(z.EncryptionConfigXML)
 			if err != nil {
@@ -97,9 +103,9 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 10
+	// map header, size 11
 	// write "Name"
-	err = en.Append(0x8a, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	err = en.Append(0x8b, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	if err != nil {
 		return
 	}
@@ -168,6 +174,16 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "ObjectLockConfigXML")
 		return
 	}
+	// write "VersioningConfigXML"
+	err = en.Append(0xb3, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.VersioningConfigXML)
+	if err != nil {
+		err = msgp.WrapError(err, "VersioningConfigXML")
+		return
+	}
 	// write "EncryptionConfigXML"
 	err = en.Append(0xb3, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
 	if err != nil {
@@ -204,9 +220,9 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 10
+	// map header, size 11
 	// string "Name"
-	o = append(o, 0x8a, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = append(o, 0x8b, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	o = msgp.AppendString(o, z.Name)
 	// string "Created"
 	o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
@@ -226,6 +242,9 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "ObjectLockConfigXML"
 	o = append(o, 0xb3, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
 	o = msgp.AppendBytes(o, z.ObjectLockConfigXML)
+	// string "VersioningConfigXML"
+	o = append(o, 0xb3, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	o = msgp.AppendBytes(o, z.VersioningConfigXML)
 	// string "EncryptionConfigXML"
 	o = append(o, 0xb3, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
 	o = msgp.AppendBytes(o, z.EncryptionConfigXML)
@@ -298,6 +317,12 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "ObjectLockConfigXML")
 				return
 			}
+		case "VersioningConfigXML":
+			z.VersioningConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.VersioningConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "VersioningConfigXML")
+				return
+			}
 		case "EncryptionConfigXML":
 			z.EncryptionConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.EncryptionConfigXML)
 			if err != nil {
@@ -330,6 +355,6 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BucketMetadata) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON)
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON)
 	return
 }

cmd/bucket-object-lock.go

@@ -52,79 +52,6 @@ func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention,
 	return config.ToRetention(), nil
 }
 
-// Similar to enforceRetentionBypassForDelete but for WebUI
-func enforceRetentionBypassForDeleteWeb(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, govBypassPerms bool) APIErrorCode {
-	opts, err := getOpts(ctx, r, bucket, object)
-	if err != nil {
-		return toAPIErrorCode(ctx, err)
-	}
-
-	oi, err := getObjectInfoFn(ctx, bucket, object, opts)
-	if err != nil {
-		return toAPIErrorCode(ctx, err)
-	}
-
-	lhold := objectlock.GetObjectLegalHoldMeta(oi.UserDefined)
-	if lhold.Status.Valid() && lhold.Status == objectlock.LegalHoldOn {
-		return ErrObjectLocked
-	}
-
-	ret := objectlock.GetObjectRetentionMeta(oi.UserDefined)
-	if ret.Mode.Valid() {
-		switch ret.Mode {
-		case objectlock.RetCompliance:
-			// In compliance mode, a protected object version can't be overwritten
-			// or deleted by any user, including the root user in your AWS account.
-			// When an object is locked in compliance mode, its retention mode can't
-			// be changed, and its retention period can't be shortened. Compliance mode
-			// ensures that an object version can't be overwritten or deleted for the
-			// duration of the retention period.
-			t, err := objectlock.UTCNowNTP()
-			if err != nil {
-				logger.LogIf(ctx, err)
-				return ErrObjectLocked
-			}
-
-			if !ret.RetainUntilDate.Before(t) {
-				return ErrObjectLocked
-			}
-			return ErrNone
-		case objectlock.RetGovernance:
-			// In governance mode, users can't overwrite or delete an object
-			// version or alter its lock settings unless they have special
-			// permissions. With governance mode, you protect objects against
-			// being deleted by most users, but you can still grant some users
-			// permission to alter the retention settings or delete the object
-			// if necessary. You can also use governance mode to test retention-period
-			// settings before creating a compliance-mode retention period.
-			// To override or remove governance-mode retention settings, a
-			// user must have the s3:BypassGovernanceRetention permission
-			// and must explicitly include x-amz-bypass-governance-retention:true
-			// as a request header with any request that requires overriding
-			// governance mode.
-			byPassSet := govBypassPerms && objectlock.IsObjectLockGovernanceBypassSet(r.Header)
-			if !byPassSet {
-				t, err := objectlock.UTCNowNTP()
-				if err != nil {
-					logger.LogIf(ctx, err)
-					return ErrObjectLocked
-				}
-
-				if !ret.RetainUntilDate.Before(t) {
-					return ErrObjectLocked
-				}
-
-				if !govBypassPerms {
-					return ErrObjectLocked
-				}
-
-				return ErrNone
-			}
-		}
-	}
-	return ErrNone
-}
-
 // enforceRetentionForDeletion checks if it is appropriate to remove an
 // object according to locking configuration when this is lifecycle/ bucket quota asking.
 func enforceRetentionForDeletion(ctx context.Context, objInfo ObjectInfo) (locked bool) {
@@ -153,14 +80,23 @@ func enforceRetentionForDeletion(ctx context.Context, objInfo ObjectInfo) (locke
 // For objects in "Governance" mode, overwrite is allowed if a) object retention date is past OR
 // governance bypass headers are set and user has governance bypass permissions.
 // Objects in "Compliance" mode can be overwritten only if retention date is past.
-func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn) APIErrorCode {
-	opts, err := getOpts(ctx, r, bucket, object)
+func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucket string, object ObjectToDelete, getObjectInfoFn GetObjectInfoFn) APIErrorCode {
+	opts, err := getOpts(ctx, r, bucket, object.ObjectName)
 	if err != nil {
 		return toAPIErrorCode(ctx, err)
 	}
 
-	oi, err := getObjectInfoFn(ctx, bucket, object, opts)
+	opts.VersionID = object.VersionID
+
+	oi, err := getObjectInfoFn(ctx, bucket, object.ObjectName, opts)
 	if err != nil {
+		switch err.(type) {
+		case MethodNotAllowed: // This happens usually for a delete marker
+			if oi.DeleteMarker {
+				// Delete marker should be present and valid.
+				return ErrNone
+			}
+		}
 		return toAPIErrorCode(ctx, err)
 	}
 
@@ -219,8 +155,8 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 			// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
 			// If you try to delete objects protected by governance mode and have s3:BypassGovernanceRetention
 			// or s3:GetBucketObjectLockConfiguration permissions, the operation will succeed.
-			govBypassPerms1 := checkRequestAuthType(ctx, r, policy.BypassGovernanceRetentionAction, bucket, object)
-			govBypassPerms2 := checkRequestAuthType(ctx, r, policy.GetBucketObjectLockConfigurationAction, bucket, object)
+			govBypassPerms1 := checkRequestAuthType(ctx, r, policy.BypassGovernanceRetentionAction, bucket, object.ObjectName)
+			govBypassPerms2 := checkRequestAuthType(ctx, r, policy.GetBucketObjectLockConfigurationAction, bucket, object.ObjectName)
 			if govBypassPerms1 != ErrNone && govBypassPerms2 != ErrNone {
 				return ErrAccessDenied
 			}
@@ -331,30 +267,32 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
 		return mode, retainDate, legalHold, ErrNone
 	}
 
-	var objExists bool
 	opts, err := getOpts(ctx, r, bucket, object)
 	if err != nil {
 		return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
 	}
 
-	t, err := objectlock.UTCNowNTP()
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return mode, retainDate, legalHold, ErrObjectLocked
-	}
+	if opts.VersionID != "" {
+		if objInfo, err := getObjectInfoFn(ctx, bucket, object, opts); err == nil {
+			r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
 
-	if objInfo, err := getObjectInfoFn(ctx, bucket, object, opts); err == nil {
-		objExists = true
-		r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
-		if r.Mode == objectlock.RetCompliance && r.RetainUntilDate.After(t) {
-			return mode, retainDate, legalHold, ErrObjectLocked
-		}
-		mode = r.Mode
-		retainDate = r.RetainUntilDate
-		legalHold = objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined)
-		// Disallow overwriting an object on legal hold
-		if legalHold.Status == objectlock.LegalHoldOn {
-			return mode, retainDate, legalHold, ErrObjectLocked
+			t, err := objectlock.UTCNowNTP()
+			if err != nil {
+				logger.LogIf(ctx, err)
+				return mode, retainDate, legalHold, ErrObjectLocked
+			}
+
+			if r.Mode == objectlock.RetCompliance && r.RetainUntilDate.After(t) {
+				return mode, retainDate, legalHold, ErrObjectLocked
+			}
+
+			mode = r.Mode
+			retainDate = r.RetainUntilDate
+			legalHold = objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined)
+			// Disallow overwriting an object on legal hold
+			if legalHold.Status == objectlock.LegalHoldOn {
+				return mode, retainDate, legalHold, ErrObjectLocked
+			}
 		}
 	}
 
@@ -374,9 +312,6 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
 		if err != nil {
 			return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
 		}
-		if objExists && retainDate.After(t) {
-			return mode, retainDate, legalHold, ErrObjectLocked
-		}
 		if retentionPermErr != ErrNone {
 			return mode, retainDate, legalHold, retentionPermErr
 		}
@@ -387,16 +322,14 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
 		if retentionPermErr != ErrNone {
 			return mode, retainDate, legalHold, retentionPermErr
 		}
+
 		t, err := objectlock.UTCNowNTP()
 		if err != nil {
 			logger.LogIf(ctx, err)
 			return mode, retainDate, legalHold, ErrObjectLocked
 		}
-		// AWS S3 just creates a new version of object when an object is being overwritten.
-		if objExists && retainDate.After(t) {
-			return mode, retainDate, legalHold, ErrObjectLocked
-		}
-		if !legalHoldRequested {
+
+		if !legalHoldRequested && retentionCfg.LockEnabled {
 			// inherit retention from bucket configuration
 			return retentionCfg.Mode, objectlock.RetentionDate{Time: t.Add(retentionCfg.Validity)}, legalHold, ErrNone
 		}

cmd/bucket-policy-handlers.go

@@ -164,7 +164,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Read bucket access policy.
-	config, err := globalBucketMetadataSys.GetPolicyConfig(bucket)
+	config, err := globalPolicySys.Get(bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return

cmd/bucket-policy-handlers_test.go

@@ -92,7 +92,7 @@ func getAnonWriteOnlyObjectPolicy(bucketName, prefix string) *policy.Policy {
 	}
 }
 
-// Wrapper for calling Put Bucket Policy HTTP handler tests for both XL multiple disks and single node setup.
+// Wrapper for calling Put Bucket Policy HTTP handler tests for both Erasure multiple disks and single node setup.
 func TestPutBucketPolicyHandler(t *testing.T) {
 	ExecObjectLayerAPITest(t, testPutBucketPolicyHandler, []string{"PutBucketPolicy"})
 }
@@ -102,7 +102,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	credentials auth.Credentials, t *testing.T) {
 
 	bucketName1 := fmt.Sprintf("%s-1", bucketName)
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, "", false); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, BucketOptions{}); err != nil {
 		t.Fatal(err)
 	}
 
@@ -314,7 +314,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 
 }
 
-// Wrapper for calling Get Bucket Policy HTTP handler tests for both XL multiple disks and single node setup.
+// Wrapper for calling Get Bucket Policy HTTP handler tests for both Erasure multiple disks and single node setup.
 func TestGetBucketPolicyHandler(t *testing.T) {
 	ExecObjectLayerAPITest(t, testGetBucketPolicyHandler, []string{"PutBucketPolicy", "GetBucketPolicy"})
 }
@@ -520,7 +520,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq)
 }
 
-// Wrapper for calling Delete Bucket Policy HTTP handler tests for both XL multiple disks and single node setup.
+// Wrapper for calling Delete Bucket Policy HTTP handler tests for both Erasure multiple disks and single node setup.
 func TestDeleteBucketPolicyHandler(t *testing.T) {
 	ExecObjectLayerAPITest(t, testDeleteBucketPolicyHandler, []string{"PutBucketPolicy", "DeleteBucketPolicy"})
 }

cmd/bucket-policy.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2018,2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -70,6 +70,16 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 		principalType = "User"
 	}
 
+	vid := r.URL.Query().Get("versionId")
+	if vid == "" {
+		if u, err := url.Parse(r.Header.Get(xhttp.AmzCopySource)); err == nil {
+			vid = u.Query().Get("versionId")
+		}
+		if vid == "" {
+			vid = r.Header.Get(xhttp.AmzCopySourceVersionID)
+		}
+	}
+
 	args := map[string][]string{
 		"CurrentTime":     {currTime.Format(time.RFC3339)},
 		"EpochTime":       {strconv.FormatInt(currTime.Unix(), 10)},
@@ -80,6 +90,7 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 		"principaltype":   {principalType},
 		"userid":          {username},
 		"username":        {username},
+		"versionid":       {vid},
 	}
 
 	if lc != "" {
@@ -142,7 +153,7 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 	return args
 }
 
-// PolicyToBucketAccessPolicy - converts policy.Policy to minio-go/policy.BucketAccessPolicy.
+// PolicyToBucketAccessPolicy converts a MinIO policy into a minio-go policy data structure.
 func PolicyToBucketAccessPolicy(bucketPolicy *policy.Policy) (*miniogopolicy.BucketAccessPolicy, error) {
 	// Return empty BucketAccessPolicy for empty bucket policy.
 	if bucketPolicy == nil {

cmd/bucket-quota.go

@@ -138,7 +138,7 @@ func startBucketQuotaEnforcement(ctx context.Context, objAPI ObjectLayer) {
 		case <-ctx.Done():
 			return
 		case <-time.NewTimer(bgQuotaInterval).C:
-			logger.LogIf(ctx, enforceFIFOQuota(ctx, objAPI))
+			enforceFIFOQuota(ctx, objAPI)
 		}
 
 	}
@@ -146,20 +146,22 @@ func startBucketQuotaEnforcement(ctx context.Context, objAPI ObjectLayer) {
 
 // enforceFIFOQuota deletes objects in FIFO order until sufficient objects
 // have been deleted so as to bring bucket usage within quota
-func enforceFIFOQuota(ctx context.Context, objectAPI ObjectLayer) error {
+func enforceFIFOQuota(ctx context.Context, objectAPI ObjectLayer) {
 	// Turn off quota enforcement if data usage info is unavailable.
 	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOff {
-		return nil
+		return
 	}
 
 	buckets, err := objectAPI.ListBuckets(ctx)
 	if err != nil {
-		return err
+		logger.LogIf(ctx, err)
+		return
 	}
 
 	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
 	if err != nil {
-		return err
+		logger.LogIf(ctx, err)
+		return
 	}
 
 	for _, binfo := range buckets {
@@ -194,9 +196,12 @@ func enforceFIFOQuota(ctx context.Context, objectAPI ObjectLayer) error {
 		// Allocate new results channel to receive ObjectInfo.
 		objInfoCh := make(chan ObjectInfo)
 
+		versioned := globalBucketVersioningSys.Enabled(bucket)
+
 		// Walk through all objects
-		if err := objectAPI.Walk(ctx, bucket, "", objInfoCh); err != nil {
-			return err
+		if err := objectAPI.Walk(ctx, bucket, "", objInfoCh, ObjectOptions{WalkVersions: versioned}); err != nil {
+			logger.LogIf(ctx, err)
+			continue
 		}
 
 		// reuse the fileScorer used by disk cache to score entries by
@@ -205,53 +210,59 @@ func enforceFIFOQuota(ctx context.Context, objectAPI ObjectLayer) error {
 		// irrelevant.
 		scorer, err := newFileScorer(toFree, time.Now().Unix(), 1)
 		if err != nil {
-			return err
+			logger.LogIf(ctx, err)
+			continue
 		}
 
 		rcfg, _ := globalBucketObjectLockSys.Get(bucket)
-
 		for obj := range objInfoCh {
+			if obj.DeleteMarker {
+				// Delete markers are automatically added for FIFO purge.
+				scorer.addFileWithObjInfo(obj, 1)
+				continue
+			}
 			// skip objects currently under retention
 			if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
 				continue
 			}
-			scorer.addFile(obj.Name, obj.ModTime, obj.Size, 1)
+			scorer.addFileWithObjInfo(obj, 1)
 		}
-		var objects []string
-		numKeys := len(scorer.fileNames())
-		for i, key := range scorer.fileNames() {
-			objects = append(objects, key)
+
+		var objects []ObjectToDelete
+		numKeys := len(scorer.fileObjInfos())
+		for i, obj := range scorer.fileObjInfos() {
+			objects = append(objects, ObjectToDelete{
+				ObjectName: obj.Name,
+				VersionID:  obj.VersionID,
+			})
 			if len(objects) < maxDeleteList && (i < numKeys-1) {
-				// skip deletion until maxObjectList or end of slice
+				// skip deletion until maxDeleteList or end of slice
 				continue
 			}
 
 			if len(objects) == 0 {
 				break
 			}
+
 			// Deletes a list of objects.
-			deleteErrs, err := objectAPI.DeleteObjects(ctx, bucket, objects)
-			if err != nil {
-				logger.LogIf(ctx, err)
-			} else {
-				for i := range deleteErrs {
-					if deleteErrs[i] != nil {
-						logger.LogIf(ctx, deleteErrs[i])
-						continue
-					}
-					// Notify object deleted event.
-					sendEvent(eventArgs{
-						EventName:  event.ObjectRemovedDelete,
-						BucketName: bucket,
-						Object: ObjectInfo{
-							Name: objects[i],
-						},
-						Host: "Internal: [FIFO-QUOTA-EXPIRY]",
-					})
+			_, deleteErrs := objectAPI.DeleteObjects(ctx, bucket, objects, ObjectOptions{
+				Versioned: versioned,
+			})
+			for i := range deleteErrs {
+				if deleteErrs[i] != nil {
+					logger.LogIf(ctx, deleteErrs[i])
+					continue
 				}
-				objects = nil
+
+				// Notify object deleted event.
+				sendEvent(eventArgs{
+					EventName:  event.ObjectRemovedDelete,
+					BucketName: bucket,
+					Object:     obj,
+					Host:       "Internal: [FIFO-QUOTA-EXPIRY]",
+				})
 			}
+			objects = nil
 		}
 	}
-	return nil
 }

cmd/bucket-versioning-handler.go

@@ -0,0 +1,130 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"io"
+	"net/http"
+
+	humanize "github.com/dustin/go-humanize"
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/bucket/versioning"
+)
+
+const (
+	bucketVersioningConfig = "versioning.xml"
+
+	// Maximum size of bucket versioning configuration payload sent to the PutBucketVersioningHandler.
+	maxBucketVersioningConfigSize = 1 * humanize.MiByte
+)
+
+// PutBucketVersioningHandler - PUT Bucket Versioning.
+// ----------
+func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutBucketVersioning")
+
+	defer logger.AuditLog(w, r, "PutBucketVersioning", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketVersioningAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	v, err := versioning.ParseConfig(io.LimitReader(r.Body, maxBucketVersioningConfigSize))
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && v.Suspended() {
+		writeErrorResponse(ctx, w, APIError{
+			Code:           "InvalidBucketState",
+			Description:    "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.",
+			HTTPStatusCode: http.StatusConflict,
+		}, r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	configData, err := xml.Marshal(v)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if err = globalBucketMetadataSys.Update(bucket, bucketVersioningConfig, configData); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	writeSuccessResponseHeadersOnly(w)
+}
+
+// GetBucketVersioningHandler - GET Bucket Versioning.
+// ----------
+func (api objectAPIHandlers) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetBucketVersioning")
+
+	defer logger.AuditLog(w, r, "GetBucketVersioning", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketVersioningAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	config, err := globalBucketVersioningSys.Get(bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	configData, err := xml.Marshal(config)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Write bucket versioning configuration to client
+	writeSuccessResponseXML(w, configData)
+
+}

cmd/bucket-versioning.go

@@ -0,0 +1,57 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "github.com/minio/minio/pkg/bucket/versioning"
+
+// BucketVersioningSys - policy subsystem.
+type BucketVersioningSys struct{}
+
+// Enabled enabled versioning?
+func (sys *BucketVersioningSys) Enabled(bucket string) bool {
+	vc, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
+	if err != nil {
+		return false
+	}
+	return vc.Enabled()
+}
+
+// Suspended suspended versioning?
+func (sys *BucketVersioningSys) Suspended(bucket string) bool {
+	vc, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
+	if err != nil {
+		return false
+	}
+	return vc.Suspended()
+}
+
+// Get returns stored bucket policy
+func (sys *BucketVersioningSys) Get(bucket string) (*versioning.Versioning, error) {
+	if globalIsGateway {
+		objAPI := newObjectLayerFn()
+		if objAPI == nil {
+			return nil, errServerNotInitialized
+		}
+		return nil, NotImplemented{}
+	}
+	return globalBucketMetadataSys.GetVersioningConfig(bucket)
+}
+
+// NewBucketVersioningSys - creates new versioning system.
+func NewBucketVersioningSys() *BucketVersioningSys {
+	return &BucketVersioningSys{}
+}

cmd/build-constants.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2015, 2016 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2015-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,15 +25,15 @@ var (
 	// GOROOT - GOROOT value at the time of build.
 	GOROOT = ""
 
-	// Go get development tag.
-	goGetTag = "DEVELOPMENT.GOGET"
-
 	// Version - version time.RFC3339.
-	Version = goGetTag
+	Version = "DEVELOPMENT.GOGET"
+
 	// ReleaseTag - release tag in TAG.%Y-%m-%dT%H-%M-%SZ.
-	ReleaseTag = goGetTag
+	ReleaseTag = "DEVELOPMENT.GOGET"
+
 	// CommitID - latest commit id.
-	CommitID = goGetTag
+	CommitID = "DEVELOPMENT.GOGET"
+
 	// ShortCommitID - first 12 characters from CommitID.
-	ShortCommitID = CommitID[:12]
+	ShortCommitID = "DEVELOPMENT.GOGET"
 )

cmd/config-common.go

@@ -50,7 +50,7 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]b
 }
 
 func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
-	err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile)
+	_, err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile, ObjectOptions{})
 	if err != nil && isErrObjectNotFound(err) {
 		return errConfigNotFound
 	}

cmd/config-current.go

@@ -59,7 +59,7 @@ func initHelp() {
 	for k, v := range notify.DefaultNotificationKVS {
 		kvs[k] = v
 	}
-	if globalIsXL {
+	if globalIsErasure {
 		kvs[config.StorageClassSubSys] = storageclass.DefaultKVS
 	}
 	config.RegisterDefaultKVS(kvs)
@@ -168,7 +168,7 @@ func initHelp() {
 		},
 	}
 
-	if globalIsXL {
+	if globalIsErasure {
 		helpSubSys = append(helpSubSys, config.HelpKV{})
 		copy(helpSubSys[2:], helpSubSys[1:])
 		helpSubSys[1] = config.HelpKV{
@@ -232,9 +232,9 @@ func validateConfig(s config.Config) error {
 		return err
 	}
 
-	if globalIsXL {
+	if globalIsErasure {
 		if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default],
-			globalXLSetDriveCount); err != nil {
+			globalErasureSetDriveCount); err != nil {
 			return err
 		}
 	}
@@ -325,13 +325,40 @@ func lookupConfigs(s config.Config) {
 
 	etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
+		if globalIsGateway {
+			logger.FatalIf(err, "Unable to initialize etcd config")
+		} else {
+			logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
+		}
 	}
 
 	if etcdCfg.Enabled {
-		globalEtcdClient, err = etcd.New(etcdCfg)
-		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
+		if globalEtcdClient == nil {
+			globalEtcdClient, err = etcd.New(etcdCfg)
+			if err != nil {
+				if globalIsGateway {
+					logger.FatalIf(err, "Unable to initialize etcd config")
+				} else {
+					logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
+				}
+			}
+		}
+
+		if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil && globalDNSConfig == nil {
+			globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
+				dns.DomainNames(globalDomainNames),
+				dns.DomainIPs(globalDomainIPs),
+				dns.DomainPort(globalMinioPort),
+				dns.CoreDNSPath(etcdCfg.CoreDNSPath),
+			)
+			if err != nil {
+				if globalIsGateway {
+					logger.FatalIf(err, "Unable to initialize DNS config")
+				} else {
+					logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
+						globalDomainNames, err))
+				}
+			}
 		}
 	}
 
@@ -342,19 +369,6 @@ func lookupConfigs(s config.Config) {
 	// but not federation.
 	globalBucketFederation = etcdCfg.PathPrefix == "" && etcdCfg.Enabled
 
-	if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
-		globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
-			dns.DomainNames(globalDomainNames),
-			dns.DomainIPs(globalDomainIPs),
-			dns.DomainPort(globalMinioPort),
-			dns.CoreDNSPath(etcdCfg.CoreDNSPath),
-		)
-		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
-				globalDomainNames, err))
-		}
-	}
-
 	globalServerRegion, err = config.LookupRegion(s[config.RegionSubSys][config.Default])
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Invalid region configuration: %w", err))
@@ -367,9 +381,9 @@ func lookupConfigs(s config.Config) {
 
 	globalAPIConfig.init(apiConfig)
 
-	if globalIsXL {
+	if globalIsErasure {
 		globalStorageClass, err = storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default],
-			globalXLSetDriveCount)
+			globalErasureSetDriveCount)
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
 		}
@@ -377,7 +391,11 @@ func lookupConfigs(s config.Config) {
 
 	globalCacheConfig, err = cache.LookupConfig(s[config.CacheSubSys][config.Default])
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
+		if globalIsGateway {
+			logger.FatalIf(err, "Unable to setup cache")
+		} else {
+			logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
+		}
 	}
 
 	if globalCacheConfig.Enabled {
@@ -464,12 +482,12 @@ func lookupConfigs(s config.Config) {
 		}
 	}
 
-	globalConfigTargetList, err = notify.GetNotificationTargets(s, GlobalContext.Done(), NewGatewayHTTPTransport())
+	globalConfigTargetList, err = notify.GetNotificationTargets(s, GlobalContext.Done(), NewGatewayHTTPTransport(), false)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
 
-	globalEnvTargetList, err = notify.GetNotificationTargets(newServerConfig(), GlobalContext.Done(), NewGatewayHTTPTransport())
+	globalEnvTargetList, err = notify.GetNotificationTargets(newServerConfig(), GlobalContext.Done(), NewGatewayHTTPTransport(), true)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
@@ -561,9 +579,6 @@ func newSrvConfig(objAPI ObjectLayer) error {
 	// Initialize server config.
 	srvCfg := newServerConfig()
 
-	// Override any values from ENVs.
-	lookupConfigs(srvCfg)
-
 	// hold the mutex lock before a new config is assigned.
 	globalServerConfigMu.Lock()
 	globalServerConfig = srvCfg

cmd/config-encrypted.go

@@ -24,17 +24,14 @@ import (
 	"time"
 	"unicode/utf8"
 
-	etcd "github.com/coreos/etcd/clientv3"
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/madmin"
+	etcd "go.etcd.io/etcd/v3/clientv3"
 )
 
-func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
-	if !server {
-		return nil
-	}
+func handleEncryptedConfigBackend(objAPI ObjectLayer) error {
 
 	encrypted, err := checkBackendEncrypted(objAPI)
 	if err != nil {

cmd/config.go

@@ -23,7 +23,6 @@ import (
 	"path"
 	"sort"
 	"strings"
-	"time"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/cmd/config"
@@ -92,7 +91,8 @@ func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData b
 
 func delServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) error {
 	historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV+kvPrefix)
-	return objAPI.DeleteObject(ctx, minioMetaBucket, historyFile)
+	_, err := objAPI.DeleteObject(ctx, minioMetaBucket, historyFile, ObjectOptions{})
+	return err
 }
 
 func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) ([]byte, error) {
@@ -149,7 +149,7 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
 	if err != nil {
 		// Config not found for some reason, allow things to continue
 		// by initializing a new fresh config in safe mode.
-		if err == errConfigNotFound && globalSafeMode {
+		if err == errConfigNotFound && newObjectLayerFn() == nil {
 			return newServerConfig(), nil
 		}
 		return nil, err
@@ -183,23 +183,6 @@ func (sys *ConfigSys) Load(objAPI ObjectLayer) error {
 	return sys.Init(objAPI)
 }
 
-// WatchConfigNASDisk - watches nas disk on periodic basis.
-func (sys *ConfigSys) WatchConfigNASDisk(ctx context.Context, objAPI ObjectLayer) {
-	configInterval := globalRefreshIAMInterval
-	watchDisk := func() {
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-time.After(configInterval):
-				loadConfig(objAPI)
-			}
-		}
-	}
-	// Refresh configSys in background for NAS gateway.
-	go watchDisk()
-}
-
 // Init - initializes config system from config.json.
 func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
 	if objAPI == nil {

cmd/config/api/help.go

@@ -33,12 +33,6 @@ var (
 			Optional:    true,
 			Type:        "duration",
 		},
-		config.HelpKV{
-			Key:         apiReadyDeadline,
-			Description: `set the deadline for health check API /minio/health/ready e.g. "1m"`,
-			Optional:    true,
-			Type:        "duration",
-		},
 		config.HelpKV{
 			Key:         apiCorsAllowOrigin,
 			Description: `set comma separated list of origins allowed for CORS requests e.g. "https://example1.com,https://example2.com"`,

cmd/config/cache/config.go

@@ -37,6 +37,7 @@ type Config struct {
 	After         int      `json:"after"`
 	WatermarkLow  int      `json:"watermark_low"`
 	WatermarkHigh int      `json:"watermark_high"`
+	Range         bool     `json:"range"`
 }
 
 // UnmarshalJSON - implements JSON unmarshal interface for unmarshalling

cmd/config/cache/help.go

@@ -68,5 +68,11 @@ var (
 			Optional:    true,
 			Type:        "number",
 		},
+		config.HelpKV{
+			Key:         Range,
+			Description: `set to "on" or "off" caching of independent range requests per object, defaults to "on"`,
+			Optional:    true,
+			Type:        "string",
+		},
 	}
 )

cmd/config/cache/lookup.go

@@ -34,6 +34,7 @@ const (
 	After         = "after"
 	WatermarkLow  = "watermark_low"
 	WatermarkHigh = "watermark_high"
+	Range         = "range"
 
 	EnvCacheDrives        = "MINIO_CACHE_DRIVES"
 	EnvCacheExclude       = "MINIO_CACHE_EXCLUDE"
@@ -43,6 +44,7 @@ const (
 	EnvCacheAfter         = "MINIO_CACHE_AFTER"
 	EnvCacheWatermarkLow  = "MINIO_CACHE_WATERMARK_LOW"
 	EnvCacheWatermarkHigh = "MINIO_CACHE_WATERMARK_HIGH"
+	EnvCacheRange         = "MINIO_CACHE_RANGE"
 
 	EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY"
 
@@ -84,6 +86,10 @@ var (
 			Key:   WatermarkHigh,
 			Value: DefaultWaterMarkHigh,
 		},
+		config.KV{
+			Key:   Range,
+			Value: config.EnableOn,
+		},
 	}
 )
 
@@ -195,5 +201,15 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 		err := errors.New("config high watermark value should be greater than low watermark value")
 		return cfg, config.ErrInvalidCacheWatermarkHigh(err)
 	}
+
+	cfg.Range = true // by default range caching is enabled.
+	if rangeStr := env.Get(EnvCacheRange, kvs.Get(Range)); rangeStr != "" {
+		rng, err := config.ParseBool(rangeStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheRange(err)
+		}
+		cfg.Range = rng
+	}
+
 	return cfg, nil
 }

cmd/config/errors.go

@@ -96,6 +96,12 @@ var (
 		"MINIO_CACHE_ENCRYPTION_MASTER_KEY: For more information, please refer to https://docs.min.io/docs/minio-disk-cache-guide",
 	)
 
+	ErrInvalidCacheRange = newErrFn(
+		"Invalid cache range value",
+		"Please check the passed value",
+		"MINIO_CACHE_RANGE: Valid expected value is `on` or `off`",
+	)
+
 	ErrInvalidRotatingCredentialsBackendEncrypted = newErrFn(
 		"Invalid rotating credentials",
 		"Please set correct rotating credentials in the environment for decryption",

cmd/config/etcd/dns/etcd_dns.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2018,2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -29,7 +29,7 @@ import (
 	"github.com/minio/minio-go/v6/pkg/set"
 
 	"github.com/coredns/coredns/plugin/etcd/msg"
-	"github.com/coreos/etcd/clientv3"
+	"go.etcd.io/etcd/v3/clientv3"
 )
 
 // ErrNoEntriesFound - Indicates no entries were found for the given key (directory)
@@ -47,6 +47,11 @@ func newCoreDNSMsg(ip string, port string, ttl uint32, t time.Time) ([]byte, err
 	})
 }
 
+// Close closes the internal etcd client and cannot be used further
+func (c *CoreDNS) Close() {
+	c.etcdClient.Close()
+}
+
 // List - Retrieves list of DNS entries for the domain.
 func (c *CoreDNS) List() (map[string][]SrvRecord, error) {
 	var srvRecords = map[string][]SrvRecord{}

cmd/config/etcd/etcd.go

@@ -22,11 +22,11 @@ import (
 	"strings"
 	"time"
 
-	"github.com/coreos/etcd/clientv3"
-	"github.com/coreos/etcd/clientv3/namespace"
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/env"
 	xnet "github.com/minio/minio/pkg/net"
+	"go.etcd.io/etcd/v3/clientv3"
+	"go.etcd.io/etcd/v3/clientv3/namespace"
 )
 
 const (

cmd/config/identity/openid/help.go

@@ -44,6 +44,12 @@ var (
 			Optional:    true,
 			Type:        "string",
 		},
+		config.HelpKV{
+			Key:         Scopes,
+			Description: `Comma separated list of OpenID scopes for server, defaults to advertised scopes from discovery document e.g. "email,admin"`,
+			Optional:    true,
+			Type:        "csv",
+		},
 		config.HelpKV{
 			Key:         config.Comment,
 			Description: config.DefaultComment,

cmd/config/identity/openid/jwt.go

@@ -24,6 +24,7 @@ import (
 	"io"
 	"net/http"
 	"strconv"
+	"strings"
 	"time"
 
 	jwtgo "github.com/dgrijalva/jwt-go"
@@ -217,12 +218,14 @@ const (
 	ClaimName   = "claim_name"
 	ClaimPrefix = "claim_prefix"
 	ClientID    = "client_id"
+	Scopes      = "scopes"
 
 	EnvIdentityOpenIDClientID    = "MINIO_IDENTITY_OPENID_CLIENT_ID"
 	EnvIdentityOpenIDJWKSURL     = "MINIO_IDENTITY_OPENID_JWKS_URL"
 	EnvIdentityOpenIDURL         = "MINIO_IDENTITY_OPENID_CONFIG_URL"
 	EnvIdentityOpenIDClaimName   = "MINIO_IDENTITY_OPENID_CLAIM_NAME"
 	EnvIdentityOpenIDClaimPrefix = "MINIO_IDENTITY_OPENID_CLAIM_PREFIX"
+	EnvIdentityOpenIDScopes      = "MINIO_IDENTITY_OPENID_SCOPES"
 )
 
 // DiscoveryDoc - parses the output from openid-configuration
@@ -287,6 +290,10 @@ var (
 			Key:   ClaimPrefix,
 			Value: "",
 		},
+		config.KV{
+			Key:   Scopes,
+			Value: "",
+		},
 		config.KV{
 			Key:   JwksURL,
 			Value: "",
@@ -331,6 +338,19 @@ func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io
 		}
 	}
 
+	if scopeList := env.Get(EnvIdentityOpenIDScopes, kvs.Get(Scopes)); scopeList != "" {
+		var scopes []string
+		for _, scope := range strings.Split(scopeList, ",") {
+			scope = strings.TrimSpace(scope)
+			if scope == "" {
+				return c, config.Errorf("empty scope value is not allowed '%s', please refer to our documentation", scopeList)
+			}
+			scopes = append(scopes, scope)
+		}
+		// Replace the discovery document scopes by client customized scopes.
+		c.DiscoveryDoc.ScopesSupported = scopes
+	}
+
 	if c.ClaimName == "" {
 		c.ClaimName = iampolicy.PolicyName
 	}

cmd/config/notify/parse.go

@@ -60,8 +60,7 @@ func TestNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transpor
 
 // GetNotificationTargets registers and initializes all notification
 // targets, returns error if any.
-func GetNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport) (*event.TargetList, error) {
-	test := false
+func GetNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport, test bool) (*event.TargetList, error) {
 	returnOnTargetError := false
 	return RegisterNotificationTargets(cfg, doneCh, transport, nil, test, returnOnTargetError)
 }
@@ -72,7 +71,6 @@ func GetNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport
 // * Add newly added target configuration to serverConfig.Notify.<TARGET_NAME>.
 // * Handle the configuration in this function to create/add into TargetList.
 func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport, targetIDs []event.TargetID, test bool, returnOnTargetError bool) (*event.TargetList, error) {
-
 	targetList, err := FetchRegisteredTargets(cfg, doneCh, transport, test, returnOnTargetError)
 	if err != nil {
 		return targetList, err

cmd/consolelogger.go

@@ -45,7 +45,7 @@ func mustGetNodeName(endpointZones EndpointZones) (nodeName string) {
 	if err != nil {
 		logger.FatalIf(err, "Unable to start console logging subsystem")
 	}
-	if globalIsDistXL {
+	if globalIsDistErasure {
 		nodeName = host.Name
 	}
 	return nodeName

cmd/copy-part-range.go

@@ -32,7 +32,9 @@ func writeCopyPartErr(ctx context.Context, w http.ResponseWriter, err error, url
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopyPartRangeSource), url, browser)
 		return
 	default:
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), url, browser)
+		apiErr := errorCodes.ToAPIErr(ErrInvalidCopyPartRangeSource)
+		apiErr.Description = err.Error()
+		writeErrorResponse(ctx, w, apiErr, url, browser)
 		return
 	}
 }

cmd/crypto/kes.go

@@ -34,9 +34,9 @@ import (
 	xnet "github.com/minio/minio/pkg/net"
 )
 
-// ErrKESKeyNotFound is the error returned a KES server
-// when a master key does not exist.
-var ErrKESKeyNotFound = NewKESError(http.StatusNotFound, "key does not exist")
+// ErrKESKeyExists is the error returned a KES server
+// when a master key does exist.
+var ErrKESKeyExists = NewKESError(http.StatusBadRequest, "key does already exist")
 
 // KesConfig contains the configuration required
 // to initialize and connect to a kes server.
@@ -134,20 +134,27 @@ func NewKes(cfg KesConfig) (KMS, error) {
 	}, nil
 }
 
-// KeyID returns the default key ID.
-func (kes *kesService) KeyID() string {
+// DefaultKeyID returns the default key ID that should be
+// used for SSE-S3 or SSE-KMS when the S3 client does not
+// provide an explicit key ID.
+func (kes *kesService) DefaultKeyID() string {
 	return kes.defaultKeyID
 }
 
-// Info returns some status information about the KMS.
+// Info returns some information about the KES,
+// configuration - like the endpoint or authentication
+// method.
 func (kes *kesService) Info() KMSInfo {
 	return KMSInfo{
 		Endpoint: kes.endpoint,
-		Name:     kes.KeyID(),
+		Name:     kes.DefaultKeyID(),
 		AuthType: "TLS",
 	}
 }
 
+// CreateKey tries to create a new master key with the given keyID.
+func (kes *kesService) CreateKey(keyID string) error { return kes.client.CreateKey(keyID) }
+
 // GenerateKey returns a new plaintext key, generated by the KMS,
 // and a sealed version of this plaintext key encrypted using the
 // named key referenced by keyID. It also binds the generated key
@@ -158,12 +165,6 @@ func (kes *kesService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
 
 	var plainKey []byte
 	plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context.Bytes())
-	if err == ErrKESKeyNotFound { // Try to create the key if it does not exist.
-		if err = kes.client.CreateKey(keyID); err != nil {
-			return key, nil, err
-		}
-		plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context.Bytes())
-	}
 	if err != nil {
 		return key, nil, err
 	}
@@ -197,28 +198,11 @@ func (kes *kesService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (k
 	return key, nil
 }
 
-// UpdateKey re-wraps the sealedKey if the master key referenced by the keyID
-// has been changed by the KMS operator - i.e. the master key has been rotated.
-// If the master key hasn't changed since the sealedKey has been created / updated
-// it may return the same sealedKey as rotatedKey.
-//
-// The context must be same context as the one provided while
-// generating the plaintext key / sealedKey.
-func (kes *kesService) UpdateKey(keyID string, sealedKey []byte, ctx Context) ([]byte, error) {
-	_, err := kes.UnsealKey(keyID, sealedKey, ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	// Currently a kes server does not support key rotation (of the same key)
-	// Therefore, we simply return the same sealedKey.
-	return sealedKey, nil
-}
-
 // kesClient implements the bare minimum functionality needed for
 // MinIO to talk to a KES server. In particular, it implements
-// GenerateDataKey (API: /v1/key/generate/) and
-// DecryptDataKey  (API: /v1/key/decrypt/).
+//   • CreateKey       (API: /v1/key/create/)
+//   • GenerateDataKey (API: /v1/key/generate/)
+//   • DecryptDataKey  (API: /v1/key/decrypt/)
 type kesClient struct {
 	addr       string
 	httpClient http.Client

cmd/crypto/kms.go

@@ -72,8 +72,14 @@ func (c Context) WriteTo(w io.Writer) (n int64, err error) {
 // data key generation and unsealing of KMS-generated
 // data keys.
 type KMS interface {
-	// KeyID - returns configured KMS key id.
-	KeyID() string
+	// DefaultKeyID returns the default master key ID. It should be
+	// used for SSE-S3 and whenever a S3 client requests SSE-KMS but
+	// does not specify an explicit SSE-KMS key ID.
+	DefaultKeyID() string
+
+	// CreateKey creates a new master key with the given key ID
+	// at the KMS.
+	CreateKey(keyID string) error
 
 	// GenerateKey generates a new random data key using
 	// the master key referenced by the keyID. It returns
@@ -90,21 +96,9 @@ type KMS interface {
 	// match the context used to generate the sealed key.
 	UnsealKey(keyID string, sealedKey []byte, context Context) (key [32]byte, err error)
 
-	// UpdateKey re-wraps the sealedKey if the master key, referenced by
-	// `keyID`, has changed in the meantime. This usually happens when the
-	// KMS operator performs a key-rotation operation of the master key.
-	// UpdateKey fails if the provided sealedKey cannot be decrypted using
-	// the master key referenced by keyID.
-	//
-	// UpdateKey makes no guarantees whatsoever about whether the returned
-	// rotatedKey is actually different from the sealedKey. If nothing has
-	// changed at the KMS or if the KMS does not support updating generated
-	// keys this method may behave like a NOP and just return the sealedKey
-	// itself.
-	UpdateKey(keyID string, sealedKey []byte, context Context) (rotatedKey []byte, err error)
-
-	// Returns KMSInfo
-	Info() (kmsInfo KMSInfo)
+	// Info returns descriptive information about the KMS,
+	// like the default key ID and authentication method.
+	Info() KMSInfo
 }
 
 type masterKeyKMS struct {
@@ -112,7 +106,8 @@ type masterKeyKMS struct {
 	masterKey [32]byte
 }
 
-// KMSInfo stores the details of KMS
+// KMSInfo contains some describing information about
+// the KMS.
 type KMSInfo struct {
 	Endpoint string
 	Name     string
@@ -125,10 +120,14 @@ type KMSInfo struct {
 // to the generated keys.
 func NewMasterKey(keyID string, key [32]byte) KMS { return &masterKeyKMS{keyID: keyID, masterKey: key} }
 
-func (kms *masterKeyKMS) KeyID() string {
+func (kms *masterKeyKMS) DefaultKeyID() string {
 	return kms.keyID
 }
 
+func (kms *masterKeyKMS) CreateKey(keyID string) error {
+	return errors.New("crypto: creating keys is not supported by a static master key")
+}
+
 func (kms *masterKeyKMS) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
 	if _, err = io.ReadFull(rand.Reader, key[:]); err != nil {
 		logger.CriticalIf(context.Background(), errOutOfEntropy)
@@ -166,13 +165,6 @@ func (kms *masterKeyKMS) UnsealKey(keyID string, sealedKey []byte, ctx Context)
 	return key, nil
 }
 
-func (kms *masterKeyKMS) UpdateKey(keyID string, sealedKey []byte, ctx Context) ([]byte, error) {
-	if _, err := kms.UnsealKey(keyID, sealedKey, ctx); err != nil {
-		return nil, err
-	}
-	return sealedKey, nil // The master key cannot update data keys -> Do nothing.
-}
-
 func (kms *masterKeyKMS) deriveKey(keyID string, context Context) (key [32]byte) {
 	if context == nil {
 		context = Context{}

cmd/crypto/kms_test.go

@@ -57,15 +57,6 @@ func TestMasterKeyKMS(t *testing.T) {
 		if !test.ShouldFail && !bytes.Equal(key[:], unsealedKey[:]) {
 			t.Errorf("Test %d: The generated and unsealed key differ", i)
 		}
-
-		rotatedKey, err := kms.UpdateKey(test.UnsealKeyID, sealedKey, test.UnsealContext)
-		if err == nil && test.ShouldFail {
-			t.Errorf("Test %d: KMS updated the generated key successfully but should have failed", i)
-		}
-		if !test.ShouldFail && !bytes.Equal(rotatedKey, sealedKey[:]) {
-			t.Errorf("Test %d: The updated and sealed key differ", i)
-		}
-
 	}
 }
 

cmd/crypto/parse_test.go

@@ -51,8 +51,8 @@ func TestParseMasterKey(t *testing.T) {
 			if !tt.success && err == nil {
 				t.Error("Unexpected failure")
 			}
-			if err == nil && kms.KeyID() != tt.expectedKeyID {
-				t.Errorf("Expected keyID %s, got %s", tt.expectedKeyID, kms.KeyID())
+			if err == nil && kms.DefaultKeyID() != tt.expectedKeyID {
+				t.Errorf("Expected keyID %s, got %s", tt.expectedKeyID, kms.DefaultKeyID())
 			}
 		})
 	}

cmd/crypto/vault.go

@@ -17,6 +17,7 @@ package crypto
 import (
 	"bytes"
 	"encoding/base64"
+	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -190,20 +191,34 @@ func (v *vaultService) authenticate() (err error) {
 	return
 }
 
-// KeyID - vault configured keyID
-func (v *vaultService) KeyID() string {
+// DefaultKeyID returns the default key ID that should be
+// used for SSE-S3 or SSE-KMS when the S3 client does not
+// provide an explicit key ID.
+func (v *vaultService) DefaultKeyID() string {
 	return v.config.Key.Name
 }
 
-// Returns - vault info
-func (v *vaultService) Info() (kmsInfo KMSInfo) {
+// Info returns some information about the Vault,
+// configuration - like the endpoint or authentication
+// method.
+func (v *vaultService) Info() KMSInfo {
 	return KMSInfo{
 		Endpoint: v.config.Endpoint,
-		Name:     v.config.Key.Name,
+		Name:     v.DefaultKeyID(),
 		AuthType: v.config.Auth.Type,
 	}
 }
 
+// CreateKey is a stub that exists such that the Vault
+// client implements the KMS interface. It always returns
+// a not-implemented error.
+//
+// Creating keys requires a KES instance between MinIO and Vault.
+func (v *vaultService) CreateKey(keyID string) error {
+	// Creating new keys requires KES.
+	return errors.New("crypto: creating keys is not supported by Vault")
+}
+
 // GenerateKey returns a new plaintext key, generated by the KMS,
 // and a sealed version of this plaintext key encrypted using the
 // named key referenced by keyID. It also binds the generated key

cmd/data-crawler.go

@@ -28,7 +28,6 @@ import (
 	"time"
 
 	"github.com/minio/minio/cmd/config"
-	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	"github.com/minio/minio/pkg/color"
@@ -510,9 +509,9 @@ func (i *crawlItem) transformMetaDir() {
 
 // actionMeta contains information used to apply actions.
 type actionMeta struct {
-	oi      ObjectInfo
-	trustOI bool // Set true if oi can be trusted and has been read with quorum.
-	meta    map[string]string
+	oi          ObjectInfo
+	trustOI     bool // Set true if oi can be trusted and has been read with quorum.
+	numVersions int  // The number of versions of this object
 }
 
 // applyActions will apply lifecycle checks on to a scanned item.
@@ -528,12 +527,22 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 		return size
 	}
 
-	action := i.lifeCycle.ComputeAction(i.objectPath(), meta.meta[xhttp.AmzObjectTagging], meta.oi.ModTime)
+	versionID := meta.oi.VersionID
+	action := i.lifeCycle.ComputeAction(
+		lifecycle.ObjectOpts{
+			Name:         i.objectPath(),
+			UserTags:     meta.oi.UserTags,
+			ModTime:      meta.oi.ModTime,
+			VersionID:    meta.oi.VersionID,
+			DeleteMarker: meta.oi.DeleteMarker,
+			IsLatest:     meta.oi.IsLatest,
+			NumVersions:  meta.numVersions,
+		})
 	if i.debug {
 		logger.Info(color.Green("applyActions:")+" lifecycle: %q, Initial scan: %v", i.objectPath(), action)
 	}
 	switch action {
-	case lifecycle.DeleteAction:
+	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
 	default:
 		// No action.
 		return size
@@ -542,28 +551,60 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 	// These (expensive) operations should only run on items we are likely to delete.
 	// Load to ensure that we have the correct version and not an unsynced version.
 	if !meta.trustOI {
-		obj, err := o.GetObjectInfo(ctx, i.bucket, i.objectPath(), ObjectOptions{})
+		obj, err := o.GetObjectInfo(ctx, i.bucket, i.objectPath(), ObjectOptions{
+			VersionID: versionID,
+		})
 		if err != nil {
-			// Do nothing - heal in the future.
-			logger.LogIf(ctx, err)
-			return size
+			switch err.(type) {
+			case MethodNotAllowed: // This happens usually for a delete marker
+				if !obj.DeleteMarker { // if this is not a delete marker log and return
+					// Do nothing - heal in the future.
+					logger.LogIf(ctx, err)
+					return size
+				}
+			case ObjectNotFound:
+				// object not found return 0
+				return 0
+			default:
+				// All other errors proceed.
+				logger.LogIf(ctx, err)
+				return size
+			}
 		}
 		size = obj.Size
 
 		// Recalculate action.
-		action = i.lifeCycle.ComputeAction(i.objectPath(), obj.UserTags, obj.ModTime)
+		action = i.lifeCycle.ComputeAction(
+			lifecycle.ObjectOpts{
+				Name:         i.objectPath(),
+				UserTags:     obj.UserTags,
+				ModTime:      obj.ModTime,
+				VersionID:    obj.VersionID,
+				DeleteMarker: obj.DeleteMarker,
+				IsLatest:     obj.IsLatest,
+				NumVersions:  meta.numVersions,
+			})
 		if i.debug {
 			logger.Info(color.Green("applyActions:")+" lifecycle: Secondary scan: %v", action)
 		}
+		versionID = obj.VersionID
 		switch action {
-		case lifecycle.DeleteAction:
+		case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
 		default:
 			// No action.
 			return size
 		}
 	}
 
-	err = o.DeleteObject(ctx, i.bucket, i.objectPath())
+	opts := ObjectOptions{}
+	switch action {
+	case lifecycle.DeleteVersionAction:
+		opts.VersionID = versionID
+	case lifecycle.DeleteAction:
+		opts.Versioned = globalBucketVersioningSys.Enabled(i.bucket)
+	}
+
+	obj, err := o.DeleteObject(ctx, i.bucket, i.objectPath(), opts)
 	if err != nil {
 		// Assume it is still there.
 		logger.LogIf(ctx, err)
@@ -574,10 +615,8 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 	sendEvent(eventArgs{
 		EventName:  event.ObjectRemovedDelete,
 		BucketName: i.bucket,
-		Object: ObjectInfo{
-			Name: i.objectPath(),
-		},
-		Host: "Internal: [ILM-EXPIRY]",
+		Object:     obj,
+		Host:       "Internal: [ILM-EXPIRY]",
 	})
 	return 0
 }

cmd/data-update-tracker.go

@@ -120,8 +120,8 @@ func (b bloomFilter) containsDir(in string) bool {
 }
 
 // bytes returns the bloom filter serialized as a byte slice.
-func (b bloomFilter) bytes() []byte {
-	if b.BloomFilter == nil {
+func (b *bloomFilter) bytes() []byte {
+	if b == nil || b.BloomFilter == nil {
 		return nil
 	}
 	var buf bytes.Buffer

cmd/disk-cache-backend.go

@@ -33,6 +33,7 @@ import (
 	"time"
 
 	"github.com/djherbis/atime"
+	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
@@ -60,7 +61,7 @@ type CacheChecksumInfoV1 struct {
 // Represents the cache metadata struct
 type cacheMeta struct {
 	Version string   `json:"version"`
-	Stat    statInfo `json:"stat"` // Stat of the current object `cache.json`.
+	Stat    StatInfo `json:"stat"` // Stat of the current object `cache.json`.
 
 	// checksums of blocks on disk.
 	Checksum CacheChecksumInfoV1 `json:"checksum,omitempty"`
@@ -125,16 +126,19 @@ func (m *cacheMeta) ToObjectInfo(bucket, object string) (o ObjectInfo) {
 
 // represents disk cache struct
 type diskCache struct {
-	gcCounter uint64 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
 	// is set to 0 if drive is offline
-	online uint32
+	online       uint32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
+	purgeRunning int32
 
-	dir           string // caching directory
-	quotaPct      int    // max usage in %
+	triggerGC     chan struct{}
+	dir           string         // caching directory
+	stats         CacheDiskStats // disk cache stats for prometheus
+	quotaPct      int            // max usage in %
 	pool          sync.Pool
 	after         int // minimum accesses before an object is cached.
 	lowWatermark  int
 	highWatermark int
+	enableRange   bool
 	// nsMutex namespace lock
 	nsMutex *nsLockMap
 	// Object functions pointing to the corresponding functions of backend implementation.
@@ -142,16 +146,24 @@ type diskCache struct {
 }
 
 // Inits the disk cache dir if it is not initialized already.
-func newDiskCache(dir string, quotaPct, after, lowWatermark, highWatermark int) (*diskCache, error) {
+func newDiskCache(ctx context.Context, dir string, config cache.Config) (*diskCache, error) {
+	quotaPct := config.MaxUse
+	if quotaPct == 0 {
+		quotaPct = config.Quota
+	}
+
 	if err := os.MkdirAll(dir, 0777); err != nil {
 		return nil, fmt.Errorf("Unable to initialize '%s' dir, %w", dir, err)
 	}
 	cache := diskCache{
 		dir:           dir,
+		triggerGC:     make(chan struct{}),
+		stats:         CacheDiskStats{Dir: dir},
 		quotaPct:      quotaPct,
-		after:         after,
-		lowWatermark:  lowWatermark,
-		highWatermark: highWatermark,
+		after:         config.After,
+		lowWatermark:  config.WatermarkLow,
+		highWatermark: config.WatermarkHigh,
+		enableRange:   config.Range,
 		online:        1,
 		pool: sync.Pool{
 			New: func() interface{} {
@@ -161,6 +173,8 @@ func newDiskCache(dir string, quotaPct, after, lowWatermark, highWatermark int)
 		},
 		nsMutex: newNSLock(false),
 	}
+	go cache.purgeWait(ctx)
+	cache.diskUsageHigh() // update if cache usage is already high.
 	cache.NewNSLockFn = func(ctx context.Context, cachePath string) RWLocker {
 		return cache.nsMutex.NewNSLock(ctx, nil, cachePath, "")
 	}
@@ -181,7 +195,12 @@ func (c *diskCache) diskUsageLow() bool {
 		return false
 	}
 	usedPercent := (di.Total - di.Free) * 100 / di.Total
-	return int(usedPercent) < gcStopPct
+	low := int(usedPercent) < gcStopPct
+	atomic.StoreUint64(&c.stats.UsagePercent, usedPercent)
+	if low {
+		atomic.StoreInt32(&c.stats.UsageState, 0)
+	}
+	return low
 }
 
 // Returns if the disk usage reaches high water mark w.r.t the configured cache quota.
@@ -196,7 +215,12 @@ func (c *diskCache) diskUsageHigh() bool {
 		return false
 	}
 	usedPercent := (di.Total - di.Free) * 100 / di.Total
-	return int(usedPercent) >= gcTriggerPct
+	high := int(usedPercent) >= gcTriggerPct
+	atomic.StoreUint64(&c.stats.UsagePercent, usedPercent)
+	if high {
+		atomic.StoreInt32(&c.stats.UsageState, 1)
+	}
+	return high
 }
 
 // Returns if size space can be allocated without exceeding
@@ -230,24 +254,36 @@ var (
 	errDoneForNow = errors.New("done for now")
 )
 
+func (c *diskCache) purgeWait(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+		case <-c.triggerGC: // wait here until someone triggers.
+			c.purge(ctx)
+		}
+	}
+}
+
 // Purge cache entries that were not accessed.
 func (c *diskCache) purge(ctx context.Context) {
-	if c.diskUsageLow() {
+	if atomic.LoadInt32(&c.purgeRunning) == 1 || c.diskUsageLow() {
 		return
 	}
+
 	toFree := c.toClear()
 	if toFree == 0 {
 		return
 	}
+
+	atomic.StoreInt32(&c.purgeRunning, 1) // do not run concurrent purge()
+	defer atomic.StoreInt32(&c.purgeRunning, 0)
+
 	// expiry for cleaning up old cache.json files that
 	// need to be cleaned up.
 	expiry := UTCNow().Add(-cacheExpiryDays)
 	// defaulting max hits count to 100
-	scorer, err := newFileScorer(toFree, time.Now().Unix(), 100)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return
-	}
+	// ignore error we know what value we are passing.
+	scorer, _ := newFileScorer(toFree, time.Now().Unix(), 100)
 
 	// this function returns FileInfo for cached range files and cache data file.
 	fiStatFn := func(ranges map[string]string, dataFile, pathPrefix string) map[string]os.FileInfo {
@@ -326,27 +362,20 @@ func (c *diskCache) purge(ctx context.Context) {
 		return
 	}
 
-	for _, path := range scorer.fileNames() {
-		removeAll(path)
-		slashIdx := strings.LastIndex(path, SlashSeparator)
-		pathPrefix := path[0:slashIdx]
-		fname := path[slashIdx+1:]
-		if fname == cacheDataFile {
-			removeAll(pathPrefix)
+	scorer.purgeFunc(func(qfile queuedFile) {
+		fileName := qfile.name
+		removeAll(fileName)
+		slashIdx := strings.LastIndex(fileName, SlashSeparator)
+		if slashIdx >= 0 {
+			fileNamePrefix := fileName[0:slashIdx]
+			fname := fileName[slashIdx+1:]
+			if fname == cacheDataFile {
+				removeAll(fileNamePrefix)
+			}
 		}
-	}
-}
+	})
 
-func (c *diskCache) incGCCounter() {
-	atomic.AddUint64(&c.gcCounter, 1)
-}
-
-func (c *diskCache) resetGCCounter() {
-	atomic.StoreUint64(&c.gcCounter, 0)
-}
-
-func (c *diskCache) gcCount() uint64 {
-	return atomic.LoadUint64(&c.gcCounter)
+	scorer.reset()
 }
 
 // sets cache drive status
@@ -553,7 +582,7 @@ func (c *diskCache) bitrotWriteToCache(cachePath, fileName string, reader io.Rea
 	}
 	f, err := os.Create(filePath)
 	if err != nil {
-		return 0, osErrToFSFileErr(err)
+		return 0, osErrToFileErr(err)
 	}
 	defer f.Close()
 
@@ -611,14 +640,14 @@ func newCacheEncryptMetadata(bucket, object string, metadata map[string]string)
 	if globalCacheKMS == nil {
 		return nil, errKMSNotConfigured
 	}
-	key, encKey, err := globalCacheKMS.GenerateKey(globalCacheKMS.KeyID(), crypto.Context{bucket: pathJoin(bucket, object)})
+	key, encKey, err := globalCacheKMS.GenerateKey(globalCacheKMS.DefaultKeyID(), crypto.Context{bucket: pathJoin(bucket, object)})
 	if err != nil {
 		return nil, err
 	}
 
 	objectKey := crypto.GenerateKey(key, rand.Reader)
 	sealedKey = objectKey.Seal(key, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
-	crypto.S3.CreateMetadata(metadata, globalCacheKMS.KeyID(), encKey, sealedKey)
+	crypto.S3.CreateMetadata(metadata, globalCacheKMS.DefaultKeyID(), encKey, sealedKey)
 
 	if etag, ok := metadata["etag"]; ok {
 		metadata["etag"] = hex.EncodeToString(objectKey.SealETag([]byte(etag)))
@@ -630,7 +659,7 @@ func newCacheEncryptMetadata(bucket, object string, metadata map[string]string)
 // Caches the object to disk
 func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions, incHitsOnly bool) error {
 	if c.diskUsageHigh() {
-		c.incGCCounter()
+		c.triggerGC <- struct{}{}
 		io.Copy(ioutil.Discard, data)
 		return errDiskFull
 	}

cmd/disk-cache-stats.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019, 2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,14 +16,27 @@
 
 package cmd
 
-import "sync/atomic"
+import (
+	"sync/atomic"
+)
+
+// CacheDiskStats represents cache disk statistics
+// such as current disk usage and available.
+type CacheDiskStats struct {
+	// indicates if usage is high or low, if high value is '1', if low its '0'
+	UsageState int32
+	// indicates the current usage percentage of this cache disk
+	UsagePercent uint64
+	Dir          string
+}
 
 // CacheStats - represents bytes served from cache,
 // cache hits and cache misses.
 type CacheStats struct {
-	BytesServed uint64
-	Hits        uint64
-	Misses      uint64
+	BytesServed  uint64
+	Hits         uint64
+	Misses       uint64
+	GetDiskStats func() []CacheDiskStats
 }
 
 // Increase total bytes served from cache

cmd/disk-cache-utils.go

@@ -187,12 +187,12 @@ func readCacheFileStream(filePath string, offset, length int64) (io.ReadCloser,
 
 	fr, err := os.Open(filePath)
 	if err != nil {
-		return nil, osErrToFSFileErr(err)
+		return nil, osErrToFileErr(err)
 	}
 	// Stat to get the size of the file at path.
 	st, err := fr.Stat()
 	if err != nil {
-		err = osErrToFSFileErr(err)
+		err = osErrToFileErr(err)
 		return nil, err
 	}
 
@@ -298,9 +298,10 @@ type fileScorer struct {
 }
 
 type queuedFile struct {
-	name  string
-	size  uint64
-	score float64
+	name      string
+	versionID string
+	size      uint64
+	score     float64
 }
 
 // newFileScorer allows to collect files to save a specific number of bytes.
@@ -321,15 +322,33 @@ func newFileScorer(saveBytes uint64, now int64, maxHits int) (*fileScorer, error
 	return &f, nil
 }
 
-func (f *fileScorer) addFile(name string, lastAccess time.Time, size int64, hits int) {
+func (f *fileScorer) addFile(name string, accTime time.Time, size int64, hits int) {
+	f.addFileWithObjInfo(ObjectInfo{
+		Name:    name,
+		AccTime: accTime,
+		Size:    size,
+	}, hits)
+}
+
+func (f *fileScorer) addFileWithObjInfo(objInfo ObjectInfo, hits int) {
 	// Calculate how much we want to delete this object.
 	file := queuedFile{
-		name: name,
-		size: uint64(size),
+		name:      objInfo.Name,
+		versionID: objInfo.VersionID,
+		size:      uint64(objInfo.Size),
 	}
-	score := float64(f.now - lastAccess.Unix())
+
+	var score float64
+	if objInfo.ModTime.IsZero() {
+		// Mod time is not available with disk cache use atime.
+		score = float64(f.now - objInfo.AccTime.Unix())
+	} else {
+		// if not used mod time when mod time is available.
+		score = float64(f.now - objInfo.ModTime.Unix())
+	}
+
 	// Size as fraction of how much we want to save, 0->1.
-	szWeight := math.Max(0, (math.Min(1, float64(size)*f.sizeMult)))
+	szWeight := math.Max(0, (math.Min(1, float64(file.size)*f.sizeMult)))
 	// 0 at f.maxHits, 1 at 0.
 	hitsWeight := (1.0 - math.Max(0, math.Min(1.0, float64(hits)/float64(f.maxHits))))
 	file.score = score * (1 + 0.25*szWeight + 0.25*hitsWeight)
@@ -404,6 +423,30 @@ func (f *fileScorer) trimQueue() {
 	}
 }
 
+// fileObjInfos returns all queued file object infos
+func (f *fileScorer) fileObjInfos() []ObjectInfo {
+	res := make([]ObjectInfo, 0, f.queue.Len())
+	e := f.queue.Front()
+	for e != nil {
+		qfile := e.Value.(queuedFile)
+		res = append(res, ObjectInfo{
+			Name:      qfile.name,
+			Size:      int64(qfile.size),
+			VersionID: qfile.versionID,
+		})
+		e = e.Next()
+	}
+	return res
+}
+
+func (f *fileScorer) purgeFunc(p func(qfile queuedFile)) {
+	e := f.queue.Front()
+	for e != nil {
+		p(e.Value.(queuedFile))
+		e = e.Next()
+	}
+}
+
 // fileNames returns all queued file names.
 func (f *fileScorer) fileNames() []string {
 	res := make([]string, 0, f.queue.Len())

cmd/disk-cache.go

@@ -24,6 +24,7 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/minio/minio/cmd/config/cache"
@@ -51,8 +52,8 @@ type CacheObjectLayer interface {
 	// Object operations.
 	GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
 	GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
-	DeleteObject(ctx context.Context, bucket, object string) error
-	DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error)
+	DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error)
+	DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error)
 	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error)
 	// Storage operations.
@@ -78,8 +79,7 @@ type cacheObjects struct {
 
 	GetObjectNInfoFn func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
 	GetObjectInfoFn  func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
-	DeleteObjectFn   func(ctx context.Context, bucket, object string) error
-	DeleteObjectsFn  func(ctx context.Context, bucket string, objects []string) ([]error, error)
+	DeleteObjectFn   func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	PutObjectFn      func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	CopyObjectFn     func(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error)
 }
@@ -120,8 +120,8 @@ func (c *cacheObjects) updateMetadataIfChanged(ctx context.Context, dcache *disk
 }
 
 // DeleteObject clears cache entry if backend delete operation succeeds
-func (c *cacheObjects) DeleteObject(ctx context.Context, bucket, object string) (err error) {
-	if err = c.DeleteObjectFn(ctx, bucket, object); err != nil {
+func (c *cacheObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+	if objInfo, err = c.DeleteObjectFn(ctx, bucket, object, opts); err != nil {
 		return
 	}
 	if c.isCacheExclude(bucket, object) || c.skipCache() {
@@ -130,19 +130,38 @@ func (c *cacheObjects) DeleteObject(ctx context.Context, bucket, object string)
 
 	dcache, cerr := c.getCacheLoc(bucket, object)
 	if cerr != nil {
-		return
+		return objInfo, cerr
 	}
 	dcache.Delete(ctx, bucket, object)
 	return
 }
 
 // DeleteObjects batch deletes objects in slice, and clears any cached entries
-func (c *cacheObjects) DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error) {
+func (c *cacheObjects) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
 	errs := make([]error, len(objects))
+	objInfos := make([]ObjectInfo, len(objects))
 	for idx, object := range objects {
-		errs[idx] = c.DeleteObject(ctx, bucket, object)
+		opts.VersionID = object.VersionID
+		objInfos[idx], errs[idx] = c.DeleteObject(ctx, bucket, object.ObjectName, opts)
 	}
-	return errs, nil
+	deletedObjects := make([]DeletedObject, len(objInfos))
+	for idx := range errs {
+		if errs[idx] != nil {
+			continue
+		}
+		if objInfos[idx].DeleteMarker {
+			deletedObjects[idx] = DeletedObject{
+				DeleteMarker:          objInfos[idx].DeleteMarker,
+				DeleteMarkerVersionID: objInfos[idx].VersionID,
+			}
+			continue
+		}
+		deletedObjects[idx] = DeletedObject{
+			ObjectName: objInfos[idx].Name,
+			VersionID:  objInfos[idx].VersionID,
+		}
+	}
+	return deletedObjects, errs
 }
 
 // construct a metadata k-v map
@@ -264,10 +283,11 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 
 	// Reaching here implies cache miss
 	c.cacheStats.incMiss()
+
 	// Since we got here, we are serving the request from backend,
 	// and also adding the object to the cache.
 	if dcache.diskUsageHigh() {
-		dcache.incGCCounter()
+		dcache.triggerGC <- struct{}{} // this is non-blocking
 	}
 
 	bkReader, bkErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
@@ -292,6 +312,10 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 
 	if rs != nil {
 		go func() {
+			// if range caching is disabled, download entire object.
+			if !dcache.enableRange {
+				rs = nil
+			}
 			// fill cache in the background for range GET requests
 			bReader, bErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 			if bErr != nil {
@@ -301,7 +325,9 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			oi, _, _, err := dcache.statRange(ctx, bucket, object, rs)
 			// avoid cache overwrite if another background routine filled cache
 			if err != nil || oi.ETag != bReader.ObjInfo.ETag {
-				dcache.Put(ctx, bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false)
+				// use a new context to avoid locker prematurely timing out operation when the GetObjectNInfo returns.
+				dcache.Put(context.Background(), bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false)
+				return
 			}
 		}()
 		return bkReader, bkErr
@@ -522,11 +548,7 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 			return nil, false, errors.New("Atime support required for disk caching")
 		}
 
-		quota := config.MaxUse
-		if quota == 0 {
-			quota = config.Quota
-		}
-		cache, err := newDiskCache(dir, quota, config.After, config.WatermarkLow, config.WatermarkHigh)
+		cache, err := newDiskCache(ctx, dir, config)
 		if err != nil {
 			return nil, false, err
 		}
@@ -649,15 +671,8 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 		GetObjectNInfoFn: func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
 			return newObjectLayerFn().GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 		},
-		DeleteObjectFn: func(ctx context.Context, bucket, object string) error {
-			return newObjectLayerFn().DeleteObject(ctx, bucket, object)
-		},
-		DeleteObjectsFn: func(ctx context.Context, bucket string, objects []string) ([]error, error) {
-			errs := make([]error, len(objects))
-			for idx, object := range objects {
-				errs[idx] = newObjectLayerFn().DeleteObject(ctx, bucket, object)
-			}
-			return errs, nil
+		DeleteObjectFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+			return newObjectLayerFn().DeleteObject(ctx, bucket, object, opts)
 		},
 		PutObjectFn: func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 			return newObjectLayerFn().PutObject(ctx, bucket, object, data, opts)
@@ -666,7 +681,17 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 			return newObjectLayerFn().CopyObject(ctx, srcBucket, srcObject, destBucket, destObject, srcInfo, srcOpts, dstOpts)
 		},
 	}
-
+	c.cacheStats.GetDiskStats = func() []CacheDiskStats {
+		cacheDiskStats := make([]CacheDiskStats, len(c.cache))
+		for i := range c.cache {
+			cacheDiskStats[i] = CacheDiskStats{
+				Dir: c.cache[i].stats.Dir,
+			}
+			atomic.StoreInt32(&cacheDiskStats[i].UsageState, atomic.LoadInt32(&c.cache[i].stats.UsageState))
+			atomic.StoreUint64(&cacheDiskStats[i].UsagePercent, atomic.LoadUint64(&c.cache[i].stats.UsagePercent))
+		}
+		return cacheDiskStats
+	}
 	if migrateSw {
 		go c.migrateCacheFromV1toV2(ctx)
 	}
@@ -686,19 +711,9 @@ func (c *cacheObjects) gc(ctx context.Context) {
 			if c.migrating {
 				continue
 			}
-			var wg sync.WaitGroup
 			for _, dcache := range c.cache {
-				if dcache.gcCount() == 0 {
-					continue
-				}
-				wg.Add(1)
-				go func(d *diskCache) {
-					defer wg.Done()
-					d.resetGCCounter()
-					d.purge(ctx)
-				}(dcache)
+				dcache.triggerGC <- struct{}{}
 			}
-			wg.Wait()
 		}
 	}
 }

cmd/encryption-v1.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2017-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bufio"
-	"context"
 	"crypto/hmac"
 	"crypto/rand"
 	"crypto/subtle"
@@ -31,7 +30,6 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/minio/minio-go/v6/pkg/encrypt"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	sha256 "github.com/minio/sha256-simd"
@@ -82,7 +80,7 @@ func isEncryptedMultipart(objInfo ObjectInfo) bool {
 		}
 	}
 	// Further check if this object is uploaded using multipart mechanism
-	// by the user and it is not about XL internally splitting the
+	// by the user and it is not about Erasure internally splitting the
 	// object into parts in PutObject()
 	return !(objInfo.backendType == BackendErasure && len(objInfo.ETag) == 32)
 }
@@ -159,12 +157,12 @@ func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map
 			return err
 		}
 
-		newKey, encKey, err := GlobalKMS.GenerateKey(GlobalKMS.KeyID(), crypto.Context{bucket: path.Join(bucket, object)})
+		newKey, encKey, err := GlobalKMS.GenerateKey(GlobalKMS.DefaultKeyID(), crypto.Context{bucket: path.Join(bucket, object)})
 		if err != nil {
 			return err
 		}
 		sealedKey = objectKey.Seal(newKey, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
-		crypto.S3.CreateMetadata(metadata, GlobalKMS.KeyID(), encKey, sealedKey)
+		crypto.S3.CreateMetadata(metadata, GlobalKMS.DefaultKeyID(), encKey, sealedKey)
 		return nil
 	}
 }
@@ -175,14 +173,14 @@ func newEncryptMetadata(key []byte, bucket, object string, metadata map[string]s
 		if GlobalKMS == nil {
 			return crypto.ObjectKey{}, errKMSNotConfigured
 		}
-		key, encKey, err := GlobalKMS.GenerateKey(GlobalKMS.KeyID(), crypto.Context{bucket: path.Join(bucket, object)})
+		key, encKey, err := GlobalKMS.GenerateKey(GlobalKMS.DefaultKeyID(), crypto.Context{bucket: path.Join(bucket, object)})
 		if err != nil {
 			return crypto.ObjectKey{}, err
 		}
 
 		objectKey := crypto.GenerateKey(key, rand.Reader)
 		sealedKey = objectKey.Seal(key, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
-		crypto.S3.CreateMetadata(metadata, GlobalKMS.KeyID(), encKey, sealedKey)
+		crypto.S3.CreateMetadata(metadata, GlobalKMS.DefaultKeyID(), encKey, sealedKey)
 		return objectKey, nil
 	}
 	var extKey [32]byte
@@ -853,133 +851,3 @@ func deriveClientKey(clientKey [32]byte, bucket, object string) [32]byte {
 	mac.Sum(key[:0])
 	return key
 }
-
-// set encryption options for pass through to backend in the case of gateway and UserDefined metadata
-func getDefaultOpts(header http.Header, copySource bool, metadata map[string]string) (opts ObjectOptions, err error) {
-	var clientKey [32]byte
-	var sse encrypt.ServerSide
-
-	if copySource {
-		if crypto.SSECopy.IsRequested(header) {
-			clientKey, err = crypto.SSECopy.ParseHTTP(header)
-			if err != nil {
-				return
-			}
-			if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil {
-				return
-			}
-			return ObjectOptions{ServerSideEncryption: encrypt.SSECopy(sse), UserDefined: metadata}, nil
-		}
-		return
-	}
-
-	if crypto.SSEC.IsRequested(header) {
-		clientKey, err = crypto.SSEC.ParseHTTP(header)
-		if err != nil {
-			return
-		}
-		if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil {
-			return
-		}
-		return ObjectOptions{ServerSideEncryption: sse, UserDefined: metadata}, nil
-	}
-	if crypto.S3.IsRequested(header) || (metadata != nil && crypto.S3.IsEncrypted(metadata)) {
-		return ObjectOptions{ServerSideEncryption: encrypt.NewSSE(), UserDefined: metadata}, nil
-	}
-	return ObjectOptions{UserDefined: metadata}, nil
-}
-
-// get ObjectOptions for GET calls from encryption headers
-func getOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) {
-	var (
-		encryption encrypt.ServerSide
-		opts       ObjectOptions
-	)
-
-	var partNumber int
-	var err error
-	if pn := r.URL.Query().Get("partNumber"); pn != "" {
-		partNumber, err = strconv.Atoi(pn)
-		if err != nil {
-			return opts, err
-		}
-		if partNumber <= 0 {
-			return opts, errInvalidArgument
-		}
-	}
-
-	if GlobalGatewaySSE.SSEC() && crypto.SSEC.IsRequested(r.Header) {
-		key, err := crypto.SSEC.ParseHTTP(r.Header)
-		if err != nil {
-			return opts, err
-		}
-		derivedKey := deriveClientKey(key, bucket, object)
-		encryption, err = encrypt.NewSSEC(derivedKey[:])
-		logger.CriticalIf(ctx, err)
-		return ObjectOptions{ServerSideEncryption: encryption, PartNumber: partNumber}, nil
-	}
-
-	// default case of passing encryption headers to backend
-	opts, err = getDefaultOpts(r.Header, false, nil)
-	if err != nil {
-		return opts, err
-	}
-	opts.PartNumber = partNumber
-	return opts, nil
-}
-
-// get ObjectOptions for PUT calls from encryption headers and metadata
-func putOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
-	// In the case of multipart custom format, the metadata needs to be checked in addition to header to see if it
-	// is SSE-S3 encrypted, primarily because S3 protocol does not require SSE-S3 headers in PutObjectPart calls
-	if GlobalGatewaySSE.SSES3() && (crypto.S3.IsRequested(r.Header) || crypto.S3.IsEncrypted(metadata)) {
-		return ObjectOptions{ServerSideEncryption: encrypt.NewSSE(), UserDefined: metadata}, nil
-	}
-	if GlobalGatewaySSE.SSEC() && crypto.SSEC.IsRequested(r.Header) {
-		opts, err = getOpts(ctx, r, bucket, object)
-		opts.UserDefined = metadata
-		return
-	}
-	if crypto.S3KMS.IsRequested(r.Header) {
-		keyID, context, err := crypto.S3KMS.ParseHTTP(r.Header)
-		if err != nil {
-			return ObjectOptions{}, err
-		}
-		sseKms, err := encrypt.NewSSEKMS(keyID, context)
-		if err != nil {
-			return ObjectOptions{}, err
-		}
-		return ObjectOptions{ServerSideEncryption: sseKms, UserDefined: metadata}, nil
-	}
-	// default case of passing encryption headers and UserDefined metadata to backend
-	return getDefaultOpts(r.Header, false, metadata)
-}
-
-// get ObjectOptions for Copy calls with encryption headers provided on the target side and source side metadata
-func copyDstOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
-	return putOpts(ctx, r, bucket, object, metadata)
-}
-
-// get ObjectOptions for Copy calls with encryption headers provided on the source side
-func copySrcOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) {
-	var (
-		ssec encrypt.ServerSide
-		opts ObjectOptions
-	)
-
-	if GlobalGatewaySSE.SSEC() && crypto.SSECopy.IsRequested(r.Header) {
-		key, err := crypto.SSECopy.ParseHTTP(r.Header)
-		if err != nil {
-			return opts, err
-		}
-		derivedKey := deriveClientKey(key, bucket, object)
-		ssec, err = encrypt.NewSSEC(derivedKey[:])
-		if err != nil {
-			return opts, err
-		}
-		return ObjectOptions{ServerSideEncryption: encrypt.SSECopy(ssec)}, nil
-	}
-
-	// default case of passing encryption headers to backend
-	return getDefaultOpts(r.Header, true, nil)
-}

cmd/endpoint.go

@@ -17,8 +17,10 @@
 package cmd
 
 import (
+	"crypto/tls"
 	"fmt"
 	"net"
+	"net/http"
 	"net/url"
 	"path"
 	"path/filepath"
@@ -31,6 +33,7 @@ import (
 	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/cmd/rest"
 	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/mountinfo"
 )
@@ -44,10 +47,15 @@ const (
 
 	// URLEndpointType - URL style endpoint type enum.
 	URLEndpointType
-
-	retryInterval = 5 // In Seconds.
 )
 
+// ProxyEndpoint - endpoint used for proxy redirects
+// See proxyRequest() for details.
+type ProxyEndpoint struct {
+	Endpoint
+	Transport *http.Transport
+}
+
 // Endpoint - any type of endpoint.
 type Endpoint struct {
 	*url.URL
@@ -302,7 +310,7 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 	resolvedList := make([]bool, len(endpoints))
 	// Mark the starting time
 	startTime := time.Now()
-	keepAliveTicker := time.NewTicker(retryInterval * time.Second)
+	keepAliveTicker := time.NewTicker(10 * time.Millisecond)
 	defer keepAliveTicker.Stop()
 	for {
 		// Break if the local endpoint is found already Or all the endpoints are resolved.
@@ -547,9 +555,9 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 		return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg("invalid number of endpoints")
 	}
 
-	// Return XL setup when all endpoints are path style.
+	// Return Erasure setup when all endpoints are path style.
 	if endpoints[0].Type() == PathEndpointType {
-		setupType = XLSetupType
+		setupType = ErasureSetupType
 		return endpoints, setupType, nil
 	}
 
@@ -614,18 +622,18 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 
 	// All endpoints are pointing to local host
 	if len(endpoints) == localEndpointCount {
-		// If all endpoints have same port number, Just treat it as distXL setup
+		// If all endpoints have same port number, Just treat it as distErasure setup
 		// using URL style endpoints.
 		if len(localPortSet) == 1 {
 			if len(localServerHostSet) > 1 {
 				return endpoints, setupType,
 					config.ErrInvalidErasureEndpoints(nil).Msg("all local endpoints should not have different hostnames/ips")
 			}
-			return endpoints, DistXLSetupType, nil
+			return endpoints, DistErasureSetupType, nil
 		}
 
 		// Even though all endpoints are local, but those endpoints use different ports.
-		// This means it is DistXL setup.
+		// This means it is DistErasure setup.
 	}
 
 	// Add missing port in all endpoints.
@@ -645,7 +653,7 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 	}
 
 	// Error out if we have less than 2 unique servers.
-	if len(uniqueArgs.ToSlice()) < 2 && setupType == DistXLSetupType {
+	if len(uniqueArgs.ToSlice()) < 2 && setupType == DistErasureSetupType {
 		err := fmt.Errorf("Unsupported number of endpoints (%s), minimum number of servers cannot be less than 2 in distributed setup", endpoints)
 		return endpoints, setupType, err
 	}
@@ -655,7 +663,7 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 		updateDomainIPs(uniqueArgs)
 	}
 
-	setupType = DistXLSetupType
+	setupType = DistErasureSetupType
 	return endpoints, setupType, nil
 }
 
@@ -710,6 +718,50 @@ func GetRemotePeers(endpointZones EndpointZones) []string {
 	return peerSet.ToSlice()
 }
 
+// GetProxyEndpointLocalIndex returns index of the local proxy endpoint
+func GetProxyEndpointLocalIndex(proxyEps []ProxyEndpoint) int {
+	for i, pep := range proxyEps {
+		if pep.IsLocal {
+			return i
+		}
+	}
+	return -1
+}
+
+// GetProxyEndpoints - get all endpoints that can be used to proxy list request.
+func GetProxyEndpoints(endpointZones EndpointZones) ([]ProxyEndpoint, error) {
+	var proxyEps []ProxyEndpoint
+
+	proxyEpSet := set.NewStringSet()
+
+	for _, ep := range endpointZones {
+		for _, endpoint := range ep.Endpoints {
+			if endpoint.Type() != URLEndpointType {
+				continue
+			}
+
+			host := endpoint.Host
+			if proxyEpSet.Contains(host) {
+				continue
+			}
+			proxyEpSet.Add(host)
+
+			var tlsConfig *tls.Config
+			if globalIsSSL {
+				tlsConfig = &tls.Config{
+					ServerName: endpoint.Hostname(),
+					RootCAs:    globalRootCAs,
+				}
+			}
+			proxyEps = append(proxyEps, ProxyEndpoint{
+				Endpoint:  endpoint,
+				Transport: newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)(),
+			})
+		}
+	}
+	return proxyEps, nil
+}
+
 func updateDomainIPs(endPoints set.StringSet) {
 	ipList := set.NewStringSet()
 	for e := range endPoints {

cmd/endpoint_test.go

@@ -232,71 +232,71 @@ func TestCreateEndpoints(t *testing.T) {
 		{"localhost:10000", [][]string{{"/d1"}}, "localhost:10000", Endpoints{Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true}}, FSSetupType, nil},
 		{"localhost:9000", [][]string{{"https://127.0.0.1:9000/d1", "https://localhost:9001/d1", "https://example.com/d1", "https://example.com/d2"}}, "", Endpoints{}, -1, fmt.Errorf("path '/d1' can not be served by different port on same address")},
 
-		// XL Setup with PathEndpointType
+		// Erasure Setup with PathEndpointType
 		{":1234", [][]string{{"/d1", "/d2", "/d3", "/d4"}}, ":1234",
 			Endpoints{
 				Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true},
 				Endpoint{URL: &url.URL{Path: mustAbs("/d2")}, IsLocal: true},
 				Endpoint{URL: &url.URL{Path: mustAbs("/d3")}, IsLocal: true},
 				Endpoint{URL: &url.URL{Path: mustAbs("/d4")}, IsLocal: true},
-			}, XLSetupType, nil},
-		// DistXL Setup with URLEndpointType
+			}, ErasureSetupType, nil},
+		// DistErasure Setup with URLEndpointType
 		{":9000", [][]string{{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}}, ":9000", Endpoints{
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d1"}, IsLocal: true},
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d2"}, IsLocal: true},
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d3"}, IsLocal: true},
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d4"}, IsLocal: true},
-		}, DistXLSetupType, nil},
-		// DistXL Setup with URLEndpointType having mixed naming to local host.
+		}, DistErasureSetupType, nil},
+		// DistErasure Setup with URLEndpointType having mixed naming to local host.
 		{"127.0.0.1:10000", [][]string{{"http://localhost/d1", "http://localhost/d2", "http://127.0.0.1/d3", "http://127.0.0.1/d4"}}, "", Endpoints{}, -1, fmt.Errorf("all local endpoints should not have different hostnames/ips")},
 
 		{":9001", [][]string{{"http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export", "http://" + nonLoopBackIP + ":9001/export", "http://10.0.0.2:9001/export"}}, "", Endpoints{}, -1, fmt.Errorf("path '/export' can not be served by different port on same address")},
 
 		{":9000", [][]string{{"http://127.0.0.1:9000/export", "http://" + nonLoopBackIP + ":9000/export", "http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export"}}, "", Endpoints{}, -1, fmt.Errorf("path '/export' cannot be served by different address on same server")},
 
-		// DistXL type
+		// DistErasure type
 		{"127.0.0.1:10000", [][]string{{case1Endpoint1, case1Endpoint2, "http://example.org/d3", "http://example.com/d4"}}, "127.0.0.1:10000", Endpoints{
 			Endpoint{URL: case1URLs[0], IsLocal: case1LocalFlags[0]},
 			Endpoint{URL: case1URLs[1], IsLocal: case1LocalFlags[1]},
 			Endpoint{URL: case1URLs[2], IsLocal: case1LocalFlags[2]},
 			Endpoint{URL: case1URLs[3], IsLocal: case1LocalFlags[3]},
-		}, DistXLSetupType, nil},
+		}, DistErasureSetupType, nil},
 
 		{"127.0.0.1:10000", [][]string{{case2Endpoint1, case2Endpoint2, "http://example.org/d3", "http://example.com/d4"}}, "127.0.0.1:10000", Endpoints{
 			Endpoint{URL: case2URLs[0], IsLocal: case2LocalFlags[0]},
 			Endpoint{URL: case2URLs[1], IsLocal: case2LocalFlags[1]},
 			Endpoint{URL: case2URLs[2], IsLocal: case2LocalFlags[2]},
 			Endpoint{URL: case2URLs[3], IsLocal: case2LocalFlags[3]},
-		}, DistXLSetupType, nil},
+		}, DistErasureSetupType, nil},
 
 		{":80", [][]string{{case3Endpoint1, "http://example.org:9000/d2", "http://example.com/d3", "http://example.net/d4"}}, ":80", Endpoints{
 			Endpoint{URL: case3URLs[0], IsLocal: case3LocalFlags[0]},
 			Endpoint{URL: case3URLs[1], IsLocal: case3LocalFlags[1]},
 			Endpoint{URL: case3URLs[2], IsLocal: case3LocalFlags[2]},
 			Endpoint{URL: case3URLs[3], IsLocal: case3LocalFlags[3]},
-		}, DistXLSetupType, nil},
+		}, DistErasureSetupType, nil},
 
 		{":9000", [][]string{{case4Endpoint1, "http://example.org/d2", "http://example.com/d3", "http://example.net/d4"}}, ":9000", Endpoints{
 			Endpoint{URL: case4URLs[0], IsLocal: case4LocalFlags[0]},
 			Endpoint{URL: case4URLs[1], IsLocal: case4LocalFlags[1]},
 			Endpoint{URL: case4URLs[2], IsLocal: case4LocalFlags[2]},
 			Endpoint{URL: case4URLs[3], IsLocal: case4LocalFlags[3]},
-		}, DistXLSetupType, nil},
+		}, DistErasureSetupType, nil},
 
 		{":9000", [][]string{{case5Endpoint1, case5Endpoint2, case5Endpoint3, case5Endpoint4}}, ":9000", Endpoints{
 			Endpoint{URL: case5URLs[0], IsLocal: case5LocalFlags[0]},
 			Endpoint{URL: case5URLs[1], IsLocal: case5LocalFlags[1]},
 			Endpoint{URL: case5URLs[2], IsLocal: case5LocalFlags[2]},
 			Endpoint{URL: case5URLs[3], IsLocal: case5LocalFlags[3]},
-		}, DistXLSetupType, nil},
+		}, DistErasureSetupType, nil},
 
-		// DistXL Setup using only local host.
+		// DistErasure Setup using only local host.
 		{":9003", [][]string{{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://127.0.0.1:9002/d3", case6Endpoint}}, ":9003", Endpoints{
 			Endpoint{URL: case6URLs[0], IsLocal: case6LocalFlags[0]},
 			Endpoint{URL: case6URLs[1], IsLocal: case6LocalFlags[1]},
 			Endpoint{URL: case6URLs[2], IsLocal: case6LocalFlags[2]},
 			Endpoint{URL: case6URLs[3], IsLocal: case6LocalFlags[3]},
-		}, DistXLSetupType, nil},
+		}, DistErasureSetupType, nil},
 	}
 
 	for _, testCase := range testCases {

cmd/erasure-bucket.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,7 +22,6 @@ import (
 
 	"github.com/minio/minio-go/v6/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
-
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
@@ -35,13 +34,13 @@ var bucketMetadataOpIgnoredErrs = append(bucketOpIgnoredErrs, errVolumeNotFound)
 /// Bucket operations
 
 // MakeBucket - make a bucket.
-func (xl xlObjects) MakeBucketWithLocation(ctx context.Context, bucket, location string, lockEnabled bool) error {
+func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts BucketOptions) error {
 	// Verify if bucket is valid.
 	if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
 		return BucketNameInvalid{Bucket: bucket}
 	}
 
-	storageDisks := xl.getDisks()
+	storageDisks := er.getDisks()
 
 	g := errgroup.WithNErrs(len(storageDisks))
 
@@ -86,9 +85,9 @@ func undoDeleteBucket(storageDisks []StorageAPI, bucket string) {
 }
 
 // getBucketInfo - returns the BucketInfo from one of the load balanced disks.
-func (xl xlObjects) getBucketInfo(ctx context.Context, bucketName string) (bucketInfo BucketInfo, err error) {
+func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string) (bucketInfo BucketInfo, err error) {
 	var bucketErrs []error
-	for _, disk := range xl.getLoadBalancedDisks() {
+	for _, disk := range er.getLoadBalancedDisks() {
 		if disk == nil {
 			bucketErrs = append(bucketErrs, errDiskNotFound)
 			continue
@@ -110,13 +109,13 @@ func (xl xlObjects) getBucketInfo(ctx context.Context, bucketName string) (bucke
 	// reduce to one error based on read quorum.
 	// `nil` is deliberately passed for ignoredErrs
 	// because these errors were already ignored.
-	readQuorum := getReadQuorum(len(xl.getDisks()))
+	readQuorum := getReadQuorum(len(er.getDisks()))
 	return BucketInfo{}, reduceReadQuorumErrs(ctx, bucketErrs, nil, readQuorum)
 }
 
 // GetBucketInfo - returns BucketInfo for a bucket.
-func (xl xlObjects) GetBucketInfo(ctx context.Context, bucket string) (bi BucketInfo, e error) {
-	bucketInfo, err := xl.getBucketInfo(ctx, bucket)
+func (er erasureObjects) GetBucketInfo(ctx context.Context, bucket string) (bi BucketInfo, e error) {
+	bucketInfo, err := er.getBucketInfo(ctx, bucket)
 	if err != nil {
 		return bi, toObjectErr(err, bucket)
 	}
@@ -124,8 +123,8 @@ func (xl xlObjects) GetBucketInfo(ctx context.Context, bucket string) (bi Bucket
 }
 
 // listBuckets - returns list of all buckets from a disk picked at random.
-func (xl xlObjects) listBuckets(ctx context.Context) (bucketsInfo []BucketInfo, err error) {
-	for _, disk := range xl.getLoadBalancedDisks() {
+func (er erasureObjects) listBuckets(ctx context.Context) (bucketsInfo []BucketInfo, err error) {
+	for _, disk := range er.getLoadBalancedDisks() {
 		if disk == nil {
 			continue
 		}
@@ -161,8 +160,8 @@ func (xl xlObjects) listBuckets(ctx context.Context) (bucketsInfo []BucketInfo,
 }
 
 // ListBuckets - lists all the buckets, sorted by its name.
-func (xl xlObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
-	bucketInfos, err := xl.listBuckets(ctx)
+func (er erasureObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
+	bucketInfos, err := er.listBuckets(ctx)
 	if err != nil {
 		return nil, toObjectErr(err)
 	}
@@ -196,9 +195,9 @@ func deleteDanglingBucket(ctx context.Context, storageDisks []StorageAPI, dErrs
 }
 
 // DeleteBucket - deletes a bucket.
-func (xl xlObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
+func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
 	// Collect if all disks report volume not found.
-	storageDisks := xl.getDisks()
+	storageDisks := er.getDisks()
 
 	g := errgroup.WithNErrs(len(storageDisks))
 
@@ -235,7 +234,7 @@ func (xl xlObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete
 
 	writeQuorum := getWriteQuorum(len(storageDisks))
 	err := reduceWriteQuorumErrs(ctx, dErrs, bucketOpIgnoredErrs, writeQuorum)
-	if err == errXLWriteQuorum {
+	if err == errErasureWriteQuorum {
 		undoDeleteBucket(storageDisks, bucket)
 	}
 	if err != nil {
@@ -251,25 +250,26 @@ func (xl xlObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete
 }
 
 // IsNotificationSupported returns whether bucket notification is applicable for this layer.
-func (xl xlObjects) IsNotificationSupported() bool {
+func (er erasureObjects) IsNotificationSupported() bool {
 	return true
 }
 
 // IsListenBucketSupported returns whether listen bucket notification is applicable for this layer.
-func (xl xlObjects) IsListenBucketSupported() bool {
+func (er erasureObjects) IsListenBucketSupported() bool {
 	return true
 }
 
 // IsEncryptionSupported returns whether server side encryption is implemented for this layer.
-func (xl xlObjects) IsEncryptionSupported() bool {
+func (er erasureObjects) IsEncryptionSupported() bool {
 	return true
 }
 
 // IsCompressionSupported returns whether compression is applicable for this layer.
-func (xl xlObjects) IsCompressionSupported() bool {
+func (er erasureObjects) IsCompressionSupported() bool {
 	return true
 }
 
-func (xl xlObjects) IsTaggingSupported() bool {
+// IsTaggingSupported indicates whether erasureObjects implements tagging support.
+func (er erasureObjects) IsTaggingSupported() bool {
 	return true
 }

cmd/erasure-coding.go

@@ -0,0 +1,143 @@
+/*
+ * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"sync"
+
+	"github.com/klauspost/reedsolomon"
+	"github.com/minio/minio/cmd/logger"
+)
+
+// Erasure - erasure encoding details.
+type Erasure struct {
+	encoder                  func() reedsolomon.Encoder
+	dataBlocks, parityBlocks int
+	blockSize                int64
+}
+
+// NewErasure creates a new ErasureStorage.
+func NewErasure(ctx context.Context, dataBlocks, parityBlocks int, blockSize int64) (e Erasure, err error) {
+	// Check the parameters for sanity now.
+	if dataBlocks <= 0 || parityBlocks <= 0 {
+		return e, reedsolomon.ErrInvShardNum
+	}
+
+	if dataBlocks+parityBlocks > 256 {
+		return e, reedsolomon.ErrMaxShardNum
+	}
+
+	e = Erasure{
+		dataBlocks:   dataBlocks,
+		parityBlocks: parityBlocks,
+		blockSize:    blockSize,
+	}
+
+	// Encoder when needed.
+	var enc reedsolomon.Encoder
+	var once sync.Once
+	e.encoder = func() reedsolomon.Encoder {
+		once.Do(func() {
+			e, err := reedsolomon.New(dataBlocks, parityBlocks, reedsolomon.WithAutoGoroutines(int(e.ShardSize())))
+			if err != nil {
+				// Error conditions should be checked above.
+				panic(err)
+			}
+			enc = e
+		})
+		return enc
+	}
+	return
+}
+
+// EncodeData encodes the given data and returns the erasure-coded data.
+// It returns an error if the erasure coding failed.
+func (e *Erasure) EncodeData(ctx context.Context, data []byte) ([][]byte, error) {
+	if len(data) == 0 {
+		return make([][]byte, e.dataBlocks+e.parityBlocks), nil
+	}
+	encoded, err := e.encoder().Split(data)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return nil, err
+	}
+	if err = e.encoder().Encode(encoded); err != nil {
+		logger.LogIf(ctx, err)
+		return nil, err
+	}
+	return encoded, nil
+}
+
+// DecodeDataBlocks decodes the given erasure-coded data.
+// It only decodes the data blocks but does not verify them.
+// It returns an error if the decoding failed.
+func (e *Erasure) DecodeDataBlocks(data [][]byte) error {
+	var isZero = 0
+	for _, b := range data[:] {
+		if len(b) == 0 {
+			isZero++
+			break
+		}
+	}
+	if isZero == 0 || isZero == len(data) {
+		// If all are zero, payload is 0 bytes.
+		return nil
+	}
+	return e.encoder().ReconstructData(data)
+}
+
+// DecodeDataAndParityBlocks decodes the given erasure-coded data and verifies it.
+// It returns an error if the decoding failed.
+func (e *Erasure) DecodeDataAndParityBlocks(ctx context.Context, data [][]byte) error {
+	if err := e.encoder().Reconstruct(data); err != nil {
+		logger.LogIf(ctx, err)
+		return err
+	}
+	return nil
+}
+
+// ShardSize - returns actual shared size from erasure blockSize.
+func (e *Erasure) ShardSize() int64 {
+	return ceilFrac(e.blockSize, int64(e.dataBlocks))
+}
+
+// ShardFileSize - returns final erasure size from original size.
+func (e *Erasure) ShardFileSize(totalLength int64) int64 {
+	if totalLength == 0 {
+		return 0
+	}
+	if totalLength == -1 {
+		return -1
+	}
+	numShards := totalLength / e.blockSize
+	lastBlockSize := totalLength % int64(e.blockSize)
+	lastShardSize := ceilFrac(lastBlockSize, int64(e.dataBlocks))
+	return numShards*e.ShardSize() + lastShardSize
+}
+
+// ShardFileOffset - returns the effective offset where erasure reading begins.
+func (e *Erasure) ShardFileOffset(startOffset, length, totalLength int64) int64 {
+	shardSize := e.ShardSize()
+	shardFileSize := e.ShardFileSize(totalLength)
+	endShard := (startOffset + int64(length)) / e.blockSize
+	tillOffset := endShard*shardSize + shardSize
+	if tillOffset > shardFileSize {
+		tillOffset = shardFileSize
+	}
+	return tillOffset
+}

cmd/erasure-common.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016, 2017 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,8 +24,8 @@ import (
 )
 
 // getLoadBalancedDisks - fetches load balanced (sufficiently randomized) disk slice.
-func (xl xlObjects) getLoadBalancedDisks() (newDisks []StorageAPI) {
-	disks := xl.getDisks()
+func (er erasureObjects) getLoadBalancedDisks() (newDisks []StorageAPI) {
+	disks := er.getDisks()
 	// Based on the random shuffling return back randomized disks.
 	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
 		newDisks = append(newDisks, disks[i-1])
@@ -36,13 +36,13 @@ func (xl xlObjects) getLoadBalancedDisks() (newDisks []StorageAPI) {
 // This function does the following check, suppose
 // object is "a/b/c/d", stat makes sure that objects ""a/b/c""
 // "a/b" and "a" do not exist.
-func (xl xlObjects) parentDirIsObject(ctx context.Context, bucket, parent string) bool {
+func (er erasureObjects) parentDirIsObject(ctx context.Context, bucket, parent string) bool {
 	var isParentDirObject func(string) bool
 	isParentDirObject = func(p string) bool {
 		if p == "." || p == SlashSeparator {
 			return false
 		}
-		if xl.isObject(bucket, p) {
+		if er.isObject(ctx, bucket, p) {
 			// If there is already a file at prefix "p", return true.
 			return true
 		}
@@ -53,9 +53,9 @@ func (xl xlObjects) parentDirIsObject(ctx context.Context, bucket, parent string
 }
 
 // isObject - returns `true` if the prefix is an object i.e if
-// `xl.json` exists at the leaf, false otherwise.
-func (xl xlObjects) isObject(bucket, prefix string) (ok bool) {
-	storageDisks := xl.getDisks()
+// `xl.meta` exists at the leaf, false otherwise.
+func (er erasureObjects) isObject(ctx context.Context, bucket, prefix string) (ok bool) {
+	storageDisks := er.getDisks()
 
 	g := errgroup.WithNErrs(len(storageDisks))
 
@@ -66,22 +66,15 @@ func (xl xlObjects) isObject(bucket, prefix string) (ok bool) {
 				return errDiskNotFound
 			}
 			// Check if 'prefix' is an object on this 'disk', else continue the check the next disk
-			fi, err := storageDisks[index].StatFile(bucket, pathJoin(prefix, xlMetaJSONFile))
-			if err != nil {
-				return err
-			}
-			if fi.Size == 0 {
-				return errCorruptedFormat
-			}
-			return nil
+			return storageDisks[index].CheckFile(bucket, prefix)
 		}, index)
 	}
 
-	// NOTE: Observe we are not trying to read `xl.json` and figure out the actual
+	// NOTE: Observe we are not trying to read `xl.meta` and figure out the actual
 	// quorum intentionally, but rely on the default case scenario. Actual quorum
 	// verification will happen by top layer by using getObjectInfo() and will be
 	// ignored if necessary.
 	readQuorum := getReadQuorum(len(storageDisks))
 
-	return reduceReadQuorumErrs(GlobalContext, g.Wait(), objectOpIgnoredErrs, readQuorum) == nil
+	return reduceReadQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, readQuorum) == nil
 }

cmd/erasure-common_test.go

@@ -24,13 +24,13 @@ import (
 )
 
 // Tests for if parent directory is object
-func TestXLParentDirIsObject(t *testing.T) {
+func TestErasureParentDirIsObject(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	obj, fsDisks, err := prepareXL16(ctx)
+	obj, fsDisks, err := prepareErasure16(ctx)
 	if err != nil {
-		t.Fatalf("Unable to initialize 'XL' object layer.")
+		t.Fatalf("Unable to initialize 'Erasure' object layer.")
 	}
 
 	// Remove all disks.
@@ -41,7 +41,7 @@ func TestXLParentDirIsObject(t *testing.T) {
 	bucketName := "testbucket"
 	objectName := "object"
 
-	if err = obj.MakeBucketWithLocation(GlobalContext, bucketName, "", false); err != nil {
+	if err = obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
 		t.Fatal(err)
 	}
 	objectContent := "12345"
@@ -54,7 +54,7 @@ func TestXLParentDirIsObject(t *testing.T) {
 		t.Fatalf("Unexpected object name returned got %s, expected %s", objInfo.Name, objectName)
 	}
 
-	z := obj.(*xlZones)
+	z := obj.(*erasureZones)
 	xl := z.zones[0].sets[0]
 	testCases := []struct {
 		parentIsObject bool

cmd/erasure-decode.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -145,9 +145,9 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 		wg.Add(1)
 		go func(i int) {
 			defer wg.Done()
-			disk := p.readers[i]
-			if disk == nil {
-				// Since disk is nil, trigger another read.
+			rr := p.readers[i]
+			if rr == nil {
+				// Since reader is nil, trigger another read.
 				readTriggerCh <- true
 				return
 			}
@@ -160,7 +160,7 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 			// For the last shard, the shardsize might be less than previous shard sizes.
 			// Hence the following statement ensures that the buffer size is reset to the right size.
 			p.buf[bufIdx] = p.buf[bufIdx][:p.shardSize]
-			_, err := disk.ReadAt(p.buf[bufIdx], p.offset)
+			_, err := rr.ReadAt(p.buf[bufIdx], p.offset)
 			if err != nil {
 				if _, ok := err.(*errHashMismatch); ok {
 					atomic.StoreInt32(&healRequired, 1)
@@ -191,7 +191,7 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 		return newBuf, nil
 	}
 
-	return nil, errXLReadQuorum
+	return nil, errErasureReadQuorum
 }
 
 type errDecodeHealRequired struct {

cmd/erasure-decode_test.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016, 2017 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -132,7 +132,7 @@ func TestErasureDecode(t *testing.T) {
 			if disk == OfflineDisk {
 				continue
 			}
-			tillOffset := erasure.ShardFileTillOffset(test.offset, test.length, test.data)
+			tillOffset := erasure.ShardFileOffset(test.offset, test.length, test.data)
 
 			bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", tillOffset, writeAlgorithm, bitrotWriterSum(writers[index]), erasure.ShardSize())
 		}
@@ -163,7 +163,7 @@ func TestErasureDecode(t *testing.T) {
 				if disk == OfflineDisk {
 					continue
 				}
-				tillOffset := erasure.ShardFileTillOffset(test.offset, test.length, test.data)
+				tillOffset := erasure.ShardFileOffset(test.offset, test.length, test.data)
 				bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", tillOffset, writeAlgorithm, bitrotWriterSum(writers[index]), erasure.ShardSize())
 			}
 			for j := range disks[:test.offDisks] {
@@ -268,7 +268,7 @@ func TestErasureDecodeRandomOffsetLength(t *testing.T) {
 			if disk == OfflineDisk {
 				continue
 			}
-			tillOffset := erasure.ShardFileTillOffset(offset, readLen, length)
+			tillOffset := erasure.ShardFileOffset(offset, readLen, length)
 			bitrotReaders[index] = newStreamingBitrotReader(disk, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
 		err = erasure.Decode(context.Background(), buf, bitrotReaders, offset, readLen, length, nil)
@@ -330,7 +330,7 @@ func benchmarkErasureDecode(data, parity, dataDown, parityDown int, size int64,
 			if writers[index] == nil {
 				continue
 			}
-			tillOffset := erasure.ShardFileTillOffset(0, size, size)
+			tillOffset := erasure.ShardFileOffset(0, size, size)
 			bitrotReaders[index] = newStreamingBitrotReader(disk, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
 		if err = erasure.Decode(context.Background(), bytes.NewBuffer(content[:0]), bitrotReaders, 0, size, size, nil); err != nil {

cmd/erasure-encode.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

cmd/erasure-encode_test.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

cmd/erasure-errors.go

@@ -18,11 +18,11 @@ package cmd
 
 import "errors"
 
-// errXLReadQuorum - did not meet read quorum.
-var errXLReadQuorum = errors.New("Read failed. Insufficient number of disks online")
+// errErasureReadQuorum - did not meet read quorum.
+var errErasureReadQuorum = errors.New("Read failed. Insufficient number of disks online")
 
-// errXLWriteQuorum - did not meet write quorum.
-var errXLWriteQuorum = errors.New("Write failed. Insufficient number of disks online")
+// errErasureWriteQuorum - did not meet write quorum.
+var errErasureWriteQuorum = errors.New("Write failed. Insufficient number of disks online")
 
 // errNoHealRequired - returned when healing is attempted on a previously healed disks.
 var errNoHealRequired = errors.New("No healing is required")

cmd/erasure-heal_test.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -71,7 +71,7 @@ func TestErasureHeal(t *testing.T) {
 		// create some test data
 		setup, err := newErasureTestSetup(test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
 		if err != nil {
-			t.Fatalf("Test %d: failed to setup XL environment: %v", i, err)
+			t.Fatalf("Test %d: failed to setup Erasure environment: %v", i, err)
 		}
 		disks := setup.disks
 		erasure, err := NewErasure(context.Background(), test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)

cmd/erasure-healing-common.go

@@ -18,10 +18,8 @@ package cmd
 
 import (
 	"context"
-	"fmt"
 	"time"
 
-	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -31,7 +29,7 @@ func commonTime(modTimes []time.Time) (modTime time.Time, count int) {
 	timeOccurenceMap := make(map[time.Time]int)
 	// Ignore the uuid sentinel and count the rest.
 	for _, time := range modTimes {
-		if time == timeSentinel {
+		if time.Equal(timeSentinel) {
 			continue
 		}
 		timeOccurenceMap[time]++
@@ -61,45 +59,45 @@ func bootModtimes(diskCount int) []time.Time {
 	return modTimes
 }
 
-// Extracts list of times from xlMetaV1 slice and returns, skips
+// Extracts list of times from FileInfo slice and returns, skips
 // slice elements which have errors.
-func listObjectModtimes(partsMetadata []xlMetaV1, errs []error) (modTimes []time.Time) {
+func listObjectModtimes(partsMetadata []FileInfo, errs []error) (modTimes []time.Time) {
 	modTimes = bootModtimes(len(partsMetadata))
 	for index, metadata := range partsMetadata {
 		if errs[index] != nil {
 			continue
 		}
 		// Once the file is found, save the uuid saved on disk.
-		modTimes[index] = metadata.Stat.ModTime
+		modTimes[index] = metadata.ModTime
 	}
 	return modTimes
 }
 
 // Notes:
 // There are 5 possible states a disk could be in,
-// 1. __online__             - has the latest copy of xl.json - returned by listOnlineDisks
+// 1. __online__             - has the latest copy of xl.meta - returned by listOnlineDisks
 //
 // 2. __offline__            - err == errDiskNotFound
 //
-// 3. __availableWithParts__ - has the latest copy of xl.json and has all
+// 3. __availableWithParts__ - has the latest copy of xl.meta and has all
 //                             parts with checksums matching; returned by disksWithAllParts
 //
 // 4. __outdated__           - returned by outDatedDisk, provided []StorageAPI
 //                             returned by diskWithAllParts is passed for latestDisks.
-//    - has an old copy of xl.json
-//    - doesn't have xl.json (errFileNotFound)
-//    - has the latest xl.json but one or more parts are corrupt
+//    - has an old copy of xl.meta
+//    - doesn't have xl.meta (errFileNotFound)
+//    - has the latest xl.meta but one or more parts are corrupt
 //
-// 5. __missingParts__       - has the latest copy of xl.json but has some parts
+// 5. __missingParts__       - has the latest copy of xl.meta but has some parts
 // missing.  This is identified separately since this may need manual
 // inspection to understand the root cause. E.g, this could be due to
 // backend filesystem corruption.
 
 // listOnlineDisks - returns
-// - a slice of disks where disk having 'older' xl.json (or nothing)
+// - a slice of disks where disk having 'older' xl.meta (or nothing)
 // are set to nil.
 // - latest (in time) of the maximally occurring modTime(s).
-func listOnlineDisks(disks []StorageAPI, partsMetadata []xlMetaV1, errs []error) (onlineDisks []StorageAPI, modTime time.Time) {
+func listOnlineDisks(disks []StorageAPI, partsMetadata []FileInfo, errs []error) (onlineDisks []StorageAPI, modTime time.Time) {
 	onlineDisks = make([]StorageAPI, len(disks))
 
 	// List all the file commit ids from parts metadata.
@@ -110,7 +108,7 @@ func listOnlineDisks(disks []StorageAPI, partsMetadata []xlMetaV1, errs []error)
 
 	// Create a new online disks slice, which have common uuid.
 	for index, t := range modTimes {
-		if t == modTime {
+		if t.Equal(modTime) {
 			onlineDisks[index] = disks[index]
 		} else {
 			onlineDisks[index] = nil
@@ -119,89 +117,67 @@ func listOnlineDisks(disks []StorageAPI, partsMetadata []xlMetaV1, errs []error)
 	return onlineDisks, modTime
 }
 
-// Returns the latest updated xlMeta files and error in case of failure.
-func getLatestXLMeta(ctx context.Context, partsMetadata []xlMetaV1, errs []error) (xlMetaV1, error) {
-
+// Returns the latest updated FileInfo files and error in case of failure.
+func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, errs []error) (FileInfo, error) {
 	// There should be atleast half correct entries, if not return failure
 	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, len(partsMetadata)/2); reducedErr != nil {
-		return xlMetaV1{}, reducedErr
+		return FileInfo{}, reducedErr
 	}
 
 	// List all the file commit ids from parts metadata.
 	modTimes := listObjectModtimes(partsMetadata, errs)
 
-	// Count all latest updated xlMeta values
+	// Count all latest updated FileInfo values
 	var count int
-	var latestXLMeta xlMetaV1
+	var latestFileInfo FileInfo
 
 	// Reduce list of UUIDs to a single common value - i.e. the last updated Time
 	modTime, _ := commonTime(modTimes)
 
-	// Interate through all the modTimes and count the xlMeta(s) with latest time.
+	// Interate through all the modTimes and count the FileInfo(s) with latest time.
 	for index, t := range modTimes {
-		if t == modTime && partsMetadata[index].IsValid() {
-			latestXLMeta = partsMetadata[index]
+		if t.Equal(modTime) && partsMetadata[index].IsValid() {
+			latestFileInfo = partsMetadata[index]
 			count++
 		}
 	}
 	if count < len(partsMetadata)/2 {
-		return xlMetaV1{}, errXLReadQuorum
+		return FileInfo{}, errErasureReadQuorum
 	}
 
-	return latestXLMeta, nil
+	return latestFileInfo, nil
 }
 
 // disksWithAllParts - This function needs to be called with
 // []StorageAPI returned by listOnlineDisks. Returns,
 //
-// - disks which have all parts specified in the latest xl.json.
+// - disks which have all parts specified in the latest xl.meta.
 //
 // - slice of errors about the state of data files on disk - can have
 //   a not-found error or a hash-mismatch error.
-func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []xlMetaV1, errs []error, bucket,
+func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []FileInfo, errs []error, bucket,
 	object string, scanMode madmin.HealScanMode) ([]StorageAPI, []error) {
 	availableDisks := make([]StorageAPI, len(onlineDisks))
 	dataErrs := make([]error, len(onlineDisks))
 
 	for i, onlineDisk := range onlineDisks {
-		if onlineDisk == nil {
+		if errs[i] != nil {
 			dataErrs[i] = errs[i]
 			continue
 		}
+		if onlineDisk == nil {
+			dataErrs[i] = errDiskNotFound
+			continue
+		}
 
 		switch scanMode {
 		case madmin.HealDeepScan:
-			erasure := partsMetadata[i].Erasure
-
-			// disk has a valid xl.json but may not have all the
+			// disk has a valid xl.meta but may not have all the
 			// parts. This is considered an outdated disk, since
 			// it needs healing too.
-			for _, part := range partsMetadata[i].Parts {
-				checksumInfo := erasure.GetChecksumInfo(part.Number)
-				partPath := pathJoin(object, fmt.Sprintf("part.%d", part.Number))
-				err := onlineDisk.VerifyFile(bucket, partPath, erasure.ShardFileSize(part.Size), checksumInfo.Algorithm, checksumInfo.Hash, erasure.ShardSize())
-				if err != nil {
-					if !IsErr(err, []error{
-						errFileNotFound,
-						errVolumeNotFound,
-						errFileCorrupt,
-					}...) {
-						logger.GetReqInfo(ctx).AppendTags("disk", onlineDisk.String())
-						logger.LogIf(ctx, err)
-					}
-					dataErrs[i] = err
-					break
-				}
-			}
+			dataErrs[i] = onlineDisk.VerifyFile(bucket, object, partsMetadata[i])
 		case madmin.HealNormalScan:
-			for _, part := range partsMetadata[i].Parts {
-				partPath := pathJoin(object, fmt.Sprintf("part.%d", part.Number))
-				_, err := onlineDisk.StatFile(bucket, partPath)
-				if err != nil {
-					dataErrs[i] = err
-					break
-				}
-			}
+			dataErrs[i] = onlineDisk.CheckParts(bucket, object, partsMetadata[i])
 		}
 
 		if dataErrs[i] == nil {