Make sure to drain body upon an error (#7197)

Also cleanup redundant code and use it at a common place

.travis.yml

@@ -16,24 +16,33 @@ matrix:
       sudo: required
       env:
         - ARCH=x86_64
-      go: 1.10.4
+        - CGO_ENABLED=0
+      go: 1.11.4
       script:
         - make
         - diff -au <(gofmt -s -d cmd) <(printf "")
         - diff -au <(gofmt -s -d pkg) <(printf "")
-        - make test GOFLAGS="-timeout 15m -race -v"
+        - for d in $(go list ./... | grep -v browser); do CGO_ENABLED=1 go test -v -race --timeout 15m "$d"; done
+        - make verifiers
+        - make crosscompile
         - make verify
         - make coverage
         - cd browser && yarn && yarn test && cd ..
     - os: windows
       env:
         - ARCH=x86_64
-      go: 1.10.4
+        - CGO_ENABLED=0
+      go: 1.11.4
       script:
         - go build --ldflags="$(go run buildscripts/gen-ldflags.go)" -o %GOPATH%\bin\minio.exe
-        - for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 20m "$d"; done
+        - for d in $(go list ./... | grep -v browser); do CGO_ENABLED=1 go test -v -race --timeout 20m "$d"; done
         - bash buildscripts/go-coverage.sh
 
+before_script:
+  # Add an IPv6 config - see the corresponding Travis issue
+  # https://github.com/travis-ci/travis-ci/issues/8361
+  - if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6'; fi
+
 before_install:
  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install stable ; fi
 

CONTRIBUTING.md

@@ -68,4 +68,4 @@ To remove a dependency
 - Run `make pkg-remove PKG=foo/bar` from top-level directory
 
 ### What are the coding guidelines for Minio?
-``Minio`` is fully conformant with Golang style. Refer: [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project. If you observe offending code, please feel free to send a pull request or ping us on [Slack](slack.minio.io).
+``Minio`` is fully conformant with Golang style. Refer: [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project. If you observe offending code, please feel free to send a pull request or ping us on [Slack](https://slack.minio.io).

Dockerfile

@@ -1,28 +1,32 @@
-FROM golang:1.10.1-alpine3.7
+FROM golang:1.11.4-alpine3.7
 
 LABEL maintainer="Minio Inc <dev@minio.io>"
 
 ENV GOPATH /go
-ENV PATH $PATH:$GOPATH/bin
 ENV CGO_ENABLED 0
+
+WORKDIR /go/src/github.com/minio/
+
+RUN  \
+     apk add --no-cache git && \
+     go get -v -d github.com/minio/minio && \
+     cd /go/src/github.com/minio/minio && \
+     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
+
+FROM alpine:3.7
+
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key
 
-WORKDIR /go/src/github.com/minio/
+EXPOSE 9000
 
+COPY --from=0 /go/bin/minio /usr/bin/minio
 COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
 
 RUN  \
      apk add --no-cache ca-certificates 'curl>7.61.0' && \
-     apk add --no-cache --virtual .build-deps git && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf
-     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
-     go get -v -d github.com/minio/minio && \
-     cd /go/src/github.com/minio/minio && \
-     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
-     rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
-
-EXPOSE 9000
 
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 

Dockerfile.simpleci

@@ -0,0 +1,49 @@
+#-------------------------------------------------------------
+# Stage 1: Build and Unit tests
+#-------------------------------------------------------------
+FROM golang:1.11.4
+
+COPY . /go/src/github.com/minio/minio
+WORKDIR /go/src/github.com/minio/minio
+
+RUN apt-get update && apt-get install -y jq
+RUN make
+RUN bash -c 'diff -au <(gofmt -s -d cmd) <(printf "")'
+RUN bash -c 'diff -au <(gofmt -s -d pkg) <(printf "")'
+RUN for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 15m "$d"; done
+RUN make verify
+RUN make coverage || true
+
+#-------------------------------------------------------------
+# Stage 2: Test Frontend
+#-------------------------------------------------------------
+FROM node:10.15-stretch-slim
+
+COPY browser /minio/browser
+WORKDIR /minio/browser
+
+RUN yarn
+RUN yarn test
+
+#-------------------------------------------------------------
+# Stage 3: Run Gateway Tests
+#-------------------------------------------------------------
+
+FROM ubuntu:16.04
+
+COPY --from=0 /go/src/github.com/minio/minio/minio ./minio
+COPY buildscripts/gateway-tests.sh ./gateway-tests.sh
+RUN apt-get update && apt-get install -y git wget jq curl dnsmasq
+
+RUN wget https://dl.google.com/go/go1.11.4.linux-amd64.tar.gz && \
+	tar -C /usr/local -xzf go1.11.4.linux-amd64.tar.gz
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV LANG C.UTF-8
+ENV GOROOT /usr/local/go
+
+RUN mkdir -p /go
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
+
+RUN ./gateway-tests.sh

Makefile

@@ -19,6 +19,9 @@ getdeps:
 	@echo "Installing misspell" && go get -u github.com/client9/misspell/cmd/misspell
 	@echo "Installing ineffassign" && go get -u github.com/gordonklaus/ineffassign
 
+crosscompile:
+	@(env bash $(PWD)/buildscripts/cross-compile.sh)
+
 verifiers: getdeps vet fmt lint cyclo deadcode spelling
 
 vet:
@@ -60,7 +63,7 @@ spelling:
 check: test
 test: verifiers build
 	@echo "Running unit tests"
-	@go test $(GOFLAGS) -tags kqueue ./...
+	@CGO_ENABLED=0 go test -tags kqueue ./...
 
 verify: build
 	@echo "Verifying build"
@@ -73,7 +76,7 @@ coverage: build
 # Builds minio locally.
 build: checks
 	@echo "Building minio binary to './minio'"
-	@CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
+	@GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
 
 docker: build
 	@docker build -t $(TAG) . -f Dockerfile.dev

README.md

@@ -53,6 +53,15 @@ chmod +x minio
 ./minio server /data
 ```
 
+| Platform| Architecture | URL|
+| ----------| -------- | ------|
+|GNU/Linux|ppc64le|https://dl.minio.io/server/minio/release/linux-ppc64le/minio |
+```sh
+wget https://dl.minio.io/server/minio/release/linux-ppc64le/minio
+chmod +x minio
+./minio server /data
+```
+
 ## Microsoft Windows
 ### Binary Download
 | Platform| Architecture | URL|

browser/app/index.html

@@ -5,13 +5,13 @@
         <meta charset="UTF-8">
         <meta name="viewport" content="width=device-width, initial-scale=1">
         <title>Minio Browser</title>
-        <link rel="stylesheet" href="loader.css" type="text/css">
+        <link rel="stylesheet" href="/minio/loader.css" type="text/css">
     </head>
 
     <body>
         <div class="page-load">
             <div class="pl-inner">
-                <img src="logo.svg" alt="">
+                <img src="/minio/logo.svg" alt="">
             </div>
         </div>
         <div id="root"></div>
@@ -51,6 +51,6 @@
         <![endif]-->
 
         <script>currentUiVersion = 'MINIO_UI_VERSION'</script>
-        <script src="index_bundle.js"></script>
+        <script src="/minio/index_bundle.js"></script>
     </body>
 </html>

browser/package.json

@@ -13,8 +13,7 @@
     "setupTestFrameworkScriptFile": "./app/js/jest/setup.js",
     "testURL": "https://localhost:8080",
     "moduleNameMapper": {
-      "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$":
+      "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "<rootDir>/app/js/jest/__mocks__/fileMock.js",
-        "<rootDir>/app/js/jest/__mocks__/fileMock.js",
       "\\.(css|scss)$": "identity-obj-proxy"
     }
   },
@@ -40,29 +39,31 @@
     "babel-preset-es2015": "^6.14.0",
     "babel-preset-react": "^6.11.1",
     "babel-register": "^6.14.0",
-    "copy-webpack-plugin": "^0.3.3",
+    "copy-webpack-plugin": "^4.6.0",
     "css-loader": "^0.23.1",
     "enzyme": "^3.3.0",
     "enzyme-adapter-react-16": "^1.1.1",
     "esformatter": "^0.10.0",
     "esformatter-jsx": "^7.4.1",
     "esformatter-jsx-ignore": "^1.0.6",
-    "html-webpack-plugin": "^2.30.1",
+    "html-webpack-plugin": "^3.2.0",
     "jest": "^22.1.4",
     "jest-enzyme": "^4.0.2",
     "json-loader": "^0.5.4",
-    "less": "^2.7.1",
+    "less": "^3.9.0",
-    "less-loader": "^2.2.3",
+    "less-loader": "^4.1.0",
-    "purifycss-webpack-plugin": "^2.0.3",
+    "purgecss-webpack-plugin": "^1.4.0",
     "style-loader": "^0.13.1",
     "url-loader": "^0.5.7",
-    "webpack-dev-server": "^2.11.1"
+    "webpack-cli": "^3.2.0",
+    "webpack-dev-server": "^3.1.14"
   },
   "dependencies": {
     "bootstrap": "^3.3.6",
     "classnames": "^2.2.3",
     "expect": "^1.20.2",
     "font-awesome": "^4.7.0",
+    "glob-all": "^3.1.0",
     "history": "^4.7.2",
     "humanize": "0.0.9",
     "identity-obj-proxy": "^3.0.0",
@@ -89,6 +90,6 @@
     "reselect": "^3.0.1",
     "superagent": "^3.8.2",
     "superagent-es6-promise": "^1.0.0",
-    "webpack": "^3.10.0"
+    "webpack": "^4.28.3"
   }
 }

browser/webpack.config.js

@@ -16,11 +16,13 @@
 
 var webpack = require('webpack')
 var path = require('path')
+var glob = require('glob-all')
 var CopyWebpackPlugin = require('copy-webpack-plugin')
-var purify = require("purifycss-webpack-plugin")
+var PurgecssPlugin = require('purgecss-webpack-plugin')
 
 var exports = {
   context: __dirname,
+  mode: 'development',
   entry: [
     path.resolve(__dirname, 'app/index.js')
   ],
@@ -99,12 +101,11 @@ var exports = {
       {from: 'app/index.html'}
     ]),
     new webpack.ContextReplacementPlugin(/moment[\\\/]locale$/, /^\.\/(en)$/),
-    new purify({
+    new PurgecssPlugin({
-        basePath: __dirname,
+      paths: glob.sync([
-        paths: [
+        path.join(__dirname, 'app/index.html'),
-            "app/index.html",
+        path.join(__dirname, 'app/js/*.js')
-            "app/js/*.js"
+      ])
-        ]
     })
   ]
 }

browser/webpack.production.config.js

@@ -16,11 +16,13 @@
 
 var webpack = require('webpack')
 var path = require('path')
+var glob = require('glob-all')
 var CopyWebpackPlugin = require('copy-webpack-plugin')
-var purify = require("purifycss-webpack-plugin")
+var PurgecssPlugin = require('purgecss-webpack-plugin')
 
 var exports = {
   context: __dirname,
+  mode: 'production',
   entry: [
     path.resolve(__dirname, 'app/index.js')
   ],
@@ -74,16 +76,12 @@ var exports = {
       {from: 'app/img/logo.svg'},
       {from: 'app/index.html'}
     ]),
-    new webpack.DefinePlugin({
-        'process.env.NODE_ENV': '"production"'
-    }),
     new webpack.ContextReplacementPlugin(/moment[\\\/]locale$/, /^\.\/(en)$/),
-    new purify({
+    new PurgecssPlugin({
-        basePath: __dirname,
+      paths: glob.sync([
-        paths: [
+        path.join(__dirname, 'app/index.html'),
-            "app/index.html",
+        path.join(__dirname, 'app/js/*.js')
-            "app/js/*.js"
+      ])
-        ]
     })
   ]
 }

buildscripts/cross-compile.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Enable tracing if set.
+[ -n "$BASH_XTRACEFD" ] && set -ex
+
+function _init() {
+    ## All binaries are static make sure to disable CGO.
+    export CGO_ENABLED=0
+
+    ## List of architectures and OS to test coss compilation.
+    SUPPORTED_OSARCH="linux/ppc64le linux/arm64 linux/s390x darwin/amd64 freebsd/amd64"
+}
+
+function _build_and_sign() {
+    local osarch=$1
+    IFS=/ read -r -a arr <<<"$osarch"
+    os="${arr[0]}"
+    arch="${arr[1]}"
+    package=$(go list -f '{{.ImportPath}}')
+    printf -- "--> %15s:%s\n" "${osarch}" "${package}"
+
+    # Go build to build the binary.
+    export GOOS=$os
+    export GOARCH=$arch
+    go build -tags kqueue -o /dev/null
+}
+
+function main() {
+    echo "Testing builds for OS/Arch: ${SUPPORTED_OSARCH}"
+    for each_osarch in ${SUPPORTED_OSARCH}; do
+        _build_and_sign "${each_osarch}"
+    done
+}
+
+_init && main "$@"

buildscripts/gateway-tests.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+#
+# Minio Cloud Storage, (C) 2019 Minio, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+set -E
+set -o pipefail
+
+export SERVER_ENDPOINT=127.0.0.1:24240
+export ENABLE_HTTPS=0
+export ACCESS_KEY=minio
+export SECRET_KEY=minio123
+export MINIO_ACCESS_KEY=minio
+export MINIO_SECRET_KEY=minio123
+export AWS_ACCESS_KEY_ID=minio
+export AWS_SECRET_ACCESS_KEY=minio123
+
+trap "cat server.log;cat gateway.log" SIGHUP SIGINT SIGTERM
+
+./minio --quiet --json server data --address 127.0.0.1:24242 > server.log &
+sleep 3
+./minio --quiet --json gateway s3 http://127.0.0.1:24242 --address 127.0.0.1:24240 > gateway.log &
+sleep 3
+
+mkdir -p /mint
+git clone https://github.com/minio/mint /mint
+cd /mint
+
+export MINT_ROOT_DIR=${MINT_ROOT_DIR:-/mint}
+export MINT_RUN_CORE_DIR="$MINT_ROOT_DIR/run/core"
+export MINT_RUN_SECURITY_DIR="$MINT_ROOT_DIR/run/security"
+export WGET="wget --quiet --no-check-certificate"
+
+go get github.com/go-ini/ini
+
+./create-data-files.sh
+./preinstall.sh
+
+# install mint app packages
+for pkg in "build"/*/install.sh; do
+    echo "Running $pkg"
+    $pkg
+done
+
+./postinstall.sh
+
+/mint/entrypoint.sh || cat server.log gateway.log fail

buildscripts/go-coverage.sh

@@ -4,7 +4,7 @@ set -e
 echo "" > coverage.txt
 
 for d in $(go list ./... | grep -v browser); do
-    go test -v -coverprofile=profile.out -covermode=atomic "$d"
+    CGO_ENABLED=0 go test -v -coverprofile=profile.out -covermode=atomic "$d"
     if [ -f profile.out ]; then
         cat profile.out >> coverage.txt
         rm profile.out

buildscripts/verify-build.sh

@@ -68,6 +68,36 @@ function start_minio_erasure_sets()
     echo "$minio_pid"
 }
 
+function start_minio_dist_erasure_sets_ipv6()
+{
+    declare -a minio_pids
+    export MINIO_ACCESS_KEY=$ACCESS_KEY
+    export MINIO_SECRET_KEY=$SECRET_KEY
+    "${MINIO[@]}" server --address="[::1]:9000" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9000.log" 2>&1 &
+    minio_pids[0]=$!
+    "${MINIO[@]}" server --address="[::1]:9001" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9001.log" 2>&1 &
+    minio_pids[1]=$!
+    "${MINIO[@]}" server --address="[::1]:9002" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9002.log" 2>&1 &
+    minio_pids[2]=$!
+    "${MINIO[@]}" server --address="[::1]:9003" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9003.log" 2>&1 &
+    minio_pids[3]=$!
+    "${MINIO[@]}" server --address="[::1]:9004" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9004.log" 2>&1 &
+    minio_pids[4]=$!
+    "${MINIO[@]}" server --address="[::1]:9005" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9005.log" 2>&1 &
+    minio_pids[5]=$!
+    "${MINIO[@]}" server --address="[::1]:9006" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9006.log" 2>&1 &
+    minio_pids[6]=$!
+    "${MINIO[@]}" server --address="[::1]:9007" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9007.log" 2>&1 &
+    minio_pids[7]=$!
+    "${MINIO[@]}" server --address="[::1]:9008" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9008.log" 2>&1 &
+    minio_pids[8]=$!
+    "${MINIO[@]}" server --address="[::1]:9009" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9009.log" 2>&1 &
+    minio_pids[9]=$!
+
+    sleep 35
+    echo "${minio_pids[@]}"
+}
+
 function start_minio_dist_erasure_sets()
 {
     declare -a minio_pids
@@ -161,6 +191,34 @@ function run_test_erasure_sets() {
     return "$rv"
 }
 
+function run_test_dist_erasure_sets_ipv6()
+{
+    minio_pids=( $(start_minio_dist_erasure_sets_ipv6) )
+
+    export SERVER_ENDPOINT="[::1]:9000"
+
+    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+    rv=$?
+
+    for pid in "${minio_pids[@]}"; do
+        kill "$pid"
+    done
+    sleep 3
+
+    if [ "$rv" -ne 0 ]; then
+        for i in $(seq 0 9); do
+            echo "server$i log:"
+            cat "$WORK_DIR/dist-minio-v6-900$i.log"
+        done
+    fi
+
+    for i in $(seq 0 9); do
+        rm -f "$WORK_DIR/dist-minio-v6-900$i.log"
+    done
+
+    return "$rv"
+}
+
 function run_test_dist_erasure_sets()
 {
     minio_pids=( $(start_minio_dist_erasure_sets) )
@@ -237,6 +295,7 @@ function run_test_gateway_s3()
 {
     minio_pid="$(start_minio_gateway_s3)"
 
+    export SERVER_ENDPOINT="127.0.0.1:9000"
     export ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
     export SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
@@ -279,6 +338,7 @@ function __init__()
         exit 1
     fi
 
+    sed -i 's|-sS|-sSg|g' "$FUNCTIONAL_TESTS"
     chmod a+x "$FUNCTIONAL_TESTS"
 }
 
@@ -319,6 +379,13 @@ function main()
         exit 1
     fi
 
+    echo "Testing in Distributed Erasure setup as sets with ipv6"
+    if ! run_test_dist_erasure_sets_ipv6; then
+        echo "FAILED"
+        rm -fr "$WORK_DIR"
+        exit 1
+    fi
+
     echo "Testing in Gateway S3 setup"
     if ! run_test_gateway_s3; then
         echo "FAILED"

cmd/admin-handlers.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017, 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,30 +17,33 @@
 package cmd
 
 import (
-	"archive/zip"
 	"bytes"
 	"context"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io"
 	"net/http"
 	"os"
+	"sort"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
+
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/cpu"
+	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/mem"
+	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/quick"
-	"github.com/tidwall/gjson"
-	"github.com/tidwall/sjson"
 )
 
 const (
@@ -94,6 +97,11 @@ func (a adminAPIHandlers) VersionHandler(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServiceStatus")
 
+	if globalNotificationSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
 	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
@@ -106,13 +114,8 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
 		CommitID: CommitID,
 	}
 
-	// Fetch uptimes from all peers. This may fail to due to lack
+	// Fetch uptimes from all peers and pick the latest.
-	// of read-quorum availability.
+	uptime := getPeerUptimes(globalNotificationSys.ServerInfo(ctx))
-	uptime, err := getPeerUptimes(globalAdminPeers)
-	if err != nil {
-		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
-		return
-	}
 
 	// Create API response
 	serverStatus := madmin.ServiceStatus{
@@ -126,6 +129,7 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
 		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
+
 	// Reply with storage information (across nodes in a
 	// distributed setup) as json.
 	writeSuccessResponseJSON(w, jsonBytes)
@@ -139,6 +143,11 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
 func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServiceStopNRestart")
 
+	if globalNotificationSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
 	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
@@ -148,7 +157,6 @@ func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *h
 	var sa madmin.ServiceAction
 	err := json.NewDecoder(r.Body).Decode(&sa)
 	if err != nil {
-		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(w, ErrRequestBodyParse, r.URL)
 		return
 	}
@@ -168,7 +176,15 @@ func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *h
 	// Reply to the client before restarting minio server.
 	writeSuccessResponseHeadersOnly(w)
 
-	sendServiceCmd(globalAdminPeers, serviceSig)
+	// Notify all other Minio peers signal service.
+	for _, nerr := range globalNotificationSys.SignalService(serviceSig) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+
+	globalServiceSignalCh <- serviceSig
 }
 
 // ServerProperties holds some server information such as, version, region
@@ -232,6 +248,12 @@ type ServerInfo struct {
 func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServerInfo")
 
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
 	// Authenticate request
 
 	// Setting the region as empty so as the mc server info command is irrespective to the region.
@@ -241,39 +263,248 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	// Web service response
+	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
-	reply := make([]ServerInfo, len(globalAdminPeers))
-
-	var wg sync.WaitGroup
-
-	// Gather server information for all nodes
-	for i, p := range globalAdminPeers {
-		wg.Add(1)
-
-		// Gather information from a peer in a goroutine
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-
-			// Initialize server info at index
-			reply[idx] = ServerInfo{Addr: peer.addr}
-
-			serverInfoData, err := peer.cmdRunner.ServerInfo()
 	if err != nil {
-				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", peer.addr)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
-				ctx := logger.SetReqInfo(ctx, reqInfo)
-				logger.LogIf(ctx, err)
-				reply[idx].Error = err.Error()
 		return
 	}
 
-			reply[idx].Data = &serverInfoData
+	serverInfo := globalNotificationSys.ServerInfo(ctx)
-		}(i, p)
+	// Once we have received all the ServerInfo from peers
-	}
+	// add the local peer server info as well.
-
+	serverInfo = append(serverInfo, ServerInfo{
-	wg.Wait()
+		Addr: thisAddr.String(),
+		Data: &ServerInfoData{
+			StorageInfo: objectAPI.StorageInfo(ctx),
+			ConnStats:   globalConnStats.toServerConnStats(),
+			HTTPStats:   globalHTTPStats.toServerHTTPStats(),
+			Properties: ServerProperties{
+				Uptime:   UTCNow().Sub(globalBootTime),
+				Version:  Version,
+				CommitID: CommitID,
+				SQSARN:   globalNotificationSys.GetARNList(),
+				Region:   globalServerConfig.GetRegion(),
+			},
+		},
+	})
 
 	// Marshal API response
-	jsonBytes, err := json.Marshal(reply)
+	jsonBytes, err := json.Marshal(serverInfo)
+	if err != nil {
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
+		return
+	}
+
+	// Reply with storage information (across nodes in a
+	// distributed setup) as json.
+	writeSuccessResponseJSON(w, jsonBytes)
+}
+
+// ServerDrivesPerfInfo holds information about address, performance
+// of all drives on one server. It also reports any errors if encountered
+// while trying to reach this server.
+type ServerDrivesPerfInfo struct {
+	Addr  string             `json:"addr"`
+	Error string             `json:"error,omitempty"`
+	Perf  []disk.Performance `json:"perf"`
+}
+
+// ServerCPULoadInfo holds informantion about cpu utilization
+// of one minio node. It also reports any errors if encountered
+// while trying to reach this server.
+type ServerCPULoadInfo struct {
+	Addr         string     `json:"addr"`
+	Error        string     `json:"error,omitempty"`
+	Load         []cpu.Load `json:"load"`
+	HistoricLoad []cpu.Load `json:"historicLoad"`
+}
+
+// ServerMemUsageInfo holds informantion about memory utilization
+// of one minio node. It also reports any errors if encountered
+// while trying to reach this server.
+type ServerMemUsageInfo struct {
+	Addr          string      `json:"addr"`
+	Error         string      `json:"error,omitempty"`
+	Usage         []mem.Usage `json:"usage"`
+	HistoricUsage []mem.Usage `json:"historicUsage"`
+}
+
+// PerfInfoHandler - GET /minio/admin/v1/performance?perfType={perfType}
+// ----------
+// Get all performance information based on input type
+// Supported types = drive
+func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PerfInfo")
+
+	// Get object layer instance.
+	objLayer := newObjectLayerFn()
+	if objLayer == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Authenticate request
+	// Setting the region as empty so as the mc server info command is irrespective to the region.
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	perfType := vars["perfType"]
+
+	if perfType == "drive" {
+		info := objLayer.StorageInfo(ctx)
+		if !(info.Backend.Type == BackendFS || info.Backend.Type == BackendErasure) {
+
+			writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+			return
+		}
+		// Get drive performance details from local server's drive(s)
+		dp := localEndpointsDrivePerf(globalEndpoints)
+
+		// Notify all other Minio peers to report drive performance numbers
+		dps := globalNotificationSys.DrivePerfInfo()
+		dps = append(dps, dp)
+
+		// Marshal API response
+		jsonBytes, err := json.Marshal(dps)
+		if err != nil {
+			writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
+			return
+		}
+
+		// Reply with performance information (across nodes in a
+		// distributed setup) as json.
+		writeSuccessResponseJSON(w, jsonBytes)
+	} else if perfType == "cpu" {
+		// Get CPU load details from local server's cpu(s)
+		cpu := localEndpointsCPULoad(globalEndpoints)
+		// Notify all other Minio peers to report cpu load numbers
+		cpus := globalNotificationSys.CPULoadInfo()
+		cpus = append(cpus, cpu)
+
+		// Marshal API response
+		jsonBytes, err := json.Marshal(cpus)
+		if err != nil {
+			writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
+			return
+		}
+
+		// Reply with cpu load information (across nodes in a
+		// distributed setup) as json.
+		writeSuccessResponseJSON(w, jsonBytes)
+	} else if perfType == "mem" {
+		// Get mem usage details from local server(s)
+		m := localEndpointsMemUsage(globalEndpoints)
+		// Notify all other Minio peers to report mem usage numbers
+		mems := globalNotificationSys.MemUsageInfo()
+		mems = append(mems, m)
+
+		// Marshal API response
+		jsonBytes, err := json.Marshal(mems)
+		if err != nil {
+			writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
+			return
+		}
+
+		// Reply with mem usage information (across nodes in a
+		// distributed setup) as json.
+		writeSuccessResponseJSON(w, jsonBytes)
+	} else {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+	}
+	return
+}
+
+func newLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntry {
+	entry := &madmin.LockEntry{Timestamp: l.Timestamp, Resource: resource, ServerList: []string{server}, Owner: l.Node, Source: l.Source, ID: l.UID}
+	if l.Writer {
+		entry.Type = "Write"
+	} else {
+		entry.Type = "Read"
+	}
+	return entry
+}
+
+func topLockEntries(peerLocks []*PeerLocks) madmin.LockEntries {
+	const listCount int = 10
+	entryMap := make(map[string]*madmin.LockEntry)
+	for _, peerLock := range peerLocks {
+		if peerLock == nil {
+			continue
+		}
+		for k, v := range peerLock.Locks {
+			for _, lockReqInfo := range v {
+				if val, ok := entryMap[lockReqInfo.UID]; ok {
+					val.ServerList = append(val.ServerList, peerLock.Addr)
+				} else {
+					entryMap[lockReqInfo.UID] = newLockEntry(lockReqInfo, k, peerLock.Addr)
+				}
+			}
+		}
+	}
+	var lockEntries = make(madmin.LockEntries, 0)
+	for _, v := range entryMap {
+		lockEntries = append(lockEntries, *v)
+	}
+	sort.Sort(lockEntries)
+	if len(lockEntries) > listCount {
+		lockEntries = lockEntries[:listCount]
+	}
+	return lockEntries
+}
+
+// PeerLocks holds server information result of one node
+type PeerLocks struct {
+	Addr  string
+	Locks GetLocksResp
+}
+
+// TopLocksHandler Get list of locks in use
+func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "TopLocks")
+
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Method only allowed in Distributed XL mode.
+	if globalIsDistXL == false {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+		return
+	}
+
+	// Authenticate request
+	// Setting the region as empty so as the mc server info command is irrespective to the region.
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
+	if err != nil {
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
+		return
+	}
+
+	peerLocks := globalNotificationSys.GetLocks(ctx)
+	// Once we have received all the locks currently used from peers
+	// add the local peer locks list as well.
+	localLocks := globalLockServer.ll.DupLockMap()
+	peerLocks = append(peerLocks, &PeerLocks{
+		Addr:  thisAddr.String(),
+		Locks: localLocks,
+	})
+
+	topLocks := topLockEntries(peerLocks)
+
+	// Marshal API response
+	jsonBytes, err := json.Marshal(topLocks)
 	if err != nil {
 		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
@@ -298,6 +529,11 @@ type StartProfilingResult struct {
 func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "StartProfiling")
 
+	if globalNotificationSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
 	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
@@ -307,31 +543,53 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
 	vars := mux.Vars(r)
 	profiler := vars["profilerType"]
 
-	startProfilingResult := make([]StartProfilingResult, len(globalAdminPeers))
+	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
-
+	if err != nil {
-	// Call StartProfiling function on all nodes and save results
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
-	wg := sync.WaitGroup{}
-	for i, peer := range globalAdminPeers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			result := StartProfilingResult{NodeName: peer.addr}
-			if err := peer.cmdRunner.StartProfiling(profiler); err != nil {
-				result.Error = err.Error()
 		return
 	}
-			result.Success = true
+
-			startProfilingResult[idx] = result
+	// Start profiling on remote servers.
-		}(i, peer)
+	hostErrs := globalNotificationSys.StartProfiling(profiler)
+
+	// Start profiling locally as well.
+	{
+		if globalProfiler != nil {
+			globalProfiler.Stop()
+		}
+		prof, err := startProfiler(profiler, "")
+		if err != nil {
+			hostErrs = append(hostErrs, NotificationPeerErr{
+				Host: *thisAddr,
+				Err:  err,
+			})
+		} else {
+			globalProfiler = prof
+			hostErrs = append(hostErrs, NotificationPeerErr{
+				Host: *thisAddr,
+			})
+		}
+	}
+
+	var startProfilingResult []StartProfilingResult
+
+	for _, nerr := range hostErrs {
+		result := StartProfilingResult{NodeName: nerr.Host.String()}
+		if nerr.Err != nil {
+			result.Error = nerr.Err.Error()
+		} else {
+			result.Success = true
+		}
+		startProfilingResult = append(startProfilingResult, result)
 	}
-	wg.Wait()
 
 	// Create JSON result and send it to the client
 	startProfilingResultInBytes, err := json.Marshal(startProfilingResult)
 	if err != nil {
-		writeCustomErrorResponseJSON(w, http.StatusInternalServerError, err.Error(), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
+
 	writeSuccessResponseJSON(w, []byte(startProfilingResultInBytes))
 }
 
@@ -359,51 +617,18 @@ func (f dummyFileInfo) Sys() interface{}   { return f.sys }
 func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DownloadProfiling")
 
+	if globalNotificationSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
 	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
 	}
 
-	profilingDataFound := false
+	if !globalNotificationSys.DownloadProfilingData(ctx, w) {
-
-	// Initialize a zip writer which will provide a zipped content
-	// of profiling data of all nodes
-	zipWriter := zip.NewWriter(w)
-	defer zipWriter.Close()
-
-	for i, peer := range globalAdminPeers {
-		// Get profiling data from a node
-		data, err := peer.cmdRunner.DownloadProfilingData()
-		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to download profiling data from node `%s`, reason: %s", peer.addr, err.Error()))
-			continue
-		}
-
-		profilingDataFound = true
-
-		// Send profiling data to zip as file
-		header, err := zip.FileInfoHeader(dummyFileInfo{
-			name:    fmt.Sprintf("profiling-%d", i),
-			size:    int64(len(data)),
-			mode:    0600,
-			modTime: time.Now().UTC(),
-			isDir:   false,
-			sys:     nil,
-		})
-		if err != nil {
-			continue
-		}
-		writer, err := zipWriter.CreateHeader(header)
-		if err != nil {
-			continue
-		}
-		if _, err = io.Copy(writer, bytes.NewBuffer(data)); err != nil {
-			return
-		}
-	}
-
-	if !profilingDataFound {
 		writeErrorResponseJSON(w, ErrAdminProfilerNotEnabled, r.URL)
 		return
 	}
@@ -833,6 +1058,11 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetUserStatus")
 
+	if globalNotificationSys == nil || globalIAMSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
@@ -867,6 +1097,14 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
+
+	// Notify all other Minio peers to reload users
+	for _, nerr := range globalNotificationSys.LoadUsers() {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
 }
 
 // AddUser - PUT /minio/admin/v1/add-user?accessKey=<access_key>
@@ -875,7 +1113,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -927,6 +1165,14 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
+
+	// Notify all other Minio peers to reload users
+	for _, nerr := range globalNotificationSys.LoadUsers() {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
 }
 
 // ListCannedPolicies - GET /minio/admin/v1/list-canned-policies
@@ -935,7 +1181,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalIAMSys == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -967,7 +1213,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalIAMSys == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -992,6 +1238,14 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
+
+	// Notify all other Minio peers to reload users
+	for _, nerr := range globalNotificationSys.LoadUsers() {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
 }
 
 // AddCannedPolicy - PUT /minio/admin/v1/add-canned-policy?name=<policy_name>
@@ -1000,7 +1254,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalIAMSys == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -1049,6 +1303,14 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
+
+	// Notify all other Minio peers to reload users
+	for _, nerr := range globalNotificationSys.LoadUsers() {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
 }
 
 // SetUserPolicy - PUT /minio/admin/v1/set-user-policy?accessKey=<access_key>&name=<policy_name>
@@ -1057,7 +1319,7 @@ func (a adminAPIHandlers) SetUserPolicy(w http.ResponseWriter, r *http.Request)
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalIAMSys == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -1088,6 +1350,14 @@ func (a adminAPIHandlers) SetUserPolicy(w http.ResponseWriter, r *http.Request)
 	if err := globalIAMSys.SetUserPolicy(accessKey, policyName); err != nil {
 		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 	}
+
+	// Notify all other Minio peers to reload users
+	for _, nerr := range globalNotificationSys.LoadUsers() {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
 }
 
 // SetConfigHandler - PUT /minio/admin/v1/config
@@ -1096,7 +1366,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -1313,7 +1583,7 @@ func (a adminAPIHandlers) UpdateAdminCredentialsHandler(w http.ResponseWriter,
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -1376,10 +1646,10 @@ func (a adminAPIHandlers) UpdateAdminCredentialsHandler(w http.ResponseWriter,
 	}
 
 	// Notify all other Minio peers to update credentials
-	for host, err := range globalNotificationSys.LoadCredentials() {
+	for _, nerr := range globalNotificationSys.LoadCredentials() {
-		if err != nil {
+		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, err)
+			logger.LogIf(ctx, nerr.Err)
 		}
 	}
 

cmd/admin-handlers_test.go

@@ -119,11 +119,11 @@ var (
         "broker": "",
         "topic": "",
         "qos": 0,
-        "clientId": "",
         "username": "",
         "password": "",
         "reconnectInterval": 0,
-        "keepAliveInterval": 0
+	"keepAliveInterval": 0,
+	"queueDir": ""
       }
     },
     "mysql": {
@@ -277,7 +277,7 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 
 	// Setup admin mgmt REST API handlers.
 	adminRouter := mux.NewRouter()
-	registerAdminRouter(adminRouter)
+	registerAdminRouter(adminRouter, true, true)
 
 	return &adminXLTestBed{
 		xlDirs:   xlDirs,
@@ -533,7 +533,6 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	// single node setup, this degenerates to a simple function
 	// call under the hood.
 	globalMinioAddr = "127.0.0.1:9000"
-	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	var wg sync.WaitGroup
 
@@ -607,7 +606,6 @@ func TestServiceSetCreds(t *testing.T) {
 	// single node setup, this degenerates to a simple function
 	// call under the hood.
 	globalMinioAddr = "127.0.0.1:9000"
-	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	credentials := globalServerConfig.GetCredential()
 
@@ -706,7 +704,6 @@ func TestGetConfigHandler(t *testing.T) {
 
 	// Initialize admin peers to make admin RPC calls.
 	globalMinioAddr = "127.0.0.1:9000"
-	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	// Prepare query params for get-config mgmt REST API.
 	queryVal := url.Values{}
@@ -735,7 +732,6 @@ func TestSetConfigHandler(t *testing.T) {
 
 	// Initialize admin peers to make admin RPC calls.
 	globalMinioAddr = "127.0.0.1:9000"
-	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	// Prepare query params for set-config mgmt REST API.
 	queryVal := url.Values{}
@@ -807,7 +803,6 @@ func TestAdminServerInfo(t *testing.T) {
 
 	// Initialize admin peers to make admin RPC calls.
 	globalMinioAddr = "127.0.0.1:9000"
-	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	// Prepare query params for set-config mgmt REST API.
 	queryVal := url.Values{}
@@ -965,81 +960,3 @@ func collectHealResults(t *testing.T, adminTestBed *adminXLTestBed, bucket,
 
 	return res
 }
-
-func TestHealStartNStatusHandler(t *testing.T) {
-	adminTestBed, err := prepareAdminXLTestBed()
-	if err != nil {
-		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
-	}
-	defer adminTestBed.TearDown()
-
-	// gen. test data
-	adminTestBed.GenerateHealTestData(t)
-	defer adminTestBed.CleanupHealTestData(t)
-
-	// Prepare heal-start request to send to the server.
-	healOpts := madmin.HealOpts{
-		Recursive: true,
-		DryRun:    false,
-	}
-	bucketName, objName := "mybucket", "myobject-0"
-	var hss madmin.HealStartSuccess
-
-	{
-		req := mkHealStartReq(t, bucketName, objName, healOpts)
-		rec := httptest.NewRecorder()
-		adminTestBed.router.ServeHTTP(rec, req)
-		if http.StatusOK != rec.Code {
-			t.Errorf("Unexpected status code - got %d but expected %d",
-				rec.Code, http.StatusOK)
-		}
-
-		err = json.Unmarshal(rec.Body.Bytes(), &hss)
-		if err != nil {
-			t.Fatal("unable to unmarshal response")
-		}
-
-		if hss.ClientToken == "" {
-			t.Errorf("unexpected result")
-		}
-	}
-
-	{
-		// test with an invalid client token
-		req := mkHealStatusReq(t, bucketName, objName, hss.ClientToken+hss.ClientToken)
-		rec := httptest.NewRecorder()
-		adminTestBed.router.ServeHTTP(rec, req)
-		if rec.Code != http.StatusBadRequest {
-			t.Errorf("Unexpected status code")
-		}
-	}
-
-	{
-		// fetch heal status
-		results := collectHealResults(t, adminTestBed, bucketName,
-			objName, hss.ClientToken, 5)
-
-		// check if we got back an expected record
-		foundIt := false
-		for _, item := range results.Items {
-			if item.Type == madmin.HealItemObject &&
-				item.Bucket == bucketName && item.Object == objName {
-				foundIt = true
-			}
-		}
-		if !foundIt {
-			t.Error("did not find expected heal record in heal results")
-		}
-
-		// check that the heal settings in the results is the
-		// same as what we started the heal seq. with.
-		if results.HealSettings != healOpts {
-			t.Errorf("unexpected heal settings: %v",
-				results.HealSettings)
-		}
-
-		if results.Summary == healStoppedStatus {
-			t.Errorf("heal sequence stopped unexpectedly")
-		}
-	}
-}

cmd/admin-heal-ops.go

@@ -133,7 +133,7 @@ func (ahs *allHealState) periodicHealSeqsClean() {
 				}
 			}
 			ahs.Unlock()
-		case <-globalServiceDoneCh:
+		case <-GlobalServiceDoneCh:
 			// server could be restarting - need
 			// to exit immediately
 			return
@@ -596,7 +596,7 @@ func (h *healSequence) healConfig() error {
 				return errHealStopSignalled
 			}
 			o := objectInfos.Objects[index]
-			res, herr := objectAPI.HealObject(h.ctx, o.Bucket, o.Name, h.settings.DryRun)
+			res, herr := objectAPI.HealObject(h.ctx, o.Bucket, o.Name, h.settings.DryRun, h.settings.Remove)
 			// Object might have been deleted, by the time heal
 			// was attempted we ignore this file an move on.
 			if isErrObjectNotFound(herr) {
@@ -641,7 +641,12 @@ func (h *healSequence) healDiskFormat() error {
 	// Healing succeeded notify the peers to reload format and re-initialize disks.
 	// We will not notify peers only if healing succeeded.
 	if err == nil {
-		peersReInitFormat(globalAdminPeers, h.settings.DryRun)
+		for _, nerr := range globalNotificationSys.ReloadFormat(h.settings.DryRun) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(h.ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(h.ctx, nerr.Err)
+			}
+		}
 	}
 
 	// Push format heal result
@@ -687,7 +692,7 @@ func (h *healSequence) healBucket(bucket string) error {
 		return errServerNotInitialized
 	}
 
-	results, err := objectAPI.HealBucket(h.ctx, bucket, h.settings.DryRun)
+	results, err := objectAPI.HealBucket(h.ctx, bucket, h.settings.DryRun, h.settings.Remove)
 	// push any available results before checking for error
 	for _, result := range results {
 		if perr := h.pushHealResultItem(result); perr != nil {
@@ -714,7 +719,7 @@ func (h *healSequence) healBucket(bucket string) error {
 		return nil
 	}
 
-	entries := runtime.NumCPU() * globalEndpoints.Nodes()
+	entries := runtime.NumCPU()
 
 	marker := ""
 	isTruncated := true
@@ -770,7 +775,7 @@ func (h *healSequence) healObject(bucket, object string) error {
 		return errServerNotInitialized
 	}
 
-	hri, err := objectAPI.HealObject(h.ctx, bucket, object, h.settings.DryRun)
+	hri, err := objectAPI.HealObject(h.ctx, bucket, object, h.settings.DryRun, h.settings.Remove)
 	if isErrObjectNotFound(err) {
 		return nil
 	}

cmd/admin-router.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017, 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -31,7 +31,7 @@ type adminAPIHandlers struct {
 }
 
 // registerAdminRouter - Add handler functions for each service REST API routes.
-func registerAdminRouter(router *mux.Router) {
+func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool) {
 
 	adminAPI := adminAPIHandlers{}
 	// Admin router
@@ -60,7 +60,12 @@ func registerAdminRouter(router *mux.Router) {
 		adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
 		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
 		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+
+		/// Health operations
+
 	}
+	// Performance command - return performance details based on input type
+	adminV1Router.Methods(http.MethodGet).Path("/performance").HandlerFunc(httpTraceAll(adminAPI.PerfInfoHandler)).Queries("perfType", "{perfType:.*}")
 
 	// Profiling operations
 	adminV1Router.Methods(http.MethodPost).Path("/profiling/start").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler)).
@@ -68,7 +73,7 @@ func registerAdminRouter(router *mux.Router) {
 	adminV1Router.Methods(http.MethodGet).Path("/profiling/download").HandlerFunc(httpTraceAll(adminAPI.DownloadProfilingHandler))
 
 	/// Config operations
-
+	if enableConfigOps {
 		// Update credentials
 		adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(httpTraceHdrs(adminAPI.UpdateAdminCredentialsHandler))
 		// Get config
@@ -80,11 +85,14 @@ func registerAdminRouter(router *mux.Router) {
 		adminV1Router.Methods(http.MethodGet).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKeysHandler))
 		// Set config keys/values
 		adminV1Router.Methods(http.MethodPut).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKeysHandler))
+	}
 
+	if enableIAMOps {
 		// -- IAM APIs --
 
 		// Add policy IAM
-	adminV1Router.Methods(http.MethodPut).Path("/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
+		adminV1Router.Methods(http.MethodPut).Path("/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name",
+			"{name:.*}")
 
 		// Add user IAM
 		adminV1Router.Methods(http.MethodPut).Path("/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
@@ -104,7 +112,12 @@ func registerAdminRouter(router *mux.Router) {
 
 		// List policies
 		adminV1Router.Methods(http.MethodGet).Path("/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
+	}
 
-	// If none of the routes match.
+	// -- Top APIs --
-	adminV1Router.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandler))
+	// Top locks
+	adminV1Router.Methods(http.MethodGet).Path("/top/locks").HandlerFunc(httpTraceHdrs(adminAPI.TopLocksHandler))
+
+	// If none of the routes match, return error.
+	adminV1Router.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandlerJSON))
 }

cmd/admin-rpc-client.go

@@ -1,295 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2014, 2015, 2016, 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"crypto/tls"
-	"fmt"
-	"net"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/minio/minio/cmd/logger"
-	xnet "github.com/minio/minio/pkg/net"
-)
-
-var errUnsupportedSignal = fmt.Errorf("unsupported signal: only restart and stop signals are supported")
-
-// AdminRPCClient - admin RPC client talks to admin RPC server.
-type AdminRPCClient struct {
-	*RPCClient
-}
-
-// SignalService - calls SignalService RPC.
-func (rpcClient *AdminRPCClient) SignalService(signal serviceSignal) (err error) {
-	args := SignalServiceArgs{Sig: signal}
-	reply := VoidReply{}
-
-	return rpcClient.Call(adminServiceName+".SignalService", &args, &reply)
-}
-
-// ReInitFormat - re-initialize disk format, remotely.
-func (rpcClient *AdminRPCClient) ReInitFormat(dryRun bool) error {
-	args := ReInitFormatArgs{DryRun: dryRun}
-	reply := VoidReply{}
-
-	return rpcClient.Call(adminServiceName+".ReInitFormat", &args, &reply)
-}
-
-// ServerInfo - returns the server info of the server to which the RPC call is made.
-func (rpcClient *AdminRPCClient) ServerInfo() (sid ServerInfoData, err error) {
-	err = rpcClient.Call(adminServiceName+".ServerInfo", &AuthArgs{}, &sid)
-	return sid, err
-}
-
-// GetConfig - returns config.json of the remote server.
-func (rpcClient *AdminRPCClient) GetConfig() ([]byte, error) {
-	args := AuthArgs{}
-	var reply []byte
-
-	err := rpcClient.Call(adminServiceName+".GetConfig", &args, &reply)
-	return reply, err
-}
-
-// StartProfiling - starts profiling in the remote server.
-func (rpcClient *AdminRPCClient) StartProfiling(profiler string) error {
-	args := StartProfilingArgs{Profiler: profiler}
-	reply := VoidReply{}
-	return rpcClient.Call(adminServiceName+".StartProfiling", &args, &reply)
-}
-
-// DownloadProfilingData - returns profiling data of the remote server.
-func (rpcClient *AdminRPCClient) DownloadProfilingData() ([]byte, error) {
-	args := AuthArgs{}
-	var reply []byte
-
-	err := rpcClient.Call(adminServiceName+".DownloadProfilingData", &args, &reply)
-	return reply, err
-}
-
-// NewAdminRPCClient - returns new admin RPC client.
-func NewAdminRPCClient(host *xnet.Host) (*AdminRPCClient, error) {
-	scheme := "http"
-	if globalIsSSL {
-		scheme = "https"
-	}
-
-	serviceURL := &xnet.URL{
-		Scheme: scheme,
-		Host:   host.String(),
-		Path:   adminServicePath,
-	}
-
-	var tlsConfig *tls.Config
-	if globalIsSSL {
-		tlsConfig = &tls.Config{
-			ServerName: host.Name,
-			RootCAs:    globalRootCAs,
-		}
-	}
-
-	rpcClient, err := NewRPCClient(
-		RPCClientArgs{
-			NewAuthTokenFunc: newAuthToken,
-			RPCVersion:       globalRPCAPIVersion,
-			ServiceName:      adminServiceName,
-			ServiceURL:       serviceURL,
-			TLSConfig:        tlsConfig,
-		},
-	)
-	if err != nil {
-		return nil, err
-	}
-
-	return &AdminRPCClient{rpcClient}, nil
-}
-
-// adminCmdRunner - abstracts local and remote execution of admin
-// commands like service stop and service restart.
-type adminCmdRunner interface {
-	SignalService(s serviceSignal) error
-	ReInitFormat(dryRun bool) error
-	ServerInfo() (ServerInfoData, error)
-	GetConfig() ([]byte, error)
-	StartProfiling(string) error
-	DownloadProfilingData() ([]byte, error)
-}
-
-// adminPeer - represents an entity that implements admin API RPCs.
-type adminPeer struct {
-	addr      string
-	cmdRunner adminCmdRunner
-	isLocal   bool
-}
-
-// type alias for a collection of adminPeer.
-type adminPeers []adminPeer
-
-// makeAdminPeers - helper function to construct a collection of adminPeer.
-func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
-	localAddr := GetLocalPeer(endpoints)
-	if strings.HasPrefix(localAddr, "127.0.0.1:") {
-		// Use first IPv4 instead of loopback address.
-		localAddr = net.JoinHostPort(sortIPs(localIP4.ToSlice())[0], globalMinioPort)
-	}
-	adminPeerList = append(adminPeerList, adminPeer{
-		addr:      localAddr,
-		cmdRunner: localAdminClient{},
-		isLocal:   true,
-	})
-
-	for _, hostStr := range GetRemotePeers(endpoints) {
-		host, err := xnet.ParseHost(hostStr)
-		logger.FatalIf(err, "Unable to parse Admin RPC Host")
-		rpcClient, err := NewAdminRPCClient(host)
-		logger.FatalIf(err, "Unable to initialize Admin RPC Client")
-		adminPeerList = append(adminPeerList, adminPeer{
-			addr:      hostStr,
-			cmdRunner: rpcClient,
-		})
-	}
-
-	return adminPeerList
-}
-
-// peersReInitFormat - reinitialize remote object layers to new format.
-func peersReInitFormat(peers adminPeers, dryRun bool) error {
-	errs := make([]error, len(peers))
-
-	// Send ReInitFormat RPC call to all nodes.
-	// for local adminPeer this is a no-op.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			if !peer.isLocal {
-				errs[idx] = peer.cmdRunner.ReInitFormat(dryRun)
-			}
-		}(i, peer)
-	}
-	wg.Wait()
-	return nil
-}
-
-// Initialize global adminPeer collection.
-func initGlobalAdminPeers(endpoints EndpointList) {
-	globalAdminPeers = makeAdminPeers(endpoints)
-}
-
-// invokeServiceCmd - Invoke Restart/Stop command.
-func invokeServiceCmd(cp adminPeer, cmd serviceSignal) (err error) {
-	switch cmd {
-	case serviceRestart, serviceStop:
-		err = cp.cmdRunner.SignalService(cmd)
-	}
-	return err
-}
-
-// sendServiceCmd - Invoke Restart command on remote peers
-// adminPeer followed by on the local peer.
-func sendServiceCmd(cps adminPeers, cmd serviceSignal) {
-	// Send service command like stop or restart to all remote nodes and finally run on local node.
-	errs := make([]error, len(cps))
-	var wg sync.WaitGroup
-	remotePeers := cps[1:]
-	for i := range remotePeers {
-		wg.Add(1)
-		go func(idx int) {
-			defer wg.Done()
-			// we use idx+1 because remotePeers slice is 1 position shifted w.r.t cps
-			errs[idx+1] = invokeServiceCmd(remotePeers[idx], cmd)
-		}(i)
-	}
-	wg.Wait()
-	errs[0] = invokeServiceCmd(cps[0], cmd)
-}
-
-// uptimeSlice - used to sort uptimes in chronological order.
-type uptimeSlice []struct {
-	err    error
-	uptime time.Duration
-}
-
-func (ts uptimeSlice) Len() int {
-	return len(ts)
-}
-
-func (ts uptimeSlice) Less(i, j int) bool {
-	return ts[i].uptime < ts[j].uptime
-}
-
-func (ts uptimeSlice) Swap(i, j int) {
-	ts[i], ts[j] = ts[j], ts[i]
-}
-
-// getPeerUptimes - returns the uptime since the last time read quorum
-// was established on success. Otherwise returns errXLReadQuorum.
-func getPeerUptimes(peers adminPeers) (time.Duration, error) {
-	// In a single node Erasure or FS backend setup the uptime of
-	// the setup is the uptime of the single minio server
-	// instance.
-	if !globalIsDistXL {
-		return UTCNow().Sub(globalBootTime), nil
-	}
-
-	uptimes := make(uptimeSlice, len(peers))
-
-	// Get up time of all servers.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			serverInfoData, rpcErr := peer.cmdRunner.ServerInfo()
-			uptimes[idx].uptime, uptimes[idx].err = serverInfoData.Properties.Uptime, rpcErr
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Sort uptimes in chronological order.
-	sort.Sort(uptimes)
-
-	// Pick the readQuorum'th uptime in chronological order. i.e,
-	// the time at which read quorum was (re-)established.
-	readQuorum := len(uptimes) / 2
-	validCount := 0
-	latestUptime := time.Duration(0)
-	for _, uptime := range uptimes {
-		if uptime.err != nil {
-			logger.LogIf(context.Background(), uptime.err)
-			continue
-		}
-
-		validCount++
-		if validCount >= readQuorum {
-			latestUptime = uptime.uptime
-			break
-		}
-	}
-
-	// Less than readQuorum "Admin.Uptime" RPC call returned
-	// successfully, so read-quorum unavailable.
-	if validCount < readQuorum {
-		return time.Duration(0), InsufficientReadQuorum{}
-	}
-
-	return latestUptime, nil
-}

cmd/admin-rpc-server.go

@@ -1,103 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"path"
-
-	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/logger"
-	xrpc "github.com/minio/minio/cmd/rpc"
-)
-
-const adminServiceName = "Admin"
-const adminServiceSubPath = "/admin"
-
-var adminServicePath = path.Join(minioReservedBucketPath, adminServiceSubPath)
-
-// adminRPCReceiver - Admin RPC receiver for admin RPC server.
-type adminRPCReceiver struct {
-	local *localAdminClient
-}
-
-// SignalServiceArgs - provides the signal argument to SignalService RPC
-type SignalServiceArgs struct {
-	AuthArgs
-	Sig serviceSignal
-}
-
-// SignalService - Send a restart or stop signal to the service
-func (receiver *adminRPCReceiver) SignalService(args *SignalServiceArgs, reply *VoidReply) error {
-	return receiver.local.SignalService(args.Sig)
-}
-
-// ServerInfo - returns the server info when object layer was initialized on this server.
-func (receiver *adminRPCReceiver) ServerInfo(args *AuthArgs, reply *ServerInfoData) (err error) {
-	*reply, err = receiver.local.ServerInfo()
-	return err
-}
-
-// StartProfilingArgs - holds the RPC argument for StartingProfiling RPC call
-type StartProfilingArgs struct {
-	AuthArgs
-	Profiler string
-}
-
-// StartProfiling - starts profiling of this server
-func (receiver *adminRPCReceiver) StartProfiling(args *StartProfilingArgs, reply *VoidReply) error {
-	return receiver.local.StartProfiling(args.Profiler)
-}
-
-// DownloadProfilingData - stops and returns profiling data of this server
-func (receiver *adminRPCReceiver) DownloadProfilingData(args *AuthArgs, reply *[]byte) (err error) {
-	*reply, err = receiver.local.DownloadProfilingData()
-	return
-}
-
-// GetConfig - returns the config.json of this server.
-func (receiver *adminRPCReceiver) GetConfig(args *AuthArgs, reply *[]byte) (err error) {
-	*reply, err = receiver.local.GetConfig()
-	return err
-}
-
-// ReInitFormatArgs - provides dry-run information to re-initialize format.json
-type ReInitFormatArgs struct {
-	AuthArgs
-	DryRun bool
-}
-
-// ReInitFormat - re-init 'format.json'
-func (receiver *adminRPCReceiver) ReInitFormat(args *ReInitFormatArgs, reply *VoidReply) error {
-	return receiver.local.ReInitFormat(args.DryRun)
-}
-
-// NewAdminRPCServer - returns new admin RPC server.
-func NewAdminRPCServer() (*xrpc.Server, error) {
-	rpcServer := xrpc.NewServer()
-	if err := rpcServer.RegisterName(adminServiceName, &adminRPCReceiver{&localAdminClient{}}); err != nil {
-		return nil, err
-	}
-	return rpcServer, nil
-}
-
-// registerAdminRPCRouter - creates and registers Admin RPC server and its router.
-func registerAdminRPCRouter(router *mux.Router) {
-	rpcServer, err := NewAdminRPCServer()
-	logger.FatalIf(err, "Unable to initialize Lock RPC Server")
-	subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
-	subrouter.Path(adminServiceSubPath).HandlerFunc(httpTraceHdrs(rpcServer.ServeHTTP))
-}

cmd/admin-rpc_test.go

@@ -1,239 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"net/http"
-	"net/http/httptest"
-	"testing"
-	"time"
-
-	xnet "github.com/minio/minio/pkg/net"
-)
-
-///////////////////////////////////////////////////////////////////////////////
-//
-// localAdminClient and AdminRPCClient are adminCmdRunner interface compatible,
-// hence below test functions are available for both clients.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-///////////////////////////////////////////////////////////////////////////////
-//
-// Admin RPC server, adminRPCReceiver and AdminRPCClient are
-// inter-dependent, below test functions are sufficient to test all of them.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-func testAdminCmdRunnerSignalService(t *testing.T, client adminCmdRunner) {
-	tmpGlobalServiceSignalCh := globalServiceSignalCh
-	globalServiceSignalCh = make(chan serviceSignal, 10)
-	defer func() {
-		globalServiceSignalCh = tmpGlobalServiceSignalCh
-	}()
-
-	testCases := []struct {
-		signal    serviceSignal
-		expectErr bool
-	}{
-		{serviceRestart, false},
-		{serviceStop, false},
-		{serviceStatus, true},
-		{serviceSignal(100), true},
-	}
-
-	for i, testCase := range testCases {
-		err := client.SignalService(testCase.signal)
-		expectErr := (err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
-		}
-	}
-}
-
-func testAdminCmdRunnerReInitFormat(t *testing.T, client adminCmdRunner) {
-	tmpGlobalObjectAPI := globalObjectAPI
-	defer func() {
-		globalObjectAPI = tmpGlobalObjectAPI
-	}()
-
-	testCases := []struct {
-		objectAPI ObjectLayer
-		dryRun    bool
-		expectErr bool
-	}{
-		{&DummyObjectLayer{}, true, false},
-		{&DummyObjectLayer{}, false, false},
-		{nil, true, true},
-		{nil, false, true},
-	}
-
-	for i, testCase := range testCases {
-		globalObjectAPI = testCase.objectAPI
-		err := client.ReInitFormat(testCase.dryRun)
-		expectErr := (err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
-		}
-	}
-}
-
-func testAdminCmdRunnerServerInfo(t *testing.T, client adminCmdRunner) {
-	tmpGlobalBootTime := globalBootTime
-	tmpGlobalObjectAPI := globalObjectAPI
-	tmpGlobalConnStats := globalConnStats
-	tmpGlobalHTTPStats := globalHTTPStats
-	tmpGlobalNotificationSys := globalNotificationSys
-	defer func() {
-		globalBootTime = tmpGlobalBootTime
-		globalObjectAPI = tmpGlobalObjectAPI
-		globalConnStats = tmpGlobalConnStats
-		globalHTTPStats = tmpGlobalHTTPStats
-		globalNotificationSys = tmpGlobalNotificationSys
-	}()
-
-	endpoints := new(EndpointList)
-
-	notificationSys := NewNotificationSys(globalServerConfig, *endpoints)
-
-	testCases := []struct {
-		bootTime        time.Time
-		objectAPI       ObjectLayer
-		connStats       *ConnStats
-		httpStats       *HTTPStats
-		notificationSys *NotificationSys
-		expectErr       bool
-	}{
-		{UTCNow(), &DummyObjectLayer{}, newConnStats(), newHTTPStats(), notificationSys, false},
-		{time.Time{}, nil, nil, nil, nil, true},
-		{UTCNow(), nil, nil, nil, nil, true},
-	}
-
-	for i, testCase := range testCases {
-		globalBootTime = testCase.bootTime
-		globalObjectAPI = testCase.objectAPI
-		globalConnStats = testCase.connStats
-		globalHTTPStats = testCase.httpStats
-		globalNotificationSys = testCase.notificationSys
-		_, err := client.ServerInfo()
-		expectErr := (err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
-		}
-	}
-}
-
-func testAdminCmdRunnerGetConfig(t *testing.T, client adminCmdRunner) {
-	tmpGlobalServerConfig := globalServerConfig
-	defer func() {
-		globalServerConfig = tmpGlobalServerConfig
-	}()
-
-	config := newServerConfig()
-
-	testCases := []struct {
-		config    *serverConfig
-		expectErr bool
-	}{
-		{globalServerConfig, false},
-		{config, false},
-	}
-
-	for i, testCase := range testCases {
-		globalServerConfig = testCase.config
-		_, err := client.GetConfig()
-		expectErr := (err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
-		}
-	}
-}
-
-func newAdminRPCHTTPServerClient(t *testing.T) (*httptest.Server, *AdminRPCClient, *serverConfig) {
-	rpcServer, err := NewAdminRPCServer()
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		rpcServer.ServeHTTP(w, r)
-	}))
-
-	url, err := xnet.ParseURL(httpServer.URL)
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	host, err := xnet.ParseHost(url.Host)
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	prevGlobalServerConfig := globalServerConfig
-	globalServerConfig = newServerConfig()
-
-	rpcClient, err := NewAdminRPCClient(host)
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	return httpServer, rpcClient, prevGlobalServerConfig
-}
-
-func TestAdminRPCClientSignalService(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
-	defer httpServer.Close()
-	defer func() {
-		globalServerConfig = prevGlobalServerConfig
-	}()
-
-	testAdminCmdRunnerSignalService(t, rpcClient)
-}
-
-func TestAdminRPCClientReInitFormat(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
-	defer httpServer.Close()
-	defer func() {
-		globalServerConfig = prevGlobalServerConfig
-	}()
-
-	testAdminCmdRunnerReInitFormat(t, rpcClient)
-}
-
-func TestAdminRPCClientServerInfo(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
-	defer httpServer.Close()
-	defer func() {
-		globalServerConfig = prevGlobalServerConfig
-	}()
-
-	testAdminCmdRunnerServerInfo(t, rpcClient)
-}
-
-func TestAdminRPCClientGetConfig(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
-	defer httpServer.Close()
-	defer func() {
-		globalServerConfig = prevGlobalServerConfig
-	}()
-
-	testAdminCmdRunnerGetConfig(t, rpcClient)
-}

cmd/api-errors.go

@@ -28,8 +28,6 @@ import (
 	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
-	"github.com/minio/minio/pkg/s3select"
-	"github.com/minio/minio/pkg/s3select/format"
 )
 
 // APIError structure
@@ -87,6 +85,7 @@ const (
 	ErrNoSuchBucketPolicy
 	ErrNoSuchKey
 	ErrNoSuchUpload
+	ErrNoSuchVersion
 	ErrNotImplemented
 	ErrPreconditionFailed
 	ErrRequestTimeTooSkewed
@@ -447,6 +446,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrNoSuchVersion: {
+		Code:           "NoSuchVersion",
+		Description:    "Indicates that the version ID specified in the request does not match an existing version.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrNotImplemented: {
 		Code:           "NotImplemented",
 		Description:    "A header you provided implies functionality that is not implemented",
@@ -1506,168 +1510,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	case errOperationTimedOut, context.Canceled, context.DeadlineExceeded:
 		apiErr = ErrOperationTimedOut
 	}
-	switch err {
-	case s3select.ErrBusy:
-		apiErr = ErrBusy
-	case s3select.ErrUnauthorizedAccess:
-		apiErr = ErrUnauthorizedAccess
-	case s3select.ErrExpressionTooLong:
-		apiErr = ErrExpressionTooLong
-	case s3select.ErrIllegalSQLFunctionArgument:
-		apiErr = ErrIllegalSQLFunctionArgument
-	case s3select.ErrInvalidKeyPath:
-		apiErr = ErrInvalidKeyPath
-	case s3select.ErrInvalidCompressionFormat:
-		apiErr = ErrInvalidCompressionFormat
-	case s3select.ErrInvalidFileHeaderInfo:
-		apiErr = ErrInvalidFileHeaderInfo
-	case s3select.ErrInvalidJSONType:
-		apiErr = ErrInvalidJSONType
-	case s3select.ErrInvalidQuoteFields:
-		apiErr = ErrInvalidQuoteFields
-	case s3select.ErrInvalidRequestParameter:
-		apiErr = ErrInvalidRequestParameter
-	case s3select.ErrInvalidDataType:
-		apiErr = ErrInvalidDataType
-	case s3select.ErrInvalidTextEncoding:
-		apiErr = ErrInvalidTextEncoding
-	case s3select.ErrInvalidTableAlias:
-		apiErr = ErrInvalidTableAlias
-	case s3select.ErrMissingRequiredParameter:
-		apiErr = ErrMissingRequiredParameter
-	case s3select.ErrObjectSerializationConflict:
-		apiErr = ErrObjectSerializationConflict
-	case s3select.ErrUnsupportedSQLOperation:
-		apiErr = ErrUnsupportedSQLOperation
-	case s3select.ErrUnsupportedSQLStructure:
-		apiErr = ErrUnsupportedSQLStructure
-	case s3select.ErrUnsupportedSyntax:
-		apiErr = ErrUnsupportedSyntax
-	case s3select.ErrUnsupportedRangeHeader:
-		apiErr = ErrUnsupportedRangeHeader
-	case s3select.ErrLexerInvalidChar:
-		apiErr = ErrLexerInvalidChar
-	case s3select.ErrLexerInvalidOperator:
-		apiErr = ErrLexerInvalidOperator
-	case s3select.ErrLexerInvalidLiteral:
-		apiErr = ErrLexerInvalidLiteral
-	case s3select.ErrLexerInvalidIONLiteral:
-		apiErr = ErrLexerInvalidIONLiteral
-	case s3select.ErrParseExpectedDatePart:
-		apiErr = ErrParseExpectedDatePart
-	case s3select.ErrParseExpectedKeyword:
-		apiErr = ErrParseExpectedKeyword
-	case s3select.ErrParseExpectedTokenType:
-		apiErr = ErrParseExpectedTokenType
-	case s3select.ErrParseExpected2TokenTypes:
-		apiErr = ErrParseExpected2TokenTypes
-	case s3select.ErrParseExpectedNumber:
-		apiErr = ErrParseExpectedNumber
-	case s3select.ErrParseExpectedRightParenBuiltinFunctionCall:
-		apiErr = ErrParseExpectedRightParenBuiltinFunctionCall
-	case s3select.ErrParseExpectedTypeName:
-		apiErr = ErrParseExpectedTypeName
-	case s3select.ErrParseExpectedWhenClause:
-		apiErr = ErrParseExpectedWhenClause
-	case s3select.ErrParseUnsupportedToken:
-		apiErr = ErrParseUnsupportedToken
-	case s3select.ErrParseUnsupportedLiteralsGroupBy:
-		apiErr = ErrParseUnsupportedLiteralsGroupBy
-	case s3select.ErrParseExpectedMember:
-		apiErr = ErrParseExpectedMember
-	case s3select.ErrParseUnsupportedSelect:
-		apiErr = ErrParseUnsupportedSelect
-	case s3select.ErrParseUnsupportedCase:
-		apiErr = ErrParseUnsupportedCase
-	case s3select.ErrParseUnsupportedCaseClause:
-		apiErr = ErrParseUnsupportedCaseClause
-	case s3select.ErrParseUnsupportedAlias:
-		apiErr = ErrParseUnsupportedAlias
-	case s3select.ErrParseUnsupportedSyntax:
-		apiErr = ErrParseUnsupportedSyntax
-	case s3select.ErrParseUnknownOperator:
-		apiErr = ErrParseUnknownOperator
-	case s3select.ErrParseMissingIdentAfterAt:
-		apiErr = ErrParseMissingIdentAfterAt
-	case s3select.ErrParseUnexpectedOperator:
-		apiErr = ErrParseUnexpectedOperator
-	case s3select.ErrParseUnexpectedTerm:
-		apiErr = ErrParseUnexpectedTerm
-	case s3select.ErrParseUnexpectedToken:
-		apiErr = ErrParseUnexpectedToken
-	case s3select.ErrParseUnexpectedKeyword:
-		apiErr = ErrParseUnexpectedKeyword
-	case s3select.ErrParseExpectedExpression:
-		apiErr = ErrParseExpectedExpression
-	case s3select.ErrParseExpectedLeftParenAfterCast:
-		apiErr = ErrParseExpectedLeftParenAfterCast
-	case s3select.ErrParseExpectedLeftParenValueConstructor:
-		apiErr = ErrParseExpectedLeftParenValueConstructor
-	case s3select.ErrParseExpectedLeftParenBuiltinFunctionCall:
-		apiErr = ErrParseExpectedLeftParenBuiltinFunctionCall
-	case s3select.ErrParseExpectedArgumentDelimiter:
-		apiErr = ErrParseExpectedArgumentDelimiter
-	case s3select.ErrParseCastArity:
-		apiErr = ErrParseCastArity
-	case s3select.ErrParseInvalidTypeParam:
-		apiErr = ErrParseInvalidTypeParam
-	case s3select.ErrParseEmptySelect:
-		apiErr = ErrParseEmptySelect
-	case s3select.ErrParseSelectMissingFrom:
-		apiErr = ErrParseSelectMissingFrom
-	case s3select.ErrParseExpectedIdentForGroupName:
-		apiErr = ErrParseExpectedIdentForGroupName
-	case s3select.ErrParseExpectedIdentForAlias:
-		apiErr = ErrParseExpectedIdentForAlias
-	case s3select.ErrParseUnsupportedCallWithStar:
-		apiErr = ErrParseUnsupportedCallWithStar
-	case s3select.ErrParseNonUnaryAgregateFunctionCall:
-		apiErr = ErrParseNonUnaryAgregateFunctionCall
-	case s3select.ErrParseMalformedJoin:
-		apiErr = ErrParseMalformedJoin
-	case s3select.ErrParseExpectedIdentForAt:
-		apiErr = ErrParseExpectedIdentForAt
-	case s3select.ErrParseAsteriskIsNotAloneInSelectList:
-		apiErr = ErrParseAsteriskIsNotAloneInSelectList
-	case s3select.ErrParseCannotMixSqbAndWildcardInSelectList:
-		apiErr = ErrParseCannotMixSqbAndWildcardInSelectList
-	case s3select.ErrParseInvalidContextForWildcardInSelectList:
-		apiErr = ErrParseInvalidContextForWildcardInSelectList
-	case s3select.ErrIncorrectSQLFunctionArgumentType:
-		apiErr = ErrIncorrectSQLFunctionArgumentType
-	case s3select.ErrValueParseFailure:
-		apiErr = ErrValueParseFailure
-	case s3select.ErrIntegerOverflow:
-		apiErr = ErrIntegerOverflow
-	case s3select.ErrLikeInvalidInputs:
-		apiErr = ErrLikeInvalidInputs
-	case s3select.ErrCastFailed:
-		apiErr = ErrCastFailed
-	case s3select.ErrInvalidCast:
-		apiErr = ErrInvalidCast
-	case s3select.ErrEvaluatorInvalidTimestampFormatPattern:
-		apiErr = ErrEvaluatorInvalidTimestampFormatPattern
-	case s3select.ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing:
-		apiErr = ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing
-	case s3select.ErrEvaluatorTimestampFormatPatternDuplicateFields:
-		apiErr = ErrEvaluatorTimestampFormatPatternDuplicateFields
-	case s3select.ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch:
-		apiErr = ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch
-	case s3select.ErrEvaluatorUnterminatedTimestampFormatPatternToken:
-		apiErr = ErrEvaluatorUnterminatedTimestampFormatPatternToken
-	case s3select.ErrEvaluatorInvalidTimestampFormatPatternToken:
-		apiErr = ErrEvaluatorInvalidTimestampFormatPatternToken
-	case s3select.ErrEvaluatorInvalidTimestampFormatPatternSymbol:
-		apiErr = ErrEvaluatorInvalidTimestampFormatPatternSymbol
-	case s3select.ErrEvaluatorBindingDoesNotExist:
-		apiErr = ErrEvaluatorBindingDoesNotExist
-	case s3select.ErrMissingHeaders:
-		apiErr = ErrMissingHeaders
-	case format.ErrParseInvalidPathComponent:
-		apiErr = ErrMissingHeaders
-	case format.ErrInvalidColumnIndex:
-		apiErr = ErrInvalidColumnIndex
-	}
 
 	// Compression errors
 	switch err {

cmd/api-headers.go

@@ -36,7 +36,7 @@ func mustGetRequestID(t time.Time) string {
 
 // Write http common headers
 func setCommonHeaders(w http.ResponseWriter) {
-	w.Header().Set("Server", globalServerUserAgent)
+	w.Header().Set("Server", "Minio/"+ReleaseTag)
 	// Set `x-amz-bucket-region` only if region is set on the server
 	// by default minio uses an empty region.
 	if region := globalServerConfig.GetRegion(); region != "" {

cmd/api-router.go

@@ -26,14 +26,19 @@ import (
 type objectAPIHandlers struct {
 	ObjectAPI func() ObjectLayer
 	CacheAPI  func() CacheObjectLayer
+	// Returns true of handlers should interpret encryption.
+	EncryptionEnabled func() bool
 }
 
 // registerAPIRouter - registers S3 compatible APIs.
-func registerAPIRouter(router *mux.Router) {
+func registerAPIRouter(router *mux.Router, encryptionEnabled bool) {
 	// Initialize API.
 	api := objectAPIHandlers{
 		ObjectAPI: newObjectLayerFn,
 		CacheAPI:  newCacheObjectsFn,
+		EncryptionEnabled: func() bool {
+			return encryptionEnabled
+		},
 	}
 
 	// API Router

cmd/auth-handler.go

@@ -283,7 +283,7 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 			AccountName:     cred.AccessKey,
 			Action:          action,
 			BucketName:      bucketName,
-			ConditionValues: getConditionValues(r, locationConstraint),
+			ConditionValues: getConditionValues(r, locationConstraint, ""),
 			IsOwner:         false,
 			ObjectName:      objectName,
 		}) {
@@ -296,7 +296,7 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 		AccountName:     cred.AccessKey,
 		Action:          iampolicy.Action(action),
 		BucketName:      bucketName,
-		ConditionValues: getConditionValues(r, ""),
+		ConditionValues: getConditionValues(r, "", cred.AccessKey),
 		ObjectName:      objectName,
 		IsOwner:         owner,
 		Claims:          claims,
@@ -448,7 +448,7 @@ func isPutAllowed(atype authType, bucketName, objectName string, r *http.Request
 			AccountName:     cred.AccessKey,
 			Action:          policy.PutObjectAction,
 			BucketName:      bucketName,
-			ConditionValues: getConditionValues(r, ""),
+			ConditionValues: getConditionValues(r, "", ""),
 			IsOwner:         false,
 			ObjectName:      objectName,
 		}) {
@@ -461,7 +461,7 @@ func isPutAllowed(atype authType, bucketName, objectName string, r *http.Request
 		AccountName:     cred.AccessKey,
 		Action:          policy.PutObjectAction,
 		BucketName:      bucketName,
-		ConditionValues: getConditionValues(r, ""),
+		ConditionValues: getConditionValues(r, "", cred.AccessKey),
 		ObjectName:      objectName,
 		IsOwner:         owner,
 		Claims:          claims,

cmd/bitrot-streaming.go

@@ -0,0 +1,172 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/hex"
+	"hash"
+	"io"
+
+	"github.com/minio/minio/cmd/logger"
+)
+
+// Calculates bitrot in chunks and writes the hash into the stream.
+type streamingBitrotWriter struct {
+	iow       *io.PipeWriter
+	h         hash.Hash
+	shardSize int64
+	canClose  chan struct{} // Needed to avoid race explained in Close() call.
+
+	// Following two fields are used only to make sure that Write(p) is called such that
+	// len(p) is always the block size except the last block, i.e prevent programmer errors.
+	currentBlockIdx int
+	verifyTillIdx   int
+}
+
+func (b *streamingBitrotWriter) Write(p []byte) (int, error) {
+	if b.currentBlockIdx < b.verifyTillIdx && int64(len(p)) != b.shardSize {
+		// All blocks except last should be of the length b.shardSize
+		logger.LogIf(context.Background(), errUnexpected)
+		return 0, errUnexpected
+	}
+	if len(p) == 0 {
+		return 0, nil
+	}
+	b.h.Reset()
+	b.h.Write(p)
+	hashBytes := b.h.Sum(nil)
+	n, err := b.iow.Write(hashBytes)
+	if n != len(hashBytes) {
+		logger.LogIf(context.Background(), err)
+		return 0, err
+	}
+	n, err = b.iow.Write(p)
+	b.currentBlockIdx++
+	return n, err
+}
+
+func (b *streamingBitrotWriter) Close() error {
+	err := b.iow.Close()
+	// Wait for all data to be written before returning else it causes race conditions.
+	// Race condition is because of io.PipeWriter implementation. i.e consider the following
+	// sequent of operations:
+	// 1) pipe.Write()
+	// 2) pipe.Close()
+	// Now pipe.Close() can return before the data is read on the other end of the pipe and written to the disk
+	// Hence an immediate Read() on the file can return incorrect data.
+	<-b.canClose
+	return err
+}
+
+// Returns streaming bitrot writer implementation.
+func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.WriteCloser {
+	r, w := io.Pipe()
+	h := algo.New()
+	bw := &streamingBitrotWriter{w, h, shardSize, make(chan struct{}), 0, int(length / shardSize)}
+	go func() {
+		bitrotSumsTotalSize := ceilFrac(length, shardSize) * int64(h.Size()) // Size used for storing bitrot checksums.
+		totalFileSize := bitrotSumsTotalSize + length
+		err := disk.CreateFile(volume, filePath, totalFileSize, r)
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+			r.CloseWithError(err)
+		}
+		close(bw.canClose)
+	}()
+	return bw
+}
+
+// ReadAt() implementation which verifies the bitrot hash available as part of the stream.
+type streamingBitrotReader struct {
+	disk       StorageAPI
+	rc         io.ReadCloser
+	volume     string
+	filePath   string
+	tillOffset int64
+	currOffset int64
+	h          hash.Hash
+	shardSize  int64
+	hashBytes  []byte
+}
+
+func (b *streamingBitrotReader) Close() error {
+	if b.rc == nil {
+		return nil
+	}
+	return b.rc.Close()
+}
+
+func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
+	var err error
+	if offset%b.shardSize != 0 {
+		// Offset should always be aligned to b.shardSize
+		logger.LogIf(context.Background(), errUnexpected)
+		return 0, errUnexpected
+	}
+	if b.rc == nil {
+		// For the first ReadAt() call we need to open the stream for reading.
+		b.currOffset = offset
+		streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
+		b.rc, err = b.disk.ReadFileStream(b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+			return 0, err
+		}
+	}
+	if offset != b.currOffset {
+		logger.LogIf(context.Background(), errUnexpected)
+		return 0, errUnexpected
+	}
+	b.h.Reset()
+	_, err = io.ReadFull(b.rc, b.hashBytes)
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return 0, err
+	}
+	_, err = io.ReadFull(b.rc, buf)
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return 0, err
+	}
+	b.h.Write(buf)
+
+	if bytes.Compare(b.h.Sum(nil), b.hashBytes) != 0 {
+		err = hashMismatchError{hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))}
+		logger.LogIf(context.Background(), err)
+		return 0, err
+	}
+	b.currOffset += int64(len(buf))
+	return len(buf), nil
+}
+
+// Returns streaming bitrot reader implementation.
+func newStreamingBitrotReader(disk StorageAPI, volume, filePath string, tillOffset int64, algo BitrotAlgorithm, shardSize int64) *streamingBitrotReader {
+	h := algo.New()
+	return &streamingBitrotReader{
+		disk,
+		nil,
+		volume,
+		filePath,
+		ceilFrac(tillOffset, shardSize)*int64(h.Size()) + tillOffset,
+		0,
+		h,
+		shardSize,
+		make([]byte, h.Size()),
+	}
+}

cmd/bitrot-whole.go

@@ -0,0 +1,109 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"hash"
+	"io"
+
+	"github.com/minio/minio/cmd/logger"
+)
+
+// Implementation to calculate bitrot for the whole file.
+type wholeBitrotWriter struct {
+	disk      StorageAPI
+	volume    string
+	filePath  string
+	shardSize int64 // This is the shard size of the erasure logic
+	hash.Hash       // For bitrot hash
+
+	// Following two fields are used only to make sure that Write(p) is called such that
+	// len(p) is always the block size except the last block and prevent programmer errors.
+	currentBlockIdx int
+	lastBlockIdx    int
+}
+
+func (b *wholeBitrotWriter) Write(p []byte) (int, error) {
+	if b.currentBlockIdx < b.lastBlockIdx && int64(len(p)) != b.shardSize {
+		// All blocks except last should be of the length b.shardSize
+		logger.LogIf(context.Background(), errUnexpected)
+		return 0, errUnexpected
+	}
+	err := b.disk.AppendFile(b.volume, b.filePath, p)
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return 0, err
+	}
+	_, err = b.Hash.Write(p)
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return 0, err
+	}
+	b.currentBlockIdx++
+	return len(p), nil
+}
+
+func (b *wholeBitrotWriter) Close() error {
+	return nil
+}
+
+// Returns whole-file bitrot writer.
+func newWholeBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.WriteCloser {
+	return &wholeBitrotWriter{disk, volume, filePath, shardSize, algo.New(), 0, int(length / shardSize)}
+}
+
+// Implementation to verify bitrot for the whole file.
+type wholeBitrotReader struct {
+	disk       StorageAPI
+	volume     string
+	filePath   string
+	verifier   *BitrotVerifier // Holds the bit-rot info
+	tillOffset int64           // Affects the length of data requested in disk.ReadFile depending on Read()'s offset
+	buf        []byte          // Holds bit-rot verified data
+}
+
+func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error) {
+	if b.buf == nil {
+		b.buf = make([]byte, b.tillOffset-offset)
+		if _, err := b.disk.ReadFile(b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
+			ctx := context.Background()
+			logger.GetReqInfo(ctx).AppendTags("disk", b.disk.String())
+			logger.LogIf(ctx, err)
+			return 0, err
+		}
+	}
+	if len(b.buf) < len(buf) {
+		logger.LogIf(context.Background(), errLessData)
+		return 0, errLessData
+	}
+	n = copy(buf, b.buf)
+	b.buf = b.buf[n:]
+	return n, nil
+}
+
+// Returns whole-file bitrot reader.
+func newWholeBitrotReader(disk StorageAPI, volume, filePath string, algo BitrotAlgorithm, tillOffset int64, sum []byte) *wholeBitrotReader {
+	return &wholeBitrotReader{
+		disk:       disk,
+		volume:     volume,
+		filePath:   filePath,
+		verifier:   &BitrotVerifier{algo, sum},
+		tillOffset: tillOffset,
+		buf:        nil,
+	}
+}

cmd/bitrot.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"hash"
+	"io"
 
 	"github.com/minio/highwayhash"
 	"github.com/minio/minio/cmd/logger"
@@ -38,19 +39,22 @@ const (
 	SHA256 BitrotAlgorithm = 1 + iota
 	// HighwayHash256 represents the HighwayHash-256 hash function
 	HighwayHash256
+	// HighwayHash256S represents the Streaming HighwayHash-256 hash function
+	HighwayHash256S
 	// BLAKE2b512 represents the BLAKE2b-512 hash function
 	BLAKE2b512
 )
 
 // DefaultBitrotAlgorithm is the default algorithm used for bitrot protection.
 const (
-	DefaultBitrotAlgorithm = HighwayHash256
+	DefaultBitrotAlgorithm = HighwayHash256S
 )
 
 var bitrotAlgorithms = map[BitrotAlgorithm]string{
 	SHA256:          "sha256",
 	BLAKE2b512:      "blake2b",
 	HighwayHash256:  "highwayhash256",
+	HighwayHash256S: "highwayhash256S",
 }
 
 // New returns a new hash.Hash calculating the given bitrot algorithm.
@@ -64,6 +68,9 @@ func (a BitrotAlgorithm) New() hash.Hash {
 	case HighwayHash256:
 		hh, _ := highwayhash.New(magicHighwayHash256Key) // New will never return error since key is 256 bit
 		return hh
+	case HighwayHash256S:
+		hh, _ := highwayhash.New(magicHighwayHash256Key) // New will never return error since key is 256 bit
+		return hh
 	default:
 		logger.CriticalIf(context.Background(), errors.New("Unsupported bitrot algorithm"))
 		return nil
@@ -109,86 +116,72 @@ func BitrotAlgorithmFromString(s string) (a BitrotAlgorithm) {
 	return
 }
 
-// To read bit-rot verified data.
+func newBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.Writer {
-type bitrotReader struct {
+	if algo == HighwayHash256S {
-	disk      StorageAPI
+		return newStreamingBitrotWriter(disk, volume, filePath, length, algo, shardSize)
-	volume    string
+	}
-	filePath  string
+	return newWholeBitrotWriter(disk, volume, filePath, length, algo, shardSize)
-	verifier  *BitrotVerifier // Holds the bit-rot info
-	endOffset int64           // Affects the length of data requested in disk.ReadFile depending on Read()'s offset
-	buf       []byte          // Holds bit-rot verified data
 }
 
-// newBitrotReader returns bitrotReader.
+func newBitrotReader(disk StorageAPI, bucket string, filePath string, tillOffset int64, algo BitrotAlgorithm, sum []byte, shardSize int64) io.ReaderAt {
-// Note that the buffer is allocated later in Read(). This is because we will know the buffer length only
+	if algo == HighwayHash256S {
-// during the bitrotReader.Read(). Depending on when parallelReader fails-over, the buffer length can be different.
+		return newStreamingBitrotReader(disk, bucket, filePath, tillOffset, algo, shardSize)
-func newBitrotReader(disk StorageAPI, volume, filePath string, algo BitrotAlgorithm, endOffset int64, sum []byte) *bitrotReader {
+	}
-	return &bitrotReader{
+	return newWholeBitrotReader(disk, bucket, filePath, algo, tillOffset, sum)
-		disk:      disk,
+}
-		volume:    volume,
+
-		filePath:  filePath,
+// Close all the readers.
-		verifier:  &BitrotVerifier{algo, sum},
+func closeBitrotReaders(rs []io.ReaderAt) {
-		endOffset: endOffset,
+	for _, r := range rs {
-		buf:       nil,
+		if br, ok := r.(*streamingBitrotReader); ok {
+			br.Close()
+		}
 	}
 }
 
-// ReadChunk returns requested data.
+// Close all the writers.
-func (b *bitrotReader) ReadChunk(offset int64, length int64) ([]byte, error) {
+func closeBitrotWriters(ws []io.Writer) {
-	if b.buf == nil {
+	for _, w := range ws {
-		b.buf = make([]byte, b.endOffset-offset)
+		if bw, ok := w.(*streamingBitrotWriter); ok {
-		if _, err := b.disk.ReadFile(b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
+			bw.Close()
-			ctx := context.Background()
-			logger.GetReqInfo(ctx).AppendTags("disk", b.disk.String())
-			logger.LogIf(ctx, err)
-			return nil, err
 		}
 	}
-	if int64(len(b.buf)) < length {
-		logger.LogIf(context.Background(), errLessData)
-		return nil, errLessData
-	}
-	retBuf := b.buf[:length]
-	b.buf = b.buf[length:]
-	return retBuf, nil
-}
-
-// To calculate the bit-rot of the written data.
-type bitrotWriter struct {
-	disk     StorageAPI
-	volume   string
-	filePath string
-	h        hash.Hash
-}
-
-// newBitrotWriter returns bitrotWriter.
-func newBitrotWriter(disk StorageAPI, volume, filePath string, algo BitrotAlgorithm) *bitrotWriter {
-	return &bitrotWriter{
-		disk:     disk,
-		volume:   volume,
-		filePath: filePath,
-		h:        algo.New(),
-	}
 }
 
-// Append appends the data and while calculating the hash.
+// Returns hash sum for whole-bitrot, nil for streaming-bitrot.
-func (b *bitrotWriter) Append(buf []byte) error {
+func bitrotWriterSum(w io.Writer) []byte {
-	n, err := b.h.Write(buf)
+	if bw, ok := w.(*wholeBitrotWriter); ok {
-	if err != nil {
+		return bw.Sum(nil)
-		return err
-	}
-	if n != len(buf) {
-		logger.LogIf(context.Background(), errUnexpected)
-		return errUnexpected
-	}
-	if err = b.disk.AppendFile(b.volume, b.filePath, buf); err != nil {
-		logger.LogIf(context.Background(), err)
-		return err
 	}
 	return nil
 }
 
-// Sum returns bit-rot sum.
+// Verify if a file has bitrot error.
-func (b *bitrotWriter) Sum() []byte {
+func bitrotCheckFile(disk StorageAPI, volume string, filePath string, tillOffset int64, algo BitrotAlgorithm, sum []byte, shardSize int64) (err error) {
-	return b.h.Sum(nil)
+	if algo != HighwayHash256S {
+		buf := []byte{}
+		// For whole-file bitrot we don't need to read the entire file as the bitrot verify happens on the server side even if we read 0-bytes.
+		_, err = disk.ReadFile(volume, filePath, 0, buf, NewBitrotVerifier(algo, sum))
+		return err
+	}
+	buf := make([]byte, shardSize)
+	r := newStreamingBitrotReader(disk, volume, filePath, tillOffset, algo, shardSize)
+	defer closeBitrotReaders([]io.ReaderAt{r})
+	var offset int64
+	for {
+		if offset == tillOffset {
+			break
+		}
+		var n int
+		tmpBuf := buf
+		if int64(len(tmpBuf)) > (tillOffset - offset) {
+			tmpBuf = tmpBuf[:(tillOffset - offset)]
+		}
+		n, err = r.ReadAt(tmpBuf, offset)
+		if err != nil {
+			return err
+		}
+		offset += int64(n)
+	}
+	return nil
 }

cmd/bitrot_test.go

@@ -17,13 +17,14 @@
 package cmd
 
 import (
+	"io"
 	"io/ioutil"
 	"log"
 	"os"
 	"testing"
 )
 
-func TestBitrotReaderWriter(t *testing.T) {
+func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 	tmpDir, err := ioutil.TempDir("", "")
 	if err != nil {
 		log.Fatal(err)
@@ -40,32 +41,44 @@ func TestBitrotReaderWriter(t *testing.T) {
 
 	disk.MakeVol(volume)
 
-	writer := newBitrotWriter(disk, volume, filePath, HighwayHash256)
+	writer := newBitrotWriter(disk, volume, filePath, 35, bitrotAlgo, 10)
 
-	err = writer.Append([]byte("aaaaaaaaa"))
+	_, err = writer.Write([]byte("aaaaaaaaaa"))
 	if err != nil {
 		log.Fatal(err)
 	}
-	err = writer.Append([]byte("a"))
+	_, err = writer.Write([]byte("aaaaaaaaaa"))
 	if err != nil {
 		log.Fatal(err)
 	}
-	err = writer.Append([]byte("aaaaaaaaaa"))
+	_, err = writer.Write([]byte("aaaaaaaaaa"))
 	if err != nil {
 		log.Fatal(err)
 	}
-	err = writer.Append([]byte("aaaaa"))
+	_, err = writer.Write([]byte("aaaaa"))
-	if err != nil {
-		log.Fatal(err)
-	}
-	err = writer.Append([]byte("aaaaaaaaaa"))
 	if err != nil {
 		log.Fatal(err)
 	}
+	writer.(io.Closer).Close()
 
-	reader := newBitrotReader(disk, volume, filePath, HighwayHash256, 35, writer.Sum())
+	reader := newBitrotReader(disk, volume, filePath, 35, bitrotAlgo, bitrotWriterSum(writer), 10)
-
+	b := make([]byte, 10)
-	if _, err = reader.ReadChunk(0, 35); err != nil {
+	if _, err = reader.ReadAt(b, 0); err != nil {
+		log.Fatal(err)
+	}
+	if _, err = reader.ReadAt(b, 10); err != nil {
+		log.Fatal(err)
+	}
+	if _, err = reader.ReadAt(b, 20); err != nil {
+		log.Fatal(err)
+	}
+	if _, err = reader.ReadAt(b[:5], 30); err != nil {
 		log.Fatal(err)
 	}
 }
+
+func TestAllBitrotAlgorithms(t *testing.T) {
+	for bitrotAlgo := range bitrotAlgorithms {
+		testBitrotReaderWriterAlgo(t, bitrotAlgo)
+	}
+}

cmd/bucket-handlers.go

@@ -482,7 +482,14 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
-
+	if crypto.S3KMS.IsRequested(r.Header) { // SSE-KMS is not supported
+		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
+		return
+	}
+	if !api.EncryptionEnabled() && hasServerSideEncryptionHeader(r.Header) {
+		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
+		return
+	}
 	bucket := mux.Vars(r)["bucket"]
 
 	// Require Content-Length to be set in the request
@@ -619,6 +626,18 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	rawReader := hashReader
 	pReader := NewPutObjReader(rawReader, nil, nil)
 	var objectEncryptionKey []byte
+
+	// This request header needs to be set prior to setting ObjectOptions
+	if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) {
+		r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
+	}
+	// get gateway encryption options
+	var opts ObjectOptions
+	opts, err = putEncryptionOpts(ctx, r, bucket, object, nil)
+	if err != nil {
+		writeErrorResponseHeadersOnly(w, toAPIErrorCode(ctx, err))
+		return
+	}
 	if objectAPI.IsEncryptionSupported() {
 		if hasServerSideEncryptionHeader(formValues) && !hasSuffix(object, slashSeparator) { // handle SSE-C and SSE-S3 requests
 			var reader io.Reader
@@ -645,7 +664,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		}
 	}
 
-	objInfo, err := objectAPI.PutObject(ctx, bucket, object, pReader, metadata, ObjectOptions{})
+	objInfo, err := objectAPI.PutObject(ctx, bucket, object, pReader, metadata, opts)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return

cmd/certs.go

@@ -68,28 +68,6 @@ func parsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err er
 }
 
 func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
-	// Get all CA file names.
-	var caFiles []string
-	fis, err := readDir(certsCAsDir)
-	if err != nil && err != errFileNotFound {
-		return nil, err
-	}
-	// Return success if CA's directory is missing.
-	if err == errFileNotFound {
-		return nil, nil
-	}
-	for _, fi := range fis {
-		// Skip all directories.
-		if hasSuffix(fi, slashSeparator) {
-			continue
-		}
-		// We are only interested in regular files here.
-		caFiles = append(caFiles, pathJoin(certsCAsDir, fi))
-	}
-	if len(caFiles) == 0 {
-		return nil, nil
-	}
-
 	rootCAs, _ := x509.SystemCertPool()
 	if rootCAs == nil {
 		// In some systems (like Windows) system cert pool is
@@ -98,16 +76,26 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 		rootCAs = x509.NewCertPool()
 	}
 
-	// Load custom root CAs for client requests
+	fis, err := readDir(certsCAsDir)
-	for _, caFile := range caFiles {
-		caCert, err := ioutil.ReadFile(caFile)
 	if err != nil {
-			return nil, err
+		if err == errFileNotFound {
+			err = nil // Return success if CA's directory is missing.
+		}
+		return rootCAs, err
 	}
 
+	// Load all custom CA files.
+	for _, fi := range fis {
+		// Skip all directories.
+		if hasSuffix(fi, slashSeparator) {
+			continue
+		}
+		caCert, err := ioutil.ReadFile(pathJoin(certsCAsDir, fi))
+		if err != nil {
+			return rootCAs, err
+		}
 		rootCAs.AppendCertsFromPEM(caCert)
 	}
-
 	return rootCAs, nil
 }
 

cmd/common-main.go

@@ -30,7 +30,6 @@ import (
 	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
 	"github.com/minio/minio-go/pkg/set"
-	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/logger/target/console"
 	"github.com/minio/minio/cmd/logger/target/http"
@@ -79,38 +78,89 @@ func loadLoggers() {
 
 }
 
-func handleCommonCmdArgs(ctx *cli.Context) {
+func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) (*ConfigDir, bool) {
-
+	var dir string
-	var configDir string
+	var dirSet bool
 
 	switch {
-	case ctx.IsSet("config-dir"):
+	case ctx.IsSet(option):
-		configDir = ctx.String("config-dir")
+		dir = ctx.String(option)
-	case ctx.GlobalIsSet("config-dir"):
+		dirSet = true
-		configDir = ctx.GlobalString("config-dir")
+	case ctx.GlobalIsSet(option):
-		// cli package does not expose parent's "config-dir" option.  Below code is workaround.
+		dir = ctx.GlobalString(option)
-		if configDir == "" || configDir == getConfigDir() {
+		dirSet = true
-			if ctx.Parent().GlobalIsSet("config-dir") {
+		// cli package does not expose parent's option option.  Below code is workaround.
-				configDir = ctx.Parent().GlobalString("config-dir")
+		if dir == "" || dir == getDefaultDir() {
+			dirSet = false // Unset to false since GlobalIsSet() true is a false positive.
+			if ctx.Parent().GlobalIsSet(option) {
+				dir = ctx.Parent().GlobalString(option)
+				dirSet = true
 			}
 		}
 	default:
-		// Neither local nor global config-dir option is provided.  In this case, try to use
+		// Neither local nor global option is provided.  In this case, try to use
-		// default config directory.
+		// default directory.
-		configDir = getConfigDir()
+		dir = getDefaultDir()
-		if configDir == "" {
+		if dir == "" {
-			logger.FatalIf(errors.New("missing option"), "config-dir option must be provided")
+			logger.FatalIf(errInvalidArgument, "%s option must be provided", option)
 		}
 	}
 
-	if configDir == "" {
+	if dir == "" {
-		logger.FatalIf(errors.New("empty directory"), "Configuration directory cannot be empty")
+		logger.FatalIf(errors.New("empty directory"), "%s directory cannot be empty", option)
 	}
 
 	// Disallow relative paths, figure out absolute paths.
-	configDirAbs, err := filepath.Abs(configDir)
+	dirAbs, err := filepath.Abs(dir)
-	logger.FatalIf(err, "Unable to fetch absolute path for config directory %s", configDir)
+	logger.FatalIf(err, "Unable to fetch absolute path for %s=%s", option, dir)
-	setConfigDir(configDirAbs)
+
+	logger.FatalIf(mkdirAllIgnorePerm(dirAbs), "Unable to create directory specified %s=%s", option, dir)
+
+	return &ConfigDir{path: dirAbs}, dirSet
+}
+
+func handleCommonCmdArgs(ctx *cli.Context) {
+
+	// Get "json" flag from command line argument and
+	// enable json and quite modes if jason flag is turned on.
+	globalCLIContext.JSON = ctx.IsSet("json") || ctx.GlobalIsSet("json")
+	if globalCLIContext.JSON {
+		logger.EnableJSON()
+	}
+
+	// Get quiet flag from command line argument.
+	globalCLIContext.Quiet = ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet")
+	if globalCLIContext.Quiet {
+		logger.EnableQuiet()
+	}
+
+	// Get anonymous flag from command line argument.
+	globalCLIContext.Anonymous = ctx.IsSet("anonymous") || ctx.GlobalIsSet("anonymous")
+	if globalCLIContext.Anonymous {
+		logger.EnableAnonymous()
+	}
+
+	// Fetch address option
+	globalCLIContext.Addr = ctx.GlobalString("address")
+	if globalCLIContext.Addr == "" || globalCLIContext.Addr == ":"+globalMinioDefaultPort {
+		globalCLIContext.Addr = ctx.String("address")
+	}
+
+	// Set all config, certs and CAs directories.
+	var configSet, certsSet bool
+	globalConfigDir, configSet = newConfigDirFromCtx(ctx, "config-dir", defaultConfigDir.Get)
+	globalCertsDir, certsSet = newConfigDirFromCtx(ctx, "certs-dir", defaultCertsDir.Get)
+
+	// Remove this code when we deprecate and remove config-dir.
+	// This code is to make sure we inherit from the config-dir
+	// option if certs-dir is not provided.
+	if !certsSet && configSet {
+		globalCertsDir = &ConfigDir{path: filepath.Join(globalConfigDir.Get(), certsDir)}
+	}
+
+	globalCertsCADir = &ConfigDir{path: filepath.Join(globalCertsDir.Get(), certsCADir)}
+
+	logger.FatalIf(mkdirAllIgnorePerm(globalCertsCADir.Get()), "Unable to create certs CA directory at %s", globalCertsCADir.Get())
 }
 
 // Parses the given compression exclude list `extensions` or `content-types`.
@@ -192,6 +242,7 @@ func handleCommonEnvVars() {
 					return &cert, terr
 				}
 			}
+
 			globalEtcdClient, err = etcd.New(etcd.Config{
 				Endpoints:         etcdEndpoints,
 				DialTimeout:       defaultDialTimeout,
@@ -221,12 +272,22 @@ func handleCommonEnvVars() {
 	minioEndpointsEnv, ok := os.LookupEnv("MINIO_PUBLIC_IPS")
 	if ok {
 		minioEndpoints := strings.Split(minioEndpointsEnv, ",")
-		for i, ip := range minioEndpoints {
+		var domainIPs = set.NewStringSet()
-			if net.ParseIP(ip) == nil {
+		for _, endpoint := range minioEndpoints {
-				logger.FatalIf(errInvalidArgument, "Unable to initialize Minio server with invalid MINIO_PUBLIC_IPS[%d]: %s", i, ip)
+			if net.ParseIP(endpoint) == nil {
+				// Checking if the IP is a DNS entry.
+				addrs, err := net.LookupHost(endpoint)
+				if err != nil {
+					logger.FatalIf(err, "Unable to initialize Minio server with [%s] invalid entry found in MINIO_PUBLIC_IPS", endpoint)
 				}
+				for _, addr := range addrs {
+					domainIPs.Add(addr)
 				}
-		updateDomainIPs(set.CreateStringSet(minioEndpoints...))
+				continue
+			}
+			domainIPs.Add(endpoint)
+		}
+		updateDomainIPs(domainIPs)
 	} else {
 		// Add found interfaces IP address to global domain IPS,
 		// loopback addresses will be naturally dropped.
@@ -322,20 +383,6 @@ func handleCommonEnvVars() {
 		globalWORMEnabled = bool(wormFlag)
 	}
 
-	kmsConf, err := crypto.NewVaultConfig()
-	if err != nil {
-		logger.Fatal(err, "Unable to initialize hashicorp vault")
-	}
-	if kmsConf.Vault.Endpoint != "" {
-		kms, err := crypto.NewVault(kmsConf)
-		if err != nil {
-			logger.Fatal(err, "Unable to initialize KMS")
-		}
-		globalKMS = kms
-		globalKMSKeyID = kmsConf.Vault.Key.Name
-		globalKMSConfig = kmsConf
-	}
-
 	if compress := os.Getenv("MINIO_COMPRESS"); compress != "" {
 		globalIsCompressionEnabled = strings.EqualFold(compress, "true")
 	}

cmd/config-common.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"fmt"
 
 	etcd "github.com/coreos/etcd/clientv3"
 	"github.com/minio/minio/cmd/logger"
@@ -60,8 +61,15 @@ func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) er
 }
 
 func saveConfigEtcd(ctx context.Context, client *etcd.Client, configFile string, data []byte) error {
-	_, err := client.Put(ctx, configFile, string(data))
+	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
-	return err
+	defer cancel()
+	_, err := client.Put(timeoutCtx, configFile, string(data))
+	if err == context.DeadlineExceeded {
+		return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints())
+	} else if err != nil {
+		return fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints())
+	}
+	return nil
 }
 
 func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data []byte) error {
@@ -75,9 +83,14 @@ func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data
 }
 
 func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) ([]byte, error) {
-	resp, err := client.Get(ctx, configFile)
+	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+	resp, err := client.Get(timeoutCtx, configFile)
 	if err != nil {
-		return nil, err
+		if err == context.DeadlineExceeded {
+			return nil, fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints())
+		}
+		return nil, fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints())
 	}
 	if resp.Count == 0 {
 		return nil, errConfigNotFound
@@ -90,23 +103,28 @@ func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string)
 	return nil, errConfigNotFound
 }
 
-// watchConfig - watches for changes on `configFile` on etcd and loads them.
+// watchConfigEtcd - watches for changes on `configFile` on etcd and loads them.
-func watchConfig(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
+func watchConfigEtcd(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
-	if globalEtcdClient != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
-		for watchResp := range globalEtcdClient.Watch(context.Background(), configFile) {
+	defer cancel()
+	for watchResp := range globalEtcdClient.Watch(ctx, configFile) {
 		for _, event := range watchResp.Events {
 			if event.IsModify() || event.IsCreate() {
 				loadCfgFn(objAPI)
 			}
 		}
 	}
-	}
 }
 
 func checkConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
-	resp, err := globalEtcdClient.Get(ctx, configFile)
+	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+	resp, err := client.Get(timeoutCtx, configFile)
 	if err != nil {
-		return err
+		if err == context.DeadlineExceeded {
+			return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints())
+		}
+		return fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints())
 	}
 	if resp.Count == 0 {
 		return errConfigNotFound

cmd/config-current.go

@@ -25,6 +25,7 @@ import (
 	"sync"
 
 	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
@@ -271,8 +272,8 @@ func (s *serverConfig) loadFromEnvs() {
 		s.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
 	}
 
-	if globalKMS != nil {
+	if err := Environment.LookupKMSConfig(s.KMS); err != nil {
-		s.KMS = globalKMSConfig
+		logger.FatalIf(err, "Unable to setup the KMS")
 	}
 
 	if globalIsEnvCompression {
@@ -282,7 +283,7 @@ func (s *serverConfig) loadFromEnvs() {
 	if jwksURL, ok := os.LookupEnv("MINIO_IAM_JWKS_URL"); ok {
 		if u, err := xnet.ParseURL(jwksURL); err == nil {
 			s.OpenID.JWKS.URL = u
-			s.OpenID.JWKS.PopulatePublicKey()
+			logger.FatalIf(s.OpenID.JWKS.PopulatePublicKey(), "Unable to populate public key from JWKS URL")
 		}
 	}
 
@@ -534,12 +535,8 @@ func (s *serverConfig) loadToCachedConfigs() {
 		globalCacheExpiry = cacheConf.Expiry
 		globalCacheMaxUse = cacheConf.MaxUse
 	}
-	if globalKMS == nil {
+	if err := Environment.LookupKMSConfig(s.KMS); err != nil {
-		globalKMSConfig = s.KMS
+		logger.FatalIf(err, "Unable to setup the KMS")
-		if kms, err := crypto.NewVault(globalKMSConfig); err == nil {
-			globalKMS = kms
-			globalKMSKeyID = globalKMSConfig.Vault.Key.Name
-		}
 	}
 
 	if !globalIsCompressionEnabled {
@@ -549,20 +546,16 @@ func (s *serverConfig) loadToCachedConfigs() {
 		globalIsCompressionEnabled = compressionConf.Enabled
 	}
 
-	if globalIAMValidators == nil {
 	globalIAMValidators = getAuthValidators(s)
-	}
 
-	if globalPolicyOPA == nil {
 	if s.Policy.OPA.URL != nil && s.Policy.OPA.URL.String() != "" {
 		globalPolicyOPA = iampolicy.NewOpa(iampolicy.OpaArgs{
 			URL:         s.Policy.OPA.URL,
 			AuthToken:   s.Policy.OPA.AuthToken,
 			Transport:   NewCustomHTTPTransport(),
-				CloseRespFn: CloseResponse,
+			CloseRespFn: xhttp.DrainBody,
 		})
 	}
-	}
 }
 
 // newSrvConfig - initialize a new server config, saves env parameters if
@@ -688,6 +681,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.MQTT {
 		if args.Enable {
+			args.RootCAs = globalRootCAs
 			newTarget, err := target.NewMQTTTarget(id, args)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
@@ -772,6 +766,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.Webhook {
 		if args.Enable {
+			args.RootCAs = globalRootCAs
 			newTarget := target.NewWebhookTarget(id, args)
 			if err := targetList.Add(newTarget); err != nil {
 				logger.LogIf(context.Background(), err)

cmd/config-current_test.go

@@ -233,7 +233,7 @@ func TestValidateConfig(t *testing.T) {
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "namespace", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
 
 		// Test 27 - Test MQTT
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "clientId": "", "username": "", "password": ""}}}}`, false},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "username": "", "password": "", "queueDir": ""}}}}`, false},
 
 		// Test 28 - Test NSQ
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nsq": { "1": { "enable": true, "nsqdAddress": "", "topic": ""} }}}`, false},

cmd/config-dir.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"os"
 	"path/filepath"
-	"sync"
 
 	homedir "github.com/mitchellh/go-homedir"
 )
@@ -41,63 +40,9 @@ const (
 	privateKeyFile = "private.key"
 )
 
-// ConfigDir - configuration directory with locking.
+// ConfigDir - points to a user set directory.
 type ConfigDir struct {
-	sync.Mutex
+	path string
-	dir string
-}
-
-// Set - saves given directory as configuration directory.
-func (config *ConfigDir) Set(dir string) {
-	config.Lock()
-	defer config.Unlock()
-
-	config.dir = dir
-}
-
-// Get - returns current configuration directory.
-func (config *ConfigDir) Get() string {
-	config.Lock()
-	defer config.Unlock()
-
-	return config.dir
-}
-
-func (config *ConfigDir) getCertsDir() string {
-	return filepath.Join(config.Get(), certsDir)
-}
-
-// GetCADir - returns certificate CA directory.
-func (config *ConfigDir) GetCADir() string {
-	return filepath.Join(config.getCertsDir(), certsCADir)
-}
-
-// Create - creates configuration directory tree.
-func (config *ConfigDir) Create() error {
-	err := os.MkdirAll(config.GetCADir(), 0700)
-	// It is possible in kubernetes like deployments this directory
-	// is already mounted and is not writable, ignore any write errors.
-	if err != nil {
-		if os.IsPermission(err) {
-			err = nil
-		}
-	}
-	return err
-}
-
-// GetMinioConfigFile - returns absolute path of config.json file.
-func (config *ConfigDir) GetMinioConfigFile() string {
-	return filepath.Join(config.Get(), minioConfigFile)
-}
-
-// GetPublicCertFile - returns absolute path of public.crt file.
-func (config *ConfigDir) GetPublicCertFile() string {
-	return filepath.Join(config.getCertsDir(), publicCertFile)
-}
-
-// GetPrivateKeyFile - returns absolute path of private.key file.
-func (config *ConfigDir) GetPrivateKeyFile() string {
-	return filepath.Join(config.getCertsDir(), privateKeyFile)
 }
 
 func getDefaultConfigDir() string {
@@ -109,32 +54,54 @@ func getDefaultConfigDir() string {
 	return filepath.Join(homeDir, defaultMinioConfigDir)
 }
 
-var configDir = &ConfigDir{dir: getDefaultConfigDir()}
+func getDefaultCertsDir() string {
-
+	return filepath.Join(getDefaultConfigDir(), certsDir)
-func setConfigDir(dir string) {
-	configDir.Set(dir)
 }
 
-func getConfigDir() string {
+func getDefaultCertsCADir() string {
-	return configDir.Get()
+	return filepath.Join(getDefaultCertsDir(), certsCADir)
 }
 
-func getCADir() string {
+var (
-	return configDir.GetCADir()
+	// Default config, certs and CA directories.
+	defaultConfigDir  = &ConfigDir{path: getDefaultConfigDir()}
+	defaultCertsDir   = &ConfigDir{path: getDefaultCertsDir()}
+	defaultCertsCADir = &ConfigDir{path: getDefaultCertsCADir()}
+
+	// Points to current configuration directory -- deprecated, to be removed in future.
+	globalConfigDir = defaultConfigDir
+	// Points to current certs directory set by user with --certs-dir
+	globalCertsDir = defaultCertsDir
+	// Points to relative path to certs directory and is <value-of-certs-dir>/CAs
+	globalCertsCADir = defaultCertsCADir
+)
+
+// Get - returns current directory.
+func (dir *ConfigDir) Get() string {
+	return dir.path
 }
 
-func createConfigDir() error {
+// Attempts to create all directories, ignores any permission denied errors.
-	return configDir.Create()
+func mkdirAllIgnorePerm(path string) error {
+	err := os.MkdirAll(path, 0700)
+	if err != nil {
+		// It is possible in kubernetes like deployments this directory
+		// is already mounted and is not writable, ignore any write errors.
+		if os.IsPermission(err) {
+			err = nil
+		}
+	}
+	return err
 }
 
 func getConfigFile() string {
-	return configDir.GetMinioConfigFile()
+	return filepath.Join(globalConfigDir.Get(), minioConfigFile)
 }
 
 func getPublicCertFile() string {
-	return configDir.GetPublicCertFile()
+	return filepath.Join(globalCertsDir.Get(), publicCertFile)
 }
 
 func getPrivateKeyFile() string {
-	return configDir.GetPrivateKeyFile()
+	return filepath.Join(globalCertsDir.Get(), privateKeyFile)
 }

cmd/config-migrate.go

@@ -27,7 +27,6 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
 	"github.com/minio/minio/pkg/iam/policy"
@@ -231,11 +230,11 @@ func migrateConfig() error {
 
 // Version '1' is not supported anymore and deprecated, safe to delete.
 func purgeV1() error {
-	configFile := filepath.Join(getConfigDir(), "fsUsers.json")
+	configFile := filepath.Join(globalConfigDir.Get(), "fsUsers.json")
 
 	cv1 := &configV1{}
 	_, err := Load(configFile, cv1)
-	if os.IsNotExist(err) || err == dns.ErrNoEntriesFound {
+	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘1’. %v", err)
@@ -2415,6 +2414,14 @@ func migrateV27ToV28() error {
 
 // Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
 func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
+	// Construct path to config.json for the given bucket.
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+
+	// Verify if config was already available in .minio.sys in which case, nothing more to be done.
+	if err = checkConfig(context.Background(), objAPI, configFile); err != errConfigNotFound {
+		return err
+	}
+
 	defer func() {
 		// Rename config.json to config.json.deprecated only upon
 		// success of this function.
@@ -2423,8 +2430,6 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 		}
 	}()
 
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	// Construct path to config.json for the given bucket.
 	transactionConfigFile := configFile + ".transaction"
 
 	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
@@ -2436,7 +2441,7 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	}
 	defer objLock.Unlock()
 
-	// Verify if backend already has the file.
+	// Verify if backend already has the file (after holding lock)
 	if err = checkConfig(context.Background(), objAPI, configFile); err != errConfigNotFound {
 		return err
 	} // if errConfigNotFound proceed to migrate..
@@ -2448,10 +2453,14 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 		}
 		// Read from deprecate file as well if necessary.
 		if _, err = Load(getConfigFile()+".deprecated", config); err != nil {
+			if !os.IsNotExist(err) {
 				return err
 			}
+			// If all else fails simply initialize the server config.
+			return newSrvConfig(objAPI)
 		}
 
+	}
 	return saveServerConfig(context.Background(), objAPI, config)
 }
 
@@ -2459,6 +2468,15 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 func migrateMinioSysConfig(objAPI ObjectLayer) error {
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 
+	// Check if the config version is latest, if not migrate.
+	ok, _, err := checkConfigVersion(objAPI, configFile, serverConfigVersion)
+	if err != nil {
+		return err
+	}
+	if ok {
+		return nil
+	}
+
 	// Construct path to config.json for the given bucket.
 	transactionConfigFile := configFile + ".transaction"
 

cmd/config-migrate_test.go

@@ -39,7 +39,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(rootPath)
-	setConfigDir(rootPath)
+	globalConfigDir = &ConfigDir{path: rootPath}
 
 	globalObjLayerMutex.Lock()
 	globalObjectAPI = objLayer
@@ -77,7 +77,7 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 	}
 	defer os.RemoveAll(rootPath)
 
-	setConfigDir(rootPath)
+	globalConfigDir = &ConfigDir{path: rootPath}
 
 	if err := migrateV2ToV3(); err != nil {
 		t.Fatal("migrate v2 to v3 should succeed when no config file is found")
@@ -166,7 +166,8 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(rootPath)
-	setConfigDir(rootPath)
+
+	globalConfigDir = &ConfigDir{path: rootPath}
 
 	objLayer, fsDir, err := prepareFS()
 	if err != nil {
@@ -235,7 +236,8 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(rootPath)
-	setConfigDir(rootPath)
+
+	globalConfigDir = &ConfigDir{path: rootPath}
 	configPath := rootPath + "/" + minioConfigFile
 
 	// Create a corrupted config file
@@ -331,7 +333,8 @@ func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(rootPath)
-	setConfigDir(rootPath)
+
+	globalConfigDir = &ConfigDir{path: rootPath}
 	configPath := rootPath + "/" + minioConfigFile
 
 	for i := 3; i <= 17; i++ {

cmd/config-versions.go

@@ -893,7 +893,7 @@ type serverConfigV32 struct {
 	} `json:"policy"`
 }
 
-// serverConfigV33 is just like version '32', removes clientID from NATS.
+// serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir with MQTT.
 type serverConfigV33 struct {
 	quick.Config `json:"-"` // ignore interfaces
 

cmd/config.go

@@ -20,11 +20,9 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"os"
 	"path"
 	"runtime"
 	"strings"
-	"time"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/quick"
@@ -52,10 +50,7 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config *serverCon
 
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 	if globalEtcdClient != nil {
-		timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
+		return saveConfigEtcd(ctx, globalEtcdClient, configFile, data)
-		_, err = globalEtcdClient.Put(timeoutCtx, configFile, string(data))
-		defer cancel()
-		return err
 	}
 
 	// Create a backup of the current config
@@ -162,20 +157,27 @@ func initConfig(objAPI ObjectLayer) error {
 		return errServerNotInitialized
 	}
 
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+
 	if globalEtcdClient != nil {
-		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+		if err := checkConfigEtcd(context.Background(), globalEtcdClient, getConfigFile()); err != nil {
-		resp, err := globalEtcdClient.Get(ctx, getConfigFile())
+			if err == errConfigNotFound {
-		cancel()
+				// Migrates all configs at old location.
-		if err == nil && resp.Count > 0 {
 				if err = migrateConfig(); err != nil {
 					return err
 				}
-
 				// Migrates etcd ${HOME}/.minio/config.json to '/config/config.json'
-			if err := migrateConfigToMinioSys(objAPI); err != nil {
+				if err = migrateConfigToMinioSys(objAPI); err != nil {
+					return err
+				}
+			} else {
 				return err
 			}
 		}
+
+		// Watch config for changes and reloads them.
+		go watchConfigEtcd(objAPI, configFile, loadConfig)
+
 	} else {
 		if isFile(getConfigFile()) {
 			if err := migrateConfig(); err != nil {
@@ -185,30 +187,15 @@ func initConfig(objAPI ObjectLayer) error {
 		// Migrates ${HOME}/.minio/config.json or config.json.deprecated
 		// to '<export_path>/.minio.sys/config/config.json'
 		// ignore if the file doesn't exist.
-		if err := migrateConfigToMinioSys(objAPI); err != nil && !os.IsNotExist(err) {
+		if err := migrateConfigToMinioSys(objAPI); err != nil {
 			return err
 		}
-	}
-
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-
-	// Watch config for changes and reloads them in-memory.
-	go watchConfig(objAPI, configFile, loadConfig)
-
-	if err := checkConfig(context.Background(), objAPI, configFile); err != nil {
-		if err == errConfigNotFound {
-			// Config file does not exist, we create it fresh and return upon success.
-			if err = newSrvConfig(objAPI); err != nil {
-				return err
-			}
-		} else {
-			return err
-		}
-	}
 
+		// Migrates backend '<export_path>/.minio.sys/config/config.json' to latest version.
 		if err := migrateMinioSysConfig(objAPI); err != nil {
 			return err
 		}
+	}
 
 	return loadConfig(objAPI)
 }

cmd/crypto/config.go

@@ -16,5 +16,6 @@ package crypto
 
 // KMSConfig has the KMS config for hashicorp vault
 type KMSConfig struct {
+	AutoEncryption bool        `json:"-"`
 	Vault          VaultConfig `json:"vault"`
 }

cmd/crypto/metadata.go

@@ -40,6 +40,14 @@ func RemoveSensitiveEntries(metadata map[string]string) { // The functions is te
 	delete(metadata, SSECopyKey)
 }
 
+// RemoveSSEHeaders removes all crypto-specific SSE
+// header entries from the metadata map.
+func RemoveSSEHeaders(metadata map[string]string) {
+	delete(metadata, SSEHeader)
+	delete(metadata, SSECKeyMD5)
+	delete(metadata, SSECAlgorithm)
+}
+
 // RemoveInternalEntries removes all crypto-specific internal
 // metadata entries from the metadata map.
 func RemoveInternalEntries(metadata map[string]string) {

cmd/crypto/vault.go

@@ -19,31 +19,11 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
-	"os"
-	"strconv"
 	"strings"
 	"time"
 
 	vault "github.com/hashicorp/vault/api"
-)
+	"github.com/minio/minio/cmd/logger"
-
-const (
-	// vaultEndpointEnv Vault endpoint environment variable
-	vaultEndpointEnv = "MINIO_SSE_VAULT_ENDPOINT"
-	// vaultAuthTypeEnv type of vault auth to be used
-	vaultAuthTypeEnv = "MINIO_SSE_VAULT_AUTH_TYPE"
-	// vaultAppRoleIDEnv  Vault AppRole ID environment variable
-	vaultAppRoleIDEnv = "MINIO_SSE_VAULT_APPROLE_ID"
-	// vaultAppSecretIDEnv Vault AppRole Secret environment variable
-	vaultAppSecretIDEnv = "MINIO_SSE_VAULT_APPROLE_SECRET"
-	// vaultKeyVersionEnv Vault Key Version environment variable
-	vaultKeyVersionEnv = "MINIO_SSE_VAULT_KEY_VERSION"
-	// vaultKeyNameEnv Vault Encryption Key Name environment variable
-	vaultKeyNameEnv = "MINIO_SSE_VAULT_KEY_NAME"
-
-	// vaultCAPath is the path to a directory of PEM-encoded CA
-	// cert files to verify the Vault server SSL certificate.
-	vaultCAPath = "MINIO_SSE_VAULT_CAPATH"
 )
 
 var (
@@ -51,220 +31,219 @@ var (
 	ErrKMSAuthLogin = errors.New("Vault service did not return auth info")
 )
 
+// VaultKey represents vault encryption key-ring.
+type VaultKey struct {
+	Name    string `json:"name"`    // The name of the encryption key-ring
+	Version int    `json:"version"` // The key version
+}
+
+// VaultAuth represents vault authentication type.
+// Currently the only supported authentication type is AppRole.
+type VaultAuth struct {
+	Type    string       `json:"type"`    // The authentication type
+	AppRole VaultAppRole `json:"approle"` // The AppRole authentication credentials
+}
+
+// VaultAppRole represents vault AppRole authentication credentials
+type VaultAppRole struct {
+	ID     string `json:"id"`     // The AppRole access ID
+	Secret string `json:"secret"` // The AppRole secret
+}
+
+// VaultConfig represents vault configuration.
+type VaultConfig struct {
+	Endpoint  string    `json:"endpoint"` // The vault API endpoint as URL
+	CAPath    string    `json:"-"`        // The path to PEM-encoded certificate files used for mTLS. Currently not used in config file.
+	Auth      VaultAuth `json:"auth"`     // The vault authentication configuration
+	Key       VaultKey  `json:"key-id"`   // The named key used for key-generation / decryption.
+	Namespace string    `json:"-"`        // The vault namespace of enterprise vault instances
+}
+
+// vaultService represents a connection to a vault KMS.
 type vaultService struct {
 	config        *VaultConfig
 	client        *vault.Client
 	leaseDuration time.Duration
+	tokenRenewer  *vault.Renewer
 }
 
-// return transit secret engine's path for generate data key operation
+var _ KMS = (*vaultService)(nil) // compiler check that *vaultService implements KMS
-func (v *vaultService) genDataKeyEndpoint(key string) string {
+
-	return "/transit/datakey/plaintext/" + key
+// empty/default vault configuration used to check whether a particular is empty.
+var emptyVaultConfig = VaultConfig{}
+
+// IsEmpty returns true if the vault config struct is an
+// empty configuration.
+func (v *VaultConfig) IsEmpty() bool { return *v == emptyVaultConfig }
+
+// Verify returns a nil error if the vault configuration
+// is valid. A valid configuration is either empty or
+// contains valid non-default values.
+func (v *VaultConfig) Verify() (err error) {
+	if v.IsEmpty() {
+		return // an empty configuration is valid
+	}
+	switch {
+	case v.Endpoint == "":
+		err = errors.New("crypto: missing hashicorp vault endpoint")
+	case strings.ToLower(v.Auth.Type) != "approle":
+		err = fmt.Errorf("crypto: invalid hashicorp vault authentication type: %s is not supported", v.Auth.Type)
+	case v.Auth.AppRole.ID == "":
+		err = errors.New("crypto: missing hashicorp vault AppRole ID")
+	case v.Auth.AppRole.Secret == "":
+		err = errors.New("crypto: missing hashicorp vault AppSecret ID")
+	case v.Key.Name == "":
+		err = errors.New("crypto: missing hashicorp vault key name")
+	case v.Key.Version < 0:
+		err = errors.New("crypto: invalid hashicorp vault key version: The key version must not be negative")
+	}
+	return
 }
 
-// return transit secret engine's path for decrypt operation
+// NewVault initializes Hashicorp Vault KMS by authenticating
-func (v *vaultService) decryptEndpoint(key string) string {
+// to Vault with the credentials in config and gets a client
-	return "/transit/decrypt/" + key
+// token for future api calls.
-}
+func NewVault(config VaultConfig) (KMS, error) {
-
+	if config.IsEmpty() {
-// VaultKey represents vault encryption key-id name & version
+		return nil, errors.New("crypto: the hashicorp vault configuration must not be empty")
-type VaultKey struct {
-	Name    string `json:"name"`
-	Version int    `json:"version"`
-}
-
-// VaultAuth represents vault auth type to use. For now, AppRole is the only supported
-// auth type.
-type VaultAuth struct {
-	Type    string       `json:"type"`
-	AppRole VaultAppRole `json:"approle"`
-}
-
-// VaultAppRole represents vault approle credentials
-type VaultAppRole struct {
-	ID     string `json:"id"`
-	Secret string `json:"secret"`
-}
-
-// VaultConfig holds config required to start vault service
-type VaultConfig struct {
-	Endpoint string    `json:"endpoint"`
-	Auth     VaultAuth `json:"auth"`
-	Key      VaultKey  `json:"key-id"`
-}
-
-// validate whether all required env variables needed to start vault service have
-// been set
-func validateVaultConfig(c *VaultConfig) error {
-	if c.Endpoint == "" {
-		return fmt.Errorf("Missing hashicorp vault endpoint - %s is empty", vaultEndpointEnv)
 	}
-	if strings.ToLower(c.Auth.Type) != "approle" {
+	if err := config.Verify(); err != nil {
-		return fmt.Errorf("Unsupported hashicorp vault auth type - %s", vaultAuthTypeEnv)
-	}
-	if c.Auth.AppRole.ID == "" {
-		return fmt.Errorf("Missing hashicorp vault AppRole ID - %s is empty", vaultAppRoleIDEnv)
-	}
-	if c.Auth.AppRole.Secret == "" {
-		return fmt.Errorf("Missing hashicorp vault AppSecret ID - %s is empty", vaultAppSecretIDEnv)
-	}
-	if c.Key.Name == "" {
-		return fmt.Errorf("Invalid value set in environment variable %s", vaultKeyNameEnv)
-	}
-	if c.Key.Version < 0 {
-		return fmt.Errorf("Invalid value set in environment variable %s", vaultKeyVersionEnv)
-	}
-	return nil
-}
-
-// authenticate to vault with app role id and app role secret, and get a client access token, lease duration
-func getVaultAccessToken(client *vault.Client, appRoleID, appSecret string) (token string, duration int, err error) {
-	data := map[string]interface{}{
-		"role_id":   appRoleID,
-		"secret_id": appSecret,
-	}
-	resp, e := client.Logical().Write("auth/approle/login", data)
-	if e != nil {
-		return token, duration, e
-	}
-	if resp.Auth == nil {
-		return token, duration, ErrKMSAuthLogin
-	}
-	return resp.Auth.ClientToken, resp.Auth.LeaseDuration, nil
-}
-
-// NewVaultConfig sets KMSConfig from environment
-// variables and performs validations.
-func NewVaultConfig() (KMSConfig, error) {
-	kc := KMSConfig{}
-	endpoint := os.Getenv(vaultEndpointEnv)
-	roleID := os.Getenv(vaultAppRoleIDEnv)
-	roleSecret := os.Getenv(vaultAppSecretIDEnv)
-	keyName := os.Getenv(vaultKeyNameEnv)
-	keyVersion := 0
-	authType := "approle"
-	if versionStr := os.Getenv(vaultKeyVersionEnv); versionStr != "" {
-		version, err := strconv.Atoi(versionStr)
-		if err != nil {
-			return kc, fmt.Errorf("Unable to parse %s value (`%s`)", vaultKeyVersionEnv, versionStr)
-		}
-		keyVersion = version
-	}
-	// return if none of the vault env variables are configured
-	if (endpoint == "") && (roleID == "") && (roleSecret == "") && (keyName == "") && (keyVersion == 0) {
-		return kc, nil
-	}
-	c := VaultConfig{
-		Endpoint: endpoint,
-		Auth: VaultAuth{
-			Type: authType,
-			AppRole: VaultAppRole{
-				ID:     roleID,
-				Secret: roleSecret,
-			},
-		},
-		Key: VaultKey{
-			Version: keyVersion,
-			Name:    keyName,
-		},
-	}
-	if err := validateVaultConfig(&c); err != nil {
-		return kc, err
-	}
-	kc.Vault = c
-	return kc, nil
-}
-
-// NewVault initializes Hashicorp Vault KMS by
-// authenticating to Vault with the credentials in KMSConfig,
-// and gets a client token for future api calls.
-func NewVault(kmsConf KMSConfig) (KMS, error) {
-	config := kmsConf.Vault
-	vconfig := &vault.Config{
-		Address: config.Endpoint,
-	}
-	if err := vconfig.ConfigureTLS(&vault.TLSConfig{
-		CAPath: os.Getenv(vaultCAPath),
-	}); err != nil {
-		return nil, err
-	}
-	c, err := vault.NewClient(vconfig)
-	if err != nil {
 		return nil, err
 	}
 
-	if ns, ok := os.LookupEnv("MINIO_SSE_VAULT_NAMESPACE"); ok {
+	vaultCfg := vault.Config{Address: config.Endpoint}
-		c.SetNamespace(ns)
+	if err := vaultCfg.ConfigureTLS(&vault.TLSConfig{CAPath: config.CAPath}); err != nil {
+		return nil, err
 	}
-
+	client, err := vault.NewClient(&vaultCfg)
-	accessToken, leaseDuration, err := getVaultAccessToken(c, config.Auth.AppRole.ID, config.Auth.AppRole.Secret)
 	if err != nil {
 		return nil, err
 	}
-	// authenticate and get the access token
+	if config.Namespace != "" {
-	c.SetToken(accessToken)
+		client.SetNamespace(config.Namespace)
-	v := vaultService{client: c, config: &config, leaseDuration: time.Duration(leaseDuration)}
+	}
-	v.renewToken(c)
+	v := &vaultService{client: client, config: &config}
-	return &v, nil
+
+	if err := v.authenticate(); err != nil {
+		return nil, err
+	}
+	return v, nil
 }
 
-func (v *vaultService) renewToken(c *vault.Client) {
+// reauthenticate() tries to login in 1 minute
+// intervals until successful.
+func (v *vaultService) reauthenticate() {
 	retryDelay := 1 * time.Minute
 	go func() {
 		for {
-			s, err := c.Auth().Token().RenewSelf(int(v.leaseDuration))
+			if err := v.authenticate(); err != nil {
-			if err != nil {
 				time.Sleep(retryDelay)
 				continue
 			}
-			nextRenew := s.Auth.LeaseDuration / 2
+			return
-			time.Sleep(time.Duration(nextRenew) * time.Second)
 		}
 	}()
 }
 
-// Generates a random plain text key, sealed plain text key from
+// renewer calls vault client's renewer that automatically
-// Vault. It returns the plaintext key and sealed plaintext key on success
+// renews secret periodically
+func (v *vaultService) renewer(secret *vault.Secret) {
+	renewer, err := v.client.NewRenewer(&vault.RenewerInput{
+		Secret: secret,
+	})
+	if err != nil {
+		logger.FatalIf(err, "crypto: hashicorp vault token renewer could not be started")
+	}
+	v.tokenRenewer = renewer
+	go renewer.Renew()
+	defer renewer.Stop()
+
+	for {
+		select {
+		case err := <-renewer.DoneCh():
+			if err != nil {
+				v.reauthenticate()
+				renewer.Stop()
+				return
+			}
+
+			// Renewal is now over
+		case renewal := <-renewer.RenewCh():
+			v.leaseDuration = time.Duration(renewal.Secret.Auth.LeaseDuration)
+		}
+	}
+}
+
+// authenticate logs the app to vault, and starts the auto renewer
+// before secret expires
+func (v *vaultService) authenticate() (err error) {
+	payload := map[string]interface{}{
+		"role_id":   v.config.Auth.AppRole.ID,
+		"secret_id": v.config.Auth.AppRole.Secret,
+	}
+	var secret *vault.Secret
+	secret, err = v.client.Logical().Write("auth/approle/login", payload)
+	if err != nil {
+		return
+	}
+	if secret.Auth == nil {
+		err = ErrKMSAuthLogin
+		return
+	}
+	v.client.SetToken(secret.Auth.ClientToken)
+	v.leaseDuration = time.Duration(secret.Auth.LeaseDuration)
+	go v.renewer(secret)
+	return
+}
+
+// GenerateKey returns a new plaintext key, generated by the KMS,
+// and a sealed version of this plaintext key encrypted using the
+// named key referenced by keyID. It also binds the generated key
+// cryptographically to the provided context.
 func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
-	contextStream := new(bytes.Buffer)
+	var contextStream bytes.Buffer
-	ctx.WriteTo(contextStream)
+	ctx.WriteTo(&contextStream)
 
 	payload := map[string]interface{}{
 		"context": base64.StdEncoding.EncodeToString(contextStream.Bytes()),
 	}
-	s, err1 := v.client.Logical().Write(v.genDataKeyEndpoint(keyID), payload)
+	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/datakey/plaintext/%s", keyID), payload)
-
+	if err != nil {
-	if err1 != nil {
+		return key, sealedKey, err
-		return key, sealedKey, err1
 	}
 	sealKey := s.Data["ciphertext"].(string)
 	plainKey, err := base64.StdEncoding.DecodeString(s.Data["plaintext"].(string))
 	if err != nil {
-		return key, sealedKey, err1
+		return key, sealedKey, err
 	}
 	copy(key[:], []byte(plainKey))
 	return key, []byte(sealKey), nil
 }
 
-// unsealKMSKey unseals the sealedKey using the Vault master key
+// UnsealKey returns the decrypted sealedKey as plaintext key.
-// referenced by the keyID. The plain text key is returned on success.
+// Therefore it sends the sealedKey to the KMS which decrypts
+// it using the named key referenced by keyID and responses with
+// the plaintext key.
+//
+// The context must be same context as the one provided while
+// generating the plaintext key / sealedKey.
 func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
-	contextStream := new(bytes.Buffer)
+	var contextStream bytes.Buffer
-	ctx.WriteTo(contextStream)
+	ctx.WriteTo(&contextStream)
+
 	payload := map[string]interface{}{
 		"ciphertext": string(sealedKey),
 		"context":    base64.StdEncoding.EncodeToString(contextStream.Bytes()),
 	}
-	s, err1 := v.client.Logical().Write(v.decryptEndpoint(keyID), payload)
+	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/decrypt/%s", keyID), payload)
-	if err1 != nil {
+	if err != nil {
-		return key, err1
+		return key, err
 	}
 	base64Key := s.Data["plaintext"].(string)
-	plainKey, err1 := base64.StdEncoding.DecodeString(base64Key)
+	plainKey, err := base64.StdEncoding.DecodeString(base64Key)
-	if err1 != nil {
+	if err != nil {
-		return key, err1
+		return key, err
 	}
 	copy(key[:], []byte(plainKey))
-
 	return key, nil
 }

cmd/disk-cache-fs.go

@@ -91,7 +91,7 @@ func newCacheFSObjects(dir string, expiry int, maxDiskUsagePct int) (*cacheFSObj
 		appendFileMap: make(map[string]*fsAppendFile),
 	}
 
-	go fsObjects.cleanupStaleMultipartUploads(context.Background(), globalMultipartCleanupInterval, globalMultipartExpiry, globalServiceDoneCh)
+	go fsObjects.cleanupStaleMultipartUploads(context.Background(), GlobalMultipartCleanupInterval, GlobalMultipartExpiry, GlobalServiceDoneCh)
 
 	cacheFS := cacheFSObjects{
 		FSObjects:       fsObjects,
@@ -158,7 +158,7 @@ func (cfs *cacheFSObjects) purgeTrash() {
 
 	for {
 		select {
-		case <-globalServiceDoneCh:
+		case <-GlobalServiceDoneCh:
 			return
 		case <-ticker.C:
 			trashPath := path.Join(cfs.fsPath, minioMetaBucket, cacheTrashDir)

cmd/disk-cache_test.go

@@ -134,7 +134,7 @@ func TestCacheExclusion(t *testing.T) {
 		t.Fatal(err)
 	}
 	cobj := cobjects.(*cacheObjects)
-	globalServiceDoneCh <- struct{}{}
+	GlobalServiceDoneCh <- struct{}{}
 	testCases := []struct {
 		bucketName     string
 		objectName     string

cmd/dummy-object-layer_test.go

@@ -99,7 +99,7 @@ func (api *DummyObjectLayer) PutObjectPart(ctx context.Context, bucket, object,
 	return
 }
 
-func (api *DummyObjectLayer) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error) {
+func (api *DummyObjectLayer) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int, opts ObjectOptions) (result ListPartsInfo, err error) {
 	return
 }
 
@@ -119,11 +119,11 @@ func (api *DummyObjectLayer) HealFormat(ctx context.Context, dryRun bool) (item
 	return
 }
 
-func (api *DummyObjectLayer) HealBucket(ctx context.Context, bucket string, dryRun bool) (items []madmin.HealResultItem, err error) {
+func (api *DummyObjectLayer) HealBucket(ctx context.Context, bucket string, dryRun, remove bool) (items []madmin.HealResultItem, err error) {
 	return
 }
 
-func (api *DummyObjectLayer) HealObject(ctx context.Context, bucket, object string, dryRun bool) (item madmin.HealResultItem, err error) {
+func (api *DummyObjectLayer) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool) (item madmin.HealResultItem, err error) {
 	return
 }
 

cmd/encryption-v1.go

@@ -29,6 +29,7 @@ import (
 	"path"
 	"strconv"
 
+	"github.com/minio/minio-go/pkg/encrypt"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/ioutil"
@@ -55,11 +56,11 @@ const (
 	// SSEIVSize is the size of the IV data
 	SSEIVSize = 32 // 32 bytes
 
-	// SSE dare package block size.
+	// SSEDAREPackageBlockSize - SSE dare package block size.
-	sseDAREPackageBlockSize = 64 * 1024 // 64KiB bytes
+	SSEDAREPackageBlockSize = 64 * 1024 // 64KiB bytes
 
-	// SSE dare package meta padding bytes.
+	// SSEDAREPackageMetaSize - SSE dare package meta padding bytes.
-	sseDAREPackageMetaSize = 32 // 32 bytes
+	SSEDAREPackageMetaSize = 32 // 32 bytes
 
 )
 
@@ -158,14 +159,14 @@ func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map
 		crypto.SSEC.CreateMetadata(metadata, sealedKey)
 		return nil
 	case crypto.S3.IsEncrypted(metadata):
-		if globalKMS == nil {
+		if GlobalKMS == nil {
 			return errKMSNotConfigured
 		}
 		keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(metadata)
 		if err != nil {
 			return err
 		}
-		oldKey, err := globalKMS.UnsealKey(keyID, kmsKey, crypto.Context{bucket: path.Join(bucket, object)})
+		oldKey, err := GlobalKMS.UnsealKey(keyID, kmsKey, crypto.Context{bucket: path.Join(bucket, object)})
 		if err != nil {
 			return err
 		}
@@ -174,7 +175,7 @@ func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map
 			return err
 		}
 
-		newKey, encKey, err := globalKMS.GenerateKey(globalKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
+		newKey, encKey, err := GlobalKMS.GenerateKey(globalKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
 		if err != nil {
 			return err
 		}
@@ -187,10 +188,10 @@ func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map
 func newEncryptMetadata(key []byte, bucket, object string, metadata map[string]string, sseS3 bool) ([]byte, error) {
 	var sealedKey crypto.SealedKey
 	if sseS3 {
-		if globalKMS == nil {
+		if GlobalKMS == nil {
 			return nil, errKMSNotConfigured
 		}
-		key, encKey, err := globalKMS.GenerateKey(globalKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
+		key, encKey, err := GlobalKMS.GenerateKey(globalKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
 		if err != nil {
 			return nil, err
 		}
@@ -279,7 +280,7 @@ func decryptObjectInfo(key []byte, bucket, object string, metadata map[string]st
 	default:
 		return nil, errObjectTampered
 	case crypto.S3.IsEncrypted(metadata):
-		if globalKMS == nil {
+		if GlobalKMS == nil {
 			return nil, errKMSNotConfigured
 		}
 		keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(metadata)
@@ -287,7 +288,7 @@ func decryptObjectInfo(key []byte, bucket, object string, metadata map[string]st
 		if err != nil {
 			return nil, err
 		}
-		extKey, err := globalKMS.UnsealKey(keyID, kmsKey, crypto.Context{bucket: path.Join(bucket, object)})
+		extKey, err := GlobalKMS.UnsealKey(keyID, kmsKey, crypto.Context{bucket: path.Join(bucket, object)})
 		if err != nil {
 			return nil, err
 		}
@@ -395,7 +396,6 @@ func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
 	io.Reader, error) {
 
 	bucket, object := oi.Bucket, oi.Name
-
 	// Single part case
 	if !isEncryptedMultipart(oi) {
 		var reader io.Reader
@@ -411,8 +411,8 @@ func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
 		return reader, nil
 	}
 
-	partDecRelOffset := int64(seqNumber) * sseDAREPackageBlockSize
+	partDecRelOffset := int64(seqNumber) * SSEDAREPackageBlockSize
-	partEncRelOffset := int64(seqNumber) * (sseDAREPackageBlockSize + sseDAREPackageMetaSize)
+	partEncRelOffset := int64(seqNumber) * (SSEDAREPackageBlockSize + SSEDAREPackageMetaSize)
 
 	w := &DecryptBlocksReader{
 		reader:            inputReader,
@@ -477,7 +477,7 @@ type DecryptBlocksReader struct {
 	// Current part index
 	partIndex int
 	// Parts information
-	parts          []objectPartInfo
+	parts          []ObjectPartInfo
 	header         http.Header
 	bucket, object string
 	metadata       map[string]string
@@ -592,7 +592,7 @@ type DecryptBlocksWriter struct {
 	// Current part index
 	partIndex int
 	// Parts information
-	parts          []objectPartInfo
+	parts          []ObjectPartInfo
 	req            *http.Request
 	bucket, object string
 	metadata       map[string]string
@@ -743,6 +743,7 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object stri
 	}
 
 	seqNumber, encStartOffset, encLength = getEncryptedMultipartsOffsetLength(startOffset, length, objInfo)
+
 	var partStartIndex int
 	var partStartOffset = startOffset
 	// Skip parts until final offset maps to a particular part offset.
@@ -765,8 +766,8 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object stri
 		partStartOffset -= int64(decryptedSize)
 	}
 
-	startSeqNum := partStartOffset / sseDAREPackageBlockSize
+	startSeqNum := partStartOffset / SSEDAREPackageBlockSize
-	partEncRelOffset := int64(startSeqNum) * (sseDAREPackageBlockSize + sseDAREPackageMetaSize)
+	partEncRelOffset := int64(startSeqNum) * (SSEDAREPackageBlockSize + SSEDAREPackageMetaSize)
 
 	w := &DecryptBlocksWriter{
 		writer:            client,
@@ -857,9 +858,9 @@ func getEncryptedMultipartsOffsetLength(offset, length int64, obj ObjectInfo) (u
 
 // getEncryptedSinglePartOffsetLength - fetch sequence number, encrypted start offset and encrypted length.
 func getEncryptedSinglePartOffsetLength(offset, length int64, objInfo ObjectInfo) (seqNumber uint32, encOffset int64, encLength int64) {
-	onePkgSize := int64(sseDAREPackageBlockSize + sseDAREPackageMetaSize)
+	onePkgSize := int64(SSEDAREPackageBlockSize + SSEDAREPackageMetaSize)
 
-	seqNumber = uint32(offset / sseDAREPackageBlockSize)
+	seqNumber = uint32(offset / SSEDAREPackageBlockSize)
 	encOffset = int64(seqNumber) * onePkgSize
 	// The math to compute the encrypted length is always
 	// originalLength i.e (offset+length-1) to be divided under
@@ -867,10 +868,10 @@ func getEncryptedSinglePartOffsetLength(offset, length int64, objInfo ObjectInfo
 	// block. This is then multiplied by final package size which
 	// is basically 64KiB + 32. Finally negate the encrypted offset
 	// to get the final encrypted length on disk.
-	encLength = ((offset+length)/sseDAREPackageBlockSize)*onePkgSize - encOffset
+	encLength = ((offset+length)/SSEDAREPackageBlockSize)*onePkgSize - encOffset
 
 	// Check for the remainder, to figure if we need one extract package to read from.
-	if (offset+length)%sseDAREPackageBlockSize > 0 {
+	if (offset+length)%SSEDAREPackageBlockSize > 0 {
 		encLength += onePkgSize
 	}
 
@@ -1041,11 +1042,11 @@ func (o *ObjectInfo) GetDecryptedRange(rs *HTTPRangeSpec) (encOff, encLength, sk
 	// partStart is always found in the loop above,
 	// because off is validated.
 
-	sseDAREEncPackageBlockSize := int64(sseDAREPackageBlockSize + sseDAREPackageMetaSize)
+	sseDAREEncPackageBlockSize := int64(SSEDAREPackageBlockSize + SSEDAREPackageMetaSize)
-	startPkgNum := (off - cumulativeSum) / sseDAREPackageBlockSize
+	startPkgNum := (off - cumulativeSum) / SSEDAREPackageBlockSize
 
 	// Now we can calculate the number of bytes to skip
-	skipLen = (off - cumulativeSum) % sseDAREPackageBlockSize
+	skipLen = (off - cumulativeSum) % SSEDAREPackageBlockSize
 
 	encOff = encCumulativeSum + startPkgNum*sseDAREEncPackageBlockSize
 	// Locate the part containing the end of the required range
@@ -1062,7 +1063,7 @@ func (o *ObjectInfo) GetDecryptedRange(rs *HTTPRangeSpec) (encOff, encLength, sk
 	}
 	// partEnd is always found in the loop above, because off and
 	// length are validated.
-	endPkgNum := (endOffset - cumulativeSum) / sseDAREPackageBlockSize
+	endPkgNum := (endOffset - cumulativeSum) / SSEDAREPackageBlockSize
 	// Compute endEncOffset with one additional DARE package (so
 	// we read the package containing the last desired byte).
 	endEncOffset := encCumulativeSum + (endPkgNum+1)*sseDAREEncPackageBlockSize
@@ -1157,3 +1158,113 @@ func DecryptObjectInfo(info *ObjectInfo, headers http.Header) (encrypted bool, e
 	}
 	return
 }
+
+// The customer key in the header is used by the gateway for encryption in the case of
+// s3 gateway double encryption. A new client key is derived from the customer provided
+// key to be sent to the s3 backend for encryption at the backend.
+func deriveClientKey(clientKey [32]byte, bucket, object string) [32]byte {
+	var key [32]byte
+	mac := hmac.New(sha256.New, clientKey[:])
+	mac.Write([]byte(crypto.SSEC.String()))
+	mac.Write([]byte(path.Join(bucket, object)))
+	mac.Sum(key[:0])
+	return key
+}
+
+// extract encryption options for pass through to backend in the case of gateway
+func extractEncryptionOption(header http.Header, copySource bool, metadata map[string]string) (opts ObjectOptions, err error) {
+	var clientKey [32]byte
+	var sse encrypt.ServerSide
+
+	if copySource {
+		if crypto.SSECopy.IsRequested(header) {
+			clientKey, err = crypto.SSECopy.ParseHTTP(header)
+			if err != nil {
+				return
+			}
+			if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil {
+				return
+			}
+			return ObjectOptions{ServerSideEncryption: encrypt.SSECopy(sse)}, nil
+		}
+		return
+	}
+
+	if crypto.SSEC.IsRequested(header) {
+		clientKey, err = crypto.SSEC.ParseHTTP(header)
+		if err != nil {
+			return
+		}
+		if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil {
+			return
+		}
+		return ObjectOptions{ServerSideEncryption: sse}, nil
+	}
+	if crypto.S3.IsRequested(header) || (metadata != nil && crypto.S3.IsEncrypted(metadata)) {
+		return ObjectOptions{ServerSideEncryption: encrypt.NewSSE()}, nil
+	}
+	return opts, nil
+}
+
+// get ObjectOptions for GET calls from encryption headers
+func getEncryptionOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) {
+	var (
+		encryption encrypt.ServerSide
+		opts       ObjectOptions
+	)
+	if GlobalGatewaySSE.SSEC() && crypto.SSEC.IsRequested(r.Header) {
+		key, err := crypto.SSEC.ParseHTTP(r.Header)
+		if err != nil {
+			return opts, err
+		}
+		derivedKey := deriveClientKey(key, bucket, object)
+		encryption, err = encrypt.NewSSEC(derivedKey[:])
+		logger.CriticalIf(ctx, err)
+		return ObjectOptions{ServerSideEncryption: encryption}, nil
+	}
+	// default case of passing encryption headers to backend
+	return extractEncryptionOption(r.Header, false, nil)
+}
+
+// get ObjectOptions for PUT calls from encryption headers
+func putEncryptionOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
+	// In the case of multipart custom format, the metadata needs to be checked in addition to header to see if it
+	// is SSE-S3 encrypted, primarily because S3 protocol does not require SSE-S3 headers in PutObjectPart calls
+	if GlobalGatewaySSE.SSES3() && (crypto.S3.IsRequested(r.Header) || crypto.S3.IsEncrypted(metadata)) {
+		return ObjectOptions{ServerSideEncryption: encrypt.NewSSE()}, nil
+	}
+	if GlobalGatewaySSE.SSEC() && crypto.SSEC.IsRequested(r.Header) {
+		return getEncryptionOpts(ctx, r, bucket, object)
+	}
+	// default case of passing encryption headers to backend
+	return extractEncryptionOption(r.Header, false, metadata)
+}
+
+// get ObjectOptions for Copy calls for encryption headers provided on the target side
+func copyDstEncryptionOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
+	return putEncryptionOpts(ctx, r, bucket, object, metadata)
+}
+
+// get ObjectOptions for Copy calls for encryption headers provided on the source side
+func copySrcEncryptionOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) {
+	var (
+		ssec encrypt.ServerSide
+		opts ObjectOptions
+	)
+
+	if GlobalGatewaySSE.SSEC() && crypto.SSECopy.IsRequested(r.Header) {
+		key, err := crypto.SSECopy.ParseHTTP(r.Header)
+		if err != nil {
+			return opts, err
+		}
+		derivedKey := deriveClientKey(key, bucket, object)
+		ssec, err = encrypt.NewSSEC(derivedKey[:])
+		if err != nil {
+			return opts, err
+		}
+		return ObjectOptions{ServerSideEncryption: encrypt.SSECopy(ssec)}, nil
+	}
+
+	// default case of passing encryption headers to backend
+	return extractEncryptionOption(r.Header, true, nil)
+}

cmd/encryption-v1_test.go

@@ -18,10 +18,12 @@ package cmd
 
 import (
 	"bytes"
+	"encoding/base64"
 	"net/http"
 	"testing"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/minio-go/pkg/encrypt"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/sio"
 )
@@ -355,7 +357,7 @@ func TestGetDecryptedRange_Issue50(t *testing.T) {
 			"content-type":                         "application/octet-stream",
 			"etag":                                 "166b1545b4c1535294ee0686678bea8c-2",
 		},
-		Parts: []objectPartInfo{
+		Parts: []ObjectPartInfo{
 			{
 				Number:     1,
 				Name:       "part.1",
@@ -503,7 +505,7 @@ func TestGetDecryptedRange(t *testing.T) {
 	var (
 		// make a multipart object-info given part sizes
 		mkMPObj = func(sizes []int64) ObjectInfo {
-			r := make([]objectPartInfo, len(sizes))
+			r := make([]ObjectPartInfo, len(sizes))
 			sum := int64(0)
 			for i, s := range sizes {
 				r[i].Number = i
@@ -675,3 +677,84 @@ func TestGetDecryptedRange(t *testing.T) {
 
 	}
 }
+
+var extractEncryptionOptionTests = []struct {
+	headers        http.Header
+	copySource     bool
+	metadata       map[string]string
+	encryptionType encrypt.Type
+	err            error
+}{
+	{headers: http.Header{crypto.SSECAlgorithm: []string{"AES256"},
+		crypto.SSECKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+		crypto.SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
+		copySource:     false,
+		metadata:       nil,
+		encryptionType: encrypt.SSEC,
+		err:            nil}, // 0
+	{headers: http.Header{crypto.SSECAlgorithm: []string{"AES256"},
+		crypto.SSECKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+		crypto.SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
+		copySource:     true,
+		metadata:       nil,
+		encryptionType: "",
+		err:            nil}, // 1
+	{headers: http.Header{crypto.SSECAlgorithm: []string{"AES256"},
+		crypto.SSECKey:    []string{"Mz"},
+		crypto.SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
+		copySource:     false,
+		metadata:       nil,
+		encryptionType: "",
+		err:            crypto.ErrInvalidCustomerKey}, // 2
+	{headers: http.Header{crypto.SSEHeader: []string{"AES256"}},
+		copySource:     false,
+		metadata:       nil,
+		encryptionType: encrypt.S3,
+		err:            nil}, // 3
+	{headers: http.Header{},
+		copySource: false,
+		metadata: map[string]string{crypto.S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)),
+			crypto.S3KMSKeyID:     "kms-key",
+			crypto.S3KMSSealedKey: "m-key"},
+		encryptionType: encrypt.S3,
+		err:            nil}, // 4
+	{headers: http.Header{},
+		copySource: true,
+		metadata: map[string]string{crypto.S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)),
+			crypto.S3KMSKeyID:     "kms-key",
+			crypto.S3KMSSealedKey: "m-key"},
+		encryptionType: "",
+		err:            nil}, // 5
+	{headers: http.Header{crypto.SSECopyAlgorithm: []string{"AES256"},
+		crypto.SSECopyKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+		crypto.SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
+		copySource:     true,
+		metadata:       nil,
+		encryptionType: encrypt.SSEC,
+		err:            nil}, // 6
+	{headers: http.Header{crypto.SSECopyAlgorithm: []string{"AES256"},
+		crypto.SSECopyKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+		crypto.SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
+		copySource:     false,
+		metadata:       nil,
+		encryptionType: "",
+		err:            nil}, // 7
+}
+
+func TestExtractEncryptionOptions(t *testing.T) {
+	for i, test := range extractEncryptionOptionTests {
+		opts, err := extractEncryptionOption(test.headers, test.copySource, test.metadata)
+		if test.err != err {
+			t.Errorf("Case %d: expected err: %v , actual err: %v", i, test.err, err)
+		}
+		if err == nil {
+			if opts.ServerSideEncryption == nil && test.encryptionType != "" {
+				t.Errorf("Case %d: expected opts to be of %v encryption type", i, test.encryptionType)
+
+			}
+			if opts.ServerSideEncryption != nil && test.encryptionType != opts.ServerSideEncryption.Type() {
+				t.Errorf("Case %d: expected opts to have encryption type %v but was %v ", i, test.encryptionType, opts.ServerSideEncryption.Type())
+			}
+		}
+	}
+}

cmd/endpoint-ellipses_test.go

@@ -226,6 +226,17 @@ func TestGetSetIndexes(t *testing.T) {
 	}
 }
 
+func getHexSequences(start int, number int, paddinglen int) (seq []string) {
+	for i := start; i <= number; i++ {
+		if paddinglen == 0 {
+			seq = append(seq, fmt.Sprintf("%x", i))
+		} else {
+			seq = append(seq, fmt.Sprintf(fmt.Sprintf("%%0%dx", paddinglen), i))
+		}
+	}
+	return seq
+}
+
 func getSequences(start int, number int, paddinglen int) (seq []string) {
 	for i := start; i <= number; i++ {
 		if paddinglen == 0 {
@@ -472,6 +483,52 @@ func TestParseEndpointSet(t *testing.T) {
 			},
 			true,
 		},
+		// IPv6 ellipses with hexadecimal expansion
+		{
+			"http://[2001:3984:3989::{1...a}]/disk{1...10}",
+			endpointSet{
+				[]ellipses.ArgPattern{
+					[]ellipses.Pattern{
+						{
+							Prefix: "",
+							Suffix: "",
+							Seq:    getSequences(1, 10, 0),
+						},
+						{
+							Prefix: "http://[2001:3984:3989::",
+							Suffix: "]/disk",
+							Seq:    getHexSequences(1, 10, 0),
+						},
+					},
+				},
+				nil,
+				[][]uint64{{10, 10, 10, 10, 10, 10, 10, 10, 10, 10}},
+			},
+			true,
+		},
+		// IPv6 ellipses with hexadecimal expansion with 3 position numerics.
+		{
+			"http://[2001:3984:3989::{001...00a}]/disk{1...10}",
+			endpointSet{
+				[]ellipses.ArgPattern{
+					[]ellipses.Pattern{
+						{
+							Prefix: "",
+							Suffix: "",
+							Seq:    getSequences(1, 10, 0),
+						},
+						{
+							Prefix: "http://[2001:3984:3989::",
+							Suffix: "]/disk",
+							Seq:    getHexSequences(1, 10, 3),
+						},
+					},
+				},
+				nil,
+				[][]uint64{{10, 10, 10, 10, 10, 10, 10, 10, 10, 10}},
+			},
+			true,
+		},
 	}
 
 	for i, testCase := range testCases {

cmd/endpoint.go

@@ -29,6 +29,9 @@ import (
 
 	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/cpu"
+	"github.com/minio/minio/pkg/disk"
+	"github.com/minio/minio/pkg/mem"
 	"github.com/minio/minio/pkg/mountinfo"
 )
 
@@ -112,6 +115,9 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 				return ep, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")
 			}
 		}
+		if i := strings.Index(host, "%"); i > -1 {
+			host = host[:i]
+		}
 
 		if host == "" {
 			return ep, fmt.Errorf("invalid URL endpoint format: empty host name")
@@ -152,7 +158,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 		// Only check if the arg is an ip address and ask for scheme since its absent.
 		// localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as
 		// /mnt/export1. So we go ahead and start the minio server in FS modes in these cases.
-		if isHostIPv4(arg) {
+		if isHostIP(arg) {
 			return ep, fmt.Errorf("invalid URL endpoint format: missing scheme http or https")
 		}
 		u = &url.URL{Path: path.Clean(arg)}
@@ -194,6 +200,90 @@ func (endpoints EndpointList) GetString(i int) string {
 	return endpoints[i].String()
 }
 
+// localEndpointsMemUsage - returns ServerMemUsageInfo for only the
+// local endpoints from given list of endpoints
+func localEndpointsMemUsage(endpoints EndpointList) ServerMemUsageInfo {
+	var memUsages []mem.Usage
+	var historicUsages []mem.Usage
+	var addr string
+	scratchSpace := map[string]bool{}
+	for _, endpoint := range endpoints {
+		// Only proceed for local endpoints
+		if endpoint.IsLocal {
+			if _, ok := scratchSpace[endpoint.Host]; ok {
+				continue
+			}
+			addr = GetLocalPeer(endpoints)
+			memUsage := mem.GetUsage()
+			memUsages = append(memUsages, memUsage)
+			historicUsage := mem.GetHistoricUsage()
+			historicUsages = append(historicUsages, historicUsage)
+			scratchSpace[endpoint.Host] = true
+		}
+	}
+	return ServerMemUsageInfo{
+		Addr:          addr,
+		Usage:         memUsages,
+		HistoricUsage: historicUsages,
+	}
+}
+
+// localEndpointsCPULoad - returns ServerCPULoadInfo for only the
+// local endpoints from given list of endpoints
+func localEndpointsCPULoad(endpoints EndpointList) ServerCPULoadInfo {
+	var cpuLoads []cpu.Load
+	var historicLoads []cpu.Load
+	var addr string
+	scratchSpace := map[string]bool{}
+	for _, endpoint := range endpoints {
+		// Only proceed for local endpoints
+		if endpoint.IsLocal {
+			if _, ok := scratchSpace[endpoint.Host]; ok {
+				continue
+			}
+			addr = GetLocalPeer(endpoints)
+			cpuLoad := cpu.GetLoad()
+			cpuLoads = append(cpuLoads, cpuLoad)
+			historicLoad := cpu.GetHistoricLoad()
+			historicLoads = append(historicLoads, historicLoad)
+			scratchSpace[endpoint.Host] = true
+		}
+	}
+	return ServerCPULoadInfo{
+		Addr:         addr,
+		Load:         cpuLoads,
+		HistoricLoad: historicLoads,
+	}
+}
+
+// localEndpointsDrivePerf - returns ServerDrivesPerfInfo for only the
+// local endpoints from given list of endpoints
+func localEndpointsDrivePerf(endpoints EndpointList) ServerDrivesPerfInfo {
+	var dps []disk.Performance
+	var addr string
+	for _, endpoint := range endpoints {
+		// Only proceed for local endpoints
+		if endpoint.IsLocal {
+			addr = GetLocalPeer(endpoints)
+			if _, err := os.Stat(endpoint.Path); err != nil {
+				// Since this drive is not available, add relevant details and proceed
+				dps = append(dps, disk.Performance{Path: endpoint.Path, Error: err.Error()})
+				continue
+			}
+			tempObj := mustGetUUID()
+			fsPath := pathJoin(endpoint.Path, minioMetaTmpBucket, tempObj)
+			dp := disk.GetPerformance(fsPath)
+			dp.Path = endpoint.Path
+			dps = append(dps, dp)
+		}
+	}
+
+	return ServerDrivesPerfInfo{
+		Addr: addr,
+		Perf: dps,
+	}
+}
+
 // NewEndpointList - returns new endpoint list based on input args.
 func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
 	var endpointType EndpointType
@@ -224,7 +314,6 @@ func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
 		uniqueArgs.Add(arg)
 		endpoints = append(endpoints, endpoint)
 	}
-
 	return endpoints, nil
 }
 
@@ -341,7 +430,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 			if err != nil {
 				host = endpoint.Host
 			}
-			hostIPSet, _ := getHostIP4(host)
+			hostIPSet, _ := getHostIP(host)
 			if IPSet, ok := pathIPMap[endpoint.Path]; ok {
 				if !IPSet.Intersection(hostIPSet).IsEmpty() {
 					return serverAddr, endpoints, setupType,
@@ -411,12 +500,12 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 				host = localServerAddr
 			}
 
-			ipList, err := getHostIP4(host)
+			ipList, err := getHostIP(host)
 			logger.FatalIf(err, "unexpected error when resolving host '%s'", host)
 
-			// Filter ipList by IPs those start with '127.'.
+			// Filter ipList by IPs those start with '127.' or '::1'
 			loopBackIPs := ipList.FuncMatch(func(ip string, matchString string) bool {
-				return strings.HasPrefix(ip, "127.")
+				return strings.HasPrefix(ip, "127.") || strings.HasPrefix(ip, "::1")
 			}, "")
 
 			// If loop back IP is found and ipList contains only loop back IPs, then error out.
@@ -485,10 +574,10 @@ func GetLocalPeer(endpoints EndpointList) (localPeer string) {
 		// Local peer can be empty in FS or Erasure coded mode.
 		// If so, return globalMinioHost + globalMinioPort value.
 		if globalMinioHost != "" {
-			return globalMinioHost + ":" + globalMinioPort
+			return net.JoinHostPort(globalMinioHost, globalMinioPort)
 		}
 
-		return "127.0.0.1:" + globalMinioPort
+		return net.JoinHostPort("127.0.0.1", globalMinioPort)
 	}
 	return peerSet.ToSlice()[0]
 }
@@ -525,10 +614,10 @@ func updateDomainIPs(endPoints set.StringSet) {
 				continue
 			}
 		}
-		IPs, _ := getHostIP4(host)
+		IPs, _ := getHostIP(host)
 		ipList = ipList.Union(IPs)
 	}
 	globalDomainIPs = ipList.FuncMatch(func(ip string, matchString string) bool {
-		return !strings.HasPrefix(ip, "127.")
+		return !strings.HasPrefix(ip, "127.") || strings.HasPrefix(ip, "::1")
 	}, "")
 }

cmd/endpoint_test.go

@@ -304,27 +304,27 @@ func TestCreateEndpoints(t *testing.T) {
 		}, DistXLSetupType, nil},
 	}
 
-	for _, testCase := range testCases {
+	for i, testCase := range testCases {
 		serverAddr, endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...)
 
 		if err == nil {
 			if testCase.expectedErr != nil {
-				t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
+				t.Fatalf("Test (%d) error: expected = %v, got = <nil>", i+1, testCase.expectedErr)
 			} else {
 				if serverAddr != testCase.expectedServerAddr {
-					t.Fatalf("serverAddr: expected = %v, got = %v", testCase.expectedServerAddr, serverAddr)
+					t.Fatalf("Test (%d) serverAddr: expected = %v, got = %v", i+1, testCase.expectedServerAddr, serverAddr)
 				}
 				if !reflect.DeepEqual(endpoints, testCase.expectedEndpoints) {
-					t.Fatalf("endpoints: expected = %v, got = %v", testCase.expectedEndpoints, endpoints)
+					t.Fatalf("Test (%d) endpoints: expected = %v, got = %v", i+1, testCase.expectedEndpoints, endpoints)
 				}
 				if setupType != testCase.expectedSetupType {
-					t.Fatalf("setupType: expected = %v, got = %v", testCase.expectedSetupType, setupType)
+					t.Fatalf("Test (%d) setupType: expected = %v, got = %v", i+1, testCase.expectedSetupType, setupType)
 				}
 			}
 		} else if testCase.expectedErr == nil {
-			t.Fatalf("error: expected = <nil>, got = %v", err)
+			t.Fatalf("Test (%d) error: expected = <nil>, got = %v", i+1, err)
 		} else if err.Error() != testCase.expectedErr.Error() {
-			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
+			t.Fatalf("Test (%d) error: expected = %v, got = %v", i+1, testCase.expectedErr, err)
 		}
 	}
 }

cmd/environment.go

@@ -0,0 +1,182 @@
+// Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/minio/minio/cmd/crypto"
+)
+
+const (
+	// EnvKMSMasterKey is the environment variable used to specify
+	// a KMS master key used to protect SSE-S3 per-object keys.
+	// Valid values must be of the from: "KEY_ID:32_BYTE_HEX_VALUE".
+	EnvKMSMasterKey = "MINIO_SSE_MASTER_KEY"
+
+	// EnvAutoEncryption is the environment variable used to en/disable
+	// SSE-S3 auto-encryption. SSE-S3 auto-encryption, if enabled,
+	// requires a valid KMS configuration and turns any non-SSE-C
+	// request into an SSE-S3 request.
+	// If present EnvAutoEncryption must be either "on" or "off".
+	EnvAutoEncryption = "MINIO_SSE_AUTO_ENCRYPTION"
+)
+
+const (
+	// EnvVaultEndpoint is the environment variable used to specify
+	// the vault HTTPS endpoint.
+	EnvVaultEndpoint = "MINIO_SSE_VAULT_ENDPOINT"
+
+	// EnvVaultAuthType is the environment variable used to specify
+	// the authentication type for vault.
+	EnvVaultAuthType = "MINIO_SSE_VAULT_AUTH_TYPE"
+
+	// EnvVaultAppRoleID is the environment variable used to specify
+	// the vault AppRole ID.
+	EnvVaultAppRoleID = "MINIO_SSE_VAULT_APPROLE_ID"
+
+	// EnvVaultAppSecretID is the environment variable used to specify
+	// the vault AppRole secret corresponding to the AppRole ID.
+	EnvVaultAppSecretID = "MINIO_SSE_VAULT_APPROLE_SECRET"
+
+	// EnvVaultKeyVersion is the environment variable used to specify
+	// the vault key version.
+	EnvVaultKeyVersion = "MINIO_SSE_VAULT_KEY_VERSION"
+
+	// EnvVaultKeyName is the environment variable used to specify
+	// the vault named key-ring. In the S3 context it's referred as
+	// customer master key ID (CMK-ID).
+	EnvVaultKeyName = "MINIO_SSE_VAULT_KEY_NAME"
+
+	// EnvVaultCAPath is the environment variable used to specify the
+	// path to a directory of PEM-encoded CA cert files. These CA cert
+	// files are used to authenticate Minio to Vault over mTLS.
+	EnvVaultCAPath = "MINIO_SSE_VAULT_CAPATH"
+
+	// EnvVaultNamespace is the environment variable used to specify
+	// vault namespace. The vault namespace is used if the enterprise
+	// version of Hashicorp Vault is used.
+	EnvVaultNamespace = "MINIO_SSE_VAULT_NAMESPACE"
+)
+
+// Environment provides functions for accessing environment
+// variables.
+var Environment = environment{}
+
+type environment struct{}
+
+// Get retrieves the value of the environment variable named
+// by the key. If the variable is present in the environment the
+// value (which may be empty) is returned. Otherwise it returns
+// the specified default value.
+func (environment) Get(key, defaultValue string) string {
+	if v, ok := os.LookupEnv(key); ok {
+		return v
+	}
+	return defaultValue
+}
+
+// Lookup retrieves the value of the environment variable named
+// by the key. If the variable is present in the environment the
+// value (which may be empty) is returned and the boolean is true.
+// Otherwise the returned value will be empty and the boolean will
+// be false.
+func (environment) Lookup(key string) (string, bool) { return os.LookupEnv(key) }
+
+// LookupKMSConfig extracts the KMS configuration provided by environment
+// variables and merge them with the provided KMS configuration. The
+// merging follows the following rules:
+//
+// 1. A valid value provided as environment variable is higher prioritized
+// than the provided configuration and overwrites the value from the
+// configuration file.
+//
+// 2. A value specified as environment variable never changes the configuration
+// file. So it is never made a persistent setting.
+//
+// It sets the global KMS configuration according to the merged configuration
+// on success.
+func (env environment) LookupKMSConfig(config crypto.KMSConfig) (err error) {
+	// Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present
+	config.Vault.Endpoint = env.Get(EnvVaultEndpoint, config.Vault.Endpoint)
+	config.Vault.CAPath = env.Get(EnvVaultCAPath, config.Vault.CAPath)
+	config.Vault.Auth.Type = env.Get(EnvVaultAuthType, config.Vault.Auth.Type)
+	config.Vault.Auth.AppRole.ID = env.Get(EnvVaultAppRoleID, config.Vault.Auth.AppRole.ID)
+	config.Vault.Auth.AppRole.Secret = env.Get(EnvVaultAppSecretID, config.Vault.Auth.AppRole.Secret)
+	config.Vault.Key.Name = env.Get(EnvVaultKeyName, config.Vault.Key.Name)
+	config.Vault.Namespace = env.Get(EnvVaultNamespace, config.Vault.Namespace)
+	keyVersion := env.Get(EnvVaultKeyVersion, strconv.Itoa(config.Vault.Key.Version))
+	config.Vault.Key.Version, err = strconv.Atoi(keyVersion)
+	if err != nil {
+		return fmt.Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)", EnvVaultKeyVersion, keyVersion)
+	}
+	if err = config.Vault.Verify(); err != nil {
+		return err
+	}
+
+	// Lookup KMS master keys - only available through ENV.
+	if masterKey, ok := env.Lookup(EnvKMSMasterKey); ok {
+		if !config.Vault.IsEmpty() { // Vault and KMS master key provided
+			return errors.New("Ambiguous KMS configuration: vault configuration and a master key are provided at the same time")
+		}
+		globalKMSKeyID, GlobalKMS, err = parseKMSMasterKey(masterKey)
+		if err != nil {
+			return err
+		}
+	}
+	if !config.Vault.IsEmpty() {
+		GlobalKMS, err = crypto.NewVault(config.Vault)
+		if err != nil {
+			return err
+		}
+		globalKMSKeyID = config.Vault.Key.Name
+	}
+
+	autoEncryption, err := ParseBoolFlag(env.Get(EnvAutoEncryption, "off"))
+	if err != nil {
+		return err
+	}
+	globalAutoEncryption = bool(autoEncryption)
+	if globalAutoEncryption && GlobalKMS == nil { // auto-encryption enabled but no KMS
+		return errors.New("Invalid KMS configuration: auto-encryption is enabled but no valid KMS configuration is present")
+	}
+	return nil
+}
+
+// parseKMSMasterKey parses the value of the environment variable
+// `EnvKMSMasterKey` and returns a key-ID and a master-key KMS on success.
+func parseKMSMasterKey(envArg string) (string, crypto.KMS, error) {
+	values := strings.SplitN(envArg, ":", 2)
+	if len(values) != 2 {
+		return "", nil, fmt.Errorf("Invalid KMS master key: %s does not contain a ':'", envArg)
+	}
+	var (
+		keyID  = values[0]
+		hexKey = values[1]
+	)
+	if len(hexKey) != 64 { // 2 hex bytes = 1 byte
+		return "", nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
+	}
+	var masterKey [32]byte
+	if _, err := hex.Decode(masterKey[:], []byte(hexKey)); err != nil {
+		return "", nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
+	}
+	return keyID, crypto.NewKMS(masterKey), nil
+}

cmd/erasure-decode.go

@@ -19,29 +19,30 @@ package cmd
 import (
 	"context"
 	"io"
+	"sync"
 
 	"github.com/minio/minio/cmd/logger"
 )
 
-// Reads in parallel from bitrotReaders.
+// Reads in parallel from readers.
 type parallelReader struct {
-	readers       []*bitrotReader
+	readers       []io.ReaderAt
 	dataBlocks    int
 	offset        int64
 	shardSize     int64
 	shardFileSize int64
+	buf           [][]byte
 }
 
 // newParallelReader returns parallelReader.
-func newParallelReader(readers []*bitrotReader, dataBlocks int, offset int64, fileSize int64, blocksize int64) *parallelReader {
+func newParallelReader(readers []io.ReaderAt, e Erasure, offset, totalLength int64) *parallelReader {
-	shardSize := ceilFrac(blocksize, int64(dataBlocks))
-	shardFileSize := getErasureShardFileSize(blocksize, fileSize, dataBlocks)
 	return &parallelReader{
 		readers,
-		dataBlocks,
+		e.dataBlocks,
-		(offset / blocksize) * shardSize,
+		(offset / e.blockSize) * e.ShardSize(),
-		shardSize,
+		e.ShardSize(),
-		shardFileSize,
+		e.ShardFileSize(totalLength),
+		make([][]byte, len(readers)),
 	}
 }
 
@@ -56,79 +57,83 @@ func (p *parallelReader) canDecode(buf [][]byte) bool {
 	return bufCount >= p.dataBlocks
 }
 
-// Read reads from bitrotReaders in parallel. Returns p.dataBlocks number of bufs.
+// Read reads from readers in parallel. Returns p.dataBlocks number of bufs.
 func (p *parallelReader) Read() ([][]byte, error) {
-	type errIdx struct {
-		idx int
-		buf []byte
-		err error
-	}
-
-	errCh := make(chan errIdx)
-	currReaderIndex := 0
 	newBuf := make([][]byte, len(p.readers))
+	var newBufLK sync.RWMutex
 
 	if p.offset+p.shardSize > p.shardFileSize {
 		p.shardSize = p.shardFileSize - p.offset
 	}
 
-	read := func(currReaderIndex int) {
+	readTriggerCh := make(chan bool, len(p.readers))
-		b, err := p.readers[currReaderIndex].ReadChunk(p.offset, p.shardSize)
+	for i := 0; i < p.dataBlocks; i++ {
-		errCh <- errIdx{currReaderIndex, b, err}
+		// Setup read triggers for p.dataBlocks number of reads so that it reads in parallel.
+		readTriggerCh <- true
 	}
 
-	readerCount := 0
+	readerIndex := 0
-	for _, r := range p.readers {
+	var wg sync.WaitGroup
-		if r != nil {
+	// if readTrigger is true, it implies next disk.ReadAt() should be tried
-			readerCount++
+	// if readTrigger is false, it implies previous disk.ReadAt() was successful and there is no need
-		}
+	// to try reading the next disk.
-	}
+	for readTrigger := range readTriggerCh {
-	if readerCount < p.dataBlocks {
+		newBufLK.RLock()
-		return nil, errXLReadQuorum
+		canDecode := p.canDecode(newBuf)
-	}
+		newBufLK.RUnlock()
-
+		if canDecode {
-	readerCount = 0
-	for i, r := range p.readers {
-		if r == nil {
-			continue
-		}
-		go read(i)
-		readerCount++
-		if readerCount == p.dataBlocks {
-			currReaderIndex = i + 1
 			break
 		}
+		if readerIndex == len(p.readers) {
+			break
 		}
+		if !readTrigger {
+			continue
+		}
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			disk := p.readers[i]
+			if disk == nil {
+				// Since disk is nil, trigger another read.
+				readTriggerCh <- true
+				return
+			}
+			if p.buf[i] == nil {
+				// Reading first time on this disk, hence the buffer needs to be allocated.
+				// Subsequent reads will re-use this buffer.
+				p.buf[i] = make([]byte, p.shardSize)
+			}
+			// For the last shard, the shardsize might be less than previous shard sizes.
+			// Hence the following statement ensures that the buffer size is reset to the right size.
+			p.buf[i] = p.buf[i][:p.shardSize]
+			_, err := disk.ReadAt(p.buf[i], p.offset)
+			if err != nil {
+				p.readers[i] = nil
+				// Since ReadAt returned error, trigger another read.
+				readTriggerCh <- true
+				return
+			}
+			newBufLK.Lock()
+			newBuf[i] = p.buf[i]
+			newBufLK.Unlock()
+			// Since ReadAt returned success, there is no need to trigger another read.
+			readTriggerCh <- false
+		}(readerIndex)
+		readerIndex++
+	}
+	wg.Wait()
 
-	for errVal := range errCh {
-		if errVal.err == nil {
-			newBuf[errVal.idx] = errVal.buf
 	if p.canDecode(newBuf) {
-				p.offset += int64(p.shardSize)
+		p.offset += p.shardSize
 		return newBuf, nil
 	}
-			continue
-		}
-		p.readers[errVal.idx] = nil
-		for currReaderIndex < len(p.readers) {
-			if p.readers[currReaderIndex] != nil {
-				break
-			}
-			currReaderIndex++
-		}
-
-		if currReaderIndex == len(p.readers) {
-			break
-		}
-		go read(currReaderIndex)
-		currReaderIndex++
-	}
 
 	return nil, errXLReadQuorum
 }
 
 // Decode reads from readers, reconstructs data if needed and writes the data to the writer.
-func (e Erasure) Decode(ctx context.Context, writer io.Writer, readers []*bitrotReader, offset, length, totalLength int64) error {
+func (e Erasure) Decode(ctx context.Context, writer io.Writer, readers []io.ReaderAt, offset, length, totalLength int64) error {
 	if offset < 0 || length < 0 {
 		logger.LogIf(ctx, errInvalidArgument)
 		return errInvalidArgument
@@ -141,7 +146,7 @@ func (e Erasure) Decode(ctx context.Context, writer io.Writer, readers []*bitrot
 		return nil
 	}
 
-	reader := newParallelReader(readers, e.dataBlocks, offset, totalLength, e.blockSize)
+	reader := newParallelReader(readers, e, offset, totalLength)
 
 	startBlock := offset / e.blockSize
 	endBlock := (offset + length) / e.blockSize

cmd/erasure-decode_test.go

@@ -45,7 +45,8 @@ var erasureDecodeTests = []struct {
 	{dataBlocks: 3, onDisks: 6, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: SHA256, shouldFail: false, shouldFailQuorum: false},                 // 1
 	{dataBlocks: 4, onDisks: 8, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 2
 	{dataBlocks: 5, onDisks: 10, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 1, length: oneMiByte - 1, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},        // 3
-	{dataBlocks: 6, onDisks: 12, offDisks: 0, blocksize: int64(oneMiByte), data: oneMiByte, offset: oneMiByte, length: 0, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                                                          // 4
+	{dataBlocks: 6, onDisks: 12, offDisks: 0, blocksize: int64(oneMiByte), data: oneMiByte, offset: oneMiByte, length: 0, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},
+	// 4
 	{dataBlocks: 7, onDisks: 14, offDisks: 0, blocksize: int64(oneMiByte), data: oneMiByte, offset: 3, length: 1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                    // 5
 	{dataBlocks: 8, onDisks: 16, offDisks: 0, blocksize: int64(oneMiByte), data: oneMiByte, offset: 4, length: 8 * 1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                // 6
 	{dataBlocks: 7, onDisks: 14, offDisks: 7, blocksize: int64(blockSizeV1), data: oneMiByte, offset: oneMiByte, length: 1, algorithm: DefaultBitrotAlgorithm, shouldFail: true, shouldFailQuorum: false},              // 7
@@ -60,7 +61,8 @@ var erasureDecodeTests = []struct {
 	{dataBlocks: 5, onDisks: 10, offDisks: 6, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},                // 16
 	{dataBlocks: 5, onDisks: 10, offDisks: 2, blocksize: int64(blockSizeV1), data: 2 * oneMiByte, offset: oneMiByte, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 17
 	{dataBlocks: 5, onDisks: 10, offDisks: 1, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                         // 18
-	{dataBlocks: 6, onDisks: 12, offDisks: 3, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: SHA256, shouldFail: false, shouldFailQuorum: false},                                                            // 19
+	{dataBlocks: 6, onDisks: 12, offDisks: 3, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: SHA256, shouldFail: false, shouldFailQuorum: false},
+	// 19
 	{dataBlocks: 6, onDisks: 12, offDisks: 7, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},                                             // 20
 	{dataBlocks: 8, onDisks: 16, offDisks: 8, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                            // 21
 	{dataBlocks: 8, onDisks: 16, offDisks: 9, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},                                               // 22
@@ -104,11 +106,12 @@ func TestErasureDecode(t *testing.T) {
 			writeAlgorithm = DefaultBitrotAlgorithm
 		}
 		buffer := make([]byte, test.blocksize, 2*test.blocksize)
-		writers := make([]*bitrotWriter, len(disks))
+		writers := make([]io.Writer, len(disks))
 		for i, disk := range disks {
-			writers[i] = newBitrotWriter(disk, "testbucket", "object", writeAlgorithm)
+			writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(test.data), writeAlgorithm, erasure.ShardSize())
 		}
 		n, err := erasure.Encode(context.Background(), bytes.NewReader(data[:]), writers, buffer, erasure.dataBlocks+1)
+		closeBitrotWriters(writers)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create erasure test file: %v", i, err)
@@ -124,17 +127,19 @@ func TestErasureDecode(t *testing.T) {
 		}
 
 		// Get the checksums of the current part.
-		bitrotReaders := make([]*bitrotReader, len(disks))
+		bitrotReaders := make([]io.ReaderAt, len(disks))
 		for index, disk := range disks {
 			if disk == OfflineDisk {
 				continue
 			}
-			endOffset := getErasureShardFileEndOffset(test.offset, test.length, test.data, test.blocksize, erasure.dataBlocks)
+			tillOffset := erasure.ShardFileTillOffset(test.offset, test.length, test.data)
-			bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", writeAlgorithm, endOffset, writers[index].Sum())
+
+			bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", tillOffset, writeAlgorithm, bitrotWriterSum(writers[index]), erasure.ShardSize())
 		}
 
 		writer := bytes.NewBuffer(nil)
 		err = erasure.Decode(context.Background(), writer, bitrotReaders, test.offset, test.length, test.data)
+		closeBitrotReaders(bitrotReaders)
 		if err != nil && !test.shouldFail {
 			t.Errorf("Test %d: should pass but failed with: %v", i, err)
 		}
@@ -143,31 +148,41 @@ func TestErasureDecode(t *testing.T) {
 		}
 		if err == nil {
 			if content := writer.Bytes(); !bytes.Equal(content, data[test.offset:test.offset+test.length]) {
-				t.Errorf("Test %d: read retruns wrong file content", i)
+				t.Errorf("Test %d: read retruns wrong file content.", i)
 			}
 		}
+
 		for i, r := range bitrotReaders {
 			if r == nil {
 				disks[i] = OfflineDisk
 			}
 		}
 		if err == nil && !test.shouldFail {
-			bitrotReaders = make([]*bitrotReader, len(disks))
+			bitrotReaders = make([]io.ReaderAt, len(disks))
 			for index, disk := range disks {
 				if disk == OfflineDisk {
 					continue
 				}
-				endOffset := getErasureShardFileEndOffset(test.offset, test.length, test.data, test.blocksize, erasure.dataBlocks)
+				tillOffset := erasure.ShardFileTillOffset(test.offset, test.length, test.data)
-				bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", writeAlgorithm, endOffset, writers[index].Sum())
+				bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", tillOffset, writeAlgorithm, bitrotWriterSum(writers[index]), erasure.ShardSize())
 			}
 			for j := range disks[:test.offDisks] {
-				bitrotReaders[j].disk = badDisk{nil}
+				if bitrotReaders[j] == nil {
+					continue
+				}
+				switch r := bitrotReaders[j].(type) {
+				case *wholeBitrotReader:
+					r.disk = badDisk{nil}
+				case *streamingBitrotReader:
+					r.disk = badDisk{nil}
+				}
 			}
 			if test.offDisks > 0 {
 				bitrotReaders[0] = nil
 			}
 			writer.Reset()
 			err = erasure.Decode(context.Background(), writer, bitrotReaders, test.offset, test.length, test.data)
+			closeBitrotReaders(bitrotReaders)
 			if err != nil && !test.shouldFailQuorum {
 				t.Errorf("Test %d: should pass but failed with: %v", i, err)
 			}
@@ -213,12 +228,12 @@ func TestErasureDecodeRandomOffsetLength(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	writers := make([]*bitrotWriter, len(disks))
+	writers := make([]io.Writer, len(disks))
 	for i, disk := range disks {
 		if disk == nil {
 			continue
 		}
-		writers[i] = newBitrotWriter(disk, "testbucket", "object", DefaultBitrotAlgorithm)
+		writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(length), DefaultBitrotAlgorithm, erasure.ShardSize())
 	}
 
 	// 10000 iterations with random offsets and lengths.
@@ -227,6 +242,7 @@ func TestErasureDecodeRandomOffsetLength(t *testing.T) {
 	// Create a test file to read from.
 	buffer := make([]byte, blockSize, 2*blockSize)
 	n, err := erasure.Encode(context.Background(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1)
+	closeBitrotWriters(writers)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -247,15 +263,16 @@ func TestErasureDecodeRandomOffsetLength(t *testing.T) {
 		expected := data[offset : offset+readLen]
 
 		// Get the checksums of the current part.
-		bitrotReaders := make([]*bitrotReader, len(disks))
+		bitrotReaders := make([]io.ReaderAt, len(disks))
 		for index, disk := range disks {
 			if disk == OfflineDisk {
 				continue
 			}
-			endOffset := getErasureShardFileEndOffset(offset, readLen, length, blockSize, erasure.dataBlocks)
+			tillOffset := erasure.ShardFileTillOffset(offset, readLen, length)
-			bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", DefaultBitrotAlgorithm, endOffset, writers[index].Sum())
+			bitrotReaders[index] = newStreamingBitrotReader(disk, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
 		err = erasure.Decode(context.Background(), buf, bitrotReaders, offset, readLen, length)
+		closeBitrotReaders(bitrotReaders)
 		if err != nil {
 			t.Fatal(err, offset, readLen)
 		}
@@ -281,17 +298,18 @@ func benchmarkErasureDecode(data, parity, dataDown, parityDown int, size int64,
 		b.Fatalf("failed to create ErasureStorage: %v", err)
 	}
 
-	writers := make([]*bitrotWriter, len(disks))
+	writers := make([]io.Writer, len(disks))
 	for i, disk := range disks {
 		if disk == nil {
 			continue
 		}
-		writers[i] = newBitrotWriter(disk, "testbucket", "object", DefaultBitrotAlgorithm)
+		writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
 	}
 
 	content := make([]byte, size)
 	buffer := make([]byte, blockSizeV1, 2*blockSizeV1)
 	_, err = erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1)
+	closeBitrotWriters(writers)
 	if err != nil {
 		b.Fatalf("failed to create erasure test file: %v", err)
 	}
@@ -307,17 +325,18 @@ func benchmarkErasureDecode(data, parity, dataDown, parityDown int, size int64,
 	b.SetBytes(size)
 	b.ReportAllocs()
 	for i := 0; i < b.N; i++ {
-		bitrotReaders := make([]*bitrotReader, len(disks))
+		bitrotReaders := make([]io.ReaderAt, len(disks))
 		for index, disk := range disks {
 			if writers[index] == nil {
 				continue
 			}
-			endOffset := getErasureShardFileEndOffset(0, size, size, erasure.blockSize, erasure.dataBlocks)
+			tillOffset := erasure.ShardFileTillOffset(0, size, size)
-			bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", DefaultBitrotAlgorithm, endOffset, writers[index].Sum())
+			bitrotReaders[index] = newStreamingBitrotReader(disk, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
 		if err = erasure.Decode(context.Background(), bytes.NewBuffer(content[:0]), bitrotReaders, 0, size, size); err != nil {
 			panic(err)
 		}
+		closeBitrotReaders(bitrotReaders)
 	}
 }
 

cmd/erasure-encode.go

@@ -25,15 +25,15 @@ import (
 	"github.com/minio/minio/cmd/logger"
 )
 
-// Writes in parallel to bitrotWriters
+// Writes in parallel to writers
 type parallelWriter struct {
-	writers     []*bitrotWriter
+	writers     []io.Writer
 	writeQuorum int
 	errs        []error
 }
 
-// Append appends data to bitrotWriters in parallel.
+// Write writes data to writers in parallel.
-func (p *parallelWriter) Append(ctx context.Context, blocks [][]byte) error {
+func (p *parallelWriter) Write(ctx context.Context, blocks [][]byte) error {
 	var wg sync.WaitGroup
 
 	for i := range p.writers {
@@ -45,7 +45,7 @@ func (p *parallelWriter) Append(ctx context.Context, blocks [][]byte) error {
 		wg.Add(1)
 		go func(i int) {
 			defer wg.Done()
-			p.errs[i] = p.writers[i].Append(blocks[i])
+			_, p.errs[i] = p.writers[i].Write(blocks[i])
 			if p.errs[i] != nil {
 				p.writers[i] = nil
 			}
@@ -70,7 +70,7 @@ func (p *parallelWriter) Append(ctx context.Context, blocks [][]byte) error {
 }
 
 // Encode reads from the reader, erasure-encodes the data and writes to the writers.
-func (e *Erasure) Encode(ctx context.Context, src io.Reader, writers []*bitrotWriter, buf []byte, quorum int) (total int64, err error) {
+func (e *Erasure) Encode(ctx context.Context, src io.Reader, writers []io.Writer, buf []byte, quorum int) (total int64, err error) {
 	writer := &parallelWriter{
 		writers:     writers,
 		writeQuorum: quorum,
@@ -96,7 +96,7 @@ func (e *Erasure) Encode(ctx context.Context, src io.Reader, writers []*bitrotWr
 			return 0, err
 		}
 
-		if err = writer.Append(ctx, blocks); err != nil {
+		if err = writer.Write(ctx, blocks); err != nil {
 			logger.LogIf(ctx, err)
 			return 0, err
 		}

cmd/erasure-encode_test.go

@@ -36,6 +36,14 @@ func (a badDisk) AppendFile(volume string, path string, buf []byte) error {
 	return errFaultyDisk
 }
 
+func (a badDisk) ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) {
+	return nil, errFaultyDisk
+}
+
+func (a badDisk) CreateFile(volume, path string, size int64, reader io.Reader) error {
+	return errFaultyDisk
+}
+
 const oneMiByte = 1 * humanize.MiByte
 
 var erasureEncodeTests = []struct {
@@ -87,14 +95,15 @@ func TestErasureEncode(t *testing.T) {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to generate random test data: %v", i, err)
 		}
-		writers := make([]*bitrotWriter, len(disks))
+		writers := make([]io.Writer, len(disks))
 		for i, disk := range disks {
 			if disk == OfflineDisk {
 				continue
 			}
-			writers[i] = newBitrotWriter(disk, "testbucket", "object", test.algorithm)
+			writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(int64(len(data[test.offset:]))), test.algorithm, erasure.ShardSize())
 		}
 		n, err := erasure.Encode(context.Background(), bytes.NewReader(data[test.offset:]), writers, buffer, erasure.dataBlocks+1)
+		closeBitrotWriters(writers)
 		if err != nil && !test.shouldFail {
 			t.Errorf("Test %d: should pass but failed with: %v", i, err)
 		}
@@ -110,20 +119,26 @@ func TestErasureEncode(t *testing.T) {
 			if length := int64(len(data[test.offset:])); n != length {
 				t.Errorf("Test %d: invalid number of bytes written: got: #%d want #%d", i, n, length)
 			}
-			writers := make([]*bitrotWriter, len(disks))
+			writers := make([]io.Writer, len(disks))
 			for i, disk := range disks {
 				if disk == nil {
 					continue
 				}
-				writers[i] = newBitrotWriter(disk, "testbucket", "object2", test.algorithm)
+				writers[i] = newBitrotWriter(disk, "testbucket", "object2", erasure.ShardFileSize(int64(len(data[test.offset:]))), test.algorithm, erasure.ShardSize())
 			}
 			for j := range disks[:test.offDisks] {
-				writers[j].disk = badDisk{nil}
+				switch w := writers[j].(type) {
+				case *wholeBitrotWriter:
+					w.disk = badDisk{nil}
+				case *streamingBitrotWriter:
+					w.iow.CloseWithError(errFaultyDisk)
+				}
 			}
 			if test.offDisks > 0 {
 				writers[0] = nil
 			}
 			n, err = erasure.Encode(context.Background(), bytes.NewReader(data[test.offset:]), writers, buffer, erasure.dataBlocks+1)
+			closeBitrotWriters(writers)
 			if err != nil && !test.shouldFailQuorum {
 				t.Errorf("Test %d: should pass but failed with: %v", i, err)
 			}
@@ -167,14 +182,16 @@ func benchmarkErasureEncode(data, parity, dataDown, parityDown int, size int64,
 	b.SetBytes(size)
 	b.ReportAllocs()
 	for i := 0; i < b.N; i++ {
-		writers := make([]*bitrotWriter, len(disks))
+		writers := make([]io.Writer, len(disks))
 		for i, disk := range disks {
 			if disk == OfflineDisk {
 				continue
 			}
-			writers[i] = newBitrotWriter(disk, "testbucket", "object", DefaultBitrotAlgorithm)
+			disk.DeleteFile("testbucket", "object")
+			writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
 		_, err := erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1)
+		closeBitrotWriters(writers)
 		if err != nil {
 			panic(err)
 		}

cmd/erasure-heal.go

@@ -25,7 +25,7 @@ import (
 
 // Heal heals the shard files on non-nil writers. Note that the quorum passed is 1
 // as healing should continue even if it has been successful healing only one shard file.
-func (e Erasure) Heal(ctx context.Context, readers []*bitrotReader, writers []*bitrotWriter, size int64) error {
+func (e Erasure) Heal(ctx context.Context, readers []io.ReaderAt, writers []io.Writer, size int64) error {
 	r, w := io.Pipe()
 	go func() {
 		if err := e.Decode(ctx, w, readers, 0, size, size); err != nil {

cmd/erasure-heal_test.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"crypto/rand"
 	"io"
+	"os"
 	"testing"
 )
 
@@ -84,20 +85,21 @@ func TestErasureHeal(t *testing.T) {
 			t.Fatalf("Test %d: failed to create random test data: %v", i, err)
 		}
 		buffer := make([]byte, test.blocksize, 2*test.blocksize)
-		writers := make([]*bitrotWriter, len(disks))
+		writers := make([]io.Writer, len(disks))
 		for i, disk := range disks {
-			writers[i] = newBitrotWriter(disk, "testbucket", "testobject", test.algorithm)
+			writers[i] = newBitrotWriter(disk, "testbucket", "testobject", erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize())
 		}
 		_, err = erasure.Encode(context.Background(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1)
+		closeBitrotWriters(writers)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create random test data: %v", i, err)
 		}
 
-		readers := make([]*bitrotReader, len(disks))
+		readers := make([]io.ReaderAt, len(disks))
 		for i, disk := range disks {
-			shardFilesize := getErasureShardFileSize(test.blocksize, test.size, erasure.dataBlocks)
+			shardFilesize := erasure.ShardFileSize(test.size)
-			readers[i] = newBitrotReader(disk, "testbucket", "testobject", test.algorithm, shardFilesize, writers[i].Sum())
+			readers[i] = newBitrotReader(disk, "testbucket", "testobject", shardFilesize, test.algorithm, bitrotWriterSum(writers[i]), erasure.ShardSize())
 		}
 
 		// setup stale disks for the test case
@@ -111,22 +113,30 @@ func TestErasureHeal(t *testing.T) {
 			}
 		}
 		for j := 0; j < test.badDisks; j++ {
-			readers[test.offDisks+j].disk = badDisk{nil}
+			switch r := readers[test.offDisks+j].(type) {
+			case *streamingBitrotReader:
+				r.disk = badDisk{nil}
+			case *wholeBitrotReader:
+				r.disk = badDisk{nil}
+			}
 		}
 		for j := 0; j < test.badStaleDisks; j++ {
 			staleDisks[j] = badDisk{nil}
 		}
 
-		staleWriters := make([]*bitrotWriter, len(staleDisks))
+		staleWriters := make([]io.Writer, len(staleDisks))
 		for i, disk := range staleDisks {
 			if disk == nil {
 				continue
 			}
-			staleWriters[i] = newBitrotWriter(disk, "testbucket", "testobject", test.algorithm)
+			os.Remove(pathJoin(disk.String(), "testbucket", "testobject"))
+			staleWriters[i] = newBitrotWriter(disk, "testbucket", "testobject", erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize())
 		}
 
-		// test case setup is complete - now call Healfile()
+		// test case setup is complete - now call Heal()
 		err = erasure.Heal(context.Background(), readers, staleWriters, test.size)
+		closeBitrotReaders(readers)
+		closeBitrotWriters(staleWriters)
 		if err != nil && !test.shouldFail {
 			t.Errorf("Test %d: should pass but it failed with: %v", i, err)
 		}
@@ -140,7 +150,7 @@ func TestErasureHeal(t *testing.T) {
 				if staleWriters[i] == nil {
 					continue
 				}
-				if !bytes.Equal(staleWriters[i].Sum(), writers[i].Sum()) {
+				if !bytes.Equal(bitrotWriterSum(staleWriters[i]), bitrotWriterSum(writers[i])) {
 					t.Errorf("Test %d: heal returned different bitrot checksums", i)
 				}
 			}

cmd/erasure-utils.go

@@ -109,25 +109,3 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
 	// Success.
 	return totalWritten, nil
 }
-
-// Returns shard-file size.
-func getErasureShardFileSize(blockSize int64, totalLength int64, dataBlocks int) int64 {
-	shardSize := ceilFrac(int64(blockSize), int64(dataBlocks))
-	numShards := totalLength / int64(blockSize)
-	lastBlockSize := totalLength % int64(blockSize)
-	lastShardSize := ceilFrac(lastBlockSize, int64(dataBlocks))
-	return shardSize*numShards + lastShardSize
-}
-
-// Returns the endOffset till which bitrotReader should read data using disk.ReadFile()
-// partOffset, partLength and partSize are values of the object's part file.
-func getErasureShardFileEndOffset(partOffset int64, partLength int64, partSize int64, erasureBlockSize int64, dataBlocks int) int64 {
-	shardSize := ceilFrac(erasureBlockSize, int64(dataBlocks))
-	shardFileSize := getErasureShardFileSize(erasureBlockSize, partSize, dataBlocks)
-	endShard := (partOffset + int64(partLength)) / erasureBlockSize
-	endOffset := endShard*shardSize + shardSize
-	if endOffset > shardFileSize {
-		endOffset = shardFileSize
-	}
-	return endOffset
-}

cmd/erasure.go

@@ -32,18 +32,16 @@ type Erasure struct {
 
 // NewErasure creates a new ErasureStorage.
 func NewErasure(ctx context.Context, dataBlocks, parityBlocks int, blockSize int64) (e Erasure, err error) {
-	shardsize := int(ceilFrac(blockSize, int64(dataBlocks)))
-	erasure, err := reedsolomon.New(dataBlocks, parityBlocks, reedsolomon.WithAutoGoroutines(shardsize))
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return e, err
-	}
 	e = Erasure{
-		encoder:      erasure,
 		dataBlocks:   dataBlocks,
 		parityBlocks: parityBlocks,
 		blockSize:    blockSize,
 	}
+	e.encoder, err = reedsolomon.New(dataBlocks, parityBlocks, reedsolomon.WithAutoGoroutines(int(e.ShardSize())))
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return e, err
+	}
 	return
 }
 
@@ -94,3 +92,31 @@ func (e *Erasure) DecodeDataAndParityBlocks(ctx context.Context, data [][]byte)
 	}
 	return nil
 }
+
+// ShardSize - returns actual shared size from erasure blockSize.
+func (e *Erasure) ShardSize() int64 {
+	return ceilFrac(e.blockSize, int64(e.dataBlocks))
+}
+
+// ShardFileSize - returns final erasure size from original size.
+func (e *Erasure) ShardFileSize(totalLength int64) int64 {
+	if totalLength == 0 {
+		return 0
+	}
+	numShards := totalLength / e.blockSize
+	lastBlockSize := totalLength % int64(e.blockSize)
+	lastShardSize := ceilFrac(lastBlockSize, int64(e.dataBlocks))
+	return numShards*e.ShardSize() + lastShardSize
+}
+
+// ShardFileTillOffset - returns the effectiv eoffset where erasure reading begins.
+func (e *Erasure) ShardFileTillOffset(startOffset, length, totalLength int64) int64 {
+	shardSize := e.ShardSize()
+	shardFileSize := e.ShardFileSize(totalLength)
+	endShard := (startOffset + int64(length)) / e.blockSize
+	tillOffset := endShard*shardSize + shardSize
+	if tillOffset > shardFileSize {
+		tillOffset = shardFileSize
+	}
+	return tillOffset
+}

cmd/format-fs.go

@@ -24,6 +24,7 @@ import (
 	"path"
 	"time"
 
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
 )
 
@@ -332,11 +333,24 @@ func formatFSFixDeploymentID(fsFormatPath string) error {
 		return nil
 	}
 
+	formatStartTime := time.Now().Round(time.Second)
+	getElapsedTime := func() string {
+		return time.Now().Round(time.Second).Sub(formatStartTime).String()
+	}
+
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	retryTimerCh := newRetryTimerSimple(doneCh)
 	for {
+		select {
+		case <-retryTimerCh:
+
 			wlk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
 			if err == lock.ErrAlreadyLocked {
-			// Lock already present, sleep and attempt again.
+				// Lock already present, sleep and attempt again
-			time.Sleep(100 * time.Millisecond)
+
+				logger.Info("Another minio process(es) might be holding a lock to the file %s. Please kill that minio process(es) (elapsed %s)\n", fsFormatPath, getElapsedTime())
 				continue
 			}
 			if err != nil {
@@ -356,5 +370,6 @@ func formatFSFixDeploymentID(fsFormatPath string) error {
 			format.ID = mustGetUUID()
 			return jsonSave(wlk, format)
 		}
+	}
 
 }

cmd/format-xl.go

@@ -896,6 +896,7 @@ func newHealFormatSets(refFormat *formatXLV3, setCount, disksPerSet int, formats
 			if errs[i*disksPerSet+j] == errUnformattedDisk || errs[i*disksPerSet+j] == nil {
 				newFormats[i][j] = &formatXLV3{}
 				newFormats[i][j].Version = refFormat.Version
+				newFormats[i][j].ID = refFormat.ID
 				newFormats[i][j].Format = refFormat.Format
 				newFormats[i][j].XL.Version = refFormat.XL.Version
 				newFormats[i][j].XL.DistributionAlgo = refFormat.XL.DistributionAlgo

cmd/format-xl_test.go

@@ -556,4 +556,13 @@ func TestNewFormatSets(t *testing.T) {
 	if newFormats == nil {
 		t.Fatal("Unexpected failure")
 	}
+
+	// Check if deployment IDs are preserved.
+	for i := range newFormats {
+		for j := range newFormats[i] {
+			if newFormats[i][j].ID != quorumFormat.ID {
+				t.Fatal("Deployment id in the new format is lost")
+			}
+		}
+	}
 }

cmd/fs-v1-metadata.go

@@ -113,7 +113,7 @@ type fsMetaV1 struct {
 	// Metadata map for current object.
 	Meta map[string]string `json:"meta,omitempty"`
 	// parts info for current object - used in encryption.
-	Parts []objectPartInfo `json:"parts,omitempty"`
+	Parts []ObjectPartInfo `json:"parts,omitempty"`
 }
 
 // IsValid - tells if the format is sane by validating the version
@@ -207,9 +207,9 @@ func parseFSMetaMap(fsMetaBuf []byte) map[string]string {
 	return metaMap
 }
 
-func parseFSPartsArray(fsMetaBuf []byte) []objectPartInfo {
+func parseFSPartsArray(fsMetaBuf []byte) []ObjectPartInfo {
 	// Get xlMetaV1.Parts array
-	var partsArray []objectPartInfo
+	var partsArray []ObjectPartInfo
 
 	partsArrayResult := gjson.GetBytes(fsMetaBuf, "parts")
 	partsArrayResult.ForEach(func(key, part gjson.Result) bool {
@@ -219,7 +219,7 @@ func parseFSPartsArray(fsMetaBuf []byte) []objectPartInfo {
 		etag := gjson.Get(partJSON, "etag").String()
 		size := gjson.Get(partJSON, "size").Int()
 		actualSize := gjson.Get(partJSON, "actualSize").Int()
-		partsArray = append(partsArray, objectPartInfo{
+		partsArray = append(partsArray, ObjectPartInfo{
 			Number:     int(number),
 			Name:       name,
 			ETag:       etag,

cmd/fs-v1-multipart.go

@@ -353,7 +353,7 @@ func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 // Implements S3 compatible ListObjectParts API. The resulting
 // ListPartsInfo structure is unmarshalled directly into XML and
 // replied back to the client.
-func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int) (result ListPartsInfo, e error) {
+func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int, opts ObjectOptions) (result ListPartsInfo, e error) {
 	if err := checkListPartsArgs(ctx, bucket, object, fs); err != nil {
 		return result, toObjectErr(err)
 	}
@@ -515,7 +515,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	fsMeta := fsMetaV1{}
 
 	// Allocate parts similar to incoming slice.
-	fsMeta.Parts = make([]objectPartInfo, len(parts))
+	fsMeta.Parts = make([]ObjectPartInfo, len(parts))
 
 	entries, err := readDir(uploadIDDir)
 	if err != nil {
@@ -560,7 +560,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 			partSize = actualSize
 		}
 
-		fsMeta.Parts[i] = objectPartInfo{
+		fsMeta.Parts[i] = ObjectPartInfo{
 			Number:     part.PartNumber,
 			ETag:       part.ETag,
 			Size:       fi.Size(),
@@ -682,6 +682,8 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 		return oi, toObjectErr(err, bucket, object)
 	}
 	fsRemoveAll(ctx, uploadIDDir)
+	// It is safe to ignore any directory not empty error (in case there were multiple uploadIDs on the same object)
+	fsRemoveDir(ctx, fs.getMultipartSHADir(bucket, object))
 	fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object))
 	if err != nil {
 		return oi, toObjectErr(err, bucket, object)
@@ -769,6 +771,9 @@ func (fs *FSObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupIn
 					}
 					if now.Sub(fi.ModTime()) > expiry {
 						fsRemoveAll(ctx, pathJoin(fs.fsPath, minioMetaMultipartBucket, entry, uploadID))
+						// It is safe to ignore any directory not empty error (in case there were multiple uploadIDs on the same object)
+						fsRemoveDir(ctx, pathJoin(fs.fsPath, minioMetaMultipartBucket, entry))
+
 					}
 				}
 			}

cmd/fs-v1-multipart_test.go

@@ -36,7 +36,7 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 
 	// Close the go-routine, we are going to
 	// manually start it and test in this test case.
-	globalServiceDoneCh <- struct{}{}
+	GlobalServiceDoneCh <- struct{}{}
 
 	bucketName := "bucket"
 	objectName := "object"
@@ -47,14 +47,14 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 		t.Fatal("Unexpected err: ", err)
 	}
 
-	go fs.cleanupStaleMultipartUploads(context.Background(), 20*time.Millisecond, 0, globalServiceDoneCh)
+	go fs.cleanupStaleMultipartUploads(context.Background(), 20*time.Millisecond, 0, GlobalServiceDoneCh)
 
 	// Wait for 40ms such that - we have given enough time for
 	// cleanup routine to kick in.
 	time.Sleep(40 * time.Millisecond)
 
 	// Close the routine we do not need it anymore.
-	globalServiceDoneCh <- struct{}{}
+	GlobalServiceDoneCh <- struct{}{}
 
 	// Check if upload id was already purged.
 	if err = obj.AbortMultipartUpload(context.Background(), bucketName, objectName, uploadID); err != nil {

cmd/fs-v1.go

@@ -152,10 +152,10 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 	fs.fsFormatRlk = rlk
 
 	if !fs.diskMount {
-		go fs.diskUsage(globalServiceDoneCh)
+		go fs.diskUsage(GlobalServiceDoneCh)
 	}
 
-	go fs.cleanupStaleMultipartUploads(ctx, globalMultipartCleanupInterval, globalMultipartExpiry, globalServiceDoneCh)
+	go fs.cleanupStaleMultipartUploads(ctx, GlobalMultipartCleanupInterval, GlobalMultipartExpiry, GlobalServiceDoneCh)
 
 	// Return successfully initialized object layer.
 	return fs, nil
@@ -1212,9 +1212,8 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de
 		fs.listPool.Set(params, walkResultCh, endWalkCh)
 	}
 
-	result := ListObjectsInfo{IsTruncated: !eof}
+	result := ListObjectsInfo{}
 	for _, objInfo := range objInfos {
-		result.NextMarker = objInfo.Name
 		if objInfo.IsDir && delimiter == slashSeparator {
 			result.Prefixes = append(result.Prefixes, objInfo.Name)
 			continue
@@ -1222,6 +1221,13 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de
 		result.Objects = append(result.Objects, objInfo)
 	}
 
+	if !eof {
+		result.IsTruncated = true
+		if len(objInfos) > 0 {
+			result.NextMarker = objInfos[len(objInfos)-1].Name
+		}
+	}
+
 	// Success.
 	return result, nil
 }
@@ -1239,14 +1245,14 @@ func (fs *FSObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealRe
 }
 
 // HealObject - no-op for fs. Valid only for XL.
-func (fs *FSObjects) HealObject(ctx context.Context, bucket, object string, dryRun bool) (
+func (fs *FSObjects) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool) (
 	res madmin.HealResultItem, err error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return res, NotImplemented{}
 }
 
 // HealBucket - no-op for fs, Valid only for XL.
-func (fs *FSObjects) HealBucket(ctx context.Context, bucket string, dryRun bool) ([]madmin.HealResultItem,
+func (fs *FSObjects) HealBucket(ctx context.Context, bucket string, dryRun, remove bool) ([]madmin.HealResultItem,
 	error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return nil, NotImplemented{}
@@ -1311,7 +1317,7 @@ func (fs *FSObjects) IsListenBucketSupported() bool {
 	return true
 }
 
-// IsEncryptionSupported returns whether server side encryption is applicable for this layer.
+// IsEncryptionSupported returns whether server side encryption is implemented for this layer.
 func (fs *FSObjects) IsEncryptionSupported() bool {
 	return true
 }

cmd/fs-v1_test.go

@@ -354,7 +354,7 @@ func TestFSListBuckets(t *testing.T) {
 		t.Fatal("Unexpected error: ", err)
 	}
 
-	globalServiceDoneCh <- struct{}{}
+	GlobalServiceDoneCh <- struct{}{}
 
 	// Create a bucket with invalid name
 	if err := os.MkdirAll(pathJoin(fs.fsPath, "vo^"), 0777); err != nil {
@@ -390,7 +390,7 @@ func TestFSHealObject(t *testing.T) {
 	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
-	_, err := obj.HealObject(context.Background(), "bucket", "object", false)
+	_, err := obj.HealObject(context.Background(), "bucket", "object", false, false)
 	if err == nil || !isSameType(err, NotImplemented{}) {
 		t.Fatalf("Heal Object should return NotImplemented error ")
 	}

cmd/gateway-common.go

@@ -18,7 +18,10 @@ package cmd
 
 import (
 	"net/http"
+	"os"
+	"strings"
 
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/hash"
 
 	minio "github.com/minio/minio-go"
@@ -30,8 +33,16 @@ var (
 
 	// MustGetUUID function alias.
 	MustGetUUID = mustGetUUID
+
+	// CleanMetadataKeys provides cleanMetadataKeys function alias.
+	CleanMetadataKeys = cleanMetadataKeys
 )
 
+// StatInfo -  alias for statInfo
+type StatInfo struct {
+	statInfo
+}
+
 // AnonErrToObjectErr - converts standard http codes into meaningful object layer errors.
 func AnonErrToObjectErr(statusCode int, params ...string) error {
 	bucket := ""
@@ -321,3 +332,30 @@ func ErrorRespToObjectError(err error, params ...string) error {
 
 	return err
 }
+
+// parse gateway sse env variable
+func parseGatewaySSE(s string) (gatewaySSE, error) {
+	l := strings.Split(s, ";")
+	var gwSlice = make([]string, 0)
+	for _, val := range l {
+		v := strings.ToUpper(val)
+		if v == gatewaySSES3 || v == gatewaySSEC {
+			gwSlice = append(gwSlice, v)
+			continue
+		}
+		return nil, uiErrInvalidGWSSEValue(nil).Msg("gateway SSE cannot be (%s) ", v)
+	}
+	return gatewaySSE(gwSlice), nil
+}
+
+// handle gateway env vars
+func handleGatewayEnvVars() {
+	gwsseVal, ok := os.LookupEnv("MINIO_GATEWAY_SSE")
+	if ok {
+		var err error
+		GlobalGatewaySSE, err = parseGatewaySSE(gwsseVal)
+		if err != nil {
+			logger.Fatal(err, "Unable to parse MINIO_GATEWAY_SSE value (`%s`)", gwsseVal)
+		}
+	}
+}

cmd/gateway-common_test.go

@@ -0,0 +1,52 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package cmd
+
+import (
+	"reflect"
+	"testing"
+)
+
+// Tests cache exclude parsing.
+func TestParseGatewaySSE(t *testing.T) {
+	testCases := []struct {
+		gwSSEStr string
+		expected gatewaySSE
+		success  bool
+	}{
+		// valid input
+		{"c;S3", []string{"C", "S3"}, true},
+		{"S3", []string{"S3"}, true},
+		{"c,S3", []string{}, false},
+		{"c;S3;KMS", []string{}, false},
+		{"C;s3", []string{"C", "S3"}, true},
+	}
+
+	for i, testCase := range testCases {
+		gwSSE, err := parseGatewaySSE(testCase.gwSSEStr)
+		if err != nil && testCase.success {
+			t.Errorf("Test %d: Expected success but failed instead %s", i+1, err)
+		}
+		if err == nil && !testCase.success {
+			t.Errorf("Test %d: Expected failure but passed instead", i+1)
+		}
+		if err == nil {
+			if !reflect.DeepEqual(gwSSE, testCase.expected) {
+				t.Errorf("Test %d: Expected %v, got %v", i+1, testCase.expected, gwSSE)
+			}
+		}
+	}
+}

cmd/gateway-env.go

@@ -14,37 +14,35 @@
  * limitations under the License.
  */
 
-package format
+package cmd
 
-import "strconv"
+type gatewaySSE []string
 
-// IsInt - returns a true or false, whether a string can
+const (
-// be represented as an int.
+	// GatewaySSES3 is set when SSE-S3 encryption needed on both gateway and backend
-func IsInt(s string) bool {
+	gatewaySSES3 = "S3"
-	_, err := strconv.Atoi(s)
+	// GatewaySSEC is set when SSE-C encryption needed on both gateway and backend
-	return err == nil
+	gatewaySSEC = "C"
-}
+)
 
-// StringInSlice - this function finds whether a string is in a list
+func (sse gatewaySSE) SSES3() bool {
-func StringInSlice(x string, list []string) bool {
+	for _, v := range sse {
-	for _, y := range list {
+		if v == gatewaySSES3 {
-		if x == y {
 			return true
 		}
 	}
 	return false
 }
 
-// ProcessSize - this function processes size so that we can calculate bytes BytesProcessed.
+func (sse gatewaySSE) SSEC() bool {
-func ProcessSize(myrecord []string) int64 {
+	for _, v := range sse {
-	if len(myrecord) > 0 {
+		if v == gatewaySSEC {
-		var size int64
+			return true
-		size = int64(len(myrecord)-1) + 1
-		for i := range myrecord {
-			size += int64(len(myrecord[i]))
 		}
-
-		return size
 	}
-	return 0
+	return false
+}
+
+func (sse gatewaySSE) IsSet() bool {
+	return sse.SSES3() || sse.SSEC()
 }

cmd/gateway-main.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"net"
 	"net/url"
 	"os"
 	"os/signal"
@@ -112,34 +111,11 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		cli.ShowCommandHelpAndExit(ctx, gatewayName, 1)
 	}
 
-	// Get "json" flag from command line argument and
-	// enable json and quite modes if jason flag is turned on.
-	jsonFlag := ctx.IsSet("json") || ctx.GlobalIsSet("json")
-	if jsonFlag {
-		logger.EnableJSON()
-	}
-
-	// Get quiet flag from command line argument.
-	quietFlag := ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet")
-	if quietFlag {
-		logger.EnableQuiet()
-	}
-
-	// Fetch address option
-	gatewayAddr := ctx.GlobalString("address")
-	if gatewayAddr == ":"+globalMinioPort {
-		gatewayAddr = ctx.String("address")
-	}
-
 	// Handle common command args.
 	handleCommonCmdArgs(ctx)
 
 	// Get port to listen on from gateway address
-	var pErr error
+	globalMinioHost, globalMinioPort = mustSplitHostPort(globalCLIContext.Addr)
-	_, globalMinioPort, pErr = net.SplitHostPort(gatewayAddr)
-	if pErr != nil {
-		logger.FatalIf(pErr, "Unable to start gateway")
-	}
 
 	// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
 	// to IPv6 address ie minio will start listening on IPv6 address whereas another
@@ -147,21 +123,21 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// To avoid this error situation we check for port availability.
 	logger.FatalIf(checkPortAvailability(globalMinioPort), "Unable to start the gateway")
 
-	// Create certs path.
-	logger.FatalIf(createConfigDir(), "Unable to create configuration directories")
-
 	// Check and load TLS certificates.
 	var err error
 	globalPublicCerts, globalTLSCerts, globalIsSSL, err = getTLSConfig()
 	logger.FatalIf(err, "Invalid TLS certificate file")
 
 	// Check and load Root CAs.
-	globalRootCAs, err = getRootCAs(getCADir())
+	globalRootCAs, err = getRootCAs(globalCertsCADir.Get())
 	logger.FatalIf(err, "Failed to read root CAs (%v)", err)
 
 	// Handle common env vars.
 	handleCommonEnvVars()
 
+	// Handle gateway specific env
+	handleGatewayEnvVars()
+
 	// Validate if we have access, secret set through environment.
 	if !globalIsEnvCreds {
 		logger.Fatal(uiErrEnvCredentialsMissingGateway(nil), "Unable to start gateway")
@@ -177,11 +153,15 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	if globalEtcdClient != nil {
 		// Enable STS router if etcd is enabled.
 		registerSTSRouter(router)
-
-		// Enable admin router if etcd is enabled.
-		registerAdminRouter(router)
 	}
 
+	enableConfigOps := globalEtcdClient != nil && gatewayName == "nas"
+	enableIAMOps := globalEtcdClient != nil
+
+	// Enable IAM admin APIs if etcd is enabled, if not just enable basic
+	// operations such as profiling, server info etc.
+	registerAdminRouter(router, enableConfigOps, enableIAMOps)
+
 	// Add healthcheck router
 	registerHealthCheckRouter(router)
 
@@ -193,24 +173,18 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		logger.FatalIf(registerWebRouter(router), "Unable to configure web browser")
 	}
 
+	// Currently only NAS and S3 gateway support encryption headers.
+	encryptionEnabled := gatewayName == "s3" || gatewayName == "nas"
+
 	// Add API router.
-	registerAPIRouter(router)
+	registerAPIRouter(router, encryptionEnabled)
-
-	// Dummy endpoint representing gateway instance.
-	globalEndpoints = []Endpoint{{
-		URL:     &url.URL{Path: "/minio/gateway"},
-		IsLocal: true,
-	}}
-
-	// Initialize Admin Peers.
-	initGlobalAdminPeers(globalEndpoints)
 
 	var getCert certs.GetCertificateFunc
 	if globalTLSCerts != nil {
 		getCert = globalTLSCerts.GetCertificate
 	}
 
-	globalHTTPServer = xhttp.NewServer([]string{gatewayAddr}, criticalErrorHandler{registerHandlers(router, globalHandlers...)}, getCert)
+	globalHTTPServer = xhttp.NewServer([]string{globalCLIContext.Addr}, criticalErrorHandler{registerHandlers(router, globalHandlers...)}, getCert)
 	globalHTTPServer.UpdateBytesReadFunc = globalConnStats.incInputBytes
 	globalHTTPServer.UpdateBytesWrittenFunc = globalConnStats.incOutputBytes
 	go func() {
@@ -219,21 +193,10 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM)
 
-	newObject, err := gw.NewGatewayLayer(globalServerConfig.GetCredential())
+	// !!! Do not move this block !!!
-	if err != nil {
+	// For all gateways, the config needs to be loaded from env
-		// Stop watching for any certificate changes.
+	// prior to initializing the gateway layer
-		globalTLSCerts.Stop()
+	{
-
-		globalHTTPServer.Shutdown()
-		logger.FatalIf(err, "Unable to initialize gateway backend")
-	}
-
-	// Create a new config system.
-	globalConfigSys = NewConfigSys()
-	if globalEtcdClient != nil && gatewayName == "nas" {
-		// Initialize server config.
-		_ = globalConfigSys.Init(newObject)
-	} else {
 		// Initialize server config.
 		srvCfg := newServerConfig()
 
@@ -249,6 +212,23 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		globalServerConfigMu.Unlock()
 	}
 
+	newObject, err := gw.NewGatewayLayer(globalServerConfig.GetCredential())
+	if err != nil {
+		// Stop watching for any certificate changes.
+		globalTLSCerts.Stop()
+
+		globalHTTPServer.Shutdown()
+		logger.FatalIf(err, "Unable to initialize gateway backend")
+	}
+
+	if enableConfigOps {
+		// Create a new config system.
+		globalConfigSys = NewConfigSys()
+
+		// Load globalServerConfig from etcd
+		logger.LogIf(context.Background(), globalConfigSys.Init(newObject))
+	}
+
 	// Load logger subsystem
 	loadLoggers()
 
@@ -268,9 +248,9 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 	// Create new IAM system.
 	globalIAMSys = NewIAMSys()
-	if globalEtcdClient != nil {
+	if enableIAMOps {
 		// Initialize IAM sys.
-		_ = globalIAMSys.Init(newObject)
+		logger.LogIf(context.Background(), globalIAMSys.Init(newObject))
 	}
 
 	// Create new policy system.
@@ -282,15 +262,30 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Create new notification system.
 	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
 	if globalEtcdClient != nil && newObject.IsNotificationSupported() {
-		_ = globalNotificationSys.Init(newObject)
+		logger.LogIf(context.Background(), globalNotificationSys.Init(newObject))
 	}
+
+	// Encryption support checks in gateway mode.
+	{
+
+		if (globalAutoEncryption || GlobalKMS != nil) && !newObject.IsEncryptionSupported() {
+			logger.Fatal(errInvalidArgument,
+				"Encryption support is requested but (%s) gateway does not support encryption", gw.Name())
+		}
+
+		if GlobalGatewaySSE.IsSet() && GlobalKMS == nil {
+			logger.Fatal(uiErrInvalidGWSSEEnvValue(nil).Msg("MINIO_GATEWAY_SSE set but KMS is not configured"),
+				"Unable to start gateway with SSE")
+		}
+	}
+
 	// Once endpoints are finalized, initialize the new object api.
 	globalObjLayerMutex.Lock()
 	globalObjectAPI = newObject
 	globalObjLayerMutex.Unlock()
 
 	// Prints the formatted startup message once object layer is initialized.
-	if !quietFlag {
+	if !globalCLIContext.Quiet {
 		mode := globalMinioModeGatewayPrefix + gatewayName
 		// Check update mode.
 		checkUpdate(mode)
@@ -301,8 +296,11 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		}
 
 		// Print gateway startup message.
-		printGatewayStartupMessage(getAPIEndpoints(gatewayAddr), gatewayName)
+		printGatewayStartupMessage(getAPIEndpoints(), gatewayName)
 	}
 
+	// Set uptime time after object layer has initialized.
+	globalBootTime = UTCNow()
+
 	handleSignals()
 }

cmd/gateway-startup-msg.go

@@ -55,9 +55,10 @@ func printGatewayCommonMsg(apiEndpoints []string) {
 	cred := globalServerConfig.GetCredential()
 
 	apiEndpointStr := strings.Join(apiEndpoints, "  ")
+
 	// Colorize the message and print.
-	logger.StartupMessage(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
+	logger.StartupMessage(colorBlue("Endpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
-	if isTerminal() {
+	if isTerminal() && !globalCLIContext.Anonymous {
 		logger.StartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
 		logger.StartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
 	}

cmd/gateway-unsupported.go

@@ -49,7 +49,7 @@ func (a GatewayUnsupported) PutObjectPart(ctx context.Context, bucket string, ob
 }
 
 // ListObjectParts returns all object parts for specified object in specified bucket
-func (a GatewayUnsupported) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi ListPartsInfo, err error) {
+func (a GatewayUnsupported) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int, opts ObjectOptions) (lpi ListPartsInfo, err error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return lpi, NotImplemented{}
 }
@@ -92,7 +92,7 @@ func (a GatewayUnsupported) HealFormat(ctx context.Context, dryRun bool) (madmin
 }
 
 // HealBucket - Not implemented stub
-func (a GatewayUnsupported) HealBucket(ctx context.Context, bucket string, dryRun bool) ([]madmin.HealResultItem, error) {
+func (a GatewayUnsupported) HealBucket(ctx context.Context, bucket string, dryRun, remove bool) ([]madmin.HealResultItem, error) {
 	return nil, NotImplemented{}
 }
 
@@ -102,7 +102,7 @@ func (a GatewayUnsupported) ListBucketsHeal(ctx context.Context) (buckets []Buck
 }
 
 // HealObject - Not implemented stub
-func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, dryRun bool) (h madmin.HealResultItem, e error) {
+func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool) (h madmin.HealResultItem, e error) {
 	return h, NotImplemented{}
 }
 
@@ -137,7 +137,7 @@ func (a GatewayUnsupported) IsListenBucketSupported() bool {
 	return false
 }
 
-// IsEncryptionSupported returns whether server side encryption is applicable for this layer.
+// IsEncryptionSupported returns whether server side encryption is implemented for this layer.
 func (a GatewayUnsupported) IsEncryptionSupported() bool {
 	return false
 }

cmd/gateway/azure/gateway-azure.go

@@ -981,7 +981,7 @@ func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, upload
 }
 
 // ListObjectParts - Use Azure equivalent GetBlockList.
-func (a *azureObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int) (result minio.ListPartsInfo, err error) {
+func (a *azureObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int, opts minio.ObjectOptions) (result minio.ListPartsInfo, err error) {
 	if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return result, err
 	}

cmd/gateway/b2/gateway-b2.go

@@ -1,850 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package b2
-
-import (
-	"context"
-	"crypto/sha1"
-	"fmt"
-	"hash"
-	"io"
-	"io/ioutil"
-	"net/http"
-
-	"strings"
-	"sync"
-	"time"
-
-	b2 "github.com/minio/blazer/base"
-	"github.com/minio/cli"
-	miniogopolicy "github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/auth"
-	h2 "github.com/minio/minio/pkg/hash"
-	"github.com/minio/minio/pkg/policy"
-	"github.com/minio/minio/pkg/policy/condition"
-
-	minio "github.com/minio/minio/cmd"
-)
-
-// Supported bucket types by B2 backend.
-const (
-	bucketTypePrivate  = "allPrivate"
-	bucketTypeReadOnly = "allPublic"
-	b2Backend          = "b2"
-)
-
-func init() {
-	const b2GatewayTemplate = `NAME:
-  {{.HelpName}} - {{.Usage}}
-
-USAGE:
-  {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}}
-{{if .VisibleFlags}}
-FLAGS:
-  {{range .VisibleFlags}}{{.}}
-  {{end}}{{end}}
-ENVIRONMENT VARIABLES:
-  ACCESS:
-     MINIO_ACCESS_KEY: B2 account id.
-     MINIO_SECRET_KEY: B2 application key.
-
-  BROWSER:
-     MINIO_BROWSER: To disable web browser access, set this value to "off".
-
-  DOMAIN:
-     MINIO_DOMAIN: To enable virtual-host-style requests, set this value to Minio host domain name.
-
-  CACHE:
-     MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
-     MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
-     MINIO_CACHE_EXPIRY: Cache expiry duration in days.
-     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
-
-EXAMPLES:
-  1. Start minio gateway server for B2 backend.
-     $ export MINIO_ACCESS_KEY=accountID
-     $ export MINIO_SECRET_KEY=applicationKey
-     $ {{.HelpName}}
-
-  2. Start minio gateway server for B2 backend with edge caching enabled.
-     $ export MINIO_ACCESS_KEY=accountID
-     $ export MINIO_SECRET_KEY=applicationKey
-     $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
-     $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
-     $ export MINIO_CACHE_EXPIRY=40
-     $ export MINIO_CACHE_MAXUSE=80
-     $ {{.HelpName}}
-`
-	minio.RegisterGatewayCommand(cli.Command{
-		Name:               b2Backend,
-		Usage:              "Backblaze B2",
-		Action:             b2GatewayMain,
-		CustomHelpTemplate: b2GatewayTemplate,
-		HideHelpCommand:    true,
-	})
-}
-
-// Handler for 'minio gateway b2' command line.
-func b2GatewayMain(ctx *cli.Context) {
-	minio.StartGateway(ctx, &B2{})
-}
-
-// B2 implements Minio Gateway
-type B2 struct{}
-
-// Name implements Gateway interface.
-func (g *B2) Name() string {
-	return b2Backend
-}
-
-// NewGatewayLayer returns b2 gateway layer, implements ObjectLayer interface to
-// talk to B2 remote backend.
-func (g *B2) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
-	ctx := context.Background()
-	client, err := b2.AuthorizeAccount(ctx, creds.AccessKey, creds.SecretKey, b2.Transport(minio.NewCustomHTTPTransport()))
-	if err != nil {
-		return nil, err
-	}
-
-	return &b2Objects{
-		creds:    creds,
-		b2Client: client,
-		ctx:      ctx,
-	}, nil
-}
-
-// Production - Ready for production use.
-func (g *B2) Production() bool {
-	return true
-}
-
-// b2Object implements gateway for Minio and BackBlaze B2 compatible object storage servers.
-type b2Objects struct {
-	minio.GatewayUnsupported
-	mu       sync.Mutex
-	creds    auth.Credentials
-	b2Client *b2.B2
-	ctx      context.Context
-}
-
-// Convert B2 errors to minio object layer errors.
-func b2ToObjectError(err error, params ...string) error {
-	if err == nil {
-		return nil
-	}
-	bucket := ""
-	object := ""
-	uploadID := ""
-	if len(params) >= 1 {
-		bucket = params[0]
-	}
-	if len(params) == 2 {
-		object = params[1]
-	}
-	if len(params) == 3 {
-		uploadID = params[2]
-	}
-
-	// Following code is a non-exhaustive check to convert
-	// B2 errors into S3 compatible errors.
-	//
-	// For a more complete information - https://www.backblaze.com/b2/docs/
-	statusCode, code, msg := b2.Code(err)
-	if statusCode == 0 {
-		// We don't interpret non B2 errors. B2 errors have statusCode
-		// to help us convert them to S3 object errors.
-		return err
-	}
-
-	switch code {
-	case "duplicate_bucket_name":
-		err = minio.BucketAlreadyOwnedByYou{Bucket: bucket}
-	case "bad_request":
-		if object != "" {
-			err = minio.ObjectNameInvalid{
-				Bucket: bucket,
-				Object: object,
-			}
-		} else if bucket != "" {
-			err = minio.BucketNotFound{Bucket: bucket}
-		}
-	case "bad_json":
-		if object != "" {
-			err = minio.ObjectNameInvalid{
-				Bucket: bucket,
-				Object: object,
-			}
-		} else if bucket != "" {
-			err = minio.BucketNameInvalid{Bucket: bucket}
-		}
-	case "bad_bucket_id":
-		err = minio.BucketNotFound{Bucket: bucket}
-	case "file_not_present", "not_found":
-		err = minio.ObjectNotFound{
-			Bucket: bucket,
-			Object: object,
-		}
-	case "cannot_delete_non_empty_bucket":
-		err = minio.BucketNotEmpty{Bucket: bucket}
-	}
-
-	// Special interpretation like this is required for Multipart sessions.
-	if strings.Contains(msg, "No active upload for") && uploadID != "" {
-		err = minio.InvalidUploadID{UploadID: uploadID}
-	}
-
-	return err
-}
-
-// Shutdown saves any gateway metadata to disk
-// if necessary and reload upon next restart.
-func (l *b2Objects) Shutdown(ctx context.Context) error {
-	// TODO
-	return nil
-}
-
-// StorageInfo is not relevant to B2 backend.
-func (l *b2Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
-	return si
-}
-
-// MakeBucket creates a new container on B2 backend.
-func (l *b2Objects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error {
-	// location is ignored for B2 backend.
-
-	// All buckets are set to private by default.
-	_, err := l.b2Client.CreateBucket(l.ctx, bucket, bucketTypePrivate, nil, nil)
-	logger.LogIf(ctx, err)
-	return b2ToObjectError(err, bucket)
-}
-
-func (l *b2Objects) reAuthorizeAccount(ctx context.Context) error {
-	client, err := b2.AuthorizeAccount(l.ctx, l.creds.AccessKey, l.creds.SecretKey, b2.Transport(minio.NewCustomHTTPTransport()))
-	if err != nil {
-		return err
-	}
-	l.mu.Lock()
-	l.b2Client.Update(client)
-	l.mu.Unlock()
-	return nil
-}
-
-// listBuckets is a wrapper similar to ListBuckets, which re-authorizes
-// the account and updates the B2 client safely. Once successfully
-// authorized performs the call again and returns list of buckets.
-// For any errors which are not actionable we return an error.
-func (l *b2Objects) listBuckets(ctx context.Context, err error) ([]*b2.Bucket, error) {
-	if err != nil {
-		if b2.Action(err) != b2.ReAuthenticate {
-			return nil, err
-		}
-		if rerr := l.reAuthorizeAccount(ctx); rerr != nil {
-			return nil, rerr
-		}
-	}
-	bktList, lerr := l.b2Client.ListBuckets(l.ctx)
-	if lerr != nil {
-		return l.listBuckets(ctx, lerr)
-	}
-	return bktList, nil
-}
-
-// Bucket - is a helper which provides a *Bucket instance
-// for performing an API operation. B2 API doesn't
-// provide a direct way to access the bucket so we need
-// to employ following technique.
-func (l *b2Objects) Bucket(ctx context.Context, bucket string) (*b2.Bucket, error) {
-	bktList, err := l.listBuckets(ctx, nil)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return nil, b2ToObjectError(err, bucket)
-	}
-	for _, bkt := range bktList {
-		if bkt.Name == bucket {
-			return bkt, nil
-		}
-	}
-	return nil, minio.BucketNotFound{Bucket: bucket}
-}
-
-// GetBucketInfo gets bucket metadata..
-func (l *b2Objects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, err error) {
-	if _, err = l.Bucket(ctx, bucket); err != nil {
-		return bi, err
-	}
-	return minio.BucketInfo{
-		Name:    bucket,
-		Created: time.Unix(0, 0),
-	}, nil
-}
-
-// ListBuckets lists all B2 buckets
-func (l *b2Objects) ListBuckets(ctx context.Context) ([]minio.BucketInfo, error) {
-	bktList, err := l.listBuckets(ctx, nil)
-	if err != nil {
-		return nil, err
-	}
-	var bktInfo []minio.BucketInfo
-	for _, bkt := range bktList {
-		bktInfo = append(bktInfo, minio.BucketInfo{
-			Name:    bkt.Name,
-			Created: time.Unix(0, 0),
-		})
-	}
-	return bktInfo, nil
-}
-
-// DeleteBucket deletes a bucket on B2
-func (l *b2Objects) DeleteBucket(ctx context.Context, bucket string) error {
-	bkt, err := l.Bucket(ctx, bucket)
-	if err != nil {
-		return err
-	}
-	err = bkt.DeleteBucket(l.ctx)
-	logger.LogIf(ctx, err)
-	return b2ToObjectError(err, bucket)
-}
-
-// ListObjects lists all objects in B2 bucket filtered by prefix, returns upto at max 1000 entries at a time.
-func (l *b2Objects) ListObjects(ctx context.Context, bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) {
-	bkt, err := l.Bucket(ctx, bucket)
-	if err != nil {
-		return loi, err
-	}
-	files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
-	if lerr != nil {
-		logger.LogIf(ctx, lerr)
-		return loi, b2ToObjectError(lerr, bucket)
-	}
-	loi.IsTruncated = next != ""
-	loi.NextMarker = next
-	for _, file := range files {
-		switch file.Status {
-		case "folder":
-			loi.Prefixes = append(loi.Prefixes, file.Name)
-		case "upload":
-			loi.Objects = append(loi.Objects, minio.ObjectInfo{
-				Bucket:      bucket,
-				Name:        file.Name,
-				ModTime:     file.Timestamp,
-				Size:        file.Size,
-				ETag:        minio.ToS3ETag(file.Info.ID),
-				ContentType: file.Info.ContentType,
-				UserDefined: file.Info.Info,
-			})
-		}
-	}
-	return loi, nil
-}
-
-// ListObjectsV2 lists all objects in B2 bucket filtered by prefix, returns upto max 1000 entries at a time.
-func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int,
-	fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) {
-	// fetchOwner is not supported and unused.
-	marker := continuationToken
-	if marker == "" {
-		// B2's continuation token is an object name to "start at" rather than "start after"
-		// startAfter plus the lowest character B2 supports is used so that the startAfter
-		// object isn't included in the results
-		marker = startAfter + " "
-	}
-
-	bkt, err := l.Bucket(ctx, bucket)
-	if err != nil {
-		return loi, err
-	}
-	files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
-	if lerr != nil {
-		logger.LogIf(ctx, lerr)
-		return loi, b2ToObjectError(lerr, bucket)
-	}
-	loi.IsTruncated = next != ""
-	loi.ContinuationToken = continuationToken
-	loi.NextContinuationToken = next
-	for _, file := range files {
-		switch file.Status {
-		case "folder":
-			loi.Prefixes = append(loi.Prefixes, file.Name)
-		case "upload":
-			loi.Objects = append(loi.Objects, minio.ObjectInfo{
-				Bucket:      bucket,
-				Name:        file.Name,
-				ModTime:     file.Timestamp,
-				Size:        file.Size,
-				ETag:        minio.ToS3ETag(file.Info.ID),
-				ContentType: file.Info.ContentType,
-				UserDefined: file.Info.Info,
-			})
-		}
-	}
-	return loi, nil
-}
-
-// GetObjectNInfo - returns object info and locked object ReadCloser
-func (l *b2Objects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
-	var objInfo minio.ObjectInfo
-	objInfo, err = l.GetObjectInfo(ctx, bucket, object, opts)
-	if err != nil {
-		return nil, err
-	}
-
-	var startOffset, length int64
-	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
-	if err != nil {
-		return nil, err
-	}
-
-	pr, pw := io.Pipe()
-	go func() {
-		err := l.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
-		pw.CloseWithError(err)
-	}()
-	// Setup cleanup function to cause the above go-routine to
-	// exit in case of partial read
-	pipeCloser := func() { pr.Close() }
-	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
-}
-
-// GetObject reads an object from B2. Supports additional
-// parameters like offset and length which are synonymous with
-// HTTP Range requests.
-//
-// startOffset indicates the starting read location of the object.
-// length indicates the total length of the object.
-func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
-	bkt, err := l.Bucket(ctx, bucket)
-	if err != nil {
-		return err
-	}
-	reader, err := bkt.DownloadFileByName(l.ctx, object, startOffset, length)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return b2ToObjectError(err, bucket, object)
-	}
-	defer reader.Close()
-	_, err = io.Copy(writer, reader)
-	logger.LogIf(ctx, err)
-	return b2ToObjectError(err, bucket, object)
-}
-
-// GetObjectInfo reads object info and replies back ObjectInfo
-func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
-	bkt, err := l.Bucket(ctx, bucket)
-	if err != nil {
-		return objInfo, err
-	}
-	f, err := bkt.DownloadFileByName(l.ctx, object, 0, 1)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return objInfo, b2ToObjectError(err, bucket, object)
-	}
-	f.Close()
-	fi, err := bkt.File(f.ID, object).GetFileInfo(l.ctx)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return objInfo, b2ToObjectError(err, bucket, object)
-	}
-	return minio.ObjectInfo{
-		Bucket:      bucket,
-		Name:        object,
-		ETag:        minio.ToS3ETag(fi.ID),
-		Size:        fi.Size,
-		ModTime:     fi.Timestamp,
-		ContentType: fi.ContentType,
-		UserDefined: fi.Info,
-	}, nil
-}
-
-// In B2 - You must always include the X-Bz-Content-Sha1 header with
-// your upload request. The value you provide can be:
-// (1) the 40-character hex checksum of the file,
-// (2) the string hex_digits_at_end, or
-// (3) the string do_not_verify.
-// For more reference - https://www.backblaze.com/b2/docs/uploading.html
-//
-// In our case we are going to use (2) option
-const sha1AtEOF = "hex_digits_at_end"
-
-// With the second option mentioned above, you append the 40-character hex sha1
-// to the end of the request body, immediately after the contents of the file
-// being uploaded. Note that the content length is the size of the file plus 40
-// of the original size of the reader.
-//
-// newB2Reader implements a B2 compatible reader by wrapping the hash.Reader into
-// a new io.Reader which will emit out the sha1 hex digits at io.EOF.
-// It also means that your overall content size is now original size + 40 bytes.
-// Additionally this reader also verifies Hash encapsulated inside hash.Reader
-// at io.EOF if the verification failed we return an error and do not send
-// the content to server.
-func newB2Reader(r *h2.Reader, size int64) *Reader {
-	return &Reader{
-		r:        r,
-		size:     size,
-		sha1Hash: sha1.New(),
-	}
-}
-
-// Reader - is a Reader wraps the hash.Reader which will emit out the sha1
-// hex digits at io.EOF. It also means that your overall content size is
-// now original size + 40 bytes. Additionally this reader also verifies
-// Hash encapsulated inside hash.Reader at io.EOF if the verification
-// failed we return an error and do not send the content to server.
-type Reader struct {
-	r        *h2.Reader
-	size     int64
-	sha1Hash hash.Hash
-
-	isEOF bool
-	buf   *strings.Reader
-}
-
-// Size - Returns the total size of Reader.
-func (nb *Reader) Size() int64 { return nb.size + 40 }
-func (nb *Reader) Read(p []byte) (int, error) {
-	if nb.isEOF {
-		return nb.buf.Read(p)
-	}
-	// Read into hash to update the on going checksum.
-	n, err := io.TeeReader(nb.r, nb.sha1Hash).Read(p)
-	if err == io.EOF {
-		// Stream is not corrupted on this end
-		// now fill in the last 40 bytes of sha1 hex
-		// so that the server can verify the stream on
-		// their end.
-		err = nil
-		nb.isEOF = true
-		nb.buf = strings.NewReader(fmt.Sprintf("%x", nb.sha1Hash.Sum(nil)))
-	}
-	return n, err
-}
-
-// PutObject uploads the single upload to B2 backend by using *b2_upload_file* API, uploads upto 5GiB.
-func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
-	data := r.Reader
-
-	bkt, err := l.Bucket(ctx, bucket)
-	if err != nil {
-		return objInfo, err
-	}
-	contentType := metadata["content-type"]
-	delete(metadata, "content-type")
-
-	var u *b2.URL
-	u, err = bkt.GetUploadURL(l.ctx)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return objInfo, b2ToObjectError(err, bucket, object)
-	}
-
-	hr := newB2Reader(data, data.Size())
-	var f *b2.File
-	f, err = u.UploadFile(l.ctx, hr, int(hr.Size()), object, contentType, sha1AtEOF, metadata)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return objInfo, b2ToObjectError(err, bucket, object)
-	}
-
-	var fi *b2.FileInfo
-	fi, err = f.GetFileInfo(l.ctx)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return objInfo, b2ToObjectError(err, bucket, object)
-	}
-
-	return minio.ObjectInfo{
-		Bucket:      bucket,
-		Name:        object,
-		ETag:        minio.ToS3ETag(fi.ID),
-		Size:        fi.Size,
-		ModTime:     fi.Timestamp,
-		ContentType: fi.ContentType,
-		UserDefined: fi.Info,
-	}, nil
-}
-
-// DeleteObject deletes a blob in bucket
-func (l *b2Objects) DeleteObject(ctx context.Context, bucket string, object string) error {
-	bkt, err := l.Bucket(ctx, bucket)
-	if err != nil {
-		return err
-	}
-	reader, err := bkt.DownloadFileByName(l.ctx, object, 0, 1)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return b2ToObjectError(err, bucket, object)
-	}
-	io.Copy(ioutil.Discard, reader)
-	reader.Close()
-	err = bkt.File(reader.ID, object).DeleteFileVersion(l.ctx)
-	logger.LogIf(ctx, err)
-	return b2ToObjectError(err, bucket, object)
-}
-
-// ListMultipartUploads lists all multipart uploads.
-func (l *b2Objects) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string,
-	delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, err error) {
-	// keyMarker, prefix, delimiter are all ignored, Backblaze B2 doesn't support any
-	// of these parameters only equivalent parameter is uploadIDMarker.
-	bkt, err := l.Bucket(ctx, bucket)
-	if err != nil {
-		return lmi, err
-	}
-	// The maximum number of files to return from this call.
-	// The default value is 100, and the maximum allowed is 100.
-	if maxUploads > 100 {
-		maxUploads = 100
-	}
-	largeFiles, nextMarker, err := bkt.ListUnfinishedLargeFiles(l.ctx, uploadIDMarker, maxUploads)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return lmi, b2ToObjectError(err, bucket)
-	}
-	lmi = minio.ListMultipartsInfo{
-		MaxUploads: maxUploads,
-	}
-	if nextMarker != "" {
-		lmi.IsTruncated = true
-		lmi.NextUploadIDMarker = nextMarker
-	}
-	for _, largeFile := range largeFiles {
-		lmi.Uploads = append(lmi.Uploads, minio.MultipartInfo{
-			Object:    largeFile.Name,
-			UploadID:  largeFile.ID,
-			Initiated: largeFile.Timestamp,
-		})
-	}
-	return lmi, nil
-}
-
-// NewMultipartUpload upload object in multiple parts, uses B2's LargeFile upload API.
-// Large files can range in size from 5MB to 10TB.
-// Each large file must consist of at least 2 parts, and all of the parts except the
-// last one must be at least 5MB in size. The last part must contain at least one byte.
-// For more information - https://www.backblaze.com/b2/docs/large_files.html
-func (l *b2Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string, o minio.ObjectOptions) (string, error) {
-	var uploadID string
-	bkt, err := l.Bucket(ctx, bucket)
-	if err != nil {
-		return uploadID, err
-	}
-
-	contentType := metadata["content-type"]
-	delete(metadata, "content-type")
-	lf, err := bkt.StartLargeFile(l.ctx, object, contentType, metadata)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return uploadID, b2ToObjectError(err, bucket, object)
-	}
-
-	return lf.ID, nil
-}
-
-// PutObjectPart puts a part of object in bucket, uses B2's LargeFile upload API.
-func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, r *minio.PutObjReader, opts minio.ObjectOptions) (pi minio.PartInfo, err error) {
-	data := r.Reader
-	bkt, err := l.Bucket(ctx, bucket)
-	if err != nil {
-		return pi, err
-	}
-
-	fc, err := bkt.File(uploadID, object).CompileParts(0, nil).GetUploadPartURL(l.ctx)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return pi, b2ToObjectError(err, bucket, object, uploadID)
-	}
-
-	hr := newB2Reader(data, data.Size())
-	sha1, err := fc.UploadPart(l.ctx, hr, sha1AtEOF, int(hr.Size()), partID)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return pi, b2ToObjectError(err, bucket, object, uploadID)
-	}
-
-	return minio.PartInfo{
-		PartNumber:   partID,
-		LastModified: minio.UTCNow(),
-		ETag:         minio.ToS3ETag(sha1),
-		Size:         data.Size(),
-	}, nil
-}
-
-// ListObjectParts returns all object parts for specified object in specified bucket, uses B2's LargeFile upload API.
-func (l *b2Objects) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi minio.ListPartsInfo, err error) {
-	bkt, err := l.Bucket(ctx, bucket)
-	if err != nil {
-		return lpi, err
-	}
-	lpi = minio.ListPartsInfo{
-		Bucket:           bucket,
-		Object:           object,
-		UploadID:         uploadID,
-		MaxParts:         maxParts,
-		PartNumberMarker: partNumberMarker,
-	}
-	// startPartNumber must be in the range 1 - 10000 for B2.
-	partNumberMarker++
-	partsList, next, err := bkt.File(uploadID, object).ListParts(l.ctx, partNumberMarker, maxParts)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return lpi, b2ToObjectError(err, bucket, object, uploadID)
-	}
-	if next != 0 {
-		lpi.IsTruncated = true
-		lpi.NextPartNumberMarker = next
-	}
-	for _, part := range partsList {
-		lpi.Parts = append(lpi.Parts, minio.PartInfo{
-			PartNumber: part.Number,
-			ETag:       minio.ToS3ETag(part.SHA1),
-			Size:       part.Size,
-		})
-	}
-	return lpi, nil
-}
-
-// AbortMultipartUpload aborts a on going multipart upload, uses B2's LargeFile upload API.
-func (l *b2Objects) AbortMultipartUpload(ctx context.Context, bucket string, object string, uploadID string) error {
-	bkt, err := l.Bucket(ctx, bucket)
-	if err != nil {
-		return err
-	}
-	err = bkt.File(uploadID, object).CompileParts(0, nil).CancelLargeFile(l.ctx)
-	logger.LogIf(ctx, err)
-	return b2ToObjectError(err, bucket, object, uploadID)
-}
-
-// CompleteMultipartUpload completes ongoing multipart upload and finalizes object, uses B2's LargeFile upload API.
-func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (oi minio.ObjectInfo, err error) {
-	bkt, err := l.Bucket(ctx, bucket)
-	if err != nil {
-		return oi, err
-	}
-	hashes := make(map[int]string)
-	for i, uploadedPart := range uploadedParts {
-		// B2 requires contigous part numbers starting with 1, they do not support
-		// hand picking part numbers, we return an S3 compatible error instead.
-		if i+1 != uploadedPart.PartNumber {
-			logger.LogIf(ctx, minio.InvalidPart{})
-			return oi, b2ToObjectError(minio.InvalidPart{}, bucket, object, uploadID)
-		}
-
-		// Trim "-1" suffix in ETag as PutObjectPart() treats B2 returned SHA1 as ETag.
-		hashes[uploadedPart.PartNumber] = strings.TrimSuffix(uploadedPart.ETag, "-1")
-	}
-
-	if _, err = bkt.File(uploadID, object).CompileParts(0, hashes).FinishLargeFile(l.ctx); err != nil {
-		logger.LogIf(ctx, err)
-		return oi, b2ToObjectError(err, bucket, object, uploadID)
-	}
-
-	return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
-}
-
-// SetBucketPolicy - B2 supports 2 types of bucket policies:
-// bucketType.AllPublic - bucketTypeReadOnly means that anybody can download the files is the bucket;
-// bucketType.AllPrivate - bucketTypePrivate means that you need an authorization token to download them.
-// Default is AllPrivate for all buckets.
-func (l *b2Objects) SetBucketPolicy(ctx context.Context, bucket string, bucketPolicy *policy.Policy) error {
-	policyInfo, err := minio.PolicyToBucketAccessPolicy(bucketPolicy)
-	if err != nil {
-		// This should not happen.
-		return b2ToObjectError(err, bucket)
-	}
-
-	var policies []minio.BucketAccessPolicy
-	for prefix, policy := range miniogopolicy.GetPolicies(policyInfo.Statements, bucket, "") {
-		policies = append(policies, minio.BucketAccessPolicy{
-			Prefix: prefix,
-			Policy: policy,
-		})
-	}
-	prefix := bucket + "/*" // For all objects inside the bucket.
-	if len(policies) != 1 {
-		logger.LogIf(ctx, minio.NotImplemented{})
-		return minio.NotImplemented{}
-	}
-	if policies[0].Prefix != prefix {
-		logger.LogIf(ctx, minio.NotImplemented{})
-		return minio.NotImplemented{}
-	}
-	if policies[0].Policy != miniogopolicy.BucketPolicyReadOnly {
-		logger.LogIf(ctx, minio.NotImplemented{})
-		return minio.NotImplemented{}
-	}
-	bkt, err := l.Bucket(ctx, bucket)
-	if err != nil {
-		return err
-	}
-	bkt.Type = bucketTypeReadOnly
-	_, err = bkt.Update(l.ctx)
-	logger.LogIf(ctx, err)
-	return b2ToObjectError(err)
-}
-
-// GetBucketPolicy, returns the current bucketType from B2 backend and convert
-// it into S3 compatible bucket policy info.
-func (l *b2Objects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
-	bkt, err := l.Bucket(ctx, bucket)
-	if err != nil {
-		return nil, err
-	}
-
-	// bkt.Type can also be snapshot, but it is only allowed through B2 browser console,
-	// just return back as policy not found for all cases.
-	// CreateBucket always sets the value to allPrivate by default.
-	if bkt.Type != bucketTypeReadOnly {
-		return nil, minio.BucketPolicyNotFound{Bucket: bucket}
-	}
-
-	return &policy.Policy{
-		Version: policy.DefaultVersion,
-		Statements: []policy.Statement{
-			policy.NewStatement(
-				policy.Allow,
-				policy.NewPrincipal("*"),
-				policy.NewActionSet(
-					policy.GetBucketLocationAction,
-					policy.ListBucketAction,
-					policy.GetObjectAction,
-				),
-				policy.NewResourceSet(
-					policy.NewResource(bucket, ""),
-					policy.NewResource(bucket, "*"),
-				),
-				condition.NewFunctions(),
-			),
-		},
-	}, nil
-}
-
-// DeleteBucketPolicy - resets the bucketType of bucket on B2 to 'allPrivate'.
-func (l *b2Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error {
-	bkt, err := l.Bucket(ctx, bucket)
-	if err != nil {
-		return err
-	}
-	bkt.Type = bucketTypePrivate
-	_, err = bkt.Update(l.ctx)
-	logger.LogIf(ctx, err)
-	return b2ToObjectError(err)
-}
-
-// IsCompressionSupported returns whether compression is applicable for this layer.
-func (l *b2Objects) IsCompressionSupported() bool {
-	return false
-}

cmd/gateway/b2/gateway-b2_test.go

@@ -1,119 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package b2
-
-import (
-	"fmt"
-	"testing"
-
-	b2 "github.com/minio/blazer/base"
-
-	minio "github.com/minio/minio/cmd"
-)
-
-// Test b2 object error.
-func TestB2ObjectError(t *testing.T) {
-	testCases := []struct {
-		params      []string
-		b2Err       error
-		expectedErr error
-	}{
-		{
-			[]string{}, nil, nil,
-		},
-		{
-			[]string{}, fmt.Errorf("Not *Error"), fmt.Errorf("Not *Error"),
-		},
-		{
-			[]string{}, fmt.Errorf("Non B2 Error"), fmt.Errorf("Non B2 Error"),
-		},
-		{
-			[]string{"bucket"}, b2.Error{
-				StatusCode: 1,
-				Code:       "duplicate_bucket_name",
-			}, minio.BucketAlreadyOwnedByYou{
-				Bucket: "bucket",
-			},
-		},
-		{
-			[]string{"bucket"}, b2.Error{
-				StatusCode: 1,
-				Code:       "bad_request",
-			}, minio.BucketNotFound{
-				Bucket: "bucket",
-			},
-		},
-		{
-			[]string{"bucket", "object"}, b2.Error{
-				StatusCode: 1,
-				Code:       "bad_request",
-			}, minio.ObjectNameInvalid{
-				Bucket: "bucket",
-				Object: "object",
-			},
-		},
-		{
-			[]string{"bucket"}, b2.Error{
-				StatusCode: 1,
-				Code:       "bad_bucket_id",
-			}, minio.BucketNotFound{Bucket: "bucket"},
-		},
-		{
-			[]string{"bucket", "object"}, b2.Error{
-				StatusCode: 1,
-				Code:       "file_not_present",
-			}, minio.ObjectNotFound{
-				Bucket: "bucket",
-				Object: "object",
-			},
-		},
-		{
-			[]string{"bucket", "object"}, b2.Error{
-				StatusCode: 1,
-				Code:       "not_found",
-			}, minio.ObjectNotFound{
-				Bucket: "bucket",
-				Object: "object",
-			},
-		},
-		{
-			[]string{"bucket"}, b2.Error{
-				StatusCode: 1,
-				Code:       "cannot_delete_non_empty_bucket",
-			}, minio.BucketNotEmpty{
-				Bucket: "bucket",
-			},
-		},
-		{
-			[]string{"bucket", "object", "uploadID"}, b2.Error{
-				StatusCode: 1,
-				Message:    "No active upload for",
-			}, minio.InvalidUploadID{
-				UploadID: "uploadID",
-			},
-		},
-	}
-
-	for i, testCase := range testCases {
-		actualErr := b2ToObjectError(testCase.b2Err, testCase.params...)
-		if actualErr != nil {
-			if actualErr.Error() != testCase.expectedErr.Error() {
-				t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.expectedErr, actualErr)
-			}
-		}
-	}
-}

cmd/gateway/gateway.go

@@ -19,12 +19,9 @@ package gateway
 import (
 	// Import all gateways.
 	_ "github.com/minio/minio/cmd/gateway/azure"
-	_ "github.com/minio/minio/cmd/gateway/b2"
 	_ "github.com/minio/minio/cmd/gateway/gcs"
-	_ "github.com/minio/minio/cmd/gateway/manta"
 	_ "github.com/minio/minio/cmd/gateway/nas"
 	_ "github.com/minio/minio/cmd/gateway/oss"
 	_ "github.com/minio/minio/cmd/gateway/s3"
-	_ "github.com/minio/minio/cmd/gateway/sia"
 	// Add your gateway here.
 )

cmd/gateway/gcs/gateway-gcs.go

@@ -1033,8 +1033,7 @@ func (l *gcsGateway) ListMultipartUploads(ctx context.Context, bucket string, pr
 				UploadID:  components[3],
 				Initiated: attrs.Created,
 			}
-			uploads = []minio.MultipartInfo{upload}
+			uploads = append(uploads, upload)
-			break
 		}
 	}
 
@@ -1121,7 +1120,7 @@ func gcsGetPartInfo(ctx context.Context, attrs *storage.ObjectAttrs) (minio.Part
 }
 
 //  ListObjectParts returns all object parts for specified object in specified bucket
-func (l *gcsGateway) ListObjectParts(ctx context.Context, bucket string, key string, uploadID string, partNumberMarker int, maxParts int) (minio.ListPartsInfo, error) {
+func (l *gcsGateway) ListObjectParts(ctx context.Context, bucket string, key string, uploadID string, partNumberMarker int, maxParts int, opts minio.ObjectOptions) (minio.ListPartsInfo, error) {
 	it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{
 		Prefix: path.Join(gcsMinioMultipartPathV1, uploadID),
 	})

cmd/gateway/manta/gateway-manta.go

@@ -1,666 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package manta
-
-import (
-	"context"
-	"encoding/pem"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"path"
-	"strings"
-
-	triton "github.com/joyent/triton-go"
-	"github.com/joyent/triton-go/authentication"
-	terrors "github.com/joyent/triton-go/errors"
-	"github.com/joyent/triton-go/storage"
-	"github.com/minio/cli"
-	minio "github.com/minio/minio/cmd"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/auth"
-)
-
-// stor is a namespace within manta where you store any documents that are deemed as private
-// and require access credentials to read them. Within the stor namespace, you can create any
-// number of directories and objects.
-const (
-	mantaBackend     = "manta"
-	defaultMantaRoot = "/stor"
-	defaultMantaURL  = "https://us-east.manta.joyent.com"
-)
-
-var mantaRoot = defaultMantaRoot
-
-func init() {
-	const mantaGatewayTemplate = `NAME:
-  {{.HelpName}} - {{.Usage}}
-USAGE:
-  {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT]
-{{if .VisibleFlags}}
-FLAGS:
-  {{range .VisibleFlags}}{{.}}
-  {{end}}{{end}}
-ENDPOINT:
-  Manta server endpoint. Default ENDPOINT is https://us-east.manta.joyent.com
-
-ENVIRONMENT VARIABLES:
-  ACCESS:
-     MINIO_ACCESS_KEY: The Manta account name.
-     MINIO_SECRET_KEY: A KeyID associated with the Manta account.
-     MANTA_KEY_MATERIAL: The path to the SSH Key associated with the Manta account if the MINIO_SECRET_KEY is not in SSH Agent.
-	 MANTA_SUBUSER: The username of a user who has limited access to your account.
-
-  BROWSER:
-     MINIO_BROWSER: To disable web browser access, set this value to "off".
-
-  DOMAIN:
-     MINIO_DOMAIN: To enable virtual-host-style requests, set this value to Minio host domain name.
-
-  CACHE:
-     MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
-     MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
-     MINIO_CACHE_EXPIRY: Cache expiry duration in days.
-     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
-
-EXAMPLES:
-  1. Start minio gateway server for Manta Object Storage backend.
-     $ export MINIO_ACCESS_KEY=manta_account_name
-     $ export MINIO_SECRET_KEY=manta_key_id
-     $ {{.HelpName}}
-
-  2. Start minio gateway server for Manta Object Storage backend on custom endpoint.
-     $ export MINIO_ACCESS_KEY=manta_account_name
-     $ export MINIO_SECRET_KEY=manta_key_id
-     $ {{.HelpName}} https://us-west.manta.joyent.com
-
-  3. Start minio gateway server for Manta Object Storage backend without using SSH Agent.
-     $ export MINIO_ACCESS_KEY=manta_account_name
-     $ export MINIO_SECRET_KEY=manta_key_id
-     $ export MANTA_KEY_MATERIAL=~/.ssh/custom_rsa
-     $ {{.HelpName}}
-
-  4. Start minio gateway server for Manta Object Storage backend with edge caching enabled.
-     $ export MINIO_ACCESS_KEY=manta_account_name
-     $ export MINIO_SECRET_KEY=manta_key_id
-     $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
-     $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
-     $ export MINIO_CACHE_EXPIRY=40
-     $ export MINIO_CACHE_MAXUSE=80
-     $ {{.HelpName}}
-`
-
-	minio.RegisterGatewayCommand(cli.Command{
-		Name:               mantaBackend,
-		Usage:              "Manta Object Storage",
-		Action:             mantaGatewayMain,
-		CustomHelpTemplate: mantaGatewayTemplate,
-		HideHelpCommand:    true,
-	})
-}
-
-func mantaGatewayMain(ctx *cli.Context) {
-	args := ctx.Args()
-	if !ctx.Args().Present() {
-		args = cli.Args{"https://us-east.manta.joyent.com"}
-	}
-
-	// Validate gateway arguments.
-	logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), args.First()), "Invalid argument")
-
-	// Start the gateway..
-	minio.StartGateway(ctx, &Manta{args.First()})
-}
-
-// Manta implements Gateway.
-type Manta struct {
-	host string
-}
-
-// Name implements Gateway interface.
-func (g *Manta) Name() string {
-	return mantaBackend
-}
-
-// NewGatewayLayer returns manta gateway layer, implements ObjectLayer interface to
-// talk to manta remote backend.
-func (g *Manta) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
-	var err error
-	var secure bool
-	var signer authentication.Signer
-	var endpoint = defaultMantaURL
-	ctx := context.Background()
-
-	if g.host != "" {
-		endpoint, secure, err = minio.ParseGatewayEndpoint(g.host)
-		if err != nil {
-			return nil, err
-		}
-		if secure {
-			endpoint = "https://" + endpoint
-		} else {
-			endpoint = "http://" + endpoint
-		}
-	}
-	if overrideRoot, ok := os.LookupEnv("MANTA_ROOT"); ok {
-		mantaRoot = overrideRoot
-	}
-
-	keyMaterial := os.Getenv("MANTA_KEY_MATERIAL")
-
-	if keyMaterial == "" {
-		input := authentication.SSHAgentSignerInput{
-			KeyID:       creds.SecretKey,
-			AccountName: creds.AccessKey,
-		}
-		if userName, ok := os.LookupEnv("MANTA_SUBUSER"); ok {
-			input.Username = userName
-		}
-		signer, err = authentication.NewSSHAgentSigner(input)
-		if err != nil {
-			logger.LogIf(ctx, err)
-			return nil, err
-		}
-	} else {
-		var keyBytes []byte
-		if _, err = os.Stat(keyMaterial); err == nil {
-			keyBytes, err = ioutil.ReadFile(keyMaterial)
-			if err != nil {
-				return nil, fmt.Errorf("Error reading key material from %s: %s",
-					keyMaterial, err)
-			}
-			block, _ := pem.Decode(keyBytes)
-			if block == nil {
-				return nil, fmt.Errorf(
-					"Failed to read key material '%s': no key found", keyMaterial)
-			}
-
-			if block.Headers["Proc-Type"] == "4,ENCRYPTED" {
-				return nil, fmt.Errorf(
-					"Failed to read key '%s': password protected keys are\n"+
-						"not currently supported. Please decrypt the key prior to use.", keyMaterial)
-			}
-
-		} else {
-			keyBytes = []byte(keyMaterial)
-		}
-
-		input := authentication.PrivateKeySignerInput{
-			KeyID:              creds.SecretKey,
-			PrivateKeyMaterial: keyBytes,
-			AccountName:        creds.AccessKey,
-		}
-		if userName, ok := os.LookupEnv("MANTA_SUBUSER"); ok {
-			input.Username = userName
-		}
-
-		signer, err = authentication.NewPrivateKeySigner(input)
-		if err != nil {
-			logger.LogIf(ctx, err)
-			return nil, err
-		}
-	}
-
-	tc, err := storage.NewClient(&triton.ClientConfig{
-		MantaURL:    endpoint,
-		AccountName: creds.AccessKey,
-		Signers:     []authentication.Signer{signer},
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	tc.Client.HTTPClient = &http.Client{
-		Transport: minio.NewCustomHTTPTransport(),
-	}
-
-	return &tritonObjects{
-		client: tc,
-	}, nil
-}
-
-// Production - Manta is production ready.
-func (g *Manta) Production() bool {
-	return true
-}
-
-// tritonObjects - Implements Object layer for Triton Manta storage
-type tritonObjects struct {
-	minio.GatewayUnsupported
-	client *storage.StorageClient
-}
-
-// Shutdown - save any gateway metadata to disk
-// if necessary and reload upon next restart.
-func (t *tritonObjects) Shutdown(ctx context.Context) error {
-	return nil
-}
-
-// StorageInfo - Not relevant to Triton backend.
-func (t *tritonObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
-	return si
-}
-
-//
-// ~~~ Buckets ~~~
-//
-
-// MakeBucketWithLocation - Create a new directory within manta.
-//
-// https://apidocs.joyent.com/manta/api.html#PutDirectory
-func (t *tritonObjects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error {
-	err := t.client.Dir().Put(ctx, &storage.PutDirectoryInput{
-		DirectoryName: path.Join(mantaRoot, bucket),
-	})
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-// GetBucketInfo - Get directory metadata..
-//
-// https://apidocs.joyent.com/manta/api.html#GetObject
-func (t *tritonObjects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, e error) {
-	var info minio.BucketInfo
-	resp, err := t.client.Objects().Get(ctx, &storage.GetObjectInput{
-		ObjectPath: path.Join(mantaRoot, bucket),
-	})
-	if err != nil {
-		return info, err
-	}
-
-	return minio.BucketInfo{
-		Name:    bucket,
-		Created: resp.LastModified,
-	}, nil
-}
-
-// ListBuckets - Lists all Manta directories, uses Manta equivalent
-// ListDirectories.
-//
-// https://apidocs.joyent.com/manta/api.html#ListDirectory
-func (t *tritonObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketInfo, err error) {
-	dirs, err := t.client.Dir().List(ctx, &storage.ListDirectoryInput{
-		DirectoryName: path.Join(mantaRoot),
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	for _, dir := range dirs.Entries {
-		if dir.Type == "directory" {
-			buckets = append(buckets, minio.BucketInfo{
-				Name:    dir.Name,
-				Created: dir.ModifiedTime,
-			})
-		}
-	}
-
-	return buckets, nil
-}
-
-// DeleteBucket - Delete a directory in Manta, uses Manta equivalent
-// DeleteDirectory.
-//
-// https://apidocs.joyent.com/manta/api.html#DeleteDirectory
-func (t *tritonObjects) DeleteBucket(ctx context.Context, bucket string) error {
-	return t.client.Dir().Delete(ctx, &storage.DeleteDirectoryInput{
-		DirectoryName: path.Join(mantaRoot, bucket),
-	})
-}
-
-//
-// ~~~ Objects ~~~
-//
-
-// ListObjects - Lists all objects in Manta with a container filtered by prefix
-// and marker, uses Manta equivalent ListDirectory.
-//
-// https://apidocs.joyent.com/manta/api.html#ListDirectory
-func (t *tritonObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result minio.ListObjectsInfo, err error) {
-	var (
-		dirName string
-		objs    *storage.ListDirectoryOutput
-		input   *storage.ListDirectoryInput
-
-		pathBase = path.Base(prefix)
-	)
-
-	// Make sure to only request a Dir.List for the parent "directory" for a
-	// given prefix first. We don't know if our prefix is referencing a
-	// directory or file name and can't send file names into Dir.List because
-	// that'll cause Manta to return file content in the response body. Dir.List
-	// expects to parse out directory entries in JSON. So, try the first
-	// directory name of the prefix path provided.
-	if pathDir := path.Dir(prefix); pathDir == "." {
-		dirName = path.Join(mantaRoot, bucket)
-	} else {
-		dirName = path.Join(mantaRoot, bucket, pathDir)
-	}
-
-	if marker != "" {
-		// Manta uses the marker as the key to start at rather than start after
-		// A space is appended to the marker so that the corresponding object is not
-		// included in the results
-		marker += " "
-	}
-
-	input = &storage.ListDirectoryInput{
-		DirectoryName: dirName,
-		Limit:         uint64(maxKeys),
-		Marker:        marker,
-	}
-	objs, err = t.client.Dir().List(ctx, input)
-	if err != nil {
-		if terrors.IsResourceNotFoundError(err) {
-			return result, nil
-		}
-		logger.LogIf(ctx, err)
-		return result, err
-	}
-
-	for _, obj := range objs.Entries {
-		// If the base name of our prefix was found to be of type "directory"
-		// than we need to pull the directory entries for that instead.
-		if obj.Name == pathBase && obj.Type == "directory" {
-			input.DirectoryName = path.Join(mantaRoot, bucket, prefix)
-			objs, err = t.client.Dir().List(ctx, input)
-			if err != nil {
-				logger.LogIf(ctx, err)
-				return result, err
-			}
-			break
-		}
-	}
-
-	isTruncated := true // Always send a second request.
-	if marker == "" && len(objs.Entries) < maxKeys {
-		isTruncated = false
-	} else if marker != "" && len(objs.Entries) < maxKeys {
-		isTruncated = false
-	}
-
-	for _, obj := range objs.Entries {
-		if obj.Type == "directory" {
-			result.Prefixes = append(result.Prefixes, obj.Name+delimiter)
-		} else {
-			result.Objects = append(result.Objects, minio.ObjectInfo{
-				Name:    obj.Name,
-				Size:    int64(obj.Size),
-				ModTime: obj.ModifiedTime,
-				ETag:    obj.ETag,
-			})
-		}
-	}
-
-	result.IsTruncated = isTruncated
-	if isTruncated {
-		result.NextMarker = result.Objects[len(result.Objects)-1].Name
-	}
-	return result, nil
-}
-
-//
-// ~~~ Objects ~~~
-//
-
-// ListObjectsV2 - Lists all objects in Manta with a container filtered by prefix
-// and continuationToken, uses Manta equivalent ListDirectory.
-//
-// https://apidocs.joyent.com/manta/api.html#ListDirectory
-func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result minio.ListObjectsV2Info, err error) {
-	var (
-		dirName string
-		objs    *storage.ListDirectoryOutput
-		input   *storage.ListDirectoryInput
-
-		pathBase = path.Base(prefix)
-	)
-
-	marker := continuationToken
-	if marker == "" {
-		marker = startAfter
-	}
-
-	if marker != "" {
-		// Manta uses the marker as the key to start at rather than start after.
-		// A space is appended to the marker so that the corresponding object is not
-		// included in the results
-		marker += " "
-	}
-
-	if pathDir := path.Dir(prefix); pathDir == "." {
-		dirName = path.Join(mantaRoot, bucket)
-	} else {
-		dirName = path.Join(mantaRoot, bucket, pathDir)
-	}
-
-	input = &storage.ListDirectoryInput{
-		DirectoryName: dirName,
-		Limit:         uint64(maxKeys),
-		Marker:        marker,
-	}
-	objs, err = t.client.Dir().List(ctx, input)
-	if err != nil {
-		if terrors.IsResourceNotFoundError(err) {
-			return result, nil
-		}
-		logger.LogIf(ctx, err)
-		return result, err
-	}
-
-	for _, obj := range objs.Entries {
-		if obj.Name == pathBase && obj.Type == "directory" {
-			input.DirectoryName = path.Join(mantaRoot, bucket, prefix)
-			objs, err = t.client.Dir().List(ctx, input)
-			if err != nil {
-				logger.LogIf(ctx, err)
-				return result, err
-			}
-			break
-		}
-	}
-
-	isTruncated := true // Always send a second request.
-	if continuationToken == "" && len(objs.Entries) < maxKeys {
-		isTruncated = false
-	} else if continuationToken != "" && len(objs.Entries) < maxKeys {
-		isTruncated = false
-	}
-
-	for _, obj := range objs.Entries {
-		if obj.Type == "directory" {
-			result.Prefixes = append(result.Prefixes, obj.Name+delimiter)
-		} else {
-			result.Objects = append(result.Objects, minio.ObjectInfo{
-				Name:    obj.Name,
-				Size:    int64(obj.Size),
-				ModTime: obj.ModifiedTime,
-				ETag:    obj.ETag,
-			})
-		}
-	}
-
-	result.IsTruncated = isTruncated
-	if isTruncated {
-		result.NextContinuationToken = result.Objects[len(result.Objects)-1].Name
-	}
-	return result, nil
-}
-
-// GetObjectNInfo - returns object info and locked object ReadCloser
-func (t *tritonObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
-	var objInfo minio.ObjectInfo
-	objInfo, err = t.GetObjectInfo(ctx, bucket, object, opts)
-	if err != nil {
-		return nil, err
-	}
-
-	var startOffset, length int64
-	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
-	if err != nil {
-		return nil, err
-	}
-
-	pr, pw := io.Pipe()
-	go func() {
-		err := t.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
-		pw.CloseWithError(err)
-	}()
-	// Setup cleanup function to cause the above go-routine to
-	// exit in case of partial read
-	pipeCloser := func() { pr.Close() }
-	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
-}
-
-// GetObject - Reads an object from Manta. Supports additional parameters like
-// offset and length which are synonymous with HTTP Range requests.
-//
-// startOffset indicates the starting read location of the object.  length
-// indicates the total length of the object.
-//
-// https://apidocs.joyent.com/manta/api.html#GetObject
-func (t *tritonObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
-	// Start offset cannot be negative.
-	if startOffset < 0 {
-		logger.LogIf(ctx, fmt.Errorf("Unexpected error"))
-		return fmt.Errorf("Unexpected error")
-	}
-
-	output, err := t.client.Objects().Get(ctx, &storage.GetObjectInput{
-		ObjectPath: path.Join(mantaRoot, bucket, object),
-	})
-	if err != nil {
-		return err
-	}
-	defer output.ObjectReader.Close()
-
-	// Read until startOffset and discard, Manta object storage doesn't support range GET requests yet.
-	if _, err = io.CopyN(ioutil.Discard, output.ObjectReader, startOffset); err != nil {
-		return err
-	}
-
-	if length > 0 {
-		_, err = io.Copy(writer, io.LimitReader(output.ObjectReader, length))
-	} else {
-		_, err = io.Copy(writer, output.ObjectReader)
-	}
-	return err
-}
-
-// GetObjectInfo - reads blob metadata properties and replies back minio.ObjectInfo,
-// uses Triton equivalent GetBlobProperties.
-//
-// https://apidocs.joyent.com/manta/api.html#GetObject
-func (t *tritonObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
-	info, err := t.client.Objects().GetInfo(ctx, &storage.GetInfoInput{
-		ObjectPath: path.Join(mantaRoot, bucket, object),
-	})
-	if err != nil {
-		if terrors.IsStatusNotFoundCode(err) {
-			return objInfo, minio.ObjectNotFound{
-				Bucket: bucket,
-				Object: object,
-			}
-		}
-
-		return objInfo, err
-	}
-
-	return minio.ObjectInfo{
-		Bucket:      bucket,
-		ContentType: info.ContentType,
-		Size:        int64(info.ContentLength),
-		ETag:        info.ETag,
-		ModTime:     info.LastModified,
-		UserDefined: info.Metadata,
-		IsDir:       strings.HasSuffix(info.ContentType, "type=directory"),
-	}, nil
-}
-
-type dummySeeker struct {
-	io.Reader
-}
-
-func (d dummySeeker) Seek(offset int64, whence int) (int64, error) {
-	return 0, nil
-}
-
-// PutObject - Create a new blob with the incoming data, uses Triton equivalent
-// CreateBlockBlobFromReader.
-//
-// https://apidocs.joyent.com/manta/api.html#PutObject
-func (t *tritonObjects) PutObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
-	data := r.Reader
-	if err = t.client.Objects().Put(ctx, &storage.PutObjectInput{
-		ContentLength: uint64(data.Size()),
-		ObjectPath:    path.Join(mantaRoot, bucket, object),
-		ContentType:   metadata["content-type"],
-		// TODO: Change to `string(data.md5sum)` if/when that becomes an exported field
-		ContentMD5:   metadata["content-md5"],
-		ObjectReader: dummySeeker{data},
-		ForceInsert:  true,
-	}); err != nil {
-		logger.LogIf(ctx, err)
-		return objInfo, err
-	}
-	if err = data.Verify(); err != nil {
-		t.DeleteObject(ctx, bucket, object)
-		logger.LogIf(ctx, err)
-		return objInfo, err
-	}
-
-	return t.GetObjectInfo(ctx, bucket, object, opts)
-}
-
-// CopyObject - Copies a blob from source container to destination container.
-// Uses Manta Snaplinks API.
-//
-// https://apidocs.joyent.com/manta/api.html#PutSnapLink
-func (t *tritonObjects) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
-	if err = t.client.SnapLinks().Put(ctx, &storage.PutSnapLinkInput{
-		SourcePath: path.Join(mantaRoot, srcBucket, srcObject),
-		LinkPath:   path.Join(mantaRoot, destBucket, destObject),
-	}); err != nil {
-		logger.LogIf(ctx, err)
-		return objInfo, err
-	}
-
-	return t.GetObjectInfo(ctx, destBucket, destObject, dstOpts)
-}
-
-// DeleteObject - Delete a blob in Manta, uses Triton equivalent DeleteBlob API.
-//
-// https://apidocs.joyent.com/manta/api.html#DeleteObject
-func (t *tritonObjects) DeleteObject(ctx context.Context, bucket, object string) error {
-	if err := t.client.Objects().Delete(ctx, &storage.DeleteObjectInput{
-		ObjectPath: path.Join(mantaRoot, bucket, object),
-	}); err != nil {
-		logger.LogIf(ctx, err)
-		return err
-	}
-
-	return nil
-}
-
-// IsCompressionSupported returns whether compression is applicable for this layer.
-func (t *tritonObjects) IsCompressionSupported() bool {
-	return false
-}

cmd/gateway/nas/gateway-nas.go

@@ -17,6 +17,8 @@
 package nas
 
 import (
+	"context"
+
 	"github.com/minio/cli"
 	minio "github.com/minio/minio/cmd"
 	"github.com/minio/minio/pkg/auth"
@@ -117,10 +119,16 @@ func (g *NAS) Production() bool {
 }
 
 // IsListenBucketSupported returns whether listen bucket notification is applicable for this gateway.
-func (g *NAS) IsListenBucketSupported() bool {
+func (n *nasObjects) IsListenBucketSupported() bool {
 	return false
 }
 
+func (n *nasObjects) StorageInfo(ctx context.Context) minio.StorageInfo {
+	sinfo := n.ObjectLayer.StorageInfo(ctx)
+	sinfo.Backend.Type = minio.Unknown
+	return sinfo
+}
+
 // nasObjects implements gateway for Minio and S3 compatible object storage servers.
 type nasObjects struct {
 	minio.ObjectLayer

cmd/gateway/oss/gateway-oss.go

@@ -31,6 +31,7 @@ import (
 	"github.com/minio/cli"
 	miniogopolicy "github.com/minio/minio-go/pkg/policy"
 	minio "github.com/minio/minio/cmd"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/hash"
@@ -844,7 +845,7 @@ func ossListObjectParts(client *oss.Client, bucket, object, uploadID string, par
 	}
 
 	// always drain output (response body)
-	defer minio.CloseResponse(resp.Body)
+	defer xhttp.DrainBody(resp.Body)
 
 	err = xml.NewDecoder(resp.Body).Decode(&lupr)
 	if err != nil {
@@ -886,7 +887,7 @@ func (l *ossObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d
 }
 
 // ListObjectParts returns all object parts for specified object in specified bucket
-func (l *ossObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int) (lpi minio.ListPartsInfo, err error) {
+func (l *ossObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int, opts minio.ObjectOptions) (lpi minio.ListPartsInfo, err error) {
 	lupr, err := ossListObjectParts(l.Client, bucket, object, uploadID, partNumberMarker, maxParts)
 	if err != nil {
 		logger.LogIf(ctx, err)

cmd/gateway/s3/gateway-s3-metadata.go

@@ -0,0 +1,238 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package s3
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"time"
+
+	minio "github.com/minio/minio/cmd"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/hash"
+	"github.com/tidwall/gjson"
+)
+
+var (
+	errGWMetaNotFound      = errors.New("dare.meta file not found")
+	errGWMetaInvalidFormat = errors.New("dare.meta format is invalid")
+)
+
+// A gwMetaV1 represents `gw.json` metadata header.
+type gwMetaV1 struct {
+	Version string         `json:"version"` // Version of the current `gw.json`.
+	Format  string         `json:"format"`  // Format of the current `gw.json`.
+	Stat    minio.StatInfo `json:"stat"`    // Stat of the current object `gw.json`.
+	ETag    string         `json:"etag"`    // ETag of the current object
+
+	// Metadata map for current object `gw.json`.
+	Meta map[string]string `json:"meta,omitempty"`
+	// Captures all the individual object `gw.json`.
+	Parts []minio.ObjectPartInfo `json:"parts,omitempty"`
+}
+
+// Gateway metadata constants.
+const (
+	// Gateway meta version.
+	gwMetaVersion = "1.0.0"
+
+	// Gateway meta version.
+	gwMetaVersion100 = "1.0.0"
+
+	// Gateway meta format string.
+	gwMetaFormat = "gw"
+
+	// Add new constants here.
+)
+
+// newGWMetaV1 - initializes new gwMetaV1, adds version.
+func newGWMetaV1() (gwMeta gwMetaV1) {
+	gwMeta = gwMetaV1{}
+	gwMeta.Version = gwMetaVersion
+	gwMeta.Format = gwMetaFormat
+	return gwMeta
+}
+
+// IsValid - tells if the format is sane by validating the version
+// string, format fields.
+func (m gwMetaV1) IsValid() bool {
+	return ((m.Version == gwMetaVersion || m.Version == gwMetaVersion100) &&
+		m.Format == gwMetaFormat)
+}
+
+// Converts metadata to object info.
+func (m gwMetaV1) ToObjectInfo(bucket, object string) minio.ObjectInfo {
+	filterKeys := append([]string{
+		"ETag",
+		"Content-Length",
+		"Last-Modified",
+		"Content-Type",
+	}, defaultFilterKeys...)
+	objInfo := minio.ObjectInfo{
+		IsDir:           false,
+		Bucket:          bucket,
+		Name:            object,
+		Size:            m.Stat.Size,
+		ModTime:         m.Stat.ModTime,
+		ContentType:     m.Meta["content-type"],
+		ContentEncoding: m.Meta["content-encoding"],
+		ETag:            minio.CanonicalizeETag(m.ETag),
+		UserDefined:     minio.CleanMetadataKeys(m.Meta, filterKeys...),
+		Parts:           m.Parts,
+	}
+
+	if sc, ok := m.Meta["x-amz-storage-class"]; ok {
+		objInfo.StorageClass = sc
+	}
+	// Success.
+	return objInfo
+}
+
+// ObjectToPartOffset - translate offset of an object to offset of its individual part.
+func (m gwMetaV1) ObjectToPartOffset(ctx context.Context, offset int64) (partIndex int, partOffset int64, err error) {
+	if offset == 0 {
+		// Special case - if offset is 0, then partIndex and partOffset are always 0.
+		return 0, 0, nil
+	}
+	partOffset = offset
+	// Seek until object offset maps to a particular part offset.
+	for i, part := range m.Parts {
+		partIndex = i
+		// Offset is smaller than size we have reached the proper part offset.
+		if partOffset < part.Size {
+			return partIndex, partOffset, nil
+		}
+		// Continue to towards the next part.
+		partOffset -= part.Size
+	}
+	logger.LogIf(ctx, minio.InvalidRange{})
+	// Offset beyond the size of the object return InvalidRange.
+	return 0, 0, minio.InvalidRange{}
+}
+
+// parses gateway metadata stat info from metadata json
+func parseGWStat(gwMetaBuf []byte) (si minio.StatInfo, e error) {
+	// obtain stat info.
+	stat := minio.StatInfo{}
+	// fetching modTime.
+	modTime, err := time.Parse(time.RFC3339, gjson.GetBytes(gwMetaBuf, "stat.modTime").String())
+	if err != nil {
+		return si, err
+	}
+	stat.ModTime = modTime
+	// obtain Stat.Size .
+	stat.Size = gjson.GetBytes(gwMetaBuf, "stat.size").Int()
+	return stat, nil
+}
+
+// parses gateway metadata version from metadata json
+func parseGWVersion(gwMetaBuf []byte) string {
+	return gjson.GetBytes(gwMetaBuf, "version").String()
+}
+
+// parses gateway ETag from metadata json
+func parseGWETag(gwMetaBuf []byte) string {
+	return gjson.GetBytes(gwMetaBuf, "etag").String()
+}
+
+// parses gateway metadata format from metadata json
+func parseGWFormat(gwMetaBuf []byte) string {
+	return gjson.GetBytes(gwMetaBuf, "format").String()
+}
+
+// parses gateway metadata json to get list of ObjectPartInfo
+func parseGWParts(gwMetaBuf []byte) []minio.ObjectPartInfo {
+	// Parse the GW Parts.
+	partsResult := gjson.GetBytes(gwMetaBuf, "parts").Array()
+	partInfo := make([]minio.ObjectPartInfo, len(partsResult))
+	for i, p := range partsResult {
+		info := minio.ObjectPartInfo{}
+		info.Number = int(p.Get("number").Int())
+		info.Name = p.Get("name").String()
+		info.ETag = p.Get("etag").String()
+		info.Size = p.Get("size").Int()
+		partInfo[i] = info
+	}
+	return partInfo
+}
+
+// parses gateway metadata json to get the metadata map
+func parseGWMetaMap(gwMetaBuf []byte) map[string]string {
+	// Get gwMetaV1.Meta map.
+	metaMapResult := gjson.GetBytes(gwMetaBuf, "meta").Map()
+	metaMap := make(map[string]string)
+	for key, valResult := range metaMapResult {
+		metaMap[key] = valResult.String()
+	}
+	return metaMap
+}
+
+// Constructs GWMetaV1 using `gjson` lib to retrieve each field.
+func gwMetaUnmarshalJSON(ctx context.Context, gwMetaBuf []byte) (gwMeta gwMetaV1, e error) {
+	// obtain version.
+	gwMeta.Version = parseGWVersion(gwMetaBuf)
+	// obtain format.
+	gwMeta.Format = parseGWFormat(gwMetaBuf)
+	// Parse gwMetaV1.Stat .
+	stat, err := parseGWStat(gwMetaBuf)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return gwMeta, err
+	}
+	gwMeta.ETag = parseGWETag(gwMetaBuf)
+	gwMeta.Stat = stat
+
+	// Parse the GW Parts.
+	gwMeta.Parts = parseGWParts(gwMetaBuf)
+	// parse gwMetaV1.
+	gwMeta.Meta = parseGWMetaMap(gwMetaBuf)
+
+	return gwMeta, nil
+}
+
+// readGWMeta reads `dare.meta` and returns back GW metadata structure.
+func readGWMetadata(ctx context.Context, buf bytes.Buffer) (gwMeta gwMetaV1, err error) {
+	if buf.Len() == 0 {
+		return gwMetaV1{}, errGWMetaNotFound
+	}
+	gwMeta, err = gwMetaUnmarshalJSON(ctx, buf.Bytes())
+	if err != nil {
+		return gwMetaV1{}, err
+	}
+	if !gwMeta.IsValid() {
+		return gwMetaV1{}, errGWMetaInvalidFormat
+	}
+	// Return structured `dare.meta`.
+	return gwMeta, nil
+}
+
+// getGWMetadata - unmarshals dare.meta into a *minio.PutObjReader
+func getGWMetadata(ctx context.Context, bucket, prefix string, gwMeta gwMetaV1) (*minio.PutObjReader, error) {
+	// Marshal json.
+	metadataBytes, err := json.Marshal(&gwMeta)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return nil, err
+	}
+	hashReader, err := hash.NewReader(bytes.NewReader(metadataBytes), int64(len(metadataBytes)), "", "", int64(len(metadataBytes)))
+	if err != nil {
+		return nil, err
+	}
+	return minio.NewPutObjReader(hashReader, nil, nil), nil
+}

cmd/gateway/s3/gateway-s3-metadata_test.go

@@ -0,0 +1,78 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package s3
+
+import (
+	"bytes"
+	"context"
+	"testing"
+)
+
+// Tests for GW metadata format validity.
+func TestGWMetaFormatValid(t *testing.T) {
+	tests := []struct {
+		name    int
+		version string
+		format  string
+		want    bool
+	}{
+		{1, "123", "fs", false},
+		{2, "123", gwMetaFormat, false},
+		{3, gwMetaVersion, "test", false},
+		{4, gwMetaVersion100, "hello", false},
+		{5, gwMetaVersion, gwMetaFormat, true},
+		{6, gwMetaVersion100, gwMetaFormat, true},
+	}
+	for _, tt := range tests {
+		m := newGWMetaV1()
+		m.Version = tt.version
+		m.Format = tt.format
+		if got := m.IsValid(); got != tt.want {
+			t.Errorf("Test %d: Expected %v but received %v", tt.name, got, tt.want)
+		}
+	}
+}
+
+// Tests for reading GW metadata info.
+func TestReadGWMetadata(t *testing.T) {
+	tests := []struct {
+		metaStr string
+		pass    bool
+	}{
+		{`{"version": "` + gwMetaVersion + `", "format":"` + gwMetaFormat + `", {"stat": {"size": "132", "modTime": "2018-08-31T22:25:39.23626461Z" }}}`, true},
+		{`{"version": "` + gwMetaVersion + `", "format":"` + gwMetaFormat + `", {"stat": {"size": "132", "modTime": "0000-00-00T00:00:00.00000000Z" }}}`, false},
+		{`{"version": "` + gwMetaVersion + `", "format":"` + gwMetaFormat + `", {"stat": {"size": "5242880", "modTime": "2018-08-31T22:25:39.23626461Z" }},"meta":{"content-type":"application/octet-stream","etag":"57c743902b2fc8eea6ba3bb4fc58c8e8"},"parts":[{"number":1,"name":"part.1","etag":"","size":5242880}]}}`, true},
+		{`{"version": "` + gwMetaVersion + `", "format":"` + gwMetaFormat + `", {"stat": {"size": "68190720", "modTime": "2018-08-31T22:25:39.23626461Z" }},"meta":{"X-Minio-Internal-Encrypted-Multipart":"","X-Minio-Internal-Server-Side-Encryption-Iv":"kdbOcKdXD3Sew8tOiHe5eI9xkX1oQ2W9JURz0oslCZA=","X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm":"DAREv2-HMAC-SHA256","X-Minio-Internal-Server-Side-Encryption-Sealed-Key":"IAAfAMfqKrxMXC9LuiI7ENP+p0xArepzAiIeB/MftFp7Xmq2OzDkKlmNbj5RKI89RrjiAbOVLSSEMvqQsrIrTQ==","content-type":"text/plain; charset=utf-8","etag":"2b137fa4ab80126af54623b010c98de6-2"},"parts":[{"number":1,"name":"part.1","etag":"c5cac075eefdab801a5198812f51b36e","size":67141632},{"number":2,"name":"part.2","etag":"ccdf4b774bc3be8eef9a8987309e8171","size":1049088}]}`, true},
+		{`{"version": "` + gwMetaVersion + `", "format":"` + gwMetaFormat + `", {"stat": {"size": "68190720", "modTime": "2018-08-31T22:25:39.23626461Z" }},"meta":{"X-Minio-Internal-Encrypted-Multipart":"","X-Minio-Internal-Server-Side-Encryption-Iv":"kdbOcKdXD3Sew8tOiHe5eI9xkX1oQ2W9JURz0oslCZA=","X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm":"DAREv2-HMAC-SHA256","X-Minio-Internal-Server-Side-Encryption-Sealed-Key":"IAAfAMfqKrxMXC9LuiI7ENP+p0xArepzAiIeB/MftFp7Xmq2OzDkKlmNbj5RKI89RrjiAbOVLSSEMvqQsrIrTQ==","content-type":"text/plain; charset=utf-8","etag":"2b137fa4ab80126af54623b010c98de6-2"},"parts":"123"}`, true},
+	}
+
+	for i, tt := range tests {
+		buf := bytes.NewBufferString(tt.metaStr)
+		m, err := readGWMetadata(context.Background(), *buf)
+		if err != nil && tt.pass {
+			t.Errorf("Test %d: Expected parse gw metadata to succeed, but failed", i)
+		}
+		if err == nil && !tt.pass {
+			t.Errorf("Test %d: Expected parse gw metadata to succeed, but failed", i)
+		}
+		if err == nil {
+			if m.Version != gwMetaVersion {
+				t.Errorf("Test %d: Expected version %s, but failed with %s", i, gwMetaVersion, m.Version)
+			}
+		}
+	}
+}

cmd/gateway/s3/gateway-s3-sse.go

@@ -0,0 +1,778 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package s3
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"net/http"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/minio/minio-go/pkg/encrypt"
+	minio "github.com/minio/minio/cmd"
+
+	"github.com/minio/minio/cmd/logger"
+)
+
+const (
+	// name of custom multipart metadata file for s3 backend.
+	gwdareMetaJSON string = "dare.meta"
+
+	// name of temporary per part metadata file
+	gwpartMetaJSON string = "part.meta"
+	// custom multipart files are stored under the defaultMinioGWPrefix
+	defaultMinioGWPrefix     = ".minio"
+	defaultGWContentFileName = "data"
+	slashSeparator           = "/"
+)
+
+// s3EncObjects is a wrapper around s3Objects and implements gateway calls for
+// custom large objects encrypted at the gateway
+type s3EncObjects struct {
+	s3Objects
+}
+
+/*
+ NOTE:
+ Custom gateway encrypted objects are stored on backend as follows:
+	 obj/.minio/data   <= encrypted content
+	 obj/.minio/dare.meta  <= metadata
+
+ When a multipart upload operation is in progress, the metadata set during
+ NewMultipartUpload is stored in obj/.minio/uploadID/dare.meta and each
+ UploadPart operation saves additional state of the part's encrypted ETag and
+ encrypted size in obj/.minio/uploadID/part1/part.meta
+
+ All the part metadata and temp dare.meta are cleaned up when upload completes
+*/
+
+// ListObjects lists all blobs in S3 bucket filtered by prefix
+func (l *s3EncObjects) ListObjects(ctx context.Context, bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, e error) {
+	var continuationToken, startAfter string
+	res, err := l.ListObjectsV2(ctx, bucket, prefix, continuationToken, delimiter, maxKeys, false, startAfter)
+	if err != nil {
+		return loi, err
+	}
+	loi.IsTruncated = res.IsTruncated
+	loi.NextMarker = res.NextContinuationToken
+	loi.Objects = res.Objects
+	loi.Prefixes = res.Prefixes
+	return loi, nil
+
+}
+
+// ListObjectsV2 lists all blobs in S3 bucket filtered by prefix
+func (l *s3EncObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, e error) {
+
+	var objects []minio.ObjectInfo
+	var prefixes []string
+	var isTruncated bool
+
+	// filter out objects that contain a .minio prefix, but is not a dare.meta metadata file.
+	for {
+		loi, e = l.s3Objects.ListObjectsV2(ctx, bucket, prefix, continuationToken, delimiter, 1000, fetchOwner, startAfter)
+		if e != nil {
+			return loi, minio.ErrorRespToObjectError(e, bucket)
+		}
+		for _, obj := range loi.Objects {
+			startAfter = obj.Name
+			continuationToken = loi.NextContinuationToken
+			isTruncated = loi.IsTruncated
+
+			if !isGWObject(obj.Name) {
+				continue
+			}
+			// get objectname and ObjectInfo from the custom metadata file
+			if strings.HasSuffix(obj.Name, gwdareMetaJSON) {
+				objSlice := strings.Split(obj.Name, slashSeparator+defaultMinioGWPrefix)
+				gwMeta, e := l.getGWMetadata(ctx, bucket, getDareMetaPath(objSlice[0]))
+				if e != nil {
+					continue
+				}
+				oInfo := gwMeta.ToObjectInfo(bucket, objSlice[0])
+				objects = append(objects, oInfo)
+			} else {
+				objects = append(objects, obj)
+			}
+			if len(objects) > maxKeys {
+				break
+			}
+		}
+		for _, p := range loi.Prefixes {
+			objName := strings.TrimSuffix(p, slashSeparator)
+			gm, err := l.getGWMetadata(ctx, bucket, getDareMetaPath(objName))
+			// if prefix is actually a custom multi-part object, append it to objects
+			if err == nil {
+				objects = append(objects, gm.ToObjectInfo(bucket, objName))
+				continue
+			}
+			isPrefix := l.isPrefix(ctx, bucket, p, fetchOwner, startAfter)
+			if isPrefix {
+				prefixes = append(prefixes, p)
+			}
+		}
+		if (len(objects) > maxKeys) || !loi.IsTruncated {
+			break
+		}
+	}
+
+	loi.IsTruncated = isTruncated
+	loi.ContinuationToken = continuationToken
+	loi.Objects = make([]minio.ObjectInfo, 0)
+	loi.Prefixes = make([]string, 0)
+
+	for _, obj := range objects {
+		loi.NextContinuationToken = obj.Name
+		loi.Objects = append(loi.Objects, obj)
+	}
+	for _, pfx := range prefixes {
+		if pfx != prefix {
+			loi.Prefixes = append(loi.Prefixes, pfx)
+		}
+	}
+	return loi, nil
+}
+
+// isGWObject returns true if it is a custom object
+func isGWObject(objName string) bool {
+	isEncrypted := strings.Contains(objName, defaultMinioGWPrefix)
+	if !isEncrypted {
+		return true
+	}
+	// ignore temp part.meta files
+	if strings.Contains(objName, gwpartMetaJSON) {
+		return false
+	}
+
+	pfxSlice := strings.Split(objName, slashSeparator)
+	var i1, i2 int
+	for i := len(pfxSlice) - 1; i >= 0; i-- {
+		p := pfxSlice[i]
+		if p == defaultMinioGWPrefix {
+			i1 = i
+		}
+		if p == gwdareMetaJSON {
+			i2 = i
+		}
+		if i1 > 0 && i2 > 0 {
+			break
+		}
+	}
+	// incomplete uploads would have a uploadID between defaultMinioGWPrefix and gwdareMetaJSON
+	return i2 > 0 && i1 > 0 && i2-i1 == 1
+}
+
+// isPrefix returns true if prefix exists and is not an incomplete multipart upload entry
+func (l *s3EncObjects) isPrefix(ctx context.Context, bucket, prefix string, fetchOwner bool, startAfter string) bool {
+	var continuationToken, delimiter string
+
+	for {
+		loi, e := l.s3Objects.ListObjectsV2(ctx, bucket, prefix, continuationToken, delimiter, 1000, fetchOwner, startAfter)
+		if e != nil {
+			return false
+		}
+		for _, obj := range loi.Objects {
+			if isGWObject(obj.Name) {
+				return true
+			}
+		}
+
+		continuationToken = loi.NextContinuationToken
+		if !loi.IsTruncated {
+			break
+		}
+	}
+	return false
+}
+
+// GetObject reads an object from S3. Supports additional
+// parameters like offset and length which are synonymous with
+// HTTP Range requests.
+func (l *s3EncObjects) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
+	return l.getObject(ctx, bucket, key, startOffset, length, writer, etag, opts)
+}
+
+func (l *s3EncObjects) isGWEncrypted(ctx context.Context, bucket, object string) bool {
+	_, err := l.s3Objects.GetObjectInfo(ctx, bucket, getDareMetaPath(object), minio.ObjectOptions{})
+	return err == nil
+}
+
+// getDaremetadata fetches dare.meta from s3 backend and marshals into a structured format.
+func (l *s3EncObjects) getGWMetadata(ctx context.Context, bucket, metaFileName string) (m gwMetaV1, err error) {
+	oi, err1 := l.s3Objects.GetObjectInfo(ctx, bucket, metaFileName, minio.ObjectOptions{})
+	if err1 != nil {
+		return m, err1
+	}
+	var buffer bytes.Buffer
+	err = l.s3Objects.GetObject(ctx, bucket, metaFileName, 0, oi.Size, &buffer, oi.ETag, minio.ObjectOptions{})
+	if err != nil {
+		return m, err
+	}
+	return readGWMetadata(ctx, buffer)
+}
+
+// writes dare metadata to the s3 backend
+func (l *s3EncObjects) writeGWMetadata(ctx context.Context, bucket, metaFileName string, m gwMetaV1, o minio.ObjectOptions) error {
+	reader, err := getGWMetadata(ctx, bucket, metaFileName, m)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return err
+	}
+	_, err = l.s3Objects.PutObject(ctx, bucket, metaFileName, reader, map[string]string{}, o)
+	return err
+}
+
+// returns path of temporary metadata json file for the upload
+func getTmpDareMetaPath(object, uploadID string) string {
+	return path.Join(getGWMetaPath(object), uploadID, gwdareMetaJSON)
+}
+
+// returns path of metadata json file for encrypted objects
+func getDareMetaPath(object string) string {
+	return path.Join(getGWMetaPath(object), gwdareMetaJSON)
+}
+
+// returns path of temporary part metadata file for multipart uploads
+func getPartMetaPath(object, uploadID string, partID int) string {
+	return path.Join(object, defaultMinioGWPrefix, uploadID, strconv.Itoa(partID), gwpartMetaJSON)
+}
+
+// deletes the custom dare metadata file saved at the backend
+func (l *s3EncObjects) deleteGWMetadata(ctx context.Context, bucket, metaFileName string) error {
+	return l.s3Objects.DeleteObject(ctx, bucket, metaFileName)
+}
+
+func (l *s3EncObjects) getObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
+	var o minio.ObjectOptions
+	if minio.GlobalGatewaySSE.SSEC() {
+		o = opts
+	}
+	dmeta, err := l.getGWMetadata(ctx, bucket, getDareMetaPath(key))
+	if err != nil {
+		// unencrypted content
+		return l.s3Objects.GetObject(ctx, bucket, key, startOffset, length, writer, etag, o)
+	}
+	if startOffset < 0 {
+		logger.LogIf(ctx, minio.InvalidRange{})
+	}
+
+	// For negative length read everything.
+	if length < 0 {
+		length = dmeta.Stat.Size - startOffset
+	}
+	// Reply back invalid range if the input offset and length fall out of range.
+	if startOffset > dmeta.Stat.Size || startOffset+length > dmeta.Stat.Size {
+		logger.LogIf(ctx, minio.InvalidRange{OffsetBegin: startOffset, OffsetEnd: length, ResourceSize: dmeta.Stat.Size})
+		return minio.InvalidRange{OffsetBegin: startOffset, OffsetEnd: length, ResourceSize: dmeta.Stat.Size}
+	}
+	// Get start part index and offset.
+	_, partOffset, err := dmeta.ObjectToPartOffset(ctx, startOffset)
+	if err != nil {
+		return minio.InvalidRange{OffsetBegin: startOffset, OffsetEnd: length, ResourceSize: dmeta.Stat.Size}
+	}
+
+	// Calculate endOffset according to length
+	endOffset := startOffset
+	if length > 0 {
+		endOffset += length - 1
+	}
+
+	// Get last part index to read given length.
+	if _, _, err := dmeta.ObjectToPartOffset(ctx, endOffset); err != nil {
+		return minio.InvalidRange{OffsetBegin: startOffset, OffsetEnd: length, ResourceSize: dmeta.Stat.Size}
+	}
+	return l.s3Objects.GetObject(ctx, bucket, key, partOffset, endOffset, writer, dmeta.ETag, o)
+}
+
+// GetObjectNInfo - returns object info and locked object ReadCloser
+func (l *s3EncObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, o minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
+	var opts minio.ObjectOptions
+	if minio.GlobalGatewaySSE.SSEC() {
+		opts = o
+	}
+	objInfo, err := l.GetObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		return l.s3Objects.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+	}
+	objInfo.UserDefined = minio.CleanMinioInternalMetadataKeys(objInfo.UserDefined)
+	fn, off, length, err := minio.NewGetObjectReader(rs, objInfo)
+	if err != nil {
+		return nil, minio.ErrorRespToObjectError(err)
+	}
+	if l.isGWEncrypted(ctx, bucket, object) {
+		object = getGWContentPath(object)
+	}
+	pr, pw := io.Pipe()
+	go func() {
+		err := l.getObject(ctx, bucket, object, off, length, pw, objInfo.ETag, opts)
+		pw.CloseWithError(err)
+	}()
+
+	// Setup cleanup function to cause the above go-routine to
+	// exit in case of partial read
+	pipeCloser := func() { pr.Close() }
+	return fn(pr, h, pipeCloser)
+}
+
+// GetObjectInfo reads object info and replies back ObjectInfo
+// For custom gateway encrypted large objects, the ObjectInfo is retrieved from the dare.meta file.
+func (l *s3EncObjects) GetObjectInfo(ctx context.Context, bucket string, object string, o minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	var opts minio.ObjectOptions
+	if minio.GlobalGatewaySSE.SSEC() {
+		opts = o
+	}
+
+	gwMeta, err := l.getGWMetadata(ctx, bucket, getDareMetaPath(object))
+	if err != nil {
+		return l.s3Objects.GetObjectInfo(ctx, bucket, object, opts)
+	}
+	return gwMeta.ToObjectInfo(bucket, object), nil
+}
+
+// CopyObject copies an object from source bucket to a destination bucket.
+func (l *s3EncObjects) CopyObject(ctx context.Context, srcBucket string, srcObject string, dstBucket string, dstObject string, srcInfo minio.ObjectInfo, s, d minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	cpSrcDstSame := strings.EqualFold(path.Join(srcBucket, srcObject), path.Join(dstBucket, dstObject))
+	if cpSrcDstSame {
+		var gwMeta gwMetaV1
+		if s.ServerSideEncryption != nil && d.ServerSideEncryption != nil &&
+			((s.ServerSideEncryption.Type() == encrypt.SSEC && d.ServerSideEncryption.Type() == encrypt.SSEC) ||
+				(s.ServerSideEncryption.Type() == encrypt.S3 && d.ServerSideEncryption.Type() == encrypt.S3)) {
+			gwMeta, err = l.getGWMetadata(ctx, srcBucket, getDareMetaPath(srcObject))
+			if err != nil {
+				return
+			}
+			header := make(http.Header)
+			if d.ServerSideEncryption != nil {
+				d.ServerSideEncryption.Marshal(header)
+			}
+			for k, v := range header {
+				srcInfo.UserDefined[k] = v[0]
+			}
+			gwMeta.Meta = srcInfo.UserDefined
+			if err = l.writeGWMetadata(ctx, dstBucket, getDareMetaPath(dstObject), gwMeta, minio.ObjectOptions{}); err != nil {
+				return objInfo, minio.ErrorRespToObjectError(err)
+			}
+			return gwMeta.ToObjectInfo(dstBucket, dstObject), nil
+		}
+	}
+	return l.PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, srcInfo.UserDefined, d)
+}
+
+// DeleteObject deletes a blob in bucket
+// For custom gateway encrypted large objects, cleans up encrypted content and metadata files
+// from the backend.
+func (l *s3EncObjects) DeleteObject(ctx context.Context, bucket string, object string) error {
+
+	// Get dare meta json
+	if _, err := l.getGWMetadata(ctx, bucket, getDareMetaPath(object)); err != nil {
+		return l.s3Objects.DeleteObject(ctx, bucket, object)
+	}
+	// delete encrypted object
+	l.s3Objects.DeleteObject(ctx, bucket, getGWContentPath(object))
+	return l.deleteGWMetadata(ctx, bucket, getDareMetaPath(object))
+}
+
+// ListMultipartUploads lists all multipart uploads.
+func (l *s3EncObjects) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, e error) {
+
+	lmi, e = l.s3Objects.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
+	if e != nil {
+		return
+	}
+	lmi.KeyMarker = strings.TrimSuffix(lmi.KeyMarker, getGWContentPath("/"))
+	lmi.NextKeyMarker = strings.TrimSuffix(lmi.NextKeyMarker, getGWContentPath("/"))
+	for i := range lmi.Uploads {
+		lmi.Uploads[i].Object = strings.TrimSuffix(lmi.Uploads[i].Object, getGWContentPath("/"))
+	}
+	return
+}
+
+// NewMultipartUpload uploads object in multiple parts
+func (l *s3EncObjects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string, o minio.ObjectOptions) (uploadID string, err error) {
+	var opts minio.ObjectOptions
+	if o.ServerSideEncryption != nil &&
+		((minio.GlobalGatewaySSE.SSEC() && o.ServerSideEncryption.Type() == encrypt.SSEC) ||
+			(minio.GlobalGatewaySSE.SSES3() && o.ServerSideEncryption.Type() == encrypt.S3)) {
+		opts = o
+	}
+	if o.ServerSideEncryption == nil {
+		return l.s3Objects.NewMultipartUpload(ctx, bucket, object, metadata, opts)
+	}
+	uploadID, err = l.s3Objects.NewMultipartUpload(ctx, bucket, getGWContentPath(object), map[string]string{}, opts)
+	if err != nil {
+		return
+	}
+	// Create uploadID and write a temporary dare.meta object under object/uploadID prefix
+	gwmeta := newGWMetaV1()
+	gwmeta.Meta = metadata
+	gwmeta.Stat.ModTime = time.Now().UTC()
+	err = l.writeGWMetadata(ctx, bucket, getTmpDareMetaPath(object, uploadID), gwmeta, minio.ObjectOptions{})
+	if err != nil {
+		return uploadID, minio.ErrorRespToObjectError(err)
+	}
+	return uploadID, nil
+}
+
+// PutObject creates a new object with the incoming data,
+func (l *s3EncObjects) PutObject(ctx context.Context, bucket string, object string, data *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	var s3Opts minio.ObjectOptions
+	if opts.ServerSideEncryption != nil &&
+		((minio.GlobalGatewaySSE.SSEC() && opts.ServerSideEncryption.Type() == encrypt.SSEC) ||
+			(minio.GlobalGatewaySSE.SSES3() && opts.ServerSideEncryption.Type() == encrypt.S3)) {
+		s3Opts = opts
+	}
+	if opts.ServerSideEncryption == nil {
+		defer l.deleteGWMetadata(ctx, bucket, getDareMetaPath(object))
+		defer l.DeleteObject(ctx, bucket, getGWContentPath(object))
+		return l.s3Objects.PutObject(ctx, bucket, object, data, metadata, s3Opts)
+	}
+
+	oi, err := l.s3Objects.PutObject(ctx, bucket, getGWContentPath(object), data, map[string]string{}, s3Opts)
+	if err != nil {
+		return objInfo, minio.ErrorRespToObjectError(err)
+	}
+
+	gwMeta := newGWMetaV1()
+	gwMeta.Meta = make(map[string]string)
+	for k, v := range oi.UserDefined {
+		gwMeta.Meta[k] = v
+	}
+	for k, v := range metadata {
+		gwMeta.Meta[k] = v
+	}
+	encMD5 := data.MD5CurrentHexString()
+
+	gwMeta.ETag = encMD5
+	gwMeta.Stat.Size = oi.Size
+	gwMeta.Stat.ModTime = time.Now().UTC()
+	if err = l.writeGWMetadata(ctx, bucket, getDareMetaPath(object), gwMeta, minio.ObjectOptions{}); err != nil {
+		return objInfo, minio.ErrorRespToObjectError(err)
+	}
+	objInfo = gwMeta.ToObjectInfo(bucket, object)
+	// delete any unencrypted content of the same name created previously
+	l.s3Objects.DeleteObject(ctx, bucket, object)
+	return objInfo, nil
+}
+
+// PutObjectPart puts a part of object in bucket
+func (l *s3EncObjects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *minio.PutObjReader, opts minio.ObjectOptions) (pi minio.PartInfo, e error) {
+
+	if opts.ServerSideEncryption == nil {
+		return l.s3Objects.PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
+	}
+
+	var s3Opts minio.ObjectOptions
+	// for sse-s3 encryption options should not be passed to backend
+	if opts.ServerSideEncryption != nil && opts.ServerSideEncryption.Type() == encrypt.SSEC && minio.GlobalGatewaySSE.SSEC() {
+		s3Opts = opts
+	}
+
+	uploadPath := getTmpGWMetaPath(object, uploadID)
+	tmpDareMeta := path.Join(uploadPath, gwdareMetaJSON)
+	_, err := l.s3Objects.GetObjectInfo(ctx, bucket, tmpDareMeta, minio.ObjectOptions{})
+	if err != nil {
+		return pi, minio.InvalidUploadID{UploadID: uploadID}
+	}
+
+	pi, e = l.s3Objects.PutObjectPart(ctx, bucket, getGWContentPath(object), uploadID, partID, data, s3Opts)
+	if e != nil {
+		return
+	}
+	gwMeta := newGWMetaV1()
+	gwMeta.Parts = make([]minio.ObjectPartInfo, 1)
+	// Add incoming part.
+	gwMeta.Parts[0] = minio.ObjectPartInfo{
+		Number: partID,
+		ETag:   pi.ETag,
+		Size:   pi.Size,
+		Name:   strconv.Itoa(partID),
+	}
+	gwMeta.ETag = data.MD5CurrentHexString() // encrypted ETag
+	gwMeta.Stat.Size = pi.Size
+	gwMeta.Stat.ModTime = pi.LastModified
+
+	if err = l.writeGWMetadata(ctx, bucket, getPartMetaPath(object, uploadID, partID), gwMeta, minio.ObjectOptions{}); err != nil {
+		return pi, minio.ErrorRespToObjectError(err)
+	}
+	return minio.PartInfo{
+		Size:         gwMeta.Stat.Size,
+		ETag:         minio.CanonicalizeETag(gwMeta.ETag),
+		LastModified: gwMeta.Stat.ModTime,
+		PartNumber:   partID,
+	}, nil
+}
+
+// CopyObjectPart creates a part in a multipart upload by copying
+// existing object or a part of it.
+func (l *s3EncObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string,
+	partID int, startOffset, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (p minio.PartInfo, err error) {
+	return l.PutObjectPart(ctx, destBucket, destObject, uploadID, partID, srcInfo.PutObjReader, dstOpts)
+}
+
+// ListObjectParts returns all object parts for specified object in specified bucket
+func (l *s3EncObjects) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int, opts minio.ObjectOptions) (lpi minio.ListPartsInfo, e error) {
+	// We do not store parts uploaded so far in the dare.meta. Only CompleteMultipartUpload finalizes the parts under upload prefix.Otherwise,
+	// there could be situations of dare.meta getting corrupted by competing upload parts.
+	dm, err := l.getGWMetadata(ctx, bucket, getTmpDareMetaPath(object, uploadID))
+	if err != nil {
+		return l.s3Objects.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
+	}
+
+	lpi, err = l.s3Objects.ListObjectParts(ctx, bucket, getGWContentPath(object), uploadID, partNumberMarker, maxParts, opts)
+	if err != nil {
+		return lpi, err
+	}
+	for i, part := range lpi.Parts {
+		partMeta, err := l.getGWMetadata(ctx, bucket, getPartMetaPath(object, uploadID, part.PartNumber))
+		if err != nil || len(partMeta.Parts) == 0 {
+			return lpi, minio.InvalidPart{}
+		}
+		lpi.Parts[i].ETag = partMeta.ETag
+	}
+	lpi.UserDefined = dm.Meta
+	lpi.Object = object
+	return lpi, nil
+}
+
+// AbortMultipartUpload aborts a ongoing multipart upload
+func (l *s3EncObjects) AbortMultipartUpload(ctx context.Context, bucket string, object string, uploadID string) error {
+	if _, err := l.getGWMetadata(ctx, bucket, getTmpDareMetaPath(object, uploadID)); err != nil {
+		return l.s3Objects.AbortMultipartUpload(ctx, bucket, object, uploadID)
+	}
+
+	if err := l.s3Objects.AbortMultipartUpload(ctx, bucket, getGWContentPath(object), uploadID); err != nil {
+		return err
+	}
+
+	uploadPrefix := getTmpGWMetaPath(object, uploadID)
+	var continuationToken, startAfter, delimiter string
+	for {
+		loi, err := l.s3Objects.ListObjectsV2(ctx, bucket, uploadPrefix, continuationToken, delimiter, 1000, false, startAfter)
+		if err != nil {
+			return minio.InvalidUploadID{UploadID: uploadID}
+		}
+		for _, obj := range loi.Objects {
+			if err := l.s3Objects.DeleteObject(ctx, bucket, obj.Name); err != nil {
+				return minio.ErrorRespToObjectError(err)
+			}
+			startAfter = obj.Name
+		}
+		continuationToken = loi.NextContinuationToken
+		if !loi.IsTruncated {
+			break
+		}
+	}
+	return nil
+}
+
+// CompleteMultipartUpload completes ongoing multipart upload and finalizes object
+func (l *s3EncObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (oi minio.ObjectInfo, e error) {
+
+	tmpMeta, err := l.getGWMetadata(ctx, bucket, getTmpDareMetaPath(object, uploadID))
+	if err != nil {
+		oi, e = l.s3Objects.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
+		if e == nil {
+			// delete any encrypted version of object that might exist
+			defer l.deleteGWMetadata(ctx, bucket, getDareMetaPath(object))
+			defer l.DeleteObject(ctx, bucket, getGWContentPath(object))
+		}
+		return oi, e
+	}
+	gwMeta := newGWMetaV1()
+	gwMeta.Meta = make(map[string]string)
+	for k, v := range tmpMeta.Meta {
+		gwMeta.Meta[k] = v
+	}
+	// Allocate parts similar to incoming slice.
+	gwMeta.Parts = make([]minio.ObjectPartInfo, len(uploadedParts))
+
+	bkUploadedParts := make([]minio.CompletePart, len(uploadedParts))
+	// Calculate full object size.
+	var objectSize int64
+
+	// Validate each part and then commit to disk.
+	for i, part := range uploadedParts {
+		partMeta, err := l.getGWMetadata(ctx, bucket, getPartMetaPath(object, uploadID, part.PartNumber))
+		if err != nil || len(partMeta.Parts) == 0 {
+			return oi, minio.InvalidPart{}
+		}
+		bkUploadedParts[i] = minio.CompletePart{PartNumber: part.PartNumber, ETag: partMeta.Parts[0].ETag}
+		gwMeta.Parts[i] = partMeta.Parts[0]
+		objectSize += partMeta.Parts[0].Size
+	}
+	oi, e = l.s3Objects.CompleteMultipartUpload(ctx, bucket, getGWContentPath(object), uploadID, bkUploadedParts, opts)
+	if e != nil {
+		return oi, e
+	}
+
+	//delete any unencrypted version of object that might be on the backend
+	defer l.s3Objects.DeleteObject(ctx, bucket, object)
+
+	// Save the final object size and modtime.
+	gwMeta.Stat.Size = objectSize
+	gwMeta.Stat.ModTime = time.Now().UTC()
+	gwMeta.ETag = oi.ETag
+
+	if err = l.writeGWMetadata(ctx, bucket, getDareMetaPath(object), gwMeta, minio.ObjectOptions{}); err != nil {
+		return oi, minio.ErrorRespToObjectError(err)
+	}
+	// Clean up any uploaded parts that are not being committed by this CompleteMultipart operation
+	var continuationToken, startAfter, delimiter string
+	uploadPrefix := getTmpGWMetaPath(object, uploadID)
+	done := false
+	for {
+		loi, lerr := l.s3Objects.ListObjectsV2(ctx, bucket, uploadPrefix, continuationToken, delimiter, 1000, false, startAfter)
+		if lerr != nil {
+			done = true
+			break
+		}
+		for _, obj := range loi.Objects {
+			if !strings.HasPrefix(obj.Name, uploadPrefix) {
+				done = true
+				break
+			}
+			startAfter = obj.Name
+			l.s3Objects.DeleteObject(ctx, bucket, obj.Name)
+		}
+		continuationToken = loi.NextContinuationToken
+		if !loi.IsTruncated || done {
+			break
+		}
+	}
+
+	return gwMeta.ToObjectInfo(bucket, object), nil
+}
+
+// getTmpGWMetaPath returns the prefix under which uploads in progress are stored on backend
+func getTmpGWMetaPath(object, uploadID string) string {
+	return path.Join(object, defaultMinioGWPrefix, uploadID)
+}
+
+// getGWMetaPath returns the prefix under which custom object metadata and object are stored on backend after upload completes
+func getGWMetaPath(object string) string {
+	return path.Join(object, defaultMinioGWPrefix)
+}
+
+// getGWContentPath returns the prefix under which custom object is stored on backend after upload completes
+func getGWContentPath(object string) string {
+	return path.Join(object, defaultMinioGWPrefix, defaultGWContentFileName)
+}
+
+// Clean-up the stale incomplete encrypted multipart uploads. Should be run in a Go routine.
+func (l *s3EncObjects) cleanupStaleEncMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh chan struct{}) {
+	ticker := time.NewTicker(cleanupInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-doneCh:
+			return
+		case <-ticker.C:
+			l.cleanupStaleEncMultipartUploadsOnGW(ctx, expiry)
+		}
+	}
+}
+
+// cleanupStaleMultipartUploads removes old custom encryption multipart uploads on backend
+func (l *s3EncObjects) cleanupStaleEncMultipartUploadsOnGW(ctx context.Context, expiry time.Duration) {
+	for {
+		buckets, err := l.s3Objects.ListBuckets(ctx)
+		if err != nil {
+			break
+		}
+		for _, b := range buckets {
+			expParts := l.getStalePartsForBucket(ctx, b.Name, expiry)
+			for k := range expParts {
+				l.s3Objects.DeleteObject(ctx, b.Name, k)
+			}
+		}
+	}
+}
+
+func (l *s3EncObjects) getStalePartsForBucket(ctx context.Context, bucket string, expiry time.Duration) (expParts map[string]string) {
+	var prefix, continuationToken, delimiter, startAfter string
+	expParts = make(map[string]string)
+	now := time.Now()
+	for {
+		loi, err := l.s3Objects.ListObjectsV2(ctx, bucket, prefix, continuationToken, delimiter, 1000, false, startAfter)
+		if err != nil {
+			break
+		}
+		for _, obj := range loi.Objects {
+			startAfter = obj.Name
+			if !strings.Contains(obj.Name, defaultMinioGWPrefix) {
+				continue
+			}
+
+			if isGWObject(obj.Name) {
+				continue
+			}
+
+			// delete temporary part.meta or dare.meta files for incomplete uploads that are past expiry
+			if (strings.HasSuffix(obj.Name, gwpartMetaJSON) || strings.HasSuffix(obj.Name, gwdareMetaJSON)) &&
+				now.Sub(obj.ModTime) > expiry {
+				expParts[obj.Name] = ""
+			}
+		}
+		continuationToken = loi.NextContinuationToken
+		if !loi.IsTruncated {
+			break
+		}
+	}
+	return
+}
+
+func (l *s3EncObjects) DeleteBucket(ctx context.Context, bucket string) error {
+	var prefix, continuationToken, delimiter, startAfter string
+	expParts := make(map[string]string)
+
+	for {
+		loi, err := l.s3Objects.ListObjectsV2(ctx, bucket, prefix, continuationToken, delimiter, 1000, false, startAfter)
+		if err != nil {
+			break
+		}
+		for _, obj := range loi.Objects {
+			startAfter = obj.Name
+			if !strings.Contains(obj.Name, defaultMinioGWPrefix) {
+				return minio.BucketNotEmpty{}
+			}
+			if isGWObject(obj.Name) {
+				return minio.BucketNotEmpty{}
+			}
+			// delete temporary part.meta or dare.meta files for incomplete uploads
+			if strings.HasSuffix(obj.Name, gwpartMetaJSON) || strings.HasSuffix(obj.Name, gwdareMetaJSON) {
+				expParts[obj.Name] = ""
+			}
+		}
+		continuationToken = loi.NextContinuationToken
+		if !loi.IsTruncated {
+			break
+		}
+	}
+	for k := range expParts {
+		l.s3Objects.DeleteObject(ctx, bucket, k)
+	}
+	err := l.Client.RemoveBucket(bucket)
+	if err != nil {
+		return minio.ErrorRespToObjectError(err, bucket)
+	}
+	return nil
+}

cmd/gateway/s3/gateway-s3-utils.go

@@ -0,0 +1,47 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package s3
+
+import minio "github.com/minio/minio/cmd"
+
+// List of header keys to be filtered, usually
+// from all S3 API http responses.
+var defaultFilterKeys = []string{
+	"Connection",
+	"Transfer-Encoding",
+	"Accept-Ranges",
+	"Date",
+	"Server",
+	"Vary",
+	"x-amz-bucket-region",
+	"x-amz-request-id",
+	"x-amz-id-2",
+	"Content-Security-Policy",
+	"X-Xss-Protection",
+
+	// Add new headers to be ignored.
+}
+
+// FromGatewayObjectPart converts ObjectInfo for custom part stored as object to PartInfo
+func FromGatewayObjectPart(partID int, oi minio.ObjectInfo) (pi minio.PartInfo) {
+	return minio.PartInfo{
+		Size:         oi.Size,
+		ETag:         minio.CanonicalizeETag(oi.ETag),
+		LastModified: oi.ModTime,
+		PartNumber:   partID,
+	}
+}

cmd/gateway/s3/gateway-s3.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2017, 2018 Minio, Inc.
+ * Minio Cloud Storage, (C) 2017, 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,18 +22,20 @@ import (
 	"io"
 	"math/rand"
 	"net/http"
+	"net/url"
 	"strings"
 	"time"
 
 	"github.com/minio/cli"
 	miniogo "github.com/minio/minio-go"
 	"github.com/minio/minio-go/pkg/credentials"
+	minio "github.com/minio/minio/cmd"
+
+	"github.com/minio/minio-go/pkg/encrypt"
 	"github.com/minio/minio-go/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/policy"
-
-	minio "github.com/minio/minio/cmd"
 )
 
 const (
@@ -167,35 +169,61 @@ func randString(n int, src rand.Source, prefix string) string {
 	return prefix + string(b[0:30-len(prefix)])
 }
 
-// newS3 - Initializes a new client by auto probing S3 server signature.
+func isAmazonS3Endpoint(urlStr string) bool {
-func newS3(url string) (*miniogo.Core, error) {
+	u, err := url.Parse(urlStr)
-	if url == "" {
-		url = "https://s3.amazonaws.com"
-	}
-
-	// Override default params if the host is provided
-	endpoint, secure, err := minio.ParseGatewayEndpoint(url)
 	if err != nil {
-		return nil, err
+		panic(err)
 	}
+	return s3utils.IsAmazonEndpoint(*u)
+}
 
-	// Chains all credential types, in the following order:
+// Chains all credential types, in the following order:
-	//  - AWS env vars (i.e. AWS_ACCESS_KEY_ID)
+//  - AWS env vars (i.e. AWS_ACCESS_KEY_ID)
-	//  - IAM profile based credentials. (performs an HTTP
+//  - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials)
-	//    call to a pre-defined endpoint, only valid inside
+//  - Static credentials provided by user (i.e. MINIO_ACCESS_KEY)
-	//    configured ec2 instances)
+var defaultProviders = []credentials.Provider{
-	//  - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials)
-	//  - Static credentials provided by user (i.e. MINIO_ACCESS_KEY)
-	creds := credentials.NewChainCredentials([]credentials.Provider{
 	&credentials.EnvAWS{},
+	&credentials.FileAWSCredentials{},
+	&credentials.EnvMinio{},
+}
+
+// Chains all credential types, in the following order:
+//  - AWS env vars (i.e. AWS_ACCESS_KEY_ID)
+//  - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials)
+//  - IAM profile based credentials. (performs an HTTP
+//    call to a pre-defined endpoint, only valid inside
+//    configured ec2 instances)
+var defaultAWSCredProviders = []credentials.Provider{
+	&credentials.EnvAWS{},
+	&credentials.FileAWSCredentials{},
 	&credentials.IAM{
 		Client: &http.Client{
 			Transport: minio.NewCustomHTTPTransport(),
 		},
 	},
-		&credentials.FileAWSCredentials{},
 	&credentials.EnvMinio{},
-	})
+}
+
+// newS3 - Initializes a new client by auto probing S3 server signature.
+func newS3(urlStr string) (*miniogo.Core, error) {
+	if urlStr == "" {
+		urlStr = "https://s3.amazonaws.com"
+	}
+
+	// Override default params if the host is provided
+	endpoint, secure, err := minio.ParseGatewayEndpoint(urlStr)
+	if err != nil {
+		return nil, err
+	}
+
+	var creds *credentials.Credentials
+	if isAmazonS3Endpoint(urlStr) {
+		// If we see an Amazon S3 endpoint, then we use more ways to fetch backend credentials.
+		// Specifically IAM style rotating credentials are only supported with AWS S3 endpoint.
+		creds = credentials.NewChainCredentials(defaultAWSCredProviders)
+	} else {
+		creds = credentials.NewChainCredentials(defaultProviders)
+	}
 
 	clnt, err := miniogo.NewWithCredentials(endpoint, creds, secure, "")
 	if err != nil {
@@ -223,9 +251,20 @@ func (g *S3) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
 		return nil, err
 	}
 
-	return &s3Objects{
+	s := s3Objects{
 		Client: clnt,
-	}, nil
+	}
+	// Enables single encyption of KMS is configured.
+	if minio.GlobalKMS != nil {
+		encS := s3EncObjects{s}
+
+		// Start stale enc multipart uploads cleanup routine.
+		go encS.cleanupStaleEncMultipartUploads(context.Background(),
+			minio.GlobalMultipartCleanupInterval, minio.GlobalMultipartExpiry, minio.GlobalServiceDoneCh)
+
+		return &encS, nil
+	}
+	return &s, nil
 }
 
 // Production - s3 gateway is production ready.
@@ -330,6 +369,7 @@ func (l *s3Objects) ListObjects(ctx context.Context, bucket string, prefix strin
 
 // ListObjectsV2 lists all blobs in S3 bucket filtered by prefix
 func (l *s3Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, e error) {
+
 	result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys, startAfter)
 	if err != nil {
 		return loi, minio.ErrorRespToObjectError(err, bucket)
@@ -387,7 +427,6 @@ func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, st
 		return minio.ErrorRespToObjectError(err, bucket, key)
 	}
 	defer object.Close()
-
 	if _, err := io.Copy(writer, object); err != nil {
 		return minio.ErrorRespToObjectError(err, bucket, key)
 	}
@@ -396,7 +435,11 @@ func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, st
 
 // GetObjectInfo reads object info and replies back ObjectInfo
 func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
-	oi, err := l.Client.StatObject(bucket, object, miniogo.StatObjectOptions{GetObjectOptions: miniogo.GetObjectOptions{ServerSideEncryption: opts.ServerSideEncryption}})
+	oi, err := l.Client.StatObject(bucket, object, miniogo.StatObjectOptions{
+		GetObjectOptions: miniogo.GetObjectOptions{
+			ServerSideEncryption: opts.ServerSideEncryption,
+		},
+	})
 	if err != nil {
 		return minio.ObjectInfo{}, minio.ErrorRespToObjectError(err, bucket, object)
 	}
@@ -426,6 +469,18 @@ func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject
 	// So preserve it by adding "REPLACE" directive to save all the metadata set by CopyObject API.
 	srcInfo.UserDefined["x-amz-metadata-directive"] = "REPLACE"
 	srcInfo.UserDefined["x-amz-copy-source-if-match"] = srcInfo.ETag
+	header := make(http.Header)
+	if srcOpts.ServerSideEncryption != nil {
+		encrypt.SSECopy(srcOpts.ServerSideEncryption).Marshal(header)
+	}
+
+	if dstOpts.ServerSideEncryption != nil {
+		dstOpts.ServerSideEncryption.Marshal(header)
+	}
+	for k, v := range header {
+		srcInfo.UserDefined[k] = v[0]
+	}
+
 	if _, err = l.Client.CopyObject(srcBucket, srcObject, dstBucket, dstObject, srcInfo.UserDefined); err != nil {
 		return objInfo, minio.ErrorRespToObjectError(err, srcBucket, srcObject)
 	}
@@ -478,10 +533,21 @@ func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object str
 // existing object or a part of it.
 func (l *s3Objects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string,
 	partID int, startOffset, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (p minio.PartInfo, err error) {
-
 	srcInfo.UserDefined = map[string]string{
 		"x-amz-copy-source-if-match": srcInfo.ETag,
 	}
+	header := make(http.Header)
+	if srcOpts.ServerSideEncryption != nil {
+		encrypt.SSECopy(srcOpts.ServerSideEncryption).Marshal(header)
+	}
+
+	if dstOpts.ServerSideEncryption != nil {
+		dstOpts.ServerSideEncryption.Marshal(header)
+	}
+	for k, v := range header {
+		srcInfo.UserDefined[k] = v[0]
+	}
+
 	completePart, err := l.Client.CopyObjectPart(srcBucket, srcObject, destBucket, destObject,
 		uploadID, partID, startOffset, length, srcInfo.UserDefined)
 	if err != nil {
@@ -493,7 +559,7 @@ func (l *s3Objects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, de
 }
 
 // ListObjectParts returns all object parts for specified object in specified bucket
-func (l *s3Objects) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi minio.ListPartsInfo, e error) {
+func (l *s3Objects) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int, opts minio.ObjectOptions) (lpi minio.ListPartsInfo, e error) {
 	result, err := l.Client.ListObjectParts(bucket, object, uploadID, partNumberMarker, maxParts)
 	if err != nil {
 		return lpi, minio.ErrorRespToObjectError(err, bucket, object)
@@ -557,3 +623,8 @@ func (l *s3Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error
 func (l *s3Objects) IsCompressionSupported() bool {
 	return false
 }
+
+// IsEncryptionSupported returns whether server side encryption is implemented for this layer.
+func (l *s3Objects) IsEncryptionSupported() bool {
+	return minio.GlobalKMS != nil || len(minio.GlobalGatewaySSE) > 0
+}

cmd/gateway/s3/gateway-s3_test.go

@@ -110,8 +110,8 @@ func TestS3ToObjectError(t *testing.T) {
 		// Special test case for error that is not of type
 		// miniogo.ErrorResponse
 		{
-			inputErr:    fmt.Errorf("not a minio.ErrorResponse"),
+			inputErr:    fmt.Errorf("not a ErrorResponse"),
-			expectedErr: fmt.Errorf("not a minio.ErrorResponse"),
+			expectedErr: fmt.Errorf("not a ErrorResponse"),
 		},
 	}
 

cmd/gateway/sia/gateway-sia.go

@@ -1,658 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package sia
-
-import (
-	"bytes"
-	"context"
-	"encoding/hex"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"os"
-	"path"
-	"path/filepath"
-	"strings"
-	"time"
-
-	humanize "github.com/dustin/go-humanize"
-	"github.com/fatih/color"
-	"github.com/minio/cli"
-	"github.com/minio/minio-go/pkg/set"
-	minio "github.com/minio/minio/cmd"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/sha256-simd"
-)
-
-const (
-	siaBackend = "sia"
-)
-
-type siaObjects struct {
-	minio.GatewayUnsupported
-	Address  string // Address and port of Sia Daemon.
-	TempDir  string // Temporary storage location for file transfers.
-	RootDir  string // Root directory to store files on Sia.
-	password string // Sia password for uploading content in authenticated manner.
-}
-
-func init() {
-	const siaGatewayTemplate = `NAME:
-  {{.HelpName}} - {{.Usage}}
-
-USAGE:
-  {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [SIA_DAEMON_ADDR]
-{{if .VisibleFlags}}
-FLAGS:
-  {{range .VisibleFlags}}{{.}}
-  {{end}}{{end}}
-ENVIRONMENT VARIABLES: (Default values in parenthesis)
-  ACCESS:
-     MINIO_ACCESS_KEY: Custom access key (Do not reuse same access keys on all instances)
-     MINIO_SECRET_KEY: Custom secret key (Do not reuse same secret keys on all instances)
-
-  BROWSER:
-     MINIO_BROWSER: To disable web browser access, set this value to "off".
-
-  DOMAIN:
-     MINIO_DOMAIN: To enable virtual-host-style requests, set this value to Minio host domain name.
-
-  CACHE:
-     MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
-     MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
-     MINIO_CACHE_EXPIRY: Cache expiry duration in days.
-     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
-
-  SIA_TEMP_DIR:        The name of the local Sia temporary storage directory. (.sia_temp)
-  SIA_API_PASSWORD:    API password for Sia daemon. (default is empty)
-
-EXAMPLES:
-  1. Start minio gateway server for Sia backend.
-     $ {{.HelpName}}
-
-  2. Start minio gateway server for Sia backend with edge caching enabled.
-     $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
-     $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
-     $ export MINIO_CACHE_EXPIRY=40
-     $ export MINIO_CACHE_MAXUSE=80
-     $ {{.HelpName}}
-`
-
-	minio.RegisterGatewayCommand(cli.Command{
-		Name:               siaBackend,
-		Usage:              "Sia Decentralized Cloud",
-		Action:             siaGatewayMain,
-		CustomHelpTemplate: siaGatewayTemplate,
-		HideHelpCommand:    true,
-	})
-}
-
-// Handler for 'minio gateway sia' command line.
-func siaGatewayMain(ctx *cli.Context) {
-	// Validate gateway arguments.
-	host := ctx.Args().First()
-	// Validate gateway arguments.
-	logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
-
-	minio.StartGateway(ctx, &Sia{host})
-}
-
-// Sia implements Gateway.
-type Sia struct {
-	host string // Sia daemon host address
-}
-
-// Name implements Gateway interface.
-func (g *Sia) Name() string {
-	return siaBackend
-}
-
-// NewGatewayLayer returns Sia gateway layer, implements ObjectLayer interface to
-// talk to Sia backend.
-func (g *Sia) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
-	sia := &siaObjects{
-		Address: g.host,
-		// RootDir uses access key directly, provides partitioning for
-		// concurrent users talking to same sia daemon.
-		RootDir:  creds.AccessKey,
-		TempDir:  os.Getenv("SIA_TEMP_DIR"),
-		password: os.Getenv("SIA_API_PASSWORD"),
-	}
-
-	// If Address not provided on command line or ENV, default to:
-	if sia.Address == "" {
-		sia.Address = "127.0.0.1:9980"
-	}
-
-	// If local Sia temp directory not specified, default to:
-	if sia.TempDir == "" {
-		sia.TempDir = ".sia_temp"
-	}
-
-	var err error
-	sia.TempDir, err = filepath.Abs(sia.TempDir)
-	if err != nil {
-		return nil, err
-	}
-
-	// Create the temp directory with proper permissions.
-	// Ignore error when dir already exists.
-	if err = os.MkdirAll(sia.TempDir, 0700); err != nil {
-		return nil, err
-	}
-
-	colorBlue := color.New(color.FgBlue).SprintfFunc()
-	colorBold := color.New(color.Bold).SprintFunc()
-
-	formatStr := "%" + fmt.Sprintf("%ds", len(sia.Address)+7)
-	logger.StartupMessage(colorBlue("\nSia Configuration:"))
-	logger.StartupMessage(colorBlue("  API Address:") + colorBold(fmt.Sprintf(formatStr, sia.Address)))
-	logger.StartupMessage(colorBlue("  Staging Directory:") + colorBold(fmt.Sprintf(" %s", sia.TempDir)))
-
-	return sia, nil
-}
-
-// Production - sia gateway is not ready for production use.
-func (g *Sia) Production() bool {
-	return false
-}
-
-// non2xx returns true for non-success HTTP status codes.
-func non2xx(code int) bool {
-	return code < 200 || code > 299
-}
-
-// decodeError returns the api.Error from a API response. This method should
-// only be called if the response's status code is non-2xx. The error returned
-// may not be of type api.Error in the event of an error unmarshalling the
-// JSON.
-type siaError struct {
-	// Message describes the error in English. Typically it is set to
-	// `err.Error()`. This field is required.
-	Message string `json:"message"`
-}
-
-func (s siaError) Error() string {
-	return s.Message
-}
-
-func decodeError(resp *http.Response) error {
-	// Error is a type that is encoded as JSON and returned in an API response in
-	// the event of an error. Only the Message field is required. More fields may
-	// be added to this struct in the future for better error reporting.
-	var apiErr siaError
-	if err := json.NewDecoder(resp.Body).Decode(&apiErr); err != nil {
-		return err
-	}
-	return apiErr
-}
-
-// MethodNotSupported - returned if call returned error.
-type MethodNotSupported struct {
-	method string
-}
-
-func (s MethodNotSupported) Error() string {
-	return fmt.Sprintf("API call not recognized: %s", s.method)
-}
-
-// apiGet wraps a GET request with a status code check, such that if the GET does
-// not return 2xx, the error will be read and returned. The response body is
-// not closed.
-func apiGet(ctx context.Context, addr, call, apiPassword string) (*http.Response, error) {
-	req, err := http.NewRequest("GET", "http://"+addr+call, nil)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return nil, err
-	}
-	req.Header.Set("User-Agent", "Sia-Agent")
-	if apiPassword != "" {
-		req.SetBasicAuth("", apiPassword)
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return nil, err
-	}
-	if resp.StatusCode == http.StatusNotFound {
-		minio.CloseResponse(resp.Body)
-		logger.LogIf(ctx, MethodNotSupported{call})
-		return nil, MethodNotSupported{call}
-	}
-	if non2xx(resp.StatusCode) {
-		err := decodeError(resp)
-		minio.CloseResponse(resp.Body)
-		logger.LogIf(ctx, err)
-		return nil, err
-	}
-	return resp, nil
-}
-
-// apiPost wraps a POST request with a status code check, such that if the POST
-// does not return 2xx, the error will be read and returned. The response body
-// is not closed.
-func apiPost(ctx context.Context, addr, call, vals, apiPassword string) (*http.Response, error) {
-	req, err := http.NewRequest("POST", "http://"+addr+call, strings.NewReader(vals))
-	if err != nil {
-		return nil, err
-	}
-	req.Header.Set("User-Agent", "Sia-Agent")
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	if apiPassword != "" {
-		req.SetBasicAuth("", apiPassword)
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return nil, err
-	}
-
-	if resp.StatusCode == http.StatusNotFound {
-		minio.CloseResponse(resp.Body)
-		return nil, MethodNotSupported{call}
-	}
-
-	if non2xx(resp.StatusCode) {
-		err := decodeError(resp)
-		minio.CloseResponse(resp.Body)
-		return nil, err
-	}
-	return resp, nil
-}
-
-// post makes an API call and discards the response. An error is returned if
-// the response status is not 2xx.
-func post(ctx context.Context, addr, call, vals, apiPassword string) error {
-	resp, err := apiPost(ctx, addr, call, vals, apiPassword)
-	if err != nil {
-		return err
-	}
-	minio.CloseResponse(resp.Body)
-	return nil
-}
-
-// list makes a lists all the uploaded files, decodes the json response.
-func list(ctx context.Context, addr string, apiPassword string, obj *renterFiles) error {
-	resp, err := apiGet(ctx, addr, "/renter/files", apiPassword)
-	if err != nil {
-		return err
-	}
-	defer minio.CloseResponse(resp.Body)
-
-	if resp.StatusCode == http.StatusNoContent {
-		logger.LogIf(ctx, fmt.Errorf("Expecting a response, but API returned %s", resp.Status))
-		return fmt.Errorf("Expecting a response, but API returned %s", resp.Status)
-	}
-	err = json.NewDecoder(resp.Body).Decode(obj)
-	logger.LogIf(ctx, err)
-	return err
-}
-
-// get makes an API call and discards the response. An error is returned if the
-// responsee status is not 2xx.
-func get(ctx context.Context, addr, call, apiPassword string) error {
-	resp, err := apiGet(ctx, addr, call, apiPassword)
-	if err != nil {
-		return err
-	}
-	minio.CloseResponse(resp.Body)
-	return nil
-}
-
-// Shutdown saves any gateway metadata to disk
-// if necessary and reload upon next restart.
-func (s *siaObjects) Shutdown(ctx context.Context) error {
-	return nil
-}
-
-// StorageInfo is not relevant to Sia backend.
-func (s *siaObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
-	return si
-}
-
-// MakeBucket creates a new container on Sia backend.
-func (s *siaObjects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error {
-	srcFile := path.Join(s.TempDir, minio.MustGetUUID())
-	defer os.Remove(srcFile)
-
-	writer, err := os.Create(srcFile)
-	if err != nil {
-		return err
-	}
-	if _, err = io.Copy(writer, bytes.NewReader([]byte(""))); err != nil {
-		return err
-	}
-
-	sha256sum := sha256.Sum256([]byte(bucket))
-	var siaObj = path.Join(s.RootDir, bucket, hex.EncodeToString(sha256sum[:]))
-	return post(ctx, s.Address, "/renter/upload/"+siaObj, "source="+srcFile, s.password)
-}
-
-// GetBucketInfo gets bucket metadata.
-func (s *siaObjects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, err error) {
-	sha256sum := sha256.Sum256([]byte(bucket))
-	var siaObj = path.Join(s.RootDir, bucket, hex.EncodeToString(sha256sum[:]))
-
-	dstFile := path.Join(s.TempDir, minio.MustGetUUID())
-	defer os.Remove(dstFile)
-
-	if err := get(ctx, s.Address, "/renter/download/"+siaObj+"?destination="+url.QueryEscape(dstFile), s.password); err != nil {
-		return bi, err
-	}
-	return minio.BucketInfo{Name: bucket}, nil
-}
-
-// ListBuckets will detect and return existing buckets on Sia.
-func (s *siaObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketInfo, err error) {
-	sObjs, serr := s.listRenterFiles(ctx, "")
-	if serr != nil {
-		return buckets, serr
-	}
-
-	m := make(set.StringSet)
-
-	prefix := s.RootDir + "/"
-	for _, sObj := range sObjs {
-		if strings.HasPrefix(sObj.SiaPath, prefix) {
-			siaObj := strings.TrimPrefix(sObj.SiaPath, prefix)
-			idx := strings.Index(siaObj, "/")
-			if idx > 0 {
-				m.Add(siaObj[0:idx])
-			}
-		}
-	}
-
-	for _, bktName := range m.ToSlice() {
-		buckets = append(buckets, minio.BucketInfo{
-			Name:    bktName,
-			Created: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
-		})
-	}
-
-	return buckets, nil
-}
-
-// DeleteBucket deletes a bucket on Sia.
-func (s *siaObjects) DeleteBucket(ctx context.Context, bucket string) error {
-	sha256sum := sha256.Sum256([]byte(bucket))
-	var siaObj = path.Join(s.RootDir, bucket, hex.EncodeToString(sha256sum[:]))
-
-	return post(ctx, s.Address, "/renter/delete/"+siaObj, "", s.password)
-}
-
-func (s *siaObjects) ListObjects(ctx context.Context, bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) {
-	siaObjs, siaErr := s.listRenterFiles(ctx, bucket)
-	if siaErr != nil {
-		return loi, siaErr
-	}
-
-	loi.IsTruncated = false
-	loi.NextMarker = ""
-
-	root := s.RootDir + "/"
-
-	sha256sum := sha256.Sum256([]byte(bucket))
-	// FIXME(harsha) - No paginated output supported for Sia backend right now, only prefix
-	// based filtering. Once list renter files API supports paginated output we can support
-	// paginated results here as well - until then Listing is an expensive operation.
-	for _, sObj := range siaObjs {
-		name := strings.TrimPrefix(sObj.SiaPath, path.Join(root, bucket)+"/")
-		// Skip the file created specially when bucket was created.
-		if name == hex.EncodeToString(sha256sum[:]) {
-			continue
-		}
-		if strings.HasPrefix(name, prefix) {
-			loi.Objects = append(loi.Objects, minio.ObjectInfo{
-				Bucket: bucket,
-				Name:   name,
-				Size:   int64(sObj.Filesize),
-				IsDir:  false,
-			})
-		}
-	}
-	return loi, nil
-}
-
-// GetObjectNInfo - returns object info and locked object ReadCloser
-func (s *siaObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
-	var objInfo minio.ObjectInfo
-	objInfo, err = s.GetObjectInfo(ctx, bucket, object, opts)
-	if err != nil {
-		return nil, err
-	}
-
-	var startOffset, length int64
-	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
-	if err != nil {
-		return nil, err
-	}
-
-	pr, pw := io.Pipe()
-	go func() {
-		err := s.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
-		pw.CloseWithError(err)
-	}()
-	// Setup cleanup function to cause the above go-routine to
-	// exit in case of partial read
-	pipeCloser := func() { pr.Close() }
-	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
-}
-
-func (s *siaObjects) GetObject(ctx context.Context, bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
-	dstFile := path.Join(s.TempDir, minio.MustGetUUID())
-	defer os.Remove(dstFile)
-
-	var siaObj = path.Join(s.RootDir, bucket, object)
-	if err := get(ctx, s.Address, "/renter/download/"+siaObj+"?destination="+url.QueryEscape(dstFile), s.password); err != nil {
-		return err
-	}
-
-	reader, err := os.Open(dstFile)
-	if err != nil {
-		return err
-	}
-	defer reader.Close()
-	st, err := reader.Stat()
-	if err != nil {
-		return err
-	}
-	size := st.Size()
-	if _, err = reader.Seek(startOffset, os.SEEK_SET); err != nil {
-		return err
-	}
-
-	// For negative length we read everything.
-	if length < 0 {
-		length = size - startOffset
-	}
-
-	bufSize := int64(1 * humanize.MiByte)
-	if bufSize > length {
-		bufSize = length
-	}
-
-	// Reply back invalid range if the input offset and length fall out of range.
-	if startOffset > size || startOffset+length > size {
-		logger.LogIf(ctx, minio.InvalidRange{
-			OffsetBegin:  startOffset,
-			OffsetEnd:    length,
-			ResourceSize: size,
-		})
-		return minio.InvalidRange{
-			OffsetBegin:  startOffset,
-			OffsetEnd:    length,
-			ResourceSize: size,
-		}
-	}
-
-	// Allocate a staging buffer.
-	buf := make([]byte, int(bufSize))
-
-	_, err = io.CopyBuffer(writer, io.LimitReader(reader, length), buf)
-
-	return err
-}
-
-// findSiaObject retrieves the siaObjectInfo for the Sia object with the given
-// Sia path name.
-func (s *siaObjects) findSiaObject(ctx context.Context, bucket, object string) (siaObjectInfo, error) {
-	siaPath := path.Join(s.RootDir, bucket, object)
-
-	sObjs, err := s.listRenterFiles(ctx, "")
-	if err != nil {
-		return siaObjectInfo{}, err
-	}
-
-	for _, sObj := range sObjs {
-		if sObj.SiaPath == siaPath {
-			return sObj, nil
-		}
-	}
-	logger.LogIf(ctx, minio.ObjectNotFound{
-		Bucket: bucket,
-		Object: object,
-	})
-	return siaObjectInfo{}, minio.ObjectNotFound{
-		Bucket: bucket,
-		Object: object,
-	}
-}
-
-// GetObjectInfo reads object info and replies back ObjectInfo
-func (s *siaObjects) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
-	so, err := s.findSiaObject(ctx, bucket, object)
-	if err != nil {
-		return minio.ObjectInfo{}, err
-	}
-
-	// Metadata about sia objects is just quite minimal. Sia only provides file size.
-	return minio.ObjectInfo{
-		Bucket:  bucket,
-		Name:    object,
-		ModTime: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
-		Size:    int64(so.Filesize),
-		IsDir:   false,
-	}, nil
-}
-
-// PutObject creates a new object with the incoming data,
-func (s *siaObjects) PutObject(ctx context.Context, bucket string, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
-	data := r.Reader
-	srcFile := path.Join(s.TempDir, minio.MustGetUUID())
-	writer, err := os.Create(srcFile)
-	if err != nil {
-		return objInfo, err
-	}
-
-	wsize, err := io.CopyN(writer, data, data.Size())
-	if err != nil {
-		os.Remove(srcFile)
-		return objInfo, err
-	}
-
-	if err = post(ctx, s.Address, "/renter/upload/"+path.Join(s.RootDir, bucket, object), "source="+srcFile, s.password); err != nil {
-		os.Remove(srcFile)
-		return objInfo, err
-	}
-	defer s.deleteTempFileWhenUploadCompletes(ctx, srcFile, bucket, object)
-
-	return minio.ObjectInfo{
-		Name:    object,
-		Bucket:  bucket,
-		ModTime: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
-		Size:    wsize,
-		ETag:    minio.GenETag(),
-	}, nil
-}
-
-// DeleteObject deletes a blob in bucket
-func (s *siaObjects) DeleteObject(ctx context.Context, bucket string, object string) error {
-	// Tell Sia daemon to delete the object
-	var siaObj = path.Join(s.RootDir, bucket, object)
-	return post(ctx, s.Address, "/renter/delete/"+siaObj, "", s.password)
-}
-
-// siaObjectInfo represents object info stored on Sia
-type siaObjectInfo struct {
-	SiaPath        string  `json:"siapath"`
-	LocalPath      string  `json:"localpath"`
-	Filesize       uint64  `json:"filesize"`
-	Available      bool    `json:"available"`
-	Renewing       bool    `json:"renewing"`
-	Redundancy     float64 `json:"redundancy"`
-	UploadProgress float64 `json:"uploadprogress"`
-}
-
-type renterFiles struct {
-	Files []siaObjectInfo `json:"files"`
-}
-
-// listRenterFiles will return a list of existing objects in the bucket provided
-func (s *siaObjects) listRenterFiles(ctx context.Context, bucket string) (siaObjs []siaObjectInfo, err error) {
-	// Get list of all renter files
-	var rf renterFiles
-	if err = list(ctx, s.Address, s.password, &rf); err != nil {
-		return siaObjs, err
-	}
-
-	var prefix string
-	root := s.RootDir + "/"
-	if bucket == "" {
-		prefix = root
-	} else {
-		prefix = root + bucket + "/"
-	}
-
-	for _, f := range rf.Files {
-		if strings.HasPrefix(f.SiaPath, prefix) {
-			siaObjs = append(siaObjs, f)
-		}
-	}
-
-	return siaObjs, nil
-}
-
-// deleteTempFileWhenUploadCompletes checks the status of a Sia file upload
-// until it reaches 100% upload progress, then deletes the local temp copy from
-// the filesystem.
-func (s *siaObjects) deleteTempFileWhenUploadCompletes(ctx context.Context, tempFile string, bucket, object string) {
-	var soi siaObjectInfo
-	// Wait until 100% upload instead of 1x redundancy because if we delete
-	// after 1x redundancy, the user has to pay the cost of other hosts
-	// redistributing the file.
-	for soi.UploadProgress < 100.0 {
-		var err error
-		soi, err = s.findSiaObject(ctx, bucket, object)
-		if err != nil {
-			break
-		}
-
-		// Sleep between each check so that we're not hammering
-		// the Sia daemon with requests.
-		time.Sleep(15 * time.Second)
-	}
-
-	os.Remove(tempFile)
-}
-
-// IsCompressionSupported returns whether compression is applicable for this layer.
-func (s *siaObjects) IsCompressionSupported() bool {
-	return false
-}

cmd/generic-handlers.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bufio"
 	"context"
-	"fmt"
 	"net"
 	"net/http"
 	"strings"
@@ -382,37 +381,9 @@ type resourceHandler struct {
 	handler http.Handler
 }
 
-// List of default allowable HTTP methods.
-var defaultAllowableHTTPMethods = []string{
-	http.MethodGet,
-	http.MethodPut,
-	http.MethodHead,
-	http.MethodPost,
-	http.MethodDelete,
-	http.MethodOptions,
-}
-
 // setCorsHandler handler for CORS (Cross Origin Resource Sharing)
 func setCorsHandler(h http.Handler) http.Handler {
-	commonS3Headers := []string{
+	c := cors.AllowAll()
-		"Date",
-		"ETag",
-		"Server",
-		"Connection",
-		"Accept-Ranges",
-		"Content-Range",
-		"Content-Encoding",
-		"Content-Length",
-		"Content-Type",
-		"x-amz-request-id",
-	}
-	c := cors.New(cors.Options{
-		AllowedOrigins:   []string{"*"},
-		AllowedMethods:   defaultAllowableHTTPMethods,
-		AllowedHeaders:   []string{"*"},
-		ExposedHeaders:   commonS3Headers,
-		AllowCredentials: true,
-	})
 	return c.Handler(h)
 }
 
@@ -497,11 +468,6 @@ func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
-	// A put method on path "/" doesn't make sense, ignore it.
-	if r.Method == http.MethodPut && r.URL.Path == "/" {
-		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
-		return
-	}
 
 	// Serve HTTP.
 	h.handler.ServeHTTP(w, r)
@@ -627,11 +593,52 @@ type bucketForwardingHandler struct {
 
 func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if globalDNSConfig == nil || globalDomainName == "" ||
-		guessIsBrowserReq(r) || guessIsHealthCheckReq(r) ||
+		guessIsHealthCheckReq(r) || guessIsMetricsReq(r) ||
-		guessIsMetricsReq(r) || guessIsRPCReq(r) || isAdminReq(r) {
+		guessIsRPCReq(r) || isAdminReq(r) {
 		f.handler.ServeHTTP(w, r)
 		return
 	}
+
+	// For browser requests, when federation is setup we need to
+	// specifically handle download and upload for browser requests.
+	if guessIsBrowserReq(r) && globalDNSConfig != nil && globalDomainName != "" {
+		var bucket, _ string
+		switch r.Method {
+		case http.MethodPut:
+			if getRequestAuthType(r) == authTypeJWT {
+				bucket, _ = urlPath2BucketObjectName(strings.TrimPrefix(r.URL.Path, minioReservedBucketPath+"/upload"))
+			}
+		case http.MethodGet:
+			if t := r.URL.Query().Get("token"); t != "" {
+				bucket, _ = urlPath2BucketObjectName(strings.TrimPrefix(r.URL.Path, minioReservedBucketPath+"/download"))
+			} else if getRequestAuthType(r) != authTypeJWT && !strings.HasPrefix(r.URL.Path, minioReservedBucketPath) {
+				bucket, _ = urlPath2BucketObjectName(r.URL.Path)
+			}
+		}
+		if bucket != "" {
+			sr, err := globalDNSConfig.Get(bucket)
+			if err != nil {
+				if err == dns.ErrNoEntriesFound {
+					writeErrorResponse(w, ErrNoSuchBucket, r.URL, guessIsBrowserReq(r))
+				} else {
+					writeErrorResponse(w, toAPIErrorCode(context.Background(), err), r.URL, guessIsBrowserReq(r))
+				}
+				return
+			}
+			if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() {
+				r.URL.Scheme = "http"
+				if globalIsSSL {
+					r.URL.Scheme = "https"
+				}
+				r.URL.Host = getHostFromSrv(sr)
+				f.fwd.ServeHTTP(w, r)
+				return
+			}
+		}
+		f.handler.ServeHTTP(w, r)
+		return
+	}
+
 	bucket, object := urlPath2BucketObjectName(r.URL.Path)
 	// ListBucket requests should be handled at current endpoint as
 	// all buckets data can be fetched from here.
@@ -663,12 +670,11 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 		return
 	}
 	if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() {
-		host, port := getRandomHostPort(sr)
 		r.URL.Scheme = "http"
 		if globalIsSSL {
 			r.URL.Scheme = "https"
 		}
-		r.URL.Host = fmt.Sprintf("%s:%d", host, port)
+		r.URL.Host = getHostFromSrv(sr)
 		f.fwd.ServeHTTP(w, r)
 		return
 	}
@@ -743,7 +749,9 @@ func (s customHeaderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Set custom headers such as x-amz-request-id and x-minio-deployment-id
 	// for each request.
 	w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow()))
+	if globalDeploymentID != "" {
 		w.Header().Set(responseDeploymentIDKey, globalDeploymentID)
+	}
 	s.handler.ServeHTTP(logger.NewResponseWriter(w), r)
 }
 

cmd/globals.go

@@ -20,7 +20,6 @@ import (
 	"crypto/x509"
 	"fmt"
 	"os"
-	"runtime"
 	"time"
 
 	isatty "github.com/mattn/go-isatty"
@@ -42,6 +41,8 @@ import (
 const (
 	globalMinioCertExpireWarnDays = time.Hour * 24 * 30 // 30 days.
 
+	globalMinioDefaultPort = "9000"
+
 	globalMinioDefaultRegion = ""
 	// This is a sha256 output of ``arn:aws:iam::minio:user/admin``,
 	// this is kept in present form to be compatible with S3 owner ID
@@ -77,10 +78,11 @@ const (
 	// date and server date during signature verification.
 	globalMaxSkewTime = 15 * time.Minute // 15 minutes skew allowed.
 
-	// Expiry duration after which the multipart uploads are deemed stale.
+	// GlobalMultipartExpiry - Expiry duration after which the multipart uploads are deemed stale.
-	globalMultipartExpiry = time.Hour * 24 * 14 // 2 weeks.
+	GlobalMultipartExpiry = time.Hour * 24 * 14 // 2 weeks.
-	// Cleanup interval when the stale multipart cleanup is initiated.
+	// GlobalMultipartCleanupInterval - Cleanup interval when the stale multipart cleanup is initiated.
-	globalMultipartCleanupInterval = time.Hour * 24 // 24 hrs.
+	GlobalMultipartCleanupInterval = time.Hour * 24 // 24 hrs.
+
 	// Refresh interval to update in-memory bucket policy cache.
 	globalRefreshBucketPolicyInterval = 5 * time.Minute
 	// Refresh interval to update in-memory iam config cache.
@@ -90,6 +92,12 @@ const (
 	maxLocationConstraintSize = 3 * humanize.MiByte
 )
 
+var globalCLIContext = struct {
+	JSON, Quiet bool
+	Anonymous   bool
+	Addr        string
+}{}
+
 var (
 	// Indicates the total number of erasure coded sets configured.
 	globalXLSetCount int
@@ -127,7 +135,7 @@ var (
 	// Minio local server address (in `host:port` format)
 	globalMinioAddr = ""
 	// Minio default port, can be changed through command line.
-	globalMinioPort = "9000"
+	globalMinioPort = globalMinioDefaultPort
 	// Holds the host that was passed using --address
 	globalMinioHost = ""
 
@@ -153,12 +161,6 @@ var (
 	// File to log HTTP request/response headers and body.
 	globalHTTPTraceFile *os.File
 
-	// List of admin peers.
-	globalAdminPeers = adminPeers{}
-
-	// Minio server user agent string.
-	globalServerUserAgent = "Minio/" + ReleaseTag + " (" + runtime.GOOS + "; " + runtime.GOARCH + ")"
-
 	globalEndpoints EndpointList
 
 	// Global server's network statistics
@@ -228,10 +230,14 @@ var (
 
 	// KMS key id
 	globalKMSKeyID string
-	// Allocated KMS
+
-	globalKMS crypto.KMS
+	// GlobalKMS initialized KMS configuration
-	// KMS config
+	GlobalKMS crypto.KMS
-	globalKMSConfig crypto.KMSConfig
+
+	// Auto-Encryption, if enabled, turns any non-SSE-C request
+	// into an SSE-S3 request. If enabled a valid, non-empty KMS
+	// configuration must be present.
+	globalAutoEncryption bool
 
 	// Is compression include extensions/content-types set.
 	globalIsEnvCompression bool
@@ -258,6 +264,9 @@ var (
 	// Deployment ID - unique per deployment
 	globalDeploymentID string
 
+	// GlobalGatewaySSE sse options
+	GlobalGatewaySSE gatewaySSE
+
 	// Add new variable global values here.
 )
 
@@ -345,7 +354,6 @@ func getGlobalInfo() (globalInfo map[string]interface{}) {
 		"isEnvRegion":      globalIsEnvRegion,
 		"isSSL":            globalIsSSL,
 		"serverRegion":     globalServerRegion,
-		"serverUserAgent":  globalServerUserAgent,
 		// Add more relevant global settings here.
 	}
 

cmd/handler-utils.go

@@ -40,7 +40,7 @@ func parseLocationConstraint(r *http.Request) (location string, s3Error APIError
 	// be created at default region.
 	locationConstraint := createBucketLocationConfiguration{}
 	err := xmlDecoder(r.Body, &locationConstraint, r.ContentLength)
-	if err != nil && err != io.EOF {
+	if err != nil && r.ContentLength != 0 {
 		logger.LogIf(context.Background(), err)
 		// Treat all other failures as XML parsing errors.
 		return "", ErrMalformedXML
@@ -134,6 +134,11 @@ func extractMetadata(ctx context.Context, r *http.Request) (metadata map[string]
 		return nil, err
 	}
 
+	// Set content-type to default value if it is not set.
+	if _, ok := metadata["content-type"]; !ok {
+		metadata["content-type"] = "application/octet-stream"
+	}
+
 	// Success.
 	return metadata, nil
 }
@@ -183,6 +188,13 @@ func getReqAccessCred(r *http.Request, region string) (cred auth.Credentials) {
 	if cred.AccessKey == "" {
 		cred, _, _ = getReqAccessKeyV2(r)
 	}
+	if cred.AccessKey == "" {
+		claims, owner, _ := webRequestAuthenticate(r)
+		if owner {
+			return globalServerConfig.GetCredential()
+		}
+		cred, _ = globalIAMSys.GetUser(claims.Subject)
+	}
 	return cred
 }
 
@@ -194,6 +206,7 @@ func extractReqParams(r *http.Request) map[string]string {
 
 	region := globalServerConfig.GetRegion()
 	cred := getReqAccessCred(r, region)
+
 	// Success.
 	return map[string]string{
 		"region":          region,
@@ -351,6 +364,12 @@ func getResource(path string, host string, domain string) (string, error) {
 	return slashSeparator + pathJoin(bucket, path), nil
 }
 
+// If none of the http routes match respond with MethodNotAllowed, in JSON
+func notFoundHandlerJSON(w http.ResponseWriter, r *http.Request) {
+	writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+	return
+}
+
 // If none of the http routes match respond with MethodNotAllowed
 func notFoundHandler(w http.ResponseWriter, r *http.Request) {
 	writeErrorResponse(w, ErrMethodNotAllowed, r.URL, guessIsBrowserReq(r))

cmd/handler-utils_test.go

@@ -39,48 +39,50 @@ func TestIsValidLocationContraint(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// Test with corrupted XML
+	// Corrupted XML
 	malformedReq := &http.Request{
 		Body:          ioutil.NopCloser(bytes.NewBuffer([]byte("<>"))),
 		ContentLength: int64(len("<>")),
 	}
-	if _, err := parseLocationConstraint(malformedReq); err != ErrMalformedXML {
+
-		t.Fatal("Unexpected error: ", err)
+	// Not an XML
+	badRequest := &http.Request{
+		Body:          ioutil.NopCloser(bytes.NewReader([]byte("garbage"))),
+		ContentLength: int64(len("garbage")),
 	}
 
 	// generates the input request with XML bucket configuration set to the request body.
-	createExpectedRequest := func(req *http.Request, location string) (*http.Request, error) {
+	createExpectedRequest := func(req *http.Request, location string) *http.Request {
 		createBucketConfig := createBucketLocationConfiguration{}
 		createBucketConfig.Location = location
-		var createBucketConfigBytes []byte
+		createBucketConfigBytes, _ := xml.Marshal(createBucketConfig)
-		createBucketConfigBytes, e := xml.Marshal(createBucketConfig)
-		if e != nil {
-			return nil, e
-		}
 		createBucketConfigBuffer := bytes.NewBuffer(createBucketConfigBytes)
 		req.Body = ioutil.NopCloser(createBucketConfigBuffer)
 		req.ContentLength = int64(createBucketConfigBuffer.Len())
-		return req, nil
+		return req
 	}
 
 	testCases := []struct {
-		locationForInputRequest string
+		request            *http.Request
 		serverConfigRegion string
 		expectedCode       APIErrorCode
 	}{
 		// Test case - 1.
-		{globalMinioDefaultRegion, globalMinioDefaultRegion, ErrNone},
+		{createExpectedRequest(&http.Request{}, "eu-central-1"), globalMinioDefaultRegion, ErrNone},
 		// Test case - 2.
 		// In case of empty request body ErrNone is returned.
-		{"", globalMinioDefaultRegion, ErrNone},
+		{createExpectedRequest(&http.Request{}, ""), globalMinioDefaultRegion, ErrNone},
+		// Test case - 3
+		// In case of garbage request body ErrMalformedXML is returned.
+		{badRequest, globalMinioDefaultRegion, ErrMalformedXML},
+		// Test case - 4
+		// In case of invalid XML request body ErrMalformedXML is returned.
+		{malformedReq, globalMinioDefaultRegion, ErrMalformedXML},
 	}
+
 	for i, testCase := range testCases {
-		inputRequest, e := createExpectedRequest(&http.Request{}, testCase.locationForInputRequest)
-		if e != nil {
-			t.Fatalf("Test %d: Failed to Marshal bucket configuration", i+1)
-		}
 		globalServerConfig.SetRegion(testCase.serverConfigRegion)
-		_, actualCode := parseLocationConstraint(inputRequest)
+		_, actualCode := parseLocationConstraint(testCase.request)
 		if testCase.expectedCode != actualCode {
 			t.Errorf("Test %d: Expected the APIErrCode to be %d, but instead found %d", i+1, testCase.expectedCode, actualCode)
 		}

cmd/healthcheck-handler.go

@@ -56,9 +56,10 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if !globalIsXL && !globalIsDistXL {
 		s := objLayer.StorageInfo(ctx)
-	// Gateways don't provide disk info, also handle special case for NAS gateway.
+		// Gateways don't provide disk info.
-	if s.Backend.Type == Unknown || s.Backend.Type == BackendFS {
+		if s.Backend.Type == Unknown {
 			// ListBuckets to confirm gateway backend is up
 			if _, err := objLayer.ListBuckets(ctx); err != nil {
 				writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
@@ -67,7 +68,9 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
 			writeResponse(w, http.StatusOK, nil, mimeNone)
 			return
 		}
+	}
 
+	// For FS and Erasure backend, check if local disks are up.
 	var totalLocalDisks int
 	var erroredDisks int
 	for _, endpoint := range globalEndpoints {

cmd/http/bufconn.go

@@ -99,10 +99,10 @@ func (c *BufConn) Write(b []byte) (n int, err error) {
 }
 
 // newBufConn - creates a new connection object wrapping net.Conn.
-func newBufConn(c net.Conn, readTimeout, writeTimeout time.Duration) *BufConn {
+func newBufConn(c net.Conn, readTimeout, writeTimeout time.Duration, maxHeaderBytes int) *BufConn {
 	return &BufConn{
 		QuirkConn:    QuirkConn{Conn: c},
-		bufReader:    bufio.NewReader(c),
+		bufReader:    bufio.NewReaderSize(c, maxHeaderBytes),
 		readTimeout:  readTimeout,
 		writeTimeout: writeTimeout,
 	}

cmd/http/bufconn_test.go

@@ -49,7 +49,7 @@ func TestBuffConnReadTimeout(t *testing.T) {
 			t.Errorf("failed to accept new connection. %v", terr)
 			return
 		}
-		bufconn := newBufConn(tcpConn, 1*time.Second, 1*time.Second)
+		bufconn := newBufConn(tcpConn, 1*time.Second, 1*time.Second, 4096)
 		defer bufconn.Close()
 
 		// Read a line

cmd/http/close.go

@@ -0,0 +1,52 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package http
+
+import (
+	"io"
+	"io/ioutil"
+	"sync"
+)
+
+var b512pool = sync.Pool{
+	New: func() interface{} {
+		buf := make([]byte, 512)
+		return &buf
+	},
+}
+
+// DrainBody close non nil response with any response Body.
+// convenient wrapper to drain any remaining data on response body.
+//
+// Subsequently this allows golang http RoundTripper
+// to re-use the same connection for future requests.
+func DrainBody(respBody io.ReadCloser) {
+	// Callers should close resp.Body when done reading from it.
+	// If resp.Body is not closed, the Client's underlying RoundTripper
+	// (typically Transport) may not be able to re-use a persistent TCP
+	// connection to the server for a subsequent "keep-alive" request.
+	if respBody != nil {
+		// Drain any remaining Body and then close the connection.
+		// Without this closing connection would disallow re-using
+		// the same connection for future uses.
+		//  - http://stackoverflow.com/a/17961593/4465767
+		bufp := b512pool.Get().(*[]byte)
+		defer b512pool.Put(bufp)
+		io.CopyBuffer(ioutil.Discard, respBody, *bufp)
+		respBody.Close()
+	}
+}

cmd/http/listener.go

@@ -178,6 +178,9 @@ type httpListener struct {
 // isRoutineNetErr returns true if error is due to a network timeout,
 // connect reset or io.EOF and false otherwise
 func isRoutineNetErr(err error) bool {
+	if err == nil {
+		return false
+	}
 	if nErr, ok := err.(*net.OpError); ok {
 		// Check if the error is a tcp connection reset
 		if syscallErr, ok := nErr.Err.(*os.SyscallError); ok {
@@ -188,7 +191,8 @@ func isRoutineNetErr(err error) bool {
 		// Check if the error is a timeout
 		return nErr.Timeout()
 	}
-	return err == io.EOF
+	// check for io.EOF and also some times io.EOF is wrapped is another error type.
+	return err == io.EOF || err.Error() == "EOF"
 }
 
 // start - starts separate goroutine for each TCP listener.  A valid insecure/TLS HTTP new connection is passed to httpListener.acceptCh.
@@ -218,7 +222,7 @@ func (listener *httpListener) start() {
 		tcpConn.SetKeepAlive(true)
 		tcpConn.SetKeepAlivePeriod(listener.tcpKeepAliveTimeout)
 
-		bufconn := newBufConn(tcpConn, listener.readTimeout, listener.writeTimeout)
+		bufconn := newBufConn(tcpConn, listener.readTimeout, listener.writeTimeout, listener.maxHeaderBytes)
 		if listener.tlsConfig != nil {
 			ok, err := getPlainText(bufconn)
 			if err != nil {
@@ -257,7 +261,7 @@ func (listener *httpListener) start() {
 				return
 			}
 
-			bufconn = newBufConn(tlsConn, listener.readTimeout, listener.writeTimeout)
+			bufconn = newBufConn(tlsConn, listener.readTimeout, listener.writeTimeout, listener.maxHeaderBytes)
 		}
 
 		method, resource, host, err := getMethodResourceHost(bufconn, listener.maxHeaderBytes)
@@ -395,8 +399,8 @@ func newHTTPListener(serverAddrs []string,
 
 	for _, serverAddr := range serverAddrs {
 		var l net.Listener
-		if l, err = listen("tcp4", serverAddr); err != nil {
+		if l, err = listen("tcp", serverAddr); err != nil {
-			if l, err = fallbackListen("tcp4", serverAddr); err != nil {
+			if l, err = fallbackListen("tcp", serverAddr); err != nil {
 				return nil, err
 			}
 		}